001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.quotas;
019
020import static org.junit.jupiter.api.Assertions.assertFalse;
021import static org.junit.jupiter.api.Assertions.assertTrue;
022import static org.junit.jupiter.api.Assertions.fail;
023
024import java.util.List;
025import java.util.Map;
026import java.util.concurrent.atomic.AtomicLong;
027import org.apache.hadoop.conf.Configuration;
028import org.apache.hadoop.hbase.DoNotRetryIOException;
029import org.apache.hadoop.hbase.HBaseTestingUtil;
030import org.apache.hadoop.hbase.MetaTableAccessor;
031import org.apache.hadoop.hbase.TableName;
032import org.apache.hadoop.hbase.Waiter;
033import org.apache.hadoop.hbase.client.Admin;
034import org.apache.hadoop.hbase.client.Append;
035import org.apache.hadoop.hbase.client.Delete;
036import org.apache.hadoop.hbase.client.Increment;
037import org.apache.hadoop.hbase.client.Put;
038import org.apache.hadoop.hbase.client.RegionInfo;
039import org.apache.hadoop.hbase.client.Table;
040import org.apache.hadoop.hbase.master.HMaster;
041import org.apache.hadoop.hbase.security.AccessDeniedException;
042import org.apache.hadoop.hbase.testclassification.LargeTests;
043import org.apache.hadoop.hbase.util.Bytes;
044import org.apache.hadoop.util.StringUtils;
045import org.junit.jupiter.api.AfterAll;
046import org.junit.jupiter.api.BeforeAll;
047import org.junit.jupiter.api.BeforeEach;
048import org.junit.jupiter.api.Tag;
049import org.junit.jupiter.api.Test;
050import org.junit.jupiter.api.TestInfo;
051import org.slf4j.Logger;
052import org.slf4j.LoggerFactory;
053
054@Tag(LargeTests.TAG)
055public class TestSpaceQuotaBasicFunctioning {
056
057  private static final Logger LOG = LoggerFactory.getLogger(TestSpaceQuotaBasicFunctioning.class);
058  private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
059  private static final int NUM_RETRIES = 10;
060
061  private SpaceQuotaHelperForTests helper;
062
063  @BeforeAll
064  public static void setUp() throws Exception {
065    Configuration conf = TEST_UTIL.getConfiguration();
066    SpaceQuotaHelperForTests.updateConfigForQuotas(conf);
067    TEST_UTIL.startMiniCluster(1);
068  }
069
070  @AfterAll
071  public static void tearDown() throws Exception {
072    TEST_UTIL.shutdownMiniCluster();
073  }
074
075  @BeforeEach
076  public void removeAllQuotas(TestInfo testInfo) throws Exception {
077    helper = new SpaceQuotaHelperForTests(TEST_UTIL, () -> testInfo.getTestMethod().get().getName(),
078      new AtomicLong(0));
079    helper.removeAllQuotas();
080  }
081
082  @Test
083  public void testNoInsertsWithPut() throws Exception {
084    Put p = new Put(Bytes.toBytes("to_reject"));
085    p.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"),
086      Bytes.toBytes("reject"));
087    helper.writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_INSERTS, p);
088  }
089
090  @Test
091  public void testNoInsertsWithAppend() throws Exception {
092    Append a = new Append(Bytes.toBytes("to_reject"));
093    a.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"),
094      Bytes.toBytes("reject"));
095    helper.writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_INSERTS, a);
096  }
097
098  @Test
099  public void testNoInsertsWithIncrement() throws Exception {
100    Increment i = new Increment(Bytes.toBytes("to_reject"));
101    i.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("count"), 0);
102    helper.writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_INSERTS, i);
103  }
104
105  @Test
106  public void testDeletesAfterNoInserts() throws Exception {
107    final TableName tn = helper.writeUntilViolation(SpaceViolationPolicy.NO_INSERTS);
108    // Try a couple of times to verify that the quota never gets enforced, same as we
109    // do when we're trying to catch the failure.
110    Delete d = new Delete(Bytes.toBytes("should_not_be_rejected"));
111    for (int i = 0; i < NUM_RETRIES; i++) {
112      try (Table t = TEST_UTIL.getConnection().getTable(tn)) {
113        t.delete(d);
114      }
115    }
116  }
117
118  @Test
119  public void testNoWritesWithPut() throws Exception {
120    Put p = new Put(Bytes.toBytes("to_reject"));
121    p.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"),
122      Bytes.toBytes("reject"));
123    helper.writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_WRITES, p);
124  }
125
126  @Test
127  public void testNoWritesWithAppend() throws Exception {
128    Append a = new Append(Bytes.toBytes("to_reject"));
129    a.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"),
130      Bytes.toBytes("reject"));
131    helper.writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_WRITES, a);
132  }
133
134  @Test
135  public void testNoWritesWithIncrement() throws Exception {
136    Increment i = new Increment(Bytes.toBytes("to_reject"));
137    i.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("count"), 0);
138    helper.writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_WRITES, i);
139  }
140
141  @Test
142  public void testNoWritesWithDelete() throws Exception {
143    Delete d = new Delete(Bytes.toBytes("to_reject"));
144    helper.writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_WRITES, d);
145  }
146
147  @Test
148  public void testNoCompactions() throws Exception {
149    Put p = new Put(Bytes.toBytes("to_reject"));
150    p.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"),
151      Bytes.toBytes("reject"));
152    final TableName tn =
153      helper.writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_WRITES_COMPACTIONS, p);
154    // We know the policy is active at this point
155
156    // Major compactions should be rejected
157    try {
158      TEST_UTIL.getAdmin().majorCompact(tn);
159      fail("Expected that invoking the compaction should throw an Exception");
160    } catch (DoNotRetryIOException e) {
161      // Expected!
162    }
163    // Minor compactions should also be rejected.
164    try {
165      TEST_UTIL.getAdmin().compact(tn);
166      fail("Expected that invoking the compaction should throw an Exception");
167    } catch (DoNotRetryIOException e) {
168      // Expected!
169    }
170  }
171
172  @Test
173  public void testNoEnableAfterDisablePolicy() throws Exception {
174    Put p = new Put(Bytes.toBytes("to_reject"));
175    p.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"),
176      Bytes.toBytes("reject"));
177    final TableName tn = helper.writeUntilViolation(SpaceViolationPolicy.DISABLE);
178    final Admin admin = TEST_UTIL.getAdmin();
179    // Disabling a table relies on some external action (over the other policies), so wait a bit
180    // more than the other tests.
181    for (int i = 0; i < NUM_RETRIES * 2; i++) {
182      if (admin.isTableEnabled(tn)) {
183        LOG.info(tn + " is still enabled, expecting it to be disabled. Will wait and re-check.");
184        Thread.sleep(2000);
185      }
186    }
187    assertFalse(admin.isTableEnabled(tn), tn + " is still enabled but it should be disabled");
188    try {
189      admin.enableTable(tn);
190    } catch (AccessDeniedException e) {
191      String exceptionContents = StringUtils.stringifyException(e);
192      final String expectedText = "violated space quota";
193      assertTrue(exceptionContents.contains(expectedText),
194        "Expected the exception to contain " + expectedText + ", but was: " + exceptionContents);
195    }
196  }
197
198  @Test
199  public void testTableQuotaOverridesNamespaceQuota() throws Exception {
200    final SpaceViolationPolicy policy = SpaceViolationPolicy.NO_INSERTS;
201    final TableName tn = helper.createTableWithRegions(10);
202
203    // 2MB limit on the table, 1GB limit on the namespace
204    final long tableLimit = 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE;
205    final long namespaceLimit = 1024L * SpaceQuotaHelperForTests.ONE_MEGABYTE;
206    TEST_UTIL.getAdmin().setQuota(QuotaSettingsFactory.limitTableSpace(tn, tableLimit, policy));
207    TEST_UTIL.getAdmin().setQuota(
208      QuotaSettingsFactory.limitNamespaceSpace(tn.getNamespaceAsString(), namespaceLimit, policy));
209
210    // Write more data than should be allowed and flush it to disk
211    helper.writeData(tn, 3L * SpaceQuotaHelperForTests.ONE_MEGABYTE);
212
213    // This should be sufficient time for the chores to run and see the change.
214    Thread.sleep(5000);
215
216    // The write should be rejected because the table quota takes priority over the namespace
217    Put p = new Put(Bytes.toBytes("to_reject"));
218    p.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"),
219      Bytes.toBytes("reject"));
220    helper.verifyViolation(policy, tn, p);
221  }
222
223  @Test
224  public void testDisablePolicyQuotaAndViolate() throws Exception {
225    TableName tableName = helper.createTable();
226    helper.setQuotaLimit(tableName, SpaceViolationPolicy.DISABLE, 1L);
227    helper.writeData(tableName, SpaceQuotaHelperForTests.ONE_MEGABYTE * 2L);
228    TEST_UTIL.getConfiguration().setLong("hbase.master.quotas.region.report.retention.millis", 100);
229
230    HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster();
231    MasterQuotaManager quotaManager = master.getMasterQuotaManager();
232
233    // Make sure the master has report for the table.
234    Waiter.waitFor(TEST_UTIL.getConfiguration(), 30 * 1000, new Waiter.Predicate<Exception>() {
235      @Override
236      public boolean evaluate() throws Exception {
237        Map<RegionInfo, Long> regionSizes = quotaManager.snapshotRegionSizes();
238        List<RegionInfo> tableRegions =
239          MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), tableName);
240        return regionSizes.containsKey(tableRegions.get(0));
241      }
242    });
243
244    // Check if disabled table region report present in the map after retention period expired.
245    // It should be present after retention period expired.
246    final long regionSizes = quotaManager.snapshotRegionSizes().keySet().stream()
247      .filter(k -> k.getTable().equals(tableName)).count();
248    assertTrue(regionSizes > 0);
249  }
250}