001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 * http://www.apache.org/licenses/LICENSE-2.0
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 */
016package org.apache.hadoop.hbase.quotas;
017
018import static org.junit.Assert.assertFalse;
019import static org.junit.Assert.assertTrue;
020import static org.junit.Assert.fail;
021
022import java.util.List;
023import java.util.Map;
024import java.util.concurrent.atomic.AtomicLong;
025
026import org.apache.hadoop.conf.Configuration;
027import org.apache.hadoop.hbase.DoNotRetryIOException;
028import org.apache.hadoop.hbase.HBaseClassTestRule;
029import org.apache.hadoop.hbase.HBaseTestingUtility;
030import org.apache.hadoop.hbase.MetaTableAccessor;
031import org.apache.hadoop.hbase.TableName;
032import org.apache.hadoop.hbase.Waiter;
033import org.apache.hadoop.hbase.client.Admin;
034import org.apache.hadoop.hbase.client.Append;
035import org.apache.hadoop.hbase.client.Delete;
036import org.apache.hadoop.hbase.client.Increment;
037import org.apache.hadoop.hbase.client.Put;
038import org.apache.hadoop.hbase.client.RegionInfo;
039import org.apache.hadoop.hbase.client.Table;
040import org.apache.hadoop.hbase.master.HMaster;
041import org.apache.hadoop.hbase.security.AccessDeniedException;
042import org.apache.hadoop.hbase.testclassification.LargeTests;
043import org.apache.hadoop.hbase.util.Bytes;
044import org.apache.hadoop.util.StringUtils;
045import org.junit.AfterClass;
046import org.junit.Assert;
047import org.junit.Before;
048import org.junit.BeforeClass;
049import org.junit.ClassRule;
050import org.junit.Rule;
051import org.junit.Test;
052import org.junit.experimental.categories.Category;
053import org.junit.rules.TestName;
054import org.slf4j.Logger;
055import org.slf4j.LoggerFactory;
056
057@Category(LargeTests.class)
058public class TestSpaceQuotaBasicFunctioning {
059
060  @ClassRule
061  public static final HBaseClassTestRule CLASS_RULE =
062      HBaseClassTestRule.forClass(TestSpaceQuotaBasicFunctioning.class);
063
064  private static final Logger LOG = LoggerFactory.getLogger(TestSpaceQuotaBasicFunctioning.class);
065  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
066  private static final int NUM_RETRIES = 10;
067
068  @Rule
069  public TestName testName = new TestName();
070  private SpaceQuotaHelperForTests helper;
071
072  @BeforeClass
073  public static void setUp() throws Exception {
074    Configuration conf = TEST_UTIL.getConfiguration();
075    SpaceQuotaHelperForTests.updateConfigForQuotas(conf);
076    TEST_UTIL.startMiniCluster(1);
077  }
078
079  @AfterClass
080  public static void tearDown() throws Exception {
081    TEST_UTIL.shutdownMiniCluster();
082  }
083
084  @Before
085  public void removeAllQuotas() throws Exception {
086    helper = new SpaceQuotaHelperForTests(TEST_UTIL, testName, new AtomicLong(0));
087    helper.removeAllQuotas();
088  }
089
090  @Test
091  public void testNoInsertsWithPut() throws Exception {
092    Put p = new Put(Bytes.toBytes("to_reject"));
093    p.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"),
094        Bytes.toBytes("reject"));
095    helper.writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_INSERTS, p);
096  }
097
098  @Test
099  public void testNoInsertsWithAppend() throws Exception {
100    Append a = new Append(Bytes.toBytes("to_reject"));
101    a.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"),
102        Bytes.toBytes("reject"));
103    helper.writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_INSERTS, a);
104  }
105
106  @Test
107  public void testNoInsertsWithIncrement() throws Exception {
108    Increment i = new Increment(Bytes.toBytes("to_reject"));
109    i.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("count"), 0);
110    helper.writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_INSERTS, i);
111  }
112
113  @Test
114  public void testDeletesAfterNoInserts() throws Exception {
115    final TableName tn = helper.writeUntilViolation(SpaceViolationPolicy.NO_INSERTS);
116    // Try a couple of times to verify that the quota never gets enforced, same as we
117    // do when we're trying to catch the failure.
118    Delete d = new Delete(Bytes.toBytes("should_not_be_rejected"));
119    for (int i = 0; i < NUM_RETRIES; i++) {
120      try (Table t = TEST_UTIL.getConnection().getTable(tn)) {
121        t.delete(d);
122      }
123    }
124  }
125
126  @Test
127  public void testNoWritesWithPut() throws Exception {
128    Put p = new Put(Bytes.toBytes("to_reject"));
129    p.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"),
130        Bytes.toBytes("reject"));
131    helper.writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_WRITES, p);
132  }
133
134  @Test
135  public void testNoWritesWithAppend() throws Exception {
136    Append a = new Append(Bytes.toBytes("to_reject"));
137    a.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"),
138        Bytes.toBytes("reject"));
139    helper.writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_WRITES, a);
140  }
141
142  @Test
143  public void testNoWritesWithIncrement() throws Exception {
144    Increment i = new Increment(Bytes.toBytes("to_reject"));
145    i.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("count"), 0);
146    helper.writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_WRITES, i);
147  }
148
149  @Test
150  public void testNoWritesWithDelete() throws Exception {
151    Delete d = new Delete(Bytes.toBytes("to_reject"));
152    helper.writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_WRITES, d);
153  }
154
155  @Test
156  public void testNoCompactions() throws Exception {
157    Put p = new Put(Bytes.toBytes("to_reject"));
158    p.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"),
159        Bytes.toBytes("reject"));
160    final TableName tn =
161        helper.writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_WRITES_COMPACTIONS, p);
162    // We know the policy is active at this point
163
164    // Major compactions should be rejected
165    try {
166      TEST_UTIL.getAdmin().majorCompact(tn);
167      fail("Expected that invoking the compaction should throw an Exception");
168    } catch (DoNotRetryIOException e) {
169      // Expected!
170    }
171    // Minor compactions should also be rejected.
172    try {
173      TEST_UTIL.getAdmin().compact(tn);
174      fail("Expected that invoking the compaction should throw an Exception");
175    } catch (DoNotRetryIOException e) {
176      // Expected!
177    }
178  }
179
180  @Test
181  public void testNoEnableAfterDisablePolicy() throws Exception {
182    Put p = new Put(Bytes.toBytes("to_reject"));
183    p.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"),
184        Bytes.toBytes("reject"));
185    final TableName tn = helper.writeUntilViolation(SpaceViolationPolicy.DISABLE);
186    final Admin admin = TEST_UTIL.getAdmin();
187    // Disabling a table relies on some external action (over the other policies), so wait a bit
188    // more than the other tests.
189    for (int i = 0; i < NUM_RETRIES * 2; i++) {
190      if (admin.isTableEnabled(tn)) {
191        LOG.info(tn + " is still enabled, expecting it to be disabled. Will wait and re-check.");
192        Thread.sleep(2000);
193      }
194    }
195    assertFalse(tn + " is still enabled but it should be disabled", admin.isTableEnabled(tn));
196    try {
197      admin.enableTable(tn);
198    } catch (AccessDeniedException e) {
199      String exceptionContents = StringUtils.stringifyException(e);
200      final String expectedText = "violated space quota";
201      assertTrue(
202          "Expected the exception to contain " + expectedText + ", but was: " + exceptionContents,
203          exceptionContents.contains(expectedText));
204    }
205  }
206
207  @Test
208  public void testTableQuotaOverridesNamespaceQuota() throws Exception {
209    final SpaceViolationPolicy policy = SpaceViolationPolicy.NO_INSERTS;
210    final TableName tn = helper.createTableWithRegions(10);
211
212    // 2MB limit on the table, 1GB limit on the namespace
213    final long tableLimit = 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE;
214    final long namespaceLimit = 1024L * SpaceQuotaHelperForTests.ONE_MEGABYTE;
215    TEST_UTIL.getAdmin().setQuota(QuotaSettingsFactory.limitTableSpace(tn, tableLimit, policy));
216    TEST_UTIL.getAdmin().setQuota(QuotaSettingsFactory
217        .limitNamespaceSpace(tn.getNamespaceAsString(), namespaceLimit, policy));
218
219    // Write more data than should be allowed and flush it to disk
220    helper.writeData(tn, 3L * SpaceQuotaHelperForTests.ONE_MEGABYTE);
221
222    // This should be sufficient time for the chores to run and see the change.
223    Thread.sleep(5000);
224
225    // The write should be rejected because the table quota takes priority over the namespace
226    Put p = new Put(Bytes.toBytes("to_reject"));
227    p.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"),
228        Bytes.toBytes("reject"));
229    helper.verifyViolation(policy, tn, p);
230  }
231
232  @Test
233  public void testDisablePolicyQuotaAndViolate() throws Exception {
234    TableName tableName = helper.createTable();
235    helper.setQuotaLimit(tableName, SpaceViolationPolicy.DISABLE, 1L);
236    helper.writeData(tableName, SpaceQuotaHelperForTests.ONE_MEGABYTE * 2L);
237    TEST_UTIL.getConfiguration()
238        .setLong("hbase.master.quotas.region.report.retention.millis", 100);
239
240    HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster();
241    MasterQuotaManager quotaManager = master.getMasterQuotaManager();
242
243    // Make sure the master has report for the table.
244    Waiter.waitFor(TEST_UTIL.getConfiguration(), 30 * 1000, new Waiter.Predicate<Exception>() {
245      @Override
246      public boolean evaluate() throws Exception {
247        Map<RegionInfo, Long> regionSizes = quotaManager.snapshotRegionSizes();
248        List<RegionInfo> tableRegions =
249            MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), tableName);
250        return regionSizes.containsKey(tableRegions.get(0));
251      }
252    });
253
254    // Check if disabled table region report present in the map after retention period expired.
255    // It should be present after retention period expired.
256    final long regionSizes = quotaManager.snapshotRegionSizes().keySet().stream()
257        .filter(k -> k.getTable().equals(tableName)).count();
258    Assert.assertTrue(regionSizes > 0);
259  }
260}