001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.quotas;
019
020import static org.junit.Assert.assertFalse;
021import static org.junit.Assert.assertTrue;
022import static org.junit.Assert.fail;
023
024import java.util.List;
025import java.util.Map;
026import java.util.concurrent.atomic.AtomicLong;
027import org.apache.hadoop.conf.Configuration;
028import org.apache.hadoop.hbase.DoNotRetryIOException;
029import org.apache.hadoop.hbase.HBaseClassTestRule;
030import org.apache.hadoop.hbase.HBaseTestingUtility;
031import org.apache.hadoop.hbase.MetaTableAccessor;
032import org.apache.hadoop.hbase.TableName;
033import org.apache.hadoop.hbase.Waiter;
034import org.apache.hadoop.hbase.client.Admin;
035import org.apache.hadoop.hbase.client.Append;
036import org.apache.hadoop.hbase.client.Delete;
037import org.apache.hadoop.hbase.client.Increment;
038import org.apache.hadoop.hbase.client.Put;
039import org.apache.hadoop.hbase.client.RegionInfo;
040import org.apache.hadoop.hbase.client.Table;
041import org.apache.hadoop.hbase.master.HMaster;
042import org.apache.hadoop.hbase.security.AccessDeniedException;
043import org.apache.hadoop.hbase.testclassification.LargeTests;
044import org.apache.hadoop.hbase.util.Bytes;
045import org.apache.hadoop.util.StringUtils;
046import org.junit.AfterClass;
047import org.junit.Assert;
048import org.junit.Before;
049import org.junit.BeforeClass;
050import org.junit.ClassRule;
051import org.junit.Rule;
052import org.junit.Test;
053import org.junit.experimental.categories.Category;
054import org.junit.rules.TestName;
055import org.slf4j.Logger;
056import org.slf4j.LoggerFactory;
057
058@Category(LargeTests.class)
059public class TestSpaceQuotaBasicFunctioning {
060
061  @ClassRule
062  public static final HBaseClassTestRule CLASS_RULE =
063    HBaseClassTestRule.forClass(TestSpaceQuotaBasicFunctioning.class);
064
065  private static final Logger LOG = LoggerFactory.getLogger(TestSpaceQuotaBasicFunctioning.class);
066  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
067  private static final int NUM_RETRIES = 10;
068
069  @Rule
070  public TestName testName = new TestName();
071  private SpaceQuotaHelperForTests helper;
072
073  @BeforeClass
074  public static void setUp() throws Exception {
075    Configuration conf = TEST_UTIL.getConfiguration();
076    SpaceQuotaHelperForTests.updateConfigForQuotas(conf);
077    TEST_UTIL.startMiniCluster(1);
078  }
079
080  @AfterClass
081  public static void tearDown() throws Exception {
082    TEST_UTIL.shutdownMiniCluster();
083  }
084
085  @Before
086  public void removeAllQuotas() throws Exception {
087    helper = new SpaceQuotaHelperForTests(TEST_UTIL, testName, new AtomicLong(0));
088    helper.removeAllQuotas();
089  }
090
091  @Test
092  public void testNoInsertsWithPut() throws Exception {
093    Put p = new Put(Bytes.toBytes("to_reject"));
094    p.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"),
095      Bytes.toBytes("reject"));
096    helper.writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_INSERTS, p);
097  }
098
099  @Test
100  public void testNoInsertsWithAppend() throws Exception {
101    Append a = new Append(Bytes.toBytes("to_reject"));
102    a.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"),
103      Bytes.toBytes("reject"));
104    helper.writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_INSERTS, a);
105  }
106
107  @Test
108  public void testNoInsertsWithIncrement() throws Exception {
109    Increment i = new Increment(Bytes.toBytes("to_reject"));
110    i.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("count"), 0);
111    helper.writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_INSERTS, i);
112  }
113
114  @Test
115  public void testDeletesAfterNoInserts() throws Exception {
116    final TableName tn = helper.writeUntilViolation(SpaceViolationPolicy.NO_INSERTS);
117    // Try a couple of times to verify that the quota never gets enforced, same as we
118    // do when we're trying to catch the failure.
119    Delete d = new Delete(Bytes.toBytes("should_not_be_rejected"));
120    for (int i = 0; i < NUM_RETRIES; i++) {
121      try (Table t = TEST_UTIL.getConnection().getTable(tn)) {
122        t.delete(d);
123      }
124    }
125  }
126
127  @Test
128  public void testNoWritesWithPut() throws Exception {
129    Put p = new Put(Bytes.toBytes("to_reject"));
130    p.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"),
131      Bytes.toBytes("reject"));
132    helper.writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_WRITES, p);
133  }
134
135  @Test
136  public void testNoWritesWithAppend() throws Exception {
137    Append a = new Append(Bytes.toBytes("to_reject"));
138    a.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"),
139      Bytes.toBytes("reject"));
140    helper.writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_WRITES, a);
141  }
142
143  @Test
144  public void testNoWritesWithIncrement() throws Exception {
145    Increment i = new Increment(Bytes.toBytes("to_reject"));
146    i.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("count"), 0);
147    helper.writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_WRITES, i);
148  }
149
150  @Test
151  public void testNoWritesWithDelete() throws Exception {
152    Delete d = new Delete(Bytes.toBytes("to_reject"));
153    helper.writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_WRITES, d);
154  }
155
156  @Test
157  public void testNoCompactions() throws Exception {
158    Put p = new Put(Bytes.toBytes("to_reject"));
159    p.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"),
160      Bytes.toBytes("reject"));
161    final TableName tn =
162      helper.writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_WRITES_COMPACTIONS, p);
163    // We know the policy is active at this point
164
165    // Major compactions should be rejected
166    try {
167      TEST_UTIL.getAdmin().majorCompact(tn);
168      fail("Expected that invoking the compaction should throw an Exception");
169    } catch (DoNotRetryIOException e) {
170      // Expected!
171    }
172    // Minor compactions should also be rejected.
173    try {
174      TEST_UTIL.getAdmin().compact(tn);
175      fail("Expected that invoking the compaction should throw an Exception");
176    } catch (DoNotRetryIOException e) {
177      // Expected!
178    }
179  }
180
181  @Test
182  public void testNoEnableAfterDisablePolicy() throws Exception {
183    Put p = new Put(Bytes.toBytes("to_reject"));
184    p.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"),
185      Bytes.toBytes("reject"));
186    final TableName tn = helper.writeUntilViolation(SpaceViolationPolicy.DISABLE);
187    final Admin admin = TEST_UTIL.getAdmin();
188    // Disabling a table relies on some external action (over the other policies), so wait a bit
189    // more than the other tests.
190    for (int i = 0; i < NUM_RETRIES * 2; i++) {
191      if (admin.isTableEnabled(tn)) {
192        LOG.info(tn + " is still enabled, expecting it to be disabled. Will wait and re-check.");
193        Thread.sleep(2000);
194      }
195    }
196    assertFalse(tn + " is still enabled but it should be disabled", admin.isTableEnabled(tn));
197    try {
198      admin.enableTable(tn);
199    } catch (AccessDeniedException e) {
200      String exceptionContents = StringUtils.stringifyException(e);
201      final String expectedText = "violated space quota";
202      assertTrue(
203        "Expected the exception to contain " + expectedText + ", but was: " + exceptionContents,
204        exceptionContents.contains(expectedText));
205    }
206  }
207
208  @Test
209  public void testTableQuotaOverridesNamespaceQuota() throws Exception {
210    final SpaceViolationPolicy policy = SpaceViolationPolicy.NO_INSERTS;
211    final TableName tn = helper.createTableWithRegions(10);
212
213    // 2MB limit on the table, 1GB limit on the namespace
214    final long tableLimit = 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE;
215    final long namespaceLimit = 1024L * SpaceQuotaHelperForTests.ONE_MEGABYTE;
216    TEST_UTIL.getAdmin().setQuota(QuotaSettingsFactory.limitTableSpace(tn, tableLimit, policy));
217    TEST_UTIL.getAdmin().setQuota(
218      QuotaSettingsFactory.limitNamespaceSpace(tn.getNamespaceAsString(), namespaceLimit, policy));
219
220    // Write more data than should be allowed and flush it to disk
221    helper.writeData(tn, 3L * SpaceQuotaHelperForTests.ONE_MEGABYTE);
222
223    // This should be sufficient time for the chores to run and see the change.
224    Thread.sleep(5000);
225
226    // The write should be rejected because the table quota takes priority over the namespace
227    Put p = new Put(Bytes.toBytes("to_reject"));
228    p.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"),
229      Bytes.toBytes("reject"));
230    helper.verifyViolation(policy, tn, p);
231  }
232
233  @Test
234  public void testDisablePolicyQuotaAndViolate() throws Exception {
235    TableName tableName = helper.createTable();
236    helper.setQuotaLimit(tableName, SpaceViolationPolicy.DISABLE, 1L);
237    helper.writeData(tableName, SpaceQuotaHelperForTests.ONE_MEGABYTE * 2L);
238    TEST_UTIL.getConfiguration().setLong("hbase.master.quotas.region.report.retention.millis", 100);
239
240    HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster();
241    MasterQuotaManager quotaManager = master.getMasterQuotaManager();
242
243    // Make sure the master has report for the table.
244    Waiter.waitFor(TEST_UTIL.getConfiguration(), 30 * 1000, new Waiter.Predicate<Exception>() {
245      @Override
246      public boolean evaluate() throws Exception {
247        Map<RegionInfo, Long> regionSizes = quotaManager.snapshotRegionSizes();
248        List<RegionInfo> tableRegions =
249          MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), tableName);
250        return regionSizes.containsKey(tableRegions.get(0));
251      }
252    });
253
254    // Check if disabled table region report present in the map after retention period expired.
255    // It should be present after retention period expired.
256    final long regionSizes = quotaManager.snapshotRegionSizes().keySet().stream()
257      .filter(k -> k.getTable().equals(tableName)).count();
258    Assert.assertTrue(regionSizes > 0);
259  }
260}