001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * http://www.apache.org/licenses/LICENSE-2.0 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 */ 016package org.apache.hadoop.hbase.quotas; 017 018import static org.junit.Assert.assertFalse; 019import static org.junit.Assert.assertTrue; 020import static org.junit.Assert.fail; 021 022import java.util.List; 023import java.util.Map; 024import java.util.concurrent.atomic.AtomicLong; 025 026import org.apache.hadoop.conf.Configuration; 027import org.apache.hadoop.hbase.DoNotRetryIOException; 028import org.apache.hadoop.hbase.HBaseClassTestRule; 029import org.apache.hadoop.hbase.HBaseTestingUtility; 030import org.apache.hadoop.hbase.MetaTableAccessor; 031import org.apache.hadoop.hbase.TableName; 032import org.apache.hadoop.hbase.Waiter; 033import org.apache.hadoop.hbase.client.Admin; 034import org.apache.hadoop.hbase.client.Append; 035import org.apache.hadoop.hbase.client.Delete; 036import org.apache.hadoop.hbase.client.Increment; 037import org.apache.hadoop.hbase.client.Put; 038import org.apache.hadoop.hbase.client.RegionInfo; 039import org.apache.hadoop.hbase.client.Table; 040import org.apache.hadoop.hbase.master.HMaster; 041import org.apache.hadoop.hbase.security.AccessDeniedException; 042import org.apache.hadoop.hbase.testclassification.LargeTests; 043import org.apache.hadoop.hbase.util.Bytes; 044import org.apache.hadoop.util.StringUtils; 045import org.junit.AfterClass; 046import org.junit.Assert; 047import org.junit.Before; 048import org.junit.BeforeClass; 049import org.junit.ClassRule; 050import org.junit.Rule; 051import org.junit.Test; 052import org.junit.experimental.categories.Category; 053import org.junit.rules.TestName; 054import org.slf4j.Logger; 055import org.slf4j.LoggerFactory; 056 057@Category(LargeTests.class) 058public class TestSpaceQuotaBasicFunctioning { 059 060 @ClassRule 061 public static final HBaseClassTestRule CLASS_RULE = 062 HBaseClassTestRule.forClass(TestSpaceQuotaBasicFunctioning.class); 063 064 private static final Logger LOG = LoggerFactory.getLogger(TestSpaceQuotaBasicFunctioning.class); 065 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); 066 private static final int NUM_RETRIES = 10; 067 068 @Rule 069 public TestName testName = new TestName(); 070 private SpaceQuotaHelperForTests helper; 071 072 @BeforeClass 073 public static void setUp() throws Exception { 074 Configuration conf = TEST_UTIL.getConfiguration(); 075 SpaceQuotaHelperForTests.updateConfigForQuotas(conf); 076 TEST_UTIL.startMiniCluster(1); 077 } 078 079 @AfterClass 080 public static void tearDown() throws Exception { 081 TEST_UTIL.shutdownMiniCluster(); 082 } 083 084 @Before 085 public void removeAllQuotas() throws Exception { 086 helper = new SpaceQuotaHelperForTests(TEST_UTIL, testName, new AtomicLong(0)); 087 helper.removeAllQuotas(); 088 } 089 090 @Test 091 public void testNoInsertsWithPut() throws Exception { 092 Put p = new Put(Bytes.toBytes("to_reject")); 093 p.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"), 094 Bytes.toBytes("reject")); 095 helper.writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_INSERTS, p); 096 } 097 098 @Test 099 public void testNoInsertsWithAppend() throws Exception { 100 Append a = new Append(Bytes.toBytes("to_reject")); 101 a.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"), 102 Bytes.toBytes("reject")); 103 helper.writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_INSERTS, a); 104 } 105 106 @Test 107 public void testNoInsertsWithIncrement() throws Exception { 108 Increment i = new Increment(Bytes.toBytes("to_reject")); 109 i.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("count"), 0); 110 helper.writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_INSERTS, i); 111 } 112 113 @Test 114 public void testDeletesAfterNoInserts() throws Exception { 115 final TableName tn = helper.writeUntilViolation(SpaceViolationPolicy.NO_INSERTS); 116 // Try a couple of times to verify that the quota never gets enforced, same as we 117 // do when we're trying to catch the failure. 118 Delete d = new Delete(Bytes.toBytes("should_not_be_rejected")); 119 for (int i = 0; i < NUM_RETRIES; i++) { 120 try (Table t = TEST_UTIL.getConnection().getTable(tn)) { 121 t.delete(d); 122 } 123 } 124 } 125 126 @Test 127 public void testNoWritesWithPut() throws Exception { 128 Put p = new Put(Bytes.toBytes("to_reject")); 129 p.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"), 130 Bytes.toBytes("reject")); 131 helper.writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_WRITES, p); 132 } 133 134 @Test 135 public void testNoWritesWithAppend() throws Exception { 136 Append a = new Append(Bytes.toBytes("to_reject")); 137 a.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"), 138 Bytes.toBytes("reject")); 139 helper.writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_WRITES, a); 140 } 141 142 @Test 143 public void testNoWritesWithIncrement() throws Exception { 144 Increment i = new Increment(Bytes.toBytes("to_reject")); 145 i.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("count"), 0); 146 helper.writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_WRITES, i); 147 } 148 149 @Test 150 public void testNoWritesWithDelete() throws Exception { 151 Delete d = new Delete(Bytes.toBytes("to_reject")); 152 helper.writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_WRITES, d); 153 } 154 155 @Test 156 public void testNoCompactions() throws Exception { 157 Put p = new Put(Bytes.toBytes("to_reject")); 158 p.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"), 159 Bytes.toBytes("reject")); 160 final TableName tn = 161 helper.writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_WRITES_COMPACTIONS, p); 162 // We know the policy is active at this point 163 164 // Major compactions should be rejected 165 try { 166 TEST_UTIL.getAdmin().majorCompact(tn); 167 fail("Expected that invoking the compaction should throw an Exception"); 168 } catch (DoNotRetryIOException e) { 169 // Expected! 170 } 171 // Minor compactions should also be rejected. 172 try { 173 TEST_UTIL.getAdmin().compact(tn); 174 fail("Expected that invoking the compaction should throw an Exception"); 175 } catch (DoNotRetryIOException e) { 176 // Expected! 177 } 178 } 179 180 @Test 181 public void testNoEnableAfterDisablePolicy() throws Exception { 182 Put p = new Put(Bytes.toBytes("to_reject")); 183 p.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"), 184 Bytes.toBytes("reject")); 185 final TableName tn = helper.writeUntilViolation(SpaceViolationPolicy.DISABLE); 186 final Admin admin = TEST_UTIL.getAdmin(); 187 // Disabling a table relies on some external action (over the other policies), so wait a bit 188 // more than the other tests. 189 for (int i = 0; i < NUM_RETRIES * 2; i++) { 190 if (admin.isTableEnabled(tn)) { 191 LOG.info(tn + " is still enabled, expecting it to be disabled. Will wait and re-check."); 192 Thread.sleep(2000); 193 } 194 } 195 assertFalse(tn + " is still enabled but it should be disabled", admin.isTableEnabled(tn)); 196 try { 197 admin.enableTable(tn); 198 } catch (AccessDeniedException e) { 199 String exceptionContents = StringUtils.stringifyException(e); 200 final String expectedText = "violated space quota"; 201 assertTrue( 202 "Expected the exception to contain " + expectedText + ", but was: " + exceptionContents, 203 exceptionContents.contains(expectedText)); 204 } 205 } 206 207 @Test 208 public void testTableQuotaOverridesNamespaceQuota() throws Exception { 209 final SpaceViolationPolicy policy = SpaceViolationPolicy.NO_INSERTS; 210 final TableName tn = helper.createTableWithRegions(10); 211 212 // 2MB limit on the table, 1GB limit on the namespace 213 final long tableLimit = 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE; 214 final long namespaceLimit = 1024L * SpaceQuotaHelperForTests.ONE_MEGABYTE; 215 TEST_UTIL.getAdmin().setQuota(QuotaSettingsFactory.limitTableSpace(tn, tableLimit, policy)); 216 TEST_UTIL.getAdmin().setQuota(QuotaSettingsFactory 217 .limitNamespaceSpace(tn.getNamespaceAsString(), namespaceLimit, policy)); 218 219 // Write more data than should be allowed and flush it to disk 220 helper.writeData(tn, 3L * SpaceQuotaHelperForTests.ONE_MEGABYTE); 221 222 // This should be sufficient time for the chores to run and see the change. 223 Thread.sleep(5000); 224 225 // The write should be rejected because the table quota takes priority over the namespace 226 Put p = new Put(Bytes.toBytes("to_reject")); 227 p.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"), 228 Bytes.toBytes("reject")); 229 helper.verifyViolation(policy, tn, p); 230 } 231 232 @Test 233 public void testDisablePolicyQuotaAndViolate() throws Exception { 234 TableName tableName = helper.createTable(); 235 helper.setQuotaLimit(tableName, SpaceViolationPolicy.DISABLE, 1L); 236 helper.writeData(tableName, SpaceQuotaHelperForTests.ONE_MEGABYTE * 2L); 237 TEST_UTIL.getConfiguration() 238 .setLong("hbase.master.quotas.region.report.retention.millis", 100); 239 240 HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster(); 241 MasterQuotaManager quotaManager = master.getMasterQuotaManager(); 242 243 // Make sure the master has report for the table. 244 Waiter.waitFor(TEST_UTIL.getConfiguration(), 30 * 1000, new Waiter.Predicate<Exception>() { 245 @Override 246 public boolean evaluate() throws Exception { 247 Map<RegionInfo, Long> regionSizes = quotaManager.snapshotRegionSizes(); 248 List<RegionInfo> tableRegions = 249 MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), tableName); 250 return regionSizes.containsKey(tableRegions.get(0)); 251 } 252 }); 253 254 // Check if disabled table region report present in the map after retention period expired. 255 // It should be present after retention period expired. 256 final long regionSizes = quotaManager.snapshotRegionSizes().keySet().stream() 257 .filter(k -> k.getTable().equals(tableName)).count(); 258 Assert.assertTrue(regionSizes > 0); 259 } 260}