001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.quotas; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertFalse; 022import static org.junit.Assert.assertNull; 023import static org.junit.Assert.assertTrue; 024import static org.junit.Assert.fail; 025 026import java.util.ArrayList; 027import java.util.HashMap; 028import java.util.List; 029import java.util.Map; 030import java.util.Map.Entry; 031import java.util.concurrent.atomic.AtomicLong; 032import org.apache.hadoop.conf.Configuration; 033import org.apache.hadoop.fs.FileStatus; 034import org.apache.hadoop.fs.FileSystem; 035import org.apache.hadoop.fs.Path; 036import org.apache.hadoop.hbase.DoNotRetryIOException; 037import org.apache.hadoop.hbase.HBaseClassTestRule; 038import org.apache.hadoop.hbase.HBaseTestingUtility; 039import org.apache.hadoop.hbase.HConstants; 040import org.apache.hadoop.hbase.TableName; 041import org.apache.hadoop.hbase.TableNotEnabledException; 042import org.apache.hadoop.hbase.client.Admin; 043import org.apache.hadoop.hbase.client.Append; 044import org.apache.hadoop.hbase.client.ClientServiceCallable; 045import org.apache.hadoop.hbase.client.Connection; 046import org.apache.hadoop.hbase.client.Delete; 047import org.apache.hadoop.hbase.client.Increment; 048import org.apache.hadoop.hbase.client.Mutation; 049import org.apache.hadoop.hbase.client.Put; 050import org.apache.hadoop.hbase.client.RegionInfo; 051import org.apache.hadoop.hbase.client.Result; 052import org.apache.hadoop.hbase.client.ResultScanner; 053import org.apache.hadoop.hbase.client.RpcRetryingCaller; 054import org.apache.hadoop.hbase.client.RpcRetryingCallerFactory; 055import org.apache.hadoop.hbase.client.Scan; 056import org.apache.hadoop.hbase.client.SecureBulkLoadClient; 057import org.apache.hadoop.hbase.client.Table; 058import org.apache.hadoop.hbase.ipc.RpcControllerFactory; 059import org.apache.hadoop.hbase.master.HMaster; 060import org.apache.hadoop.hbase.quotas.policies.DefaultViolationPolicyEnforcement; 061import org.apache.hadoop.hbase.regionserver.HRegionServer; 062import org.apache.hadoop.hbase.regionserver.TestHRegionServerBulkLoad; 063import org.apache.hadoop.hbase.security.AccessDeniedException; 064import org.apache.hadoop.hbase.testclassification.LargeTests; 065import org.apache.hadoop.hbase.util.Bytes; 066import org.apache.hadoop.hbase.util.Pair; 067import org.apache.hadoop.util.StringUtils; 068import org.junit.AfterClass; 069import org.junit.Before; 070import org.junit.BeforeClass; 071import org.junit.ClassRule; 072import org.junit.Rule; 073import org.junit.Test; 074import org.junit.experimental.categories.Category; 075import org.junit.rules.TestName; 076import org.slf4j.Logger; 077import org.slf4j.LoggerFactory; 078 079/** 080 * End-to-end test class for filesystem space quotas. 081 */ 082@Category(LargeTests.class) 083public class TestSpaceQuotas { 084 085 @ClassRule 086 public static final HBaseClassTestRule CLASS_RULE = 087 HBaseClassTestRule.forClass(TestSpaceQuotas.class); 088 089 private static final Logger LOG = LoggerFactory.getLogger(TestSpaceQuotas.class); 090 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); 091 // Global for all tests in the class 092 private static final AtomicLong COUNTER = new AtomicLong(0); 093 private static final int NUM_RETRIES = 10; 094 095 @Rule 096 public TestName testName = new TestName(); 097 private SpaceQuotaHelperForTests helper; 098 private final TableName NON_EXISTENT_TABLE = TableName.valueOf("NON_EXISTENT_TABLE"); 099 100 @BeforeClass 101 public static void setUp() throws Exception { 102 Configuration conf = TEST_UTIL.getConfiguration(); 103 SpaceQuotaHelperForTests.updateConfigForQuotas(conf); 104 TEST_UTIL.startMiniCluster(1); 105 } 106 107 @AfterClass 108 public static void tearDown() throws Exception { 109 TEST_UTIL.shutdownMiniCluster(); 110 } 111 112 @Before 113 public void removeAllQuotas() throws Exception { 114 final Connection conn = TEST_UTIL.getConnection(); 115 if (helper == null) { 116 helper = new SpaceQuotaHelperForTests(TEST_UTIL, testName, COUNTER); 117 } 118 // Wait for the quota table to be created 119 if (!conn.getAdmin().tableExists(QuotaUtil.QUOTA_TABLE_NAME)) { 120 helper.waitForQuotaTable(conn); 121 } else { 122 // Or, clean up any quotas from previous test runs. 123 helper.removeAllQuotas(conn); 124 assertEquals(0, helper.listNumDefinedQuotas(conn)); 125 } 126 } 127 128 @Test 129 public void testNoInsertsWithPut() throws Exception { 130 Put p = new Put(Bytes.toBytes("to_reject")); 131 p.addColumn( 132 Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"), Bytes.toBytes("reject")); 133 writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_INSERTS, p); 134 } 135 136 @Test 137 public void testNoInsertsWithAppend() throws Exception { 138 Append a = new Append(Bytes.toBytes("to_reject")); 139 a.addColumn( 140 Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"), Bytes.toBytes("reject")); 141 writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_INSERTS, a); 142 } 143 144 @Test 145 public void testNoInsertsWithIncrement() throws Exception { 146 Increment i = new Increment(Bytes.toBytes("to_reject")); 147 i.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("count"), 0); 148 writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_INSERTS, i); 149 } 150 151 @Test 152 public void testDeletesAfterNoInserts() throws Exception { 153 final TableName tn = writeUntilViolation(SpaceViolationPolicy.NO_INSERTS); 154 // Try a couple of times to verify that the quota never gets enforced, same as we 155 // do when we're trying to catch the failure. 156 Delete d = new Delete(Bytes.toBytes("should_not_be_rejected")); 157 for (int i = 0; i < NUM_RETRIES; i++) { 158 try (Table t = TEST_UTIL.getConnection().getTable(tn)) { 159 t.delete(d); 160 } 161 } 162 } 163 164 @Test 165 public void testNoWritesWithPut() throws Exception { 166 Put p = new Put(Bytes.toBytes("to_reject")); 167 p.addColumn( 168 Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"), Bytes.toBytes("reject")); 169 writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_WRITES, p); 170 } 171 172 @Test 173 public void testNoWritesWithAppend() throws Exception { 174 Append a = new Append(Bytes.toBytes("to_reject")); 175 a.addColumn( 176 Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"), Bytes.toBytes("reject")); 177 writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_WRITES, a); 178 } 179 180 @Test 181 public void testNoWritesWithIncrement() throws Exception { 182 Increment i = new Increment(Bytes.toBytes("to_reject")); 183 i.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("count"), 0); 184 writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_WRITES, i); 185 } 186 187 @Test 188 public void testNoWritesWithDelete() throws Exception { 189 Delete d = new Delete(Bytes.toBytes("to_reject")); 190 writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_WRITES, d); 191 } 192 193 @Test 194 public void testNoCompactions() throws Exception { 195 Put p = new Put(Bytes.toBytes("to_reject")); 196 p.addColumn( 197 Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"), Bytes.toBytes("reject")); 198 final TableName tn = writeUntilViolationAndVerifyViolation( 199 SpaceViolationPolicy.NO_WRITES_COMPACTIONS, p); 200 // We know the policy is active at this point 201 202 // Major compactions should be rejected 203 try { 204 TEST_UTIL.getAdmin().majorCompact(tn); 205 fail("Expected that invoking the compaction should throw an Exception"); 206 } catch (DoNotRetryIOException e) { 207 // Expected! 208 } 209 // Minor compactions should also be rejected. 210 try { 211 TEST_UTIL.getAdmin().compact(tn); 212 fail("Expected that invoking the compaction should throw an Exception"); 213 } catch (DoNotRetryIOException e) { 214 // Expected! 215 } 216 } 217 218 @Test 219 public void testNoEnableAfterDisablePolicy() throws Exception { 220 Put p = new Put(Bytes.toBytes("to_reject")); 221 p.addColumn( 222 Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"), Bytes.toBytes("reject")); 223 final TableName tn = writeUntilViolation(SpaceViolationPolicy.DISABLE); 224 final Admin admin = TEST_UTIL.getAdmin(); 225 // Disabling a table relies on some external action (over the other policies), so wait a bit 226 // more than the other tests. 227 for (int i = 0; i < NUM_RETRIES * 2; i++) { 228 if (admin.isTableEnabled(tn)) { 229 LOG.info(tn + " is still enabled, expecting it to be disabled. Will wait and re-check."); 230 Thread.sleep(2000); 231 } 232 } 233 assertFalse(tn + " is still enabled but it should be disabled", admin.isTableEnabled(tn)); 234 try { 235 admin.enableTable(tn); 236 } catch (AccessDeniedException e) { 237 String exceptionContents = StringUtils.stringifyException(e); 238 final String expectedText = "violated space quota"; 239 assertTrue("Expected the exception to contain " + expectedText + ", but was: " 240 + exceptionContents, exceptionContents.contains(expectedText)); 241 } 242 } 243 244 @Test 245 public void testNoBulkLoadsWithNoWrites() throws Exception { 246 Put p = new Put(Bytes.toBytes("to_reject")); 247 p.addColumn( 248 Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"), Bytes.toBytes("reject")); 249 TableName tableName = writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_WRITES, p); 250 251 // The table is now in violation. Try to do a bulk load 252 ClientServiceCallable<Void> callable = generateFileToLoad(tableName, 1, 50); 253 RpcRetryingCallerFactory factory = new RpcRetryingCallerFactory(TEST_UTIL.getConfiguration()); 254 RpcRetryingCaller<Void> caller = factory.<Void> newCaller(); 255 try { 256 caller.callWithRetries(callable, Integer.MAX_VALUE); 257 fail("Expected the bulk load call to fail!"); 258 } catch (SpaceLimitingException e) { 259 // Pass 260 LOG.trace("Caught expected exception", e); 261 } 262 } 263 264 @Test 265 public void testAtomicBulkLoadUnderQuota() throws Exception { 266 // Need to verify that if the batch of hfiles cannot be loaded, none are loaded. 267 TableName tn = helper.createTableWithRegions(10); 268 269 final long sizeLimit = 50L * SpaceQuotaHelperForTests.ONE_KILOBYTE; 270 QuotaSettings settings = QuotaSettingsFactory.limitTableSpace( 271 tn, sizeLimit, SpaceViolationPolicy.NO_INSERTS); 272 TEST_UTIL.getAdmin().setQuota(settings); 273 274 HRegionServer rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0); 275 RegionServerSpaceQuotaManager spaceQuotaManager = rs.getRegionServerSpaceQuotaManager(); 276 Map<TableName,SpaceQuotaSnapshot> snapshots = spaceQuotaManager.copyQuotaSnapshots(); 277 Map<RegionInfo,Long> regionSizes = getReportedSizesForTable(tn); 278 while (true) { 279 SpaceQuotaSnapshot snapshot = snapshots.get(tn); 280 if (snapshot != null && snapshot.getLimit() > 0) { 281 break; 282 } 283 LOG.debug( 284 "Snapshot does not yet realize quota limit: " + snapshots + ", regionsizes: " + 285 regionSizes); 286 Thread.sleep(3000); 287 snapshots = spaceQuotaManager.copyQuotaSnapshots(); 288 regionSizes = getReportedSizesForTable(tn); 289 } 290 // Our quota limit should be reflected in the latest snapshot 291 SpaceQuotaSnapshot snapshot = snapshots.get(tn); 292 assertEquals(0L, snapshot.getUsage()); 293 assertEquals(sizeLimit, snapshot.getLimit()); 294 295 // We would also not have a "real" policy in violation 296 ActivePolicyEnforcement activePolicies = spaceQuotaManager.getActiveEnforcements(); 297 SpaceViolationPolicyEnforcement enforcement = activePolicies.getPolicyEnforcement(tn); 298 assertTrue( 299 "Expected to find Noop policy, but got " + enforcement.getClass().getSimpleName(), 300 enforcement instanceof DefaultViolationPolicyEnforcement); 301 302 // Should generate two files, each of which is over 25KB each 303 ClientServiceCallable<Void> callable = generateFileToLoad(tn, 2, 500); 304 FileSystem fs = TEST_UTIL.getTestFileSystem(); 305 FileStatus[] files = fs.listStatus( 306 new Path(fs.getHomeDirectory(), testName.getMethodName() + "_files")); 307 for (FileStatus file : files) { 308 assertTrue( 309 "Expected the file, " + file.getPath() + ", length to be larger than 25KB, but was " 310 + file.getLen(), 311 file.getLen() > 25 * SpaceQuotaHelperForTests.ONE_KILOBYTE); 312 LOG.debug(file.getPath() + " -> " + file.getLen() +"B"); 313 } 314 315 RpcRetryingCallerFactory factory = new RpcRetryingCallerFactory(TEST_UTIL.getConfiguration()); 316 RpcRetryingCaller<Void> caller = factory.<Void> newCaller(); 317 try { 318 caller.callWithRetries(callable, Integer.MAX_VALUE); 319 fail("Expected the bulk load call to fail!"); 320 } catch (SpaceLimitingException e) { 321 // Pass 322 LOG.trace("Caught expected exception", e); 323 } 324 // Verify that we have no data in the table because neither file should have been 325 // loaded even though one of the files could have. 326 Table table = TEST_UTIL.getConnection().getTable(tn); 327 ResultScanner scanner = table.getScanner(new Scan()); 328 try { 329 assertNull("Expected no results", scanner.next()); 330 } finally{ 331 scanner.close(); 332 } 333 } 334 335 @Test 336 public void testTableQuotaOverridesNamespaceQuota() throws Exception { 337 final SpaceViolationPolicy policy = SpaceViolationPolicy.NO_INSERTS; 338 final TableName tn = helper.createTableWithRegions(10); 339 340 // 2MB limit on the table, 1GB limit on the namespace 341 final long tableLimit = 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE; 342 final long namespaceLimit = 1024L * SpaceQuotaHelperForTests.ONE_MEGABYTE; 343 TEST_UTIL.getAdmin().setQuota(QuotaSettingsFactory.limitTableSpace(tn, tableLimit, policy)); 344 TEST_UTIL.getAdmin().setQuota(QuotaSettingsFactory.limitNamespaceSpace( 345 tn.getNamespaceAsString(), namespaceLimit, policy)); 346 347 // Write more data than should be allowed and flush it to disk 348 helper.writeData(tn, 3L * SpaceQuotaHelperForTests.ONE_MEGABYTE); 349 350 // This should be sufficient time for the chores to run and see the change. 351 Thread.sleep(5000); 352 353 // The write should be rejected because the table quota takes priority over the namespace 354 Put p = new Put(Bytes.toBytes("to_reject")); 355 p.addColumn( 356 Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"), Bytes.toBytes("reject")); 357 verifyViolation(policy, tn, p); 358 } 359 360 @Test 361 public void testSetQuotaAndThenRemoveWithNoInserts() throws Exception { 362 setQuotaAndThenRemove(SpaceViolationPolicy.NO_INSERTS); 363 } 364 365 @Test 366 public void testSetQuotaAndThenRemoveWithNoWrite() throws Exception { 367 setQuotaAndThenRemove(SpaceViolationPolicy.NO_WRITES); 368 } 369 370 @Test 371 public void testSetQuotaAndThenRemoveWithNoWritesCompactions() throws Exception { 372 setQuotaAndThenRemove(SpaceViolationPolicy.NO_WRITES_COMPACTIONS); 373 } 374 375 @Test 376 public void testSetQuotaAndThenRemoveWithDisable() throws Exception { 377 setQuotaAndThenRemove(SpaceViolationPolicy.DISABLE); 378 } 379 380 @Test 381 public void testSetQuotaAndThenDropTableWithNoInserts() throws Exception { 382 setQuotaAndThenDropTable(SpaceViolationPolicy.NO_INSERTS); 383 } 384 385 @Test 386 public void testSetQuotaAndThenDropTableWithNoWrite() throws Exception { 387 setQuotaAndThenDropTable(SpaceViolationPolicy.NO_WRITES); 388 } 389 390 @Test 391 public void testSetQuotaAndThenDropTableWithNoWritesCompactions() throws Exception { 392 setQuotaAndThenDropTable(SpaceViolationPolicy.NO_WRITES_COMPACTIONS); 393 } 394 395 @Test 396 public void testSetQuotaAndThenDropTableWithDisable() throws Exception { 397 setQuotaAndThenDropTable(SpaceViolationPolicy.DISABLE); 398 } 399 400 @Test 401 public void testSetQuotaAndThenIncreaseQuotaWithNoInserts() throws Exception { 402 setQuotaAndThenIncreaseQuota(SpaceViolationPolicy.NO_INSERTS); 403 } 404 405 @Test 406 public void testSetQuotaAndThenIncreaseQuotaWithNoWrite() throws Exception { 407 setQuotaAndThenIncreaseQuota(SpaceViolationPolicy.NO_WRITES); 408 } 409 410 @Test 411 public void testSetQuotaAndThenIncreaseQuotaWithNoWritesCompactions() throws Exception { 412 setQuotaAndThenIncreaseQuota(SpaceViolationPolicy.NO_WRITES_COMPACTIONS); 413 } 414 415 @Test 416 public void testSetQuotaAndThenIncreaseQuotaWithDisable() throws Exception { 417 setQuotaAndThenIncreaseQuota(SpaceViolationPolicy.DISABLE); 418 } 419 420 @Test 421 public void testSetQuotaAndThenDisableIncrEnableWithDisable() throws Exception { 422 setQuotaNextDisableThenIncreaseFinallyEnable(SpaceViolationPolicy.DISABLE); 423 } 424 425 @Test 426 public void testSetQuotaAndThenRemoveInOneWithNoInserts() throws Exception { 427 setQuotaAndThenRemoveInOneAmongTwoTables(SpaceViolationPolicy.NO_INSERTS); 428 } 429 430 @Test 431 public void testSetQuotaAndThenRemoveInOneWithNoWrite() throws Exception { 432 setQuotaAndThenRemoveInOneAmongTwoTables(SpaceViolationPolicy.NO_WRITES); 433 } 434 435 @Test 436 public void testSetQuotaAndThenRemoveInOneWithNoWritesCompaction() throws Exception { 437 setQuotaAndThenRemoveInOneAmongTwoTables(SpaceViolationPolicy.NO_WRITES_COMPACTIONS); 438 } 439 440 @Test 441 public void testSetQuotaAndThenRemoveInOneWithDisable() throws Exception { 442 setQuotaAndThenRemoveInOneAmongTwoTables(SpaceViolationPolicy.DISABLE); 443 } 444 445 @Test 446 public void testSetQuotaFirstWithDisableNextNoWrites() throws Exception { 447 setQuotaAndViolateNextSwitchPoliciesAndValidate(SpaceViolationPolicy.DISABLE, 448 SpaceViolationPolicy.NO_WRITES); 449 } 450 451 @Test 452 public void testSetQuotaFirstWithDisableNextAgainDisable() throws Exception { 453 setQuotaAndViolateNextSwitchPoliciesAndValidate(SpaceViolationPolicy.DISABLE, 454 SpaceViolationPolicy.DISABLE); 455 } 456 457 @Test 458 public void testSetQuotaFirstWithDisableNextNoInserts() throws Exception { 459 setQuotaAndViolateNextSwitchPoliciesAndValidate(SpaceViolationPolicy.DISABLE, 460 SpaceViolationPolicy.NO_INSERTS); 461 } 462 463 @Test 464 public void testSetQuotaFirstWithDisableNextNoWritesCompaction() throws Exception { 465 setQuotaAndViolateNextSwitchPoliciesAndValidate(SpaceViolationPolicy.DISABLE, 466 SpaceViolationPolicy.NO_WRITES_COMPACTIONS); 467 } 468 469 @Test 470 public void testSetQuotaFirstWithNoWritesNextWithDisable() throws Exception { 471 setQuotaAndViolateNextSwitchPoliciesAndValidate(SpaceViolationPolicy.NO_WRITES, 472 SpaceViolationPolicy.DISABLE); 473 } 474 475 @Test 476 public void testSetQuotaOnNonExistingTableWithNoInserts() throws Exception { 477 setQuotaLimit(NON_EXISTENT_TABLE, SpaceViolationPolicy.NO_INSERTS, 2L); 478 } 479 480 @Test 481 public void testSetQuotaOnNonExistingTableWithNoWrites() throws Exception { 482 setQuotaLimit(NON_EXISTENT_TABLE, SpaceViolationPolicy.NO_WRITES, 2L); 483 } 484 485 @Test 486 public void testSetQuotaOnNonExistingTableWithNoWritesCompaction() throws Exception { 487 setQuotaLimit(NON_EXISTENT_TABLE, SpaceViolationPolicy.NO_WRITES_COMPACTIONS, 2L); 488 } 489 490 @Test 491 public void testSetQuotaOnNonExistingTableWithDisable() throws Exception { 492 setQuotaLimit(NON_EXISTENT_TABLE, SpaceViolationPolicy.DISABLE, 2L); 493 } 494 495 public void setQuotaAndViolateNextSwitchPoliciesAndValidate(SpaceViolationPolicy policy1, 496 SpaceViolationPolicy policy2) throws Exception { 497 Put put = new Put(Bytes.toBytes("to_reject")); 498 put.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"), 499 Bytes.toBytes("reject")); 500 501 // Do puts until we violate space violation policy1 502 final TableName tn = writeUntilViolationAndVerifyViolation(policy1, put); 503 504 // Now, change violation policy to policy2 505 setQuotaLimit(tn, policy2, 2L); 506 507 // The table should be in enabled state on changing violation policy 508 if (policy1.equals(SpaceViolationPolicy.DISABLE) && !policy1.equals(policy2)) { 509 TEST_UTIL.waitTableEnabled(tn, 20000); 510 } 511 // Put some row now: should still violate as quota limit still violated 512 verifyViolation(policy2, tn, put); 513 } 514 515 private void setQuotaAndThenRemove(SpaceViolationPolicy policy) throws Exception { 516 Put put = new Put(Bytes.toBytes("to_reject")); 517 put.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"), 518 Bytes.toBytes("reject")); 519 520 // Do puts until we violate space policy 521 final TableName tn = writeUntilViolationAndVerifyViolation(policy, put); 522 523 // Now, remove the quota 524 removeQuotaFromtable(tn); 525 526 // Put some rows now: should not violate as quota settings removed 527 verifyNoViolation(policy, tn, put); 528 } 529 530 private void setQuotaAndThenDropTable(SpaceViolationPolicy policy) throws Exception { 531 Put put = new Put(Bytes.toBytes("to_reject")); 532 put.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"), 533 Bytes.toBytes("reject")); 534 535 // Do puts until we violate space policy 536 final TableName tn = writeUntilViolationAndVerifyViolation(policy, put); 537 538 // Now, drop the table 539 TEST_UTIL.deleteTable(tn); 540 LOG.debug("Successfully deleted table ", tn); 541 542 // Now re-create the table 543 TEST_UTIL.createTable(tn, Bytes.toBytes(SpaceQuotaHelperForTests.F1)); 544 LOG.debug("Successfully re-created table ", tn); 545 546 // Put some rows now: should not violate as table/quota was dropped 547 verifyNoViolation(policy, tn, put); 548 } 549 550 private void setQuotaAndThenIncreaseQuota(SpaceViolationPolicy policy) throws Exception { 551 Put put = new Put(Bytes.toBytes("to_reject")); 552 put.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"), 553 Bytes.toBytes("reject")); 554 555 // Do puts until we violate space policy 556 final TableName tn = writeUntilViolationAndVerifyViolation(policy, put); 557 558 // Now, increase limit and perform put 559 setQuotaLimit(tn, policy, 4L); 560 561 // Put some row now: should not violate as quota limit increased 562 verifyNoViolation(policy, tn, put); 563 } 564 565 private void setQuotaNextDisableThenIncreaseFinallyEnable(SpaceViolationPolicy policy) 566 throws Exception { 567 Put put = new Put(Bytes.toBytes("to_reject")); 568 put.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"), 569 Bytes.toBytes("reject")); 570 571 // Do puts until we violate space policy 572 final TableName tn = writeUntilViolationAndVerifyViolation(policy, put); 573 574 // Disable the table; in case of SpaceViolationPolicy.DISABLE already disabled 575 if (!policy.equals(SpaceViolationPolicy.DISABLE)) { 576 TEST_UTIL.getAdmin().disableTable(tn); 577 TEST_UTIL.waitTableDisabled(tn, 10000); 578 } 579 580 // Now, increase limit and perform put 581 setQuotaLimit(tn, policy, 4L); 582 583 // in case of disable policy quota manager will enable it 584 if (!policy.equals(SpaceViolationPolicy.DISABLE)) { 585 TEST_UTIL.getAdmin().enableTable(tn); 586 } 587 TEST_UTIL.waitTableEnabled(tn, 10000); 588 589 // Put some row now: should not violate as quota limit increased 590 verifyNoViolation(policy, tn, put); 591 } 592 593 public void setQuotaAndThenRemoveInOneAmongTwoTables(SpaceViolationPolicy policy) 594 throws Exception { 595 Put put = new Put(Bytes.toBytes("to_reject")); 596 put.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"), 597 Bytes.toBytes("reject")); 598 599 // Do puts until we violate space policy on table tn1 600 final TableName tn1 = writeUntilViolationAndVerifyViolation(policy, put); 601 602 // Do puts until we violate space policy on table tn2 603 final TableName tn2 = writeUntilViolationAndVerifyViolation(policy, put); 604 605 // Now, remove the quota from table tn1 606 removeQuotaFromtable(tn1); 607 608 // Put a new row now on tn1: should not violate as quota settings removed 609 verifyNoViolation(policy, tn1, put); 610 // Put a new row now on tn2: should violate as quota settings exists 611 verifyViolation(policy, tn2, put); 612 } 613 614 private void removeQuotaFromtable(final TableName tn) throws Exception { 615 QuotaSettings removeQuota = QuotaSettingsFactory.removeTableSpaceLimit(tn); 616 TEST_UTIL.getAdmin().setQuota(removeQuota); 617 LOG.debug("Space quota settings removed from the table ", tn); 618 } 619 620 private void setQuotaLimit(final TableName tn, SpaceViolationPolicy policy, long sizeInMBs) 621 throws Exception { 622 final long sizeLimit = sizeInMBs * SpaceQuotaHelperForTests.ONE_MEGABYTE; 623 QuotaSettings settings = QuotaSettingsFactory.limitTableSpace(tn, sizeLimit, policy); 624 TEST_UTIL.getAdmin().setQuota(settings); 625 LOG.debug("Quota limit set for table = {}, limit = {}", tn, sizeLimit); 626 } 627 628 private Map<RegionInfo,Long> getReportedSizesForTable(TableName tn) { 629 HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster(); 630 MasterQuotaManager quotaManager = master.getMasterQuotaManager(); 631 Map<RegionInfo,Long> filteredRegionSizes = new HashMap<>(); 632 for (Entry<RegionInfo,Long> entry : quotaManager.snapshotRegionSizes().entrySet()) { 633 if (entry.getKey().getTable().equals(tn)) { 634 filteredRegionSizes.put(entry.getKey(), entry.getValue()); 635 } 636 } 637 return filteredRegionSizes; 638 } 639 640 private TableName writeUntilViolation(SpaceViolationPolicy policyToViolate) throws Exception { 641 TableName tn = helper.createTableWithRegions(10); 642 setQuotaLimit(tn, policyToViolate, 2L); 643 // Write more data than should be allowed and flush it to disk 644 helper.writeData(tn, 3L * SpaceQuotaHelperForTests.ONE_MEGABYTE); 645 646 // This should be sufficient time for the chores to run and see the change. 647 Thread.sleep(5000); 648 649 return tn; 650 } 651 652 private TableName writeUntilViolationAndVerifyViolation( 653 SpaceViolationPolicy policyToViolate, Mutation m) throws Exception { 654 final TableName tn = writeUntilViolation(policyToViolate); 655 verifyViolation(policyToViolate, tn, m); 656 return tn; 657 } 658 659 private void verifyViolation( 660 SpaceViolationPolicy policyToViolate, TableName tn, Mutation m) throws Exception { 661 // But let's try a few times to get the exception before failing 662 boolean sawError = false; 663 String msg = ""; 664 for (int i = 0; i < NUM_RETRIES && !sawError; i++) { 665 try (Table table = TEST_UTIL.getConnection().getTable(tn)) { 666 if (m instanceof Put) { 667 table.put((Put) m); 668 } else if (m instanceof Delete) { 669 table.delete((Delete) m); 670 } else if (m instanceof Append) { 671 table.append((Append) m); 672 } else if (m instanceof Increment) { 673 table.increment((Increment) m); 674 } else { 675 fail( 676 "Failed to apply " + m.getClass().getSimpleName() + 677 " to the table. Programming error"); 678 } 679 LOG.info("Did not reject the " + m.getClass().getSimpleName() + ", will sleep and retry"); 680 Thread.sleep(2000); 681 } catch (Exception e) { 682 msg = StringUtils.stringifyException(e); 683 if ((policyToViolate.equals(SpaceViolationPolicy.DISABLE) 684 && e instanceof TableNotEnabledException) || msg.contains(policyToViolate.name())) { 685 LOG.info("Got the expected exception={}", msg); 686 sawError = true; 687 break; 688 } else { 689 LOG.info("Did not get the expected exception, will sleep and retry"); 690 Thread.sleep(2000); 691 } 692 } 693 } 694 if (!sawError) { 695 try (Table quotaTable = TEST_UTIL.getConnection().getTable(QuotaUtil.QUOTA_TABLE_NAME)) { 696 ResultScanner scanner = quotaTable.getScanner(new Scan()); 697 Result result = null; 698 LOG.info("Dumping contents of hbase:quota table"); 699 while ((result = scanner.next()) != null) { 700 LOG.info(Bytes.toString(result.getRow()) + " => " + result.toString()); 701 } 702 scanner.close(); 703 } 704 } else { 705 if (policyToViolate.equals(SpaceViolationPolicy.DISABLE)) { 706 assertTrue( 707 msg.contains("TableNotEnabledException") || msg.contains(policyToViolate.name())); 708 } else { 709 assertTrue("Expected exception message to contain the word '" + policyToViolate.name() 710 + "', but was " + msg, 711 msg.contains(policyToViolate.name())); 712 } 713 } 714 assertTrue( 715 "Expected to see an exception writing data to a table exceeding its quota", sawError); 716 } 717 718 private ClientServiceCallable<Void> generateFileToLoad( 719 TableName tn, int numFiles, int numRowsPerFile) throws Exception { 720 Connection conn = TEST_UTIL.getConnection(); 721 FileSystem fs = TEST_UTIL.getTestFileSystem(); 722 Configuration conf = TEST_UTIL.getConfiguration(); 723 Path baseDir = new Path(fs.getHomeDirectory(), testName.getMethodName() + "_files"); 724 fs.mkdirs(baseDir); 725 final List<Pair<byte[], String>> famPaths = new ArrayList<Pair<byte[], String>>(); 726 for (int i = 1; i <= numFiles; i++) { 727 Path hfile = new Path(baseDir, "file" + i); 728 TestHRegionServerBulkLoad.createHFile( 729 fs, hfile, Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"), 730 Bytes.toBytes("reject"), numRowsPerFile); 731 famPaths.add(new Pair<>(Bytes.toBytes(SpaceQuotaHelperForTests.F1), hfile.toString())); 732 } 733 734 // bulk load HFiles 735 Table table = conn.getTable(tn); 736 final String bulkToken = new SecureBulkLoadClient(conf, table).prepareBulkLoad(conn); 737 return new ClientServiceCallable<Void>(conn, 738 tn, Bytes.toBytes("row"), new RpcControllerFactory(conf).newController(), HConstants.PRIORITY_UNSET) { 739 @Override 740 public Void rpcCall() throws Exception { 741 SecureBulkLoadClient secureClient = null; 742 byte[] regionName = getLocation().getRegionInfo().getRegionName(); 743 try (Table table = conn.getTable(getTableName())) { 744 secureClient = new SecureBulkLoadClient(conf, table); 745 secureClient.secureBulkLoadHFiles(getStub(), famPaths, regionName, 746 true, null, bulkToken); 747 } 748 return null; 749 } 750 }; 751 } 752 753 private void verifyNoViolation(SpaceViolationPolicy policyToViolate, TableName tn, Mutation m) 754 throws Exception { 755 // But let's try a few times to write data before failing 756 boolean sawSuccess = false; 757 for (int i = 0; i < NUM_RETRIES && !sawSuccess; i++) { 758 try (Table table = TEST_UTIL.getConnection().getTable(tn)) { 759 if (m instanceof Put) { 760 table.put((Put) m); 761 } else if (m instanceof Delete) { 762 table.delete((Delete) m); 763 } else if (m instanceof Append) { 764 table.append((Append) m); 765 } else if (m instanceof Increment) { 766 table.increment((Increment) m); 767 } else { 768 fail( 769 "Failed to apply " + m.getClass().getSimpleName() + " to the table. Programming error"); 770 } 771 sawSuccess = true; 772 } catch (Exception e) { 773 LOG.info("Rejected the " + m.getClass().getSimpleName() + ", will sleep and retry"); 774 Thread.sleep(2000); 775 } 776 } 777 if (!sawSuccess) { 778 try (Table quotaTable = TEST_UTIL.getConnection().getTable(QuotaUtil.QUOTA_TABLE_NAME)) { 779 ResultScanner scanner = quotaTable.getScanner(new Scan()); 780 Result result = null; 781 LOG.info("Dumping contents of hbase:quota table"); 782 while ((result = scanner.next()) != null) { 783 LOG.info(Bytes.toString(result.getRow()) + " => " + result.toString()); 784 } 785 scanner.close(); 786 } 787 } 788 assertTrue("Expected to succeed in writing data to a table not having quota ", sawSuccess); 789 } 790}