001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.quotas; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertNotNull; 022import static org.junit.Assert.assertTrue; 023 024import java.io.IOException; 025import java.util.Map; 026import java.util.concurrent.atomic.AtomicInteger; 027import java.util.concurrent.atomic.AtomicLong; 028import org.apache.hadoop.conf.Configuration; 029import org.apache.hadoop.hbase.Cell; 030import org.apache.hadoop.hbase.CellScanner; 031import org.apache.hadoop.hbase.HBaseClassTestRule; 032import org.apache.hadoop.hbase.HBaseTestingUtility; 033import org.apache.hadoop.hbase.MetaTableAccessor; 034import org.apache.hadoop.hbase.TableName; 035import org.apache.hadoop.hbase.Waiter; 036import org.apache.hadoop.hbase.Waiter.Predicate; 037import org.apache.hadoop.hbase.client.Admin; 038import org.apache.hadoop.hbase.client.Connection; 039import org.apache.hadoop.hbase.client.Result; 040import org.apache.hadoop.hbase.client.ResultScanner; 041import org.apache.hadoop.hbase.client.Scan; 042import org.apache.hadoop.hbase.client.SnapshotType; 043import org.apache.hadoop.hbase.client.Table; 044import org.apache.hadoop.hbase.quotas.SpaceQuotaHelperForTests.SpaceQuotaSnapshotPredicate; 045import org.apache.hadoop.hbase.testclassification.LargeTests; 046import org.junit.AfterClass; 047import org.junit.Before; 048import org.junit.BeforeClass; 049import org.junit.ClassRule; 050import org.junit.Rule; 051import org.junit.Test; 052import org.junit.experimental.categories.Category; 053import org.junit.rules.TestName; 054import org.slf4j.Logger; 055import org.slf4j.LoggerFactory; 056 057import org.apache.hbase.thirdparty.com.google.common.collect.Iterables; 058import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations; 059 060import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos; 061 062/** 063 * Test class to exercise the inclusion of snapshots in space quotas 064 */ 065@Category({LargeTests.class}) 066public class TestSpaceQuotasWithSnapshots { 067 068 @ClassRule 069 public static final HBaseClassTestRule CLASS_RULE = 070 HBaseClassTestRule.forClass(TestSpaceQuotasWithSnapshots.class); 071 072 private static final Logger LOG = LoggerFactory.getLogger(TestSpaceQuotasWithSnapshots.class); 073 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); 074 // Global for all tests in the class 075 private static final AtomicLong COUNTER = new AtomicLong(0); 076 private static final long FUDGE_FOR_TABLE_SIZE = 500L * SpaceQuotaHelperForTests.ONE_KILOBYTE; 077 078 @Rule 079 public TestName testName = new TestName(); 080 private SpaceQuotaHelperForTests helper; 081 private Connection conn; 082 private Admin admin; 083 084 @BeforeClass 085 public static void setUp() throws Exception { 086 Configuration conf = TEST_UTIL.getConfiguration(); 087 SpaceQuotaHelperForTests.updateConfigForQuotas(conf); 088 TEST_UTIL.startMiniCluster(1); 089 // Wait till quota table onlined. 090 TEST_UTIL.waitFor(10000, new Waiter.Predicate<Exception>() { 091 @Override public boolean evaluate() throws Exception { 092 return MetaTableAccessor.tableExists(TEST_UTIL.getConnection(), 093 QuotaTableUtil.QUOTA_TABLE_NAME); 094 } 095 }); 096 } 097 098 @AfterClass 099 public static void tearDown() throws Exception { 100 TEST_UTIL.shutdownMiniCluster(); 101 } 102 103 @Before 104 public void removeAllQuotas() throws Exception { 105 helper = new SpaceQuotaHelperForTests(TEST_UTIL, testName, COUNTER); 106 conn = TEST_UTIL.getConnection(); 107 admin = TEST_UTIL.getAdmin(); 108 } 109 110 @Test 111 public void testTablesInheritSnapshotSize() throws Exception { 112 TableName tn = helper.createTableWithRegions(1); 113 LOG.info("Writing data"); 114 // Set a quota 115 QuotaSettings settings = QuotaSettingsFactory.limitTableSpace( 116 tn, SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS); 117 admin.setQuota(settings); 118 // Write some data 119 final long initialSize = 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE; 120 helper.writeData(tn, initialSize); 121 122 LOG.info("Waiting until table size reflects written data"); 123 // Wait until that data is seen by the master 124 TEST_UTIL.waitFor(30 * 1000, 500, new SpaceQuotaSnapshotPredicate(conn, tn) { 125 @Override boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception { 126 return snapshot.getUsage() >= initialSize; 127 } 128 }); 129 130 // Make sure we see the final quota usage size 131 waitForStableQuotaSize(conn, tn, null); 132 133 // The actual size on disk after we wrote our data the first time 134 final long actualInitialSize = conn.getAdmin().getCurrentSpaceQuotaSnapshot(tn).getUsage(); 135 LOG.info("Initial table size was " + actualInitialSize); 136 137 LOG.info("Snapshot the table"); 138 final String snapshot1 = tn.toString() + "_snapshot1"; 139 admin.snapshot(snapshot1, tn); 140 141 // Write the same data again, then flush+compact. This should make sure that 142 // the snapshot is referencing files that the table no longer references. 143 LOG.info("Write more data"); 144 helper.writeData(tn, initialSize); 145 LOG.info("Flush the table"); 146 admin.flush(tn); 147 LOG.info("Synchronously compacting the table"); 148 TEST_UTIL.compact(tn, true); 149 150 final long upperBound = initialSize + FUDGE_FOR_TABLE_SIZE; 151 final long lowerBound = initialSize - FUDGE_FOR_TABLE_SIZE; 152 153 // Store the actual size after writing more data and then compacting it down to one file 154 LOG.info("Waiting for the region reports to reflect the correct size, between (" 155 + lowerBound + ", " + upperBound + ")"); 156 TEST_UTIL.waitFor(30 * 1000, 500, new Predicate<Exception>() { 157 @Override 158 public boolean evaluate() throws Exception { 159 long size = getRegionSizeReportForTable(conn, tn); 160 return size < upperBound && size > lowerBound; 161 } 162 }); 163 164 // Make sure we see the "final" new size for the table, not some intermediate 165 waitForStableRegionSizeReport(conn, tn); 166 final long finalSize = getRegionSizeReportForTable(conn, tn); 167 assertNotNull("Did not expect to see a null size", finalSize); 168 LOG.info("Last seen size: " + finalSize); 169 170 // Make sure the QuotaObserverChore has time to reflect the new region size reports 171 // (we saw above). The usage of the table should *not* decrease when we check it below, 172 // though, because the snapshot on our table will cause the table to "retain" the size. 173 TEST_UTIL.waitFor(20 * 1000, 500, new SpaceQuotaSnapshotPredicate(conn, tn) { 174 @Override 175 public boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception { 176 return snapshot.getUsage() >= finalSize; 177 } 178 }); 179 180 // The final usage should be the sum of the initial size (referenced by the snapshot) and the 181 // new size we just wrote above. 182 long expectedFinalSize = actualInitialSize + finalSize; 183 LOG.info( 184 "Expecting table usage to be " + actualInitialSize + " + " + finalSize 185 + " = " + expectedFinalSize); 186 // The size of the table (WRT quotas) should now be approximately double what it was previously 187 TEST_UTIL.waitFor(30 * 1000, 1000, new SpaceQuotaSnapshotPredicate(conn, tn) { 188 @Override boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception { 189 LOG.debug("Checking for " + expectedFinalSize + " == " + snapshot.getUsage()); 190 return expectedFinalSize == snapshot.getUsage(); 191 } 192 }); 193 194 Map<String,Long> snapshotSizes = QuotaTableUtil.getObservedSnapshotSizes(conn); 195 Long size = snapshotSizes.get(snapshot1); 196 assertNotNull("Did not observe the size of the snapshot", size); 197 assertEquals( 198 "The recorded size of the HBase snapshot was not the size we expected", actualInitialSize, 199 size.longValue()); 200 } 201 202 @Test 203 public void testNamespacesInheritSnapshotSize() throws Exception { 204 String ns = helper.createNamespace().getName(); 205 TableName tn = helper.createTableWithRegions(ns, 1); 206 LOG.info("Writing data"); 207 // Set a quota 208 QuotaSettings settings = QuotaSettingsFactory.limitNamespaceSpace( 209 ns, SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS); 210 admin.setQuota(settings); 211 212 // Write some data and flush it to disk 213 final long initialSize = 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE; 214 helper.writeData(tn, initialSize); 215 admin.flush(tn); 216 217 LOG.info("Waiting until namespace size reflects written data"); 218 // Wait until that data is seen by the master 219 TEST_UTIL.waitFor(30 * 1000, 500, new SpaceQuotaSnapshotPredicate(conn, ns) { 220 @Override boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception { 221 return snapshot.getUsage() >= initialSize; 222 } 223 }); 224 225 // Make sure we see the "final" new size for the table, not some intermediate 226 waitForStableQuotaSize(conn, null, ns); 227 228 // The actual size on disk after we wrote our data the first time 229 final long actualInitialSize = conn.getAdmin().getCurrentSpaceQuotaSnapshot(ns).getUsage(); 230 LOG.info("Initial table size was " + actualInitialSize); 231 232 LOG.info("Snapshot the table"); 233 final String snapshot1 = tn.getQualifierAsString() + "_snapshot1"; 234 admin.snapshot(snapshot1, tn); 235 236 // Write the same data again, then flush+compact. This should make sure that 237 // the snapshot is referencing files that the table no longer references. 238 LOG.info("Write more data"); 239 helper.writeData(tn, initialSize); 240 LOG.info("Flush the table"); 241 admin.flush(tn); 242 LOG.info("Synchronously compacting the table"); 243 TEST_UTIL.compact(tn, true); 244 245 final long upperBound = initialSize + FUDGE_FOR_TABLE_SIZE; 246 final long lowerBound = initialSize - FUDGE_FOR_TABLE_SIZE; 247 248 LOG.info("Waiting for the region reports to reflect the correct size, between (" 249 + lowerBound + ", " + upperBound + ")"); 250 TEST_UTIL.waitFor(30 * 1000, 500, new Predicate<Exception>() { 251 @Override 252 public boolean evaluate() throws Exception { 253 Map<TableName, Long> sizes = conn.getAdmin().getSpaceQuotaTableSizes(); 254 LOG.debug("Master observed table sizes from region size reports: " + sizes); 255 Long size = sizes.get(tn); 256 if (null == size) { 257 return false; 258 } 259 return size < upperBound && size > lowerBound; 260 } 261 }); 262 263 // Make sure we see the "final" new size for the table, not some intermediate 264 waitForStableRegionSizeReport(conn, tn); 265 final long finalSize = getRegionSizeReportForTable(conn, tn); 266 assertNotNull("Did not expect to see a null size", finalSize); 267 LOG.info("Final observed size of table: " + finalSize); 268 269 // Make sure the QuotaObserverChore has time to reflect the new region size reports 270 // (we saw above). The usage of the table should *not* decrease when we check it below, 271 // though, because the snapshot on our table will cause the table to "retain" the size. 272 TEST_UTIL.waitFor(20 * 1000, 500, new SpaceQuotaSnapshotPredicate(conn, ns) { 273 @Override 274 public boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception { 275 return snapshot.getUsage() >= finalSize; 276 } 277 }); 278 279 // The final usage should be the sum of the initial size (referenced by the snapshot) and the 280 // new size we just wrote above. 281 long expectedFinalSize = actualInitialSize + finalSize; 282 LOG.info( 283 "Expecting namespace usage to be " + actualInitialSize + " + " + finalSize 284 + " = " + expectedFinalSize); 285 // The size of the table (WRT quotas) should now be approximately double what it was previously 286 TEST_UTIL.waitFor(30 * 1000, 1000, new SpaceQuotaSnapshotPredicate(conn, ns) { 287 @Override boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception { 288 LOG.debug("Checking for " + expectedFinalSize + " == " + snapshot.getUsage()); 289 return expectedFinalSize == snapshot.getUsage(); 290 } 291 }); 292 293 Map<String,Long> snapshotSizes = QuotaTableUtil.getObservedSnapshotSizes(conn); 294 Long size = snapshotSizes.get(snapshot1); 295 assertNotNull("Did not observe the size of the snapshot", size); 296 assertEquals( 297 "The recorded size of the HBase snapshot was not the size we expected", actualInitialSize, 298 size.longValue()); 299 } 300 301 @Test 302 public void testTablesWithSnapshots() throws Exception { 303 final Connection conn = TEST_UTIL.getConnection(); 304 final SpaceViolationPolicy policy = SpaceViolationPolicy.NO_INSERTS; 305 final TableName tn = helper.createTableWithRegions(10); 306 307 // 3MB limit on the table 308 final long tableLimit = 3L * SpaceQuotaHelperForTests.ONE_MEGABYTE; 309 TEST_UTIL.getAdmin().setQuota(QuotaSettingsFactory.limitTableSpace(tn, tableLimit, policy)); 310 311 LOG.info("Writing first data set"); 312 // Write more data than should be allowed and flush it to disk 313 helper.writeData(tn, 1L * SpaceQuotaHelperForTests.ONE_MEGABYTE, "q1"); 314 315 LOG.info("Creating snapshot"); 316 TEST_UTIL.getAdmin().snapshot(tn.toString() + "snap1", tn, SnapshotType.FLUSH); 317 318 LOG.info("Writing second data set"); 319 // Write some more data 320 helper.writeData(tn, 1L * SpaceQuotaHelperForTests.ONE_MEGABYTE, "q2"); 321 322 LOG.info("Flushing and major compacting table"); 323 // Compact the table to force the snapshot to own all of its files 324 TEST_UTIL.getAdmin().flush(tn); 325 TEST_UTIL.compact(tn, true); 326 327 LOG.info("Checking for quota violation"); 328 // Wait to observe the quota moving into violation 329 TEST_UTIL.waitFor(60_000, 1_000, new Predicate<Exception>() { 330 @Override 331 public boolean evaluate() throws Exception { 332 Scan s = QuotaTableUtil.makeQuotaSnapshotScanForTable(tn); 333 try (Table t = conn.getTable(QuotaTableUtil.QUOTA_TABLE_NAME)) { 334 ResultScanner rs = t.getScanner(s); 335 try { 336 Result r = Iterables.getOnlyElement(rs); 337 CellScanner cs = r.cellScanner(); 338 assertTrue(cs.advance()); 339 Cell c = cs.current(); 340 SpaceQuotaSnapshot snapshot = SpaceQuotaSnapshot.toSpaceQuotaSnapshot( 341 QuotaProtos.SpaceQuotaSnapshot.parseFrom( 342 UnsafeByteOperations.unsafeWrap( 343 c.getValueArray(), c.getValueOffset(), c.getValueLength()))); 344 LOG.info( 345 snapshot.getUsage() + "/" + snapshot.getLimit() + " " + snapshot.getQuotaStatus()); 346 // We expect to see the table move to violation 347 return snapshot.getQuotaStatus().isInViolation(); 348 } finally { 349 if (null != rs) { 350 rs.close(); 351 } 352 } 353 } 354 } 355 }); 356 } 357 358 @Test 359 public void testRematerializedTablesDoNoInheritSpace() throws Exception { 360 TableName tn = helper.createTableWithRegions(1); 361 TableName tn2 = helper.getNextTableName(); 362 LOG.info("Writing data"); 363 // Set a quota on both tables 364 QuotaSettings settings = QuotaSettingsFactory.limitTableSpace( 365 tn, SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS); 366 admin.setQuota(settings); 367 QuotaSettings settings2 = QuotaSettingsFactory.limitTableSpace( 368 tn2, SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS); 369 admin.setQuota(settings2); 370 // Write some data 371 final long initialSize = 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE; 372 helper.writeData(tn, initialSize); 373 374 LOG.info("Waiting until table size reflects written data"); 375 // Wait until that data is seen by the master 376 TEST_UTIL.waitFor(30 * 1000, 500, new SpaceQuotaSnapshotPredicate(conn, tn) { 377 @Override boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception { 378 return snapshot.getUsage() >= initialSize; 379 } 380 }); 381 382 // Make sure we see the final quota usage size 383 waitForStableQuotaSize(conn, tn, null); 384 385 // The actual size on disk after we wrote our data the first time 386 final long actualInitialSize = conn.getAdmin().getCurrentSpaceQuotaSnapshot(tn).getUsage(); 387 LOG.info("Initial table size was " + actualInitialSize); 388 389 LOG.info("Snapshot the table"); 390 final String snapshot1 = tn.toString() + "_snapshot1"; 391 admin.snapshot(snapshot1, tn); 392 393 admin.cloneSnapshot(snapshot1, tn2); 394 395 // Write some more data to the first table 396 helper.writeData(tn, initialSize, "q2"); 397 admin.flush(tn); 398 399 // Watch the usage of the first table with some more data to know when the new 400 // region size reports were sent to the master 401 TEST_UTIL.waitFor(30_000, 1_000, new SpaceQuotaSnapshotPredicate(conn, tn) { 402 @Override 403 boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception { 404 return snapshot.getUsage() >= actualInitialSize * 2; 405 } 406 }); 407 408 // We know that reports were sent by our RS, verify that they take up zero size. 409 SpaceQuotaSnapshot snapshot = 410 (SpaceQuotaSnapshot) conn.getAdmin().getCurrentSpaceQuotaSnapshot(tn2); 411 assertNotNull(snapshot); 412 assertEquals(0, snapshot.getUsage()); 413 414 // Compact the cloned table to force it to own its own files. 415 TEST_UTIL.compact(tn2, true); 416 // After the table is compacted, it should have its own files and be the same size as originally 417 // But The compaction result file has an additional compaction event tracker 418 TEST_UTIL.waitFor(30_000, 1_000, new SpaceQuotaSnapshotPredicate(conn, tn2) { 419 @Override 420 boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception { 421 return snapshot.getUsage() >= actualInitialSize; 422 } 423 }); 424 } 425 426 void waitForStableQuotaSize(Connection conn, TableName tn, String ns) throws Exception { 427 // For some stability in the value before proceeding 428 // Helps make sure that we got the actual last value, not some inbetween 429 AtomicLong lastValue = new AtomicLong(-1); 430 AtomicInteger counter = new AtomicInteger(0); 431 TEST_UTIL.waitFor(15_000, 500, new SpaceQuotaSnapshotPredicate(conn, tn, ns) { 432 @Override boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception { 433 LOG.debug("Last observed size=" + lastValue.get()); 434 if (snapshot.getUsage() == lastValue.get()) { 435 int numMatches = counter.incrementAndGet(); 436 if (numMatches >= 5) { 437 return true; 438 } 439 // Not yet.. 440 return false; 441 } 442 counter.set(0); 443 lastValue.set(snapshot.getUsage()); 444 return false; 445 } 446 }); 447 } 448 449 long getRegionSizeReportForTable(Connection conn, TableName tn) throws IOException { 450 Map<TableName, Long> sizes = conn.getAdmin().getSpaceQuotaTableSizes(); 451 Long value = sizes.get(tn); 452 if (null == value) { 453 return 0L; 454 } 455 return value.longValue(); 456 } 457 458 void waitForStableRegionSizeReport(Connection conn, TableName tn) throws Exception { 459 // For some stability in the value before proceeding 460 // Helps make sure that we got the actual last value, not some inbetween 461 AtomicLong lastValue = new AtomicLong(-1); 462 AtomicInteger counter = new AtomicInteger(0); 463 TEST_UTIL.waitFor(15_000, 500, new Predicate<Exception>() { 464 @Override public boolean evaluate() throws Exception { 465 LOG.debug("Last observed size=" + lastValue.get()); 466 long actual = getRegionSizeReportForTable(conn, tn); 467 if (actual == lastValue.get()) { 468 int numMatches = counter.incrementAndGet(); 469 if (numMatches >= 5) { 470 return true; 471 } 472 // Not yet.. 473 return false; 474 } 475 counter.set(0); 476 lastValue.set(actual); 477 return false; 478 } 479 }); 480 } 481}