001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.quotas;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertNotNull;
023import static org.junit.Assert.assertTrue;
024
025import java.io.IOException;
026import java.util.Arrays;
027import java.util.Collection;
028import java.util.HashSet;
029import java.util.Map;
030import java.util.Map.Entry;
031import java.util.Set;
032import java.util.concurrent.TimeUnit;
033import java.util.concurrent.atomic.AtomicLong;
034import java.util.concurrent.atomic.AtomicReference;
035
036import org.apache.hadoop.conf.Configuration;
037import org.apache.hadoop.fs.FileSystem;
038import org.apache.hadoop.fs.Path;
039import org.apache.hadoop.hbase.Cell;
040import org.apache.hadoop.hbase.HBaseClassTestRule;
041import org.apache.hadoop.hbase.HBaseTestingUtility;
042import org.apache.hadoop.hbase.HConstants;
043import org.apache.hadoop.hbase.NamespaceDescriptor;
044import org.apache.hadoop.hbase.TableName;
045import org.apache.hadoop.hbase.Waiter.Predicate;
046import org.apache.hadoop.hbase.client.Admin;
047import org.apache.hadoop.hbase.client.Connection;
048import org.apache.hadoop.hbase.client.Get;
049import org.apache.hadoop.hbase.client.RegionInfo;
050import org.apache.hadoop.hbase.client.Result;
051import org.apache.hadoop.hbase.client.SnapshotDescription;
052import org.apache.hadoop.hbase.client.SnapshotType;
053import org.apache.hadoop.hbase.client.Table;
054import org.apache.hadoop.hbase.master.HMaster;
055import org.apache.hadoop.hbase.quotas.SpaceQuotaHelperForTests.NoFilesToDischarge;
056import org.apache.hadoop.hbase.quotas.SpaceQuotaHelperForTests.SpaceQuotaSnapshotPredicate;
057import org.apache.hadoop.hbase.regionserver.HStore;
058import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
059import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil.SnapshotVisitor;
060import org.apache.hadoop.hbase.testclassification.MediumTests;
061import org.junit.AfterClass;
062import org.junit.Before;
063import org.junit.BeforeClass;
064import org.junit.ClassRule;
065import org.junit.Rule;
066import org.junit.Test;
067import org.junit.experimental.categories.Category;
068import org.junit.rules.TestName;
069import org.slf4j.Logger;
070import org.slf4j.LoggerFactory;
071
072import org.apache.hbase.thirdparty.com.google.common.collect.HashMultimap;
073import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap;
074import org.apache.hbase.thirdparty.com.google.common.collect.Multimap;
075
076import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile;
077
078/**
079 * Test class for the {@link SnapshotQuotaObserverChore}.
080 */
081@Category(MediumTests.class)
082public class TestSnapshotQuotaObserverChore {
083
084  @ClassRule
085  public static final HBaseClassTestRule CLASS_RULE =
086      HBaseClassTestRule.forClass(TestSnapshotQuotaObserverChore.class);
087
088  private static final Logger LOG = LoggerFactory.getLogger(TestSnapshotQuotaObserverChore.class);
089  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
090  private static final AtomicLong COUNTER = new AtomicLong();
091
092  @Rule
093  public TestName testName = new TestName();
094
095  private Connection conn;
096  private Admin admin;
097  private SpaceQuotaHelperForTests helper;
098  private HMaster master;
099  private SnapshotQuotaObserverChore testChore;
100
101  @BeforeClass
102  public static void setUp() throws Exception {
103    Configuration conf = TEST_UTIL.getConfiguration();
104    SpaceQuotaHelperForTests.updateConfigForQuotas(conf);
105    // Clean up the compacted files faster than normal (15s instead of 2mins)
106    conf.setInt("hbase.hfile.compaction.discharger.interval", 15 * 1000);
107    TEST_UTIL.startMiniCluster(1);
108  }
109
110  @AfterClass
111  public static void tearDown() throws Exception {
112    TEST_UTIL.shutdownMiniCluster();
113  }
114
115  @Before
116  public void setup() throws Exception {
117    conn = TEST_UTIL.getConnection();
118    admin = TEST_UTIL.getAdmin();
119    helper = new SpaceQuotaHelperForTests(TEST_UTIL, testName, COUNTER);
120    master = TEST_UTIL.getHBaseCluster().getMaster();
121    helper.removeAllQuotas(conn);
122    testChore = new SnapshotQuotaObserverChore(
123        TEST_UTIL.getConnection(), TEST_UTIL.getConfiguration(), master.getFileSystem(), master,
124        null);
125  }
126
127  @Test
128  public void testSnapshotsFromTables() throws Exception {
129    TableName tn1 = helper.createTableWithRegions(1);
130    TableName tn2 = helper.createTableWithRegions(1);
131    TableName tn3 = helper.createTableWithRegions(1);
132
133    // Set a space quota on table 1 and 2 (but not 3)
134    admin.setQuota(QuotaSettingsFactory.limitTableSpace(
135        tn1, SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS));
136    admin.setQuota(QuotaSettingsFactory.limitTableSpace(
137        tn2, SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS));
138
139    // Create snapshots on each table (we didn't write any data, so just skipflush)
140    admin.snapshot(new SnapshotDescription(tn1 + "snapshot", tn1, SnapshotType.SKIPFLUSH));
141    admin.snapshot(new SnapshotDescription(tn2 + "snapshot", tn2, SnapshotType.SKIPFLUSH));
142    admin.snapshot(new SnapshotDescription(tn3 + "snapshot", tn3, SnapshotType.SKIPFLUSH));
143
144    Multimap<TableName,String> mapping = testChore.getSnapshotsToComputeSize();
145    assertEquals(2, mapping.size());
146    assertEquals(1, mapping.get(tn1).size());
147    assertEquals(tn1 + "snapshot", mapping.get(tn1).iterator().next());
148    assertEquals(1, mapping.get(tn2).size());
149    assertEquals(tn2 + "snapshot", mapping.get(tn2).iterator().next());
150
151    admin.snapshot(new SnapshotDescription(tn2 + "snapshot1", tn2, SnapshotType.SKIPFLUSH));
152    admin.snapshot(new SnapshotDescription(tn3 + "snapshot1", tn3, SnapshotType.SKIPFLUSH));
153
154    mapping = testChore.getSnapshotsToComputeSize();
155    assertEquals(3, mapping.size());
156    assertEquals(1, mapping.get(tn1).size());
157    assertEquals(tn1 + "snapshot", mapping.get(tn1).iterator().next());
158    assertEquals(2, mapping.get(tn2).size());
159    assertEquals(
160        new HashSet<String>(Arrays.asList(tn2 + "snapshot", tn2 + "snapshot1")), mapping.get(tn2));
161  }
162
163  @Test
164  public void testSnapshotsFromNamespaces() throws Exception {
165    NamespaceDescriptor ns = NamespaceDescriptor.create("snapshots_from_namespaces").build();
166    admin.createNamespace(ns);
167
168    TableName tn1 = helper.createTableWithRegions(ns.getName(), 1);
169    TableName tn2 = helper.createTableWithRegions(ns.getName(), 1);
170    TableName tn3 = helper.createTableWithRegions(1);
171
172    // Set a throttle quota on 'default' namespace
173    admin.setQuota(QuotaSettingsFactory.throttleNamespace(tn3.getNamespaceAsString(),
174      ThrottleType.WRITE_NUMBER, 100, TimeUnit.SECONDS));
175    // Set a user throttle quota
176    admin.setQuota(
177      QuotaSettingsFactory.throttleUser("user", ThrottleType.WRITE_NUMBER, 100, TimeUnit.MINUTES));
178
179    // Set a space quota on the namespace
180    admin.setQuota(QuotaSettingsFactory.limitNamespaceSpace(
181        ns.getName(), SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS));
182
183    // Create snapshots on each table (we didn't write any data, so just skipflush)
184    admin.snapshot(new SnapshotDescription(
185        tn1.getQualifierAsString() + "snapshot", tn1, SnapshotType.SKIPFLUSH));
186    admin.snapshot(new SnapshotDescription(
187        tn2.getQualifierAsString() + "snapshot", tn2, SnapshotType.SKIPFLUSH));
188    admin.snapshot(new SnapshotDescription(
189        tn3.getQualifierAsString() + "snapshot", tn3, SnapshotType.SKIPFLUSH));
190
191    Multimap<TableName,String> mapping = testChore.getSnapshotsToComputeSize();
192    assertEquals(2, mapping.size());
193    assertEquals(1, mapping.get(tn1).size());
194    assertEquals(tn1.getQualifierAsString() + "snapshot", mapping.get(tn1).iterator().next());
195    assertEquals(1, mapping.get(tn2).size());
196    assertEquals(tn2.getQualifierAsString() + "snapshot", mapping.get(tn2).iterator().next());
197
198    admin.snapshot(new SnapshotDescription(
199        tn2.getQualifierAsString() + "snapshot1", tn2, SnapshotType.SKIPFLUSH));
200    admin.snapshot(new SnapshotDescription(
201        tn3.getQualifierAsString() + "snapshot2", tn3, SnapshotType.SKIPFLUSH));
202
203    mapping = testChore.getSnapshotsToComputeSize();
204    assertEquals(3, mapping.size());
205    assertEquals(1, mapping.get(tn1).size());
206    assertEquals(tn1.getQualifierAsString() + "snapshot", mapping.get(tn1).iterator().next());
207    assertEquals(2, mapping.get(tn2).size());
208    assertEquals(
209        new HashSet<String>(Arrays.asList(tn2.getQualifierAsString() + "snapshot",
210            tn2.getQualifierAsString() + "snapshot1")), mapping.get(tn2));
211  }
212
213  @Test
214  public void testSnapshotSize() throws Exception {
215    // Create a table and set a quota
216    TableName tn1 = helper.createTableWithRegions(5);
217    admin.setQuota(QuotaSettingsFactory.limitTableSpace(
218        tn1, SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS));
219
220    // Write some data and flush it
221    helper.writeData(tn1, 256L * SpaceQuotaHelperForTests.ONE_KILOBYTE);
222    admin.flush(tn1);
223
224    final long snapshotSize = TEST_UTIL.getMiniHBaseCluster().getRegions(tn1).stream()
225        .flatMap(r -> r.getStores().stream()).mapToLong(HStore::getHFilesSize).sum();
226
227    // Wait for the Master chore to run to see the usage (with a fudge factor)
228    TEST_UTIL.waitFor(30_000, new SpaceQuotaSnapshotPredicate(conn, tn1) {
229      @Override
230      boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
231        return snapshot.getUsage() == snapshotSize;
232      }
233    });
234
235    // Create a snapshot on the table
236    final String snapshotName = tn1 + "snapshot";
237    admin.snapshot(new SnapshotDescription(snapshotName, tn1, SnapshotType.SKIPFLUSH));
238
239    // Get the snapshots
240    Multimap<TableName,String> snapshotsToCompute = testChore.getSnapshotsToComputeSize();
241    assertEquals(
242        "Expected to see the single snapshot: " + snapshotsToCompute, 1, snapshotsToCompute.size());
243
244    // Get the size of our snapshot
245    Map<String,Long> namespaceSnapshotSizes = testChore.computeSnapshotSizes(
246        snapshotsToCompute);
247    assertEquals(1, namespaceSnapshotSizes.size());
248    Long size = namespaceSnapshotSizes.get(tn1.getNamespaceAsString());
249    assertNotNull(size);
250    // The snapshot should take up no space since the table refers to it completely
251    assertEquals(0, size.longValue());
252
253    // Write some more data, flush it, and then major_compact the table
254    helper.writeData(tn1, 256L * SpaceQuotaHelperForTests.ONE_KILOBYTE);
255    admin.flush(tn1);
256    TEST_UTIL.compact(tn1, true);
257
258    // Test table should reflect it's original size since ingest was deterministic
259    TEST_UTIL.waitFor(30_000, new SpaceQuotaSnapshotPredicate(conn, tn1) {
260      private final long regionSize = TEST_UTIL.getMiniHBaseCluster().getRegions(tn1).stream()
261          .flatMap(r -> r.getStores().stream()).mapToLong(HStore::getHFilesSize).sum();
262
263      @Override
264      boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
265        LOG.debug("Current usage=" + snapshot.getUsage() + " snapshotSize=" + snapshotSize);
266        // The usage of table space consists of region size and snapshot size
267        return closeInSize(snapshot.getUsage(), snapshotSize + regionSize,
268            SpaceQuotaHelperForTests.ONE_KILOBYTE);
269      }
270    });
271
272    // Wait for no compacted files on the regions of our table
273    TEST_UTIL.waitFor(30_000, new NoFilesToDischarge(TEST_UTIL.getMiniHBaseCluster(), tn1));
274
275    // Still should see only one snapshot
276    snapshotsToCompute = testChore.getSnapshotsToComputeSize();
277    assertEquals(
278        "Expected to see the single snapshot: " + snapshotsToCompute, 1, snapshotsToCompute.size());
279    namespaceSnapshotSizes = testChore.computeSnapshotSizes(
280            snapshotsToCompute);
281    assertEquals(1, namespaceSnapshotSizes.size());
282    size = namespaceSnapshotSizes.get(tn1.getNamespaceAsString());
283    assertNotNull(size);
284    // The snapshot should take up the size the table originally took up
285    assertEquals(snapshotSize, size.longValue());
286  }
287
288  @Test
289  public void testPersistingSnapshotsForNamespaces() throws Exception {
290    TableName tn1 = TableName.valueOf("ns1:tn1");
291    TableName tn2 = TableName.valueOf("ns1:tn2");
292    TableName tn3 = TableName.valueOf("ns2:tn1");
293    TableName tn4 = TableName.valueOf("ns2:tn2");
294    TableName tn5 = TableName.valueOf("tn1");
295    // Shim in a custom factory to avoid computing snapshot sizes.
296    FileArchiverNotifierFactory test = new FileArchiverNotifierFactory() {
297      Map<TableName,Long> tableToSize = ImmutableMap.of(
298          tn1, 1024L, tn2, 1024L, tn3, 512L, tn4, 1024L, tn5, 3072L);
299      @Override
300      public FileArchiverNotifier get(
301          Connection conn, Configuration conf, FileSystem fs, TableName tn) {
302        return new FileArchiverNotifier() {
303          @Override public void addArchivedFiles(Set<Entry<String,Long>> fileSizes)
304              throws IOException {}
305
306          @Override
307          public long computeAndStoreSnapshotSizes(Collection<String> currentSnapshots)
308              throws IOException {
309            return tableToSize.get(tn);
310          }
311        };
312      }
313    };
314    try {
315      FileArchiverNotifierFactoryImpl.setInstance(test);
316
317      Multimap<TableName,String> snapshotsToCompute = HashMultimap.create();
318      snapshotsToCompute.put(tn1, "");
319      snapshotsToCompute.put(tn2, "");
320      snapshotsToCompute.put(tn3, "");
321      snapshotsToCompute.put(tn4, "");
322      snapshotsToCompute.put(tn5, "");
323      Map<String,Long> nsSizes = testChore.computeSnapshotSizes(snapshotsToCompute);
324      assertEquals(3, nsSizes.size());
325      assertEquals(2048L, (long) nsSizes.get("ns1"));
326      assertEquals(1536L, (long) nsSizes.get("ns2"));
327      assertEquals(3072L, (long) nsSizes.get(NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR));
328    } finally {
329      FileArchiverNotifierFactoryImpl.reset();
330    }
331  }
332
333  @Test
334  public void testRemovedSnapshots() throws Exception {
335    // Create a table and set a quota
336    TableName tn1 = helper.createTableWithRegions(1);
337    admin.setQuota(QuotaSettingsFactory.limitTableSpace(tn1, SpaceQuotaHelperForTests.ONE_GIGABYTE,
338        SpaceViolationPolicy.NO_INSERTS));
339
340    // Write some data and flush it
341    helper.writeData(tn1, 256L * SpaceQuotaHelperForTests.ONE_KILOBYTE); // 256 KB
342
343    final AtomicReference<Long> lastSeenSize = new AtomicReference<>();
344    // Wait for the Master chore to run to see the usage (with a fudge factor)
345    TEST_UTIL.waitFor(30_000, new SpaceQuotaSnapshotPredicate(conn, tn1) {
346      @Override
347      boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
348        lastSeenSize.set(snapshot.getUsage());
349        return snapshot.getUsage() > 230L * SpaceQuotaHelperForTests.ONE_KILOBYTE;
350      }
351    });
352
353    // Create a snapshot on the table
354    final String snapshotName1 = tn1 + "snapshot1";
355    admin.snapshot(new SnapshotDescription(snapshotName1, tn1, SnapshotType.SKIPFLUSH));
356
357    // Snapshot size has to be 0 as the snapshot shares the data with the table
358    final Table quotaTable = conn.getTable(QuotaUtil.QUOTA_TABLE_NAME);
359    TEST_UTIL.waitFor(30_000, new Predicate<Exception>() {
360      @Override
361      public boolean evaluate() throws Exception {
362        Get g = QuotaTableUtil.makeGetForSnapshotSize(tn1, snapshotName1);
363        Result r = quotaTable.get(g);
364        if (r == null || r.isEmpty()) {
365          return false;
366        }
367        r.advance();
368        Cell c = r.current();
369        return QuotaTableUtil.parseSnapshotSize(c) == 0;
370      }
371    });
372    // Total usage has to remain same as what we saw before taking a snapshot
373    TEST_UTIL.waitFor(30_000, new SpaceQuotaSnapshotPredicate(conn, tn1) {
374      @Override
375      boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
376        return snapshot.getUsage() == lastSeenSize.get();
377      }
378    });
379
380    // Major compact the table to force a rewrite
381    TEST_UTIL.compact(tn1, true);
382    // Now the snapshot size has to prev total size
383    TEST_UTIL.waitFor(30_000, new Predicate<Exception>() {
384      @Override
385      public boolean evaluate() throws Exception {
386        Get g = QuotaTableUtil.makeGetForSnapshotSize(tn1, snapshotName1);
387        Result r = quotaTable.get(g);
388        if (r == null || r.isEmpty()) {
389          return false;
390        }
391        r.advance();
392        Cell c = r.current();
393        // The compaction result file has an additional compaction event tracker
394        return lastSeenSize.get() == QuotaTableUtil.parseSnapshotSize(c);
395      }
396    });
397    // The total size now has to be equal/more than double of prev total size
398    // as double the number of store files exist now.
399    final AtomicReference<Long> sizeAfterCompaction = new AtomicReference<>();
400    TEST_UTIL.waitFor(30_000, new SpaceQuotaSnapshotPredicate(conn, tn1) {
401      @Override
402      boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
403        sizeAfterCompaction.set(snapshot.getUsage());
404        return snapshot.getUsage() >= 2 * lastSeenSize.get();
405      }
406    });
407
408    // Delete the snapshot
409    admin.deleteSnapshot(snapshotName1);
410    // Total size has to come down to prev totalsize - snapshot size(which was removed)
411    TEST_UTIL.waitFor(30_000, new SpaceQuotaSnapshotPredicate(conn, tn1) {
412      @Override
413      boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
414        return snapshot.getUsage() == (sizeAfterCompaction.get() - lastSeenSize.get());
415      }
416    });
417  }
418
419  @Test
420  public void testBucketingFilesToSnapshots() throws Exception {
421    // Create a table and set a quota
422    TableName tn1 = helper.createTableWithRegions(1);
423    admin.setQuota(QuotaSettingsFactory.limitTableSpace(
424        tn1, SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS));
425
426    // Write some data and flush it
427    helper.writeData(tn1, 256L * SpaceQuotaHelperForTests.ONE_KILOBYTE);
428    admin.flush(tn1);
429
430    final AtomicReference<Long> lastSeenSize = new AtomicReference<>();
431    // Wait for the Master chore to run to see the usage (with a fudge factor)
432    TEST_UTIL.waitFor(30_000, new SpaceQuotaSnapshotPredicate(conn, tn1) {
433      @Override
434      boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
435        lastSeenSize.set(snapshot.getUsage());
436        return snapshot.getUsage() > 230L * SpaceQuotaHelperForTests.ONE_KILOBYTE;
437      }
438    });
439
440    // Create a snapshot on the table
441    final String snapshotName1 = tn1 + "snapshot1";
442    admin.snapshot(new SnapshotDescription(snapshotName1, tn1, SnapshotType.SKIPFLUSH));
443    // Major compact the table to force a rewrite
444    TEST_UTIL.compact(tn1, true);
445
446    // Make sure that the snapshot owns the size
447    final Table quotaTable = conn.getTable(QuotaUtil.QUOTA_TABLE_NAME);
448    TEST_UTIL.waitFor(30_000, new Predicate<Exception>() {
449      @Override
450      public boolean evaluate() throws Exception {
451        LOG.info("Waiting to see quota snapshot1 size");
452        debugFilesForSnapshot(tn1, snapshotName1);
453        Get g = QuotaTableUtil.makeGetForSnapshotSize(tn1, snapshotName1);
454        Result r = quotaTable.get(g);
455        if (r == null || r.isEmpty()) {
456          return false;
457        }
458        r.advance();
459        Cell c = r.current();
460        // The compaction result file has an additional compaction event tracker
461        return lastSeenSize.get() == QuotaTableUtil.parseSnapshotSize(c);
462      }
463    });
464
465    LOG.info("Snapshotting table again");
466    // Create another snapshot on the table
467    final String snapshotName2 = tn1 + "snapshot2";
468    admin.snapshot(new SnapshotDescription(snapshotName2, tn1, SnapshotType.SKIPFLUSH));
469    LOG.info("Compacting table");
470    // Major compact the table to force a rewrite
471    TEST_UTIL.compact(tn1, true);
472
473    // Make sure that the snapshot owns the size
474    TEST_UTIL.waitFor(30_000, new Predicate<Exception>() {
475      @Override
476      public boolean evaluate() throws Exception {
477        LOG.info("Waiting to see quota snapshot2 size");
478        debugFilesForSnapshot(tn1, snapshotName2);
479        Get g = QuotaTableUtil.makeGetForSnapshotSize(tn1, snapshotName2);
480        Result r = quotaTable.get(g);
481        if (r == null || r.isEmpty()) {
482          return false;
483        }
484        r.advance();
485        Cell c = r.current();
486        return closeInSize(lastSeenSize.get(),
487            QuotaTableUtil.parseSnapshotSize(c), SpaceQuotaHelperForTests.ONE_KILOBYTE);
488      }
489    });
490
491    Get g = QuotaTableUtil.createGetNamespaceSnapshotSize(tn1.getNamespaceAsString());
492    Result r = quotaTable.get(g);
493    assertNotNull(r);
494    assertFalse(r.isEmpty());
495    r.advance();
496    long size = QuotaTableUtil.parseSnapshotSize(r.current());
497    // Two snapshots of equal size.
498    assertTrue(closeInSize(lastSeenSize.get() * 2, size, SpaceQuotaHelperForTests.ONE_KILOBYTE));
499  }
500
501  /**
502   * Prints details about every file referenced by the snapshot with the given name.
503   */
504  void debugFilesForSnapshot(TableName table, String snapshot) throws IOException {
505    final Configuration conf = TEST_UTIL.getConfiguration();
506    final FileSystem fs = TEST_UTIL.getTestFileSystem();
507    final Path snapshotDir = new Path(conf.get("hbase.rootdir"), HConstants.SNAPSHOT_DIR_NAME);
508    SnapshotReferenceUtil.visitReferencedFiles(conf, fs, new Path(snapshotDir, snapshot),
509        new SnapshotVisitor() {
510          @Override
511          public void storeFile(
512              RegionInfo regionInfo, String familyName, StoreFile storeFile) throws IOException {
513            LOG.info("Snapshot={} references file={}, size={}", snapshot, storeFile.getName(),
514                storeFile.getFileSize());
515          }
516        }
517    );
518  }
519
520  /**
521   * Computes if {@code size2} is within {@code delta} of {@code size1}, inclusive.
522   *
523   * The size of our store files will change after the first major compaction as the last
524   * compaction gets serialized into the store file (see the fields referenced by
525   * COMPACTION_EVENT_KEY in HFilePrettyPrinter).
526   */
527  boolean closeInSize(long size1, long size2, long delta) {
528    long lower = size1 - delta;
529    long upper = size1 + delta;
530    return lower <= size2 && size2 <= upper;
531  }
532}