001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.quotas;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNotNull;
022
023import java.io.IOException;
024import java.util.Arrays;
025import java.util.HashSet;
026import java.util.Map;
027import java.util.concurrent.atomic.AtomicLong;
028import java.util.concurrent.atomic.AtomicReference;
029
030import org.apache.hadoop.conf.Configuration;
031import org.apache.hadoop.hbase.Cell;
032import org.apache.hadoop.hbase.CellScanner;
033import org.apache.hadoop.hbase.HBaseClassTestRule;
034import org.apache.hadoop.hbase.HBaseTestingUtility;
035import org.apache.hadoop.hbase.HColumnDescriptor;
036import org.apache.hadoop.hbase.HTableDescriptor;
037import org.apache.hadoop.hbase.NamespaceDescriptor;
038import org.apache.hadoop.hbase.TableName;
039import org.apache.hadoop.hbase.Waiter.Predicate;
040import org.apache.hadoop.hbase.client.Admin;
041import org.apache.hadoop.hbase.client.Connection;
042import org.apache.hadoop.hbase.client.Get;
043import org.apache.hadoop.hbase.client.Result;
044import org.apache.hadoop.hbase.client.ResultScanner;
045import org.apache.hadoop.hbase.client.Scan;
046import org.apache.hadoop.hbase.client.SnapshotDescription;
047import org.apache.hadoop.hbase.client.SnapshotType;
048import org.apache.hadoop.hbase.client.Table;
049import org.apache.hadoop.hbase.master.HMaster;
050import org.apache.hadoop.hbase.quotas.SnapshotQuotaObserverChore.SnapshotWithSize;
051import org.apache.hadoop.hbase.quotas.SpaceQuotaHelperForTests.NoFilesToDischarge;
052import org.apache.hadoop.hbase.quotas.SpaceQuotaHelperForTests.SpaceQuotaSnapshotPredicate;
053import org.apache.hadoop.hbase.regionserver.HStore;
054import org.apache.hadoop.hbase.testclassification.MediumTests;
055import org.junit.AfterClass;
056import org.junit.Before;
057import org.junit.BeforeClass;
058import org.junit.ClassRule;
059import org.junit.Rule;
060import org.junit.Test;
061import org.junit.experimental.categories.Category;
062import org.junit.rules.TestName;
063import org.slf4j.Logger;
064import org.slf4j.LoggerFactory;
065
066import org.apache.hbase.thirdparty.com.google.common.collect.HashMultimap;
067import org.apache.hbase.thirdparty.com.google.common.collect.Iterables;
068import org.apache.hbase.thirdparty.com.google.common.collect.Multimap;
069
070/**
071 * Test class for the {@link SnapshotQuotaObserverChore}.
072 */
073@Category(MediumTests.class)
074public class TestSnapshotQuotaObserverChore {
075
076  @ClassRule
077  public static final HBaseClassTestRule CLASS_RULE =
078      HBaseClassTestRule.forClass(TestSnapshotQuotaObserverChore.class);
079
080  private static final Logger LOG = LoggerFactory.getLogger(TestSnapshotQuotaObserverChore.class);
081  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
082  private static final AtomicLong COUNTER = new AtomicLong();
083
084  @Rule
085  public TestName testName = new TestName();
086
087  private Connection conn;
088  private Admin admin;
089  private SpaceQuotaHelperForTests helper;
090  private HMaster master;
091  private SnapshotQuotaObserverChore testChore;
092
093  @BeforeClass
094  public static void setUp() throws Exception {
095    Configuration conf = TEST_UTIL.getConfiguration();
096    SpaceQuotaHelperForTests.updateConfigForQuotas(conf);
097    // Clean up the compacted files faster than normal (15s instead of 2mins)
098    conf.setInt("hbase.hfile.compaction.discharger.interval", 15 * 1000);
099    TEST_UTIL.startMiniCluster(1);
100  }
101
102  @AfterClass
103  public static void tearDown() throws Exception {
104    TEST_UTIL.shutdownMiniCluster();
105  }
106
107  @Before
108  public void setup() throws Exception {
109    conn = TEST_UTIL.getConnection();
110    admin = TEST_UTIL.getAdmin();
111    helper = new SpaceQuotaHelperForTests(TEST_UTIL, testName, COUNTER);
112    master = TEST_UTIL.getHBaseCluster().getMaster();
113    helper.removeAllQuotas(conn);
114    testChore = new SnapshotQuotaObserverChore(
115        TEST_UTIL.getConnection(), TEST_UTIL.getConfiguration(), master.getFileSystem(), master,
116        null);
117  }
118
119  @Test
120  public void testSnapshotSizePersistence() throws IOException {
121    final Admin admin = TEST_UTIL.getAdmin();
122    final TableName tn = TableName.valueOf("quota_snapshotSizePersistence");
123    if (admin.tableExists(tn)) {
124      admin.disableTable(tn);
125      admin.deleteTable(tn);
126    }
127    HTableDescriptor desc = new HTableDescriptor(tn);
128    desc.addFamily(new HColumnDescriptor(QuotaTableUtil.QUOTA_FAMILY_USAGE));
129    admin.createTable(desc);
130
131    Multimap<TableName,SnapshotWithSize> snapshotsWithSizes = HashMultimap.create();
132    try (Table table = conn.getTable(tn)) {
133      // Writing no values will result in no records written.
134      verify(table, () -> {
135        testChore.persistSnapshotSizes(table, snapshotsWithSizes);
136        assertEquals(0, count(table));
137      });
138
139      verify(table, () -> {
140        TableName originatingTable = TableName.valueOf("t1");
141        snapshotsWithSizes.put(originatingTable, new SnapshotWithSize("ss1", 1024L));
142        snapshotsWithSizes.put(originatingTable, new SnapshotWithSize("ss2", 4096L));
143        testChore.persistSnapshotSizes(table, snapshotsWithSizes);
144        assertEquals(2, count(table));
145        assertEquals(1024L, extractSnapshotSize(table, originatingTable, "ss1"));
146        assertEquals(4096L, extractSnapshotSize(table, originatingTable, "ss2"));
147      });
148
149      snapshotsWithSizes.clear();
150      verify(table, () -> {
151        snapshotsWithSizes.put(TableName.valueOf("t1"), new SnapshotWithSize("ss1", 1024L));
152        snapshotsWithSizes.put(TableName.valueOf("t2"), new SnapshotWithSize("ss2", 4096L));
153        snapshotsWithSizes.put(TableName.valueOf("t3"), new SnapshotWithSize("ss3", 8192L));
154        testChore.persistSnapshotSizes(table, snapshotsWithSizes);
155        assertEquals(3, count(table));
156        assertEquals(1024L, extractSnapshotSize(table, TableName.valueOf("t1"), "ss1"));
157        assertEquals(4096L, extractSnapshotSize(table, TableName.valueOf("t2"), "ss2"));
158        assertEquals(8192L, extractSnapshotSize(table, TableName.valueOf("t3"), "ss3"));
159      });
160    }
161  }
162
163  @Test
164  public void testSnapshotsFromTables() throws Exception {
165    TableName tn1 = helper.createTableWithRegions(1);
166    TableName tn2 = helper.createTableWithRegions(1);
167    TableName tn3 = helper.createTableWithRegions(1);
168
169    // Set a space quota on table 1 and 2 (but not 3)
170    admin.setQuota(QuotaSettingsFactory.limitTableSpace(
171        tn1, SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS));
172    admin.setQuota(QuotaSettingsFactory.limitTableSpace(
173        tn2, SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS));
174
175    // Create snapshots on each table (we didn't write any data, so just skipflush)
176    admin.snapshot(new SnapshotDescription(tn1 + "snapshot", tn1, SnapshotType.SKIPFLUSH));
177    admin.snapshot(new SnapshotDescription(tn2 + "snapshot", tn2, SnapshotType.SKIPFLUSH));
178    admin.snapshot(new SnapshotDescription(tn3 + "snapshot", tn3, SnapshotType.SKIPFLUSH));
179
180    Multimap<TableName,String> mapping = testChore.getSnapshotsToComputeSize();
181    assertEquals(2, mapping.size());
182    assertEquals(1, mapping.get(tn1).size());
183    assertEquals(tn1 + "snapshot", mapping.get(tn1).iterator().next());
184    assertEquals(1, mapping.get(tn2).size());
185    assertEquals(tn2 + "snapshot", mapping.get(tn2).iterator().next());
186
187    admin.snapshot(new SnapshotDescription(tn2 + "snapshot1", tn2, SnapshotType.SKIPFLUSH));
188    admin.snapshot(new SnapshotDescription(tn3 + "snapshot1", tn3, SnapshotType.SKIPFLUSH));
189
190    mapping = testChore.getSnapshotsToComputeSize();
191    assertEquals(3, mapping.size());
192    assertEquals(1, mapping.get(tn1).size());
193    assertEquals(tn1 + "snapshot", mapping.get(tn1).iterator().next());
194    assertEquals(2, mapping.get(tn2).size());
195    assertEquals(
196        new HashSet<String>(Arrays.asList(tn2 + "snapshot", tn2 + "snapshot1")), mapping.get(tn2));
197  }
198
199  @Test
200  public void testSnapshotsFromNamespaces() throws Exception {
201    NamespaceDescriptor ns = NamespaceDescriptor.create("snapshots_from_namespaces").build();
202    admin.createNamespace(ns);
203
204    TableName tn1 = helper.createTableWithRegions(ns.getName(), 1);
205    TableName tn2 = helper.createTableWithRegions(ns.getName(), 1);
206    TableName tn3 = helper.createTableWithRegions(1);
207
208    // Set a space quota on the namespace
209    admin.setQuota(QuotaSettingsFactory.limitNamespaceSpace(
210        ns.getName(), SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS));
211
212    // Create snapshots on each table (we didn't write any data, so just skipflush)
213    admin.snapshot(new SnapshotDescription(
214        tn1.getQualifierAsString() + "snapshot", tn1, SnapshotType.SKIPFLUSH));
215    admin.snapshot(new SnapshotDescription(
216        tn2.getQualifierAsString() + "snapshot", tn2, SnapshotType.SKIPFLUSH));
217    admin.snapshot(new SnapshotDescription(
218        tn3.getQualifierAsString() + "snapshot", tn3, SnapshotType.SKIPFLUSH));
219
220    Multimap<TableName,String> mapping = testChore.getSnapshotsToComputeSize();
221    assertEquals(2, mapping.size());
222    assertEquals(1, mapping.get(tn1).size());
223    assertEquals(tn1.getQualifierAsString() + "snapshot", mapping.get(tn1).iterator().next());
224    assertEquals(1, mapping.get(tn2).size());
225    assertEquals(tn2.getQualifierAsString() + "snapshot", mapping.get(tn2).iterator().next());
226
227    admin.snapshot(new SnapshotDescription(
228        tn2.getQualifierAsString() + "snapshot1", tn2, SnapshotType.SKIPFLUSH));
229    admin.snapshot(new SnapshotDescription(
230        tn3.getQualifierAsString() + "snapshot2", tn3, SnapshotType.SKIPFLUSH));
231
232    mapping = testChore.getSnapshotsToComputeSize();
233    assertEquals(3, mapping.size());
234    assertEquals(1, mapping.get(tn1).size());
235    assertEquals(tn1.getQualifierAsString() + "snapshot", mapping.get(tn1).iterator().next());
236    assertEquals(2, mapping.get(tn2).size());
237    assertEquals(
238        new HashSet<String>(Arrays.asList(tn2.getQualifierAsString() + "snapshot",
239            tn2.getQualifierAsString() + "snapshot1")), mapping.get(tn2));
240  }
241
242  @Test
243  public void testSnapshotSize() throws Exception {
244    // Create a table and set a quota
245    TableName tn1 = helper.createTableWithRegions(5);
246    admin.setQuota(QuotaSettingsFactory.limitTableSpace(
247        tn1, SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS));
248
249    // Write some data and flush it
250    helper.writeData(tn1, 256L * SpaceQuotaHelperForTests.ONE_KILOBYTE);
251    admin.flush(tn1);
252
253    final long snapshotSize = TEST_UTIL.getMiniHBaseCluster().getRegions(tn1).stream()
254        .flatMap(r -> r.getStores().stream()).mapToLong(HStore::getHFilesSize).sum();
255
256    // Wait for the Master chore to run to see the usage (with a fudge factor)
257    TEST_UTIL.waitFor(30_000, new SpaceQuotaSnapshotPredicate(conn, tn1) {
258      @Override
259      boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
260        return snapshot.getUsage() == snapshotSize;
261      }
262    });
263
264    // Create a snapshot on the table
265    final String snapshotName = tn1 + "snapshot";
266    admin.snapshot(new SnapshotDescription(snapshotName, tn1, SnapshotType.SKIPFLUSH));
267
268    // Get the snapshots
269    Multimap<TableName,String> snapshotsToCompute = testChore.getSnapshotsToComputeSize();
270    assertEquals(
271        "Expected to see the single snapshot: " + snapshotsToCompute, 1, snapshotsToCompute.size());
272
273    // Get the size of our snapshot
274    Multimap<TableName,SnapshotWithSize> snapshotsWithSize = testChore.computeSnapshotSizes(
275        snapshotsToCompute);
276    assertEquals(1, snapshotsWithSize.size());
277    SnapshotWithSize sws = Iterables.getOnlyElement(snapshotsWithSize.get(tn1));
278    assertEquals(snapshotName, sws.getName());
279    // The snapshot should take up no space since the table refers to it completely
280    assertEquals(0, sws.getSize());
281
282    // Write some more data, flush it, and then major_compact the table
283    helper.writeData(tn1, 256L * SpaceQuotaHelperForTests.ONE_KILOBYTE);
284    admin.flush(tn1);
285    TEST_UTIL.compact(tn1, true);
286
287    // Test table should reflect it's original size since ingest was deterministic
288    TEST_UTIL.waitFor(30_000, new SpaceQuotaSnapshotPredicate(conn, tn1) {
289      private final long regionSize = TEST_UTIL.getMiniHBaseCluster().getRegions(tn1).stream()
290          .flatMap(r -> r.getStores().stream()).mapToLong(HStore::getHFilesSize).sum();
291
292      @Override
293      boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
294        LOG.debug("Current usage=" + snapshot.getUsage() + " snapshotSize=" + snapshotSize);
295        // The usage of table space consists of region size and snapshot size
296        return closeInSize(snapshot.getUsage(), snapshotSize + regionSize,
297            SpaceQuotaHelperForTests.ONE_KILOBYTE);
298      }
299    });
300
301    // Wait for no compacted files on the regions of our table
302    TEST_UTIL.waitFor(30_000, new NoFilesToDischarge(TEST_UTIL.getMiniHBaseCluster(), tn1));
303
304    // Still should see only one snapshot
305    snapshotsToCompute = testChore.getSnapshotsToComputeSize();
306    assertEquals(
307        "Expected to see the single snapshot: " + snapshotsToCompute, 1, snapshotsToCompute.size());
308    snapshotsWithSize = testChore.computeSnapshotSizes(
309            snapshotsToCompute);
310    assertEquals(1, snapshotsWithSize.size());
311    sws = Iterables.getOnlyElement(snapshotsWithSize.get(tn1));
312    assertEquals(snapshotName, sws.getName());
313    // The snapshot should take up the size the table originally took up
314    assertEquals(snapshotSize, sws.getSize());
315  }
316
317  @Test
318  public void testPersistingSnapshotsForNamespaces() throws Exception {
319    Multimap<TableName,SnapshotWithSize> snapshotsWithSizes = HashMultimap.create();
320    TableName tn1 = TableName.valueOf("ns1:tn1");
321    TableName tn2 = TableName.valueOf("ns1:tn2");
322    TableName tn3 = TableName.valueOf("ns2:tn1");
323    TableName tn4 = TableName.valueOf("ns2:tn2");
324    TableName tn5 = TableName.valueOf("tn1");
325
326    snapshotsWithSizes.put(tn1, new SnapshotWithSize("", 1024L));
327    snapshotsWithSizes.put(tn2, new SnapshotWithSize("", 1024L));
328    snapshotsWithSizes.put(tn3, new SnapshotWithSize("", 512L));
329    snapshotsWithSizes.put(tn4, new SnapshotWithSize("", 1024L));
330    snapshotsWithSizes.put(tn5, new SnapshotWithSize("", 3072L));
331
332    Map<String,Long> nsSizes = testChore.groupSnapshotSizesByNamespace(snapshotsWithSizes);
333    assertEquals(3, nsSizes.size());
334    assertEquals(2048L, (long) nsSizes.get("ns1"));
335    assertEquals(1536L, (long) nsSizes.get("ns2"));
336    assertEquals(3072L, (long) nsSizes.get(NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR));
337  }
338
339  @Test
340  public void testRemovedSnapshots() throws Exception {
341    // Create a table and set a quota
342    TableName tn1 = helper.createTableWithRegions(1);
343    admin.setQuota(QuotaSettingsFactory.limitTableSpace(tn1, SpaceQuotaHelperForTests.ONE_GIGABYTE,
344        SpaceViolationPolicy.NO_INSERTS));
345
346    // Write some data and flush it
347    helper.writeData(tn1, 256L * SpaceQuotaHelperForTests.ONE_KILOBYTE); // 256 KB
348
349    final AtomicReference<Long> lastSeenSize = new AtomicReference<>();
350    // Wait for the Master chore to run to see the usage (with a fudge factor)
351    TEST_UTIL.waitFor(30_000, new SpaceQuotaSnapshotPredicate(conn, tn1) {
352      @Override
353      boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
354        lastSeenSize.set(snapshot.getUsage());
355        return snapshot.getUsage() > 230L * SpaceQuotaHelperForTests.ONE_KILOBYTE;
356      }
357    });
358
359    // Create a snapshot on the table
360    final String snapshotName1 = tn1 + "snapshot1";
361    admin.snapshot(new SnapshotDescription(snapshotName1, tn1, SnapshotType.SKIPFLUSH));
362
363    // Snapshot size has to be 0 as the snapshot shares the data with the table
364    final Table quotaTable = conn.getTable(QuotaUtil.QUOTA_TABLE_NAME);
365    TEST_UTIL.waitFor(30_000, new Predicate<Exception>() {
366      @Override
367      public boolean evaluate() throws Exception {
368        Get g = QuotaTableUtil.makeGetForSnapshotSize(tn1, snapshotName1);
369        Result r = quotaTable.get(g);
370        if (r == null || r.isEmpty()) {
371          return false;
372        }
373        r.advance();
374        Cell c = r.current();
375        return QuotaTableUtil.parseSnapshotSize(c) == 0;
376      }
377    });
378    // Total usage has to remain same as what we saw before taking a snapshot
379    TEST_UTIL.waitFor(30_000, new SpaceQuotaSnapshotPredicate(conn, tn1) {
380      @Override
381      boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
382        return snapshot.getUsage() == lastSeenSize.get();
383      }
384    });
385
386    // Major compact the table to force a rewrite
387    TEST_UTIL.compact(tn1, true);
388    // Now the snapshot size has to prev total size
389    TEST_UTIL.waitFor(30_000, new Predicate<Exception>() {
390      @Override
391      public boolean evaluate() throws Exception {
392        Get g = QuotaTableUtil.makeGetForSnapshotSize(tn1, snapshotName1);
393        Result r = quotaTable.get(g);
394        if (r == null || r.isEmpty()) {
395          return false;
396        }
397        r.advance();
398        Cell c = r.current();
399        // The compaction result file has an additional compaction event tracker
400        return lastSeenSize.get() == QuotaTableUtil.parseSnapshotSize(c);
401      }
402    });
403    // The total size now has to be equal/more than double of prev total size
404    // as double the number of store files exist now.
405    final AtomicReference<Long> sizeAfterCompaction = new AtomicReference<>();
406    TEST_UTIL.waitFor(30_000, new SpaceQuotaSnapshotPredicate(conn, tn1) {
407      @Override
408      boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
409        sizeAfterCompaction.set(snapshot.getUsage());
410        return snapshot.getUsage() >= 2 * lastSeenSize.get();
411      }
412    });
413
414    // Delete the snapshot
415    admin.deleteSnapshot(snapshotName1);
416    // Total size has to come down to prev totalsize - snapshot size(which was removed)
417    TEST_UTIL.waitFor(30_000, new SpaceQuotaSnapshotPredicate(conn, tn1) {
418      @Override
419      boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
420        return snapshot.getUsage() == (sizeAfterCompaction.get() - lastSeenSize.get());
421      }
422    });
423  }
424
425  private long count(Table t) throws IOException {
426    try (ResultScanner rs = t.getScanner(new Scan())) {
427      long sum = 0;
428      for (Result r : rs) {
429        while (r.advance()) {
430          sum++;
431        }
432      }
433      return sum;
434    }
435  }
436
437  private long extractSnapshotSize(
438      Table quotaTable, TableName tn, String snapshot) throws IOException {
439    Get g = QuotaTableUtil.makeGetForSnapshotSize(tn, snapshot);
440    Result r = quotaTable.get(g);
441    assertNotNull(r);
442    CellScanner cs = r.cellScanner();
443    cs.advance();
444    Cell c = cs.current();
445    assertNotNull(c);
446    return QuotaTableUtil.extractSnapshotSize(
447        c.getValueArray(), c.getValueOffset(), c.getValueLength());
448  }
449
450  private void verify(Table t, IOThrowingRunnable test) throws IOException {
451    admin.disableTable(t.getName());
452    admin.truncateTable(t.getName(), false);
453    test.run();
454  }
455
456  @FunctionalInterface
457  private interface IOThrowingRunnable {
458    void run() throws IOException;
459  }
460
461  /**
462   * Computes if {@code size2} is within {@code delta} of {@code size1}, inclusive.
463   */
464  boolean closeInSize(long size1, long size2, long delta) {
465    long lower = size1 - delta;
466    long upper = size1 + delta;
467    return lower <= size2 && size2 <= upper;
468  }
469}