Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.quotas;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertFalse;
022import static org.junit.jupiter.api.Assertions.assertNotNull;
023import static org.junit.jupiter.api.Assertions.assertTrue;
024
025import java.io.IOException;
026import java.util.Arrays;
027import java.util.Collection;
028import java.util.HashSet;
029import java.util.Map;
030import java.util.Map.Entry;
031import java.util.Set;
032import java.util.concurrent.TimeUnit;
033import java.util.concurrent.atomic.AtomicLong;
034import java.util.concurrent.atomic.AtomicReference;
035import org.apache.hadoop.conf.Configuration;
036import org.apache.hadoop.fs.FileSystem;
037import org.apache.hadoop.hbase.Cell;
038import org.apache.hadoop.hbase.HBaseTestingUtil;
039import org.apache.hadoop.hbase.NamespaceDescriptor;
040import org.apache.hadoop.hbase.TableName;
041import org.apache.hadoop.hbase.Waiter.Predicate;
042import org.apache.hadoop.hbase.client.Admin;
043import org.apache.hadoop.hbase.client.Connection;
044import org.apache.hadoop.hbase.client.Get;
045import org.apache.hadoop.hbase.client.Result;
046import org.apache.hadoop.hbase.client.SnapshotDescription;
047import org.apache.hadoop.hbase.client.SnapshotType;
048import org.apache.hadoop.hbase.client.Table;
049import org.apache.hadoop.hbase.master.HMaster;
050import org.apache.hadoop.hbase.quotas.SpaceQuotaHelperForTests.NoFilesToDischarge;
051import org.apache.hadoop.hbase.quotas.SpaceQuotaHelperForTests.SpaceQuotaSnapshotPredicate;
052import org.apache.hadoop.hbase.regionserver.HStore;
053import org.apache.hadoop.hbase.testclassification.LargeTests;
054import org.junit.jupiter.api.AfterAll;
055import org.junit.jupiter.api.BeforeAll;
056import org.junit.jupiter.api.BeforeEach;
057import org.junit.jupiter.api.Tag;
058import org.junit.jupiter.api.Test;
059import org.junit.jupiter.api.TestInfo;
060import org.slf4j.Logger;
061import org.slf4j.LoggerFactory;
062
063import org.apache.hbase.thirdparty.com.google.common.collect.HashMultimap;
064import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap;
065import org.apache.hbase.thirdparty.com.google.common.collect.Multimap;
066
067/**
068 * Test class for the {@link SnapshotQuotaObserverChore}.
069 */
070@Tag(LargeTests.TAG)
071public class TestSnapshotQuotaObserverChore {
072
073  private static final Logger LOG = LoggerFactory.getLogger(TestSnapshotQuotaObserverChore.class);
074  private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
075  private static final AtomicLong COUNTER = new AtomicLong();
076
077  private Connection conn;
078  private Admin admin;
079  private SpaceQuotaHelperForTests helper;
080  private HMaster master;
081  private SnapshotQuotaObserverChore testChore;
082  private String testName;
083
084  @BeforeAll
085  public static void setUp() throws Exception {
086    Configuration conf = TEST_UTIL.getConfiguration();
087    SpaceQuotaHelperForTests.updateConfigForQuotas(conf);
088    // Clean up the compacted files faster than normal (15s instead of 2mins)
089    conf.setInt("hbase.hfile.compaction.discharger.interval", 15 * 1000);
090    TEST_UTIL.startMiniCluster(1);
091  }
092
093  @AfterAll
094  public static void tearDown() throws Exception {
095    TEST_UTIL.shutdownMiniCluster();
096  }
097
098  @BeforeEach
099  public void setup(TestInfo testInfo) throws Exception {
100    testName = testInfo.getTestMethod().get().getName();
101    conn = TEST_UTIL.getConnection();
102    admin = TEST_UTIL.getAdmin();
103    helper = new SpaceQuotaHelperForTests(TEST_UTIL, () -> testName, COUNTER);
104    master = TEST_UTIL.getHBaseCluster().getMaster();
105    helper.removeAllQuotas(conn);
106    testChore = new SnapshotQuotaObserverChore(TEST_UTIL.getConnection(),
107      TEST_UTIL.getConfiguration(), master.getFileSystem(), master, null);
108  }
109
110  @Test
111  public void testSnapshotsFromTables() throws Exception {
112    TableName tn1 = helper.createTableWithRegions(1);
113    TableName tn2 = helper.createTableWithRegions(1);
114    TableName tn3 = helper.createTableWithRegions(1);
115
116    // Set a space quota on table 1 and 2 (but not 3)
117    admin.setQuota(QuotaSettingsFactory.limitTableSpace(tn1, SpaceQuotaHelperForTests.ONE_GIGABYTE,
118      SpaceViolationPolicy.NO_INSERTS));
119    admin.setQuota(QuotaSettingsFactory.limitTableSpace(tn2, SpaceQuotaHelperForTests.ONE_GIGABYTE,
120      SpaceViolationPolicy.NO_INSERTS));
121
122    // Create snapshots on each table (we didn't write any data, so just skipflush)
123    admin.snapshot(new SnapshotDescription(tn1 + "snapshot", tn1, SnapshotType.SKIPFLUSH));
124    admin.snapshot(new SnapshotDescription(tn2 + "snapshot", tn2, SnapshotType.SKIPFLUSH));
125    admin.snapshot(new SnapshotDescription(tn3 + "snapshot", tn3, SnapshotType.SKIPFLUSH));
126
127    Multimap<TableName, String> mapping = testChore.getSnapshotsToComputeSize();
128    assertEquals(2, mapping.size());
129    assertEquals(1, mapping.get(tn1).size());
130    assertEquals(tn1 + "snapshot", mapping.get(tn1).iterator().next());
131    assertEquals(1, mapping.get(tn2).size());
132    assertEquals(tn2 + "snapshot", mapping.get(tn2).iterator().next());
133
134    admin.snapshot(new SnapshotDescription(tn2 + "snapshot1", tn2, SnapshotType.SKIPFLUSH));
135    admin.snapshot(new SnapshotDescription(tn3 + "snapshot1", tn3, SnapshotType.SKIPFLUSH));
136
137    mapping = testChore.getSnapshotsToComputeSize();
138    assertEquals(3, mapping.size());
139    assertEquals(1, mapping.get(tn1).size());
140    assertEquals(tn1 + "snapshot", mapping.get(tn1).iterator().next());
141    assertEquals(2, mapping.get(tn2).size());
142    assertEquals(new HashSet<String>(Arrays.asList(tn2 + "snapshot", tn2 + "snapshot1")),
143      mapping.get(tn2));
144  }
145
146  @Test
147  public void testSnapshotsFromNamespaces() throws Exception {
148    NamespaceDescriptor ns = NamespaceDescriptor.create("snapshots_from_namespaces").build();
149    admin.createNamespace(ns);
150
151    TableName tn1 = helper.createTableWithRegions(ns.getName(), 1);
152    TableName tn2 = helper.createTableWithRegions(ns.getName(), 1);
153    TableName tn3 = helper.createTableWithRegions(1);
154
155    // Set a throttle quota on 'default' namespace
156    admin.setQuota(QuotaSettingsFactory.throttleNamespace(tn3.getNamespaceAsString(),
157      ThrottleType.WRITE_NUMBER, 100, TimeUnit.SECONDS));
158    // Set a user throttle quota
159    admin.setQuota(
160      QuotaSettingsFactory.throttleUser("user", ThrottleType.WRITE_NUMBER, 100, TimeUnit.MINUTES));
161
162    // Set a space quota on the namespace
163    admin.setQuota(QuotaSettingsFactory.limitNamespaceSpace(ns.getName(),
164      SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS));
165
166    // Create snapshots on each table (we didn't write any data, so just skipflush)
167    admin.snapshot(new SnapshotDescription(tn1.getQualifierAsString() + "snapshot", tn1,
168      SnapshotType.SKIPFLUSH));
169    admin.snapshot(new SnapshotDescription(tn2.getQualifierAsString() + "snapshot", tn2,
170      SnapshotType.SKIPFLUSH));
171    admin.snapshot(new SnapshotDescription(tn3.getQualifierAsString() + "snapshot", tn3,
172      SnapshotType.SKIPFLUSH));
173
174    Multimap<TableName, String> mapping = testChore.getSnapshotsToComputeSize();
175    assertEquals(2, mapping.size());
176    assertEquals(1, mapping.get(tn1).size());
177    assertEquals(tn1.getQualifierAsString() + "snapshot", mapping.get(tn1).iterator().next());
178    assertEquals(1, mapping.get(tn2).size());
179    assertEquals(tn2.getQualifierAsString() + "snapshot", mapping.get(tn2).iterator().next());
180
181    admin.snapshot(new SnapshotDescription(tn2.getQualifierAsString() + "snapshot1", tn2,
182      SnapshotType.SKIPFLUSH));
183    admin.snapshot(new SnapshotDescription(tn3.getQualifierAsString() + "snapshot2", tn3,
184      SnapshotType.SKIPFLUSH));
185
186    mapping = testChore.getSnapshotsToComputeSize();
187    assertEquals(3, mapping.size());
188    assertEquals(1, mapping.get(tn1).size());
189    assertEquals(tn1.getQualifierAsString() + "snapshot", mapping.get(tn1).iterator().next());
190    assertEquals(2, mapping.get(tn2).size());
191    assertEquals(new HashSet<String>(Arrays.asList(tn2.getQualifierAsString() + "snapshot",
192      tn2.getQualifierAsString() + "snapshot1")), mapping.get(tn2));
193  }
194
195  @Test
196  public void testSnapshotSize() throws Exception {
197    // Create a table and set a quota
198    TableName tn1 = helper.createTableWithRegions(5);
199    admin.setQuota(QuotaSettingsFactory.limitTableSpace(tn1, SpaceQuotaHelperForTests.ONE_GIGABYTE,
200      SpaceViolationPolicy.NO_INSERTS));
201
202    // Write some data and flush it
203    helper.writeData(tn1, 256L * SpaceQuotaHelperForTests.ONE_KILOBYTE);
204    admin.flush(tn1);
205
206    final long snapshotSize = TEST_UTIL.getMiniHBaseCluster().getRegions(tn1).stream()
207      .flatMap(r -> r.getStores().stream()).mapToLong(HStore::getHFilesSize).sum();
208
209    // Wait for the Master chore to run to see the usage (with a fudge factor)
210    TEST_UTIL.waitFor(30_000, new SpaceQuotaSnapshotPredicate(conn, tn1) {
211      @Override
212      boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
213        return snapshot.getUsage() == snapshotSize;
214      }
215    });
216
217    // Create a snapshot on the table
218    final String snapshotName = tn1 + "snapshot";
219    admin.snapshot(new SnapshotDescription(snapshotName, tn1, SnapshotType.SKIPFLUSH));
220
221    // Get the snapshots
222    Multimap<TableName, String> snapshotsToCompute = testChore.getSnapshotsToComputeSize();
223    assertEquals(1, snapshotsToCompute.size(),
224      "Expected to see the single snapshot: " + snapshotsToCompute);
225
226    // Get the size of our snapshot
227    Map<String, Long> namespaceSnapshotSizes = testChore.computeSnapshotSizes(snapshotsToCompute);
228    assertEquals(1, namespaceSnapshotSizes.size());
229    Long size = namespaceSnapshotSizes.get(tn1.getNamespaceAsString());
230    assertNotNull(size);
231    // The snapshot should take up no space since the table refers to it completely
232    assertEquals(0, size.longValue());
233
234    // Write some more data, flush it, and then major_compact the table
235    helper.writeData(tn1, 256L * SpaceQuotaHelperForTests.ONE_KILOBYTE);
236    admin.flush(tn1);
237    TEST_UTIL.compact(tn1, true);
238
239    // Test table should reflect it's original size since ingest was deterministic
240    TEST_UTIL.waitFor(30_000, new SpaceQuotaSnapshotPredicate(conn, tn1) {
241      private final long regionSize = TEST_UTIL.getMiniHBaseCluster().getRegions(tn1).stream()
242        .flatMap(r -> r.getStores().stream()).mapToLong(HStore::getHFilesSize).sum();
243
244      @Override
245      boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
246        LOG.debug("Current usage=" + snapshot.getUsage() + " snapshotSize=" + snapshotSize);
247        // The usage of table space consists of region size and snapshot size
248        return closeInSize(snapshot.getUsage(), snapshotSize + regionSize,
249          SpaceQuotaHelperForTests.ONE_KILOBYTE);
250      }
251    });
252
253    // Wait for no compacted files on the regions of our table
254    TEST_UTIL.waitFor(30_000, new NoFilesToDischarge(TEST_UTIL.getMiniHBaseCluster(), tn1));
255
256    // Still should see only one snapshot
257    snapshotsToCompute = testChore.getSnapshotsToComputeSize();
258    assertEquals(1, snapshotsToCompute.size(),
259      "Expected to see the single snapshot: " + snapshotsToCompute);
260    namespaceSnapshotSizes = testChore.computeSnapshotSizes(snapshotsToCompute);
261    assertEquals(1, namespaceSnapshotSizes.size());
262    size = namespaceSnapshotSizes.get(tn1.getNamespaceAsString());
263    assertNotNull(size);
264    // The snapshot should take up the size the table originally took up
265    assertEquals(snapshotSize, size.longValue());
266  }
267
268  @Test
269  public void testPersistingSnapshotsForNamespaces() throws Exception {
270    TableName tn1 = TableName.valueOf("ns1:tn1");
271    TableName tn2 = TableName.valueOf("ns1:tn2");
272    TableName tn3 = TableName.valueOf("ns2:tn1");
273    TableName tn4 = TableName.valueOf("ns2:tn2");
274    TableName tn5 = TableName.valueOf("tn1");
275    // Shim in a custom factory to avoid computing snapshot sizes.
276    FileArchiverNotifierFactory test = new FileArchiverNotifierFactory() {
277      Map<TableName, Long> tableToSize =
278        ImmutableMap.of(tn1, 1024L, tn2, 1024L, tn3, 512L, tn4, 1024L, tn5, 3072L);
279
280      @Override
281      public FileArchiverNotifier get(Connection conn, Configuration conf, FileSystem fs,
282        TableName tn) {
283        return new FileArchiverNotifier() {
284          @Override
285          public void addArchivedFiles(Set<Entry<String, Long>> fileSizes) throws IOException {
286          }
287
288          @Override
289          public long computeAndStoreSnapshotSizes(Collection<String> currentSnapshots)
290            throws IOException {
291            return tableToSize.get(tn);
292          }
293        };
294      }
295    };
296    try {
297      FileArchiverNotifierFactoryImpl.setInstance(test);
298
299      Multimap<TableName, String> snapshotsToCompute = HashMultimap.create();
300      snapshotsToCompute.put(tn1, "");
301      snapshotsToCompute.put(tn2, "");
302      snapshotsToCompute.put(tn3, "");
303      snapshotsToCompute.put(tn4, "");
304      snapshotsToCompute.put(tn5, "");
305      Map<String, Long> nsSizes = testChore.computeSnapshotSizes(snapshotsToCompute);
306      assertEquals(3, nsSizes.size());
307      assertEquals(2048L, (long) nsSizes.get("ns1"));
308      assertEquals(1536L, (long) nsSizes.get("ns2"));
309      assertEquals(3072L, (long) nsSizes.get(NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR));
310    } finally {
311      FileArchiverNotifierFactoryImpl.reset();
312    }
313  }
314
315  @Test
316  public void testRemovedSnapshots() throws Exception {
317    // Create a table and set a quota
318    TableName tn1 = helper.createTableWithRegions(1);
319    admin.setQuota(QuotaSettingsFactory.limitTableSpace(tn1, SpaceQuotaHelperForTests.ONE_GIGABYTE,
320      SpaceViolationPolicy.NO_INSERTS));
321
322    // Write some data and flush it
323    helper.writeData(tn1, 256L * SpaceQuotaHelperForTests.ONE_KILOBYTE); // 256 KB
324
325    final AtomicReference<Long> lastSeenSize = new AtomicReference<>();
326    // Wait for the Master chore to run to see the usage (with a fudge factor)
327    TEST_UTIL.waitFor(30_000, new SpaceQuotaSnapshotPredicate(conn, tn1) {
328      @Override
329      boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
330        lastSeenSize.set(snapshot.getUsage());
331        return snapshot.getUsage() > 230L * SpaceQuotaHelperForTests.ONE_KILOBYTE;
332      }
333    });
334
335    // Create a snapshot on the table
336    final String snapshotName1 = tn1 + "snapshot1";
337    admin.snapshot(new SnapshotDescription(snapshotName1, tn1, SnapshotType.SKIPFLUSH));
338
339    // Snapshot size has to be 0 as the snapshot shares the data with the table
340    final Table quotaTable = conn.getTable(QuotaUtil.QUOTA_TABLE_NAME);
341    TEST_UTIL.waitFor(30_000, new Predicate<Exception>() {
342      @Override
343      public boolean evaluate() throws Exception {
344        Get g = QuotaTableUtil.makeGetForSnapshotSize(tn1, snapshotName1);
345        Result r = quotaTable.get(g);
346        if (r == null || r.isEmpty()) {
347          return false;
348        }
349        r.advance();
350        Cell c = r.current();
351        return QuotaTableUtil.parseSnapshotSize(c) == 0;
352      }
353    });
354    // Total usage has to remain same as what we saw before taking a snapshot
355    TEST_UTIL.waitFor(30_000, new SpaceQuotaSnapshotPredicate(conn, tn1) {
356      @Override
357      boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
358        return snapshot.getUsage() == lastSeenSize.get();
359      }
360    });
361
362    // Major compact the table to force a rewrite
363    TEST_UTIL.compact(tn1, true);
364    // Now the snapshot size has to prev total size
365    TEST_UTIL.waitFor(30_000, new Predicate<Exception>() {
366      @Override
367      public boolean evaluate() throws Exception {
368        Get g = QuotaTableUtil.makeGetForSnapshotSize(tn1, snapshotName1);
369        Result r = quotaTable.get(g);
370        if (r == null || r.isEmpty()) {
371          return false;
372        }
373        r.advance();
374        Cell c = r.current();
375        // The compaction result file has an additional compaction event tracker
376        return lastSeenSize.get() == QuotaTableUtil.parseSnapshotSize(c);
377      }
378    });
379    // The total size now has to be equal/more than double of prev total size
380    // as double the number of store files exist now.
381    final AtomicReference<Long> sizeAfterCompaction = new AtomicReference<>();
382    TEST_UTIL.waitFor(30_000, new SpaceQuotaSnapshotPredicate(conn, tn1) {
383      @Override
384      boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
385        sizeAfterCompaction.set(snapshot.getUsage());
386        return snapshot.getUsage() >= 2 * lastSeenSize.get();
387      }
388    });
389
390    // Delete the snapshot
391    admin.deleteSnapshot(snapshotName1);
392    // Total size has to come down to prev totalsize - snapshot size(which was removed)
393    TEST_UTIL.waitFor(30_000, new SpaceQuotaSnapshotPredicate(conn, tn1) {
394      @Override
395      boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
396        return snapshot.getUsage() == (sizeAfterCompaction.get() - lastSeenSize.get());
397      }
398    });
399  }
400
401  @Test
402  public void testBucketingFilesToSnapshots() throws Exception {
403    // Create a table and set a quota
404    TableName tn1 = helper.createTableWithRegions(1);
405    admin.setQuota(QuotaSettingsFactory.limitTableSpace(tn1, SpaceQuotaHelperForTests.ONE_GIGABYTE,
406      SpaceViolationPolicy.NO_INSERTS));
407
408    // Write some data and flush it
409    helper.writeData(tn1, 256L * SpaceQuotaHelperForTests.ONE_KILOBYTE);
410    admin.flush(tn1);
411
412    final AtomicReference<Long> lastSeenSize = new AtomicReference<>();
413    // Wait for the Master chore to run to see the usage (with a fudge factor)
414    TEST_UTIL.waitFor(30_000, new SpaceQuotaSnapshotPredicate(conn, tn1) {
415      @Override
416      boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
417        lastSeenSize.set(snapshot.getUsage());
418        return snapshot.getUsage() > 230L * SpaceQuotaHelperForTests.ONE_KILOBYTE;
419      }
420    });
421
422    // Create a snapshot on the table
423    final String snapshotName1 = tn1 + "snapshot1";
424    admin.snapshot(new SnapshotDescription(snapshotName1, tn1, SnapshotType.SKIPFLUSH));
425    // Major compact the table to force a rewrite
426    TEST_UTIL.compact(tn1, true);
427
428    // Make sure that the snapshot owns the size
429    final Table quotaTable = conn.getTable(QuotaUtil.QUOTA_TABLE_NAME);
430    TEST_UTIL.waitFor(30_000, new Predicate<Exception>() {
431      @Override
432      public boolean evaluate() throws Exception {
433        Get g = QuotaTableUtil.makeGetForSnapshotSize(tn1, snapshotName1);
434        Result r = quotaTable.get(g);
435        if (r == null || r.isEmpty()) {
436          return false;
437        }
438        r.advance();
439        Cell c = r.current();
440        // The compaction result file has an additional compaction event tracker
441        return lastSeenSize.get() <= QuotaTableUtil.parseSnapshotSize(c);
442      }
443    });
444
445    // Create another snapshot on the table
446    final String snapshotName2 = tn1 + "snapshot2";
447    admin.snapshot(new SnapshotDescription(snapshotName2, tn1, SnapshotType.SKIPFLUSH));
448    // Major compact the table to force a rewrite
449    TEST_UTIL.compact(tn1, true);
450
451    // Make sure that the snapshot owns the size
452    TEST_UTIL.waitFor(30_000, new Predicate<Exception>() {
453      @Override
454      public boolean evaluate() throws Exception {
455        Get g = QuotaTableUtil.makeGetForSnapshotSize(tn1, snapshotName2);
456        Result r = quotaTable.get(g);
457        if (r == null || r.isEmpty()) {
458          return false;
459        }
460        r.advance();
461        Cell c = r.current();
462        // The compaction result file has an additional compaction event tracker
463        return lastSeenSize.get() <= QuotaTableUtil.parseSnapshotSize(c);
464      }
465    });
466
467    Get g = QuotaTableUtil.createGetNamespaceSnapshotSize(tn1.getNamespaceAsString());
468    Result r = quotaTable.get(g);
469    assertNotNull(r);
470    assertFalse(r.isEmpty());
471    r.advance();
472    long size = QuotaTableUtil.parseSnapshotSize(r.current());
473    assertTrue(lastSeenSize.get() * 2 <= size);
474  }
475
476  /**
477   * Computes if {@code size2} is within {@code delta} of {@code size1}, inclusive.
478   */
479  boolean closeInSize(long size1, long size2, long delta) {
480    long lower = size1 - delta;
481    long upper = size1 + delta;
482    return lower <= size2 && size2 <= upper;
483  }
484}