001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.quotas;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNotNull;
022import static org.junit.Assert.assertTrue;
023
024import java.io.IOException;
025import java.util.Map;
026import java.util.concurrent.atomic.AtomicInteger;
027import java.util.concurrent.atomic.AtomicLong;
028import org.apache.hadoop.conf.Configuration;
029import org.apache.hadoop.hbase.Cell;
030import org.apache.hadoop.hbase.CellScanner;
031import org.apache.hadoop.hbase.HBaseClassTestRule;
032import org.apache.hadoop.hbase.HBaseTestingUtil;
033import org.apache.hadoop.hbase.TableName;
034import org.apache.hadoop.hbase.Waiter;
035import org.apache.hadoop.hbase.Waiter.Predicate;
036import org.apache.hadoop.hbase.client.Admin;
037import org.apache.hadoop.hbase.client.Connection;
038import org.apache.hadoop.hbase.client.Result;
039import org.apache.hadoop.hbase.client.ResultScanner;
040import org.apache.hadoop.hbase.client.Scan;
041import org.apache.hadoop.hbase.client.SnapshotType;
042import org.apache.hadoop.hbase.client.Table;
043import org.apache.hadoop.hbase.quotas.SpaceQuotaHelperForTests.SpaceQuotaSnapshotPredicate;
044import org.apache.hadoop.hbase.testclassification.LargeTests;
045import org.junit.AfterClass;
046import org.junit.Before;
047import org.junit.BeforeClass;
048import org.junit.ClassRule;
049import org.junit.Rule;
050import org.junit.Test;
051import org.junit.experimental.categories.Category;
052import org.junit.rules.TestName;
053import org.slf4j.Logger;
054import org.slf4j.LoggerFactory;
055
056import org.apache.hbase.thirdparty.com.google.common.collect.Iterables;
057import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;
058
059import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos;
060
061/**
062 * Test class to exercise the inclusion of snapshots in space quotas
063 */
064@Category({LargeTests.class})
065public class TestSpaceQuotasWithSnapshots {
066
067  @ClassRule
068  public static final HBaseClassTestRule CLASS_RULE =
069      HBaseClassTestRule.forClass(TestSpaceQuotasWithSnapshots.class);
070
071  private static final Logger LOG = LoggerFactory.getLogger(TestSpaceQuotasWithSnapshots.class);
072  private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
073  // Global for all tests in the class
074  private static final AtomicLong COUNTER = new AtomicLong(0);
075  private static final long FUDGE_FOR_TABLE_SIZE = 500L * SpaceQuotaHelperForTests.ONE_KILOBYTE;
076
077  @Rule
078  public TestName testName = new TestName();
079  private SpaceQuotaHelperForTests helper;
080  private Connection conn;
081  private Admin admin;
082
083  @BeforeClass
084  public static void setUp() throws Exception {
085    Configuration conf = TEST_UTIL.getConfiguration();
086    SpaceQuotaHelperForTests.updateConfigForQuotas(conf);
087    TEST_UTIL.startMiniCluster(1);
088    // Wait till quota table onlined.
089    TEST_UTIL.waitFor(10000, new Waiter.Predicate<Exception>() {
090      @Override
091      public boolean evaluate() throws Exception {
092        return TEST_UTIL.getAdmin().tableExists(QuotaTableUtil.QUOTA_TABLE_NAME);
093      }
094    });
095  }
096
097  @AfterClass
098  public static void tearDown() throws Exception {
099    TEST_UTIL.shutdownMiniCluster();
100  }
101
102  @Before
103  public void removeAllQuotas() throws Exception {
104    helper = new SpaceQuotaHelperForTests(TEST_UTIL, testName, COUNTER);
105    conn = TEST_UTIL.getConnection();
106    admin = TEST_UTIL.getAdmin();
107  }
108
109  @Test
110  public void testTablesInheritSnapshotSize() throws Exception {
111    TableName tn = helper.createTableWithRegions(1);
112    LOG.info("Writing data");
113    // Set a quota
114    QuotaSettings settings = QuotaSettingsFactory.limitTableSpace(
115        tn, SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS);
116    admin.setQuota(settings);
117    // Write some data
118    final long initialSize = 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE;
119    helper.writeData(tn, initialSize);
120
121    LOG.info("Waiting until table size reflects written data");
122    // Wait until that data is seen by the master
123    TEST_UTIL.waitFor(30 * 1000, 500, new SpaceQuotaSnapshotPredicate(conn, tn) {
124      @Override boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
125        return snapshot.getUsage() >= initialSize;
126      }
127    });
128
129    // Make sure we see the final quota usage size
130    waitForStableQuotaSize(conn, tn, null);
131
132    // The actual size on disk after we wrote our data the first time
133    final long actualInitialSize = conn.getAdmin().getCurrentSpaceQuotaSnapshot(tn).getUsage();
134    LOG.info("Initial table size was " + actualInitialSize);
135
136    LOG.info("Snapshot the table");
137    final String snapshot1 = tn.toString() + "_snapshot1";
138    admin.snapshot(snapshot1, tn);
139
140    // Write the same data again, then flush+compact. This should make sure that
141    // the snapshot is referencing files that the table no longer references.
142    LOG.info("Write more data");
143    helper.writeData(tn, initialSize);
144    LOG.info("Flush the table");
145    admin.flush(tn);
146    LOG.info("Synchronously compacting the table");
147    TEST_UTIL.compact(tn, true);
148
149    final long upperBound = initialSize + FUDGE_FOR_TABLE_SIZE;
150    final long lowerBound = initialSize - FUDGE_FOR_TABLE_SIZE;
151
152    // Store the actual size after writing more data and then compacting it down to one file
153    LOG.info("Waiting for the region reports to reflect the correct size, between ("
154        + lowerBound + ", " + upperBound + ")");
155    TEST_UTIL.waitFor(30 * 1000, 500, new Predicate<Exception>() {
156      @Override
157      public boolean evaluate() throws Exception {
158        long size = getRegionSizeReportForTable(conn, tn);
159        return size < upperBound && size > lowerBound;
160      }
161    });
162
163    // Make sure we see the "final" new size for the table, not some intermediate
164    waitForStableRegionSizeReport(conn, tn);
165    final long finalSize = getRegionSizeReportForTable(conn, tn);
166    assertNotNull("Did not expect to see a null size", finalSize);
167    LOG.info("Last seen size: " + finalSize);
168
169    // Make sure the QuotaObserverChore has time to reflect the new region size reports
170    // (we saw above). The usage of the table should *not* decrease when we check it below,
171    // though, because the snapshot on our table will cause the table to "retain" the size.
172    TEST_UTIL.waitFor(20 * 1000, 500, new SpaceQuotaSnapshotPredicate(conn, tn) {
173      @Override
174      public boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
175        return snapshot.getUsage() >= finalSize;
176      }
177    });
178
179    // The final usage should be the sum of the initial size (referenced by the snapshot) and the
180    // new size we just wrote above.
181    long expectedFinalSize = actualInitialSize + finalSize;
182    LOG.info(
183        "Expecting table usage to be " + actualInitialSize + " + " + finalSize
184        + " = " + expectedFinalSize);
185    // The size of the table (WRT quotas) should now be approximately double what it was previously
186    TEST_UTIL.waitFor(30 * 1000, 1000, new SpaceQuotaSnapshotPredicate(conn, tn) {
187      @Override boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
188        LOG.debug("Checking for " + expectedFinalSize + " == " + snapshot.getUsage());
189        return expectedFinalSize == snapshot.getUsage();
190      }
191    });
192
193    Map<String,Long> snapshotSizes = QuotaTableUtil.getObservedSnapshotSizes(conn);
194    Long size = snapshotSizes.get(snapshot1);
195    assertNotNull("Did not observe the size of the snapshot", size);
196    assertEquals(
197        "The recorded size of the HBase snapshot was not the size we expected", actualInitialSize,
198        size.longValue());
199  }
200
201  @Test
202  public void testNamespacesInheritSnapshotSize() throws Exception {
203    String ns = helper.createNamespace().getName();
204    TableName tn = helper.createTableWithRegions(ns, 1);
205    LOG.info("Writing data");
206    // Set a quota
207    QuotaSettings settings = QuotaSettingsFactory.limitNamespaceSpace(
208        ns, SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS);
209    admin.setQuota(settings);
210
211    // Write some data and flush it to disk
212    final long initialSize = 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE;
213    helper.writeData(tn, initialSize);
214    admin.flush(tn);
215
216    LOG.info("Waiting until namespace size reflects written data");
217    // Wait until that data is seen by the master
218    TEST_UTIL.waitFor(30 * 1000, 500, new SpaceQuotaSnapshotPredicate(conn, ns) {
219      @Override boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
220        return snapshot.getUsage() >= initialSize;
221      }
222    });
223
224    // Make sure we see the "final" new size for the table, not some intermediate
225    waitForStableQuotaSize(conn, null, ns);
226
227    // The actual size on disk after we wrote our data the first time
228    final long actualInitialSize = conn.getAdmin().getCurrentSpaceQuotaSnapshot(ns).getUsage();
229    LOG.info("Initial table size was " + actualInitialSize);
230
231    LOG.info("Snapshot the table");
232    final String snapshot1 = tn.getQualifierAsString() + "_snapshot1";
233    admin.snapshot(snapshot1, tn);
234
235    // Write the same data again, then flush+compact. This should make sure that
236    // the snapshot is referencing files that the table no longer references.
237    LOG.info("Write more data");
238    helper.writeData(tn, initialSize);
239    LOG.info("Flush the table");
240    admin.flush(tn);
241    LOG.info("Synchronously compacting the table");
242    TEST_UTIL.compact(tn, true);
243
244    final long upperBound = initialSize + FUDGE_FOR_TABLE_SIZE;
245    final long lowerBound = initialSize - FUDGE_FOR_TABLE_SIZE;
246
247    LOG.info("Waiting for the region reports to reflect the correct size, between ("
248        + lowerBound + ", " + upperBound + ")");
249    TEST_UTIL.waitFor(30 * 1000, 500, new Predicate<Exception>() {
250      @Override
251      public boolean evaluate() throws Exception {
252        Map<TableName, Long> sizes = conn.getAdmin().getSpaceQuotaTableSizes();
253        LOG.debug("Master observed table sizes from region size reports: " + sizes);
254        Long size = sizes.get(tn);
255        if (null == size) {
256          return false;
257        }
258        return size < upperBound && size > lowerBound;
259      }
260    });
261
262    // Make sure we see the "final" new size for the table, not some intermediate
263    waitForStableRegionSizeReport(conn, tn);
264    final long finalSize = getRegionSizeReportForTable(conn, tn);
265    assertNotNull("Did not expect to see a null size", finalSize);
266    LOG.info("Final observed size of table: " + finalSize);
267
268    // Make sure the QuotaObserverChore has time to reflect the new region size reports
269    // (we saw above). The usage of the table should *not* decrease when we check it below,
270    // though, because the snapshot on our table will cause the table to "retain" the size.
271    TEST_UTIL.waitFor(20 * 1000, 500, new SpaceQuotaSnapshotPredicate(conn, ns) {
272      @Override
273      public boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
274        return snapshot.getUsage() >= finalSize;
275      }
276    });
277
278    // The final usage should be the sum of the initial size (referenced by the snapshot) and the
279    // new size we just wrote above.
280    long expectedFinalSize = actualInitialSize + finalSize;
281    LOG.info(
282        "Expecting namespace usage to be " + actualInitialSize + " + " + finalSize
283        + " = " + expectedFinalSize);
284    // The size of the table (WRT quotas) should now be approximately double what it was previously
285    TEST_UTIL.waitFor(30 * 1000, 1000, new SpaceQuotaSnapshotPredicate(conn, ns) {
286      @Override boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
287        LOG.debug("Checking for " + expectedFinalSize + " == " + snapshot.getUsage());
288        return expectedFinalSize == snapshot.getUsage();
289      }
290    });
291
292    Map<String,Long> snapshotSizes = QuotaTableUtil.getObservedSnapshotSizes(conn);
293    Long size = snapshotSizes.get(snapshot1);
294    assertNotNull("Did not observe the size of the snapshot", size);
295    assertEquals(
296        "The recorded size of the HBase snapshot was not the size we expected", actualInitialSize,
297        size.longValue());
298  }
299
300  @Test
301  public void testTablesWithSnapshots() throws Exception {
302    final Connection conn = TEST_UTIL.getConnection();
303    final SpaceViolationPolicy policy = SpaceViolationPolicy.NO_INSERTS;
304    final TableName tn = helper.createTableWithRegions(10);
305
306    // 3MB limit on the table
307    final long tableLimit = 3L * SpaceQuotaHelperForTests.ONE_MEGABYTE;
308    TEST_UTIL.getAdmin().setQuota(QuotaSettingsFactory.limitTableSpace(tn, tableLimit, policy));
309
310    LOG.info("Writing first data set");
311    // Write more data than should be allowed and flush it to disk
312    helper.writeData(tn, 1L * SpaceQuotaHelperForTests.ONE_MEGABYTE, "q1");
313
314    LOG.info("Creating snapshot");
315    TEST_UTIL.getAdmin().snapshot(tn.toString() + "snap1", tn, SnapshotType.FLUSH);
316
317    LOG.info("Writing second data set");
318    // Write some more data
319    helper.writeData(tn, 1L * SpaceQuotaHelperForTests.ONE_MEGABYTE, "q2");
320
321    LOG.info("Flushing and major compacting table");
322    // Compact the table to force the snapshot to own all of its files
323    TEST_UTIL.getAdmin().flush(tn);
324    TEST_UTIL.compact(tn, true);
325
326    LOG.info("Checking for quota violation");
327    // Wait to observe the quota moving into violation
328    TEST_UTIL.waitFor(60_000, 1_000, new Predicate<Exception>() {
329      @Override
330      public boolean evaluate() throws Exception {
331        Scan s = QuotaTableUtil.makeQuotaSnapshotScanForTable(tn);
332        try (Table t = conn.getTable(QuotaTableUtil.QUOTA_TABLE_NAME)) {
333          ResultScanner rs = t.getScanner(s);
334          try {
335            Result r = Iterables.getOnlyElement(rs);
336            CellScanner cs = r.cellScanner();
337            assertTrue(cs.advance());
338            Cell c = cs.current();
339            SpaceQuotaSnapshot snapshot = SpaceQuotaSnapshot.toSpaceQuotaSnapshot(
340                QuotaProtos.SpaceQuotaSnapshot.parseFrom(
341                  UnsafeByteOperations.unsafeWrap(
342                      c.getValueArray(), c.getValueOffset(), c.getValueLength())));
343            LOG.info(
344                snapshot.getUsage() + "/" + snapshot.getLimit() + " " + snapshot.getQuotaStatus());
345            // We expect to see the table move to violation
346            return snapshot.getQuotaStatus().isInViolation();
347          } finally {
348            if (null != rs) {
349              rs.close();
350            }
351          }
352        }
353      }
354    });
355  }
356
357  @Test
358  public void testRematerializedTablesDoNoInheritSpace() throws Exception {
359    TableName tn = helper.createTableWithRegions(1);
360    TableName tn2 = helper.getNextTableName();
361    LOG.info("Writing data");
362    // Set a quota on both tables
363    QuotaSettings settings = QuotaSettingsFactory.limitTableSpace(
364        tn, SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS);
365    admin.setQuota(settings);
366    QuotaSettings settings2 = QuotaSettingsFactory.limitTableSpace(
367        tn2, SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS);
368    admin.setQuota(settings2);
369    // Write some data
370    final long initialSize = 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE;
371    helper.writeData(tn, initialSize);
372
373    LOG.info("Waiting until table size reflects written data");
374    // Wait until that data is seen by the master
375    TEST_UTIL.waitFor(30 * 1000, 500, new SpaceQuotaSnapshotPredicate(conn, tn) {
376      @Override boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
377        return snapshot.getUsage() >= initialSize;
378      }
379    });
380
381    // Make sure we see the final quota usage size
382    waitForStableQuotaSize(conn, tn, null);
383
384    // The actual size on disk after we wrote our data the first time
385    final long actualInitialSize = conn.getAdmin().getCurrentSpaceQuotaSnapshot(tn).getUsage();
386    LOG.info("Initial table size was " + actualInitialSize);
387
388    LOG.info("Snapshot the table");
389    final String snapshot1 = tn.toString() + "_snapshot1";
390    admin.snapshot(snapshot1, tn);
391
392    admin.cloneSnapshot(snapshot1, tn2);
393
394    // Write some more data to the first table
395    helper.writeData(tn, initialSize, "q2");
396    admin.flush(tn);
397
398    // Watch the usage of the first table with some more data to know when the new
399    // region size reports were sent to the master
400    TEST_UTIL.waitFor(30_000, 1_000, new SpaceQuotaSnapshotPredicate(conn, tn) {
401      @Override
402      boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
403        return snapshot.getUsage() >= actualInitialSize * 2;
404      }
405    });
406
407    // We know that reports were sent by our RS, verify that they take up zero size.
408    SpaceQuotaSnapshot snapshot =
409      (SpaceQuotaSnapshot) conn.getAdmin().getCurrentSpaceQuotaSnapshot(tn2);
410    assertNotNull(snapshot);
411    assertEquals(0, snapshot.getUsage());
412
413    // Compact the cloned table to force it to own its own files.
414    TEST_UTIL.compact(tn2, true);
415    // After the table is compacted, it should have its own files and be the same size as originally
416    // But The compaction result file has an additional compaction event tracker
417    TEST_UTIL.waitFor(30_000, 1_000, new SpaceQuotaSnapshotPredicate(conn, tn2) {
418      @Override
419      boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
420        return snapshot.getUsage() >= actualInitialSize;
421      }
422    });
423  }
424
425  void waitForStableQuotaSize(Connection conn, TableName tn, String ns) throws Exception {
426    // For some stability in the value before proceeding
427    // Helps make sure that we got the actual last value, not some inbetween
428    AtomicLong lastValue = new AtomicLong(-1);
429    AtomicInteger counter = new AtomicInteger(0);
430    TEST_UTIL.waitFor(15_000, 500, new SpaceQuotaSnapshotPredicate(conn, tn, ns) {
431      @Override boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
432        LOG.debug("Last observed size=" + lastValue.get());
433        if (snapshot.getUsage() == lastValue.get()) {
434          int numMatches = counter.incrementAndGet();
435          if (numMatches >= 5) {
436            return true;
437          }
438          // Not yet..
439          return false;
440        }
441        counter.set(0);
442        lastValue.set(snapshot.getUsage());
443        return false;
444      }
445    });
446  }
447
448  long getRegionSizeReportForTable(Connection conn, TableName tn) throws IOException {
449    Map<TableName, Long> sizes = conn.getAdmin().getSpaceQuotaTableSizes();
450    Long value = sizes.get(tn);
451    if (null == value) {
452      return 0L;
453    }
454    return value.longValue();
455  }
456
457  void waitForStableRegionSizeReport(Connection conn, TableName tn) throws Exception {
458    // For some stability in the value before proceeding
459    // Helps make sure that we got the actual last value, not some inbetween
460    AtomicLong lastValue = new AtomicLong(-1);
461    AtomicInteger counter = new AtomicInteger(0);
462    TEST_UTIL.waitFor(15_000, 500, new Predicate<Exception>() {
463      @Override public boolean evaluate() throws Exception {
464        LOG.debug("Last observed size=" + lastValue.get());
465        long actual = getRegionSizeReportForTable(conn, tn);
466        if (actual == lastValue.get()) {
467          int numMatches = counter.incrementAndGet();
468          if (numMatches >= 5) {
469            return true;
470          }
471          // Not yet..
472          return false;
473        }
474        counter.set(0);
475        lastValue.set(actual);
476        return false;
477      }
478    });
479  }
480}