001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.quotas;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNotNull;
022import static org.junit.Assert.assertTrue;
023
024import java.io.IOException;
025import java.util.Map;
026import java.util.concurrent.atomic.AtomicInteger;
027import java.util.concurrent.atomic.AtomicLong;
028import org.apache.hadoop.conf.Configuration;
029import org.apache.hadoop.hbase.Cell;
030import org.apache.hadoop.hbase.CellScanner;
031import org.apache.hadoop.hbase.HBaseClassTestRule;
032import org.apache.hadoop.hbase.HBaseTestingUtil;
033import org.apache.hadoop.hbase.TableName;
034import org.apache.hadoop.hbase.Waiter;
035import org.apache.hadoop.hbase.Waiter.Predicate;
036import org.apache.hadoop.hbase.client.Admin;
037import org.apache.hadoop.hbase.client.Connection;
038import org.apache.hadoop.hbase.client.Result;
039import org.apache.hadoop.hbase.client.ResultScanner;
040import org.apache.hadoop.hbase.client.Scan;
041import org.apache.hadoop.hbase.client.SnapshotType;
042import org.apache.hadoop.hbase.client.Table;
043import org.apache.hadoop.hbase.quotas.SpaceQuotaHelperForTests.SpaceQuotaSnapshotPredicate;
044import org.apache.hadoop.hbase.testclassification.LargeTests;
045import org.junit.AfterClass;
046import org.junit.Before;
047import org.junit.BeforeClass;
048import org.junit.ClassRule;
049import org.junit.Rule;
050import org.junit.Test;
051import org.junit.experimental.categories.Category;
052import org.junit.rules.TestName;
053import org.slf4j.Logger;
054import org.slf4j.LoggerFactory;
055
056import org.apache.hbase.thirdparty.com.google.common.collect.Iterables;
057import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;
058
059import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos;
060
061/**
062 * Test class to exercise the inclusion of snapshots in space quotas
063 */
064@Category({ LargeTests.class })
065public class TestSpaceQuotasWithSnapshots {
066
067  @ClassRule
068  public static final HBaseClassTestRule CLASS_RULE =
069    HBaseClassTestRule.forClass(TestSpaceQuotasWithSnapshots.class);
070
071  private static final Logger LOG = LoggerFactory.getLogger(TestSpaceQuotasWithSnapshots.class);
072  private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
073  // Global for all tests in the class
074  private static final AtomicLong COUNTER = new AtomicLong(0);
075  private static final long FUDGE_FOR_TABLE_SIZE = 500L * SpaceQuotaHelperForTests.ONE_KILOBYTE;
076
077  @Rule
078  public TestName testName = new TestName();
079  private SpaceQuotaHelperForTests helper;
080  private Connection conn;
081  private Admin admin;
082
083  @BeforeClass
084  public static void setUp() throws Exception {
085    Configuration conf = TEST_UTIL.getConfiguration();
086    SpaceQuotaHelperForTests.updateConfigForQuotas(conf);
087    TEST_UTIL.startMiniCluster(1);
088    // Wait till quota table onlined.
089    TEST_UTIL.waitFor(10000, new Waiter.Predicate<Exception>() {
090      @Override
091      public boolean evaluate() throws Exception {
092        return TEST_UTIL.getAdmin().tableExists(QuotaTableUtil.QUOTA_TABLE_NAME);
093      }
094    });
095  }
096
097  @AfterClass
098  public static void tearDown() throws Exception {
099    TEST_UTIL.shutdownMiniCluster();
100  }
101
102  @Before
103  public void removeAllQuotas() throws Exception {
104    helper = new SpaceQuotaHelperForTests(TEST_UTIL, testName, COUNTER);
105    conn = TEST_UTIL.getConnection();
106    admin = TEST_UTIL.getAdmin();
107  }
108
109  @Test
110  public void testTablesInheritSnapshotSize() throws Exception {
111    TableName tn = helper.createTableWithRegions(1);
112    LOG.info("Writing data");
113    // Set a quota
114    QuotaSettings settings = QuotaSettingsFactory.limitTableSpace(tn,
115      SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS);
116    admin.setQuota(settings);
117    // Write some data
118    final long initialSize = 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE;
119    helper.writeData(tn, initialSize);
120
121    LOG.info("Waiting until table size reflects written data");
122    // Wait until that data is seen by the master
123    TEST_UTIL.waitFor(30 * 1000, 500, new SpaceQuotaSnapshotPredicate(conn, tn) {
124      @Override
125      boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
126        return snapshot.getUsage() >= initialSize;
127      }
128    });
129
130    // Make sure we see the final quota usage size
131    waitForStableQuotaSize(conn, tn, null);
132
133    // The actual size on disk after we wrote our data the first time
134    final long actualInitialSize = conn.getAdmin().getCurrentSpaceQuotaSnapshot(tn).getUsage();
135    LOG.info("Initial table size was " + actualInitialSize);
136
137    LOG.info("Snapshot the table");
138    final String snapshot1 = tn.toString() + "_snapshot1";
139    admin.snapshot(snapshot1, tn);
140
141    // Write the same data again, then flush+compact. This should make sure that
142    // the snapshot is referencing files that the table no longer references.
143    LOG.info("Write more data");
144    helper.writeData(tn, initialSize);
145    LOG.info("Flush the table");
146    admin.flush(tn);
147    LOG.info("Synchronously compacting the table");
148    TEST_UTIL.compact(tn, true);
149
150    final long upperBound = initialSize + FUDGE_FOR_TABLE_SIZE;
151    final long lowerBound = initialSize - FUDGE_FOR_TABLE_SIZE;
152
153    // Store the actual size after writing more data and then compacting it down to one file
154    LOG.info("Waiting for the region reports to reflect the correct size, between (" + lowerBound
155      + ", " + upperBound + ")");
156    TEST_UTIL.waitFor(30 * 1000, 500, new Predicate<Exception>() {
157      @Override
158      public boolean evaluate() throws Exception {
159        long size = getRegionSizeReportForTable(conn, tn);
160        return size < upperBound && size > lowerBound;
161      }
162    });
163
164    // Make sure we see the "final" new size for the table, not some intermediate
165    waitForStableRegionSizeReport(conn, tn);
166    final long finalSize = getRegionSizeReportForTable(conn, tn);
167    assertNotNull("Did not expect to see a null size", finalSize);
168    LOG.info("Last seen size: " + finalSize);
169
170    // Make sure the QuotaObserverChore has time to reflect the new region size reports
171    // (we saw above). The usage of the table should *not* decrease when we check it below,
172    // though, because the snapshot on our table will cause the table to "retain" the size.
173    TEST_UTIL.waitFor(20 * 1000, 500, new SpaceQuotaSnapshotPredicate(conn, tn) {
174      @Override
175      public boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
176        return snapshot.getUsage() >= finalSize;
177      }
178    });
179
180    // The final usage should be the sum of the initial size (referenced by the snapshot) and the
181    // new size we just wrote above.
182    long expectedFinalSize = actualInitialSize + finalSize;
183    LOG.info("Expecting table usage to be " + actualInitialSize + " + " + finalSize + " = "
184      + expectedFinalSize);
185    // The size of the table (WRT quotas) should now be approximately double what it was previously
186    TEST_UTIL.waitFor(30 * 1000, 1000, new SpaceQuotaSnapshotPredicate(conn, tn) {
187      @Override
188      boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
189        LOG.debug("Checking for " + expectedFinalSize + " == " + snapshot.getUsage());
190        return expectedFinalSize == snapshot.getUsage();
191      }
192    });
193
194    Map<String, Long> snapshotSizes = QuotaTableUtil.getObservedSnapshotSizes(conn);
195    Long size = snapshotSizes.get(snapshot1);
196    assertNotNull("Did not observe the size of the snapshot", size);
197    assertEquals("The recorded size of the HBase snapshot was not the size we expected",
198      actualInitialSize, size.longValue());
199  }
200
201  @Test
202  public void testNamespacesInheritSnapshotSize() throws Exception {
203    String ns = helper.createNamespace().getName();
204    TableName tn = helper.createTableWithRegions(ns, 1);
205    LOG.info("Writing data");
206    // Set a quota
207    QuotaSettings settings = QuotaSettingsFactory.limitNamespaceSpace(ns,
208      SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS);
209    admin.setQuota(settings);
210
211    // Write some data and flush it to disk
212    final long initialSize = 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE;
213    helper.writeData(tn, initialSize);
214    admin.flush(tn);
215
216    LOG.info("Waiting until namespace size reflects written data");
217    // Wait until that data is seen by the master
218    TEST_UTIL.waitFor(30 * 1000, 500, new SpaceQuotaSnapshotPredicate(conn, ns) {
219      @Override
220      boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
221        return snapshot.getUsage() >= initialSize;
222      }
223    });
224
225    // Make sure we see the "final" new size for the table, not some intermediate
226    waitForStableQuotaSize(conn, null, ns);
227
228    // The actual size on disk after we wrote our data the first time
229    final long actualInitialSize = conn.getAdmin().getCurrentSpaceQuotaSnapshot(ns).getUsage();
230    LOG.info("Initial table size was " + actualInitialSize);
231
232    LOG.info("Snapshot the table");
233    final String snapshot1 = tn.getQualifierAsString() + "_snapshot1";
234    admin.snapshot(snapshot1, tn);
235
236    // Write the same data again, then flush+compact. This should make sure that
237    // the snapshot is referencing files that the table no longer references.
238    LOG.info("Write more data");
239    helper.writeData(tn, initialSize);
240    LOG.info("Flush the table");
241    admin.flush(tn);
242    LOG.info("Synchronously compacting the table");
243    TEST_UTIL.compact(tn, true);
244
245    final long upperBound = initialSize + FUDGE_FOR_TABLE_SIZE;
246    final long lowerBound = initialSize - FUDGE_FOR_TABLE_SIZE;
247
248    LOG.info("Waiting for the region reports to reflect the correct size, between (" + lowerBound
249      + ", " + upperBound + ")");
250    TEST_UTIL.waitFor(30 * 1000, 500, new Predicate<Exception>() {
251      @Override
252      public boolean evaluate() throws Exception {
253        Map<TableName, Long> sizes = conn.getAdmin().getSpaceQuotaTableSizes();
254        LOG.debug("Master observed table sizes from region size reports: " + sizes);
255        Long size = sizes.get(tn);
256        if (null == size) {
257          return false;
258        }
259        return size < upperBound && size > lowerBound;
260      }
261    });
262
263    // Make sure we see the "final" new size for the table, not some intermediate
264    waitForStableRegionSizeReport(conn, tn);
265    final long finalSize = getRegionSizeReportForTable(conn, tn);
266    assertNotNull("Did not expect to see a null size", finalSize);
267    LOG.info("Final observed size of table: " + finalSize);
268
269    // Make sure the QuotaObserverChore has time to reflect the new region size reports
270    // (we saw above). The usage of the table should *not* decrease when we check it below,
271    // though, because the snapshot on our table will cause the table to "retain" the size.
272    TEST_UTIL.waitFor(20 * 1000, 500, new SpaceQuotaSnapshotPredicate(conn, ns) {
273      @Override
274      public boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
275        return snapshot.getUsage() >= finalSize;
276      }
277    });
278
279    // The final usage should be the sum of the initial size (referenced by the snapshot) and the
280    // new size we just wrote above.
281    long expectedFinalSize = actualInitialSize + finalSize;
282    LOG.info("Expecting namespace usage to be " + actualInitialSize + " + " + finalSize + " = "
283      + expectedFinalSize);
284    // The size of the table (WRT quotas) should now be approximately double what it was previously
285    TEST_UTIL.waitFor(30 * 1000, 1000, new SpaceQuotaSnapshotPredicate(conn, ns) {
286      @Override
287      boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
288        LOG.debug("Checking for " + expectedFinalSize + " == " + snapshot.getUsage());
289        return expectedFinalSize == snapshot.getUsage();
290      }
291    });
292
293    Map<String, Long> snapshotSizes = QuotaTableUtil.getObservedSnapshotSizes(conn);
294    Long size = snapshotSizes.get(snapshot1);
295    assertNotNull("Did not observe the size of the snapshot", size);
296    assertEquals("The recorded size of the HBase snapshot was not the size we expected",
297      actualInitialSize, size.longValue());
298  }
299
300  @Test
301  public void testTablesWithSnapshots() throws Exception {
302    final Connection conn = TEST_UTIL.getConnection();
303    final SpaceViolationPolicy policy = SpaceViolationPolicy.NO_INSERTS;
304    final TableName tn = helper.createTableWithRegions(10);
305
306    // 3MB limit on the table
307    final long tableLimit = 3L * SpaceQuotaHelperForTests.ONE_MEGABYTE;
308    TEST_UTIL.getAdmin().setQuota(QuotaSettingsFactory.limitTableSpace(tn, tableLimit, policy));
309
310    LOG.info("Writing first data set");
311    // Write more data than should be allowed and flush it to disk
312    helper.writeData(tn, 1L * SpaceQuotaHelperForTests.ONE_MEGABYTE, "q1");
313
314    LOG.info("Creating snapshot");
315    TEST_UTIL.getAdmin().snapshot(tn.toString() + "snap1", tn, SnapshotType.FLUSH);
316
317    LOG.info("Writing second data set");
318    // Write some more data
319    helper.writeData(tn, 1L * SpaceQuotaHelperForTests.ONE_MEGABYTE, "q2");
320
321    LOG.info("Flushing and major compacting table");
322    // Compact the table to force the snapshot to own all of its files
323    TEST_UTIL.getAdmin().flush(tn);
324    TEST_UTIL.compact(tn, true);
325
326    LOG.info("Checking for quota violation");
327    // Wait to observe the quota moving into violation
328    TEST_UTIL.waitFor(60_000, 1_000, new Predicate<Exception>() {
329      @Override
330      public boolean evaluate() throws Exception {
331        Scan s = QuotaTableUtil.makeQuotaSnapshotScanForTable(tn);
332        try (Table t = conn.getTable(QuotaTableUtil.QUOTA_TABLE_NAME)) {
333          ResultScanner rs = t.getScanner(s);
334          try {
335            Result r = Iterables.getOnlyElement(rs);
336            CellScanner cs = r.cellScanner();
337            assertTrue(cs.advance());
338            Cell c = cs.current();
339            SpaceQuotaSnapshot snapshot = SpaceQuotaSnapshot
340              .toSpaceQuotaSnapshot(QuotaProtos.SpaceQuotaSnapshot.parseFrom(UnsafeByteOperations
341                .unsafeWrap(c.getValueArray(), c.getValueOffset(), c.getValueLength())));
342            LOG.info(
343              snapshot.getUsage() + "/" + snapshot.getLimit() + " " + snapshot.getQuotaStatus());
344            // We expect to see the table move to violation
345            return snapshot.getQuotaStatus().isInViolation();
346          } finally {
347            if (null != rs) {
348              rs.close();
349            }
350          }
351        }
352      }
353    });
354  }
355
356  @Test
357  public void testRematerializedTablesDoNoInheritSpace() throws Exception {
358    TableName tn = helper.createTableWithRegions(1);
359    TableName tn2 = helper.getNextTableName();
360    LOG.info("Writing data");
361    // Set a quota on both tables
362    QuotaSettings settings = QuotaSettingsFactory.limitTableSpace(tn,
363      SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS);
364    admin.setQuota(settings);
365    QuotaSettings settings2 = QuotaSettingsFactory.limitTableSpace(tn2,
366      SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS);
367    admin.setQuota(settings2);
368    // Write some data
369    final long initialSize = 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE;
370    helper.writeData(tn, initialSize);
371
372    LOG.info("Waiting until table size reflects written data");
373    // Wait until that data is seen by the master
374    TEST_UTIL.waitFor(30 * 1000, 500, new SpaceQuotaSnapshotPredicate(conn, tn) {
375      @Override
376      boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
377        return snapshot.getUsage() >= initialSize;
378      }
379    });
380
381    // Make sure we see the final quota usage size
382    waitForStableQuotaSize(conn, tn, null);
383
384    // The actual size on disk after we wrote our data the first time
385    final long actualInitialSize = conn.getAdmin().getCurrentSpaceQuotaSnapshot(tn).getUsage();
386    LOG.info("Initial table size was " + actualInitialSize);
387
388    LOG.info("Snapshot the table");
389    final String snapshot1 = tn.toString() + "_snapshot1";
390    admin.snapshot(snapshot1, tn);
391
392    admin.cloneSnapshot(snapshot1, tn2);
393
394    // Write some more data to the first table
395    helper.writeData(tn, initialSize, "q2");
396    admin.flush(tn);
397
398    // Watch the usage of the first table with some more data to know when the new
399    // region size reports were sent to the master
400    TEST_UTIL.waitFor(30_000, 1_000, new SpaceQuotaSnapshotPredicate(conn, tn) {
401      @Override
402      boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
403        return snapshot.getUsage() >= actualInitialSize * 2;
404      }
405    });
406
407    // We know that reports were sent by our RS, verify that they take up zero size.
408    SpaceQuotaSnapshot snapshot =
409      (SpaceQuotaSnapshot) conn.getAdmin().getCurrentSpaceQuotaSnapshot(tn2);
410    assertNotNull(snapshot);
411    assertEquals(0, snapshot.getUsage());
412
413    // Compact the cloned table to force it to own its own files.
414    TEST_UTIL.compact(tn2, true);
415    // After the table is compacted, it should have its own files and be the same size as originally
416    // But The compaction result file has an additional compaction event tracker
417    TEST_UTIL.waitFor(30_000, 1_000, new SpaceQuotaSnapshotPredicate(conn, tn2) {
418      @Override
419      boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
420        return snapshot.getUsage() >= actualInitialSize;
421      }
422    });
423  }
424
425  void waitForStableQuotaSize(Connection conn, TableName tn, String ns) throws Exception {
426    // For some stability in the value before proceeding
427    // Helps make sure that we got the actual last value, not some inbetween
428    AtomicLong lastValue = new AtomicLong(-1);
429    AtomicInteger counter = new AtomicInteger(0);
430    TEST_UTIL.waitFor(15_000, 500, new SpaceQuotaSnapshotPredicate(conn, tn, ns) {
431      @Override
432      boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
433        LOG.debug("Last observed size=" + lastValue.get());
434        if (snapshot.getUsage() == lastValue.get()) {
435          int numMatches = counter.incrementAndGet();
436          if (numMatches >= 5) {
437            return true;
438          }
439          // Not yet..
440          return false;
441        }
442        counter.set(0);
443        lastValue.set(snapshot.getUsage());
444        return false;
445      }
446    });
447  }
448
449  long getRegionSizeReportForTable(Connection conn, TableName tn) throws IOException {
450    Map<TableName, Long> sizes = conn.getAdmin().getSpaceQuotaTableSizes();
451    Long value = sizes.get(tn);
452    if (null == value) {
453      return 0L;
454    }
455    return value.longValue();
456  }
457
458  void waitForStableRegionSizeReport(Connection conn, TableName tn) throws Exception {
459    // For some stability in the value before proceeding
460    // Helps make sure that we got the actual last value, not some inbetween
461    AtomicLong lastValue = new AtomicLong(-1);
462    AtomicInteger counter = new AtomicInteger(0);
463    TEST_UTIL.waitFor(15_000, 500, new Predicate<Exception>() {
464      @Override
465      public boolean evaluate() throws Exception {
466        LOG.debug("Last observed size=" + lastValue.get());
467        long actual = getRegionSizeReportForTable(conn, tn);
468        if (actual == lastValue.get()) {
469          int numMatches = counter.incrementAndGet();
470          if (numMatches >= 5) {
471            return true;
472          }
473          // Not yet..
474          return false;
475        }
476        counter.set(0);
477        lastValue.set(actual);
478        return false;
479      }
480    });
481  }
482}