001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.apache.hadoop.hbase.HBaseTestingUtil.START_KEY;
021import static org.apache.hadoop.hbase.HBaseTestingUtil.START_KEY_BYTES;
022import static org.apache.hadoop.hbase.HBaseTestingUtil.fam1;
023import static org.apache.hadoop.hbase.regionserver.Store.PRIORITY_USER;
024import static org.junit.Assert.assertEquals;
025import static org.junit.Assert.assertNotNull;
026import static org.junit.Assert.assertTrue;
027
028import java.io.IOException;
029import java.util.ArrayList;
030import java.util.Collection;
031import java.util.HashMap;
032import java.util.List;
033import java.util.Map;
034import java.util.Map.Entry;
035import org.apache.hadoop.conf.Configuration;
036import org.apache.hadoop.hbase.Cell;
037import org.apache.hadoop.hbase.CellUtil;
038import org.apache.hadoop.hbase.HBaseClassTestRule;
039import org.apache.hadoop.hbase.HBaseTestingUtil;
040import org.apache.hadoop.hbase.HConstants;
041import org.apache.hadoop.hbase.HTestConst;
042import org.apache.hadoop.hbase.KeepDeletedCells;
043import org.apache.hadoop.hbase.TableName;
044import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
045import org.apache.hadoop.hbase.client.Delete;
046import org.apache.hadoop.hbase.client.Get;
047import org.apache.hadoop.hbase.client.Result;
048import org.apache.hadoop.hbase.client.Scan;
049import org.apache.hadoop.hbase.client.Table;
050import org.apache.hadoop.hbase.client.TableDescriptor;
051import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
052import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder;
053import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoderImpl;
054import org.apache.hadoop.hbase.io.hfile.HFileScanner;
055import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker;
056import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequestImpl;
057import org.apache.hadoop.hbase.regionserver.compactions.RatioBasedCompactionPolicy;
058import org.apache.hadoop.hbase.testclassification.LargeTests;
059import org.apache.hadoop.hbase.testclassification.RegionServerTests;
060import org.apache.hadoop.hbase.util.Bytes;
061import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
062import org.apache.hadoop.hbase.wal.WAL;
063import org.junit.After;
064import org.junit.Before;
065import org.junit.ClassRule;
066import org.junit.Rule;
067import org.junit.Test;
068import org.junit.experimental.categories.Category;
069import org.junit.rules.TestName;
070import org.junit.runner.RunWith;
071import org.junit.runners.Parameterized;
072import org.slf4j.Logger;
073import org.slf4j.LoggerFactory;
074
075/**
076 * Test major compactions
077 */
078@Category({ RegionServerTests.class, LargeTests.class })
079@RunWith(Parameterized.class)
080public class TestMajorCompaction {
081
082  @ClassRule
083  public static final HBaseClassTestRule CLASS_RULE =
084    HBaseClassTestRule.forClass(TestMajorCompaction.class);
085
086  @Parameterized.Parameters
087  public static Object[] data() {
088    return new Object[] { "NONE", "BASIC", "EAGER" };
089  }
090
091  @Rule
092  public TestName name;
093  private static final Logger LOG = LoggerFactory.getLogger(TestMajorCompaction.class.getName());
094  private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
095  protected Configuration conf = UTIL.getConfiguration();
096
097  private HRegion r = null;
098  private TableDescriptor htd = null;
099  private static final byte[] COLUMN_FAMILY = fam1;
100  private final byte[] STARTROW = Bytes.toBytes(START_KEY);
101  private static final byte[] COLUMN_FAMILY_TEXT = COLUMN_FAMILY;
102  private int compactionThreshold;
103  private byte[] secondRowBytes, thirdRowBytes;
104  private static final long MAX_FILES_TO_COMPACT = 10;
105
106  /** constructor */
107  public TestMajorCompaction(String compType) {
108    super();
109    name = new TestName();
110    // Set cache flush size to 1MB
111    conf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024 * 1024);
112    conf.setInt(HConstants.HREGION_MEMSTORE_BLOCK_MULTIPLIER, 100);
113    compactionThreshold = conf.getInt("hbase.hstore.compactionThreshold", 3);
114    conf.set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY, String.valueOf(compType));
115
116    secondRowBytes = START_KEY_BYTES.clone();
117    // Increment the least significant character so we get to next row.
118    secondRowBytes[START_KEY_BYTES.length - 1]++;
119    thirdRowBytes = START_KEY_BYTES.clone();
120    thirdRowBytes[START_KEY_BYTES.length - 1] =
121      (byte) (thirdRowBytes[START_KEY_BYTES.length - 1] + 2);
122  }
123
124  @Before
125  public void setUp() throws Exception {
126    this.htd = UTIL.createTableDescriptor(
127      TableName.valueOf(name.getMethodName().replace('[', 'i').replace(']', 'i')),
128      ColumnFamilyDescriptorBuilder.DEFAULT_MIN_VERSIONS, 3, HConstants.FOREVER,
129      ColumnFamilyDescriptorBuilder.DEFAULT_KEEP_DELETED);
130    this.r = UTIL.createLocalHRegion(htd, null, null);
131  }
132
133  @After
134  public void tearDown() throws Exception {
135    WAL wal = ((HRegion) r).getWAL();
136    ((HRegion) r).close();
137    wal.close();
138  }
139
140  /**
141   * Test that on a major compaction, if all cells are expired or deleted, then we'll end up with no
142   * product. Make sure scanner over region returns right answer in this case - and that it just
143   * basically works.
144   * @throws IOException exception encountered
145   */
146  @Test
147  public void testMajorCompactingToNoOutput() throws IOException {
148    testMajorCompactingWithDeletes(KeepDeletedCells.FALSE);
149  }
150
151  /**
152   * Test that on a major compaction,Deleted cells are retained if keep deleted cells is set to true
153   * @throws IOException exception encountered
154   */
155  @Test
156  public void testMajorCompactingWithKeepDeletedCells() throws IOException {
157    testMajorCompactingWithDeletes(KeepDeletedCells.TRUE);
158  }
159
160  /**
161   * Run compaction and flushing memstore Assert deletes get cleaned up.
162   */
163  @Test
164  public void testMajorCompaction() throws Exception {
165    majorCompaction();
166  }
167
168  @Test
169  public void testDataBlockEncodingInCacheOnly() throws Exception {
170    majorCompactionWithDataBlockEncoding(true);
171  }
172
173  @Test
174  public void testDataBlockEncodingEverywhere() throws Exception {
175    majorCompactionWithDataBlockEncoding(false);
176  }
177
178  public void majorCompactionWithDataBlockEncoding(boolean inCacheOnly) throws Exception {
179    Map<HStore, HFileDataBlockEncoder> replaceBlockCache = new HashMap<>();
180    for (HStore store : r.getStores()) {
181      HFileDataBlockEncoder blockEncoder = store.getDataBlockEncoder();
182      replaceBlockCache.put(store, blockEncoder);
183      final DataBlockEncoding inCache = DataBlockEncoding.PREFIX;
184      final DataBlockEncoding onDisk = inCacheOnly ? DataBlockEncoding.NONE : inCache;
185      ((HStore) store).setDataBlockEncoderInTest(new HFileDataBlockEncoderImpl(onDisk));
186    }
187
188    majorCompaction();
189
190    // restore settings
191    for (Entry<HStore, HFileDataBlockEncoder> entry : replaceBlockCache.entrySet()) {
192      ((HStore) entry.getKey()).setDataBlockEncoderInTest(entry.getValue());
193    }
194  }
195
196  private void majorCompaction() throws Exception {
197    createStoreFile(r);
198    for (int i = 0; i < compactionThreshold; i++) {
199      createStoreFile(r);
200    }
201    // Add more content.
202    HTestConst.addContent(new RegionAsTable(r), Bytes.toString(COLUMN_FAMILY));
203
204    // Now there are about 5 versions of each column.
205    // Default is that there only 3 (MAXVERSIONS) versions allowed per column.
206    //
207    // Assert == 3 when we ask for versions.
208    Result result = r.get(new Get(STARTROW).addFamily(COLUMN_FAMILY_TEXT).readVersions(100));
209    assertEquals(compactionThreshold, result.size());
210
211    r.flush(true);
212    r.compact(true);
213
214    // look at the second row
215    // Increment the least significant character so we get to next row.
216    byte[] secondRowBytes = START_KEY_BYTES.clone();
217    secondRowBytes[START_KEY_BYTES.length - 1]++;
218
219    // Always 3 versions if that is what max versions is.
220    result = r.get(new Get(secondRowBytes).addFamily(COLUMN_FAMILY_TEXT).readVersions(100));
221    LOG.debug(
222      "Row " + Bytes.toStringBinary(secondRowBytes) + " after " + "initial compaction: " + result);
223    assertEquals("Invalid number of versions of row " + Bytes.toStringBinary(secondRowBytes) + ".",
224      compactionThreshold, result.size());
225
226    // Now add deletes to memstore and then flush it.
227    // That will put us over
228    // the compaction threshold of 3 store files. Compacting these store files
229    // should result in a compacted store file that has no references to the
230    // deleted row.
231    LOG.debug("Adding deletes to memstore and flushing");
232    Delete delete = new Delete(secondRowBytes, EnvironmentEdgeManager.currentTime());
233    byte[][] famAndQf = { COLUMN_FAMILY, null };
234    delete.addFamily(famAndQf[0]);
235    r.delete(delete);
236
237    // Assert deleted.
238    result = r.get(new Get(secondRowBytes).addFamily(COLUMN_FAMILY_TEXT).readVersions(100));
239    assertTrue("Second row should have been deleted", result.isEmpty());
240
241    r.flush(true);
242
243    result = r.get(new Get(secondRowBytes).addFamily(COLUMN_FAMILY_TEXT).readVersions(100));
244    assertTrue("Second row should have been deleted", result.isEmpty());
245
246    // Add a bit of data and flush. Start adding at 'bbb'.
247    createSmallerStoreFile(this.r);
248    r.flush(true);
249    // Assert that the second row is still deleted.
250    result = r.get(new Get(secondRowBytes).addFamily(COLUMN_FAMILY_TEXT).readVersions(100));
251    assertTrue("Second row should still be deleted", result.isEmpty());
252
253    // Force major compaction.
254    r.compact(true);
255    assertEquals(1, r.getStore(COLUMN_FAMILY_TEXT).getStorefiles().size());
256
257    result = r.get(new Get(secondRowBytes).addFamily(COLUMN_FAMILY_TEXT).readVersions(100));
258    assertTrue("Second row should still be deleted", result.isEmpty());
259
260    // Make sure the store files do have some 'aaa' keys in them -- exactly 3.
261    // Also, that compacted store files do not have any secondRowBytes because
262    // they were deleted.
263    verifyCounts(3, 0);
264
265    // Multiple versions allowed for an entry, so the delete isn't enough
266    // Lower TTL and expire to ensure that all our entries have been wiped
267    final int ttl = 1000;
268    for (HStore store : r.getStores()) {
269      ScanInfo old = store.getScanInfo();
270      ScanInfo si = old.customize(old.getMaxVersions(), ttl, old.getKeepDeletedCells());
271      store.setScanInfo(si);
272    }
273    Thread.sleep(1000);
274
275    r.compact(true);
276    int count = count();
277    assertEquals("Should not see anything after TTL has expired", 0, count);
278  }
279
280  @Test
281  public void testTimeBasedMajorCompaction() throws Exception {
282    // create 2 storefiles and force a major compaction to reset the time
283    int delay = 10 * 1000; // 10 sec
284    float jitterPct = 0.20f; // 20%
285    conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, delay);
286    conf.setFloat("hbase.hregion.majorcompaction.jitter", jitterPct);
287
288    HStore s = ((HStore) r.getStore(COLUMN_FAMILY));
289    s.storeEngine.getCompactionPolicy().setConf(conf);
290    try {
291      createStoreFile(r);
292      createStoreFile(r);
293      r.compact(true);
294
295      // add one more file & verify that a regular compaction won't work
296      createStoreFile(r);
297      r.compact(false);
298      assertEquals(2, s.getStorefilesCount());
299
300      // ensure that major compaction time is deterministic
301      RatioBasedCompactionPolicy c =
302        (RatioBasedCompactionPolicy) s.storeEngine.getCompactionPolicy();
303      Collection<HStoreFile> storeFiles = s.getStorefiles();
304      long mcTime = c.getNextMajorCompactTime(storeFiles);
305      for (int i = 0; i < 10; ++i) {
306        assertEquals(mcTime, c.getNextMajorCompactTime(storeFiles));
307      }
308
309      // ensure that the major compaction time is within the variance
310      long jitter = Math.round(delay * jitterPct);
311      assertTrue(delay - jitter <= mcTime && mcTime <= delay + jitter);
312
313      // wait until the time-based compaction interval
314      Thread.sleep(mcTime);
315
316      // trigger a compaction request and ensure that it's upgraded to major
317      r.compact(false);
318      assertEquals(1, s.getStorefilesCount());
319    } finally {
320      // reset the timed compaction settings
321      conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 1000 * 60 * 60 * 24);
322      conf.setFloat("hbase.hregion.majorcompaction.jitter", 0.20F);
323      // run a major to reset the cache
324      createStoreFile(r);
325      r.compact(true);
326      assertEquals(1, s.getStorefilesCount());
327    }
328  }
329
330  private void verifyCounts(int countRow1, int countRow2) throws Exception {
331    int count1 = 0;
332    int count2 = 0;
333    for (HStoreFile f : r.getStore(COLUMN_FAMILY_TEXT).getStorefiles()) {
334      HFileScanner scanner = f.getReader().getScanner(false, false);
335      scanner.seekTo();
336      do {
337        byte[] row = CellUtil.cloneRow(scanner.getCell());
338        if (Bytes.equals(row, STARTROW)) {
339          count1++;
340        } else if (Bytes.equals(row, secondRowBytes)) {
341          count2++;
342        }
343      } while (scanner.next());
344    }
345    assertEquals(countRow1, count1);
346    assertEquals(countRow2, count2);
347  }
348
349  private int count() throws IOException {
350    int count = 0;
351    for (HStoreFile f : r.getStore(COLUMN_FAMILY_TEXT).getStorefiles()) {
352      HFileScanner scanner = f.getReader().getScanner(false, false);
353      if (!scanner.seekTo()) {
354        continue;
355      }
356      do {
357        count++;
358      } while (scanner.next());
359    }
360    return count;
361  }
362
363  private void createStoreFile(final HRegion region) throws IOException {
364    createStoreFile(region, Bytes.toString(COLUMN_FAMILY));
365  }
366
367  private void createStoreFile(final HRegion region, String family) throws IOException {
368    Table loader = new RegionAsTable(region);
369    HTestConst.addContent(loader, family);
370    region.flush(true);
371  }
372
373  private void createSmallerStoreFile(final HRegion region) throws IOException {
374    Table loader = new RegionAsTable(region);
375    HTestConst.addContent(loader, Bytes.toString(COLUMN_FAMILY), Bytes.toBytes("" + "bbb"), null);
376    region.flush(true);
377  }
378
379  /**
380   * Test for HBASE-5920 - Test user requested major compactions always occurring
381   */
382  @Test
383  public void testNonUserMajorCompactionRequest() throws Exception {
384    HStore store = r.getStore(COLUMN_FAMILY);
385    createStoreFile(r);
386    for (int i = 0; i < MAX_FILES_TO_COMPACT + 1; i++) {
387      createStoreFile(r);
388    }
389    store.triggerMajorCompaction();
390
391    CompactionRequestImpl request = store.requestCompaction().get().getRequest();
392    assertNotNull("Expected to receive a compaction request", request);
393    assertEquals(
394      "System-requested major compaction should not occur if there are too many store files", false,
395      request.isMajor());
396  }
397
398  /**
399   * Test for HBASE-5920
400   */
401  @Test
402  public void testUserMajorCompactionRequest() throws IOException {
403    HStore store = r.getStore(COLUMN_FAMILY);
404    createStoreFile(r);
405    for (int i = 0; i < MAX_FILES_TO_COMPACT + 1; i++) {
406      createStoreFile(r);
407    }
408    store.triggerMajorCompaction();
409    CompactionRequestImpl request = store
410      .requestCompaction(PRIORITY_USER, CompactionLifeCycleTracker.DUMMY, null).get().getRequest();
411    assertNotNull("Expected to receive a compaction request", request);
412    assertEquals(
413      "User-requested major compaction should always occur, even if there are too many store files",
414      true, request.isMajor());
415  }
416
417  /**
418   * Test that on a major compaction, if all cells are expired or deleted, then we'll end up with no
419   * product. Make sure scanner over region returns right answer in this case - and that it just
420   * basically works.
421   */
422  @Test
423  public void testMajorCompactingToNoOutputWithReverseScan() throws IOException {
424    createStoreFile(r);
425    for (int i = 0; i < compactionThreshold; i++) {
426      createStoreFile(r);
427    }
428    // Now delete everything.
429    Scan scan = new Scan();
430    scan.setReversed(true);
431    InternalScanner s = r.getScanner(scan);
432    do {
433      List<Cell> results = new ArrayList<>();
434      boolean result = s.next(results);
435      assertTrue(!results.isEmpty());
436      r.delete(new Delete(CellUtil.cloneRow(results.get(0))));
437      if (!result) {
438        break;
439      }
440    } while (true);
441    s.close();
442    // Flush
443    r.flush(true);
444    // Major compact.
445    r.compact(true);
446    scan = new Scan();
447    scan.setReversed(true);
448    s = r.getScanner(scan);
449    int counter = 0;
450    do {
451      List<Cell> results = new ArrayList<>();
452      boolean result = s.next(results);
453      if (!result) {
454        break;
455      }
456      counter++;
457    } while (true);
458    s.close();
459    assertEquals(0, counter);
460  }
461
462  private void testMajorCompactingWithDeletes(KeepDeletedCells keepDeletedCells)
463    throws IOException {
464    createStoreFile(r);
465    for (int i = 0; i < compactionThreshold; i++) {
466      createStoreFile(r);
467    }
468    // Now delete everything.
469    InternalScanner s = r.getScanner(new Scan());
470    int originalCount = 0;
471    do {
472      List<Cell> results = new ArrayList<>();
473      boolean result = s.next(results);
474      r.delete(new Delete(CellUtil.cloneRow(results.get(0))));
475      if (!result) break;
476      originalCount++;
477    } while (true);
478    s.close();
479    // Flush
480    r.flush(true);
481
482    for (HStore store : this.r.stores.values()) {
483      ScanInfo old = store.getScanInfo();
484      ScanInfo si = old.customize(old.getMaxVersions(), old.getTtl(), keepDeletedCells);
485      store.setScanInfo(si);
486    }
487    // Major compact.
488    r.compact(true);
489    s = r.getScanner(new Scan().setRaw(true));
490    int counter = 0;
491    do {
492      List<Cell> results = new ArrayList<>();
493      boolean result = s.next(results);
494      if (!result) break;
495      counter++;
496    } while (true);
497    assertEquals(keepDeletedCells == KeepDeletedCells.TRUE ? originalCount : 0, counter);
498
499  }
500}