001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.mob;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertTrue;
022import static org.junit.Assert.fail;
023
024import java.io.IOException;
025import java.util.Arrays;
026import java.util.Collection;
027import java.util.List;
028import java.util.stream.Collectors;
029import org.apache.hadoop.conf.Configuration;
030import org.apache.hadoop.fs.FileStatus;
031import org.apache.hadoop.fs.FileSystem;
032import org.apache.hadoop.fs.Path;
033import org.apache.hadoop.hbase.HBaseClassTestRule;
034import org.apache.hadoop.hbase.HBaseTestingUtil;
035import org.apache.hadoop.hbase.ServerName;
036import org.apache.hadoop.hbase.TableName;
037import org.apache.hadoop.hbase.client.Admin;
038import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
039import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
040import org.apache.hadoop.hbase.client.CompactionState;
041import org.apache.hadoop.hbase.client.Put;
042import org.apache.hadoop.hbase.client.Result;
043import org.apache.hadoop.hbase.client.ResultScanner;
044import org.apache.hadoop.hbase.client.Table;
045import org.apache.hadoop.hbase.client.TableDescriptor;
046import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
047import org.apache.hadoop.hbase.testclassification.LargeTests;
048import org.apache.hadoop.hbase.util.Bytes;
049import org.apache.hadoop.hbase.util.RegionSplitter;
050import org.junit.After;
051import org.junit.Before;
052import org.junit.ClassRule;
053import org.junit.Rule;
054import org.junit.Test;
055import org.junit.experimental.categories.Category;
056import org.junit.rules.TestName;
057import org.junit.runner.RunWith;
058import org.junit.runners.Parameterized;
059import org.slf4j.Logger;
060import org.slf4j.LoggerFactory;
061
062/**
063 * Mob file compaction base test. 1. Enables batch mode for regular MOB compaction, Sets batch size
064 * to 7 regions. (Optional) 2. Disables periodic MOB compactions, sets minimum age to archive to 10
065 * sec 3. Creates MOB table with 20 regions 4. Loads MOB data (randomized keys, 1000 rows), flushes
066 * data. 5. Repeats 4. two more times 6. Verifies that we have 20 *3 = 60 mob files (equals to
067 * number of regions x 3) 7. Runs major MOB compaction. 8. Verifies that number of MOB files in a
068 * mob directory is 20 x4 = 80 9. Waits for a period of time larger than minimum age to archive 10.
069 * Runs Mob cleaner chore 11 Verifies that number of MOB files in a mob directory is 20. 12 Runs
070 * scanner and checks all 3 * 1000 rows.
071 */
072@RunWith(Parameterized.class)
073@Category(LargeTests.class)
074public class TestMobCompactionWithDefaults {
075  private static final Logger LOG = LoggerFactory.getLogger(TestMobCompactionWithDefaults.class);
076  @ClassRule
077  public static final HBaseClassTestRule CLASS_RULE =
078    HBaseClassTestRule.forClass(TestMobCompactionWithDefaults.class);
079
080  protected HBaseTestingUtil HTU;
081  protected static Configuration conf;
082  protected static long minAgeToArchive = 10000;
083
084  protected final static String famStr = "f1";
085  protected final static byte[] fam = Bytes.toBytes(famStr);
086  protected final static byte[] qualifier = Bytes.toBytes("q1");
087  protected final static long mobLen = 10;
088  protected final static byte[] mobVal = Bytes
089    .toBytes("01234567890123456789012345678901234567890123456789012345678901234567890123456789");
090
091  @Rule
092  public TestName test = new TestName();
093  protected TableDescriptor tableDescriptor;
094  private ColumnFamilyDescriptor familyDescriptor;
095  protected Admin admin;
096  protected TableName table = null;
097  protected int numRegions = 20;
098  protected int rows = 1000;
099
100  protected MobFileCleanerChore cleanerChore;
101
102  protected Boolean useFileBasedSFT;
103
104  public TestMobCompactionWithDefaults(Boolean useFileBasedSFT) {
105    this.useFileBasedSFT = useFileBasedSFT;
106  }
107
108  @Parameterized.Parameters
109  public static Collection<Boolean> data() {
110    Boolean[] data = { false, true };
111    return Arrays.asList(data);
112  }
113
114  protected void htuStart() throws Exception {
115    HTU = new HBaseTestingUtil();
116    conf = HTU.getConfiguration();
117    conf.setInt("hfile.format.version", 3);
118    // Disable automatic MOB compaction
119    conf.setLong(MobConstants.MOB_COMPACTION_CHORE_PERIOD, 0);
120    // Disable automatic MOB file cleaner chore
121    conf.setLong(MobConstants.MOB_CLEANER_PERIOD, 0);
122    // Set minimum age to archive to 10 sec
123    conf.setLong(MobConstants.MIN_AGE_TO_ARCHIVE_KEY, minAgeToArchive);
124    // Set compacted file discharger interval to a half minAgeToArchive
125    conf.setLong("hbase.hfile.compaction.discharger.interval", minAgeToArchive / 2);
126    conf.setBoolean("hbase.regionserver.compaction.enabled", false);
127    if (useFileBasedSFT) {
128      conf.set(StoreFileTrackerFactory.TRACKER_IMPL,
129        "org.apache.hadoop.hbase.regionserver.storefiletracker.FileBasedStoreFileTracker");
130    }
131    additonalConfigSetup();
132    HTU.startMiniCluster();
133  }
134
135  protected void additonalConfigSetup() {
136  }
137
138  @Before
139  public void setUp() throws Exception {
140    htuStart();
141    admin = HTU.getAdmin();
142    cleanerChore = new MobFileCleanerChore();
143    familyDescriptor = ColumnFamilyDescriptorBuilder.newBuilder(fam).setMobEnabled(true)
144      .setMobThreshold(mobLen).setMaxVersions(1).build();
145    tableDescriptor = HTU.createModifyableTableDescriptor(TestMobUtils.getTableName(test))
146      .setColumnFamily(familyDescriptor).build();
147    RegionSplitter.UniformSplit splitAlgo = new RegionSplitter.UniformSplit();
148    byte[][] splitKeys = splitAlgo.split(numRegions);
149    table = HTU.createTable(tableDescriptor, splitKeys).getName();
150  }
151
152  private void loadData(TableName tableName, int num) {
153    LOG.info("Started loading {} rows into {}", num, tableName);
154    try (final Table table = HTU.getConnection().getTable(tableName)) {
155      for (int i = 0; i < num; i++) {
156        byte[] key = new byte[32];
157        Bytes.random(key);
158        Put p = new Put(key);
159        p.addColumn(fam, qualifier, mobVal);
160        table.put(p);
161      }
162      admin.flush(tableName);
163      LOG.info("Finished loading {} rows into {}", num, tableName);
164    } catch (Exception e) {
165      LOG.error("MOB file compaction chore test FAILED", e);
166      fail("MOB file compaction chore test FAILED");
167    }
168  }
169
170  @After
171  public void tearDown() throws Exception {
172    admin.disableTable(tableDescriptor.getTableName());
173    admin.deleteTable(tableDescriptor.getTableName());
174    HTU.shutdownMiniCluster();
175  }
176
177  @Test
178  public void baseTestMobFileCompaction() throws InterruptedException, IOException {
179    LOG.info("MOB compaction " + description() + " started");
180    loadAndFlushThreeTimes(rows, table, famStr);
181    mobCompact(tableDescriptor, familyDescriptor);
182    assertEquals("Should have 4 MOB files per region due to 3xflush + compaction.", numRegions * 4,
183      getNumberOfMobFiles(table, famStr));
184    cleanupAndVerifyCounts(table, famStr, 3 * rows);
185    LOG.info("MOB compaction " + description() + " finished OK");
186  }
187
188  @Test
189  public void testMobFileCompactionAfterSnapshotClone() throws InterruptedException, IOException {
190    final TableName clone = TableName.valueOf(TestMobUtils.getTableName(test) + "-clone");
191    LOG.info("MOB compaction of cloned snapshot, " + description() + " started");
192    loadAndFlushThreeTimes(rows, table, famStr);
193    LOG.debug("Taking snapshot and cloning table {}", table);
194    admin.snapshot(TestMobUtils.getTableName(test), table);
195    admin.cloneSnapshot(TestMobUtils.getTableName(test), clone);
196    assertEquals("Should have 3 hlinks per region in MOB area from snapshot clone", 3 * numRegions,
197      getNumberOfMobFiles(clone, famStr));
198    mobCompact(admin.getDescriptor(clone), familyDescriptor);
199    assertEquals("Should have 3 hlinks + 1 MOB file per region due to clone + compact",
200      4 * numRegions, getNumberOfMobFiles(clone, famStr));
201    cleanupAndVerifyCounts(clone, famStr, 3 * rows);
202    LOG.info("MOB compaction of cloned snapshot, " + description() + " finished OK");
203  }
204
205  @Test
206  public void testMobFileCompactionAfterSnapshotCloneAndFlush()
207    throws InterruptedException, IOException {
208    final TableName clone = TableName.valueOf(TestMobUtils.getTableName(test) + "-clone");
209    LOG.info("MOB compaction of cloned snapshot after flush, " + description() + " started");
210    loadAndFlushThreeTimes(rows, table, famStr);
211    LOG.debug("Taking snapshot and cloning table {}", table);
212    admin.snapshot(TestMobUtils.getTableName(test), table);
213    admin.cloneSnapshot(TestMobUtils.getTableName(test), clone);
214    assertEquals("Should have 3 hlinks per region in MOB area from snapshot clone", 3 * numRegions,
215      getNumberOfMobFiles(clone, famStr));
216    loadAndFlushThreeTimes(rows, clone, famStr);
217    mobCompact(admin.getDescriptor(clone), familyDescriptor);
218    assertEquals("Should have 7 MOB file per region due to clone + 3xflush + compact",
219      7 * numRegions, getNumberOfMobFiles(clone, famStr));
220    cleanupAndVerifyCounts(clone, famStr, 6 * rows);
221    LOG.info("MOB compaction of cloned snapshot w flush, " + description() + " finished OK");
222  }
223
224  protected void loadAndFlushThreeTimes(int rows, TableName table, String family)
225    throws IOException {
226    final long start = getNumberOfMobFiles(table, family);
227    // Load and flush data 3 times
228    loadData(table, rows);
229    loadData(table, rows);
230    loadData(table, rows);
231    assertEquals("Should have 3 more mob files per region from flushing.", start + numRegions * 3,
232      getNumberOfMobFiles(table, family));
233  }
234
235  protected String description() {
236    return "regular mode";
237  }
238
239  protected void enableCompactions() throws IOException {
240    final List<String> serverList =
241      admin.getRegionServers().stream().map(sn -> sn.getServerName()).collect(Collectors.toList());
242    admin.compactionSwitch(true, serverList);
243  }
244
245  protected void disableCompactions() throws IOException {
246    final List<String> serverList =
247      admin.getRegionServers().stream().map(sn -> sn.getServerName()).collect(Collectors.toList());
248    admin.compactionSwitch(false, serverList);
249  }
250
251  /**
252   * compact the given table and return once it is done. should presume compactions are disabled
253   * when called. should ensure compactions are disabled before returning.
254   */
255  protected void mobCompact(TableDescriptor tableDescriptor,
256    ColumnFamilyDescriptor familyDescriptor) throws IOException, InterruptedException {
257    LOG.debug("Major compact MOB table " + tableDescriptor.getTableName());
258    enableCompactions();
259    mobCompactImpl(tableDescriptor, familyDescriptor);
260    waitUntilCompactionIsComplete(tableDescriptor.getTableName());
261    disableCompactions();
262  }
263
264  /**
265   * Call the API for compaction specific to the test set. should not wait for compactions to
266   * finish. may assume compactions are enabled when called.
267   */
268  protected void mobCompactImpl(TableDescriptor tableDescriptor,
269    ColumnFamilyDescriptor familyDescriptor) throws IOException, InterruptedException {
270    admin.majorCompact(tableDescriptor.getTableName(), familyDescriptor.getName());
271  }
272
273  protected void waitUntilCompactionIsComplete(TableName table)
274    throws IOException, InterruptedException {
275    CompactionState state = admin.getCompactionState(table);
276    while (state != CompactionState.NONE) {
277      LOG.debug("Waiting for compaction on {} to complete. current state {}", table, state);
278      Thread.sleep(100);
279      state = admin.getCompactionState(table);
280    }
281    LOG.debug("done waiting for compaction on {}", table);
282  }
283
284  protected void cleanupAndVerifyCounts(TableName table, String family, int rows)
285    throws InterruptedException, IOException {
286    // We have guarantee, that compacted file discharger will run during this pause
287    // because it has interval less than this wait time
288    LOG.info("Waiting for {}ms", minAgeToArchive + 1000);
289
290    Thread.sleep(minAgeToArchive + 1000);
291    LOG.info("Cleaning up MOB files");
292
293    // run cleaner chore on each RS
294    for (ServerName sn : admin.getRegionServers()) {
295      HTU.getMiniHBaseCluster().getRegionServer(sn).getRSMobFileCleanerChore().chore();
296    }
297
298    assertEquals("After cleaning, we should have 1 MOB file per region based on size.", numRegions,
299      getNumberOfMobFiles(table, family));
300
301    LOG.debug("checking count of rows");
302    long scanned = scanTable(table);
303    assertEquals("Got the wrong number of rows in table " + table + " cf " + family, rows, scanned);
304
305  }
306
307  protected long getNumberOfMobFiles(TableName tableName, String family) throws IOException {
308    FileSystem fs = FileSystem.get(conf);
309    Path dir = MobUtils.getMobFamilyPath(conf, tableName, family);
310    FileStatus[] stat = fs.listStatus(dir);
311    for (FileStatus st : stat) {
312      LOG.debug("MOB Directory content: {}", st.getPath());
313    }
314    LOG.debug("MOB Directory content total files: {}", stat.length);
315
316    return stat.length;
317  }
318
319  protected long scanTable(TableName tableName) {
320    try (final Table table = HTU.getConnection().getTable(tableName);
321      final ResultScanner scanner = table.getScanner(fam)) {
322      Result result;
323      long counter = 0;
324      while ((result = scanner.next()) != null) {
325        assertTrue(Arrays.equals(result.getValue(fam, qualifier), mobVal));
326        counter++;
327      }
328      return counter;
329    } catch (Exception e) {
330      LOG.error("MOB file compaction test FAILED", e);
331      if (HTU != null) {
332        fail(e.getMessage());
333      } else {
334        System.exit(-1);
335      }
336    }
337    return 0;
338  }
339}