001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.cleaner;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertTrue;
023
024import java.io.IOException;
025import java.util.List;
026import java.util.Random;
027
028import org.apache.hadoop.conf.Configuration;
029import org.apache.hadoop.fs.FSDataOutputStream;
030import org.apache.hadoop.fs.FileStatus;
031import org.apache.hadoop.fs.FileSystem;
032import org.apache.hadoop.fs.Path;
033import org.apache.hadoop.hbase.HBaseClassTestRule;
034import org.apache.hadoop.hbase.HBaseTestingUtil;
035import org.apache.hadoop.hbase.HConstants;
036import org.apache.hadoop.hbase.Server;
037import org.apache.hadoop.hbase.TableName;
038import org.apache.hadoop.hbase.client.RegionInfoBuilder;
039import org.apache.hadoop.hbase.mob.ManualMobMaintHFileCleaner;
040import org.apache.hadoop.hbase.mob.MobUtils;
041import org.apache.hadoop.hbase.testclassification.MasterTests;
042import org.apache.hadoop.hbase.testclassification.MediumTests;
043import org.apache.hadoop.hbase.util.EnvironmentEdge;
044import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
045import org.apache.hadoop.hbase.util.HFileArchiveUtil;
046import org.apache.hadoop.hbase.util.MockServer;
047import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
048import org.junit.AfterClass;
049import org.junit.Assert;
050import org.junit.BeforeClass;
051import org.junit.ClassRule;
052import org.junit.Test;
053import org.junit.experimental.categories.Category;
054import org.slf4j.Logger;
055import org.slf4j.LoggerFactory;
056
057@Category({MasterTests.class, MediumTests.class})
058public class TestHFileCleaner {
059
060  @ClassRule
061  public static final HBaseClassTestRule CLASS_RULE =
062      HBaseClassTestRule.forClass(TestHFileCleaner.class);
063
064  private static final Logger LOG = LoggerFactory.getLogger(TestHFileCleaner.class);
065
066  private final static HBaseTestingUtil UTIL = new HBaseTestingUtil();
067
068  private static DirScanPool POOL;
069
070  @BeforeClass
071  public static void setupCluster() throws Exception {
072    // have to use a minidfs cluster because the localfs doesn't modify file times correctly
073    UTIL.startMiniDFSCluster(1);
074    POOL = DirScanPool.getHFileCleanerScanPool(UTIL.getConfiguration());
075  }
076
077  @AfterClass
078  public static void shutdownCluster() throws IOException {
079    UTIL.shutdownMiniDFSCluster();
080    POOL.shutdownNow();
081  }
082
083  @Test
084  public void testTTLCleaner() throws IOException {
085    FileSystem fs = UTIL.getDFSCluster().getFileSystem();
086    Path root = UTIL.getDataTestDirOnTestFS();
087    Path file = new Path(root, "file");
088    fs.createNewFile(file);
089    long createTime = EnvironmentEdgeManager.currentTime();
090    assertTrue("Test file not created!", fs.exists(file));
091    TimeToLiveHFileCleaner cleaner = new TimeToLiveHFileCleaner();
092    // update the time info for the file, so the cleaner removes it
093    fs.setTimes(file, createTime - 100, -1);
094    Configuration conf = UTIL.getConfiguration();
095    conf.setLong(TimeToLiveHFileCleaner.TTL_CONF_KEY, 100);
096    cleaner.setConf(conf);
097    assertTrue("File not set deletable - check mod time:" + getFileStats(file, fs)
098        + " with create time:" + createTime, cleaner.isFileDeletable(fs.getFileStatus(file)));
099  }
100
101  @Test
102  public void testManualMobCleanerStopsMobRemoval() throws IOException {
103    FileSystem fs = UTIL.getDFSCluster().getFileSystem();
104    Path root = UTIL.getDataTestDirOnTestFS();
105    TableName table = TableName.valueOf("testManualMobCleanerStopsMobRemoval");
106    Path mob = HFileArchiveUtil.getRegionArchiveDir(root, table,
107        MobUtils.getMobRegionInfo(table).getEncodedName());
108    Path family= new Path(mob, "family");
109
110    Path file = new Path(family, "someHFileThatWouldBeAUUID");
111    fs.createNewFile(file);
112    assertTrue("Test file not created!", fs.exists(file));
113
114    ManualMobMaintHFileCleaner cleaner = new ManualMobMaintHFileCleaner();
115
116    assertFalse("Mob File shouldn't have been deletable. check path. '"+file+"'",
117        cleaner.isFileDeletable(fs.getFileStatus(file)));
118  }
119
120  @Test
121  public void testManualMobCleanerLetsNonMobGo() throws IOException {
122    FileSystem fs = UTIL.getDFSCluster().getFileSystem();
123    Path root = UTIL.getDataTestDirOnTestFS();
124    TableName table = TableName.valueOf("testManualMobCleanerLetsNonMobGo");
125    Path nonmob = HFileArchiveUtil.getRegionArchiveDir(root, table,
126      RegionInfoBuilder.newBuilder(table).build().getEncodedName());
127    Path family= new Path(nonmob, "family");
128
129    Path file = new Path(family, "someHFileThatWouldBeAUUID");
130    fs.createNewFile(file);
131    assertTrue("Test file not created!", fs.exists(file));
132
133    ManualMobMaintHFileCleaner cleaner = new ManualMobMaintHFileCleaner();
134
135    assertTrue("Non-Mob File should have been deletable. check path. '"+file+"'",
136        cleaner.isFileDeletable(fs.getFileStatus(file)));
137  }
138
139  /**
140   * @param file to check
141   * @return loggable information about the file
142   */
143  private String getFileStats(Path file, FileSystem fs) throws IOException {
144    FileStatus status = fs.getFileStatus(file);
145    return "File" + file + ", mtime:" + status.getModificationTime() + ", atime:"
146        + status.getAccessTime();
147  }
148
149  @Test
150  public void testHFileCleaning() throws Exception {
151    final EnvironmentEdge originalEdge = EnvironmentEdgeManager.getDelegate();
152    String prefix = "someHFileThatWouldBeAUUID";
153    Configuration conf = UTIL.getConfiguration();
154    // set TTL
155    long ttl = 2000;
156    conf.set(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS,
157        "org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner," +
158        "org.apache.hadoop.hbase.mob.ManualMobMaintHFileCleaner");
159    conf.setLong(TimeToLiveHFileCleaner.TTL_CONF_KEY, ttl);
160    Server server = new DummyServer();
161    Path archivedHfileDir =
162      new Path(UTIL.getDataTestDirOnTestFS(), HConstants.HFILE_ARCHIVE_DIRECTORY);
163    FileSystem fs = FileSystem.get(conf);
164    HFileCleaner cleaner = new HFileCleaner(1000, server, conf, fs, archivedHfileDir, POOL);
165
166    // Create 2 invalid files, 1 "recent" file, 1 very new file and 30 old files
167    final long createTime = EnvironmentEdgeManager.currentTime();
168    fs.delete(archivedHfileDir, true);
169    fs.mkdirs(archivedHfileDir);
170    // Case 1: 1 invalid file, which should be deleted directly
171    fs.createNewFile(new Path(archivedHfileDir, "dfd-dfd"));
172    // Case 2: 1 "recent" file, not even deletable for the first log cleaner
173    // (TimeToLiveLogCleaner), so we are not going down the chain
174    LOG.debug("Now is: " + createTime);
175    for (int i = 1; i < 32; i++) {
176      // Case 3: old files which would be deletable for the first log cleaner
177      // (TimeToLiveHFileCleaner),
178      Path fileName = new Path(archivedHfileDir, (prefix + "." + (createTime + i)));
179      fs.createNewFile(fileName);
180      // set the creation time past ttl to ensure that it gets removed
181      fs.setTimes(fileName, createTime - ttl - 1, -1);
182      LOG.debug("Creating " + getFileStats(fileName, fs));
183    }
184
185    // Case 2: 1 newer file, not even deletable for the first log cleaner
186    // (TimeToLiveLogCleaner), so we are not going down the chain
187    Path saved = new Path(archivedHfileDir, prefix + ".00000000000");
188    fs.createNewFile(saved);
189    // set creation time within the ttl
190    fs.setTimes(saved, createTime - ttl / 2, -1);
191    LOG.debug("Creating " + getFileStats(saved, fs));
192    for (FileStatus stat : fs.listStatus(archivedHfileDir)) {
193      LOG.debug(stat.getPath().toString());
194    }
195
196    assertEquals(33, fs.listStatus(archivedHfileDir).length);
197
198    // set a custom edge manager to handle time checking
199    EnvironmentEdge setTime = new EnvironmentEdge() {
200      @Override
201      public long currentTime() {
202        return createTime;
203      }
204    };
205    EnvironmentEdgeManager.injectEdge(setTime);
206
207    // run the chore
208    cleaner.chore();
209
210    // ensure we only end up with the saved file
211    assertEquals(1, fs.listStatus(archivedHfileDir).length);
212
213    for (FileStatus file : fs.listStatus(archivedHfileDir)) {
214      LOG.debug("Kept hfiles: " + file.getPath().getName());
215    }
216
217    // reset the edge back to the original edge
218    EnvironmentEdgeManager.injectEdge(originalEdge);
219  }
220
221  @Test
222  public void testRemovesEmptyDirectories() throws Exception {
223    Configuration conf = UTIL.getConfiguration();
224    // no cleaner policies = delete all files
225    conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, "");
226    Server server = new DummyServer();
227    Path archivedHfileDir =
228      new Path(UTIL.getDataTestDirOnTestFS(), HConstants.HFILE_ARCHIVE_DIRECTORY);
229
230    // setup the cleaner
231    FileSystem fs = UTIL.getDFSCluster().getFileSystem();
232    HFileCleaner cleaner = new HFileCleaner(1000, server, conf, fs, archivedHfileDir, POOL);
233
234    // make all the directories for archiving files
235    Path table = new Path(archivedHfileDir, "table");
236    Path region = new Path(table, "regionsomthing");
237    Path family = new Path(region, "fam");
238    Path file = new Path(family, "file12345");
239    fs.mkdirs(family);
240    if (!fs.exists(family)) throw new RuntimeException("Couldn't create test family:" + family);
241    fs.create(file).close();
242    if (!fs.exists(file)) throw new RuntimeException("Test file didn't get created:" + file);
243
244    // run the chore to cleanup the files (and the directories above it)
245    cleaner.chore();
246
247    // make sure all the parent directories get removed
248    assertFalse("family directory not removed for empty directory", fs.exists(family));
249    assertFalse("region directory not removed for empty directory", fs.exists(region));
250    assertFalse("table directory not removed for empty directory", fs.exists(table));
251    assertTrue("archive directory", fs.exists(archivedHfileDir));
252  }
253
254  static class DummyServer extends MockServer {
255    @Override
256    public Configuration getConfiguration() {
257      return UTIL.getConfiguration();
258    }
259
260    @Override
261    public ZKWatcher getZooKeeper() {
262      try {
263        return new ZKWatcher(getConfiguration(), "dummy server", this);
264      } catch (IOException e) {
265        e.printStackTrace();
266      }
267      return null;
268    }
269  }
270
271  @Test
272  public void testThreadCleanup() throws Exception {
273    Configuration conf = UTIL.getConfiguration();
274    conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, "");
275    Server server = new DummyServer();
276    Path archivedHfileDir =
277        new Path(UTIL.getDataTestDirOnTestFS(), HConstants.HFILE_ARCHIVE_DIRECTORY);
278
279    // setup the cleaner
280    FileSystem fs = UTIL.getDFSCluster().getFileSystem();
281    HFileCleaner cleaner = new HFileCleaner(1000, server, conf, fs, archivedHfileDir, POOL);
282    // clean up archive directory
283    fs.delete(archivedHfileDir, true);
284    fs.mkdirs(archivedHfileDir);
285    // create some file to delete
286    fs.createNewFile(new Path(archivedHfileDir, "dfd-dfd"));
287    // launch the chore
288    cleaner.chore();
289    // call cleanup
290    cleaner.cleanup();
291    // wait awhile for thread to die
292    Thread.sleep(100);
293    for (Thread thread : cleaner.getCleanerThreads()) {
294      Assert.assertFalse(thread.isAlive());
295    }
296  }
297
298  @Test
299  public void testLargeSmallIsolation() throws Exception {
300    Configuration conf = UTIL.getConfiguration();
301    // no cleaner policies = delete all files
302    conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, "");
303    conf.setInt(HFileCleaner.HFILE_DELETE_THROTTLE_THRESHOLD, 512 * 1024);
304    Server server = new DummyServer();
305    Path archivedHfileDir =
306        new Path(UTIL.getDataTestDirOnTestFS(), HConstants.HFILE_ARCHIVE_DIRECTORY);
307
308    // setup the cleaner
309    FileSystem fs = UTIL.getDFSCluster().getFileSystem();
310    HFileCleaner cleaner = new HFileCleaner(1000, server, conf, fs, archivedHfileDir, POOL);
311    // clean up archive directory
312    fs.delete(archivedHfileDir, true);
313    fs.mkdirs(archivedHfileDir);
314    // necessary set up
315    final int LARGE_FILE_NUM = 5;
316    final int SMALL_FILE_NUM = 20;
317    createFilesForTesting(LARGE_FILE_NUM, SMALL_FILE_NUM, fs, archivedHfileDir);
318    // call cleanup
319    cleaner.chore();
320
321    Assert.assertEquals(LARGE_FILE_NUM, cleaner.getNumOfDeletedLargeFiles());
322    Assert.assertEquals(SMALL_FILE_NUM, cleaner.getNumOfDeletedSmallFiles());
323  }
324
325  @Test
326  public void testOnConfigurationChange() throws Exception {
327    // constants
328    final int ORIGINAL_THROTTLE_POINT = 512 * 1024;
329    final int ORIGINAL_QUEUE_INIT_SIZE = 512;
330    final int UPDATE_THROTTLE_POINT = 1024;// small enough to change large/small check
331    final int UPDATE_QUEUE_INIT_SIZE = 1024;
332    final int LARGE_FILE_NUM = 5;
333    final int SMALL_FILE_NUM = 20;
334    final int LARGE_THREAD_NUM = 2;
335    final int SMALL_THREAD_NUM = 4;
336    final long THREAD_TIMEOUT_MSEC = 30 * 1000L;
337    final long THREAD_CHECK_INTERVAL_MSEC = 500L;
338
339    Configuration conf = UTIL.getConfiguration();
340    // no cleaner policies = delete all files
341    conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, "");
342    conf.setInt(HFileCleaner.HFILE_DELETE_THROTTLE_THRESHOLD, ORIGINAL_THROTTLE_POINT);
343    conf.setInt(HFileCleaner.LARGE_HFILE_QUEUE_INIT_SIZE, ORIGINAL_QUEUE_INIT_SIZE);
344    conf.setInt(HFileCleaner.SMALL_HFILE_QUEUE_INIT_SIZE, ORIGINAL_QUEUE_INIT_SIZE);
345    Server server = new DummyServer();
346    Path archivedHfileDir =
347        new Path(UTIL.getDataTestDirOnTestFS(), HConstants.HFILE_ARCHIVE_DIRECTORY);
348
349    // setup the cleaner
350    FileSystem fs = UTIL.getDFSCluster().getFileSystem();
351    final HFileCleaner cleaner = new HFileCleaner(1000, server, conf, fs, archivedHfileDir, POOL);
352    Assert.assertEquals(ORIGINAL_THROTTLE_POINT, cleaner.getThrottlePoint());
353    Assert.assertEquals(ORIGINAL_QUEUE_INIT_SIZE, cleaner.getLargeQueueInitSize());
354    Assert.assertEquals(ORIGINAL_QUEUE_INIT_SIZE, cleaner.getSmallQueueInitSize());
355    Assert.assertEquals(HFileCleaner.DEFAULT_HFILE_DELETE_THREAD_TIMEOUT_MSEC,
356        cleaner.getCleanerThreadTimeoutMsec());
357    Assert.assertEquals(HFileCleaner.DEFAULT_HFILE_DELETE_THREAD_CHECK_INTERVAL_MSEC,
358        cleaner.getCleanerThreadCheckIntervalMsec());
359
360    // clean up archive directory and create files for testing
361    fs.delete(archivedHfileDir, true);
362    fs.mkdirs(archivedHfileDir);
363    createFilesForTesting(LARGE_FILE_NUM, SMALL_FILE_NUM, fs, archivedHfileDir);
364
365    // call cleaner, run as daemon to test the interrupt-at-middle case
366    Thread t = new Thread() {
367      @Override
368      public void run() {
369        cleaner.chore();
370      }
371    };
372    t.setDaemon(true);
373    t.start();
374    // wait until file clean started
375    while (cleaner.getNumOfDeletedSmallFiles() == 0) {
376      Thread.yield();
377    }
378
379    // trigger configuration change
380    Configuration newConf = new Configuration(conf);
381    newConf.setInt(HFileCleaner.HFILE_DELETE_THROTTLE_THRESHOLD, UPDATE_THROTTLE_POINT);
382    newConf.setInt(HFileCleaner.LARGE_HFILE_QUEUE_INIT_SIZE, UPDATE_QUEUE_INIT_SIZE);
383    newConf.setInt(HFileCleaner.SMALL_HFILE_QUEUE_INIT_SIZE, UPDATE_QUEUE_INIT_SIZE);
384    newConf.setInt(HFileCleaner.LARGE_HFILE_DELETE_THREAD_NUMBER, LARGE_THREAD_NUM);
385    newConf.setInt(HFileCleaner.SMALL_HFILE_DELETE_THREAD_NUMBER, SMALL_THREAD_NUM);
386    newConf.setLong(HFileCleaner.HFILE_DELETE_THREAD_TIMEOUT_MSEC, THREAD_TIMEOUT_MSEC);
387    newConf.setLong(HFileCleaner.HFILE_DELETE_THREAD_CHECK_INTERVAL_MSEC,
388        THREAD_CHECK_INTERVAL_MSEC);
389
390    LOG.debug("File deleted from large queue: " + cleaner.getNumOfDeletedLargeFiles()
391        + "; from small queue: " + cleaner.getNumOfDeletedSmallFiles());
392    cleaner.onConfigurationChange(newConf);
393
394    // check values after change
395    Assert.assertEquals(UPDATE_THROTTLE_POINT, cleaner.getThrottlePoint());
396    Assert.assertEquals(UPDATE_QUEUE_INIT_SIZE, cleaner.getLargeQueueInitSize());
397    Assert.assertEquals(UPDATE_QUEUE_INIT_SIZE, cleaner.getSmallQueueInitSize());
398    Assert.assertEquals(LARGE_THREAD_NUM + SMALL_THREAD_NUM, cleaner.getCleanerThreads().size());
399    Assert.assertEquals(THREAD_TIMEOUT_MSEC, cleaner.getCleanerThreadTimeoutMsec());
400    Assert.assertEquals(THREAD_CHECK_INTERVAL_MSEC, cleaner.getCleanerThreadCheckIntervalMsec());
401
402    // make sure no cost when onConfigurationChange called with no change
403    List<Thread> oldThreads = cleaner.getCleanerThreads();
404    cleaner.onConfigurationChange(newConf);
405    List<Thread> newThreads = cleaner.getCleanerThreads();
406    Assert.assertArrayEquals(oldThreads.toArray(), newThreads.toArray());
407
408    // wait until clean done and check
409    t.join();
410    LOG.debug("File deleted from large queue: " + cleaner.getNumOfDeletedLargeFiles()
411        + "; from small queue: " + cleaner.getNumOfDeletedSmallFiles());
412    Assert.assertTrue("Should delete more than " + LARGE_FILE_NUM
413        + " files from large queue but actually " + cleaner.getNumOfDeletedLargeFiles(),
414      cleaner.getNumOfDeletedLargeFiles() > LARGE_FILE_NUM);
415    Assert.assertTrue("Should delete less than " + SMALL_FILE_NUM
416        + " files from small queue but actually " + cleaner.getNumOfDeletedSmallFiles(),
417      cleaner.getNumOfDeletedSmallFiles() < SMALL_FILE_NUM);
418  }
419
420  private void createFilesForTesting(int largeFileNum, int smallFileNum, FileSystem fs,
421      Path archivedHfileDir) throws IOException {
422    final Random rand = new Random();
423    final byte[] large = new byte[1024 * 1024];
424    for (int i = 0; i < large.length; i++) {
425      large[i] = (byte) rand.nextInt(128);
426    }
427    final byte[] small = new byte[1024];
428    for (int i = 0; i < small.length; i++) {
429      small[i] = (byte) rand.nextInt(128);
430    }
431    // create large and small files
432    for (int i = 1; i <= largeFileNum; i++) {
433      FSDataOutputStream out = fs.create(new Path(archivedHfileDir, "large-file-" + i));
434      out.write(large);
435      out.close();
436    }
437    for (int i = 1; i <= smallFileNum; i++) {
438      FSDataOutputStream out = fs.create(new Path(archivedHfileDir, "small-file-" + i));
439      out.write(small);
440      out.close();
441    }
442  }
443}