001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.cleaner;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertTrue;
023
024import java.io.IOException;
025import java.util.List;
026import java.util.Random;
027import org.apache.hadoop.conf.Configuration;
028import org.apache.hadoop.fs.FSDataOutputStream;
029import org.apache.hadoop.fs.FileStatus;
030import org.apache.hadoop.fs.FileSystem;
031import org.apache.hadoop.fs.Path;
032import org.apache.hadoop.hbase.ChoreService;
033import org.apache.hadoop.hbase.CoordinatedStateManager;
034import org.apache.hadoop.hbase.HBaseClassTestRule;
035import org.apache.hadoop.hbase.HBaseTestingUtility;
036import org.apache.hadoop.hbase.HConstants;
037import org.apache.hadoop.hbase.HRegionInfo;
038import org.apache.hadoop.hbase.Server;
039import org.apache.hadoop.hbase.ServerName;
040import org.apache.hadoop.hbase.TableName;
041import org.apache.hadoop.hbase.client.ClusterConnection;
042import org.apache.hadoop.hbase.client.Connection;
043import org.apache.hadoop.hbase.mob.ManualMobMaintHFileCleaner;
044import org.apache.hadoop.hbase.mob.MobUtils;
045import org.apache.hadoop.hbase.testclassification.MasterTests;
046import org.apache.hadoop.hbase.testclassification.MediumTests;
047import org.apache.hadoop.hbase.util.EnvironmentEdge;
048import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
049import org.apache.hadoop.hbase.util.HFileArchiveUtil;
050import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
051import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
052import org.junit.AfterClass;
053import org.junit.Assert;
054import org.junit.BeforeClass;
055import org.junit.ClassRule;
056import org.junit.Test;
057import org.junit.experimental.categories.Category;
058import org.slf4j.Logger;
059import org.slf4j.LoggerFactory;
060
061@Category({MasterTests.class, MediumTests.class})
062public class TestHFileCleaner {
063
064  @ClassRule
065  public static final HBaseClassTestRule CLASS_RULE =
066      HBaseClassTestRule.forClass(TestHFileCleaner.class);
067
068  private static final Logger LOG = LoggerFactory.getLogger(TestHFileCleaner.class);
069
070  private final static HBaseTestingUtility UTIL = new HBaseTestingUtility();
071
072  private static DirScanPool POOL;
073
074  @BeforeClass
075  public static void setupCluster() throws Exception {
076    // have to use a minidfs cluster because the localfs doesn't modify file times correctly
077    UTIL.startMiniDFSCluster(1);
078    POOL = new DirScanPool(UTIL.getConfiguration());
079  }
080
081  @AfterClass
082  public static void shutdownCluster() throws IOException {
083    UTIL.shutdownMiniDFSCluster();
084    POOL.shutdownNow();
085  }
086
087  @Test
088  public void testTTLCleaner() throws IOException, InterruptedException {
089    FileSystem fs = UTIL.getDFSCluster().getFileSystem();
090    Path root = UTIL.getDataTestDirOnTestFS();
091    Path file = new Path(root, "file");
092    fs.createNewFile(file);
093    long createTime = System.currentTimeMillis();
094    assertTrue("Test file not created!", fs.exists(file));
095    TimeToLiveHFileCleaner cleaner = new TimeToLiveHFileCleaner();
096    // update the time info for the file, so the cleaner removes it
097    fs.setTimes(file, createTime - 100, -1);
098    Configuration conf = UTIL.getConfiguration();
099    conf.setLong(TimeToLiveHFileCleaner.TTL_CONF_KEY, 100);
100    cleaner.setConf(conf);
101    assertTrue("File not set deletable - check mod time:" + getFileStats(file, fs)
102        + " with create time:" + createTime, cleaner.isFileDeletable(fs.getFileStatus(file)));
103  }
104
105  @Test
106  public void testManualMobCleanerStopsMobRemoval() throws IOException {
107    FileSystem fs = UTIL.getDFSCluster().getFileSystem();
108    Path root = UTIL.getDataTestDirOnTestFS();
109    TableName table = TableName.valueOf("testManualMobCleanerStopsMobRemoval");
110    Path mob = HFileArchiveUtil.getRegionArchiveDir(root, table,
111        MobUtils.getMobRegionInfo(table).getEncodedName());
112    Path family= new Path(mob, "family");
113
114    Path file = new Path(family, "someHFileThatWouldBeAUUID");
115    fs.createNewFile(file);
116    assertTrue("Test file not created!", fs.exists(file));
117
118    ManualMobMaintHFileCleaner cleaner = new ManualMobMaintHFileCleaner();
119
120    assertFalse("Mob File shouldn't have been deletable. check path. '"+file+"'",
121        cleaner.isFileDeletable(fs.getFileStatus(file)));
122  }
123
124  @Test
125  public void testManualMobCleanerLetsNonMobGo() throws IOException {
126    FileSystem fs = UTIL.getDFSCluster().getFileSystem();
127    Path root = UTIL.getDataTestDirOnTestFS();
128    TableName table = TableName.valueOf("testManualMobCleanerLetsNonMobGo");
129    Path nonmob = HFileArchiveUtil.getRegionArchiveDir(root, table,
130        new HRegionInfo(table).getEncodedName());
131    Path family= new Path(nonmob, "family");
132
133    Path file = new Path(family, "someHFileThatWouldBeAUUID");
134    fs.createNewFile(file);
135    assertTrue("Test file not created!", fs.exists(file));
136
137    ManualMobMaintHFileCleaner cleaner = new ManualMobMaintHFileCleaner();
138
139    assertTrue("Non-Mob File should have been deletable. check path. '"+file+"'",
140        cleaner.isFileDeletable(fs.getFileStatus(file)));
141  }
142
143  /**
144   * @param file to check
145   * @return loggable information about the file
146   */
147  private String getFileStats(Path file, FileSystem fs) throws IOException {
148    FileStatus status = fs.getFileStatus(file);
149    return "File" + file + ", mtime:" + status.getModificationTime() + ", atime:"
150        + status.getAccessTime();
151  }
152
153  @Test
154  public void testHFileCleaning() throws Exception {
155    final EnvironmentEdge originalEdge = EnvironmentEdgeManager.getDelegate();
156    String prefix = "someHFileThatWouldBeAUUID";
157    Configuration conf = UTIL.getConfiguration();
158    // set TTL
159    long ttl = 2000;
160    conf.set(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS,
161        "org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner," +
162        "org.apache.hadoop.hbase.mob.ManualMobMaintHFileCleaner");
163    conf.setLong(TimeToLiveHFileCleaner.TTL_CONF_KEY, ttl);
164    Server server = new DummyServer();
165    Path archivedHfileDir =
166      new Path(UTIL.getDataTestDirOnTestFS(), HConstants.HFILE_ARCHIVE_DIRECTORY);
167    FileSystem fs = FileSystem.get(conf);
168    HFileCleaner cleaner = new HFileCleaner(1000, server, conf, fs, archivedHfileDir, POOL);
169
170    // Create 2 invalid files, 1 "recent" file, 1 very new file and 30 old files
171    final long createTime = System.currentTimeMillis();
172    fs.delete(archivedHfileDir, true);
173    fs.mkdirs(archivedHfileDir);
174    // Case 1: 1 invalid file, which should be deleted directly
175    fs.createNewFile(new Path(archivedHfileDir, "dfd-dfd"));
176    // Case 2: 1 "recent" file, not even deletable for the first log cleaner
177    // (TimeToLiveLogCleaner), so we are not going down the chain
178    LOG.debug("Now is: " + createTime);
179    for (int i = 1; i < 32; i++) {
180      // Case 3: old files which would be deletable for the first log cleaner
181      // (TimeToLiveHFileCleaner),
182      Path fileName = new Path(archivedHfileDir, (prefix + "." + (createTime + i)));
183      fs.createNewFile(fileName);
184      // set the creation time past ttl to ensure that it gets removed
185      fs.setTimes(fileName, createTime - ttl - 1, -1);
186      LOG.debug("Creating " + getFileStats(fileName, fs));
187    }
188
189    // Case 2: 1 newer file, not even deletable for the first log cleaner
190    // (TimeToLiveLogCleaner), so we are not going down the chain
191    Path saved = new Path(archivedHfileDir, prefix + ".00000000000");
192    fs.createNewFile(saved);
193    // set creation time within the ttl
194    fs.setTimes(saved, createTime - ttl / 2, -1);
195    LOG.debug("Creating " + getFileStats(saved, fs));
196    for (FileStatus stat : fs.listStatus(archivedHfileDir)) {
197      LOG.debug(stat.getPath().toString());
198    }
199
200    assertEquals(33, fs.listStatus(archivedHfileDir).length);
201
202    // set a custom edge manager to handle time checking
203    EnvironmentEdge setTime = new EnvironmentEdge() {
204      @Override
205      public long currentTime() {
206        return createTime;
207      }
208    };
209    EnvironmentEdgeManager.injectEdge(setTime);
210
211    // run the chore
212    cleaner.chore();
213
214    // ensure we only end up with the saved file
215    assertEquals(1, fs.listStatus(archivedHfileDir).length);
216
217    for (FileStatus file : fs.listStatus(archivedHfileDir)) {
218      LOG.debug("Kept hfiles: " + file.getPath().getName());
219    }
220
221    // reset the edge back to the original edge
222    EnvironmentEdgeManager.injectEdge(originalEdge);
223  }
224
225  @Test
226  public void testRemovesEmptyDirectories() throws Exception {
227    Configuration conf = UTIL.getConfiguration();
228    // no cleaner policies = delete all files
229    conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, "");
230    Server server = new DummyServer();
231    Path archivedHfileDir =
232      new Path(UTIL.getDataTestDirOnTestFS(), HConstants.HFILE_ARCHIVE_DIRECTORY);
233
234    // setup the cleaner
235    FileSystem fs = UTIL.getDFSCluster().getFileSystem();
236    HFileCleaner cleaner = new HFileCleaner(1000, server, conf, fs, archivedHfileDir, POOL);
237
238    // make all the directories for archiving files
239    Path table = new Path(archivedHfileDir, "table");
240    Path region = new Path(table, "regionsomthing");
241    Path family = new Path(region, "fam");
242    Path file = new Path(family, "file12345");
243    fs.mkdirs(family);
244    if (!fs.exists(family)) throw new RuntimeException("Couldn't create test family:" + family);
245    fs.create(file).close();
246    if (!fs.exists(file)) throw new RuntimeException("Test file didn't get created:" + file);
247
248    // run the chore to cleanup the files (and the directories above it)
249    cleaner.chore();
250
251    // make sure all the parent directories get removed
252    assertFalse("family directory not removed for empty directory", fs.exists(family));
253    assertFalse("region directory not removed for empty directory", fs.exists(region));
254    assertFalse("table directory not removed for empty directory", fs.exists(table));
255    assertTrue("archive directory", fs.exists(archivedHfileDir));
256  }
257
258  static class DummyServer implements Server {
259    @Override
260    public Configuration getConfiguration() {
261      return UTIL.getConfiguration();
262    }
263
264    @Override
265    public ZKWatcher getZooKeeper() {
266      try {
267        return new ZKWatcher(getConfiguration(), "dummy server", this);
268      } catch (IOException e) {
269        e.printStackTrace();
270      }
271      return null;
272    }
273
274    @Override
275    public CoordinatedStateManager getCoordinatedStateManager() {
276      return null;
277    }
278
279    @Override
280    public ClusterConnection getConnection() {
281      return null;
282    }
283
284    @Override
285    public MetaTableLocator getMetaTableLocator() {
286      return null;
287    }
288
289    @Override
290    public ServerName getServerName() {
291      return ServerName.valueOf("regionserver,60020,000000");
292    }
293
294    @Override
295    public void abort(String why, Throwable e) {
296    }
297
298    @Override
299    public boolean isAborted() {
300      return false;
301    }
302
303    @Override
304    public void stop(String why) {
305    }
306
307    @Override
308    public boolean isStopped() {
309      return false;
310    }
311
312    @Override
313    public ChoreService getChoreService() {
314      return null;
315    }
316
317    @Override
318    public ClusterConnection getClusterConnection() {
319      // TODO Auto-generated method stub
320      return null;
321    }
322
323    @Override
324    public FileSystem getFileSystem() {
325      return null;
326    }
327
328    @Override
329    public boolean isStopping() {
330      return false;
331    }
332
333    @Override
334    public Connection createConnection(Configuration conf) throws IOException {
335      return null;
336    }
337  }
338
339  @Test
340  public void testThreadCleanup() throws Exception {
341    Configuration conf = UTIL.getConfiguration();
342    conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, "");
343    Server server = new DummyServer();
344    Path archivedHfileDir =
345        new Path(UTIL.getDataTestDirOnTestFS(), HConstants.HFILE_ARCHIVE_DIRECTORY);
346
347    // setup the cleaner
348    FileSystem fs = UTIL.getDFSCluster().getFileSystem();
349    HFileCleaner cleaner = new HFileCleaner(1000, server, conf, fs, archivedHfileDir, POOL);
350    // clean up archive directory
351    fs.delete(archivedHfileDir, true);
352    fs.mkdirs(archivedHfileDir);
353    // create some file to delete
354    fs.createNewFile(new Path(archivedHfileDir, "dfd-dfd"));
355    // launch the chore
356    cleaner.chore();
357    // call cleanup
358    cleaner.cleanup();
359    // wait awhile for thread to die
360    Thread.sleep(100);
361    for (Thread thread : cleaner.getCleanerThreads()) {
362      Assert.assertFalse(thread.isAlive());
363    }
364  }
365
366  @Test
367  public void testLargeSmallIsolation() throws Exception {
368    Configuration conf = UTIL.getConfiguration();
369    // no cleaner policies = delete all files
370    conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, "");
371    conf.setInt(HFileCleaner.HFILE_DELETE_THROTTLE_THRESHOLD, 512 * 1024);
372    Server server = new DummyServer();
373    Path archivedHfileDir =
374        new Path(UTIL.getDataTestDirOnTestFS(), HConstants.HFILE_ARCHIVE_DIRECTORY);
375
376    // setup the cleaner
377    FileSystem fs = UTIL.getDFSCluster().getFileSystem();
378    HFileCleaner cleaner = new HFileCleaner(1000, server, conf, fs, archivedHfileDir, POOL);
379    // clean up archive directory
380    fs.delete(archivedHfileDir, true);
381    fs.mkdirs(archivedHfileDir);
382    // necessary set up
383    final int LARGE_FILE_NUM = 5;
384    final int SMALL_FILE_NUM = 20;
385    createFilesForTesting(LARGE_FILE_NUM, SMALL_FILE_NUM, fs, archivedHfileDir);
386    // call cleanup
387    cleaner.chore();
388
389    Assert.assertEquals(LARGE_FILE_NUM, cleaner.getNumOfDeletedLargeFiles());
390    Assert.assertEquals(SMALL_FILE_NUM, cleaner.getNumOfDeletedSmallFiles());
391  }
392
393  @Test
394  public void testOnConfigurationChange() throws Exception {
395    // constants
396    final int ORIGINAL_THROTTLE_POINT = 512 * 1024;
397    final int ORIGINAL_QUEUE_INIT_SIZE = 512;
398    final int UPDATE_THROTTLE_POINT = 1024;// small enough to change large/small check
399    final int UPDATE_QUEUE_INIT_SIZE = 1024;
400    final int LARGE_FILE_NUM = 5;
401    final int SMALL_FILE_NUM = 20;
402    final int LARGE_THREAD_NUM = 2;
403    final int SMALL_THREAD_NUM = 4;
404    final long THREAD_TIMEOUT_MSEC = 30 * 1000L;
405    final long THREAD_CHECK_INTERVAL_MSEC = 500L;
406
407    Configuration conf = UTIL.getConfiguration();
408    // no cleaner policies = delete all files
409    conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, "");
410    conf.setInt(HFileCleaner.HFILE_DELETE_THROTTLE_THRESHOLD, ORIGINAL_THROTTLE_POINT);
411    conf.setInt(HFileCleaner.LARGE_HFILE_QUEUE_INIT_SIZE, ORIGINAL_QUEUE_INIT_SIZE);
412    conf.setInt(HFileCleaner.SMALL_HFILE_QUEUE_INIT_SIZE, ORIGINAL_QUEUE_INIT_SIZE);
413    Server server = new DummyServer();
414    Path archivedHfileDir =
415        new Path(UTIL.getDataTestDirOnTestFS(), HConstants.HFILE_ARCHIVE_DIRECTORY);
416
417    // setup the cleaner
418    FileSystem fs = UTIL.getDFSCluster().getFileSystem();
419    final HFileCleaner cleaner = new HFileCleaner(1000, server, conf, fs, archivedHfileDir, POOL);
420    Assert.assertEquals(ORIGINAL_THROTTLE_POINT, cleaner.getThrottlePoint());
421    Assert.assertEquals(ORIGINAL_QUEUE_INIT_SIZE, cleaner.getLargeQueueInitSize());
422    Assert.assertEquals(ORIGINAL_QUEUE_INIT_SIZE, cleaner.getSmallQueueInitSize());
423    Assert.assertEquals(HFileCleaner.DEFAULT_HFILE_DELETE_THREAD_TIMEOUT_MSEC,
424        cleaner.getCleanerThreadTimeoutMsec());
425    Assert.assertEquals(HFileCleaner.DEFAULT_HFILE_DELETE_THREAD_CHECK_INTERVAL_MSEC,
426        cleaner.getCleanerThreadCheckIntervalMsec());
427
428    // clean up archive directory and create files for testing
429    fs.delete(archivedHfileDir, true);
430    fs.mkdirs(archivedHfileDir);
431    createFilesForTesting(LARGE_FILE_NUM, SMALL_FILE_NUM, fs, archivedHfileDir);
432
433    // call cleaner, run as daemon to test the interrupt-at-middle case
434    Thread t = new Thread() {
435      @Override
436      public void run() {
437        cleaner.chore();
438      }
439    };
440    t.setDaemon(true);
441    t.start();
442    // wait until file clean started
443    while (cleaner.getNumOfDeletedSmallFiles() == 0) {
444      Thread.yield();
445    }
446
447    // trigger configuration change
448    Configuration newConf = new Configuration(conf);
449    newConf.setInt(HFileCleaner.HFILE_DELETE_THROTTLE_THRESHOLD, UPDATE_THROTTLE_POINT);
450    newConf.setInt(HFileCleaner.LARGE_HFILE_QUEUE_INIT_SIZE, UPDATE_QUEUE_INIT_SIZE);
451    newConf.setInt(HFileCleaner.SMALL_HFILE_QUEUE_INIT_SIZE, UPDATE_QUEUE_INIT_SIZE);
452    newConf.setInt(HFileCleaner.LARGE_HFILE_DELETE_THREAD_NUMBER, LARGE_THREAD_NUM);
453    newConf.setInt(HFileCleaner.SMALL_HFILE_DELETE_THREAD_NUMBER, SMALL_THREAD_NUM);
454    newConf.setLong(HFileCleaner.HFILE_DELETE_THREAD_TIMEOUT_MSEC, THREAD_TIMEOUT_MSEC);
455    newConf.setLong(HFileCleaner.HFILE_DELETE_THREAD_CHECK_INTERVAL_MSEC,
456        THREAD_CHECK_INTERVAL_MSEC);
457
458    LOG.debug("File deleted from large queue: " + cleaner.getNumOfDeletedLargeFiles()
459        + "; from small queue: " + cleaner.getNumOfDeletedSmallFiles());
460    cleaner.onConfigurationChange(newConf);
461
462    // check values after change
463    Assert.assertEquals(UPDATE_THROTTLE_POINT, cleaner.getThrottlePoint());
464    Assert.assertEquals(UPDATE_QUEUE_INIT_SIZE, cleaner.getLargeQueueInitSize());
465    Assert.assertEquals(UPDATE_QUEUE_INIT_SIZE, cleaner.getSmallQueueInitSize());
466    Assert.assertEquals(LARGE_THREAD_NUM + SMALL_THREAD_NUM, cleaner.getCleanerThreads().size());
467    Assert.assertEquals(THREAD_TIMEOUT_MSEC, cleaner.getCleanerThreadTimeoutMsec());
468    Assert.assertEquals(THREAD_CHECK_INTERVAL_MSEC, cleaner.getCleanerThreadCheckIntervalMsec());
469
470    // make sure no cost when onConfigurationChange called with no change
471    List<Thread> oldThreads = cleaner.getCleanerThreads();
472    cleaner.onConfigurationChange(newConf);
473    List<Thread> newThreads = cleaner.getCleanerThreads();
474    Assert.assertArrayEquals(oldThreads.toArray(), newThreads.toArray());
475
476    // wait until clean done and check
477    t.join();
478    LOG.debug("File deleted from large queue: " + cleaner.getNumOfDeletedLargeFiles()
479        + "; from small queue: " + cleaner.getNumOfDeletedSmallFiles());
480    Assert.assertTrue("Should delete more than " + LARGE_FILE_NUM
481        + " files from large queue but actually " + cleaner.getNumOfDeletedLargeFiles(),
482      cleaner.getNumOfDeletedLargeFiles() > LARGE_FILE_NUM);
483    Assert.assertTrue("Should delete less than " + SMALL_FILE_NUM
484        + " files from small queue but actually " + cleaner.getNumOfDeletedSmallFiles(),
485      cleaner.getNumOfDeletedSmallFiles() < SMALL_FILE_NUM);
486  }
487
488  private void createFilesForTesting(int largeFileNum, int smallFileNum, FileSystem fs,
489      Path archivedHfileDir) throws IOException {
490    final Random rand = new Random();
491    final byte[] large = new byte[1024 * 1024];
492    for (int i = 0; i < large.length; i++) {
493      large[i] = (byte) rand.nextInt(128);
494    }
495    final byte[] small = new byte[1024];
496    for (int i = 0; i < small.length; i++) {
497      small[i] = (byte) rand.nextInt(128);
498    }
499    // create large and small files
500    for (int i = 1; i <= largeFileNum; i++) {
501      FSDataOutputStream out = fs.create(new Path(archivedHfileDir, "large-file-" + i));
502      out.write(large);
503      out.close();
504    }
505    for (int i = 1; i <= smallFileNum; i++) {
506      FSDataOutputStream out = fs.create(new Path(archivedHfileDir, "small-file-" + i));
507      out.write(small);
508      out.close();
509    }
510  }
511}