001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.cleaner;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertTrue;
023
024import java.io.IOException;
025import java.util.List;
026import java.util.Random;
027import org.apache.hadoop.conf.Configuration;
028import org.apache.hadoop.fs.FSDataOutputStream;
029import org.apache.hadoop.fs.FileStatus;
030import org.apache.hadoop.fs.FileSystem;
031import org.apache.hadoop.fs.Path;
032import org.apache.hadoop.hbase.ChoreService;
033import org.apache.hadoop.hbase.CoordinatedStateManager;
034import org.apache.hadoop.hbase.HBaseClassTestRule;
035import org.apache.hadoop.hbase.HBaseTestingUtility;
036import org.apache.hadoop.hbase.HConstants;
037import org.apache.hadoop.hbase.Server;
038import org.apache.hadoop.hbase.ServerName;
039import org.apache.hadoop.hbase.client.AsyncClusterConnection;
040import org.apache.hadoop.hbase.client.Connection;
041import org.apache.hadoop.hbase.testclassification.MasterTests;
042import org.apache.hadoop.hbase.testclassification.MediumTests;
043import org.apache.hadoop.hbase.util.EnvironmentEdge;
044import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
045import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
046import org.junit.AfterClass;
047import org.junit.Assert;
048import org.junit.BeforeClass;
049import org.junit.ClassRule;
050import org.junit.Test;
051import org.junit.experimental.categories.Category;
052import org.slf4j.Logger;
053import org.slf4j.LoggerFactory;
054
055@Category({MasterTests.class, MediumTests.class})
056public class TestHFileCleaner {
057
058  @ClassRule
059  public static final HBaseClassTestRule CLASS_RULE =
060      HBaseClassTestRule.forClass(TestHFileCleaner.class);
061
062  private static final Logger LOG = LoggerFactory.getLogger(TestHFileCleaner.class);
063
064  private final static HBaseTestingUtility UTIL = new HBaseTestingUtility();
065
066  private static DirScanPool POOL;
067
068  @BeforeClass
069  public static void setupCluster() throws Exception {
070    // have to use a minidfs cluster because the localfs doesn't modify file times correctly
071    UTIL.startMiniDFSCluster(1);
072    POOL = new DirScanPool(UTIL.getConfiguration());
073  }
074
075  @AfterClass
076  public static void shutdownCluster() throws IOException {
077    UTIL.shutdownMiniDFSCluster();
078    POOL.shutdownNow();
079  }
080
081  @Test
082  public void testTTLCleaner() throws IOException, InterruptedException {
083    FileSystem fs = UTIL.getDFSCluster().getFileSystem();
084    Path root = UTIL.getDataTestDirOnTestFS();
085    Path file = new Path(root, "file");
086    fs.createNewFile(file);
087    long createTime = System.currentTimeMillis();
088    assertTrue("Test file not created!", fs.exists(file));
089    TimeToLiveHFileCleaner cleaner = new TimeToLiveHFileCleaner();
090    // update the time info for the file, so the cleaner removes it
091    fs.setTimes(file, createTime - 100, -1);
092    Configuration conf = UTIL.getConfiguration();
093    conf.setLong(TimeToLiveHFileCleaner.TTL_CONF_KEY, 100);
094    cleaner.setConf(conf);
095    assertTrue("File not set deletable - check mod time:" + getFileStats(file, fs)
096        + " with create time:" + createTime, cleaner.isFileDeletable(fs.getFileStatus(file)));
097  }
098
099  /**
100   * @param file to check
101   * @return loggable information about the file
102   */
103  private String getFileStats(Path file, FileSystem fs) throws IOException {
104    FileStatus status = fs.getFileStatus(file);
105    return "File" + file + ", mtime:" + status.getModificationTime() + ", atime:"
106        + status.getAccessTime();
107  }
108
109  @Test
110  public void testHFileCleaning() throws Exception {
111    final EnvironmentEdge originalEdge = EnvironmentEdgeManager.getDelegate();
112    String prefix = "someHFileThatWouldBeAUUID";
113    Configuration conf = UTIL.getConfiguration();
114    // set TTL
115    long ttl = 2000;
116    conf.set(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS,
117      "org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner");
118    conf.setLong(TimeToLiveHFileCleaner.TTL_CONF_KEY, ttl);
119    Server server = new DummyServer();
120    Path archivedHfileDir =
121      new Path(UTIL.getDataTestDirOnTestFS(), HConstants.HFILE_ARCHIVE_DIRECTORY);
122    FileSystem fs = FileSystem.get(conf);
123    HFileCleaner cleaner = new HFileCleaner(1000, server, conf, fs, archivedHfileDir, POOL);
124
125    // Create 2 invalid files, 1 "recent" file, 1 very new file and 30 old files
126    final long createTime = System.currentTimeMillis();
127    fs.delete(archivedHfileDir, true);
128    fs.mkdirs(archivedHfileDir);
129    // Case 1: 1 invalid file, which should be deleted directly
130    fs.createNewFile(new Path(archivedHfileDir, "dfd-dfd"));
131    // Case 2: 1 "recent" file, not even deletable for the first log cleaner
132    // (TimeToLiveLogCleaner), so we are not going down the chain
133    LOG.debug("Now is: " + createTime);
134    for (int i = 1; i < 32; i++) {
135      // Case 3: old files which would be deletable for the first log cleaner
136      // (TimeToLiveHFileCleaner),
137      Path fileName = new Path(archivedHfileDir, (prefix + "." + (createTime + i)));
138      fs.createNewFile(fileName);
139      // set the creation time past ttl to ensure that it gets removed
140      fs.setTimes(fileName, createTime - ttl - 1, -1);
141      LOG.debug("Creating " + getFileStats(fileName, fs));
142    }
143
144    // Case 2: 1 newer file, not even deletable for the first log cleaner
145    // (TimeToLiveLogCleaner), so we are not going down the chain
146    Path saved = new Path(archivedHfileDir, prefix + ".00000000000");
147    fs.createNewFile(saved);
148    // set creation time within the ttl
149    fs.setTimes(saved, createTime - ttl / 2, -1);
150    LOG.debug("Creating " + getFileStats(saved, fs));
151    for (FileStatus stat : fs.listStatus(archivedHfileDir)) {
152      LOG.debug(stat.getPath().toString());
153    }
154
155    assertEquals(33, fs.listStatus(archivedHfileDir).length);
156
157    // set a custom edge manager to handle time checking
158    EnvironmentEdge setTime = new EnvironmentEdge() {
159      @Override
160      public long currentTime() {
161        return createTime;
162      }
163    };
164    EnvironmentEdgeManager.injectEdge(setTime);
165
166    // run the chore
167    cleaner.chore();
168
169    // ensure we only end up with the saved file
170    assertEquals(1, fs.listStatus(archivedHfileDir).length);
171
172    for (FileStatus file : fs.listStatus(archivedHfileDir)) {
173      LOG.debug("Kept hfiles: " + file.getPath().getName());
174    }
175
176    // reset the edge back to the original edge
177    EnvironmentEdgeManager.injectEdge(originalEdge);
178  }
179
180  @Test
181  public void testRemovesEmptyDirectories() throws Exception {
182    Configuration conf = UTIL.getConfiguration();
183    // no cleaner policies = delete all files
184    conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, "");
185    Server server = new DummyServer();
186    Path archivedHfileDir =
187      new Path(UTIL.getDataTestDirOnTestFS(), HConstants.HFILE_ARCHIVE_DIRECTORY);
188
189    // setup the cleaner
190    FileSystem fs = UTIL.getDFSCluster().getFileSystem();
191    HFileCleaner cleaner = new HFileCleaner(1000, server, conf, fs, archivedHfileDir, POOL);
192
193    // make all the directories for archiving files
194    Path table = new Path(archivedHfileDir, "table");
195    Path region = new Path(table, "regionsomthing");
196    Path family = new Path(region, "fam");
197    Path file = new Path(family, "file12345");
198    fs.mkdirs(family);
199    if (!fs.exists(family)) throw new RuntimeException("Couldn't create test family:" + family);
200    fs.create(file).close();
201    if (!fs.exists(file)) throw new RuntimeException("Test file didn't get created:" + file);
202
203    // run the chore to cleanup the files (and the directories above it)
204    cleaner.chore();
205
206    // make sure all the parent directories get removed
207    assertFalse("family directory not removed for empty directory", fs.exists(family));
208    assertFalse("region directory not removed for empty directory", fs.exists(region));
209    assertFalse("table directory not removed for empty directory", fs.exists(table));
210    assertTrue("archive directory", fs.exists(archivedHfileDir));
211  }
212
213  static class DummyServer implements Server {
214    @Override
215    public Configuration getConfiguration() {
216      return UTIL.getConfiguration();
217    }
218
219    @Override
220    public ZKWatcher getZooKeeper() {
221      try {
222        return new ZKWatcher(getConfiguration(), "dummy server", this);
223      } catch (IOException e) {
224        e.printStackTrace();
225      }
226      return null;
227    }
228
229    @Override
230    public CoordinatedStateManager getCoordinatedStateManager() {
231      return null;
232    }
233
234    @Override
235    public Connection getConnection() {
236      return null;
237    }
238
239    @Override
240    public ServerName getServerName() {
241      return ServerName.valueOf("regionserver,60020,000000");
242    }
243
244    @Override
245    public void abort(String why, Throwable e) {
246    }
247
248    @Override
249    public boolean isAborted() {
250      return false;
251    }
252
253    @Override
254    public void stop(String why) {
255    }
256
257    @Override
258    public boolean isStopped() {
259      return false;
260    }
261
262    @Override
263    public ChoreService getChoreService() {
264      return null;
265    }
266
267    @Override
268    public FileSystem getFileSystem() {
269      return null;
270    }
271
272    @Override
273    public boolean isStopping() {
274      return false;
275    }
276
277    @Override
278    public Connection createConnection(Configuration conf) throws IOException {
279      return null;
280    }
281
282    @Override
283    public AsyncClusterConnection getAsyncClusterConnection() {
284      return null;
285    }
286  }
287
288  @Test
289  public void testThreadCleanup() throws Exception {
290    Configuration conf = UTIL.getConfiguration();
291    conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, "");
292    Server server = new DummyServer();
293    Path archivedHfileDir =
294        new Path(UTIL.getDataTestDirOnTestFS(), HConstants.HFILE_ARCHIVE_DIRECTORY);
295
296    // setup the cleaner
297    FileSystem fs = UTIL.getDFSCluster().getFileSystem();
298    HFileCleaner cleaner = new HFileCleaner(1000, server, conf, fs, archivedHfileDir, POOL);
299    // clean up archive directory
300    fs.delete(archivedHfileDir, true);
301    fs.mkdirs(archivedHfileDir);
302    // create some file to delete
303    fs.createNewFile(new Path(archivedHfileDir, "dfd-dfd"));
304    // launch the chore
305    cleaner.chore();
306    // call cleanup
307    cleaner.cleanup();
308    // wait awhile for thread to die
309    Thread.sleep(100);
310    for (Thread thread : cleaner.getCleanerThreads()) {
311      Assert.assertFalse(thread.isAlive());
312    }
313  }
314
315  @Test
316  public void testLargeSmallIsolation() throws Exception {
317    Configuration conf = UTIL.getConfiguration();
318    // no cleaner policies = delete all files
319    conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, "");
320    conf.setInt(HFileCleaner.HFILE_DELETE_THROTTLE_THRESHOLD, 512 * 1024);
321    Server server = new DummyServer();
322    Path archivedHfileDir =
323        new Path(UTIL.getDataTestDirOnTestFS(), HConstants.HFILE_ARCHIVE_DIRECTORY);
324
325    // setup the cleaner
326    FileSystem fs = UTIL.getDFSCluster().getFileSystem();
327    HFileCleaner cleaner = new HFileCleaner(1000, server, conf, fs, archivedHfileDir, POOL);
328    // clean up archive directory
329    fs.delete(archivedHfileDir, true);
330    fs.mkdirs(archivedHfileDir);
331    // necessary set up
332    final int LARGE_FILE_NUM = 5;
333    final int SMALL_FILE_NUM = 20;
334    createFilesForTesting(LARGE_FILE_NUM, SMALL_FILE_NUM, fs, archivedHfileDir);
335    // call cleanup
336    cleaner.chore();
337
338    Assert.assertEquals(LARGE_FILE_NUM, cleaner.getNumOfDeletedLargeFiles());
339    Assert.assertEquals(SMALL_FILE_NUM, cleaner.getNumOfDeletedSmallFiles());
340  }
341
342  @Test
343  public void testOnConfigurationChange() throws Exception {
344    // constants
345    final int ORIGINAL_THROTTLE_POINT = 512 * 1024;
346    final int ORIGINAL_QUEUE_INIT_SIZE = 512;
347    final int UPDATE_THROTTLE_POINT = 1024;// small enough to change large/small check
348    final int UPDATE_QUEUE_INIT_SIZE = 1024;
349    final int LARGE_FILE_NUM = 5;
350    final int SMALL_FILE_NUM = 20;
351    final int LARGE_THREAD_NUM = 2;
352    final int SMALL_THREAD_NUM = 4;
353    final long THREAD_TIMEOUT_MSEC = 30 * 1000L;
354    final long THREAD_CHECK_INTERVAL_MSEC = 500L;
355
356    Configuration conf = UTIL.getConfiguration();
357    // no cleaner policies = delete all files
358    conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, "");
359    conf.setInt(HFileCleaner.HFILE_DELETE_THROTTLE_THRESHOLD, ORIGINAL_THROTTLE_POINT);
360    conf.setInt(HFileCleaner.LARGE_HFILE_QUEUE_INIT_SIZE, ORIGINAL_QUEUE_INIT_SIZE);
361    conf.setInt(HFileCleaner.SMALL_HFILE_QUEUE_INIT_SIZE, ORIGINAL_QUEUE_INIT_SIZE);
362    Server server = new DummyServer();
363    Path archivedHfileDir =
364        new Path(UTIL.getDataTestDirOnTestFS(), HConstants.HFILE_ARCHIVE_DIRECTORY);
365
366    // setup the cleaner
367    FileSystem fs = UTIL.getDFSCluster().getFileSystem();
368    final HFileCleaner cleaner = new HFileCleaner(1000, server, conf, fs, archivedHfileDir, POOL);
369    Assert.assertEquals(ORIGINAL_THROTTLE_POINT, cleaner.getThrottlePoint());
370    Assert.assertEquals(ORIGINAL_QUEUE_INIT_SIZE, cleaner.getLargeQueueInitSize());
371    Assert.assertEquals(ORIGINAL_QUEUE_INIT_SIZE, cleaner.getSmallQueueInitSize());
372    Assert.assertEquals(HFileCleaner.DEFAULT_HFILE_DELETE_THREAD_TIMEOUT_MSEC,
373        cleaner.getCleanerThreadTimeoutMsec());
374    Assert.assertEquals(HFileCleaner.DEFAULT_HFILE_DELETE_THREAD_CHECK_INTERVAL_MSEC,
375        cleaner.getCleanerThreadCheckIntervalMsec());
376
377    // clean up archive directory and create files for testing
378    fs.delete(archivedHfileDir, true);
379    fs.mkdirs(archivedHfileDir);
380    createFilesForTesting(LARGE_FILE_NUM, SMALL_FILE_NUM, fs, archivedHfileDir);
381
382    // call cleaner, run as daemon to test the interrupt-at-middle case
383    Thread t = new Thread() {
384      @Override
385      public void run() {
386        cleaner.chore();
387      }
388    };
389    t.setDaemon(true);
390    t.start();
391    // wait until file clean started
392    while (cleaner.getNumOfDeletedSmallFiles() == 0) {
393      Thread.yield();
394    }
395
396    // trigger configuration change
397    Configuration newConf = new Configuration(conf);
398    newConf.setInt(HFileCleaner.HFILE_DELETE_THROTTLE_THRESHOLD, UPDATE_THROTTLE_POINT);
399    newConf.setInt(HFileCleaner.LARGE_HFILE_QUEUE_INIT_SIZE, UPDATE_QUEUE_INIT_SIZE);
400    newConf.setInt(HFileCleaner.SMALL_HFILE_QUEUE_INIT_SIZE, UPDATE_QUEUE_INIT_SIZE);
401    newConf.setInt(HFileCleaner.LARGE_HFILE_DELETE_THREAD_NUMBER, LARGE_THREAD_NUM);
402    newConf.setInt(HFileCleaner.SMALL_HFILE_DELETE_THREAD_NUMBER, SMALL_THREAD_NUM);
403    newConf.setLong(HFileCleaner.HFILE_DELETE_THREAD_TIMEOUT_MSEC, THREAD_TIMEOUT_MSEC);
404    newConf.setLong(HFileCleaner.HFILE_DELETE_THREAD_CHECK_INTERVAL_MSEC,
405        THREAD_CHECK_INTERVAL_MSEC);
406
407    LOG.debug("File deleted from large queue: " + cleaner.getNumOfDeletedLargeFiles()
408        + "; from small queue: " + cleaner.getNumOfDeletedSmallFiles());
409    cleaner.onConfigurationChange(newConf);
410
411    // check values after change
412    Assert.assertEquals(UPDATE_THROTTLE_POINT, cleaner.getThrottlePoint());
413    Assert.assertEquals(UPDATE_QUEUE_INIT_SIZE, cleaner.getLargeQueueInitSize());
414    Assert.assertEquals(UPDATE_QUEUE_INIT_SIZE, cleaner.getSmallQueueInitSize());
415    Assert.assertEquals(LARGE_THREAD_NUM + SMALL_THREAD_NUM, cleaner.getCleanerThreads().size());
416    Assert.assertEquals(THREAD_TIMEOUT_MSEC, cleaner.getCleanerThreadTimeoutMsec());
417    Assert.assertEquals(THREAD_CHECK_INTERVAL_MSEC, cleaner.getCleanerThreadCheckIntervalMsec());
418
419    // make sure no cost when onConfigurationChange called with no change
420    List<Thread> oldThreads = cleaner.getCleanerThreads();
421    cleaner.onConfigurationChange(newConf);
422    List<Thread> newThreads = cleaner.getCleanerThreads();
423    Assert.assertArrayEquals(oldThreads.toArray(), newThreads.toArray());
424
425    // wait until clean done and check
426    t.join();
427    LOG.debug("File deleted from large queue: " + cleaner.getNumOfDeletedLargeFiles()
428        + "; from small queue: " + cleaner.getNumOfDeletedSmallFiles());
429    Assert.assertTrue("Should delete more than " + LARGE_FILE_NUM
430        + " files from large queue but actually " + cleaner.getNumOfDeletedLargeFiles(),
431      cleaner.getNumOfDeletedLargeFiles() > LARGE_FILE_NUM);
432    Assert.assertTrue("Should delete less than " + SMALL_FILE_NUM
433        + " files from small queue but actually " + cleaner.getNumOfDeletedSmallFiles(),
434      cleaner.getNumOfDeletedSmallFiles() < SMALL_FILE_NUM);
435  }
436
437  private void createFilesForTesting(int largeFileNum, int smallFileNum, FileSystem fs,
438      Path archivedHfileDir) throws IOException {
439    final Random rand = new Random();
440    final byte[] large = new byte[1024 * 1024];
441    for (int i = 0; i < large.length; i++) {
442      large[i] = (byte) rand.nextInt(128);
443    }
444    final byte[] small = new byte[1024];
445    for (int i = 0; i < small.length; i++) {
446      small[i] = (byte) rand.nextInt(128);
447    }
448    // create large and small files
449    for (int i = 1; i <= largeFileNum; i++) {
450      FSDataOutputStream out = fs.create(new Path(archivedHfileDir, "large-file-" + i));
451      out.write(large);
452      out.close();
453    }
454    for (int i = 1; i <= smallFileNum; i++) {
455      FSDataOutputStream out = fs.create(new Path(archivedHfileDir, "small-file-" + i));
456      out.write(small);
457      out.close();
458    }
459  }
460}