001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.cleaner;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertTrue;
023
024import java.io.IOException;
025import java.util.Random;
026import java.util.concurrent.atomic.AtomicBoolean;
027import org.apache.hadoop.conf.Configuration;
028import org.apache.hadoop.fs.FSDataOutputStream;
029import org.apache.hadoop.fs.FileStatus;
030import org.apache.hadoop.fs.FileSystem;
031import org.apache.hadoop.fs.FilterFileSystem;
032import org.apache.hadoop.fs.Path;
033import org.apache.hadoop.hbase.HBaseClassTestRule;
034import org.apache.hadoop.hbase.HBaseTestingUtility;
035import org.apache.hadoop.hbase.Stoppable;
036import org.apache.hadoop.hbase.testclassification.MasterTests;
037import org.apache.hadoop.hbase.testclassification.SmallTests;
038import org.apache.hadoop.hbase.util.FSUtils;
039import org.apache.hadoop.hbase.util.StoppableImplementation;
040import org.junit.AfterClass;
041import org.junit.BeforeClass;
042import org.junit.ClassRule;
043import org.junit.Test;
044import org.junit.experimental.categories.Category;
045import org.mockito.Mockito;
046import org.mockito.invocation.InvocationOnMock;
047import org.mockito.stubbing.Answer;
048import org.slf4j.Logger;
049import org.slf4j.LoggerFactory;
050
051@Category({MasterTests.class, SmallTests.class})
052public class TestCleanerChore {
053
054  @ClassRule
055  public static final HBaseClassTestRule CLASS_RULE =
056    HBaseClassTestRule.forClass(TestCleanerChore.class);
057
058  private static final Logger LOG = LoggerFactory.getLogger(TestCleanerChore.class);
059  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
060  private static DirScanPool POOL;
061
062  @BeforeClass
063  public static void setup() {
064    POOL = new DirScanPool(UTIL.getConfiguration());
065  }
066
067  @AfterClass
068  public static void cleanup() throws Exception {
069    // delete and recreate the test directory, ensuring a clean test dir between tests
070    UTIL.cleanupTestDir();
071    POOL.shutdownNow();
072  }
073
074  @Test
075  public void testSavesFilesOnRequest() throws Exception {
076    Stoppable stop = new StoppableImplementation();
077    Configuration conf = UTIL.getConfiguration();
078    Path testDir = UTIL.getDataTestDir();
079    FileSystem fs = UTIL.getTestFileSystem();
080    String confKey = "hbase.test.cleaner.delegates";
081    conf.set(confKey, NeverDelete.class.getName());
082
083    AllValidPaths chore =
084      new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey, POOL);
085
086    // create the directory layout in the directory to clean
087    Path parent = new Path(testDir, "parent");
088    Path file = new Path(parent, "someFile");
089    fs.mkdirs(parent);
090    // touch a new file
091    fs.create(file).close();
092    assertTrue("Test file didn't get created.", fs.exists(file));
093
094    // run the chore
095    chore.chore();
096
097    // verify all the files were preserved
098    assertTrue("File shouldn't have been deleted", fs.exists(file));
099    assertTrue("directory shouldn't have been deleted", fs.exists(parent));
100  }
101
102  @Test
103  public void retriesIOExceptionInStatus() throws Exception {
104    Stoppable stop = new StoppableImplementation();
105    Configuration conf = UTIL.getConfiguration();
106    Path testDir = UTIL.getDataTestDir();
107    FileSystem fs = UTIL.getTestFileSystem();
108    String confKey = "hbase.test.cleaner.delegates";
109
110    Path child = new Path(testDir, "child");
111    Path file = new Path(child, "file");
112    fs.mkdirs(child);
113    fs.create(file).close();
114    assertTrue("test file didn't get created.", fs.exists(file));
115    final AtomicBoolean fails = new AtomicBoolean(true);
116
117    FilterFileSystem filtered = new FilterFileSystem(fs) {
118      public FileStatus[] listStatus(Path f) throws IOException {
119        if (fails.get()) {
120          throw new IOException("whomp whomp.");
121        }
122        return fs.listStatus(f);
123      }
124    };
125
126    AllValidPaths chore =
127      new AllValidPaths("test-retry-ioe", stop, conf, filtered, testDir, confKey, POOL);
128
129    // trouble talking to the filesystem
130    Boolean result = chore.runCleaner();
131
132    // verify that it couldn't clean the files.
133    assertTrue("test rig failed to inject failure.", fs.exists(file));
134    assertTrue("test rig failed to inject failure.", fs.exists(child));
135    // and verify that it accurately reported the failure.
136    assertFalse("chore should report that it failed.", result);
137
138    // filesystem is back
139    fails.set(false);
140    result = chore.runCleaner();
141
142    // verify everything is gone.
143    assertFalse("file should have been destroyed.", fs.exists(file));
144    assertFalse("directory should have been destroyed.", fs.exists(child));
145    // and verify that it accurately reported success.
146    assertTrue("chore should claim it succeeded.", result);
147  }
148
149  @Test
150  public void testDeletesEmptyDirectories() throws Exception {
151    Stoppable stop = new StoppableImplementation();
152    Configuration conf = UTIL.getConfiguration();
153    Path testDir = UTIL.getDataTestDir();
154    FileSystem fs = UTIL.getTestFileSystem();
155    String confKey = "hbase.test.cleaner.delegates";
156    conf.set(confKey, AlwaysDelete.class.getName());
157
158    AllValidPaths chore =
159      new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey, POOL);
160
161    // create the directory layout in the directory to clean
162    Path parent = new Path(testDir, "parent");
163    Path child = new Path(parent, "child");
164    Path emptyChild = new Path(parent, "emptyChild");
165    Path file = new Path(child, "someFile");
166    fs.mkdirs(child);
167    fs.mkdirs(emptyChild);
168    // touch a new file
169    fs.create(file).close();
170    // also create a file in the top level directory
171    Path topFile = new Path(testDir, "topFile");
172    fs.create(topFile).close();
173    assertTrue("Test file didn't get created.", fs.exists(file));
174    assertTrue("Test file didn't get created.", fs.exists(topFile));
175
176    // run the chore
177    chore.chore();
178
179    // verify all the files got deleted
180    assertFalse("File didn't get deleted", fs.exists(topFile));
181    assertFalse("File didn't get deleted", fs.exists(file));
182    assertFalse("Empty directory didn't get deleted", fs.exists(child));
183    assertFalse("Empty directory didn't get deleted", fs.exists(parent));
184  }
185
186  /**
187   * Test to make sure that we don't attempt to ask the delegate whether or not we should preserve a
188   * directory.
189   * @throws Exception on failure
190   */
191  @Test
192  public void testDoesNotCheckDirectories() throws Exception {
193    Stoppable stop = new StoppableImplementation();
194    Configuration conf = UTIL.getConfiguration();
195    Path testDir = UTIL.getDataTestDir();
196    FileSystem fs = UTIL.getTestFileSystem();
197    String confKey = "hbase.test.cleaner.delegates";
198    conf.set(confKey, AlwaysDelete.class.getName());
199
200    AllValidPaths chore =
201      new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey, POOL);
202    // spy on the delegate to ensure that we don't check for directories
203    AlwaysDelete delegate = (AlwaysDelete) chore.cleanersChain.get(0);
204    AlwaysDelete spy = Mockito.spy(delegate);
205    chore.cleanersChain.set(0, spy);
206
207    // create the directory layout in the directory to clean
208    Path parent = new Path(testDir, "parent");
209    Path file = new Path(parent, "someFile");
210    fs.mkdirs(parent);
211    assertTrue("Test parent didn't get created.", fs.exists(parent));
212    // touch a new file
213    fs.create(file).close();
214    assertTrue("Test file didn't get created.", fs.exists(file));
215
216    FileStatus fStat = fs.getFileStatus(parent);
217    chore.chore();
218    // make sure we never checked the directory
219    Mockito.verify(spy, Mockito.never()).isFileDeletable(fStat);
220    Mockito.reset(spy);
221  }
222
223  @Test
224  public void testStoppedCleanerDoesNotDeleteFiles() throws Exception {
225    Stoppable stop = new StoppableImplementation();
226    Configuration conf = UTIL.getConfiguration();
227    Path testDir = UTIL.getDataTestDir();
228    FileSystem fs = UTIL.getTestFileSystem();
229    String confKey = "hbase.test.cleaner.delegates";
230    conf.set(confKey, AlwaysDelete.class.getName());
231
232    AllValidPaths chore =
233      new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey, POOL);
234
235    // also create a file in the top level directory
236    Path topFile = new Path(testDir, "topFile");
237    fs.create(topFile).close();
238    assertTrue("Test file didn't get created.", fs.exists(topFile));
239
240    // stop the chore
241    stop.stop("testing stop");
242
243    // run the chore
244    chore.chore();
245
246    // test that the file still exists
247    assertTrue("File got deleted while chore was stopped", fs.exists(topFile));
248  }
249
250  /**
251   * While cleaning a directory, all the files in the directory may be deleted, but there may be
252   * another file added, in which case the directory shouldn't be deleted.
253   * @throws IOException on failure
254   */
255  @Test
256  public void testCleanerDoesNotDeleteDirectoryWithLateAddedFiles() throws IOException {
257    Stoppable stop = new StoppableImplementation();
258    Configuration conf = UTIL.getConfiguration();
259    final Path testDir = UTIL.getDataTestDir();
260    final FileSystem fs = UTIL.getTestFileSystem();
261    String confKey = "hbase.test.cleaner.delegates";
262    conf.set(confKey, AlwaysDelete.class.getName());
263
264    AllValidPaths chore =
265      new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey, POOL);
266    // spy on the delegate to ensure that we don't check for directories
267    AlwaysDelete delegate = (AlwaysDelete) chore.cleanersChain.get(0);
268    AlwaysDelete spy = Mockito.spy(delegate);
269    chore.cleanersChain.set(0, spy);
270
271    // create the directory layout in the directory to clean
272    final Path parent = new Path(testDir, "parent");
273    Path file = new Path(parent, "someFile");
274    fs.mkdirs(parent);
275    // touch a new file
276    fs.create(file).close();
277    assertTrue("Test file didn't get created.", fs.exists(file));
278    final Path addedFile = new Path(parent, "addedFile");
279
280    // when we attempt to delete the original file, add another file in the same directory
281    Mockito.doAnswer(new Answer<Boolean>() {
282      @Override
283      public Boolean answer(InvocationOnMock invocation) throws Throwable {
284        fs.create(addedFile).close();
285        FSUtils.logFileSystemState(fs, testDir, LOG);
286        return (Boolean) invocation.callRealMethod();
287      }
288    }).when(spy).isFileDeletable(Mockito.any());
289
290    // run the chore
291    chore.chore();
292
293    // make sure all the directories + added file exist, but the original file is deleted
294    assertTrue("Added file unexpectedly deleted", fs.exists(addedFile));
295    assertTrue("Parent directory deleted unexpectedly", fs.exists(parent));
296    assertFalse("Original file unexpectedly retained", fs.exists(file));
297    Mockito.verify(spy, Mockito.times(1)).isFileDeletable(Mockito.any());
298    Mockito.reset(spy);
299  }
300
301  /**
302   * The cleaner runs in a loop, where it first checks to see all the files under a directory can be
303   * deleted. If they all can, then we try to delete the directory. However, a file may be added
304   * that directory to after the original check. This ensures that we don't accidentally delete that
305   * directory on and don't get spurious IOExceptions.
306   * <p>
307   * This was from HBASE-7465.
308   * @throws Exception on failure
309   */
310  @Test
311  public void testNoExceptionFromDirectoryWithRacyChildren() throws Exception {
312    UTIL.cleanupTestDir();
313    Stoppable stop = new StoppableImplementation();
314    // need to use a localutil to not break the rest of the test that runs on the local FS, which
315    // gets hosed when we start to use a minicluster.
316    HBaseTestingUtility localUtil = new HBaseTestingUtility();
317    Configuration conf = localUtil.getConfiguration();
318    final Path testDir = UTIL.getDataTestDir();
319    final FileSystem fs = UTIL.getTestFileSystem();
320    LOG.debug("Writing test data to: " + testDir);
321    String confKey = "hbase.test.cleaner.delegates";
322    conf.set(confKey, AlwaysDelete.class.getName());
323
324    AllValidPaths chore =
325      new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey, POOL);
326    // spy on the delegate to ensure that we don't check for directories
327    AlwaysDelete delegate = (AlwaysDelete) chore.cleanersChain.get(0);
328    AlwaysDelete spy = Mockito.spy(delegate);
329    chore.cleanersChain.set(0, spy);
330
331    // create the directory layout in the directory to clean
332    final Path parent = new Path(testDir, "parent");
333    Path file = new Path(parent, "someFile");
334    fs.mkdirs(parent);
335    // touch a new file
336    fs.create(file).close();
337    assertTrue("Test file didn't get created.", fs.exists(file));
338    final Path racyFile = new Path(parent, "addedFile");
339
340    // when we attempt to delete the original file, add another file in the same directory
341    Mockito.doAnswer(new Answer<Boolean>() {
342      @Override
343      public Boolean answer(InvocationOnMock invocation) throws Throwable {
344        fs.create(racyFile).close();
345        FSUtils.logFileSystemState(fs, testDir, LOG);
346        return (Boolean) invocation.callRealMethod();
347      }
348    }).when(spy).isFileDeletable(Mockito.any());
349
350    // run the chore
351    chore.chore();
352
353    // make sure all the directories + added file exist, but the original file is deleted
354    assertTrue("Added file unexpectedly deleted", fs.exists(racyFile));
355    assertTrue("Parent directory deleted unexpectedly", fs.exists(parent));
356    assertFalse("Original file unexpectedly retained", fs.exists(file));
357    Mockito.verify(spy, Mockito.times(1)).isFileDeletable(Mockito.any());
358  }
359
360  @Test
361  public void testDeleteFileWithCleanerEnabled() throws Exception {
362    Stoppable stop = new StoppableImplementation();
363    Configuration conf = UTIL.getConfiguration();
364    Path testDir = UTIL.getDataTestDir();
365    FileSystem fs = UTIL.getTestFileSystem();
366    String confKey = "hbase.test.cleaner.delegates";
367    conf.set(confKey, AlwaysDelete.class.getName());
368
369    AllValidPaths chore =
370      new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey, POOL);
371
372    // Enable cleaner
373    chore.setEnabled(true);
374
375    // create the directory layout in the directory to clean
376    Path parent = new Path(testDir, "parent");
377    Path child = new Path(parent, "child");
378    Path file = new Path(child, "someFile");
379    fs.mkdirs(child);
380
381    // touch a new file
382    fs.create(file).close();
383    assertTrue("Test file didn't get created.", fs.exists(file));
384
385    // run the chore
386    chore.chore();
387
388    // verify all the files got deleted
389    assertFalse("File didn't get deleted", fs.exists(file));
390    assertFalse("Empty directory didn't get deleted", fs.exists(child));
391    assertFalse("Empty directory didn't get deleted", fs.exists(parent));
392  }
393
394  @Test
395  public void testDeleteFileWithCleanerDisabled() throws Exception {
396    Stoppable stop = new StoppableImplementation();
397    Configuration conf = UTIL.getConfiguration();
398    Path testDir = UTIL.getDataTestDir();
399    FileSystem fs = UTIL.getTestFileSystem();
400    String confKey = "hbase.test.cleaner.delegates";
401    conf.set(confKey, AlwaysDelete.class.getName());
402
403    AllValidPaths chore =
404      new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey, POOL);
405
406    // Disable cleaner
407    chore.setEnabled(false);
408
409    // create the directory layout in the directory to clean
410    Path parent = new Path(testDir, "parent");
411    Path child = new Path(parent, "child");
412    Path file = new Path(child, "someFile");
413    fs.mkdirs(child);
414
415    // touch a new file
416    fs.create(file).close();
417    assertTrue("Test file didn't get created.", fs.exists(file));
418
419    // run the chore
420    chore.chore();
421
422    // verify all the files exist
423    assertTrue("File got deleted with cleaner disabled", fs.exists(file));
424    assertTrue("Directory got deleted", fs.exists(child));
425    assertTrue("Directory got deleted", fs.exists(parent));
426  }
427
428  @Test
429  public void testOnConfigurationChange() throws Exception {
430    int availableProcessorNum = Runtime.getRuntime().availableProcessors();
431    if (availableProcessorNum == 1) { // no need to run this test
432      return;
433    }
434
435    // have at least 2 available processors/cores
436    int initPoolSize = availableProcessorNum / 2;
437    int changedPoolSize = availableProcessorNum;
438
439    Stoppable stop = new StoppableImplementation();
440    Configuration conf = UTIL.getConfiguration();
441    Path testDir = UTIL.getDataTestDir();
442    FileSystem fs = UTIL.getTestFileSystem();
443    String confKey = "hbase.test.cleaner.delegates";
444    conf.set(confKey, AlwaysDelete.class.getName());
445    conf.set(CleanerChore.CHORE_POOL_SIZE, String.valueOf(initPoolSize));
446    AllValidPaths chore =
447      new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey, POOL);
448    chore.setEnabled(true);
449    // Create subdirs under testDir
450    int dirNums = 6;
451    Path[] subdirs = new Path[dirNums];
452    for (int i = 0; i < dirNums; i++) {
453      subdirs[i] = new Path(testDir, "subdir-" + i);
454      fs.mkdirs(subdirs[i]);
455    }
456    // Under each subdirs create 6 files
457    for (Path subdir : subdirs) {
458      createFiles(fs, subdir, 6);
459    }
460    // Start chore
461    Thread t = new Thread(() -> chore.chore());
462    t.setDaemon(true);
463    t.start();
464    // Change size of chore's pool
465    conf.set(CleanerChore.CHORE_POOL_SIZE, String.valueOf(changedPoolSize));
466    POOL.onConfigurationChange(conf);
467    assertEquals(changedPoolSize, chore.getChorePoolSize());
468    // Stop chore
469    t.join();
470  }
471
472  @Test
473  public void testMinimumNumberOfThreads() throws Exception {
474    Configuration conf = UTIL.getConfiguration();
475    String confKey = "hbase.test.cleaner.delegates";
476    conf.set(confKey, AlwaysDelete.class.getName());
477    conf.set(CleanerChore.CHORE_POOL_SIZE, "2");
478    int numProcs = Runtime.getRuntime().availableProcessors();
479    // Sanity
480    assertEquals(numProcs, CleanerChore.calculatePoolSize(Integer.toString(numProcs)));
481    // The implementation does not allow us to set more threads than we have processors
482    assertEquals(numProcs, CleanerChore.calculatePoolSize(Integer.toString(numProcs + 2)));
483    // Force us into the branch that is multiplying 0.0 against the number of processors
484    assertEquals(1, CleanerChore.calculatePoolSize("0.0"));
485  }
486
487  private void createFiles(FileSystem fs, Path parentDir, int numOfFiles) throws IOException {
488    Random random = new Random();
489    for (int i = 0; i < numOfFiles; i++) {
490      int xMega = 1 + random.nextInt(3); // size of each file is between 1~3M
491      try (FSDataOutputStream fsdos = fs.create(new Path(parentDir, "file-" + i))) {
492        for (int m = 0; m < xMega; m++) {
493          byte[] M = new byte[1024 * 1024];
494          random.nextBytes(M);
495          fsdos.write(M);
496        }
497      }
498    }
499  }
500
501  private static class AllValidPaths extends CleanerChore<BaseHFileCleanerDelegate> {
502
503    public AllValidPaths(String name, Stoppable s, Configuration conf, FileSystem fs,
504      Path oldFileDir, String confkey, DirScanPool pool) {
505      super(name, Integer.MAX_VALUE, s, conf, fs, oldFileDir, confkey, pool);
506    }
507
508    // all paths are valid
509    @Override
510    protected boolean validate(Path file) {
511      return true;
512    }
513  };
514
515  public static class AlwaysDelete extends BaseHFileCleanerDelegate {
516    @Override
517    public boolean isFileDeletable(FileStatus fStat) {
518      return true;
519    }
520  }
521
522  public static class NeverDelete extends BaseHFileCleanerDelegate {
523    @Override
524    public boolean isFileDeletable(FileStatus fStat) {
525      return false;
526    }
527  }
528}