001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.cleaner;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertTrue;
023
024import java.io.IOException;
025import java.util.Random;
026import java.util.concurrent.atomic.AtomicBoolean;
027import org.apache.hadoop.conf.Configuration;
028import org.apache.hadoop.fs.FSDataOutputStream;
029import org.apache.hadoop.fs.FileStatus;
030import org.apache.hadoop.fs.FileSystem;
031import org.apache.hadoop.fs.FilterFileSystem;
032import org.apache.hadoop.fs.Path;
033import org.apache.hadoop.hbase.HBaseClassTestRule;
034import org.apache.hadoop.hbase.HBaseTestingUtility;
035import org.apache.hadoop.hbase.Stoppable;
036import org.apache.hadoop.hbase.testclassification.MasterTests;
037import org.apache.hadoop.hbase.testclassification.SmallTests;
038import org.apache.hadoop.hbase.util.FSUtils;
039import org.apache.hadoop.hbase.util.StoppableImplementation;
040import org.junit.AfterClass;
041import org.junit.BeforeClass;
042import org.junit.ClassRule;
043import org.junit.Test;
044import org.junit.experimental.categories.Category;
045import org.mockito.Mockito;
046import org.mockito.invocation.InvocationOnMock;
047import org.mockito.stubbing.Answer;
048import org.slf4j.Logger;
049import org.slf4j.LoggerFactory;
050
051@Category({MasterTests.class, SmallTests.class})
052public class TestCleanerChore {
053
054  @ClassRule
055  public static final HBaseClassTestRule CLASS_RULE =
056      HBaseClassTestRule.forClass(TestCleanerChore.class);
057
058  private static final Logger LOG = LoggerFactory.getLogger(TestCleanerChore.class);
059  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
060
061  @BeforeClass
062  public static void setup() {
063    CleanerChore.initChorePool(UTIL.getConfiguration());
064  }
065
066  @AfterClass
067  public static void cleanup() throws Exception {
068    // delete and recreate the test directory, ensuring a clean test dir between tests
069    UTIL.cleanupTestDir();
070    CleanerChore.shutDownChorePool();
071  }
072
073  @Test
074  public void testSavesFilesOnRequest() throws Exception {
075    Stoppable stop = new StoppableImplementation();
076    Configuration conf = UTIL.getConfiguration();
077    Path testDir = UTIL.getDataTestDir();
078    FileSystem fs = UTIL.getTestFileSystem();
079    String confKey = "hbase.test.cleaner.delegates";
080    conf.set(confKey, NeverDelete.class.getName());
081
082    AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey);
083
084    // create the directory layout in the directory to clean
085    Path parent = new Path(testDir, "parent");
086    Path file = new Path(parent, "someFile");
087    fs.mkdirs(parent);
088    // touch a new file
089    fs.create(file).close();
090    assertTrue("Test file didn't get created.", fs.exists(file));
091
092    // run the chore
093    chore.chore();
094
095    // verify all the files were preserved
096    assertTrue("File shouldn't have been deleted", fs.exists(file));
097    assertTrue("directory shouldn't have been deleted", fs.exists(parent));
098  }
099
100  @Test
101  public void retriesIOExceptionInStatus() throws Exception {
102    Stoppable stop = new StoppableImplementation();
103    Configuration conf = UTIL.getConfiguration();
104    Path testDir = UTIL.getDataTestDir();
105    FileSystem fs = UTIL.getTestFileSystem();
106    String confKey = "hbase.test.cleaner.delegates";
107
108    Path child = new Path(testDir, "child");
109    Path file = new Path(child, "file");
110    fs.mkdirs(child);
111    fs.create(file).close();
112    assertTrue("test file didn't get created.", fs.exists(file));
113    final AtomicBoolean fails = new AtomicBoolean(true);
114
115    FilterFileSystem filtered = new FilterFileSystem(fs) {
116      public FileStatus[] listStatus(Path f) throws IOException {
117        if (fails.get()) {
118          throw new IOException("whomp whomp.");
119        }
120        return fs.listStatus(f);
121      }
122    };
123
124    AllValidPaths chore = new AllValidPaths("test-retry-ioe", stop, conf, filtered, testDir, confKey);
125
126    // trouble talking to the filesystem
127    Boolean result = chore.runCleaner();
128
129    // verify that it couldn't clean the files.
130    assertTrue("test rig failed to inject failure.", fs.exists(file));
131    assertTrue("test rig failed to inject failure.", fs.exists(child));
132    // and verify that it accurately reported the failure.
133    assertFalse("chore should report that it failed.", result);
134
135    // filesystem is back
136    fails.set(false);
137    result = chore.runCleaner();
138
139    // verify everything is gone.
140    assertFalse("file should have been destroyed.", fs.exists(file));
141    assertFalse("directory should have been destroyed.", fs.exists(child));
142    // and verify that it accurately reported success.
143    assertTrue("chore should claim it succeeded.", result);
144  }
145
146  @Test
147  public void testDeletesEmptyDirectories() throws Exception {
148    Stoppable stop = new StoppableImplementation();
149    Configuration conf = UTIL.getConfiguration();
150    Path testDir = UTIL.getDataTestDir();
151    FileSystem fs = UTIL.getTestFileSystem();
152    String confKey = "hbase.test.cleaner.delegates";
153    conf.set(confKey, AlwaysDelete.class.getName());
154
155    AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey);
156
157    // create the directory layout in the directory to clean
158    Path parent = new Path(testDir, "parent");
159    Path child = new Path(parent, "child");
160    Path emptyChild = new Path(parent, "emptyChild");
161    Path file = new Path(child, "someFile");
162    fs.mkdirs(child);
163    fs.mkdirs(emptyChild);
164    // touch a new file
165    fs.create(file).close();
166    // also create a file in the top level directory
167    Path topFile = new Path(testDir, "topFile");
168    fs.create(topFile).close();
169    assertTrue("Test file didn't get created.", fs.exists(file));
170    assertTrue("Test file didn't get created.", fs.exists(topFile));
171
172    // run the chore
173    chore.chore();
174
175    // verify all the files got deleted
176    assertFalse("File didn't get deleted", fs.exists(topFile));
177    assertFalse("File didn't get deleted", fs.exists(file));
178    assertFalse("Empty directory didn't get deleted", fs.exists(child));
179    assertFalse("Empty directory didn't get deleted", fs.exists(parent));
180  }
181
182  /**
183   * Test to make sure that we don't attempt to ask the delegate whether or not we should preserve a
184   * directory.
185   * @throws Exception on failure
186   */
187  @Test
188  public void testDoesNotCheckDirectories() throws Exception {
189    Stoppable stop = new StoppableImplementation();
190    Configuration conf = UTIL.getConfiguration();
191    Path testDir = UTIL.getDataTestDir();
192    FileSystem fs = UTIL.getTestFileSystem();
193    String confKey = "hbase.test.cleaner.delegates";
194    conf.set(confKey, AlwaysDelete.class.getName());
195
196    AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey);
197    // spy on the delegate to ensure that we don't check for directories
198    AlwaysDelete delegate = (AlwaysDelete) chore.cleanersChain.get(0);
199    AlwaysDelete spy = Mockito.spy(delegate);
200    chore.cleanersChain.set(0, spy);
201
202    // create the directory layout in the directory to clean
203    Path parent = new Path(testDir, "parent");
204    Path file = new Path(parent, "someFile");
205    fs.mkdirs(parent);
206    assertTrue("Test parent didn't get created.", fs.exists(parent));
207    // touch a new file
208    fs.create(file).close();
209    assertTrue("Test file didn't get created.", fs.exists(file));
210
211    FileStatus fStat = fs.getFileStatus(parent);
212    chore.chore();
213    // make sure we never checked the directory
214    Mockito.verify(spy, Mockito.never()).isFileDeletable(fStat);
215    Mockito.reset(spy);
216  }
217
218  @Test
219  public void testStoppedCleanerDoesNotDeleteFiles() throws Exception {
220    Stoppable stop = new StoppableImplementation();
221    Configuration conf = UTIL.getConfiguration();
222    Path testDir = UTIL.getDataTestDir();
223    FileSystem fs = UTIL.getTestFileSystem();
224    String confKey = "hbase.test.cleaner.delegates";
225    conf.set(confKey, AlwaysDelete.class.getName());
226
227    AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey);
228
229    // also create a file in the top level directory
230    Path topFile = new Path(testDir, "topFile");
231    fs.create(topFile).close();
232    assertTrue("Test file didn't get created.", fs.exists(topFile));
233
234    // stop the chore
235    stop.stop("testing stop");
236
237    // run the chore
238    chore.chore();
239
240    // test that the file still exists
241    assertTrue("File got deleted while chore was stopped", fs.exists(topFile));
242  }
243
244  /**
245   * While cleaning a directory, all the files in the directory may be deleted, but there may be
246   * another file added, in which case the directory shouldn't be deleted.
247   * @throws IOException on failure
248   */
249  @Test
250  public void testCleanerDoesNotDeleteDirectoryWithLateAddedFiles() throws IOException {
251    Stoppable stop = new StoppableImplementation();
252    Configuration conf = UTIL.getConfiguration();
253    final Path testDir = UTIL.getDataTestDir();
254    final FileSystem fs = UTIL.getTestFileSystem();
255    String confKey = "hbase.test.cleaner.delegates";
256    conf.set(confKey, AlwaysDelete.class.getName());
257
258    AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey);
259    // spy on the delegate to ensure that we don't check for directories
260    AlwaysDelete delegate = (AlwaysDelete) chore.cleanersChain.get(0);
261    AlwaysDelete spy = Mockito.spy(delegate);
262    chore.cleanersChain.set(0, spy);
263
264    // create the directory layout in the directory to clean
265    final Path parent = new Path(testDir, "parent");
266    Path file = new Path(parent, "someFile");
267    fs.mkdirs(parent);
268    // touch a new file
269    fs.create(file).close();
270    assertTrue("Test file didn't get created.", fs.exists(file));
271    final Path addedFile = new Path(parent, "addedFile");
272
273    // when we attempt to delete the original file, add another file in the same directory
274    Mockito.doAnswer(new Answer<Boolean>() {
275      @Override
276      public Boolean answer(InvocationOnMock invocation) throws Throwable {
277        fs.create(addedFile).close();
278        FSUtils.logFileSystemState(fs, testDir, LOG);
279        return (Boolean) invocation.callRealMethod();
280      }
281    }).when(spy).isFileDeletable(Mockito.any());
282
283    // run the chore
284    chore.chore();
285
286    // make sure all the directories + added file exist, but the original file is deleted
287    assertTrue("Added file unexpectedly deleted", fs.exists(addedFile));
288    assertTrue("Parent directory deleted unexpectedly", fs.exists(parent));
289    assertFalse("Original file unexpectedly retained", fs.exists(file));
290    Mockito.verify(spy, Mockito.times(1)).isFileDeletable(Mockito.any());
291    Mockito.reset(spy);
292  }
293
294  /**
295   * The cleaner runs in a loop, where it first checks to see all the files under a directory can be
296   * deleted. If they all can, then we try to delete the directory. However, a file may be added
297   * that directory to after the original check. This ensures that we don't accidentally delete that
298   * directory on and don't get spurious IOExceptions.
299   * <p>
300   * This was from HBASE-7465.
301   * @throws Exception on failure
302   */
303  @Test
304  public void testNoExceptionFromDirectoryWithRacyChildren() throws Exception {
305    UTIL.cleanupTestDir();
306    Stoppable stop = new StoppableImplementation();
307    // need to use a localutil to not break the rest of the test that runs on the local FS, which
308    // gets hosed when we start to use a minicluster.
309    HBaseTestingUtility localUtil = new HBaseTestingUtility();
310    Configuration conf = localUtil.getConfiguration();
311    final Path testDir = UTIL.getDataTestDir();
312    final FileSystem fs = UTIL.getTestFileSystem();
313    LOG.debug("Writing test data to: " + testDir);
314    String confKey = "hbase.test.cleaner.delegates";
315    conf.set(confKey, AlwaysDelete.class.getName());
316
317    AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey);
318    // spy on the delegate to ensure that we don't check for directories
319    AlwaysDelete delegate = (AlwaysDelete) chore.cleanersChain.get(0);
320    AlwaysDelete spy = Mockito.spy(delegate);
321    chore.cleanersChain.set(0, spy);
322
323    // create the directory layout in the directory to clean
324    final Path parent = new Path(testDir, "parent");
325    Path file = new Path(parent, "someFile");
326    fs.mkdirs(parent);
327    // touch a new file
328    fs.create(file).close();
329    assertTrue("Test file didn't get created.", fs.exists(file));
330    final Path racyFile = new Path(parent, "addedFile");
331
332    // when we attempt to delete the original file, add another file in the same directory
333    Mockito.doAnswer(new Answer<Boolean>() {
334      @Override
335      public Boolean answer(InvocationOnMock invocation) throws Throwable {
336        fs.create(racyFile).close();
337        FSUtils.logFileSystemState(fs, testDir, LOG);
338        return (Boolean) invocation.callRealMethod();
339      }
340    }).when(spy).isFileDeletable(Mockito.any());
341
342    // run the chore
343    chore.chore();
344
345    // make sure all the directories + added file exist, but the original file is deleted
346    assertTrue("Added file unexpectedly deleted", fs.exists(racyFile));
347    assertTrue("Parent directory deleted unexpectedly", fs.exists(parent));
348    assertFalse("Original file unexpectedly retained", fs.exists(file));
349    Mockito.verify(spy, Mockito.times(1)).isFileDeletable(Mockito.any());
350  }
351
352  @Test
353  public void testDeleteFileWithCleanerEnabled() throws Exception {
354    Stoppable stop = new StoppableImplementation();
355    Configuration conf = UTIL.getConfiguration();
356    Path testDir = UTIL.getDataTestDir();
357    FileSystem fs = UTIL.getTestFileSystem();
358    String confKey = "hbase.test.cleaner.delegates";
359    conf.set(confKey, AlwaysDelete.class.getName());
360
361    AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey);
362
363    // Enable cleaner
364    chore.setEnabled(true);
365
366    // create the directory layout in the directory to clean
367    Path parent = new Path(testDir, "parent");
368    Path child = new Path(parent, "child");
369    Path file = new Path(child, "someFile");
370    fs.mkdirs(child);
371
372    // touch a new file
373    fs.create(file).close();
374    assertTrue("Test file didn't get created.", fs.exists(file));
375
376    // run the chore
377    chore.chore();
378
379    // verify all the files got deleted
380    assertFalse("File didn't get deleted", fs.exists(file));
381    assertFalse("Empty directory didn't get deleted", fs.exists(child));
382    assertFalse("Empty directory didn't get deleted", fs.exists(parent));
383  }
384
385  @Test
386  public void testDeleteFileWithCleanerDisabled() throws Exception {
387    Stoppable stop = new StoppableImplementation();
388    Configuration conf = UTIL.getConfiguration();
389    Path testDir = UTIL.getDataTestDir();
390    FileSystem fs = UTIL.getTestFileSystem();
391    String confKey = "hbase.test.cleaner.delegates";
392    conf.set(confKey, AlwaysDelete.class.getName());
393
394    AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey);
395
396    // Disable cleaner
397    chore.setEnabled(false);
398
399    // create the directory layout in the directory to clean
400    Path parent = new Path(testDir, "parent");
401    Path child = new Path(parent, "child");
402    Path file = new Path(child, "someFile");
403    fs.mkdirs(child);
404
405    // touch a new file
406    fs.create(file).close();
407    assertTrue("Test file didn't get created.", fs.exists(file));
408
409    // run the chore
410    chore.chore();
411
412    // verify all the files exist
413    assertTrue("File got deleted with cleaner disabled", fs.exists(file));
414    assertTrue("Directory got deleted", fs.exists(child));
415    assertTrue("Directory got deleted", fs.exists(parent));
416  }
417
418  @Test
419  public void testOnConfigurationChange() throws Exception {
420    int availableProcessorNum = Runtime.getRuntime().availableProcessors();
421    if (availableProcessorNum == 1) { // no need to run this test
422      return;
423    }
424
425    // have at least 2 available processors/cores
426    int    initPoolSize = availableProcessorNum / 2;
427    int changedPoolSize = availableProcessorNum;
428
429    Stoppable stop = new StoppableImplementation();
430    Configuration conf = UTIL.getConfiguration();
431    Path testDir = UTIL.getDataTestDir();
432    FileSystem fs = UTIL.getTestFileSystem();
433    String confKey = "hbase.test.cleaner.delegates";
434    conf.set(confKey, AlwaysDelete.class.getName());
435    conf.set(CleanerChore.CHORE_POOL_SIZE, String.valueOf(initPoolSize));
436    AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey);
437    chore.setEnabled(true);
438    // Create subdirs under testDir
439    int dirNums = 6;
440    Path[] subdirs = new Path[dirNums];
441    for (int i = 0; i < dirNums; i++) {
442      subdirs[i] = new Path(testDir, "subdir-" + i);
443      fs.mkdirs(subdirs[i]);
444    }
445    // Under each subdirs create 6 files
446    for (Path subdir : subdirs) {
447      createFiles(fs, subdir, 6);
448    }
449    // Start chore
450    Thread t = new Thread(() -> chore.chore());
451    t.setDaemon(true);
452    t.start();
453    // Change size of chore's pool
454    conf.set(CleanerChore.CHORE_POOL_SIZE, String.valueOf(changedPoolSize));
455    chore.onConfigurationChange(conf);
456    assertEquals(changedPoolSize, chore.getChorePoolSize());
457    // Stop chore
458    t.join();
459  }
460
461  @Test
462  public void testMinimumNumberOfThreads() throws Exception {
463    Stoppable stop = new StoppableImplementation();
464    Configuration conf = UTIL.getConfiguration();
465    Path testDir = UTIL.getDataTestDir();
466    FileSystem fs = UTIL.getTestFileSystem();
467    String confKey = "hbase.test.cleaner.delegates";
468    conf.set(confKey, AlwaysDelete.class.getName());
469    conf.set(CleanerChore.CHORE_POOL_SIZE, "2");
470    AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey);
471    int numProcs = Runtime.getRuntime().availableProcessors();
472    // Sanity
473    assertEquals(numProcs, chore.calculatePoolSize(Integer.toString(numProcs)));
474    // The implementation does not allow us to set more threads than we have processors
475    assertEquals(numProcs, chore.calculatePoolSize(Integer.toString(numProcs + 2)));
476    // Force us into the branch that is multiplying 0.0 against the number of processors
477    assertEquals(1, chore.calculatePoolSize("0.0"));
478  }
479
480  private void createFiles(FileSystem fs, Path parentDir, int numOfFiles) throws IOException {
481    Random random = new Random();
482    for (int i = 0; i < numOfFiles; i++) {
483      int xMega = 1 + random.nextInt(3); // size of each file is between 1~3M
484      try (FSDataOutputStream fsdos = fs.create(new Path(parentDir, "file-" + i))) {
485        for (int m = 0; m < xMega; m++) {
486          byte[] M = new byte[1024 * 1024];
487          random.nextBytes(M);
488          fsdos.write(M);
489        }
490      }
491    }
492  }
493
494  private static class AllValidPaths extends CleanerChore<BaseHFileCleanerDelegate> {
495
496    public AllValidPaths(String name, Stoppable s, Configuration conf, FileSystem fs,
497        Path oldFileDir, String confkey) {
498      super(name, Integer.MAX_VALUE, s, conf, fs, oldFileDir, confkey);
499    }
500
501    // all paths are valid
502    @Override
503    protected boolean validate(Path file) {
504      return true;
505    }
506  };
507
508  public static class AlwaysDelete extends BaseHFileCleanerDelegate {
509    @Override
510    public boolean isFileDeletable(FileStatus fStat) {
511      return true;
512    }
513  }
514
515  public static class NeverDelete extends BaseHFileCleanerDelegate {
516    @Override
517    public boolean isFileDeletable(FileStatus fStat) {
518      return false;
519    }
520  }
521}