001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.master.cleaner; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertFalse; 022import static org.junit.Assert.assertTrue; 023 024import java.io.IOException; 025import java.util.Random; 026import java.util.concurrent.atomic.AtomicBoolean; 027import org.apache.hadoop.conf.Configuration; 028import org.apache.hadoop.fs.FSDataOutputStream; 029import org.apache.hadoop.fs.FileStatus; 030import org.apache.hadoop.fs.FileSystem; 031import org.apache.hadoop.fs.FilterFileSystem; 032import org.apache.hadoop.fs.Path; 033import org.apache.hadoop.hbase.HBaseClassTestRule; 034import org.apache.hadoop.hbase.HBaseTestingUtility; 035import org.apache.hadoop.hbase.Stoppable; 036import org.apache.hadoop.hbase.testclassification.MasterTests; 037import org.apache.hadoop.hbase.testclassification.SmallTests; 038import org.apache.hadoop.hbase.util.FSUtils; 039import org.apache.hadoop.hbase.util.StoppableImplementation; 040import org.junit.AfterClass; 041import org.junit.BeforeClass; 042import org.junit.ClassRule; 043import org.junit.Test; 044import org.junit.experimental.categories.Category; 045import org.mockito.Mockito; 046import org.mockito.invocation.InvocationOnMock; 047import org.mockito.stubbing.Answer; 048import org.slf4j.Logger; 049import org.slf4j.LoggerFactory; 050 051@Category({MasterTests.class, SmallTests.class}) 052public class TestCleanerChore { 053 054 @ClassRule 055 public static final HBaseClassTestRule CLASS_RULE = 056 HBaseClassTestRule.forClass(TestCleanerChore.class); 057 058 private static final Logger LOG = LoggerFactory.getLogger(TestCleanerChore.class); 059 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); 060 private static DirScanPool POOL; 061 062 @BeforeClass 063 public static void setup() { 064 POOL = new DirScanPool(UTIL.getConfiguration()); 065 } 066 067 @AfterClass 068 public static void cleanup() throws Exception { 069 // delete and recreate the test directory, ensuring a clean test dir between tests 070 UTIL.cleanupTestDir(); 071 POOL.shutdownNow(); 072 } 073 074 @Test 075 public void testSavesFilesOnRequest() throws Exception { 076 Stoppable stop = new StoppableImplementation(); 077 Configuration conf = UTIL.getConfiguration(); 078 Path testDir = UTIL.getDataTestDir(); 079 FileSystem fs = UTIL.getTestFileSystem(); 080 String confKey = "hbase.test.cleaner.delegates"; 081 conf.set(confKey, NeverDelete.class.getName()); 082 083 AllValidPaths chore = 084 new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey, POOL); 085 086 // create the directory layout in the directory to clean 087 Path parent = new Path(testDir, "parent"); 088 Path file = new Path(parent, "someFile"); 089 fs.mkdirs(parent); 090 // touch a new file 091 fs.create(file).close(); 092 assertTrue("Test file didn't get created.", fs.exists(file)); 093 094 // run the chore 095 chore.chore(); 096 097 // verify all the files were preserved 098 assertTrue("File shouldn't have been deleted", fs.exists(file)); 099 assertTrue("directory shouldn't have been deleted", fs.exists(parent)); 100 } 101 102 @Test 103 public void retriesIOExceptionInStatus() throws Exception { 104 Stoppable stop = new StoppableImplementation(); 105 Configuration conf = UTIL.getConfiguration(); 106 Path testDir = UTIL.getDataTestDir(); 107 FileSystem fs = UTIL.getTestFileSystem(); 108 String confKey = "hbase.test.cleaner.delegates"; 109 110 Path child = new Path(testDir, "child"); 111 Path file = new Path(child, "file"); 112 fs.mkdirs(child); 113 fs.create(file).close(); 114 assertTrue("test file didn't get created.", fs.exists(file)); 115 final AtomicBoolean fails = new AtomicBoolean(true); 116 117 FilterFileSystem filtered = new FilterFileSystem(fs) { 118 public FileStatus[] listStatus(Path f) throws IOException { 119 if (fails.get()) { 120 throw new IOException("whomp whomp."); 121 } 122 return fs.listStatus(f); 123 } 124 }; 125 126 AllValidPaths chore = 127 new AllValidPaths("test-retry-ioe", stop, conf, filtered, testDir, confKey, POOL); 128 129 // trouble talking to the filesystem 130 Boolean result = chore.runCleaner(); 131 132 // verify that it couldn't clean the files. 133 assertTrue("test rig failed to inject failure.", fs.exists(file)); 134 assertTrue("test rig failed to inject failure.", fs.exists(child)); 135 // and verify that it accurately reported the failure. 136 assertFalse("chore should report that it failed.", result); 137 138 // filesystem is back 139 fails.set(false); 140 result = chore.runCleaner(); 141 142 // verify everything is gone. 143 assertFalse("file should have been destroyed.", fs.exists(file)); 144 assertFalse("directory should have been destroyed.", fs.exists(child)); 145 // and verify that it accurately reported success. 146 assertTrue("chore should claim it succeeded.", result); 147 } 148 149 @Test 150 public void testDeletesEmptyDirectories() throws Exception { 151 Stoppable stop = new StoppableImplementation(); 152 Configuration conf = UTIL.getConfiguration(); 153 Path testDir = UTIL.getDataTestDir(); 154 FileSystem fs = UTIL.getTestFileSystem(); 155 String confKey = "hbase.test.cleaner.delegates"; 156 conf.set(confKey, AlwaysDelete.class.getName()); 157 158 AllValidPaths chore = 159 new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey, POOL); 160 161 // create the directory layout in the directory to clean 162 Path parent = new Path(testDir, "parent"); 163 Path child = new Path(parent, "child"); 164 Path emptyChild = new Path(parent, "emptyChild"); 165 Path file = new Path(child, "someFile"); 166 fs.mkdirs(child); 167 fs.mkdirs(emptyChild); 168 // touch a new file 169 fs.create(file).close(); 170 // also create a file in the top level directory 171 Path topFile = new Path(testDir, "topFile"); 172 fs.create(topFile).close(); 173 assertTrue("Test file didn't get created.", fs.exists(file)); 174 assertTrue("Test file didn't get created.", fs.exists(topFile)); 175 176 // run the chore 177 chore.chore(); 178 179 // verify all the files got deleted 180 assertFalse("File didn't get deleted", fs.exists(topFile)); 181 assertFalse("File didn't get deleted", fs.exists(file)); 182 assertFalse("Empty directory didn't get deleted", fs.exists(child)); 183 assertFalse("Empty directory didn't get deleted", fs.exists(parent)); 184 } 185 186 /** 187 * Test to make sure that we don't attempt to ask the delegate whether or not we should preserve a 188 * directory. 189 * @throws Exception on failure 190 */ 191 @Test 192 public void testDoesNotCheckDirectories() throws Exception { 193 Stoppable stop = new StoppableImplementation(); 194 Configuration conf = UTIL.getConfiguration(); 195 Path testDir = UTIL.getDataTestDir(); 196 FileSystem fs = UTIL.getTestFileSystem(); 197 String confKey = "hbase.test.cleaner.delegates"; 198 conf.set(confKey, AlwaysDelete.class.getName()); 199 200 AllValidPaths chore = 201 new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey, POOL); 202 // spy on the delegate to ensure that we don't check for directories 203 AlwaysDelete delegate = (AlwaysDelete) chore.cleanersChain.get(0); 204 AlwaysDelete spy = Mockito.spy(delegate); 205 chore.cleanersChain.set(0, spy); 206 207 // create the directory layout in the directory to clean 208 Path parent = new Path(testDir, "parent"); 209 Path file = new Path(parent, "someFile"); 210 fs.mkdirs(parent); 211 assertTrue("Test parent didn't get created.", fs.exists(parent)); 212 // touch a new file 213 fs.create(file).close(); 214 assertTrue("Test file didn't get created.", fs.exists(file)); 215 216 FileStatus fStat = fs.getFileStatus(parent); 217 chore.chore(); 218 // make sure we never checked the directory 219 Mockito.verify(spy, Mockito.never()).isFileDeletable(fStat); 220 Mockito.reset(spy); 221 } 222 223 @Test 224 public void testStoppedCleanerDoesNotDeleteFiles() throws Exception { 225 Stoppable stop = new StoppableImplementation(); 226 Configuration conf = UTIL.getConfiguration(); 227 Path testDir = UTIL.getDataTestDir(); 228 FileSystem fs = UTIL.getTestFileSystem(); 229 String confKey = "hbase.test.cleaner.delegates"; 230 conf.set(confKey, AlwaysDelete.class.getName()); 231 232 AllValidPaths chore = 233 new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey, POOL); 234 235 // also create a file in the top level directory 236 Path topFile = new Path(testDir, "topFile"); 237 fs.create(topFile).close(); 238 assertTrue("Test file didn't get created.", fs.exists(topFile)); 239 240 // stop the chore 241 stop.stop("testing stop"); 242 243 // run the chore 244 chore.chore(); 245 246 // test that the file still exists 247 assertTrue("File got deleted while chore was stopped", fs.exists(topFile)); 248 } 249 250 /** 251 * While cleaning a directory, all the files in the directory may be deleted, but there may be 252 * another file added, in which case the directory shouldn't be deleted. 253 * @throws IOException on failure 254 */ 255 @Test 256 public void testCleanerDoesNotDeleteDirectoryWithLateAddedFiles() throws IOException { 257 Stoppable stop = new StoppableImplementation(); 258 Configuration conf = UTIL.getConfiguration(); 259 final Path testDir = UTIL.getDataTestDir(); 260 final FileSystem fs = UTIL.getTestFileSystem(); 261 String confKey = "hbase.test.cleaner.delegates"; 262 conf.set(confKey, AlwaysDelete.class.getName()); 263 264 AllValidPaths chore = 265 new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey, POOL); 266 // spy on the delegate to ensure that we don't check for directories 267 AlwaysDelete delegate = (AlwaysDelete) chore.cleanersChain.get(0); 268 AlwaysDelete spy = Mockito.spy(delegate); 269 chore.cleanersChain.set(0, spy); 270 271 // create the directory layout in the directory to clean 272 final Path parent = new Path(testDir, "parent"); 273 Path file = new Path(parent, "someFile"); 274 fs.mkdirs(parent); 275 // touch a new file 276 fs.create(file).close(); 277 assertTrue("Test file didn't get created.", fs.exists(file)); 278 final Path addedFile = new Path(parent, "addedFile"); 279 280 // when we attempt to delete the original file, add another file in the same directory 281 Mockito.doAnswer(new Answer<Boolean>() { 282 @Override 283 public Boolean answer(InvocationOnMock invocation) throws Throwable { 284 fs.create(addedFile).close(); 285 FSUtils.logFileSystemState(fs, testDir, LOG); 286 return (Boolean) invocation.callRealMethod(); 287 } 288 }).when(spy).isFileDeletable(Mockito.any()); 289 290 // run the chore 291 chore.chore(); 292 293 // make sure all the directories + added file exist, but the original file is deleted 294 assertTrue("Added file unexpectedly deleted", fs.exists(addedFile)); 295 assertTrue("Parent directory deleted unexpectedly", fs.exists(parent)); 296 assertFalse("Original file unexpectedly retained", fs.exists(file)); 297 Mockito.verify(spy, Mockito.times(1)).isFileDeletable(Mockito.any()); 298 Mockito.reset(spy); 299 } 300 301 /** 302 * The cleaner runs in a loop, where it first checks to see all the files under a directory can be 303 * deleted. If they all can, then we try to delete the directory. However, a file may be added 304 * that directory to after the original check. This ensures that we don't accidentally delete that 305 * directory on and don't get spurious IOExceptions. 306 * <p> 307 * This was from HBASE-7465. 308 * @throws Exception on failure 309 */ 310 @Test 311 public void testNoExceptionFromDirectoryWithRacyChildren() throws Exception { 312 UTIL.cleanupTestDir(); 313 Stoppable stop = new StoppableImplementation(); 314 // need to use a localutil to not break the rest of the test that runs on the local FS, which 315 // gets hosed when we start to use a minicluster. 316 HBaseTestingUtility localUtil = new HBaseTestingUtility(); 317 Configuration conf = localUtil.getConfiguration(); 318 final Path testDir = UTIL.getDataTestDir(); 319 final FileSystem fs = UTIL.getTestFileSystem(); 320 LOG.debug("Writing test data to: " + testDir); 321 String confKey = "hbase.test.cleaner.delegates"; 322 conf.set(confKey, AlwaysDelete.class.getName()); 323 324 AllValidPaths chore = 325 new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey, POOL); 326 // spy on the delegate to ensure that we don't check for directories 327 AlwaysDelete delegate = (AlwaysDelete) chore.cleanersChain.get(0); 328 AlwaysDelete spy = Mockito.spy(delegate); 329 chore.cleanersChain.set(0, spy); 330 331 // create the directory layout in the directory to clean 332 final Path parent = new Path(testDir, "parent"); 333 Path file = new Path(parent, "someFile"); 334 fs.mkdirs(parent); 335 // touch a new file 336 fs.create(file).close(); 337 assertTrue("Test file didn't get created.", fs.exists(file)); 338 final Path racyFile = new Path(parent, "addedFile"); 339 340 // when we attempt to delete the original file, add another file in the same directory 341 Mockito.doAnswer(new Answer<Boolean>() { 342 @Override 343 public Boolean answer(InvocationOnMock invocation) throws Throwable { 344 fs.create(racyFile).close(); 345 FSUtils.logFileSystemState(fs, testDir, LOG); 346 return (Boolean) invocation.callRealMethod(); 347 } 348 }).when(spy).isFileDeletable(Mockito.any()); 349 350 // run the chore 351 chore.chore(); 352 353 // make sure all the directories + added file exist, but the original file is deleted 354 assertTrue("Added file unexpectedly deleted", fs.exists(racyFile)); 355 assertTrue("Parent directory deleted unexpectedly", fs.exists(parent)); 356 assertFalse("Original file unexpectedly retained", fs.exists(file)); 357 Mockito.verify(spy, Mockito.times(1)).isFileDeletable(Mockito.any()); 358 } 359 360 @Test 361 public void testDeleteFileWithCleanerEnabled() throws Exception { 362 Stoppable stop = new StoppableImplementation(); 363 Configuration conf = UTIL.getConfiguration(); 364 Path testDir = UTIL.getDataTestDir(); 365 FileSystem fs = UTIL.getTestFileSystem(); 366 String confKey = "hbase.test.cleaner.delegates"; 367 conf.set(confKey, AlwaysDelete.class.getName()); 368 369 AllValidPaths chore = 370 new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey, POOL); 371 372 // Enable cleaner 373 chore.setEnabled(true); 374 375 // create the directory layout in the directory to clean 376 Path parent = new Path(testDir, "parent"); 377 Path child = new Path(parent, "child"); 378 Path file = new Path(child, "someFile"); 379 fs.mkdirs(child); 380 381 // touch a new file 382 fs.create(file).close(); 383 assertTrue("Test file didn't get created.", fs.exists(file)); 384 385 // run the chore 386 chore.chore(); 387 388 // verify all the files got deleted 389 assertFalse("File didn't get deleted", fs.exists(file)); 390 assertFalse("Empty directory didn't get deleted", fs.exists(child)); 391 assertFalse("Empty directory didn't get deleted", fs.exists(parent)); 392 } 393 394 @Test 395 public void testDeleteFileWithCleanerDisabled() throws Exception { 396 Stoppable stop = new StoppableImplementation(); 397 Configuration conf = UTIL.getConfiguration(); 398 Path testDir = UTIL.getDataTestDir(); 399 FileSystem fs = UTIL.getTestFileSystem(); 400 String confKey = "hbase.test.cleaner.delegates"; 401 conf.set(confKey, AlwaysDelete.class.getName()); 402 403 AllValidPaths chore = 404 new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey, POOL); 405 406 // Disable cleaner 407 chore.setEnabled(false); 408 409 // create the directory layout in the directory to clean 410 Path parent = new Path(testDir, "parent"); 411 Path child = new Path(parent, "child"); 412 Path file = new Path(child, "someFile"); 413 fs.mkdirs(child); 414 415 // touch a new file 416 fs.create(file).close(); 417 assertTrue("Test file didn't get created.", fs.exists(file)); 418 419 // run the chore 420 chore.chore(); 421 422 // verify all the files exist 423 assertTrue("File got deleted with cleaner disabled", fs.exists(file)); 424 assertTrue("Directory got deleted", fs.exists(child)); 425 assertTrue("Directory got deleted", fs.exists(parent)); 426 } 427 428 @Test 429 public void testOnConfigurationChange() throws Exception { 430 int availableProcessorNum = Runtime.getRuntime().availableProcessors(); 431 if (availableProcessorNum == 1) { // no need to run this test 432 return; 433 } 434 435 // have at least 2 available processors/cores 436 int initPoolSize = availableProcessorNum / 2; 437 int changedPoolSize = availableProcessorNum; 438 439 Stoppable stop = new StoppableImplementation(); 440 Configuration conf = UTIL.getConfiguration(); 441 Path testDir = UTIL.getDataTestDir(); 442 FileSystem fs = UTIL.getTestFileSystem(); 443 String confKey = "hbase.test.cleaner.delegates"; 444 conf.set(confKey, AlwaysDelete.class.getName()); 445 conf.set(CleanerChore.CHORE_POOL_SIZE, String.valueOf(initPoolSize)); 446 AllValidPaths chore = 447 new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey, POOL); 448 chore.setEnabled(true); 449 // Create subdirs under testDir 450 int dirNums = 6; 451 Path[] subdirs = new Path[dirNums]; 452 for (int i = 0; i < dirNums; i++) { 453 subdirs[i] = new Path(testDir, "subdir-" + i); 454 fs.mkdirs(subdirs[i]); 455 } 456 // Under each subdirs create 6 files 457 for (Path subdir : subdirs) { 458 createFiles(fs, subdir, 6); 459 } 460 // Start chore 461 Thread t = new Thread(() -> chore.chore()); 462 t.setDaemon(true); 463 t.start(); 464 // Change size of chore's pool 465 conf.set(CleanerChore.CHORE_POOL_SIZE, String.valueOf(changedPoolSize)); 466 POOL.onConfigurationChange(conf); 467 assertEquals(changedPoolSize, chore.getChorePoolSize()); 468 // Stop chore 469 t.join(); 470 } 471 472 @Test 473 public void testMinimumNumberOfThreads() throws Exception { 474 Configuration conf = UTIL.getConfiguration(); 475 String confKey = "hbase.test.cleaner.delegates"; 476 conf.set(confKey, AlwaysDelete.class.getName()); 477 conf.set(CleanerChore.CHORE_POOL_SIZE, "2"); 478 int numProcs = Runtime.getRuntime().availableProcessors(); 479 // Sanity 480 assertEquals(numProcs, CleanerChore.calculatePoolSize(Integer.toString(numProcs))); 481 // The implementation does not allow us to set more threads than we have processors 482 assertEquals(numProcs, CleanerChore.calculatePoolSize(Integer.toString(numProcs + 2))); 483 // Force us into the branch that is multiplying 0.0 against the number of processors 484 assertEquals(1, CleanerChore.calculatePoolSize("0.0")); 485 } 486 487 private void createFiles(FileSystem fs, Path parentDir, int numOfFiles) throws IOException { 488 Random random = new Random(); 489 for (int i = 0; i < numOfFiles; i++) { 490 int xMega = 1 + random.nextInt(3); // size of each file is between 1~3M 491 try (FSDataOutputStream fsdos = fs.create(new Path(parentDir, "file-" + i))) { 492 for (int m = 0; m < xMega; m++) { 493 byte[] M = new byte[1024 * 1024]; 494 random.nextBytes(M); 495 fsdos.write(M); 496 } 497 } 498 } 499 } 500 501 private static class AllValidPaths extends CleanerChore<BaseHFileCleanerDelegate> { 502 503 public AllValidPaths(String name, Stoppable s, Configuration conf, FileSystem fs, 504 Path oldFileDir, String confkey, DirScanPool pool) { 505 super(name, Integer.MAX_VALUE, s, conf, fs, oldFileDir, confkey, pool); 506 } 507 508 // all paths are valid 509 @Override 510 protected boolean validate(Path file) { 511 return true; 512 } 513 }; 514 515 public static class AlwaysDelete extends BaseHFileCleanerDelegate { 516 @Override 517 public boolean isFileDeletable(FileStatus fStat) { 518 return true; 519 } 520 } 521 522 public static class NeverDelete extends BaseHFileCleanerDelegate { 523 @Override 524 public boolean isFileDeletable(FileStatus fStat) { 525 return false; 526 } 527 } 528}