001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.master.cleaner; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertFalse; 022import static org.junit.Assert.assertTrue; 023 024import java.io.IOException; 025import java.util.List; 026import java.util.Random; 027import org.apache.hadoop.conf.Configuration; 028import org.apache.hadoop.fs.FSDataOutputStream; 029import org.apache.hadoop.fs.FileStatus; 030import org.apache.hadoop.fs.FileSystem; 031import org.apache.hadoop.fs.Path; 032import org.apache.hadoop.hbase.ChoreService; 033import org.apache.hadoop.hbase.CoordinatedStateManager; 034import org.apache.hadoop.hbase.HBaseClassTestRule; 035import org.apache.hadoop.hbase.HBaseTestingUtility; 036import org.apache.hadoop.hbase.HConstants; 037import org.apache.hadoop.hbase.HRegionInfo; 038import org.apache.hadoop.hbase.Server; 039import org.apache.hadoop.hbase.ServerName; 040import org.apache.hadoop.hbase.TableName; 041import org.apache.hadoop.hbase.client.ClusterConnection; 042import org.apache.hadoop.hbase.client.Connection; 043import org.apache.hadoop.hbase.mob.ManualMobMaintHFileCleaner; 044import org.apache.hadoop.hbase.mob.MobUtils; 045import org.apache.hadoop.hbase.testclassification.MasterTests; 046import org.apache.hadoop.hbase.testclassification.MediumTests; 047import org.apache.hadoop.hbase.util.EnvironmentEdge; 048import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; 049import org.apache.hadoop.hbase.util.HFileArchiveUtil; 050import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; 051import org.apache.hadoop.hbase.zookeeper.ZKWatcher; 052import org.junit.AfterClass; 053import org.junit.Assert; 054import org.junit.BeforeClass; 055import org.junit.ClassRule; 056import org.junit.Test; 057import org.junit.experimental.categories.Category; 058import org.slf4j.Logger; 059import org.slf4j.LoggerFactory; 060 061@Category({MasterTests.class, MediumTests.class}) 062public class TestHFileCleaner { 063 064 @ClassRule 065 public static final HBaseClassTestRule CLASS_RULE = 066 HBaseClassTestRule.forClass(TestHFileCleaner.class); 067 068 private static final Logger LOG = LoggerFactory.getLogger(TestHFileCleaner.class); 069 070 private final static HBaseTestingUtility UTIL = new HBaseTestingUtility(); 071 072 private static DirScanPool POOL; 073 074 @BeforeClass 075 public static void setupCluster() throws Exception { 076 // have to use a minidfs cluster because the localfs doesn't modify file times correctly 077 UTIL.startMiniDFSCluster(1); 078 POOL = new DirScanPool(UTIL.getConfiguration()); 079 } 080 081 @AfterClass 082 public static void shutdownCluster() throws IOException { 083 UTIL.shutdownMiniDFSCluster(); 084 POOL.shutdownNow(); 085 } 086 087 @Test 088 public void testTTLCleaner() throws IOException, InterruptedException { 089 FileSystem fs = UTIL.getDFSCluster().getFileSystem(); 090 Path root = UTIL.getDataTestDirOnTestFS(); 091 Path file = new Path(root, "file"); 092 fs.createNewFile(file); 093 long createTime = System.currentTimeMillis(); 094 assertTrue("Test file not created!", fs.exists(file)); 095 TimeToLiveHFileCleaner cleaner = new TimeToLiveHFileCleaner(); 096 // update the time info for the file, so the cleaner removes it 097 fs.setTimes(file, createTime - 100, -1); 098 Configuration conf = UTIL.getConfiguration(); 099 conf.setLong(TimeToLiveHFileCleaner.TTL_CONF_KEY, 100); 100 cleaner.setConf(conf); 101 assertTrue("File not set deletable - check mod time:" + getFileStats(file, fs) 102 + " with create time:" + createTime, cleaner.isFileDeletable(fs.getFileStatus(file))); 103 } 104 105 @Test 106 public void testManualMobCleanerStopsMobRemoval() throws IOException { 107 FileSystem fs = UTIL.getDFSCluster().getFileSystem(); 108 Path root = UTIL.getDataTestDirOnTestFS(); 109 TableName table = TableName.valueOf("testManualMobCleanerStopsMobRemoval"); 110 Path mob = HFileArchiveUtil.getRegionArchiveDir(root, table, 111 MobUtils.getMobRegionInfo(table).getEncodedName()); 112 Path family= new Path(mob, "family"); 113 114 Path file = new Path(family, "someHFileThatWouldBeAUUID"); 115 fs.createNewFile(file); 116 assertTrue("Test file not created!", fs.exists(file)); 117 118 ManualMobMaintHFileCleaner cleaner = new ManualMobMaintHFileCleaner(); 119 120 assertFalse("Mob File shouldn't have been deletable. check path. '"+file+"'", 121 cleaner.isFileDeletable(fs.getFileStatus(file))); 122 } 123 124 @Test 125 public void testManualMobCleanerLetsNonMobGo() throws IOException { 126 FileSystem fs = UTIL.getDFSCluster().getFileSystem(); 127 Path root = UTIL.getDataTestDirOnTestFS(); 128 TableName table = TableName.valueOf("testManualMobCleanerLetsNonMobGo"); 129 Path nonmob = HFileArchiveUtil.getRegionArchiveDir(root, table, 130 new HRegionInfo(table).getEncodedName()); 131 Path family= new Path(nonmob, "family"); 132 133 Path file = new Path(family, "someHFileThatWouldBeAUUID"); 134 fs.createNewFile(file); 135 assertTrue("Test file not created!", fs.exists(file)); 136 137 ManualMobMaintHFileCleaner cleaner = new ManualMobMaintHFileCleaner(); 138 139 assertTrue("Non-Mob File should have been deletable. check path. '"+file+"'", 140 cleaner.isFileDeletable(fs.getFileStatus(file))); 141 } 142 143 /** 144 * @param file to check 145 * @return loggable information about the file 146 */ 147 private String getFileStats(Path file, FileSystem fs) throws IOException { 148 FileStatus status = fs.getFileStatus(file); 149 return "File" + file + ", mtime:" + status.getModificationTime() + ", atime:" 150 + status.getAccessTime(); 151 } 152 153 @Test 154 public void testHFileCleaning() throws Exception { 155 final EnvironmentEdge originalEdge = EnvironmentEdgeManager.getDelegate(); 156 String prefix = "someHFileThatWouldBeAUUID"; 157 Configuration conf = UTIL.getConfiguration(); 158 // set TTL 159 long ttl = 2000; 160 conf.set(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, 161 "org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner," + 162 "org.apache.hadoop.hbase.mob.ManualMobMaintHFileCleaner"); 163 conf.setLong(TimeToLiveHFileCleaner.TTL_CONF_KEY, ttl); 164 Server server = new DummyServer(); 165 Path archivedHfileDir = 166 new Path(UTIL.getDataTestDirOnTestFS(), HConstants.HFILE_ARCHIVE_DIRECTORY); 167 FileSystem fs = FileSystem.get(conf); 168 HFileCleaner cleaner = new HFileCleaner(1000, server, conf, fs, archivedHfileDir, POOL); 169 170 // Create 2 invalid files, 1 "recent" file, 1 very new file and 30 old files 171 final long createTime = System.currentTimeMillis(); 172 fs.delete(archivedHfileDir, true); 173 fs.mkdirs(archivedHfileDir); 174 // Case 1: 1 invalid file, which should be deleted directly 175 fs.createNewFile(new Path(archivedHfileDir, "dfd-dfd")); 176 // Case 2: 1 "recent" file, not even deletable for the first log cleaner 177 // (TimeToLiveLogCleaner), so we are not going down the chain 178 LOG.debug("Now is: " + createTime); 179 for (int i = 1; i < 32; i++) { 180 // Case 3: old files which would be deletable for the first log cleaner 181 // (TimeToLiveHFileCleaner), 182 Path fileName = new Path(archivedHfileDir, (prefix + "." + (createTime + i))); 183 fs.createNewFile(fileName); 184 // set the creation time past ttl to ensure that it gets removed 185 fs.setTimes(fileName, createTime - ttl - 1, -1); 186 LOG.debug("Creating " + getFileStats(fileName, fs)); 187 } 188 189 // Case 2: 1 newer file, not even deletable for the first log cleaner 190 // (TimeToLiveLogCleaner), so we are not going down the chain 191 Path saved = new Path(archivedHfileDir, prefix + ".00000000000"); 192 fs.createNewFile(saved); 193 // set creation time within the ttl 194 fs.setTimes(saved, createTime - ttl / 2, -1); 195 LOG.debug("Creating " + getFileStats(saved, fs)); 196 for (FileStatus stat : fs.listStatus(archivedHfileDir)) { 197 LOG.debug(stat.getPath().toString()); 198 } 199 200 assertEquals(33, fs.listStatus(archivedHfileDir).length); 201 202 // set a custom edge manager to handle time checking 203 EnvironmentEdge setTime = new EnvironmentEdge() { 204 @Override 205 public long currentTime() { 206 return createTime; 207 } 208 }; 209 EnvironmentEdgeManager.injectEdge(setTime); 210 211 // run the chore 212 cleaner.chore(); 213 214 // ensure we only end up with the saved file 215 assertEquals(1, fs.listStatus(archivedHfileDir).length); 216 217 for (FileStatus file : fs.listStatus(archivedHfileDir)) { 218 LOG.debug("Kept hfiles: " + file.getPath().getName()); 219 } 220 221 // reset the edge back to the original edge 222 EnvironmentEdgeManager.injectEdge(originalEdge); 223 } 224 225 @Test 226 public void testRemovesEmptyDirectories() throws Exception { 227 Configuration conf = UTIL.getConfiguration(); 228 // no cleaner policies = delete all files 229 conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, ""); 230 Server server = new DummyServer(); 231 Path archivedHfileDir = 232 new Path(UTIL.getDataTestDirOnTestFS(), HConstants.HFILE_ARCHIVE_DIRECTORY); 233 234 // setup the cleaner 235 FileSystem fs = UTIL.getDFSCluster().getFileSystem(); 236 HFileCleaner cleaner = new HFileCleaner(1000, server, conf, fs, archivedHfileDir, POOL); 237 238 // make all the directories for archiving files 239 Path table = new Path(archivedHfileDir, "table"); 240 Path region = new Path(table, "regionsomthing"); 241 Path family = new Path(region, "fam"); 242 Path file = new Path(family, "file12345"); 243 fs.mkdirs(family); 244 if (!fs.exists(family)) throw new RuntimeException("Couldn't create test family:" + family); 245 fs.create(file).close(); 246 if (!fs.exists(file)) throw new RuntimeException("Test file didn't get created:" + file); 247 248 // run the chore to cleanup the files (and the directories above it) 249 cleaner.chore(); 250 251 // make sure all the parent directories get removed 252 assertFalse("family directory not removed for empty directory", fs.exists(family)); 253 assertFalse("region directory not removed for empty directory", fs.exists(region)); 254 assertFalse("table directory not removed for empty directory", fs.exists(table)); 255 assertTrue("archive directory", fs.exists(archivedHfileDir)); 256 } 257 258 static class DummyServer implements Server { 259 @Override 260 public Configuration getConfiguration() { 261 return UTIL.getConfiguration(); 262 } 263 264 @Override 265 public ZKWatcher getZooKeeper() { 266 try { 267 return new ZKWatcher(getConfiguration(), "dummy server", this); 268 } catch (IOException e) { 269 e.printStackTrace(); 270 } 271 return null; 272 } 273 274 @Override 275 public CoordinatedStateManager getCoordinatedStateManager() { 276 return null; 277 } 278 279 @Override 280 public ClusterConnection getConnection() { 281 return null; 282 } 283 284 @Override 285 public MetaTableLocator getMetaTableLocator() { 286 return null; 287 } 288 289 @Override 290 public ServerName getServerName() { 291 return ServerName.valueOf("regionserver,60020,000000"); 292 } 293 294 @Override 295 public void abort(String why, Throwable e) { 296 } 297 298 @Override 299 public boolean isAborted() { 300 return false; 301 } 302 303 @Override 304 public void stop(String why) { 305 } 306 307 @Override 308 public boolean isStopped() { 309 return false; 310 } 311 312 @Override 313 public ChoreService getChoreService() { 314 return null; 315 } 316 317 @Override 318 public ClusterConnection getClusterConnection() { 319 // TODO Auto-generated method stub 320 return null; 321 } 322 323 @Override 324 public FileSystem getFileSystem() { 325 return null; 326 } 327 328 @Override 329 public boolean isStopping() { 330 return false; 331 } 332 333 @Override 334 public Connection createConnection(Configuration conf) throws IOException { 335 return null; 336 } 337 } 338 339 @Test 340 public void testThreadCleanup() throws Exception { 341 Configuration conf = UTIL.getConfiguration(); 342 conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, ""); 343 Server server = new DummyServer(); 344 Path archivedHfileDir = 345 new Path(UTIL.getDataTestDirOnTestFS(), HConstants.HFILE_ARCHIVE_DIRECTORY); 346 347 // setup the cleaner 348 FileSystem fs = UTIL.getDFSCluster().getFileSystem(); 349 HFileCleaner cleaner = new HFileCleaner(1000, server, conf, fs, archivedHfileDir, POOL); 350 // clean up archive directory 351 fs.delete(archivedHfileDir, true); 352 fs.mkdirs(archivedHfileDir); 353 // create some file to delete 354 fs.createNewFile(new Path(archivedHfileDir, "dfd-dfd")); 355 // launch the chore 356 cleaner.chore(); 357 // call cleanup 358 cleaner.cleanup(); 359 // wait awhile for thread to die 360 Thread.sleep(100); 361 for (Thread thread : cleaner.getCleanerThreads()) { 362 Assert.assertFalse(thread.isAlive()); 363 } 364 } 365 366 @Test 367 public void testLargeSmallIsolation() throws Exception { 368 Configuration conf = UTIL.getConfiguration(); 369 // no cleaner policies = delete all files 370 conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, ""); 371 conf.setInt(HFileCleaner.HFILE_DELETE_THROTTLE_THRESHOLD, 512 * 1024); 372 Server server = new DummyServer(); 373 Path archivedHfileDir = 374 new Path(UTIL.getDataTestDirOnTestFS(), HConstants.HFILE_ARCHIVE_DIRECTORY); 375 376 // setup the cleaner 377 FileSystem fs = UTIL.getDFSCluster().getFileSystem(); 378 HFileCleaner cleaner = new HFileCleaner(1000, server, conf, fs, archivedHfileDir, POOL); 379 // clean up archive directory 380 fs.delete(archivedHfileDir, true); 381 fs.mkdirs(archivedHfileDir); 382 // necessary set up 383 final int LARGE_FILE_NUM = 5; 384 final int SMALL_FILE_NUM = 20; 385 createFilesForTesting(LARGE_FILE_NUM, SMALL_FILE_NUM, fs, archivedHfileDir); 386 // call cleanup 387 cleaner.chore(); 388 389 Assert.assertEquals(LARGE_FILE_NUM, cleaner.getNumOfDeletedLargeFiles()); 390 Assert.assertEquals(SMALL_FILE_NUM, cleaner.getNumOfDeletedSmallFiles()); 391 } 392 393 @Test 394 public void testOnConfigurationChange() throws Exception { 395 // constants 396 final int ORIGINAL_THROTTLE_POINT = 512 * 1024; 397 final int ORIGINAL_QUEUE_INIT_SIZE = 512; 398 final int UPDATE_THROTTLE_POINT = 1024;// small enough to change large/small check 399 final int UPDATE_QUEUE_INIT_SIZE = 1024; 400 final int LARGE_FILE_NUM = 5; 401 final int SMALL_FILE_NUM = 20; 402 final int LARGE_THREAD_NUM = 2; 403 final int SMALL_THREAD_NUM = 4; 404 final long THREAD_TIMEOUT_MSEC = 30 * 1000L; 405 final long THREAD_CHECK_INTERVAL_MSEC = 500L; 406 407 Configuration conf = UTIL.getConfiguration(); 408 // no cleaner policies = delete all files 409 conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, ""); 410 conf.setInt(HFileCleaner.HFILE_DELETE_THROTTLE_THRESHOLD, ORIGINAL_THROTTLE_POINT); 411 conf.setInt(HFileCleaner.LARGE_HFILE_QUEUE_INIT_SIZE, ORIGINAL_QUEUE_INIT_SIZE); 412 conf.setInt(HFileCleaner.SMALL_HFILE_QUEUE_INIT_SIZE, ORIGINAL_QUEUE_INIT_SIZE); 413 Server server = new DummyServer(); 414 Path archivedHfileDir = 415 new Path(UTIL.getDataTestDirOnTestFS(), HConstants.HFILE_ARCHIVE_DIRECTORY); 416 417 // setup the cleaner 418 FileSystem fs = UTIL.getDFSCluster().getFileSystem(); 419 final HFileCleaner cleaner = new HFileCleaner(1000, server, conf, fs, archivedHfileDir, POOL); 420 Assert.assertEquals(ORIGINAL_THROTTLE_POINT, cleaner.getThrottlePoint()); 421 Assert.assertEquals(ORIGINAL_QUEUE_INIT_SIZE, cleaner.getLargeQueueInitSize()); 422 Assert.assertEquals(ORIGINAL_QUEUE_INIT_SIZE, cleaner.getSmallQueueInitSize()); 423 Assert.assertEquals(HFileCleaner.DEFAULT_HFILE_DELETE_THREAD_TIMEOUT_MSEC, 424 cleaner.getCleanerThreadTimeoutMsec()); 425 Assert.assertEquals(HFileCleaner.DEFAULT_HFILE_DELETE_THREAD_CHECK_INTERVAL_MSEC, 426 cleaner.getCleanerThreadCheckIntervalMsec()); 427 428 // clean up archive directory and create files for testing 429 fs.delete(archivedHfileDir, true); 430 fs.mkdirs(archivedHfileDir); 431 createFilesForTesting(LARGE_FILE_NUM, SMALL_FILE_NUM, fs, archivedHfileDir); 432 433 // call cleaner, run as daemon to test the interrupt-at-middle case 434 Thread t = new Thread() { 435 @Override 436 public void run() { 437 cleaner.chore(); 438 } 439 }; 440 t.setDaemon(true); 441 t.start(); 442 // wait until file clean started 443 while (cleaner.getNumOfDeletedSmallFiles() == 0) { 444 Thread.yield(); 445 } 446 447 // trigger configuration change 448 Configuration newConf = new Configuration(conf); 449 newConf.setInt(HFileCleaner.HFILE_DELETE_THROTTLE_THRESHOLD, UPDATE_THROTTLE_POINT); 450 newConf.setInt(HFileCleaner.LARGE_HFILE_QUEUE_INIT_SIZE, UPDATE_QUEUE_INIT_SIZE); 451 newConf.setInt(HFileCleaner.SMALL_HFILE_QUEUE_INIT_SIZE, UPDATE_QUEUE_INIT_SIZE); 452 newConf.setInt(HFileCleaner.LARGE_HFILE_DELETE_THREAD_NUMBER, LARGE_THREAD_NUM); 453 newConf.setInt(HFileCleaner.SMALL_HFILE_DELETE_THREAD_NUMBER, SMALL_THREAD_NUM); 454 newConf.setLong(HFileCleaner.HFILE_DELETE_THREAD_TIMEOUT_MSEC, THREAD_TIMEOUT_MSEC); 455 newConf.setLong(HFileCleaner.HFILE_DELETE_THREAD_CHECK_INTERVAL_MSEC, 456 THREAD_CHECK_INTERVAL_MSEC); 457 458 LOG.debug("File deleted from large queue: " + cleaner.getNumOfDeletedLargeFiles() 459 + "; from small queue: " + cleaner.getNumOfDeletedSmallFiles()); 460 cleaner.onConfigurationChange(newConf); 461 462 // check values after change 463 Assert.assertEquals(UPDATE_THROTTLE_POINT, cleaner.getThrottlePoint()); 464 Assert.assertEquals(UPDATE_QUEUE_INIT_SIZE, cleaner.getLargeQueueInitSize()); 465 Assert.assertEquals(UPDATE_QUEUE_INIT_SIZE, cleaner.getSmallQueueInitSize()); 466 Assert.assertEquals(LARGE_THREAD_NUM + SMALL_THREAD_NUM, cleaner.getCleanerThreads().size()); 467 Assert.assertEquals(THREAD_TIMEOUT_MSEC, cleaner.getCleanerThreadTimeoutMsec()); 468 Assert.assertEquals(THREAD_CHECK_INTERVAL_MSEC, cleaner.getCleanerThreadCheckIntervalMsec()); 469 470 // make sure no cost when onConfigurationChange called with no change 471 List<Thread> oldThreads = cleaner.getCleanerThreads(); 472 cleaner.onConfigurationChange(newConf); 473 List<Thread> newThreads = cleaner.getCleanerThreads(); 474 Assert.assertArrayEquals(oldThreads.toArray(), newThreads.toArray()); 475 476 // wait until clean done and check 477 t.join(); 478 LOG.debug("File deleted from large queue: " + cleaner.getNumOfDeletedLargeFiles() 479 + "; from small queue: " + cleaner.getNumOfDeletedSmallFiles()); 480 Assert.assertTrue("Should delete more than " + LARGE_FILE_NUM 481 + " files from large queue but actually " + cleaner.getNumOfDeletedLargeFiles(), 482 cleaner.getNumOfDeletedLargeFiles() > LARGE_FILE_NUM); 483 Assert.assertTrue("Should delete less than " + SMALL_FILE_NUM 484 + " files from small queue but actually " + cleaner.getNumOfDeletedSmallFiles(), 485 cleaner.getNumOfDeletedSmallFiles() < SMALL_FILE_NUM); 486 } 487 488 private void createFilesForTesting(int largeFileNum, int smallFileNum, FileSystem fs, 489 Path archivedHfileDir) throws IOException { 490 final Random rand = new Random(); 491 final byte[] large = new byte[1024 * 1024]; 492 for (int i = 0; i < large.length; i++) { 493 large[i] = (byte) rand.nextInt(128); 494 } 495 final byte[] small = new byte[1024]; 496 for (int i = 0; i < small.length; i++) { 497 small[i] = (byte) rand.nextInt(128); 498 } 499 // create large and small files 500 for (int i = 1; i <= largeFileNum; i++) { 501 FSDataOutputStream out = fs.create(new Path(archivedHfileDir, "large-file-" + i)); 502 out.write(large); 503 out.close(); 504 } 505 for (int i = 1; i <= smallFileNum; i++) { 506 FSDataOutputStream out = fs.create(new Path(archivedHfileDir, "small-file-" + i)); 507 out.write(small); 508 out.close(); 509 } 510 } 511}