001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.backup; 019 020import java.io.FileNotFoundException; 021import java.io.IOException; 022import java.io.InterruptedIOException; 023import java.util.ArrayList; 024import java.util.Arrays; 025import java.util.Collection; 026import java.util.Collections; 027import java.util.List; 028import java.util.concurrent.ExecutionException; 029import java.util.concurrent.Future; 030import java.util.concurrent.ThreadFactory; 031import java.util.concurrent.ThreadPoolExecutor; 032import java.util.concurrent.TimeUnit; 033import java.util.concurrent.atomic.AtomicInteger; 034 035import org.apache.hadoop.conf.Configuration; 036import org.apache.hadoop.fs.FileStatus; 037import org.apache.hadoop.fs.FileSystem; 038import org.apache.hadoop.fs.Path; 039import org.apache.hadoop.fs.PathFilter; 040import org.apache.hadoop.hbase.HConstants; 041import org.apache.hadoop.hbase.client.RegionInfo; 042import org.apache.hadoop.hbase.regionserver.HStoreFile; 043import org.apache.hadoop.hbase.util.Bytes; 044import org.apache.hadoop.hbase.util.CommonFSUtils; 045import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; 046import org.apache.hadoop.hbase.util.FSUtils; 047import org.apache.hadoop.hbase.util.HFileArchiveUtil; 048import org.apache.hadoop.hbase.util.Threads; 049import org.apache.hadoop.io.MultipleIOException; 050import org.apache.yetus.audience.InterfaceAudience; 051import org.slf4j.Logger; 052import org.slf4j.LoggerFactory; 053import org.apache.hbase.thirdparty.com.google.common.base.Function; 054import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; 055import org.apache.hbase.thirdparty.com.google.common.collect.Collections2; 056import org.apache.hbase.thirdparty.com.google.common.collect.Lists; 057 058/** 059 * Utility class to handle the removal of HFiles (or the respective {@link HStoreFile StoreFiles}) 060 * for a HRegion from the {@link FileSystem}. The hfiles will be archived or deleted, depending on 061 * the state of the system. 062 */ 063@InterfaceAudience.Private 064public class HFileArchiver { 065 private static final Logger LOG = LoggerFactory.getLogger(HFileArchiver.class); 066 private static final String SEPARATOR = "."; 067 068 /** Number of retries in case of fs operation failure */ 069 private static final int DEFAULT_RETRIES_NUMBER = 3; 070 071 private static final Function<File, Path> FUNC_FILE_TO_PATH = 072 new Function<File, Path>() { 073 @Override 074 public Path apply(File file) { 075 return file == null ? null : file.getPath(); 076 } 077 }; 078 079 private static ThreadPoolExecutor archiveExecutor; 080 081 private HFileArchiver() { 082 // hidden ctor since this is just a util 083 } 084 085 /** 086 * @return True if the Region exits in the filesystem. 087 */ 088 public static boolean exists(Configuration conf, FileSystem fs, RegionInfo info) 089 throws IOException { 090 Path rootDir = FSUtils.getRootDir(conf); 091 Path regionDir = FSUtils.getRegionDirFromRootDir(rootDir, info); 092 return fs.exists(regionDir); 093 } 094 095 /** 096 * Cleans up all the files for a HRegion by archiving the HFiles to the archive directory 097 * @param conf the configuration to use 098 * @param fs the file system object 099 * @param info RegionInfo for region to be deleted 100 */ 101 public static void archiveRegion(Configuration conf, FileSystem fs, RegionInfo info) 102 throws IOException { 103 Path rootDir = FSUtils.getRootDir(conf); 104 archiveRegion(fs, rootDir, FSUtils.getTableDir(rootDir, info.getTable()), 105 FSUtils.getRegionDirFromRootDir(rootDir, info)); 106 } 107 108 /** 109 * Remove an entire region from the table directory via archiving the region's hfiles. 110 * @param fs {@link FileSystem} from which to remove the region 111 * @param rootdir {@link Path} to the root directory where hbase files are stored (for building 112 * the archive path) 113 * @param tableDir {@link Path} to where the table is being stored (for building the archive path) 114 * @param regionDir {@link Path} to where a region is being stored (for building the archive path) 115 * @return <tt>true</tt> if the region was successfully deleted. <tt>false</tt> if the filesystem 116 * operations could not complete. 117 * @throws IOException if the request cannot be completed 118 */ 119 public static boolean archiveRegion(FileSystem fs, Path rootdir, Path tableDir, Path regionDir) 120 throws IOException { 121 // otherwise, we archive the files 122 // make sure we can archive 123 if (tableDir == null || regionDir == null) { 124 LOG.error("No archive directory could be found because tabledir (" + tableDir 125 + ") or regiondir (" + regionDir + "was null. Deleting files instead."); 126 if (regionDir != null) { 127 deleteRegionWithoutArchiving(fs, regionDir); 128 } 129 // we should have archived, but failed to. Doesn't matter if we deleted 130 // the archived files correctly or not. 131 return false; 132 } 133 134 LOG.debug("ARCHIVING {}", regionDir); 135 136 // make sure the regiondir lives under the tabledir 137 Preconditions.checkArgument(regionDir.toString().startsWith(tableDir.toString())); 138 Path regionArchiveDir = HFileArchiveUtil.getRegionArchiveDir(rootdir, 139 FSUtils.getTableName(tableDir), 140 regionDir.getName()); 141 142 FileStatusConverter getAsFile = new FileStatusConverter(fs); 143 // otherwise, we attempt to archive the store files 144 145 // build collection of just the store directories to archive 146 Collection<File> toArchive = new ArrayList<>(); 147 final PathFilter dirFilter = new FSUtils.DirFilter(fs); 148 PathFilter nonHidden = new PathFilter() { 149 @Override 150 public boolean accept(Path file) { 151 return dirFilter.accept(file) && !file.getName().startsWith("."); 152 } 153 }; 154 FileStatus[] storeDirs = FSUtils.listStatus(fs, regionDir, nonHidden); 155 // if there no files, we can just delete the directory and return; 156 if (storeDirs == null) { 157 LOG.debug("Directory {} empty.", regionDir); 158 return deleteRegionWithoutArchiving(fs, regionDir); 159 } 160 161 // convert the files in the region to a File 162 toArchive.addAll(Lists.transform(Arrays.asList(storeDirs), getAsFile)); 163 LOG.debug("Archiving " + toArchive); 164 List<File> failedArchive = resolveAndArchive(fs, regionArchiveDir, toArchive, 165 EnvironmentEdgeManager.currentTime()); 166 if (!failedArchive.isEmpty()) { 167 throw new FailedArchiveException("Failed to archive/delete all the files for region:" 168 + regionDir.getName() + " into " + regionArchiveDir 169 + ". Something is probably awry on the filesystem.", 170 Collections2.transform(failedArchive, FUNC_FILE_TO_PATH)); 171 } 172 // if that was successful, then we delete the region 173 return deleteRegionWithoutArchiving(fs, regionDir); 174 } 175 176 /** 177 * Archive the specified regions in parallel. 178 * @param conf the configuration to use 179 * @param fs {@link FileSystem} from which to remove the region 180 * @param rootDir {@link Path} to the root directory where hbase files are stored (for building 181 * the archive path) 182 * @param tableDir {@link Path} to where the table is being stored (for building the archive 183 * path) 184 * @param regionDirList {@link Path} to where regions are being stored (for building the archive 185 * path) 186 * @throws IOException if the request cannot be completed 187 */ 188 public static void archiveRegions(Configuration conf, FileSystem fs, Path rootDir, Path tableDir, 189 List<Path> regionDirList) throws IOException { 190 List<Future<Void>> futures = new ArrayList<>(regionDirList.size()); 191 for (Path regionDir: regionDirList) { 192 Future<Void> future = getArchiveExecutor(conf).submit(() -> { 193 archiveRegion(fs, rootDir, tableDir, regionDir); 194 return null; 195 }); 196 futures.add(future); 197 } 198 try { 199 for (Future<Void> future: futures) { 200 future.get(); 201 } 202 } catch (InterruptedException e) { 203 throw new InterruptedIOException(e.getMessage()); 204 } catch (ExecutionException e) { 205 throw new IOException(e.getCause()); 206 } 207 } 208 209 private static synchronized ThreadPoolExecutor getArchiveExecutor(final Configuration conf) { 210 if (archiveExecutor == null) { 211 int maxThreads = conf.getInt("hbase.hfilearchiver.thread.pool.max", 8); 212 archiveExecutor = Threads.getBoundedCachedThreadPool(maxThreads, 30L, TimeUnit.SECONDS, 213 getThreadFactory()); 214 215 // Shutdown this ThreadPool in a shutdown hook 216 Runtime.getRuntime().addShutdownHook(new Thread(() -> archiveExecutor.shutdown())); 217 } 218 return archiveExecutor; 219 } 220 221 // We need this method instead of Threads.getNamedThreadFactory() to pass some tests. 222 // The difference from Threads.getNamedThreadFactory() is that it doesn't fix ThreadGroup for 223 // new threads. If we use Threads.getNamedThreadFactory(), we will face ThreadGroup related 224 // issues in some tests. 225 private static ThreadFactory getThreadFactory() { 226 return new ThreadFactory() { 227 final AtomicInteger threadNumber = new AtomicInteger(1); 228 229 @Override 230 public Thread newThread(Runnable r) { 231 final String name = "HFileArchiver-" + threadNumber.getAndIncrement(); 232 Thread t = new Thread(r, name); 233 t.setDaemon(true); 234 return t; 235 } 236 }; 237 } 238 239 /** 240 * Remove from the specified region the store files of the specified column family, 241 * either by archiving them or outright deletion 242 * @param fs the filesystem where the store files live 243 * @param conf {@link Configuration} to examine to determine the archive directory 244 * @param parent Parent region hosting the store files 245 * @param tableDir {@link Path} to where the table is being stored (for building the archive path) 246 * @param family the family hosting the store files 247 * @throws IOException if the files could not be correctly disposed. 248 */ 249 public static void archiveFamily(FileSystem fs, Configuration conf, 250 RegionInfo parent, Path tableDir, byte[] family) throws IOException { 251 Path familyDir = new Path(tableDir, new Path(parent.getEncodedName(), Bytes.toString(family))); 252 archiveFamilyByFamilyDir(fs, conf, parent, familyDir, family); 253 } 254 255 /** 256 * Removes from the specified region the store files of the specified column family, 257 * either by archiving them or outright deletion 258 * @param fs the filesystem where the store files live 259 * @param conf {@link Configuration} to examine to determine the archive directory 260 * @param parent Parent region hosting the store files 261 * @param familyDir {@link Path} to where the family is being stored 262 * @param family the family hosting the store files 263 * @throws IOException if the files could not be correctly disposed. 264 */ 265 public static void archiveFamilyByFamilyDir(FileSystem fs, Configuration conf, 266 RegionInfo parent, Path familyDir, byte[] family) throws IOException { 267 FileStatus[] storeFiles = FSUtils.listStatus(fs, familyDir); 268 if (storeFiles == null) { 269 LOG.debug("No files to dispose of in {}, family={}", parent.getRegionNameAsString(), 270 Bytes.toString(family)); 271 return; 272 } 273 274 FileStatusConverter getAsFile = new FileStatusConverter(fs); 275 Collection<File> toArchive = Lists.transform(Arrays.asList(storeFiles), getAsFile); 276 Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf, parent, family); 277 278 // do the actual archive 279 List<File> failedArchive = resolveAndArchive(fs, storeArchiveDir, toArchive, 280 EnvironmentEdgeManager.currentTime()); 281 if (!failedArchive.isEmpty()){ 282 throw new FailedArchiveException("Failed to archive/delete all the files for region:" 283 + Bytes.toString(parent.getRegionName()) + ", family:" + Bytes.toString(family) 284 + " into " + storeArchiveDir + ". Something is probably awry on the filesystem.", 285 Collections2.transform(failedArchive, FUNC_FILE_TO_PATH)); 286 } 287 } 288 289 /** 290 * Remove the store files, either by archiving them or outright deletion 291 * @param conf {@link Configuration} to examine to determine the archive directory 292 * @param fs the filesystem where the store files live 293 * @param regionInfo {@link RegionInfo} of the region hosting the store files 294 * @param family the family hosting the store files 295 * @param compactedFiles files to be disposed of. No further reading of these files should be 296 * attempted; otherwise likely to cause an {@link IOException} 297 * @throws IOException if the files could not be correctly disposed. 298 */ 299 public static void archiveStoreFiles(Configuration conf, FileSystem fs, RegionInfo regionInfo, 300 Path tableDir, byte[] family, Collection<HStoreFile> compactedFiles) 301 throws IOException { 302 Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf, regionInfo, tableDir, family); 303 archive(fs, regionInfo, family, compactedFiles, storeArchiveDir); 304 } 305 306 /** 307 * Archive recovered edits using existing logic for archiving store files. This is currently only 308 * relevant when <b>hbase.region.archive.recovered.edits</b> is true, as recovered edits shouldn't 309 * be kept after replay. In theory, we could use very same method available for archiving 310 * store files, but supporting WAL dir and store files on different FileSystems added the need for 311 * extra validation of the passed FileSystem instance and the path where the archiving edits 312 * should be placed. 313 * @param conf {@link Configuration} to determine the archive directory. 314 * @param fs the filesystem used for storing WAL files. 315 * @param regionInfo {@link RegionInfo} a pseudo region representation for the archiving logic. 316 * @param family a pseudo familiy representation for the archiving logic. 317 * @param replayedEdits the recovered edits to be archived. 318 * @throws IOException if files can't be achived due to some internal error. 319 */ 320 public static void archiveRecoveredEdits(Configuration conf, FileSystem fs, RegionInfo regionInfo, 321 byte[] family, Collection<HStoreFile> replayedEdits) 322 throws IOException { 323 String workingDir = conf.get(CommonFSUtils.HBASE_WAL_DIR, conf.get(HConstants.HBASE_DIR)); 324 //extra sanity checks for the right FS 325 Path path = new Path(workingDir); 326 if(path.isAbsoluteAndSchemeAuthorityNull()){ 327 //no schema specified on wal dir value, so it's on same FS as StoreFiles 328 path = new Path(conf.get(HConstants.HBASE_DIR)); 329 } 330 if(path.toUri().getScheme()!=null && !path.toUri().getScheme().equals(fs.getScheme())){ 331 throw new IOException("Wrong file system! Should be " + path.toUri().getScheme() + 332 ", but got " + fs.getScheme()); 333 } 334 path = HFileArchiveUtil.getStoreArchivePathForRootDir(path, regionInfo, family); 335 archive(fs, regionInfo, family, replayedEdits, path); 336 } 337 338 private static void archive(FileSystem fs, RegionInfo regionInfo, byte[] family, 339 Collection<HStoreFile> compactedFiles, Path storeArchiveDir) throws IOException { 340 // sometimes in testing, we don't have rss, so we need to check for that 341 if (fs == null) { 342 LOG.warn("Passed filesystem is null, so just deleting files without archiving for {}," + 343 "family={}", Bytes.toString(regionInfo.getRegionName()), Bytes.toString(family)); 344 deleteStoreFilesWithoutArchiving(compactedFiles); 345 return; 346 } 347 348 // short circuit if we don't have any files to delete 349 if (compactedFiles.isEmpty()) { 350 LOG.debug("No files to dispose of, done!"); 351 return; 352 } 353 354 // build the archive path 355 if (regionInfo == null || family == null) throw new IOException( 356 "Need to have a region and a family to archive from."); 357 // make sure we don't archive if we can't and that the archive dir exists 358 if (!fs.mkdirs(storeArchiveDir)) { 359 throw new IOException("Could not make archive directory (" + storeArchiveDir + ") for store:" 360 + Bytes.toString(family) + ", deleting compacted files instead."); 361 } 362 363 // otherwise we attempt to archive the store files 364 LOG.debug("Archiving compacted files."); 365 366 // Wrap the storefile into a File 367 StoreToFile getStorePath = new StoreToFile(fs); 368 Collection<File> storeFiles = Collections2.transform(compactedFiles, getStorePath); 369 370 // do the actual archive 371 List<File> failedArchive = resolveAndArchive(fs, storeArchiveDir, storeFiles, 372 EnvironmentEdgeManager.currentTime()); 373 374 if (!failedArchive.isEmpty()){ 375 throw new FailedArchiveException("Failed to archive/delete all the files for region:" 376 + Bytes.toString(regionInfo.getRegionName()) + ", family:" + Bytes.toString(family) 377 + " into " + storeArchiveDir + ". Something is probably awry on the filesystem.", 378 Collections2.transform(failedArchive, FUNC_FILE_TO_PATH)); 379 } 380 } 381 382 /** 383 * Archive the store file 384 * @param fs the filesystem where the store files live 385 * @param regionInfo region hosting the store files 386 * @param conf {@link Configuration} to examine to determine the archive directory 387 * @param tableDir {@link Path} to where the table is being stored (for building the archive path) 388 * @param family the family hosting the store files 389 * @param storeFile file to be archived 390 * @throws IOException if the files could not be correctly disposed. 391 */ 392 public static void archiveStoreFile(Configuration conf, FileSystem fs, RegionInfo regionInfo, 393 Path tableDir, byte[] family, Path storeFile) throws IOException { 394 Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf, regionInfo, tableDir, family); 395 // make sure we don't archive if we can't and that the archive dir exists 396 if (!fs.mkdirs(storeArchiveDir)) { 397 throw new IOException("Could not make archive directory (" + storeArchiveDir + ") for store:" 398 + Bytes.toString(family) + ", deleting compacted files instead."); 399 } 400 401 // do the actual archive 402 long start = EnvironmentEdgeManager.currentTime(); 403 File file = new FileablePath(fs, storeFile); 404 if (!resolveAndArchiveFile(storeArchiveDir, file, Long.toString(start))) { 405 throw new IOException("Failed to archive/delete the file for region:" 406 + regionInfo.getRegionNameAsString() + ", family:" + Bytes.toString(family) 407 + " into " + storeArchiveDir + ". Something is probably awry on the filesystem."); 408 } 409 } 410 411 /** 412 * Resolve any conflict with an existing archive file via timestamp-append 413 * renaming of the existing file and then archive the passed in files. 414 * @param fs {@link FileSystem} on which to archive the files 415 * @param baseArchiveDir base archive directory to store the files. If any of 416 * the files to archive are directories, will append the name of the 417 * directory to the base archive directory name, creating a parallel 418 * structure. 419 * @param toArchive files/directories that need to be archvied 420 * @param start time the archiving started - used for resolving archive 421 * conflicts. 422 * @return the list of failed to archive files. 423 * @throws IOException if an unexpected file operation exception occurred 424 */ 425 private static List<File> resolveAndArchive(FileSystem fs, Path baseArchiveDir, 426 Collection<File> toArchive, long start) throws IOException { 427 // short circuit if no files to move 428 if (toArchive.isEmpty()) { 429 return Collections.emptyList(); 430 } 431 432 LOG.trace("Moving files to the archive directory {}", baseArchiveDir); 433 434 // make sure the archive directory exists 435 if (!fs.exists(baseArchiveDir)) { 436 if (!fs.mkdirs(baseArchiveDir)) { 437 throw new IOException("Failed to create the archive directory:" + baseArchiveDir 438 + ", quitting archive attempt."); 439 } 440 LOG.trace("Created archive directory {}", baseArchiveDir); 441 } 442 443 List<File> failures = new ArrayList<>(); 444 String startTime = Long.toString(start); 445 for (File file : toArchive) { 446 // if its a file archive it 447 try { 448 LOG.trace("Archiving {}", file); 449 if (file.isFile()) { 450 // attempt to archive the file 451 if (!resolveAndArchiveFile(baseArchiveDir, file, startTime)) { 452 LOG.warn("Couldn't archive " + file + " into backup directory: " + baseArchiveDir); 453 failures.add(file); 454 } 455 } else { 456 // otherwise its a directory and we need to archive all files 457 LOG.trace("{} is a directory, archiving children files", file); 458 // so we add the directory name to the one base archive 459 Path parentArchiveDir = new Path(baseArchiveDir, file.getName()); 460 // and then get all the files from that directory and attempt to 461 // archive those too 462 Collection<File> children = file.getChildren(); 463 failures.addAll(resolveAndArchive(fs, parentArchiveDir, children, start)); 464 } 465 } catch (IOException e) { 466 LOG.warn("Failed to archive {}", file, e); 467 failures.add(file); 468 } 469 } 470 return failures; 471 } 472 473 /** 474 * Attempt to archive the passed in file to the archive directory. 475 * <p> 476 * If the same file already exists in the archive, it is moved to a timestamped directory under 477 * the archive directory and the new file is put in its place. 478 * @param archiveDir {@link Path} to the directory that stores the archives of the hfiles 479 * @param currentFile {@link Path} to the original HFile that will be archived 480 * @param archiveStartTime time the archiving started, to resolve naming conflicts 481 * @return <tt>true</tt> if the file is successfully archived. <tt>false</tt> if there was a 482 * problem, but the operation still completed. 483 * @throws IOException on failure to complete {@link FileSystem} operations. 484 */ 485 private static boolean resolveAndArchiveFile(Path archiveDir, File currentFile, 486 String archiveStartTime) throws IOException { 487 // build path as it should be in the archive 488 String filename = currentFile.getName(); 489 Path archiveFile = new Path(archiveDir, filename); 490 FileSystem fs = currentFile.getFileSystem(); 491 492 // if the file already exists in the archive, move that one to a timestamped backup. This is a 493 // really, really unlikely situtation, where we get the same name for the existing file, but 494 // is included just for that 1 in trillion chance. 495 if (fs.exists(archiveFile)) { 496 LOG.debug("{} already exists in archive, moving to timestamped backup and " + 497 "overwriting current.", archiveFile); 498 499 // move the archive file to the stamped backup 500 Path backedupArchiveFile = new Path(archiveDir, filename + SEPARATOR + archiveStartTime); 501 if (!fs.rename(archiveFile, backedupArchiveFile)) { 502 LOG.error("Could not rename archive file to backup: " + backedupArchiveFile 503 + ", deleting existing file in favor of newer."); 504 // try to delete the exisiting file, if we can't rename it 505 if (!fs.delete(archiveFile, false)) { 506 throw new IOException("Couldn't delete existing archive file (" + archiveFile 507 + ") or rename it to the backup file (" + backedupArchiveFile 508 + ") to make room for similarly named file."); 509 } 510 } 511 LOG.debug("Backed up archive file from " + archiveFile); 512 } 513 514 LOG.trace("No existing file in archive for {}, free to archive original file.", archiveFile); 515 516 // at this point, we should have a free spot for the archive file 517 boolean success = false; 518 for (int i = 0; !success && i < DEFAULT_RETRIES_NUMBER; ++i) { 519 if (i > 0) { 520 // Ensure that the archive directory exists. 521 // The previous "move to archive" operation has failed probably because 522 // the cleaner has removed our archive directory (HBASE-7643). 523 // (we're in a retry loop, so don't worry too much about the exception) 524 try { 525 if (!fs.exists(archiveDir)) { 526 if (fs.mkdirs(archiveDir)) { 527 LOG.debug("Created archive directory {}", archiveDir); 528 } 529 } 530 } catch (IOException e) { 531 LOG.warn("Failed to create directory {}", archiveDir, e); 532 } 533 } 534 535 try { 536 success = currentFile.moveAndClose(archiveFile); 537 } catch (FileNotFoundException fnfe) { 538 LOG.warn("Failed to archive " + currentFile + 539 " because it does not exist! Skipping and continuing on.", fnfe); 540 success = true; 541 } catch (IOException e) { 542 LOG.warn("Failed to archive " + currentFile + " on try #" + i, e); 543 success = false; 544 } 545 } 546 547 if (!success) { 548 LOG.error("Failed to archive " + currentFile); 549 return false; 550 } 551 552 LOG.debug("Archived from {} to {}", currentFile, archiveFile); 553 return true; 554 } 555 556 /** 557 * Without regard for backup, delete a region. Should be used with caution. 558 * @param regionDir {@link Path} to the region to be deleted. 559 * @param fs FileSystem from which to delete the region 560 * @return <tt>true</tt> on successful deletion, <tt>false</tt> otherwise 561 * @throws IOException on filesystem operation failure 562 */ 563 private static boolean deleteRegionWithoutArchiving(FileSystem fs, Path regionDir) 564 throws IOException { 565 if (fs.delete(regionDir, true)) { 566 LOG.debug("Deleted {}", regionDir); 567 return true; 568 } 569 LOG.debug("Failed to delete directory {}", regionDir); 570 return false; 571 } 572 573 /** 574 * Just do a simple delete of the given store files 575 * <p> 576 * A best effort is made to delete each of the files, rather than bailing on the first failure. 577 * <p> 578 * @param compactedFiles store files to delete from the file system. 579 * @throws IOException if a file cannot be deleted. All files will be attempted to deleted before 580 * throwing the exception, rather than failing at the first file. 581 */ 582 private static void deleteStoreFilesWithoutArchiving(Collection<HStoreFile> compactedFiles) 583 throws IOException { 584 LOG.debug("Deleting files without archiving."); 585 List<IOException> errors = new ArrayList<>(0); 586 for (HStoreFile hsf : compactedFiles) { 587 try { 588 hsf.deleteStoreFile(); 589 } catch (IOException e) { 590 LOG.error("Failed to delete {}", hsf.getPath()); 591 errors.add(e); 592 } 593 } 594 if (errors.size() > 0) { 595 throw MultipleIOException.createIOException(errors); 596 } 597 } 598 599 /** 600 * Adapt a type to match the {@link File} interface, which is used internally for handling 601 * archival/removal of files 602 * @param <T> type to adapt to the {@link File} interface 603 */ 604 private static abstract class FileConverter<T> implements Function<T, File> { 605 protected final FileSystem fs; 606 607 public FileConverter(FileSystem fs) { 608 this.fs = fs; 609 } 610 } 611 612 /** 613 * Convert a FileStatus to something we can manage in the archiving 614 */ 615 private static class FileStatusConverter extends FileConverter<FileStatus> { 616 public FileStatusConverter(FileSystem fs) { 617 super(fs); 618 } 619 620 @Override 621 public File apply(FileStatus input) { 622 return new FileablePath(fs, input.getPath()); 623 } 624 } 625 626 /** 627 * Convert the {@link HStoreFile} into something we can manage in the archive 628 * methods 629 */ 630 private static class StoreToFile extends FileConverter<HStoreFile> { 631 public StoreToFile(FileSystem fs) { 632 super(fs); 633 } 634 635 @Override 636 public File apply(HStoreFile input) { 637 return new FileableStoreFile(fs, input); 638 } 639 } 640 641 /** 642 * Wrapper to handle file operations uniformly 643 */ 644 private static abstract class File { 645 protected final FileSystem fs; 646 647 public File(FileSystem fs) { 648 this.fs = fs; 649 } 650 651 /** 652 * Delete the file 653 * @throws IOException on failure 654 */ 655 abstract void delete() throws IOException; 656 657 /** 658 * Check to see if this is a file or a directory 659 * @return <tt>true</tt> if it is a file, <tt>false</tt> otherwise 660 * @throws IOException on {@link FileSystem} connection error 661 */ 662 abstract boolean isFile() throws IOException; 663 664 /** 665 * @return if this is a directory, returns all the children in the 666 * directory, otherwise returns an empty list 667 * @throws IOException 668 */ 669 abstract Collection<File> getChildren() throws IOException; 670 671 /** 672 * close any outside readers of the file 673 * @throws IOException 674 */ 675 abstract void close() throws IOException; 676 677 /** 678 * @return the name of the file (not the full fs path, just the individual 679 * file name) 680 */ 681 abstract String getName(); 682 683 /** 684 * @return the path to this file 685 */ 686 abstract Path getPath(); 687 688 /** 689 * Move the file to the given destination 690 * @param dest 691 * @return <tt>true</tt> on success 692 * @throws IOException 693 */ 694 public boolean moveAndClose(Path dest) throws IOException { 695 this.close(); 696 Path p = this.getPath(); 697 return FSUtils.renameAndSetModifyTime(fs, p, dest); 698 } 699 700 /** 701 * @return the {@link FileSystem} on which this file resides 702 */ 703 public FileSystem getFileSystem() { 704 return this.fs; 705 } 706 707 @Override 708 public String toString() { 709 return this.getClass().getSimpleName() + ", " + getPath().toString(); 710 } 711 } 712 713 /** 714 * A {@link File} that wraps a simple {@link Path} on a {@link FileSystem}. 715 */ 716 private static class FileablePath extends File { 717 private final Path file; 718 private final FileStatusConverter getAsFile; 719 720 public FileablePath(FileSystem fs, Path file) { 721 super(fs); 722 this.file = file; 723 this.getAsFile = new FileStatusConverter(fs); 724 } 725 726 @Override 727 public void delete() throws IOException { 728 if (!fs.delete(file, true)) throw new IOException("Failed to delete:" + this.file); 729 } 730 731 @Override 732 public String getName() { 733 return file.getName(); 734 } 735 736 @Override 737 public Collection<File> getChildren() throws IOException { 738 if (fs.isFile(file)) return Collections.emptyList(); 739 return Collections2.transform(Arrays.asList(fs.listStatus(file)), getAsFile); 740 } 741 742 @Override 743 public boolean isFile() throws IOException { 744 return fs.isFile(file); 745 } 746 747 @Override 748 public void close() throws IOException { 749 // NOOP - files are implicitly closed on removal 750 } 751 752 @Override 753 Path getPath() { 754 return file; 755 } 756 } 757 758 /** 759 * {@link File} adapter for a {@link HStoreFile} living on a {@link FileSystem} 760 * . 761 */ 762 private static class FileableStoreFile extends File { 763 HStoreFile file; 764 765 public FileableStoreFile(FileSystem fs, HStoreFile store) { 766 super(fs); 767 this.file = store; 768 } 769 770 @Override 771 public void delete() throws IOException { 772 file.deleteStoreFile(); 773 } 774 775 @Override 776 public String getName() { 777 return file.getPath().getName(); 778 } 779 780 @Override 781 public boolean isFile() { 782 return true; 783 } 784 785 @Override 786 public Collection<File> getChildren() throws IOException { 787 // storefiles don't have children 788 return Collections.emptyList(); 789 } 790 791 @Override 792 public void close() throws IOException { 793 file.closeStoreFile(true); 794 } 795 796 @Override 797 Path getPath() { 798 return file.getPath(); 799 } 800 } 801}