001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.master.snapshot; 019 020import java.io.IOException; 021import java.util.Collection; 022import java.util.Collections; 023import java.util.Map; 024import org.apache.hadoop.conf.Configuration; 025import org.apache.hadoop.fs.FileStatus; 026import org.apache.hadoop.fs.FileSystem; 027import org.apache.hadoop.fs.Path; 028import org.apache.hadoop.hbase.HBaseInterfaceAudience; 029import org.apache.hadoop.hbase.master.HMaster; 030import org.apache.hadoop.hbase.master.MasterServices; 031import org.apache.hadoop.hbase.master.cleaner.BaseHFileCleanerDelegate; 032import org.apache.hadoop.hbase.snapshot.CorruptedSnapshotException; 033import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; 034import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil; 035import org.apache.hadoop.hbase.util.CommonFSUtils; 036import org.apache.yetus.audience.InterfaceAudience; 037import org.apache.yetus.audience.InterfaceStability; 038import org.slf4j.Logger; 039import org.slf4j.LoggerFactory; 040 041/** 042 * Implementation of a file cleaner that checks if a hfile is still used by snapshots of HBase 043 * tables. 044 */ 045@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG) 046@InterfaceStability.Evolving 047public class SnapshotHFileCleaner extends BaseHFileCleanerDelegate { 048 private static final Logger LOG = LoggerFactory.getLogger(SnapshotHFileCleaner.class); 049 050 /** 051 * Conf key for the frequency to attempt to refresh the cache of hfiles currently used in 052 * snapshots (ms) 053 */ 054 public static final String HFILE_CACHE_REFRESH_PERIOD_CONF_KEY = 055 "hbase.master.hfilecleaner.plugins.snapshot.period"; 056 057 /** Refresh cache, by default, every 5 minutes */ 058 private static final long DEFAULT_HFILE_CACHE_REFRESH_PERIOD = 300000; 059 060 /** File cache for HFiles in the completed and currently running snapshots */ 061 private SnapshotFileCache cache; 062 063 private MasterServices master; 064 065 @Override 066 public Iterable<FileStatus> getDeletableFiles(Iterable<FileStatus> files) { 067 try { 068 return cache.getUnreferencedFiles(files, master.getSnapshotManager()); 069 } catch (CorruptedSnapshotException cse) { 070 LOG.debug("Corrupted in-progress snapshot file exception, ignored ", cse); 071 } catch (IOException e) { 072 LOG.error("Exception while checking if files were valid, keeping them just in case.", e); 073 } 074 return Collections.emptyList(); 075 } 076 077 @Override 078 public void init(Map<String, Object> params) { 079 if (params.containsKey(HMaster.MASTER)) { 080 this.master = (MasterServices) params.get(HMaster.MASTER); 081 } 082 } 083 084 @Override 085 protected boolean isFileDeletable(FileStatus fStat) { 086 return false; 087 } 088 089 @Override 090 public void setConf(final Configuration conf) { 091 super.setConf(conf); 092 try { 093 long cacheRefreshPeriod = 094 conf.getLong(HFILE_CACHE_REFRESH_PERIOD_CONF_KEY, DEFAULT_HFILE_CACHE_REFRESH_PERIOD); 095 final FileSystem fs = CommonFSUtils.getCurrentFileSystem(conf); 096 Path rootDir = CommonFSUtils.getRootDir(conf); 097 Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir, conf); 098 FileSystem workingFs = workingDir.getFileSystem(conf); 099 100 cache = new SnapshotFileCache(fs, rootDir, workingFs, workingDir, cacheRefreshPeriod, 101 cacheRefreshPeriod, "snapshot-hfile-cleaner-cache-refresher", 102 new SnapshotFileCache.SnapshotFileInspector() { 103 @Override 104 public Collection<String> filesUnderSnapshot(final FileSystem fs, final Path snapshotDir) 105 throws IOException { 106 return SnapshotReferenceUtil.getHFileNames(conf, fs, snapshotDir); 107 } 108 }); 109 } catch (IOException e) { 110 LOG.error("Failed to create cleaner util", e); 111 } 112 } 113 114 @Override 115 public void stop(String why) { 116 this.cache.stop(why); 117 } 118 119 @Override 120 public boolean isStopped() { 121 return this.cache.isStopped(); 122 } 123 124 /** 125 * Exposed for Testing! 126 * @return the cache of all hfiles 127 */ 128 public SnapshotFileCache getFileCacheForTesting() { 129 return this.cache; 130 } 131}