001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.snapshot;
019
020import java.io.IOException;
021import java.util.Collection;
022import java.util.Collections;
023import java.util.Map;
024
025import org.apache.yetus.audience.InterfaceAudience;
026import org.apache.yetus.audience.InterfaceStability;
027import org.slf4j.Logger;
028import org.slf4j.LoggerFactory;
029import org.apache.hadoop.conf.Configuration;
030import org.apache.hadoop.fs.FileStatus;
031import org.apache.hadoop.fs.FileSystem;
032import org.apache.hadoop.fs.Path;
033import org.apache.hadoop.hbase.HBaseInterfaceAudience;
034import org.apache.hadoop.hbase.master.HMaster;
035import org.apache.hadoop.hbase.master.MasterServices;
036import org.apache.hadoop.hbase.master.cleaner.BaseHFileCleanerDelegate;
037import org.apache.hadoop.hbase.snapshot.CorruptedSnapshotException;
038import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
039import org.apache.hadoop.hbase.util.FSUtils;
040
041/**
042 * Implementation of a file cleaner that checks if a hfile is still used by snapshots of HBase
043 * tables.
044 */
045@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)
046@InterfaceStability.Evolving
047public class SnapshotHFileCleaner extends BaseHFileCleanerDelegate {
048  private static final Logger LOG = LoggerFactory.getLogger(SnapshotHFileCleaner.class);
049
050  /**
051   * Conf key for the frequency to attempt to refresh the cache of hfiles currently used in
052   * snapshots (ms)
053   */
054  public static final String HFILE_CACHE_REFRESH_PERIOD_CONF_KEY =
055      "hbase.master.hfilecleaner.plugins.snapshot.period";
056
057  /** Refresh cache, by default, every 5 minutes */
058  private static final long DEFAULT_HFILE_CACHE_REFRESH_PERIOD = 300000;
059
060  /** File cache for HFiles in the completed and currently running snapshots */
061  private SnapshotFileCache cache;
062
063  private MasterServices master;
064
065  @Override
066  public synchronized Iterable<FileStatus> getDeletableFiles(Iterable<FileStatus> files) {
067    try {
068      return cache.getUnreferencedFiles(files, master.getSnapshotManager());
069    } catch (CorruptedSnapshotException cse) {
070      LOG.debug("Corrupted in-progress snapshot file exception, ignored ", cse);
071    } catch (IOException e) {
072      LOG.error("Exception while checking if files were valid, keeping them just in case.", e);
073    }
074    return Collections.emptyList();
075  }
076
077  @Override
078  public void init(Map<String, Object> params) {
079    if (params.containsKey(HMaster.MASTER)) {
080      this.master = (MasterServices) params.get(HMaster.MASTER);
081    }
082  }
083
084  @Override
085  protected boolean isFileDeletable(FileStatus fStat) {
086    return false;
087  }
088
089  @Override
090  public void setConf(final Configuration conf) {
091    super.setConf(conf);
092    try {
093      long cacheRefreshPeriod = conf.getLong(HFILE_CACHE_REFRESH_PERIOD_CONF_KEY,
094        DEFAULT_HFILE_CACHE_REFRESH_PERIOD);
095      final FileSystem fs = FSUtils.getCurrentFileSystem(conf);
096      Path rootDir = FSUtils.getRootDir(conf);
097      cache = new SnapshotFileCache(fs, rootDir, cacheRefreshPeriod, cacheRefreshPeriod,
098          "snapshot-hfile-cleaner-cache-refresher", new SnapshotFileCache.SnapshotFileInspector() {
099            @Override
100            public Collection<String> filesUnderSnapshot(final Path snapshotDir)
101                throws IOException {
102              return SnapshotReferenceUtil.getHFileNames(conf, fs, snapshotDir);
103            }
104          });
105    } catch (IOException e) {
106      LOG.error("Failed to create cleaner util", e);
107    }
108  }
109
110
111  @Override
112  public void stop(String why) {
113    this.cache.stop(why);
114  }
115
116  @Override
117  public boolean isStopped() {
118    return this.cache.isStopped();
119  }
120
121  /**
122   * Exposed for Testing!
123   * @return the cache of all hfiles
124   */
125  public SnapshotFileCache getFileCacheForTesting() {
126    return this.cache;
127  }
128}