001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.snapshot;
019
020import java.io.IOException;
021import java.util.Collection;
022import java.util.Collections;
023import java.util.Map;
024import org.apache.hadoop.conf.Configuration;
025import org.apache.hadoop.fs.FileStatus;
026import org.apache.hadoop.fs.FileSystem;
027import org.apache.hadoop.fs.Path;
028import org.apache.hadoop.hbase.HBaseInterfaceAudience;
029import org.apache.hadoop.hbase.master.HMaster;
030import org.apache.hadoop.hbase.master.MasterServices;
031import org.apache.hadoop.hbase.master.cleaner.BaseHFileCleanerDelegate;
032import org.apache.hadoop.hbase.snapshot.CorruptedSnapshotException;
033import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
034import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
035import org.apache.hadoop.hbase.util.CommonFSUtils;
036import org.apache.yetus.audience.InterfaceAudience;
037import org.apache.yetus.audience.InterfaceStability;
038import org.slf4j.Logger;
039import org.slf4j.LoggerFactory;
040
041/**
042 * Implementation of a file cleaner that checks if a hfile is still used by snapshots of HBase
043 * tables.
044 */
045@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)
046@InterfaceStability.Evolving
047public class SnapshotHFileCleaner extends BaseHFileCleanerDelegate {
048  private static final Logger LOG = LoggerFactory.getLogger(SnapshotHFileCleaner.class);
049
050  /**
051   * Conf key for the frequency to attempt to refresh the cache of hfiles currently used in
052   * snapshots (ms)
053   */
054  public static final String HFILE_CACHE_REFRESH_PERIOD_CONF_KEY =
055      "hbase.master.hfilecleaner.plugins.snapshot.period";
056
057  /** Refresh cache, by default, every 5 minutes */
058  private static final long DEFAULT_HFILE_CACHE_REFRESH_PERIOD = 300000;
059
060  /** File cache for HFiles in the completed and currently running snapshots */
061  private SnapshotFileCache cache;
062
063  private MasterServices master;
064
065  @Override
066  public synchronized Iterable<FileStatus> getDeletableFiles(Iterable<FileStatus> files) {
067    try {
068      return cache.getUnreferencedFiles(files, master.getSnapshotManager());
069    } catch (CorruptedSnapshotException cse) {
070      LOG.debug("Corrupted in-progress snapshot file exception, ignored ", cse);
071    } catch (IOException e) {
072      LOG.error("Exception while checking if files were valid, keeping them just in case.", e);
073    }
074    return Collections.emptyList();
075  }
076
077  @Override
078  public void init(Map<String, Object> params) {
079    if (params.containsKey(HMaster.MASTER)) {
080      this.master = (MasterServices) params.get(HMaster.MASTER);
081    }
082  }
083
084  @Override
085  protected boolean isFileDeletable(FileStatus fStat) {
086    return false;
087  }
088
089  @Override
090  public void setConf(final Configuration conf) {
091    super.setConf(conf);
092    try {
093      long cacheRefreshPeriod = conf.getLong(HFILE_CACHE_REFRESH_PERIOD_CONF_KEY,
094        DEFAULT_HFILE_CACHE_REFRESH_PERIOD);
095      final FileSystem fs = CommonFSUtils.getCurrentFileSystem(conf);
096      Path rootDir = CommonFSUtils.getRootDir(conf);
097      Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir, conf);
098      FileSystem workingFs = workingDir.getFileSystem(conf);
099
100      cache = new SnapshotFileCache(fs, rootDir, workingFs, workingDir, cacheRefreshPeriod,
101        cacheRefreshPeriod, "snapshot-hfile-cleaner-cache-refresher",
102        new SnapshotFileCache.SnapshotFileInspector() {
103            @Override
104            public Collection<String> filesUnderSnapshot(final FileSystem fs,
105              final Path snapshotDir)
106                throws IOException {
107              return SnapshotReferenceUtil.getHFileNames(conf, fs, snapshotDir);
108            }
109          });
110    } catch (IOException e) {
111      LOG.error("Failed to create cleaner util", e);
112    }
113  }
114
115
116  @Override
117  public void stop(String why) {
118    this.cache.stop(why);
119  }
120
121  @Override
122  public boolean isStopped() {
123    return this.cache.isStopped();
124  }
125
126  /**
127   * Exposed for Testing!
128   * @return the cache of all hfiles
129   */
130  public SnapshotFileCache getFileCacheForTesting() {
131    return this.cache;
132  }
133}