001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.snapshot;
019
020import java.io.IOException;
021import java.util.Collection;
022import java.util.Collections;
023import java.util.Map;
024import org.apache.hadoop.conf.Configuration;
025import org.apache.hadoop.fs.FileStatus;
026import org.apache.hadoop.fs.FileSystem;
027import org.apache.hadoop.fs.Path;
028import org.apache.hadoop.hbase.HBaseInterfaceAudience;
029import org.apache.hadoop.hbase.master.HMaster;
030import org.apache.hadoop.hbase.master.MasterServices;
031import org.apache.hadoop.hbase.master.cleaner.BaseHFileCleanerDelegate;
032import org.apache.hadoop.hbase.snapshot.CorruptedSnapshotException;
033import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
034import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
035import org.apache.hadoop.hbase.util.CommonFSUtils;
036import org.apache.yetus.audience.InterfaceAudience;
037import org.apache.yetus.audience.InterfaceStability;
038import org.slf4j.Logger;
039import org.slf4j.LoggerFactory;
040
041/**
042 * Implementation of a file cleaner that checks if a hfile is still used by snapshots of HBase
043 * tables.
044 */
045@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)
046@InterfaceStability.Evolving
047public class SnapshotHFileCleaner extends BaseHFileCleanerDelegate {
048  private static final Logger LOG = LoggerFactory.getLogger(SnapshotHFileCleaner.class);
049
050  /**
051   * Conf key for the frequency to attempt to refresh the cache of hfiles currently used in
052   * snapshots (ms)
053   */
054  public static final String HFILE_CACHE_REFRESH_PERIOD_CONF_KEY =
055    "hbase.master.hfilecleaner.plugins.snapshot.period";
056
057  /** Refresh cache, by default, every 5 minutes */
058  private static final long DEFAULT_HFILE_CACHE_REFRESH_PERIOD = 300000;
059
060  /** File cache for HFiles in the completed and currently running snapshots */
061  private SnapshotFileCache cache;
062
063  private MasterServices master;
064
065  @Override
066  public Iterable<FileStatus> getDeletableFiles(Iterable<FileStatus> files) {
067    try {
068      return cache.getUnreferencedFiles(files, master.getSnapshotManager());
069    } catch (CorruptedSnapshotException cse) {
070      LOG.debug("Corrupted in-progress snapshot file exception, ignored ", cse);
071    } catch (IOException e) {
072      LOG.error("Exception while checking if files were valid, keeping them just in case.", e);
073    }
074    return Collections.emptyList();
075  }
076
077  @Override
078  public void init(Map<String, Object> params) {
079    if (params.containsKey(HMaster.MASTER)) {
080      this.master = (MasterServices) params.get(HMaster.MASTER);
081    }
082  }
083
084  @Override
085  protected boolean isFileDeletable(FileStatus fStat) {
086    return false;
087  }
088
089  @Override
090  public void setConf(final Configuration conf) {
091    super.setConf(conf);
092    try {
093      long cacheRefreshPeriod =
094        conf.getLong(HFILE_CACHE_REFRESH_PERIOD_CONF_KEY, DEFAULT_HFILE_CACHE_REFRESH_PERIOD);
095      final FileSystem fs = CommonFSUtils.getCurrentFileSystem(conf);
096      Path rootDir = CommonFSUtils.getRootDir(conf);
097      Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir, conf);
098      FileSystem workingFs = workingDir.getFileSystem(conf);
099
100      cache = new SnapshotFileCache(fs, rootDir, workingFs, workingDir, cacheRefreshPeriod,
101        cacheRefreshPeriod, "snapshot-hfile-cleaner-cache-refresher",
102        new SnapshotFileCache.SnapshotFileInspector() {
103          @Override
104          public Collection<String> filesUnderSnapshot(final FileSystem fs, final Path snapshotDir)
105            throws IOException {
106            return SnapshotReferenceUtil.getHFileNames(conf, fs, snapshotDir);
107          }
108        });
109    } catch (IOException e) {
110      LOG.error("Failed to create cleaner util", e);
111    }
112  }
113
114  @Override
115  public void stop(String why) {
116    this.cache.stop(why);
117  }
118
119  @Override
120  public boolean isStopped() {
121    return this.cache.isStopped();
122  }
123
124  /**
125   * Exposed for Testing!
126   * @return the cache of all hfiles
127   */
128  public SnapshotFileCache getFileCacheForTesting() {
129    return this.cache;
130  }
131}