View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.master.snapshot;
19  
20  import java.io.FileNotFoundException;
21  import java.io.IOException;
22  import java.util.*;
23  
24  import com.google.common.annotations.VisibleForTesting;
25  import com.google.common.collect.Lists;
26  import org.apache.commons.logging.Log;
27  import org.apache.commons.logging.LogFactory;
28  import org.apache.hadoop.classification.InterfaceAudience;
29  import org.apache.hadoop.classification.InterfaceStability;
30  import org.apache.hadoop.conf.Configuration;
31  import org.apache.hadoop.fs.FileStatus;
32  import org.apache.hadoop.fs.FileSystem;
33  import org.apache.hadoop.fs.Path;
34  import org.apache.hadoop.hbase.Stoppable;
35  import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
36  import org.apache.hadoop.hbase.util.FSUtils;
37  
38  
39  /**
40   * Intelligently keep track of all the files for all the snapshots.
41   * <p>
42   * A cache of files is kept to avoid querying the {@link FileSystem} frequently. If there is a cache
43   * miss the directory modification time is used to ensure that we don't rescan directories that we
44   * already have in cache. We only check the modification times of the snapshot directories
45   * (/hbase/.hbase-snapshot/[snapshot_name]) to determine if the files need to be loaded into the cache.
46   * <p>
47   * New snapshots will be added to the cache and deleted snapshots will be removed when we refresh
48   * the cache. If the files underneath a snapshot directory are changed, but not the snapshot itself,
49   * we will ignore updates to that snapshot's files.
50   * <p>
51   * This is sufficient because each snapshot has its own directory and is added via an atomic rename
52   * <i>once</i>, when the snapshot is created. We don't need to worry about the data in the snapshot
53   * being run.
54   * <p>
55   * Further, the cache is periodically refreshed ensure that files in snapshots that were deleted are
56   * also removed from the cache.
57   * <p>
58   * A SnapshotFileInspector must be passed when creating <tt>this</tt> to allow extraction
59   * of files under the /hbase/.snapshot/[snapshot name] directory, for each snapshot.
60   * This allows you to only cache files under, for instance, all the logs in the .logs directory or
61   * all the files under all the regions.
62   * <p>
63   * <tt>this</tt> also considers all running snapshots (those under /hbase/.hbase-snapshot/.tmp) as valid
64   * snapshots but will not attempt to cache files from that directory.
65   * <p>
66   * Queries about a given file are thread-safe with respect to multiple queries and cache refreshes.
67   */
68  @InterfaceAudience.Private
69  @InterfaceStability.Evolving
70  public class SnapshotFileCache implements Stoppable {
71    interface SnapshotFileInspector {
72      /**
73       * Returns a collection of file names needed by the snapshot.
74       * @param snapshotDir {@link Path} to the snapshot directory to scan.
75       * @return the collection of file names needed by the snapshot.
76       */
77      Collection<String> filesUnderSnapshot(final Path snapshotDir) throws IOException;
78    }
79  
80    private static final Log LOG = LogFactory.getLog(SnapshotFileCache.class);
81    private volatile boolean stop = false;
82    private final FileSystem fs;
83    private final SnapshotFileInspector fileInspector;
84    private final Path snapshotDir;
85    private final Set<String> cache = new HashSet<String>();
86    /**
87     * This is a helper map of information about the snapshot directories so we don't need to rescan
88     * them if they haven't changed since the last time we looked.
89     */
90    private final Map<String, SnapshotDirectoryInfo> snapshots =
91        new HashMap<String, SnapshotDirectoryInfo>();
92    private final Timer refreshTimer;
93  
94    private long lastModifiedTime = Long.MIN_VALUE;
95  
96    /**
97     * Create a snapshot file cache for all snapshots under the specified [root]/.snapshot on the
98     * filesystem.
99     * <p>
100    * Immediately loads the file cache.
101    * @param conf to extract the configured {@link FileSystem} where the snapshots are stored and
102    *          hbase root directory
103    * @param cacheRefreshPeriod frequency (ms) with which the cache should be refreshed
104    * @param refreshThreadName name of the cache refresh thread
105    * @param inspectSnapshotFiles Filter to apply to each snapshot to extract the files.
106    * @throws IOException if the {@link FileSystem} or root directory cannot be loaded
107    */
108   public SnapshotFileCache(Configuration conf, long cacheRefreshPeriod, String refreshThreadName,
109       SnapshotFileInspector inspectSnapshotFiles) throws IOException {
110     this(FSUtils.getCurrentFileSystem(conf), FSUtils.getRootDir(conf), 0, cacheRefreshPeriod,
111         refreshThreadName, inspectSnapshotFiles);
112   }
113 
114   /**
115    * Create a snapshot file cache for all snapshots under the specified [root]/.snapshot on the
116    * filesystem
117    * @param fs {@link FileSystem} where the snapshots are stored
118    * @param rootDir hbase root directory
119    * @param cacheRefreshPeriod period (ms) with which the cache should be refreshed
120    * @param cacheRefreshDelay amount of time to wait for the cache to be refreshed
121    * @param refreshThreadName name of the cache refresh thread
122    * @param inspectSnapshotFiles Filter to apply to each snapshot to extract the files.
123    */
124   public SnapshotFileCache(FileSystem fs, Path rootDir, long cacheRefreshPeriod,
125       long cacheRefreshDelay, String refreshThreadName, SnapshotFileInspector inspectSnapshotFiles) {
126     this.fs = fs;
127     this.fileInspector = inspectSnapshotFiles;
128     this.snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
129     // periodically refresh the file cache to make sure we aren't superfluously saving files.
130     this.refreshTimer = new Timer(refreshThreadName, true);
131     this.refreshTimer.scheduleAtFixedRate(new RefreshCacheTask(), cacheRefreshDelay,
132       cacheRefreshPeriod);
133   }
134 
135   /**
136    * Trigger a cache refresh, even if its before the next cache refresh. Does not affect pending
137    * cache refreshes.
138    * <p>
139    * Blocks until the cache is refreshed.
140    * <p>
141    * Exposed for TESTING.
142    */
143   public void triggerCacheRefreshForTesting() {
144     try {
145       SnapshotFileCache.this.refreshCache();
146     } catch (IOException e) {
147       LOG.warn("Failed to refresh snapshot hfile cache!", e);
148     }
149     LOG.debug("Current cache:" + cache);
150   }
151 
152   /**
153    * Check to see if the passed file name is contained in any of the snapshots. First checks an
154    * in-memory cache of the files to keep. If its not in the cache, then the cache is refreshed and
155    * the cache checked again for that file. This ensures that we always return <tt>true</tt> for a
156    * files that exists.
157    * <p>
158    * Note this may lead to periodic false positives for the file being referenced. Periodically, the
159    * cache is refreshed even if there are no requests to ensure that the false negatives get removed
160    * eventually. For instance, suppose you have a file in the snapshot and it gets loaded into the
161    * cache. Then at some point later that snapshot is deleted. If the cache has not been refreshed
162    * at that point, cache will still think the file system contains that file and return
163    * <tt>true</tt>, even if it is no longer present (false positive). However, if the file never was
164    * on the filesystem, we will never find it and always return <tt>false</tt>.
165    * @param files file to check, NOTE: Relies that files are loaded from hdfs before method is called (NOT LAZY)
166    * @return <tt>unReferencedFiles</tt> the collection of files that do not have snapshot references
167    * @throws IOException if there is an unexpected error reaching the filesystem.
168    */
169   // XXX this is inefficient to synchronize on the method, when what we really need to guard against
170   // is an illegal access to the cache. Really we could do a mutex-guarded pointer swap on the
171   // cache, but that seems overkill at the moment and isn't necessarily a bottleneck.
172   public synchronized Iterable<FileStatus> getUnreferencedFiles(Iterable<FileStatus> files) throws IOException {
173     List<FileStatus> unReferencedFiles = Lists.newArrayList();
174     List<String> snapshotsInProgress = null;
175     boolean refreshed = false;
176     for (FileStatus file : files) {
177       String fileName = file.getPath().getName();
178       if (!refreshed && !cache.contains(fileName)) {
179     refreshCache();
180         refreshed = true;
181       }
182       if (cache.contains(fileName)) {
183         continue;
184       }
185       if (snapshotsInProgress == null) {
186         snapshotsInProgress = getSnapshotsInProgress();
187       }
188       if (snapshotsInProgress.contains(fileName)) {
189         continue;
190       }
191       unReferencedFiles.add(file);
192     }
193     return unReferencedFiles;
194   }
195 
196   private synchronized void refreshCache() throws IOException {
197     // get the status of the snapshots directory 
198     FileStatus dirStatus;
199     try {
200       dirStatus = fs.getFileStatus(snapshotDir);
201     } catch (FileNotFoundException e) {
202       if (this.cache.size() > 0) {
203         LOG.error("Snapshot directory: " + snapshotDir + " doesn't exist");
204       }
205       return;
206     }
207 
208     if (dirStatus.getModificationTime() <= lastModifiedTime) {
209       return;
210     }
211 
212     // directory was modified, so we need to reload our cache
213     // there could be a slight race here where we miss the cache, check the directory modification
214     // time, then someone updates the directory, causing us to not scan the directory again.
215     // However, snapshot directories are only created once, so this isn't an issue.
216 
217     // 1. update the modified time
218     this.lastModifiedTime = dirStatus.getModificationTime();
219 
220     // 2.clear the cache
221     this.cache.clear();
222     Map<String, SnapshotDirectoryInfo> known = new HashMap<String, SnapshotDirectoryInfo>();
223 
224     // 3. check each of the snapshot directories
225     FileStatus[] snapshots = FSUtils.listStatus(fs, snapshotDir);
226     if (snapshots == null) {
227       // remove all the remembered snapshots because we don't have any left
228       if (LOG.isDebugEnabled() && this.snapshots.size() > 0) {
229         LOG.debug("No snapshots on-disk, cache empty");
230       }
231       this.snapshots.clear();
232       return;
233     }
234 
235     // 3.1 iterate through the on-disk snapshots
236     for (FileStatus snapshot : snapshots) {
237       String name = snapshot.getPath().getName();
238       if (!name.equals(SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME)) {
239         SnapshotDirectoryInfo files = this.snapshots.remove(name);
240         // 3.1.1 if we don't know about the snapshot or its been modified, we need to update the files
241         // the latter could occur where I create a snapshot, then delete it, and then make a new
242         // snapshot with the same name. We will need to update the cache the information from that new
243         // snapshot, even though it has the same name as the files referenced have probably changed.
244         if (files == null || files.hasBeenModified(snapshot.getModificationTime())) {
245           // get all files for the snapshot and create a new info
246           Collection<String> storedFiles = fileInspector.filesUnderSnapshot(snapshot.getPath());
247           files = new SnapshotDirectoryInfo(snapshot.getModificationTime(), storedFiles);
248         }
249         // 3.2 add all the files to cache
250         this.cache.addAll(files.getFiles());
251         known.put(name, files);
252       }
253     }
254 
255     // 4. set the snapshots we are tracking
256     this.snapshots.clear();
257     this.snapshots.putAll(known);
258   }
259 
260   @VisibleForTesting List<String> getSnapshotsInProgress() throws IOException {
261     List<String> snapshotInProgress = Lists.newArrayList();
262     // only add those files to the cache, but not to the known snapshots
263     Path snapshotTmpDir = new Path(snapshotDir, SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME);
264     // only add those files to the cache, but not to the known snapshots
265     FileStatus[] running = FSUtils.listStatus(fs, snapshotTmpDir);
266     if (running != null) {
267       for (FileStatus run : running) {
268         snapshotInProgress.addAll(fileInspector.filesUnderSnapshot(run.getPath()));
269       }
270     }
271     return snapshotInProgress;
272   }
273 
274   /**
275    * Simple helper task that just periodically attempts to refresh the cache
276    */
277   public class RefreshCacheTask extends TimerTask {
278     @Override
279     public void run() {
280       try {
281         SnapshotFileCache.this.refreshCache();
282       } catch (IOException e) {
283         LOG.warn("Failed to refresh snapshot hfile cache!", e);
284       }
285     }
286   }
287 
288   @Override
289   public void stop(String why) {
290     if (!this.stop) {
291       this.stop = true;
292       this.refreshTimer.cancel();
293     }
294 
295   }
296 
297   @Override
298   public boolean isStopped() {
299     return this.stop;
300   }
301 
302   /**
303    * Information about a snapshot directory
304    */
305   private static class SnapshotDirectoryInfo {
306     long lastModified;
307     Collection<String> files;
308 
309     public SnapshotDirectoryInfo(long mtime, Collection<String> files) {
310       this.lastModified = mtime;
311       this.files = files;
312     }
313 
314     /**
315      * @return the hfiles in the snapshot when <tt>this</tt> was made.
316      */
317     public Collection<String> getFiles() {
318       return this.files;
319     }
320 
321     /**
322      * Check if the snapshot directory has been modified
323      * @param mtime current modification time of the directory
324      * @return <tt>true</tt> if it the modification time of the directory is newer time when we
325      *         created <tt>this</tt>
326      */
327     public boolean hasBeenModified(long mtime) {
328       return this.lastModified < mtime;
329     }
330   }
331 
332 }