View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.master.snapshot;
19  
20  import java.io.FileNotFoundException;
21  import java.io.IOException;
22  import java.util.Collection;
23  import java.util.HashMap;
24  import java.util.HashSet;
25  import java.util.List;
26  import java.util.Map;
27  import java.util.Set;
28  import java.util.Timer;
29  import java.util.TimerTask;
30  import java.util.concurrent.locks.ReentrantLock;
31
32  import com.google.common.annotations.VisibleForTesting;
33  import com.google.common.collect.Lists;
34  import org.apache.commons.logging.Log;
35  import org.apache.commons.logging.LogFactory;
36  import org.apache.hadoop.hbase.classification.InterfaceAudience;
37  import org.apache.hadoop.hbase.classification.InterfaceStability;
38  import org.apache.hadoop.conf.Configuration;
39  import org.apache.hadoop.fs.FileStatus;
40  import org.apache.hadoop.fs.FileSystem;
41  import org.apache.hadoop.fs.Path;
42  import org.apache.hadoop.hbase.Stoppable;
43  import org.apache.hadoop.hbase.snapshot.CorruptedSnapshotException;
44  import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
45  import org.apache.hadoop.hbase.util.FSUtils;
46
47  /**
48   * Intelligently keep track of all the files for all the snapshots.
49   * <p>
50   * A cache of files is kept to avoid querying the {@link FileSystem} frequently. If there is a cache
51   * miss the directory modification time is used to ensure that we don't rescan directories that we
52   * already have in cache. We only check the modification times of the snapshot directories
53   * (/hbase/.snapshot/[snapshot_name]) to determine if the files need to be loaded into the cache.
54   * <p>
55   * New snapshots will be added to the cache and deleted snapshots will be removed when we refresh
56   * the cache. If the files underneath a snapshot directory are changed, but not the snapshot itself,
57   * we will ignore updates to that snapshot's files.
58   * <p>
59   * This is sufficient because each snapshot has its own directory and is added via an atomic rename
60   * <i>once</i>, when the snapshot is created. We don't need to worry about the data in the snapshot
61   * being run.
62   * <p>
63   * Further, the cache is periodically refreshed ensure that files in snapshots that were deleted are
64   * also removed from the cache.
65   * <p>
66   * A {@link SnapshotFileCache.SnapshotFileInspector} must be passed when creating <tt>this</tt> to
67   * allow extraction of files under /hbase/.snapshot/[snapshot name] directory, for each snapshot.
68   * This allows you to only cache files under, for instance, all the logs in the .logs directory or
69   * all the files under all the regions.
70   * <p>
71   * <tt>this</tt> also considers all running snapshots (those under /hbase/.snapshot/.tmp) as valid
72   * snapshots and will attempt to cache files from those snapshots as well.
73   * <p>
74   * Queries about a given file are thread-safe with respect to multiple queries and cache refreshes.
75   */
76  @InterfaceAudience.Private
77  @InterfaceStability.Evolving
78  public class SnapshotFileCache implements Stoppable {
79    interface SnapshotFileInspector {
80      /**
81       * Returns a collection of file names needed by the snapshot.
82       * @param snapshotDir {@link Path} to the snapshot directory to scan.
83       * @return the collection of file names needed by the snapshot.
84       */
85      Collection<String> filesUnderSnapshot(final Path snapshotDir) throws IOException;
86    }
87
88    private static final Log LOG = LogFactory.getLog(SnapshotFileCache.class);
89    private volatile boolean stop = false;
90    private final FileSystem fs;
91    private final SnapshotFileInspector fileInspector;
92    private final Path snapshotDir;
93    private final Set<String> cache = new HashSet<String>();
94    /**
95     * This is a helper map of information about the snapshot directories so we don't need to rescan
96     * them if they haven't changed since the last time we looked.
97     */
98    private final Map<String, SnapshotDirectoryInfo> snapshots =
99        new HashMap<String, SnapshotDirectoryInfo>();
100   private final Timer refreshTimer;
101
102   private long lastModifiedTime = Long.MIN_VALUE;
103
104   /**
105    * Create a snapshot file cache for all snapshots under the specified [root]/.snapshot on the
106    * filesystem.
107    * <p>
108    * Immediately loads the file cache.
109    * @param conf to extract the configured {@link FileSystem} where the snapshots are stored and
110    *          hbase root directory
111    * @param cacheRefreshPeriod frequency (ms) with which the cache should be refreshed
112    * @param refreshThreadName name of the cache refresh thread
113    * @param inspectSnapshotFiles Filter to apply to each snapshot to extract the files.
114    * @throws IOException if the {@link FileSystem} or root directory cannot be loaded
115    */
116   public SnapshotFileCache(Configuration conf, long cacheRefreshPeriod, String refreshThreadName,
117       SnapshotFileInspector inspectSnapshotFiles) throws IOException {
118     this(FSUtils.getCurrentFileSystem(conf), FSUtils.getRootDir(conf), 0, cacheRefreshPeriod,
119         refreshThreadName, inspectSnapshotFiles);
120   }
121
122   /**
123    * Create a snapshot file cache for all snapshots under the specified [root]/.snapshot on the
124    * filesystem
125    * @param fs {@link FileSystem} where the snapshots are stored
126    * @param rootDir hbase root directory
127    * @param cacheRefreshPeriod period (ms) with which the cache should be refreshed
128    * @param cacheRefreshDelay amount of time to wait for the cache to be refreshed
129    * @param refreshThreadName name of the cache refresh thread
130    * @param inspectSnapshotFiles Filter to apply to each snapshot to extract the files.
131    */
132   public SnapshotFileCache(FileSystem fs, Path rootDir, long cacheRefreshPeriod,
133       long cacheRefreshDelay, String refreshThreadName, SnapshotFileInspector inspectSnapshotFiles) {
134     this.fs = fs;
135     this.fileInspector = inspectSnapshotFiles;
136     this.snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
137     // periodically refresh the file cache to make sure we aren't superfluously saving files.
138     this.refreshTimer = new Timer(refreshThreadName, true);
139     this.refreshTimer.scheduleAtFixedRate(new RefreshCacheTask(), cacheRefreshDelay,
140       cacheRefreshPeriod);
141   }
142
143   /**
144    * Trigger a cache refresh, even if its before the next cache refresh. Does not affect pending
145    * cache refreshes.
146    * <p>
147    * Blocks until the cache is refreshed.
148    * <p>
149    * Exposed for TESTING.
150    */
151   public void triggerCacheRefreshForTesting() {
152     try {
153       SnapshotFileCache.this.refreshCache();
154     } catch (IOException e) {
155       LOG.warn("Failed to refresh snapshot hfile cache!", e);
156     }
157     LOG.debug("Current cache:" + cache);
158   }
159
160   /**
161    * Check to see if any of the passed file names is contained in any of the snapshots.
162    * First checks an in-memory cache of the files to keep. If its not in the cache, then the cache
163    * is refreshed and the cache checked again for that file.
164    * This ensures that we never return files that exist.
165    * <p>
166    * Note this may lead to periodic false positives for the file being referenced. Periodically, the
167    * cache is refreshed even if there are no requests to ensure that the false negatives get removed
168    * eventually. For instance, suppose you have a file in the snapshot and it gets loaded into the
169    * cache. Then at some point later that snapshot is deleted. If the cache has not been refreshed
170    * at that point, cache will still think the file system contains that file and return
171    * <tt>true</tt>, even if it is no longer present (false positive). However, if the file never was
172    * on the filesystem, we will never find it and always return <tt>false</tt>.
173    * @param files file to check, NOTE: Relies that files are loaded from hdfs before method
174    *              is called (NOT LAZY)
175    * @return <tt>unReferencedFiles</tt> the collection of files that do not have snapshot references
176    * @throws IOException if there is an unexpected error reaching the filesystem.
177    */
178   // XXX this is inefficient to synchronize on the method, when what we really need to guard against
179   // is an illegal access to the cache. Really we could do a mutex-guarded pointer swap on the
180   // cache, but that seems overkill at the moment and isn't necessarily a bottleneck.
181   public synchronized Iterable<FileStatus> getUnreferencedFiles(Iterable<FileStatus> files,
182       final SnapshotManager snapshotManager)
183       throws IOException {
184     List<FileStatus> unReferencedFiles = Lists.newArrayList();
185     List<String> snapshotsInProgress = null;
186     boolean refreshed = false;
187     for (FileStatus file : files) {
188       String fileName = file.getPath().getName();
189       if (!refreshed && !cache.contains(fileName)) {
190         refreshCache();
191         refreshed = true;
192       }
193       if (cache.contains(fileName)) {
194         continue;
195       }
196       if (snapshotsInProgress == null) {
197         snapshotsInProgress = getSnapshotsInProgress(snapshotManager);
198       }
199       if (snapshotsInProgress.contains(fileName)) {
200         continue;
201       }
202       unReferencedFiles.add(file);
203     }
204     return unReferencedFiles;
205   }
206
207   private synchronized void refreshCache() throws IOException {
208     long lastTimestamp = Long.MAX_VALUE;
209     boolean hasChanges = false;
210
211     // get the status of the snapshots directory and check if it is has changes
212     try {
213       FileStatus dirStatus = fs.getFileStatus(snapshotDir);
214       lastTimestamp = dirStatus.getModificationTime();
215       hasChanges |= (lastTimestamp >= lastModifiedTime);
216     } catch (FileNotFoundException e) {
217       if (this.cache.size() > 0) {
218         LOG.error("Snapshot directory: " + snapshotDir + " doesn't exist");
219       }
220       return;
221     }
222
223     // get the status of the snapshots temporary directory and check if it has changes
224     // The top-level directory timestamp is not updated, so we have to check the inner-level.
225     try {
226       Path snapshotTmpDir = new Path(snapshotDir, SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME);
227       FileStatus tempDirStatus = fs.getFileStatus(snapshotTmpDir);
228       lastTimestamp = Math.min(lastTimestamp, tempDirStatus.getModificationTime());
229       hasChanges |= (lastTimestamp >= lastModifiedTime);
230       if (!hasChanges) {
231         FileStatus[] tmpSnapshots = FSUtils.listStatus(fs, snapshotDir);
232         if (tmpSnapshots != null) {
233           for (FileStatus dirStatus: tmpSnapshots) {
234             lastTimestamp = Math.min(lastTimestamp, dirStatus.getModificationTime());
235           }
236           hasChanges |= (lastTimestamp >= lastModifiedTime);
237         }
238       }
239     } catch (FileNotFoundException e) {
240       // Nothing todo, if the tmp dir is empty
241     }
242
243     // if the snapshot directory wasn't modified since we last check, we are done
244     if (!hasChanges) {
245       return;
246     }
247
248     // directory was modified, so we need to reload our cache
249     // there could be a slight race here where we miss the cache, check the directory modification
250     // time, then someone updates the directory, causing us to not scan the directory again.
251     // However, snapshot directories are only created once, so this isn't an issue.
252
253     // 1. update the modified time
254     this.lastModifiedTime = lastTimestamp;
255
256     // 2.clear the cache
257     this.cache.clear();
258     Map<String, SnapshotDirectoryInfo> known = new HashMap<String, SnapshotDirectoryInfo>();
259
260     // 3. check each of the snapshot directories
261     FileStatus[] snapshots = FSUtils.listStatus(fs, snapshotDir);
262     if (snapshots == null) {
263       // remove all the remembered snapshots because we don't have any left
264       if (LOG.isDebugEnabled() && this.snapshots.size() > 0) {
265         LOG.debug("No snapshots on-disk, cache empty");
266       }
267       this.snapshots.clear();
268       return;
269     }
270
271     // 3.1 iterate through the on-disk snapshots
272     for (FileStatus snapshot : snapshots) {
273       String name = snapshot.getPath().getName();
274       // its not the tmp dir,
275       if (!name.equals(SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME)) {
276         SnapshotDirectoryInfo files = this.snapshots.remove(name);
277         // 3.1.1 if we don't know about the snapshot or its been modified, we need to update the
278         // files the latter could occur where I create a snapshot, then delete it, and then make a
279         // new snapshot with the same name. We will need to update the cache the information from
280         // that new snapshot, even though it has the same name as the files referenced have
281         // probably changed.
282         if (files == null || files.hasBeenModified(snapshot.getModificationTime())) {
283           // get all files for the snapshot and create a new info
284           Collection<String> storedFiles = fileInspector.filesUnderSnapshot(snapshot.getPath());
285           files = new SnapshotDirectoryInfo(snapshot.getModificationTime(), storedFiles);
286         }
287         // 3.2 add all the files to cache
288         this.cache.addAll(files.getFiles());
289         known.put(name, files);
290       }
291     }
292
293     // 4. set the snapshots we are tracking
294     this.snapshots.clear();
295     this.snapshots.putAll(known);
296   }
297
298   @VisibleForTesting List<String> getSnapshotsInProgress(
299     final SnapshotManager snapshotManager) throws IOException {
300     List<String> snapshotInProgress = Lists.newArrayList();
301     // only add those files to the cache, but not to the known snapshots
302     Path snapshotTmpDir = new Path(snapshotDir, SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME);
303     // only add those files to the cache, but not to the known snapshots
304     FileStatus[] running = FSUtils.listStatus(fs, snapshotTmpDir);
305     if (running != null) {
306       for (FileStatus run : running) {
307         ReentrantLock lock = null;
308         if (snapshotManager != null) {
309           lock = snapshotManager.getLocks().acquireLock(run.getPath().getName());
310         }
311         try {
312           snapshotInProgress.addAll(fileInspector.filesUnderSnapshot(run.getPath()));
313         } catch (CorruptedSnapshotException e) {
314           // See HBASE-16464
315           if (e.getCause() instanceof FileNotFoundException) {
316             // If the snapshot is corrupt, we will delete it
317             fs.delete(run.getPath(), true);
318             LOG.warn("delete the " + run.getPath() + " due to exception:", e.getCause());
319           } else {
320             throw e;
321           }
322         } finally {
323           if (lock != null) {
324             lock.unlock();
325           }
326         }
327       }
328     }
329     return snapshotInProgress;
330   }
331
332   /**
333    * Simple helper task that just periodically attempts to refresh the cache
334    */
335   public class RefreshCacheTask extends TimerTask {
336     @Override
337     public void run() {
338       try {
339         SnapshotFileCache.this.refreshCache();
340       } catch (IOException e) {
341         LOG.warn("Failed to refresh snapshot hfile cache!", e);
342       }
343     }
344   }
345
346   @Override
347   public void stop(String why) {
348     if (!this.stop) {
349       this.stop = true;
350       this.refreshTimer.cancel();
351     }
352
353   }
354
355   @Override
356   public boolean isStopped() {
357     return this.stop;
358   }
359
360   /**
361    * Information about a snapshot directory
362    */
363   private static class SnapshotDirectoryInfo {
364     long lastModified;
365     Collection<String> files;
366
367     public SnapshotDirectoryInfo(long mtime, Collection<String> files) {
368       this.lastModified = mtime;
369       this.files = files;
370     }
371
372     /**
373      * @return the hfiles in the snapshot when <tt>this</tt> was made.
374      */
375     public Collection<String> getFiles() {
376       return this.files;
377     }
378
379     /**
380      * Check if the snapshot directory has been modified
381      * @param mtime current modification time of the directory
382      * @return <tt>true</tt> if it the modification time of the directory is newer time when we
383      *         created <tt>this</tt>
384      */
385     public boolean hasBeenModified(long mtime) {
386       return this.lastModified < mtime;
387     }
388   }
389 }