View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.snapshot;
19  
20  import java.io.IOException;
21  import java.util.Collections;
22  
23  import org.apache.commons.logging.Log;
24  import org.apache.commons.logging.LogFactory;
25  import org.apache.hadoop.conf.Configuration;
26  import org.apache.hadoop.fs.FSDataInputStream;
27  import org.apache.hadoop.fs.FSDataOutputStream;
28  import org.apache.hadoop.fs.FileSystem;
29  import org.apache.hadoop.fs.Path;
30  import org.apache.hadoop.fs.permission.FsPermission;
31  import org.apache.hadoop.hbase.HConstants;
32  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
33  import org.apache.hadoop.hbase.security.User;
34  import org.apache.hadoop.hbase.snapshot.SnapshotManifestV2;
35  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
36  import org.apache.hadoop.hbase.util.FSUtils;
37  
38  /**
39   * Utility class to help manage {@link SnapshotDescription SnapshotDesriptions}.
40   * <p>
41   * Snapshots are laid out on disk like this:
42   *
43   * <pre>
44   * /hbase/.snapshots
45   *          /.tmp                &lt;---- working directory
46   *          /[snapshot name]     &lt;----- completed snapshot
47   * </pre>
48   *
49   * A completed snapshot named 'completed' then looks like (multiple regions, servers, files, etc.
50   * signified by '...' on the same directory depth).
51   *
52   * <pre>
53   * /hbase/.snapshots/completed
54   *                   .snapshotinfo          &lt;--- Description of the snapshot
55   *                   .tableinfo             &lt;--- Copy of the tableinfo
56   *                    /.logs
57   *                        /[server_name]
58   *                            /... [log files]
59   *                         ...
60   *                   /[region name]           &lt;---- All the region's information
61   *                   .regioninfo              &lt;---- Copy of the HRegionInfo
62   *                      /[column family name]
63   *                          /[hfile name]     &lt;--- name of the hfile in the real region
64   *                          ...
65   *                      ...
66   *                    ...
67   * </pre>
68   *
69   * Utility methods in this class are useful for getting the correct locations for different parts of
70   * the snapshot, as well as moving completed snapshots into place (see
71   * {@link #completeSnapshot}, and writing the
72   * {@link SnapshotDescription} to the working snapshot directory.
73   */
74  public class SnapshotDescriptionUtils {
75  
76    /**
77     * Filter that only accepts completed snapshot directories
78     */
79    public static class CompletedSnaphotDirectoriesFilter extends FSUtils.BlackListDirFilter {
80  
81      /**
82       * @param fs
83       */
84      public CompletedSnaphotDirectoriesFilter(FileSystem fs) {
85        super(fs, Collections.singletonList(SNAPSHOT_TMP_DIR_NAME));
86      }
87    }
88  
89    private static final Log LOG = LogFactory.getLog(SnapshotDescriptionUtils.class);
90    /**
91     * Version of the fs layout for a snapshot. Future snapshots may have different file layouts,
92     * which we may need to read in differently.
93     */
94    public static final int SNAPSHOT_LAYOUT_VERSION = SnapshotManifestV2.DESCRIPTOR_VERSION;
95  
96    // snapshot directory constants
97    /**
98     * The file contains the snapshot basic information and it is under the directory of a snapshot.
99     */
100   public static final String SNAPSHOTINFO_FILE = ".snapshotinfo";
101 
102   /** Temporary directory under the snapshot directory to store in-progress snapshots */
103   public static final String SNAPSHOT_TMP_DIR_NAME = ".tmp";
104   // snapshot operation values
105   /** Default value if no start time is specified */
106   public static final long NO_SNAPSHOT_START_TIME_SPECIFIED = 0;
107 
108 
109   public static final String MASTER_SNAPSHOT_TIMEOUT_MILLIS = "hbase.snapshot.master.timeout.millis";
110 
111   /** By default, wait 300 seconds for a snapshot to complete */
112   public static final long DEFAULT_MAX_WAIT_TIME = 60000 * 5 ;
113 
114 
115   /**
116    * By default, check to see if the snapshot is complete (ms)
117    * @deprecated Use {@link #DEFAULT_MAX_WAIT_TIME} instead.
118    * */
119   @Deprecated
120   public static final int SNAPSHOT_TIMEOUT_MILLIS_DEFAULT = 60000 * 5;
121 
122   /**
123    * Conf key for # of ms elapsed before injecting a snapshot timeout error when waiting for
124    * completion.
125    * @deprecated Use {@link #MASTER_SNAPSHOT_TIMEOUT_MILLIS} instead.
126    */
127   @Deprecated
128   public static final String SNAPSHOT_TIMEOUT_MILLIS_KEY = "hbase.snapshot.master.timeoutMillis";
129 
130   private SnapshotDescriptionUtils() {
131     // private constructor for utility class
132   }
133 
134   /**
135    * @param conf {@link Configuration} from which to check for the timeout
136    * @param type type of snapshot being taken
137    * @param defaultMaxWaitTime Default amount of time to wait, if none is in the configuration
138    * @return the max amount of time the master should wait for a snapshot to complete
139    */
140   public static long getMaxMasterTimeout(Configuration conf, SnapshotDescription.Type type,
141       long defaultMaxWaitTime) {
142     String confKey;
143     switch (type) {
144     case DISABLED:
145     default:
146       confKey = MASTER_SNAPSHOT_TIMEOUT_MILLIS;
147     }
148     return Math.max(conf.getLong(confKey, defaultMaxWaitTime),
149         conf.getLong(SNAPSHOT_TIMEOUT_MILLIS_KEY, defaultMaxWaitTime));
150   }
151 
152   /**
153    * Get the snapshot root directory. All the snapshots are kept under this directory, i.e.
154    * ${hbase.rootdir}/.snapshot
155    * @param rootDir hbase root directory
156    * @return the base directory in which all snapshots are kept
157    */
158   public static Path getSnapshotRootDir(final Path rootDir) {
159     return new Path(rootDir, HConstants.SNAPSHOT_DIR_NAME);
160   }
161 
162   /**
163    * Get the directory for a specified snapshot. This directory is a sub-directory of snapshot root
164    * directory and all the data files for a snapshot are kept under this directory.
165    * @param snapshot snapshot being taken
166    * @param rootDir hbase root directory
167    * @return the final directory for the completed snapshot
168    */
169   public static Path getCompletedSnapshotDir(final SnapshotDescription snapshot, final Path rootDir) {
170     return getCompletedSnapshotDir(snapshot.getName(), rootDir);
171   }
172 
173   /**
174    * Get the directory for a completed snapshot. This directory is a sub-directory of snapshot root
175    * directory and all the data files for a snapshot are kept under this directory.
176    * @param snapshotName name of the snapshot being taken
177    * @param rootDir hbase root directory
178    * @return the final directory for the completed snapshot
179    */
180   public static Path getCompletedSnapshotDir(final String snapshotName, final Path rootDir) {
181     return getCompletedSnapshotDir(getSnapshotsDir(rootDir), snapshotName);
182   }
183 
184   /**
185    * Get the general working directory for snapshots - where they are built, where they are
186    * temporarily copied on export, etc.
187    * @param rootDir root directory of the HBase installation
188    * @return Path to the snapshot tmp directory, relative to the passed root directory
189    */
190   public static Path getWorkingSnapshotDir(final Path rootDir) {
191     return new Path(getSnapshotsDir(rootDir), SNAPSHOT_TMP_DIR_NAME);
192   }
193 
194   /**
195    * Get the directory to build a snapshot, before it is finalized
196    * @param snapshot snapshot that will be built
197    * @param rootDir root directory of the hbase installation
198    * @return {@link Path} where one can build a snapshot
199    */
200   public static Path getWorkingSnapshotDir(SnapshotDescription snapshot, final Path rootDir) {
201     return getCompletedSnapshotDir(getWorkingSnapshotDir(rootDir), snapshot.getName());
202   }
203 
204   /**
205    * Get the directory to build a snapshot, before it is finalized
206    * @param snapshotName name of the snapshot
207    * @param rootDir root directory of the hbase installation
208    * @return {@link Path} where one can build a snapshot
209    */
210   public static Path getWorkingSnapshotDir(String snapshotName, final Path rootDir) {
211     return getCompletedSnapshotDir(getWorkingSnapshotDir(rootDir), snapshotName);
212   }
213 
214   /**
215    * Get the directory to store the snapshot instance
216    * @param snapshotsDir hbase-global directory for storing all snapshots
217    * @param snapshotName name of the snapshot to take
218    * @return the final directory for the completed snapshot
219    */
220   private static final Path getCompletedSnapshotDir(final Path snapshotsDir, String snapshotName) {
221     return new Path(snapshotsDir, snapshotName);
222   }
223 
224   /**
225    * @param rootDir hbase root directory
226    * @return the directory for all completed snapshots;
227    */
228   public static final Path getSnapshotsDir(Path rootDir) {
229     return new Path(rootDir, HConstants.SNAPSHOT_DIR_NAME);
230   }
231 
232   /**
233    * Convert the passed snapshot description into a 'full' snapshot description based on default
234    * parameters, if none have been supplied. This resolves any 'optional' parameters that aren't
235    * supplied to their default values.
236    * @param snapshot general snapshot descriptor
237    * @param conf Configuration to read configured snapshot defaults if snapshot is not complete
238    * @return a valid snapshot description
239    * @throws IllegalArgumentException if the {@link SnapshotDescription} is not a complete
240    *           {@link SnapshotDescription}.
241    */
242   public static SnapshotDescription validate(SnapshotDescription snapshot, Configuration conf)
243       throws IllegalArgumentException {
244     if (!snapshot.hasTable()) {
245       throw new IllegalArgumentException(
246         "Descriptor doesn't apply to a table, so we can't build it.");
247     }
248 
249     // set the creation time, if one hasn't been set
250     long time = snapshot.getCreationTime();
251     if (time == SnapshotDescriptionUtils.NO_SNAPSHOT_START_TIME_SPECIFIED) {
252       time = EnvironmentEdgeManager.currentTime();
253       LOG.debug("Creation time not specified, setting to:" + time + " (current time:"
254           + EnvironmentEdgeManager.currentTime() + ").");
255       SnapshotDescription.Builder builder = snapshot.toBuilder();
256       builder.setCreationTime(time);
257       snapshot = builder.build();
258     }
259     return snapshot;
260   }
261 
262   /**
263    * Write the snapshot description into the working directory of a snapshot
264    * @param snapshot description of the snapshot being taken
265    * @param workingDir working directory of the snapshot
266    * @param fs {@link FileSystem} on which the snapshot should be taken
267    * @throws IOException if we can't reach the filesystem and the file cannot be cleaned up on
268    *           failure
269    */
270   public static void writeSnapshotInfo(SnapshotDescription snapshot, Path workingDir, FileSystem fs)
271       throws IOException {
272     FsPermission perms = FSUtils.getFilePermissions(fs, fs.getConf(),
273       HConstants.DATA_FILE_UMASK_KEY);
274     Path snapshotInfo = new Path(workingDir, SnapshotDescriptionUtils.SNAPSHOTINFO_FILE);
275     try {
276       FSDataOutputStream out = FSUtils.create(fs, snapshotInfo, perms, true);
277       try {
278         snapshot.writeTo(out);
279       } finally {
280         out.close();
281       }
282     } catch (IOException e) {
283       // if we get an exception, try to remove the snapshot info
284       if (!fs.delete(snapshotInfo, false)) {
285         String msg = "Couldn't delete snapshot info file: " + snapshotInfo;
286         LOG.error(msg);
287         throw new IOException(msg);
288       }
289     }
290   }
291 
292   /**
293    * Read in the {@link org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription} stored for the snapshot in the passed directory
294    * @param fs filesystem where the snapshot was taken
295    * @param snapshotDir directory where the snapshot was stored
296    * @return the stored snapshot description
297    * @throws CorruptedSnapshotException if the
298    * snapshot cannot be read
299    */
300   public static SnapshotDescription readSnapshotInfo(FileSystem fs, Path snapshotDir)
301       throws CorruptedSnapshotException {
302     Path snapshotInfo = new Path(snapshotDir, SNAPSHOTINFO_FILE);
303     try {
304       FSDataInputStream in = null;
305       try {
306         in = fs.open(snapshotInfo);
307         SnapshotDescription desc = SnapshotDescription.parseFrom(in);
308         return desc;
309       } finally {
310         if (in != null) in.close();
311       }
312     } catch (IOException e) {
313       throw new CorruptedSnapshotException("Couldn't read snapshot info from:" + snapshotInfo, e);
314     }
315   }
316 
317   /**
318    * Move the finished snapshot to its final, publicly visible directory - this marks the snapshot
319    * as 'complete'.
320    * @param snapshot description of the snapshot being tabken
321    * @param rootdir root directory of the hbase installation
322    * @param workingDir directory where the in progress snapshot was built
323    * @param fs {@link FileSystem} where the snapshot was built
324    * @throws org.apache.hadoop.hbase.snapshot.SnapshotCreationException if the
325    * snapshot could not be moved
326    * @throws IOException the filesystem could not be reached
327    */
328   public static void completeSnapshot(SnapshotDescription snapshot, Path rootdir, Path workingDir,
329       FileSystem fs) throws SnapshotCreationException, IOException {
330     Path finishedDir = getCompletedSnapshotDir(snapshot, rootdir);
331     LOG.debug("Snapshot is done, just moving the snapshot from " + workingDir + " to "
332         + finishedDir);
333     if (!fs.rename(workingDir, finishedDir)) {
334       throw new SnapshotCreationException("Failed to move working directory(" + workingDir
335           + ") to completed directory(" + finishedDir + ").", snapshot);
336     }
337   }
338 
339   /**
340    * Check if the user is this table snapshot's owner
341    * @param snapshot the table snapshot description
342    * @param user the user
343    * @return true if the user is the owner of the snapshot,
344    *         false otherwise or the snapshot owner field is not present.
345    */
346   public static boolean isSnapshotOwner(final SnapshotDescription snapshot, final User user) {
347     if (user == null) return false;
348     if (!snapshot.hasOwner()) return false;
349     return snapshot.getOwner().equals(user.getShortName());
350   }
351 }