View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.snapshot;
19  
20  import java.io.IOException;
21  import java.util.Collections;
22  
23  import org.apache.commons.logging.Log;
24  import org.apache.commons.logging.LogFactory;
25  import org.apache.hadoop.conf.Configuration;
26  import org.apache.hadoop.fs.FSDataInputStream;
27  import org.apache.hadoop.fs.FSDataOutputStream;
28  import org.apache.hadoop.fs.FileSystem;
29  import org.apache.hadoop.fs.Path;
30  import org.apache.hadoop.fs.permission.FsPermission;
31  import org.apache.hadoop.hbase.HConstants;
32  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
33  import org.apache.hadoop.hbase.classification.InterfaceAudience;
34  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
35  import org.apache.hadoop.hbase.security.User;
36  import org.apache.hadoop.hbase.snapshot.SnapshotManifestV2;
37  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
38  import org.apache.hadoop.hbase.util.FSUtils;
39
40  /**
41   * Utility class to help manage {@link SnapshotDescription SnapshotDesriptions}.
42   * <p>
43   * Snapshots are laid out on disk like this:
44   *
45   * <pre>
46   * /hbase/.snapshots
47   *          /.tmp                &lt;---- working directory
48   *          /[snapshot name]     &lt;----- completed snapshot
49   * </pre>
50   *
51   * A completed snapshot named 'completed' then looks like (multiple regions, servers, files, etc.
52   * signified by '...' on the same directory depth).
53   *
54   * <pre>
55   * /hbase/.snapshots/completed
56   *                   .snapshotinfo          &lt;--- Description of the snapshot
57   *                   .tableinfo             &lt;--- Copy of the tableinfo
58   *                    /.logs
59   *                        /[server_name]
60   *                            /... [log files]
61   *                         ...
62   *                   /[region name]           &lt;---- All the region's information
63   *                   .regioninfo              &lt;---- Copy of the HRegionInfo
64   *                      /[column family name]
65   *                          /[hfile name]     &lt;--- name of the hfile in the real region
66   *                          ...
67   *                      ...
68   *                    ...
69   * </pre>
70   *
71   * Utility methods in this class are useful for getting the correct locations for different parts of
72   * the snapshot, as well as moving completed snapshots into place (see
73   * {@link #completeSnapshot}, and writing the
74   * {@link SnapshotDescription} to the working snapshot directory.
75   */
76  @InterfaceAudience.Private
77  public final class SnapshotDescriptionUtils {
78
79    /**
80     * Filter that only accepts completed snapshot directories
81     */
82    public static class CompletedSnaphotDirectoriesFilter extends FSUtils.BlackListDirFilter {
83
84      /**
85       * @param fs
86       */
87      public CompletedSnaphotDirectoriesFilter(FileSystem fs) {
88        super(fs, Collections.singletonList(SNAPSHOT_TMP_DIR_NAME));
89      }
90    }
91
92    private static final Log LOG = LogFactory.getLog(SnapshotDescriptionUtils.class);
93    /**
94     * Version of the fs layout for a snapshot. Future snapshots may have different file layouts,
95     * which we may need to read in differently.
96     */
97    public static final int SNAPSHOT_LAYOUT_VERSION = SnapshotManifestV2.DESCRIPTOR_VERSION;
98
99    // snapshot directory constants
100   /**
101    * The file contains the snapshot basic information and it is under the directory of a snapshot.
102    */
103   public static final String SNAPSHOTINFO_FILE = ".snapshotinfo";
104
105   /** Temporary directory under the snapshot directory to store in-progress snapshots */
106   public static final String SNAPSHOT_TMP_DIR_NAME = ".tmp";
107   // snapshot operation values
108   /** Default value if no start time is specified */
109   public static final long NO_SNAPSHOT_START_TIME_SPECIFIED = 0;
110 
111
112   public static final String MASTER_SNAPSHOT_TIMEOUT_MILLIS = "hbase.snapshot.master.timeout.millis";
113 
114   /** By default, wait 300 seconds for a snapshot to complete */
115   public static final long DEFAULT_MAX_WAIT_TIME = 60000 * 5 ;
116
117
118   /**
119    * By default, check to see if the snapshot is complete (ms)
120    * @deprecated Use {@link #DEFAULT_MAX_WAIT_TIME} instead.
121    * */
122   @Deprecated
123   public static final int SNAPSHOT_TIMEOUT_MILLIS_DEFAULT = 60000 * 5;
124
125   /**
126    * Conf key for # of ms elapsed before injecting a snapshot timeout error when waiting for
127    * completion.
128    * @deprecated Use {@link #MASTER_SNAPSHOT_TIMEOUT_MILLIS} instead.
129    */
130   @Deprecated
131   public static final String SNAPSHOT_TIMEOUT_MILLIS_KEY = "hbase.snapshot.master.timeoutMillis";
132
133   private SnapshotDescriptionUtils() {
134     // private constructor for utility class
135   }
136
137   /**
138    * @param conf {@link Configuration} from which to check for the timeout
139    * @param type type of snapshot being taken
140    * @param defaultMaxWaitTime Default amount of time to wait, if none is in the configuration
141    * @return the max amount of time the master should wait for a snapshot to complete
142    */
143   public static long getMaxMasterTimeout(Configuration conf, SnapshotDescription.Type type,
144       long defaultMaxWaitTime) {
145     String confKey;
146     switch (type) {
147     case DISABLED:
148     default:
149       confKey = MASTER_SNAPSHOT_TIMEOUT_MILLIS;
150     }
151     return Math.max(conf.getLong(confKey, defaultMaxWaitTime),
152         conf.getLong(SNAPSHOT_TIMEOUT_MILLIS_KEY, defaultMaxWaitTime));
153   }
154
155   /**
156    * Get the snapshot root directory. All the snapshots are kept under this directory, i.e.
157    * ${hbase.rootdir}/.snapshot
158    * @param rootDir hbase root directory
159    * @return the base directory in which all snapshots are kept
160    */
161   public static Path getSnapshotRootDir(final Path rootDir) {
162     return new Path(rootDir, HConstants.SNAPSHOT_DIR_NAME);
163   }
164
165   /**
166    * Get the directory for a specified snapshot. This directory is a sub-directory of snapshot root
167    * directory and all the data files for a snapshot are kept under this directory.
168    * @param snapshot snapshot being taken
169    * @param rootDir hbase root directory
170    * @return the final directory for the completed snapshot
171    */
172   public static Path getCompletedSnapshotDir(final SnapshotDescription snapshot, final Path rootDir) {
173     return getCompletedSnapshotDir(snapshot.getName(), rootDir);
174   }
175
176   /**
177    * Get the directory for a completed snapshot. This directory is a sub-directory of snapshot root
178    * directory and all the data files for a snapshot are kept under this directory.
179    * @param snapshotName name of the snapshot being taken
180    * @param rootDir hbase root directory
181    * @return the final directory for the completed snapshot
182    */
183   public static Path getCompletedSnapshotDir(final String snapshotName, final Path rootDir) {
184     return getCompletedSnapshotDir(getSnapshotsDir(rootDir), snapshotName);
185   }
186
187   /**
188    * Get the general working directory for snapshots - where they are built, where they are
189    * temporarily copied on export, etc.
190    * @param rootDir root directory of the HBase installation
191    * @return Path to the snapshot tmp directory, relative to the passed root directory
192    */
193   public static Path getWorkingSnapshotDir(final Path rootDir) {
194     return new Path(getSnapshotsDir(rootDir), SNAPSHOT_TMP_DIR_NAME);
195   }
196
197   /**
198    * Get the directory to build a snapshot, before it is finalized
199    * @param snapshot snapshot that will be built
200    * @param rootDir root directory of the hbase installation
201    * @return {@link Path} where one can build a snapshot
202    */
203   public static Path getWorkingSnapshotDir(SnapshotDescription snapshot, final Path rootDir) {
204     return getCompletedSnapshotDir(getWorkingSnapshotDir(rootDir), snapshot.getName());
205   }
206
207   /**
208    * Get the directory to build a snapshot, before it is finalized
209    * @param snapshotName name of the snapshot
210    * @param rootDir root directory of the hbase installation
211    * @return {@link Path} where one can build a snapshot
212    */
213   public static Path getWorkingSnapshotDir(String snapshotName, final Path rootDir) {
214     return getCompletedSnapshotDir(getWorkingSnapshotDir(rootDir), snapshotName);
215   }
216
217   /**
218    * Get the directory to store the snapshot instance
219    * @param snapshotsDir hbase-global directory for storing all snapshots
220    * @param snapshotName name of the snapshot to take
221    * @return the final directory for the completed snapshot
222    */
223   private static final Path getCompletedSnapshotDir(final Path snapshotsDir, String snapshotName) {
224     return new Path(snapshotsDir, snapshotName);
225   }
226
227   /**
228    * @param rootDir hbase root directory
229    * @return the directory for all completed snapshots;
230    */
231   public static final Path getSnapshotsDir(Path rootDir) {
232     return new Path(rootDir, HConstants.SNAPSHOT_DIR_NAME);
233   }
234
235   /**
236    * Convert the passed snapshot description into a 'full' snapshot description based on default
237    * parameters, if none have been supplied. This resolves any 'optional' parameters that aren't
238    * supplied to their default values.
239    * @param snapshot general snapshot descriptor
240    * @param conf Configuration to read configured snapshot defaults if snapshot is not complete
241    * @return a valid snapshot description
242    * @throws IllegalArgumentException if the {@link SnapshotDescription} is not a complete
243    *           {@link SnapshotDescription}.
244    */
245   public static SnapshotDescription validate(SnapshotDescription snapshot, Configuration conf)
246       throws IllegalArgumentException {
247     if (!snapshot.hasTable()) {
248       throw new IllegalArgumentException(
249         "Descriptor doesn't apply to a table, so we can't build it.");
250     }
251
252     // set the creation time, if one hasn't been set
253     long time = snapshot.getCreationTime();
254     if (time == SnapshotDescriptionUtils.NO_SNAPSHOT_START_TIME_SPECIFIED) {
255       time = EnvironmentEdgeManager.currentTime();
256       LOG.debug("Creation time not specified, setting to:" + time + " (current time:"
257           + EnvironmentEdgeManager.currentTime() + ").");
258       SnapshotDescription.Builder builder = snapshot.toBuilder();
259       builder.setCreationTime(time);
260       snapshot = builder.build();
261     }
262     return snapshot;
263   }
264
265   /**
266    * Write the snapshot description into the working directory of a snapshot
267    * @param snapshot description of the snapshot being taken
268    * @param workingDir working directory of the snapshot
269    * @param fs {@link FileSystem} on which the snapshot should be taken
270    * @throws IOException if we can't reach the filesystem and the file cannot be cleaned up on
271    *           failure
272    */
273   public static void writeSnapshotInfo(SnapshotDescription snapshot, Path workingDir, FileSystem fs)
274       throws IOException {
275     FsPermission perms = FSUtils.getFilePermissions(fs, fs.getConf(),
276       HConstants.DATA_FILE_UMASK_KEY);
277     Path snapshotInfo = new Path(workingDir, SnapshotDescriptionUtils.SNAPSHOTINFO_FILE);
278     try {
279       FSDataOutputStream out = FSUtils.create(fs, snapshotInfo, perms, true);
280       try {
281         snapshot.writeTo(out);
282       } finally {
283         out.close();
284       }
285     } catch (IOException e) {
286       // if we get an exception, try to remove the snapshot info
287       if (!fs.delete(snapshotInfo, false)) {
288         String msg = "Couldn't delete snapshot info file: " + snapshotInfo;
289         LOG.error(msg);
290         throw new IOException(msg);
291       }
292     }
293   }
294
295   /**
296    * Read in the {@link org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription} stored for the snapshot in the passed directory
297    * @param fs filesystem where the snapshot was taken
298    * @param snapshotDir directory where the snapshot was stored
299    * @return the stored snapshot description
300    * @throws CorruptedSnapshotException if the
301    * snapshot cannot be read
302    */
303   public static SnapshotDescription readSnapshotInfo(FileSystem fs, Path snapshotDir)
304       throws CorruptedSnapshotException {
305     Path snapshotInfo = new Path(snapshotDir, SNAPSHOTINFO_FILE);
306     try {
307       FSDataInputStream in = null;
308       try {
309         in = fs.open(snapshotInfo);
310         SnapshotDescription desc = SnapshotDescription.parseFrom(in);
311         return desc;
312       } finally {
313         if (in != null) in.close();
314       }
315     } catch (IOException e) {
316       throw new CorruptedSnapshotException("Couldn't read snapshot info from:" + snapshotInfo, e);
317     }
318   }
319
320   /**
321    * Move the finished snapshot to its final, publicly visible directory - this marks the snapshot
322    * as 'complete'.
323    * @param snapshot description of the snapshot being tabken
324    * @param rootdir root directory of the hbase installation
325    * @param workingDir directory where the in progress snapshot was built
326    * @param fs {@link FileSystem} where the snapshot was built
327    * @throws org.apache.hadoop.hbase.snapshot.SnapshotCreationException if the
328    * snapshot could not be moved
329    * @throws IOException the filesystem could not be reached
330    */
331   public static void completeSnapshot(SnapshotDescription snapshot, Path rootdir, Path workingDir,
332       FileSystem fs) throws SnapshotCreationException, IOException {
333     Path finishedDir = getCompletedSnapshotDir(snapshot, rootdir);
334     LOG.debug("Snapshot is done, just moving the snapshot from " + workingDir + " to "
335         + finishedDir);
336     if (!fs.rename(workingDir, finishedDir)) {
337       throw new SnapshotCreationException(
338           "Failed to move working directory(" + workingDir + ") to completed directory("
339               + finishedDir + ").", ProtobufUtil.createSnapshotDesc(snapshot));
340     }
341   }
342
343   /**
344    * Check if the user is this table snapshot's owner
345    * @param snapshot the table snapshot description
346    * @param user the user
347    * @return true if the user is the owner of the snapshot,
348    *         false otherwise or the snapshot owner field is not present.
349    */
350   public static boolean isSnapshotOwner(final SnapshotDescription snapshot, final User user) {
351     if (user == null) return false;
352     if (!snapshot.hasOwner()) return false;
353     return snapshot.getOwner().equals(user.getShortName());
354   }
355 }