View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.snapshot;
19  
20  import java.io.FileNotFoundException;
21  import java.io.IOException;
22  import java.util.Collections;
23
24  import org.apache.commons.logging.Log;
25  import org.apache.commons.logging.LogFactory;
26  import org.apache.hadoop.conf.Configuration;
27  import org.apache.hadoop.fs.FSDataInputStream;
28  import org.apache.hadoop.fs.FSDataOutputStream;
29  import org.apache.hadoop.fs.FileSystem;
30  import org.apache.hadoop.fs.Path;
31  import org.apache.hadoop.fs.permission.FsPermission;
32  import org.apache.hadoop.hbase.HConstants;
33  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
34  import org.apache.hadoop.hbase.classification.InterfaceAudience;
35  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
36  import org.apache.hadoop.hbase.security.User;
37  import org.apache.hadoop.hbase.snapshot.SnapshotManifestV2;
38  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
39  import org.apache.hadoop.hbase.util.FSUtils;
40
41  /**
42   * Utility class to help manage {@link SnapshotDescription SnapshotDesriptions}.
43   * <p>
44   * Snapshots are laid out on disk like this:
45   *
46   * <pre>
47   * /hbase/.snapshots
48   *          /.tmp                &lt;---- working directory
49   *          /[snapshot name]     &lt;----- completed snapshot
50   * </pre>
51   *
52   * A completed snapshot named 'completed' then looks like (multiple regions, servers, files, etc.
53   * signified by '...' on the same directory depth).
54   *
55   * <pre>
56   * /hbase/.snapshots/completed
57   *                   .snapshotinfo          &lt;--- Description of the snapshot
58   *                   .tableinfo             &lt;--- Copy of the tableinfo
59   *                    /.logs
60   *                        /[server_name]
61   *                            /... [log files]
62   *                         ...
63   *                   /[region name]           &lt;---- All the region's information
64   *                   .regioninfo              &lt;---- Copy of the HRegionInfo
65   *                      /[column family name]
66   *                          /[hfile name]     &lt;--- name of the hfile in the real region
67   *                          ...
68   *                      ...
69   *                    ...
70   * </pre>
71   *
72   * Utility methods in this class are useful for getting the correct locations for different parts of
73   * the snapshot, as well as moving completed snapshots into place (see
74   * {@link #completeSnapshot}, and writing the
75   * {@link SnapshotDescription} to the working snapshot directory.
76   */
77  @InterfaceAudience.Private
78  public final class SnapshotDescriptionUtils {
79
80    /**
81     * Filter that only accepts completed snapshot directories
82     */
83    public static class CompletedSnaphotDirectoriesFilter extends FSUtils.BlackListDirFilter {
84
85      /**
86       * @param fs
87       */
88      public CompletedSnaphotDirectoriesFilter(FileSystem fs) {
89        super(fs, Collections.singletonList(SNAPSHOT_TMP_DIR_NAME));
90      }
91    }
92
93    private static final Log LOG = LogFactory.getLog(SnapshotDescriptionUtils.class);
94    /**
95     * Version of the fs layout for a snapshot. Future snapshots may have different file layouts,
96     * which we may need to read in differently.
97     */
98    public static final int SNAPSHOT_LAYOUT_VERSION = SnapshotManifestV2.DESCRIPTOR_VERSION;
99
100   // snapshot directory constants
101   /**
102    * The file contains the snapshot basic information and it is under the directory of a snapshot.
103    */
104   public static final String SNAPSHOTINFO_FILE = ".snapshotinfo";
105
106   /** Temporary directory under the snapshot directory to store in-progress snapshots */
107   public static final String SNAPSHOT_TMP_DIR_NAME = ".tmp";
108
109   /** This tag will be created in in-progess snapshots */
110   public static final String SNAPSHOT_IN_PROGRESS = ".inprogress";
111   // snapshot operation values
112   /** Default value if no start time is specified */
113   public static final long NO_SNAPSHOT_START_TIME_SPECIFIED = 0;
114
115
116   public static final String MASTER_SNAPSHOT_TIMEOUT_MILLIS = "hbase.snapshot.master.timeout.millis";
117
118   /** By default, wait 300 seconds for a snapshot to complete */
119   public static final long DEFAULT_MAX_WAIT_TIME = 60000 * 5 ;
120
121
122   /**
123    * By default, check to see if the snapshot is complete (ms)
124    * @deprecated Use {@link #DEFAULT_MAX_WAIT_TIME} instead.
125    * */
126   @Deprecated
127   public static final int SNAPSHOT_TIMEOUT_MILLIS_DEFAULT = 60000 * 5;
128
129   /**
130    * Conf key for # of ms elapsed before injecting a snapshot timeout error when waiting for
131    * completion.
132    * @deprecated Use {@link #MASTER_SNAPSHOT_TIMEOUT_MILLIS} instead.
133    */
134   @Deprecated
135   public static final String SNAPSHOT_TIMEOUT_MILLIS_KEY = "hbase.snapshot.master.timeoutMillis";
136
137   private SnapshotDescriptionUtils() {
138     // private constructor for utility class
139   }
140
141   /**
142    * @param conf {@link Configuration} from which to check for the timeout
143    * @param type type of snapshot being taken
144    * @param defaultMaxWaitTime Default amount of time to wait, if none is in the configuration
145    * @return the max amount of time the master should wait for a snapshot to complete
146    */
147   public static long getMaxMasterTimeout(Configuration conf, SnapshotDescription.Type type,
148       long defaultMaxWaitTime) {
149     String confKey;
150     switch (type) {
151     case DISABLED:
152     default:
153       confKey = MASTER_SNAPSHOT_TIMEOUT_MILLIS;
154     }
155     return Math.max(conf.getLong(confKey, defaultMaxWaitTime),
156         conf.getLong(SNAPSHOT_TIMEOUT_MILLIS_KEY, defaultMaxWaitTime));
157   }
158
159   /**
160    * Get the snapshot root directory. All the snapshots are kept under this directory, i.e.
161    * ${hbase.rootdir}/.snapshot
162    * @param rootDir hbase root directory
163    * @return the base directory in which all snapshots are kept
164    */
165   public static Path getSnapshotRootDir(final Path rootDir) {
166     return new Path(rootDir, HConstants.SNAPSHOT_DIR_NAME);
167   }
168
169   /**
170    * Get the directory for a specified snapshot. This directory is a sub-directory of snapshot root
171    * directory and all the data files for a snapshot are kept under this directory.
172    * @param snapshot snapshot being taken
173    * @param rootDir hbase root directory
174    * @return the final directory for the completed snapshot
175    */
176   public static Path getCompletedSnapshotDir(final SnapshotDescription snapshot, final Path rootDir) {
177     return getCompletedSnapshotDir(snapshot.getName(), rootDir);
178   }
179
180   /**
181    * Get the directory for a completed snapshot. This directory is a sub-directory of snapshot root
182    * directory and all the data files for a snapshot are kept under this directory.
183    * @param snapshotName name of the snapshot being taken
184    * @param rootDir hbase root directory
185    * @return the final directory for the completed snapshot
186    */
187   public static Path getCompletedSnapshotDir(final String snapshotName, final Path rootDir) {
188     return getCompletedSnapshotDir(getSnapshotsDir(rootDir), snapshotName);
189   }
190
191   /**
192    * Get the general working directory for snapshots - where they are built, where they are
193    * temporarily copied on export, etc.
194    * @param rootDir root directory of the HBase installation
195    * @return Path to the snapshot tmp directory, relative to the passed root directory
196    */
197   public static Path getWorkingSnapshotDir(final Path rootDir) {
198     return new Path(getSnapshotsDir(rootDir), SNAPSHOT_TMP_DIR_NAME);
199   }
200
201   /**
202    * Get the directory to build a snapshot, before it is finalized
203    * @param snapshot snapshot that will be built
204    * @param rootDir root directory of the hbase installation
205    * @return {@link Path} where one can build a snapshot
206    */
207   public static Path getWorkingSnapshotDir(SnapshotDescription snapshot, final Path rootDir) {
208     return getCompletedSnapshotDir(getWorkingSnapshotDir(rootDir), snapshot.getName());
209   }
210
211   /**
212    * Get the directory to build a snapshot, before it is finalized
213    * @param snapshotName name of the snapshot
214    * @param rootDir root directory of the hbase installation
215    * @return {@link Path} where one can build a snapshot
216    */
217   public static Path getWorkingSnapshotDir(String snapshotName, final Path rootDir) {
218     return getCompletedSnapshotDir(getWorkingSnapshotDir(rootDir), snapshotName);
219   }
220
221   /**
222    * Get the directory to store the snapshot instance
223    * @param snapshotsDir hbase-global directory for storing all snapshots
224    * @param snapshotName name of the snapshot to take
225    * @return the final directory for the completed snapshot
226    */
227   private static final Path getCompletedSnapshotDir(final Path snapshotsDir, String snapshotName) {
228     return new Path(snapshotsDir, snapshotName);
229   }
230
231   /**
232    * @param rootDir hbase root directory
233    * @return the directory for all completed snapshots;
234    */
235   public static final Path getSnapshotsDir(Path rootDir) {
236     return new Path(rootDir, HConstants.SNAPSHOT_DIR_NAME);
237   }
238
239   /**
240    * Convert the passed snapshot description into a 'full' snapshot description based on default
241    * parameters, if none have been supplied. This resolves any 'optional' parameters that aren't
242    * supplied to their default values.
243    * @param snapshot general snapshot descriptor
244    * @param conf Configuration to read configured snapshot defaults if snapshot is not complete
245    * @return a valid snapshot description
246    * @throws IllegalArgumentException if the {@link SnapshotDescription} is not a complete
247    *           {@link SnapshotDescription}.
248    */
249   public static SnapshotDescription validate(SnapshotDescription snapshot, Configuration conf)
250       throws IllegalArgumentException {
251     if (!snapshot.hasTable()) {
252       throw new IllegalArgumentException(
253         "Descriptor doesn't apply to a table, so we can't build it.");
254     }
255
256     // set the creation time, if one hasn't been set
257     long time = snapshot.getCreationTime();
258     if (time == SnapshotDescriptionUtils.NO_SNAPSHOT_START_TIME_SPECIFIED) {
259       time = EnvironmentEdgeManager.currentTime();
260       LOG.debug("Creation time not specified, setting to:" + time + " (current time:"
261           + EnvironmentEdgeManager.currentTime() + ").");
262       SnapshotDescription.Builder builder = snapshot.toBuilder();
263       builder.setCreationTime(time);
264       snapshot = builder.build();
265     }
266     return snapshot;
267   }
268
269   /**
270    * Write the snapshot description into the working directory of a snapshot
271    * @param snapshot description of the snapshot being taken
272    * @param workingDir working directory of the snapshot
273    * @param fs {@link FileSystem} on which the snapshot should be taken
274    * @throws IOException if we can't reach the filesystem and the file cannot be cleaned up on
275    *           failure
276    */
277   public static void writeSnapshotInfo(SnapshotDescription snapshot, Path workingDir, FileSystem fs)
278       throws IOException {
279     FsPermission perms = FSUtils.getFilePermissions(fs, fs.getConf(),
280       HConstants.DATA_FILE_UMASK_KEY);
281     Path snapshotInfo = new Path(workingDir, SnapshotDescriptionUtils.SNAPSHOTINFO_FILE);
282     try {
283       FSDataOutputStream out = FSUtils.create(fs, snapshotInfo, perms, true);
284       try {
285         snapshot.writeTo(out);
286       } finally {
287         out.close();
288       }
289     } catch (IOException e) {
290       // if we get an exception, try to remove the snapshot info
291       if (!fs.delete(snapshotInfo, false)) {
292         String msg = "Couldn't delete snapshot info file: " + snapshotInfo;
293         LOG.error(msg);
294         throw new IOException(msg);
295       }
296     }
297   }
298
299   /**
300    * Create in-progress tag under .tmp of in-progress snapshot
301    * */
302   public static void createInProgressTag(Path workingDir, FileSystem fs) throws IOException {
303     FsPermission perms = FSUtils.getFilePermissions(fs, fs.getConf(),
304       HConstants.DATA_FILE_UMASK_KEY);
305     Path snapshot_in_progress = new Path(workingDir, SnapshotDescriptionUtils.SNAPSHOT_IN_PROGRESS);
306     FSUtils.create(fs, snapshot_in_progress, perms, true);
307   }
308
309   /**
310    * Read in the {@link org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription} stored for the snapshot in the passed directory
311    * @param fs filesystem where the snapshot was taken
312    * @param snapshotDir directory where the snapshot was stored
313    * @return the stored snapshot description
314    * @throws CorruptedSnapshotException if the
315    * snapshot cannot be read
316    */
317   public static SnapshotDescription readSnapshotInfo(FileSystem fs, Path snapshotDir)
318       throws CorruptedSnapshotException {
319     Path snapshotInfo = new Path(snapshotDir, SNAPSHOTINFO_FILE);
320     try {
321       FSDataInputStream in = null;
322       try {
323         in = fs.open(snapshotInfo);
324         SnapshotDescription desc = SnapshotDescription.parseFrom(in);
325         return desc;
326       } finally {
327         if (in != null) in.close();
328       }
329     } catch (IOException e) {
330       throw new CorruptedSnapshotException("Couldn't read snapshot info from:" + snapshotInfo, e);
331     }
332   }
333
334   /**
335    * Move the finished snapshot to its final, publicly visible directory - this marks the snapshot
336    * as 'complete'.
337    * @param snapshot description of the snapshot being tabken
338    * @param rootdir root directory of the hbase installation
339    * @param workingDir directory where the in progress snapshot was built
340    * @param fs {@link FileSystem} where the snapshot was built
341    * @throws org.apache.hadoop.hbase.snapshot.SnapshotCreationException if the
342    * snapshot could not be moved
343    * @throws IOException the filesystem could not be reached
344    */
345   public static void completeSnapshot(SnapshotDescription snapshot, Path rootdir, Path workingDir,
346       FileSystem fs) throws SnapshotCreationException, IOException {
347     Path finishedDir = getCompletedSnapshotDir(snapshot, rootdir);
348     LOG.debug("Snapshot is done, just moving the snapshot from " + workingDir + " to "
349         + finishedDir);
350     if (!fs.rename(workingDir, finishedDir)) {
351       throw new SnapshotCreationException(
352           "Failed to move working directory(" + workingDir + ") to completed directory("
353               + finishedDir + ").", ProtobufUtil.createSnapshotDesc(snapshot));
354     }
355   }
356
357   /**
358    * Check if the user is this table snapshot's owner
359    * @param snapshot the table snapshot description
360    * @param user the user
361    * @return true if the user is the owner of the snapshot,
362    *         false otherwise or the snapshot owner field is not present.
363    */
364   public static boolean isSnapshotOwner(final SnapshotDescription snapshot, final User user) {
365     if (user == null) return false;
366     if (!snapshot.hasOwner()) return false;
367     return snapshot.getOwner().equals(user.getShortName());
368   }
369 }