View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.snapshot;
19  
20  import java.io.IOException;
21  
22  import org.apache.commons.logging.Log;
23  import org.apache.commons.logging.LogFactory;
24  import org.apache.hadoop.conf.Configuration;
25  import org.apache.hadoop.fs.FSDataInputStream;
26  import org.apache.hadoop.fs.FSDataOutputStream;
27  import org.apache.hadoop.fs.FileSystem;
28  import org.apache.hadoop.fs.Path;
29  import org.apache.hadoop.fs.permission.FsPermission;
30  import org.apache.hadoop.hbase.HConstants;
31  import org.apache.hadoop.hbase.HTableDescriptor;
32  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
33  import org.apache.hadoop.hbase.util.Bytes;
34  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
35  import org.apache.hadoop.hbase.util.FSUtils;
36  
37  /**
38   * Utility class to help manage {@link SnapshotDescription SnapshotDesriptions}.
39   * <p>
40   * Snapshots are laid out on disk like this:
41   *
42   * <pre>
43   * /hbase/.snapshots
44   *          /.tmp                <---- working directory
45   *          /[snapshot name]     <----- completed snapshot
46   * </pre>
47   *
48   * A completed snapshot named 'completed' then looks like (multiple regions, servers, files, etc.
49   * signified by '...' on the same directory depth).
50   *
51   * <pre>
52   * /hbase/.snapshots/completed
53   *                   .snapshotinfo          <--- Description of the snapshot
54   *                   .tableinfo             <--- Copy of the tableinfo
55   *                    /.logs
56   *                        /[server_name]
57   *                            /... [log files]
58   *                         ...
59   *                   /[region name]           <---- All the region's information
60   *                   .regioninfo              <---- Copy of the HRegionInfo
61   *                      /[column family name]
62   *                          /[hfile name]     <--- name of the hfile in the real region
63   *                          ...
64   *                      ...
65   *                    ...
66   * </pre>
67   *
68   * Utility methods in this class are useful for getting the correct locations for different parts of
69   * the snapshot, as well as moving completed snapshots into place (see
70   * {@link #completeSnapshot}, and writing the
71   * {@link SnapshotDescription} to the working snapshot directory.
72   */
73  public class SnapshotDescriptionUtils {
74  
75    /**
76     * Filter that only accepts completed snapshot directories
77     */
78    public static class CompletedSnaphotDirectoriesFilter extends FSUtils.DirFilter {
79  
80      /**
81       * @param fs
82       */
83      public CompletedSnaphotDirectoriesFilter(FileSystem fs) {
84        super(fs);
85      }
86  
87      @Override
88      public boolean accept(Path path) {
89        // only accept directories that aren't the tmp directory
90        if (super.accept(path)) {
91          return !path.getName().equals(SNAPSHOT_TMP_DIR_NAME);
92        }
93        return false;
94      }
95  
96    }
97  
98    private static final Log LOG = LogFactory.getLog(SnapshotDescriptionUtils.class);
99    /**
100    * Version of the fs layout for a snapshot. Future snapshots may have different file layouts,
101    * which we may need to read in differently.
102    */
103   public static final int SNAPSHOT_LAYOUT_VERSION = 0;
104 
105   // snapshot directory constants
106   /**
107    * The file contains the snapshot basic information and it is under the directory of a snapshot.
108    */
109   public static final String SNAPSHOTINFO_FILE = ".snapshotinfo";
110 
111   /** Temporary directory under the snapshot directory to store in-progress snapshots */
112   public static final String SNAPSHOT_TMP_DIR_NAME = ".tmp";
113   // snapshot operation values
114   /** Default value if no start time is specified */
115   public static final long NO_SNAPSHOT_START_TIME_SPECIFIED = 0;
116 
117   public static final String MASTER_SNAPSHOT_TIMEOUT_MILLIS = "hbase.snapshot.master.timeout.millis";
118 
119   /** By default, wait 60 seconds for a snapshot to complete */
120   public static final long DEFAULT_MAX_WAIT_TIME = 60000;
121 
122   private SnapshotDescriptionUtils() {
123     // private constructor for utility class
124   }
125 
126   /**
127    * Check to make sure that the description of the snapshot requested is valid
128    * @param snapshot description of the snapshot
129    * @throws IllegalArgumentException if the name of the snapshot or the name of the table to
130    *           snapshot are not valid names.
131    */
132   public static void assertSnapshotRequestIsValid(SnapshotDescription snapshot)
133       throws IllegalArgumentException {
134     // FIXME these method names is really bad - trunk will probably change
135     // .META. and -ROOT- snapshots are not allowed
136     if (HTableDescriptor.isMetaTable(Bytes.toBytes(snapshot.getTable()))) {
137       throw new IllegalArgumentException(".META. and -ROOT- snapshots are not allowed");
138     }
139     // make sure the snapshot name is valid
140     HTableDescriptor.isLegalTableName(Bytes.toBytes(snapshot.getName()), true);
141     // make sure the table name is valid
142     HTableDescriptor.isLegalTableName(Bytes.toBytes(snapshot.getTable()));
143   }
144 
145   /**
146    * @param conf {@link Configuration} from which to check for the timeout
147    * @param type type of snapshot being taken
148    * @param defaultMaxWaitTime Default amount of time to wait, if none is in the configuration
149    * @return the max amount of time the master should wait for a snapshot to complete
150    */
151   public static long getMaxMasterTimeout(Configuration conf, SnapshotDescription.Type type,
152       long defaultMaxWaitTime) {
153     String confKey;
154     switch (type) {
155     case DISABLED:
156     default:
157       confKey = MASTER_SNAPSHOT_TIMEOUT_MILLIS;
158     }
159     return conf.getLong(confKey, defaultMaxWaitTime);
160   }
161 
162   /**
163    * Get the snapshot root directory. All the snapshots are kept under this directory, i.e.
164    * ${hbase.rootdir}/.snapshot
165    * @param rootDir hbase root directory
166    * @return the base directory in which all snapshots are kept
167    */
168   public static Path getSnapshotRootDir(final Path rootDir) {
169     return new Path(rootDir, HConstants.SNAPSHOT_DIR_NAME);
170   }
171 
172   /**
173    * Get the directory for a specified snapshot. This directory is a sub-directory of snapshot root
174    * directory and all the data files for a snapshot are kept under this directory.
175    * @param snapshot snapshot being taken
176    * @param rootDir hbase root directory
177    * @return the final directory for the completed snapshot
178    */
179   public static Path getCompletedSnapshotDir(final SnapshotDescription snapshot, final Path rootDir) {
180     return getCompletedSnapshotDir(snapshot.getName(), rootDir);
181   }
182 
183   /**
184    * Get the directory for a completed snapshot. This directory is a sub-directory of snapshot root
185    * directory and all the data files for a snapshot are kept under this directory.
186    * @param snapshotName name of the snapshot being taken
187    * @param rootDir hbase root directory
188    * @return the final directory for the completed snapshot
189    */
190   public static Path getCompletedSnapshotDir(final String snapshotName, final Path rootDir) {
191     return getCompletedSnapshotDir(getSnapshotsDir(rootDir), snapshotName);
192   }
193 
194   /**
195    * Get the general working directory for snapshots - where they are built, where they are
196    * temporarily copied on export, etc.
197    * @param rootDir root directory of the HBase installation
198    * @return Path to the snapshot tmp directory, relative to the passed root directory
199    */
200   public static Path getWorkingSnapshotDir(final Path rootDir) {
201     return new Path(getSnapshotsDir(rootDir), SNAPSHOT_TMP_DIR_NAME);
202   }
203 
204   /**
205    * Get the directory to build a snapshot, before it is finalized
206    * @param snapshot snapshot that will be built
207    * @param rootDir root directory of the hbase installation
208    * @return {@link Path} where one can build a snapshot
209    */
210   public static Path getWorkingSnapshotDir(SnapshotDescription snapshot, final Path rootDir) {
211     return getCompletedSnapshotDir(getWorkingSnapshotDir(rootDir), snapshot.getName());
212   }
213 
214   /**
215    * Get the directory to build a snapshot, before it is finalized
216    * @param snapshotName name of the snapshot
217    * @param rootDir root directory of the hbase installation
218    * @return {@link Path} where one can build a snapshot
219    */
220   public static Path getWorkingSnapshotDir(String snapshotName, final Path rootDir) {
221     return getCompletedSnapshotDir(getWorkingSnapshotDir(rootDir), snapshotName);
222   }
223 
224   /**
225    * Get the directory to store the snapshot instance
226    * @param snapshotsDir hbase-global directory for storing all snapshots
227    * @param snapshotName name of the snapshot to take
228    * @return
229    */
230   private static final Path getCompletedSnapshotDir(final Path snapshotsDir, String snapshotName) {
231     return new Path(snapshotsDir, snapshotName);
232   }
233 
234   /**
235    * @param rootDir hbase root directory
236    * @return the directory for all completed snapshots;
237    */
238   public static final Path getSnapshotsDir(Path rootDir) {
239     return new Path(rootDir, HConstants.SNAPSHOT_DIR_NAME);
240   }
241 
242   /**
243    * Convert the passed snapshot description into a 'full' snapshot description based on default
244    * parameters, if none have been supplied. This resolves any 'optional' parameters that aren't
245    * supplied to their default values.
246    * @param snapshot general snapshot descriptor
247    * @param conf Configuration to read configured snapshot defaults if snapshot is not complete
248    * @return a valid snapshot description
249    * @throws IllegalArgumentException if the {@link SnapshotDescription} is not a complete
250    *           {@link SnapshotDescription}.
251    */
252   public static SnapshotDescription validate(SnapshotDescription snapshot, Configuration conf)
253       throws IllegalArgumentException {
254     if (!snapshot.hasTable()) {
255       throw new IllegalArgumentException(
256         "Descriptor doesn't apply to a table, so we can't build it.");
257     }
258 
259     // set the creation time, if one hasn't been set
260     long time = snapshot.getCreationTime();
261     if (time == SnapshotDescriptionUtils.NO_SNAPSHOT_START_TIME_SPECIFIED) {
262       time = EnvironmentEdgeManager.currentTimeMillis();
263       LOG.debug("Creation time not specified, setting to:" + time + " (current time:"
264           + EnvironmentEdgeManager.currentTimeMillis() + ").");
265       SnapshotDescription.Builder builder = snapshot.toBuilder();
266       builder.setCreationTime(time);
267       snapshot = builder.build();
268     }
269     return snapshot;
270   }
271 
272   /**
273    * Write the snapshot description into the working directory of a snapshot
274    * @param snapshot description of the snapshot being taken
275    * @param workingDir working directory of the snapshot
276    * @param fs {@link FileSystem} on which the snapshot should be taken
277    * @throws IOException if we can't reach the filesystem and the file cannot be cleaned up on
278    *           failure
279    */
280   public static void writeSnapshotInfo(SnapshotDescription snapshot, Path workingDir, FileSystem fs)
281       throws IOException {
282     FsPermission perms = FSUtils.getFilePermissions(fs, fs.getConf(),
283       HConstants.DATA_FILE_UMASK_KEY);
284     Path snapshotInfo = new Path(workingDir, SnapshotDescriptionUtils.SNAPSHOTINFO_FILE);
285     try {
286       FSDataOutputStream out = FSUtils.create(fs, snapshotInfo, perms, true);
287       try {
288         snapshot.writeTo(out);
289       } finally {
290         out.close();
291       }
292     } catch (IOException e) {
293       // if we get an exception, try to remove the snapshot info
294       if (!fs.delete(snapshotInfo, false)) {
295         String msg = "Couldn't delete snapshot info file: " + snapshotInfo;
296         LOG.error(msg);
297         throw new IOException(msg);
298       }
299     }
300   }
301 
302   /**
303    * Read in the {@link SnapshotDescription} stored for the snapshot in the passed directory
304    * @param fs filesystem where the snapshot was taken
305    * @param snapshotDir directory where the snapshot was stored
306    * @return the stored snapshot description
307    * @throws CorruptedSnapshotException if the snapshot cannot be read
308    */
309   public static SnapshotDescription readSnapshotInfo(FileSystem fs, Path snapshotDir)
310       throws CorruptedSnapshotException {
311     Path snapshotInfo = new Path(snapshotDir, SNAPSHOTINFO_FILE);
312     try {
313       FSDataInputStream in = null;
314       try {
315         in = fs.open(snapshotInfo);
316         return SnapshotDescription.parseFrom(in);
317       } finally {
318         if (in != null) in.close();
319       }
320     } catch (IOException e) {
321       throw new CorruptedSnapshotException("Couldn't read snapshot info from:" + snapshotInfo, e);
322     }
323   }
324 
325   /**
326    * Move the finished snapshot to its final, publicly visible directory - this marks the snapshot
327    * as 'complete'.
328    * @param snapshot description of the snapshot being tabken
329    * @param rootdir root directory of the hbase installation
330    * @param workingDir directory where the in progress snapshot was built
331    * @param fs {@link FileSystem} where the snapshot was built
332    * @throws SnapshotCreationException if the snapshot could not be moved
333    * @throws IOException the filesystem could not be reached
334    */
335   public static void completeSnapshot(SnapshotDescription snapshot, Path rootdir, Path workingDir,
336       FileSystem fs) throws SnapshotCreationException, IOException {
337     Path finishedDir = getCompletedSnapshotDir(snapshot, rootdir);
338     LOG.debug("Snapshot is done, just moving the snapshot from " + workingDir + " to "
339         + finishedDir);
340     if (!fs.rename(workingDir, finishedDir)) {
341       throw new SnapshotCreationException("Failed to move working directory(" + workingDir
342           + ") to completed directory(" + finishedDir + ").", snapshot);
343     }
344   }
345 
346   /**
347    * Returns a single line (no \n) representation of snapshot metadata.  Use this instead of
348    * {@link SnapshotDescription#toString()}.  We don't replace SnapshotDescrpition's toString
349    * because it is auto-generated by protoc.
350    * @param ssd
351    * @return Single line string with a summary of the snapshot parameters
352    */
353   public static String toString(SnapshotDescription ssd) {
354     if (ssd == null) {
355       return null;
356     }
357     return "{ ss=" + ssd.getName() + " table=" + ssd.getTable()
358         + " type=" + ssd.getType() + " }";
359   }
360 }