View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.master.snapshot;
19  
20  import java.io.FileNotFoundException;
21  import java.io.IOException;
22  import java.util.ArrayList;
23  import java.util.Collections;
24  import java.util.HashMap;
25  import java.util.HashSet;
26  import java.util.Iterator;
27  import java.util.List;
28  import java.util.Map;
29  import java.util.Set;
30  import java.util.concurrent.ThreadPoolExecutor;
31  
32  import org.apache.commons.logging.Log;
33  import org.apache.commons.logging.LogFactory;
34  import org.apache.hadoop.classification.InterfaceAudience;
35  import org.apache.hadoop.classification.InterfaceStability;
36  import org.apache.hadoop.conf.Configuration;
37  import org.apache.hadoop.fs.FSDataInputStream;
38  import org.apache.hadoop.fs.FileStatus;
39  import org.apache.hadoop.fs.FileSystem;
40  import org.apache.hadoop.fs.Path;
41  import org.apache.hadoop.hbase.TableName;
42  import org.apache.hadoop.hbase.HBaseInterfaceAudience;
43  import org.apache.hadoop.hbase.HConstants;
44  import org.apache.hadoop.hbase.HTableDescriptor;
45  import org.apache.hadoop.hbase.Stoppable;
46  import org.apache.hadoop.hbase.MetaTableAccessor;
47  import org.apache.hadoop.hbase.errorhandling.ForeignException;
48  import org.apache.hadoop.hbase.executor.ExecutorService;
49  import org.apache.hadoop.hbase.ipc.RequestContext;
50  import org.apache.hadoop.hbase.master.AssignmentManager;
51  import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
52  import org.apache.hadoop.hbase.master.MasterFileSystem;
53  import org.apache.hadoop.hbase.master.MasterServices;
54  import org.apache.hadoop.hbase.master.MetricsMaster;
55  import org.apache.hadoop.hbase.master.SnapshotSentinel;
56  import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
57  import org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner;
58  import org.apache.hadoop.hbase.procedure.MasterProcedureManager;
59  import org.apache.hadoop.hbase.procedure.Procedure;
60  import org.apache.hadoop.hbase.procedure.ProcedureCoordinator;
61  import org.apache.hadoop.hbase.procedure.ProcedureCoordinatorRpcs;
62  import org.apache.hadoop.hbase.procedure.ZKProcedureCoordinatorRpcs;
63  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameStringPair;
64  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ProcedureDescription;
65  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
66  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Type;
67  import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
68  import org.apache.hadoop.hbase.security.User;
69  import org.apache.hadoop.hbase.snapshot.ClientSnapshotDescriptionUtils;
70  import org.apache.hadoop.hbase.snapshot.HBaseSnapshotException;
71  import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
72  import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper;
73  import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
74  import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
75  import org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException;
76  import org.apache.hadoop.hbase.snapshot.SnapshotExistsException;
77  import org.apache.hadoop.hbase.snapshot.SnapshotManifest;
78  import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
79  import org.apache.hadoop.hbase.snapshot.TablePartiallyOpenException;
80  import org.apache.hadoop.hbase.snapshot.UnknownSnapshotException;
81  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
82  import org.apache.hadoop.hbase.util.FSUtils;
83  import org.apache.zookeeper.KeeperException;
84  
85  /**
86   * This class manages the procedure of taking and restoring snapshots. There is only one
87   * SnapshotManager for the master.
88   * <p>
89   * The class provides methods for monitoring in-progress snapshot actions.
90   * <p>
91   * Note: Currently there can only be one snapshot being taken at a time over the cluster. This is a
92   * simplification in the current implementation.
93   */
94  @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)
95  @InterfaceStability.Unstable
96  public class SnapshotManager extends MasterProcedureManager implements Stoppable {
97    private static final Log LOG = LogFactory.getLog(SnapshotManager.class);
98  
99    /** By default, check to see if the snapshot is complete every WAKE MILLIS (ms) */
100   private static final int SNAPSHOT_WAKE_MILLIS_DEFAULT = 500;
101 
102   /**
103    * Wait time before removing a finished sentinel from the in-progress map
104    *
105    * NOTE: This is used as a safety auto cleanup.
106    * The snapshot and restore handlers map entries are removed when a user asks if a snapshot or
107    * restore is completed. This operation is part of the HBaseAdmin snapshot/restore API flow.
108    * In case something fails on the client side and the snapshot/restore state is not reclaimed
109    * after a default timeout, the entry is removed from the in-progress map.
110    * At this point, if the user asks for the snapshot/restore status, the result will be
111    * snapshot done if exists or failed if it doesn't exists.
112    */
113   private static final int SNAPSHOT_SENTINELS_CLEANUP_TIMEOUT = 60 * 1000;
114 
115   /** Enable or disable snapshot support */
116   public static final String HBASE_SNAPSHOT_ENABLED = "hbase.snapshot.enabled";
117 
118   /**
119    * Conf key for # of ms elapsed between checks for snapshot errors while waiting for
120    * completion.
121    */
122   private static final String SNAPSHOT_WAKE_MILLIS_KEY = "hbase.snapshot.master.wakeMillis";
123 
124   /** By default, check to see if the snapshot is complete (ms) */
125   private static final int SNAPSHOT_TIMEOUT_MILLIS_DEFAULT = 60000;
126 
127   /**
128    * Conf key for # of ms elapsed before injecting a snapshot timeout error when waiting for
129    * completion.
130    */
131   private static final String SNAPSHOT_TIMEOUT_MILLIS_KEY = "hbase.snapshot.master.timeoutMillis";
132 
133   /** Name of the operation to use in the controller */
134   public static final String ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION = "online-snapshot";
135 
136   /** Conf key for # of threads used by the SnapshotManager thread pool */
137   private static final String SNAPSHOT_POOL_THREADS_KEY = "hbase.snapshot.master.threads";
138 
139   /** number of current operations running on the master */
140   private static final int SNAPSHOT_POOL_THREADS_DEFAULT = 1;
141 
142   private boolean stopped;
143   private MasterServices master;  // Needed by TableEventHandlers
144   private ProcedureCoordinator coordinator;
145 
146   // Is snapshot feature enabled?
147   private boolean isSnapshotSupported = false;
148 
149   // Snapshot handlers map, with table name as key.
150   // The map is always accessed and modified under the object lock using synchronized.
151   // snapshotTable() will insert an Handler in the table.
152   // isSnapshotDone() will remove the handler requested if the operation is finished.
153   private Map<TableName, SnapshotSentinel> snapshotHandlers =
154       new HashMap<TableName, SnapshotSentinel>();
155 
156   // Restore Sentinels map, with table name as key.
157   // The map is always accessed and modified under the object lock using synchronized.
158   // restoreSnapshot()/cloneSnapshot() will insert an Handler in the table.
159   // isRestoreDone() will remove the handler requested if the operation is finished.
160   private Map<TableName, SnapshotSentinel> restoreHandlers =
161       new HashMap<TableName, SnapshotSentinel>();
162 
163   private Path rootDir;
164   private ExecutorService executorService;
165 
166   public SnapshotManager() {}
167 
168   /**
169    * Fully specify all necessary components of a snapshot manager. Exposed for testing.
170    * @param master services for the master where the manager is running
171    * @param coordinator procedure coordinator instance.  exposed for testing.
172    * @param pool HBase ExecutorServcie instance, exposed for testing.
173    */
174   public SnapshotManager(final MasterServices master, final MetricsMaster metricsMaster,
175       ProcedureCoordinator coordinator, ExecutorService pool)
176       throws IOException, UnsupportedOperationException {
177     this.master = master;
178 
179     this.rootDir = master.getMasterFileSystem().getRootDir();
180     checkSnapshotSupport(master.getConfiguration(), master.getMasterFileSystem());
181 
182     this.coordinator = coordinator;
183     this.executorService = pool;
184     resetTempDir();
185   }
186 
187   /**
188    * Gets the list of all completed snapshots.
189    * @return list of SnapshotDescriptions
190    * @throws IOException File system exception
191    */
192   public List<SnapshotDescription> getCompletedSnapshots() throws IOException {
193     return getCompletedSnapshots(SnapshotDescriptionUtils.getSnapshotsDir(rootDir));
194   }
195 
196   /**
197    * Gets the list of all completed snapshots.
198    * @param snapshotDir snapshot directory
199    * @return list of SnapshotDescriptions
200    * @throws IOException File system exception
201    */
202   private List<SnapshotDescription> getCompletedSnapshots(Path snapshotDir) throws IOException {
203     List<SnapshotDescription> snapshotDescs = new ArrayList<SnapshotDescription>();
204     // first create the snapshot root path and check to see if it exists
205     FileSystem fs = master.getMasterFileSystem().getFileSystem();
206     if (snapshotDir == null) snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
207 
208     // if there are no snapshots, return an empty list
209     if (!fs.exists(snapshotDir)) {
210       return snapshotDescs;
211     }
212 
213     // ignore all the snapshots in progress
214     FileStatus[] snapshots = fs.listStatus(snapshotDir,
215       new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
216     // loop through all the completed snapshots
217     for (FileStatus snapshot : snapshots) {
218       Path info = new Path(snapshot.getPath(), SnapshotDescriptionUtils.SNAPSHOTINFO_FILE);
219       // if the snapshot is bad
220       if (!fs.exists(info)) {
221         LOG.error("Snapshot information for " + snapshot.getPath() + " doesn't exist");
222         continue;
223       }
224       FSDataInputStream in = null;
225       try {
226         in = fs.open(info);
227         SnapshotDescription desc = SnapshotDescription.parseFrom(in);
228         snapshotDescs.add(desc);
229       } catch (IOException e) {
230         LOG.warn("Found a corrupted snapshot " + snapshot.getPath(), e);
231       } finally {
232         if (in != null) {
233           in.close();
234         }
235       }
236     }
237     return snapshotDescs;
238   }
239 
240   /**
241    * Cleans up any snapshots in the snapshot/.tmp directory that were left from failed
242    * snapshot attempts.
243    *
244    * @throws IOException if we can't reach the filesystem
245    */
246   void resetTempDir() throws IOException {
247     // cleanup any existing snapshots.
248     Path tmpdir = SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir);
249     if (master.getMasterFileSystem().getFileSystem().exists(tmpdir)) {
250       if (!master.getMasterFileSystem().getFileSystem().delete(tmpdir, true)) {
251         LOG.warn("Couldn't delete working snapshot directory: " + tmpdir);
252       }
253     }
254   }
255 
256   /**
257    * Delete the specified snapshot
258    * @param snapshot
259    * @throws SnapshotDoesNotExistException If the specified snapshot does not exist.
260    * @throws IOException For filesystem IOExceptions
261    */
262   public void deleteSnapshot(SnapshotDescription snapshot) throws SnapshotDoesNotExistException, IOException {
263     // check to see if it is completed
264     if (!isSnapshotCompleted(snapshot)) {
265       throw new SnapshotDoesNotExistException(snapshot);
266     }
267 
268     String snapshotName = snapshot.getName();
269     // first create the snapshot description and check to see if it exists
270     FileSystem fs = master.getMasterFileSystem().getFileSystem();
271     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
272     // Get snapshot info from file system. The one passed as parameter is a "fake" snapshotInfo with
273     // just the "name" and it does not contains the "real" snapshot information
274     snapshot = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
275 
276     // call coproc pre hook
277     MasterCoprocessorHost cpHost = master.getMasterCoprocessorHost();
278     if (cpHost != null) {
279       cpHost.preDeleteSnapshot(snapshot);
280     }
281 
282     LOG.debug("Deleting snapshot: " + snapshotName);
283     // delete the existing snapshot
284     if (!fs.delete(snapshotDir, true)) {
285       throw new HBaseSnapshotException("Failed to delete snapshot directory: " + snapshotDir);
286     }
287 
288     // call coproc post hook
289     if (cpHost != null) {
290       cpHost.postDeleteSnapshot(snapshot);
291     }
292 
293   }
294 
295   /**
296    * Check if the specified snapshot is done
297    *
298    * @param expected
299    * @return true if snapshot is ready to be restored, false if it is still being taken.
300    * @throws IOException IOException if error from HDFS or RPC
301    * @throws UnknownSnapshotException if snapshot is invalid or does not exist.
302    */
303   public boolean isSnapshotDone(SnapshotDescription expected) throws IOException {
304     // check the request to make sure it has a snapshot
305     if (expected == null) {
306       throw new UnknownSnapshotException(
307          "No snapshot name passed in request, can't figure out which snapshot you want to check.");
308     }
309 
310     String ssString = ClientSnapshotDescriptionUtils.toString(expected);
311 
312     // check to see if the sentinel exists,
313     // and if the task is complete removes it from the in-progress snapshots map.
314     SnapshotSentinel handler = removeSentinelIfFinished(this.snapshotHandlers, expected);
315 
316     // stop tracking "abandoned" handlers
317     cleanupSentinels();
318 
319     if (handler == null) {
320       // If there's no handler in the in-progress map, it means one of the following:
321       //   - someone has already requested the snapshot state
322       //   - the requested snapshot was completed long time ago (cleanupSentinels() timeout)
323       //   - the snapshot was never requested
324       // In those cases returns to the user the "done state" if the snapshots exists on disk,
325       // otherwise raise an exception saying that the snapshot is not running and doesn't exist.
326       if (!isSnapshotCompleted(expected)) {
327         throw new UnknownSnapshotException("Snapshot " + ssString
328             + " is not currently running or one of the known completed snapshots.");
329       }
330       // was done, return true;
331       return true;
332     }
333 
334     // pass on any failure we find in the sentinel
335     try {
336       handler.rethrowExceptionIfFailed();
337     } catch (ForeignException e) {
338       // Give some procedure info on an exception.
339       String status;
340       Procedure p = coordinator.getProcedure(expected.getName());
341       if (p != null) {
342         status = p.getStatus();
343       } else {
344         status = expected.getName() + " not found in proclist " + coordinator.getProcedureNames();
345       }
346       throw new HBaseSnapshotException("Snapshot " + ssString +  " had an error.  " + status, e,
347           expected);
348     }
349 
350     // check to see if we are done
351     if (handler.isFinished()) {
352       LOG.debug("Snapshot '" + ssString + "' has completed, notifying client.");
353       return true;
354     } else if (LOG.isDebugEnabled()) {
355       LOG.debug("Snapshoting '" + ssString + "' is still in progress!");
356     }
357     return false;
358   }
359 
360   /**
361    * Check to see if there is a snapshot in progress with the same name or on the same table.
362    * Currently we have a limitation only allowing a single snapshot per table at a time. Also we
363    * don't allow snapshot with the same name.
364    * @param snapshot description of the snapshot being checked.
365    * @return <tt>true</tt> if there is a snapshot in progress with the same name or on the same
366    *         table.
367    */
368   synchronized boolean isTakingSnapshot(final SnapshotDescription snapshot) {
369     TableName snapshotTable = TableName.valueOf(snapshot.getTable());
370     if (isTakingSnapshot(snapshotTable)) {
371       return true;
372     }
373     Iterator<Map.Entry<TableName, SnapshotSentinel>> it = this.snapshotHandlers.entrySet().iterator();
374     while (it.hasNext()) {
375       Map.Entry<TableName, SnapshotSentinel> entry = it.next();
376       SnapshotSentinel sentinel = entry.getValue();
377       if (snapshot.getName().equals(sentinel.getSnapshot().getName()) && !sentinel.isFinished()) {
378         return true;
379       }
380     }
381     return false;
382   }
383 
384   /**
385    * Check to see if the specified table has a snapshot in progress.  Currently we have a
386    * limitation only allowing a single snapshot per table at a time.
387    * @param tableName name of the table being snapshotted.
388    * @return <tt>true</tt> if there is a snapshot in progress on the specified table.
389    */
390   synchronized boolean isTakingSnapshot(final TableName tableName) {
391     SnapshotSentinel handler = this.snapshotHandlers.get(tableName);
392     return handler != null && !handler.isFinished();
393   }
394 
395   /**
396    * Check to make sure that we are OK to run the passed snapshot. Checks to make sure that we
397    * aren't already running a snapshot or restore on the requested table.
398    * @param snapshot description of the snapshot we want to start
399    * @throws HBaseSnapshotException if the filesystem could not be prepared to start the snapshot
400    */
401   private synchronized void prepareToTakeSnapshot(SnapshotDescription snapshot)
402       throws HBaseSnapshotException {
403     FileSystem fs = master.getMasterFileSystem().getFileSystem();
404     Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
405     TableName snapshotTable =
406         TableName.valueOf(snapshot.getTable());
407 
408     // make sure we aren't already running a snapshot
409     if (isTakingSnapshot(snapshot)) {
410       SnapshotSentinel handler = this.snapshotHandlers.get(snapshotTable);
411       throw new SnapshotCreationException("Rejected taking "
412           + ClientSnapshotDescriptionUtils.toString(snapshot)
413           + " because we are already running another snapshot "
414           + (handler != null ? ("on the same table " +
415               ClientSnapshotDescriptionUtils.toString(handler.getSnapshot()))
416               : "with the same name"), snapshot);
417     }
418 
419     // make sure we aren't running a restore on the same table
420     if (isRestoringTable(snapshotTable)) {
421       SnapshotSentinel handler = restoreHandlers.get(snapshotTable);
422       throw new SnapshotCreationException("Rejected taking "
423           + ClientSnapshotDescriptionUtils.toString(snapshot)
424           + " because we are already have a restore in progress on the same snapshot "
425           + ClientSnapshotDescriptionUtils.toString(handler.getSnapshot()), snapshot);
426     }
427 
428     try {
429       // delete the working directory, since we aren't running the snapshot. Likely leftovers
430       // from a failed attempt.
431       fs.delete(workingDir, true);
432 
433       // recreate the working directory for the snapshot
434       if (!fs.mkdirs(workingDir)) {
435         throw new SnapshotCreationException("Couldn't create working directory (" + workingDir
436             + ") for snapshot" , snapshot);
437       }
438     } catch (HBaseSnapshotException e) {
439       throw e;
440     } catch (IOException e) {
441       throw new SnapshotCreationException(
442           "Exception while checking to see if snapshot could be started.", e, snapshot);
443     }
444   }
445 
446   /**
447    * Take a snapshot of a disabled table.
448    * @param snapshot description of the snapshot to take. Modified to be {@link Type#DISABLED}.
449    * @throws HBaseSnapshotException if the snapshot could not be started
450    */
451   private synchronized void snapshotDisabledTable(SnapshotDescription snapshot)
452       throws HBaseSnapshotException {
453     // setup the snapshot
454     prepareToTakeSnapshot(snapshot);
455 
456     // set the snapshot to be a disabled snapshot, since the client doesn't know about that
457     snapshot = snapshot.toBuilder().setType(Type.DISABLED).build();
458 
459     // Take the snapshot of the disabled table
460     DisabledTableSnapshotHandler handler =
461         new DisabledTableSnapshotHandler(snapshot, master);
462     snapshotTable(snapshot, handler);
463   }
464 
465   /**
466    * Take a snapshot of an enabled table.
467    * @param snapshot description of the snapshot to take.
468    * @throws HBaseSnapshotException if the snapshot could not be started
469    */
470   private synchronized void snapshotEnabledTable(SnapshotDescription snapshot)
471       throws HBaseSnapshotException {
472     // setup the snapshot
473     prepareToTakeSnapshot(snapshot);
474 
475     // Take the snapshot of the enabled table
476     EnabledTableSnapshotHandler handler =
477         new EnabledTableSnapshotHandler(snapshot, master, this);
478     snapshotTable(snapshot, handler);
479   }
480 
481   /**
482    * Take a snapshot using the specified handler.
483    * On failure the snapshot temporary working directory is removed.
484    * NOTE: prepareToTakeSnapshot() called before this one takes care of the rejecting the
485    *       snapshot request if the table is busy with another snapshot/restore operation.
486    * @param snapshot the snapshot description
487    * @param handler the snapshot handler
488    */
489   private synchronized void snapshotTable(SnapshotDescription snapshot,
490       final TakeSnapshotHandler handler) throws HBaseSnapshotException {
491     try {
492       handler.prepare();
493       this.executorService.submit(handler);
494       this.snapshotHandlers.put(TableName.valueOf(snapshot.getTable()), handler);
495     } catch (Exception e) {
496       // cleanup the working directory by trying to delete it from the fs.
497       Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
498       try {
499         if (!this.master.getMasterFileSystem().getFileSystem().delete(workingDir, true)) {
500           LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" +
501               ClientSnapshotDescriptionUtils.toString(snapshot));
502         }
503       } catch (IOException e1) {
504         LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" +
505             ClientSnapshotDescriptionUtils.toString(snapshot));
506       }
507       // fail the snapshot
508       throw new SnapshotCreationException("Could not build snapshot handler", e, snapshot);
509     }
510   }
511 
512   /**
513    * Take a snapshot based on the enabled/disabled state of the table.
514    *
515    * @param snapshot
516    * @throws HBaseSnapshotException when a snapshot specific exception occurs.
517    * @throws IOException when some sort of generic IO exception occurs.
518    */
519   public void takeSnapshot(SnapshotDescription snapshot) throws IOException {
520     // check to see if we already completed the snapshot
521     if (isSnapshotCompleted(snapshot)) {
522       throw new SnapshotExistsException("Snapshot '" + snapshot.getName()
523           + "' already stored on the filesystem.", snapshot);
524     }
525 
526     LOG.debug("No existing snapshot, attempting snapshot...");
527 
528     // stop tracking "abandoned" handlers
529     cleanupSentinels();
530 
531     // check to see if the table exists
532     HTableDescriptor desc = null;
533     try {
534       desc = master.getTableDescriptors().get(
535           TableName.valueOf(snapshot.getTable()));
536     } catch (FileNotFoundException e) {
537       String msg = "Table:" + snapshot.getTable() + " info doesn't exist!";
538       LOG.error(msg);
539       throw new SnapshotCreationException(msg, e, snapshot);
540     } catch (IOException e) {
541       throw new SnapshotCreationException("Error while geting table description for table "
542           + snapshot.getTable(), e, snapshot);
543     }
544     if (desc == null) {
545       throw new SnapshotCreationException("Table '" + snapshot.getTable()
546           + "' doesn't exist, can't take snapshot.", snapshot);
547     }
548     SnapshotDescription.Builder builder = snapshot.toBuilder();
549     // if not specified, set the snapshot format
550     if (!snapshot.hasVersion()) {
551       builder.setVersion(SnapshotDescriptionUtils.SNAPSHOT_LAYOUT_VERSION);
552     }
553     User user = RequestContext.getRequestUser();
554     if (User.isHBaseSecurityEnabled(master.getConfiguration()) && user != null) {
555       builder.setOwner(user.getShortName());
556     }
557     snapshot = builder.build();
558 
559     // call pre coproc hook
560     MasterCoprocessorHost cpHost = master.getMasterCoprocessorHost();
561     if (cpHost != null) {
562       cpHost.preSnapshot(snapshot, desc);
563     }
564 
565     // if the table is enabled, then have the RS run actually the snapshot work
566     TableName snapshotTable = TableName.valueOf(snapshot.getTable());
567     AssignmentManager assignmentMgr = master.getAssignmentManager();
568     if (assignmentMgr.getTableStateManager().isTableState(snapshotTable,
569         ZooKeeperProtos.Table.State.ENABLED)) {
570       LOG.debug("Table enabled, starting distributed snapshot.");
571       snapshotEnabledTable(snapshot);
572       LOG.debug("Started snapshot: " + ClientSnapshotDescriptionUtils.toString(snapshot));
573     }
574     // For disabled table, snapshot is created by the master
575     else if (assignmentMgr.getTableStateManager().isTableState(snapshotTable,
576         ZooKeeperProtos.Table.State.DISABLED)) {
577       LOG.debug("Table is disabled, running snapshot entirely on master.");
578       snapshotDisabledTable(snapshot);
579       LOG.debug("Started snapshot: " + ClientSnapshotDescriptionUtils.toString(snapshot));
580     } else {
581       LOG.error("Can't snapshot table '" + snapshot.getTable()
582           + "', isn't open or closed, we don't know what to do!");
583       TablePartiallyOpenException tpoe = new TablePartiallyOpenException(snapshot.getTable()
584           + " isn't fully open.");
585       throw new SnapshotCreationException("Table is not entirely open or closed", tpoe, snapshot);
586     }
587 
588     // call post coproc hook
589     if (cpHost != null) {
590       cpHost.postSnapshot(snapshot, desc);
591     }
592   }
593 
594   /**
595    * Set the handler for the current snapshot
596    * <p>
597    * Exposed for TESTING
598    * @param tableName
599    * @param handler handler the master should use
600    *
601    * TODO get rid of this if possible, repackaging, modify tests.
602    */
603   public synchronized void setSnapshotHandlerForTesting(
604       final TableName tableName,
605       final SnapshotSentinel handler) {
606     if (handler != null) {
607       this.snapshotHandlers.put(tableName, handler);
608     } else {
609       this.snapshotHandlers.remove(tableName);
610     }
611   }
612 
613   /**
614    * @return distributed commit coordinator for all running snapshots
615    */
616   ProcedureCoordinator getCoordinator() {
617     return coordinator;
618   }
619 
620   /**
621    * Check to see if the snapshot is one of the currently completed snapshots
622    * Returns true if the snapshot exists in the "completed snapshots folder".
623    *
624    * @param snapshot expected snapshot to check
625    * @return <tt>true</tt> if the snapshot is stored on the {@link FileSystem}, <tt>false</tt> if is
626    *         not stored
627    * @throws IOException if the filesystem throws an unexpected exception,
628    * @throws IllegalArgumentException if snapshot name is invalid.
629    */
630   private boolean isSnapshotCompleted(SnapshotDescription snapshot) throws IOException {
631     try {
632       final Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
633       FileSystem fs = master.getMasterFileSystem().getFileSystem();
634       // check to see if the snapshot already exists
635       return fs.exists(snapshotDir);
636     } catch (IllegalArgumentException iae) {
637       throw new UnknownSnapshotException("Unexpected exception thrown", iae);
638     }
639   }
640 
641   /**
642    * Clone the specified snapshot into a new table.
643    * The operation will fail if the destination table has a snapshot or restore in progress.
644    *
645    * @param snapshot Snapshot Descriptor
646    * @param hTableDescriptor Table Descriptor of the table to create
647    */
648   synchronized void cloneSnapshot(final SnapshotDescription snapshot,
649       final HTableDescriptor hTableDescriptor) throws HBaseSnapshotException {
650     TableName tableName = hTableDescriptor.getTableName();
651 
652     // make sure we aren't running a snapshot on the same table
653     if (isTakingSnapshot(tableName)) {
654       throw new RestoreSnapshotException("Snapshot in progress on the restore table=" + tableName);
655     }
656 
657     // make sure we aren't running a restore on the same table
658     if (isRestoringTable(tableName)) {
659       throw new RestoreSnapshotException("Restore already in progress on the table=" + tableName);
660     }
661 
662     try {
663       CloneSnapshotHandler handler =
664         new CloneSnapshotHandler(master, snapshot, hTableDescriptor).prepare();
665       this.executorService.submit(handler);
666       this.restoreHandlers.put(tableName, handler);
667     } catch (Exception e) {
668       String msg = "Couldn't clone the snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot) +
669         " on table=" + tableName;
670       LOG.error(msg, e);
671       throw new RestoreSnapshotException(msg, e);
672     }
673   }
674 
675   /**
676    * Restore the specified snapshot
677    * @param reqSnapshot
678    * @throws IOException
679    */
680   public void restoreSnapshot(SnapshotDescription reqSnapshot) throws IOException {
681     FileSystem fs = master.getMasterFileSystem().getFileSystem();
682     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(reqSnapshot, rootDir);
683     MasterCoprocessorHost cpHost = master.getMasterCoprocessorHost();
684 
685     // check if the snapshot exists
686     if (!fs.exists(snapshotDir)) {
687       LOG.error("A Snapshot named '" + reqSnapshot.getName() + "' does not exist.");
688       throw new SnapshotDoesNotExistException(reqSnapshot);
689     }
690 
691     // Get snapshot info from file system. The reqSnapshot is a "fake" snapshotInfo with
692     // just the snapshot "name" and table name to restore. It does not contains the "real" snapshot
693     // information.
694     SnapshotDescription snapshot = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
695     SnapshotManifest manifest = SnapshotManifest.open(master.getConfiguration(), fs,
696         snapshotDir, snapshot);
697     HTableDescriptor snapshotTableDesc = manifest.getTableDescriptor();
698     TableName tableName = TableName.valueOf(reqSnapshot.getTable());
699 
700     // stop tracking "abandoned" handlers
701     cleanupSentinels();
702 
703     // Verify snapshot validity
704     SnapshotReferenceUtil.verifySnapshot(master.getConfiguration(), fs, manifest);
705 
706     // Execute the restore/clone operation
707     if (MetaTableAccessor.tableExists(master.getShortCircuitConnection(), tableName)) {
708       if (master.getAssignmentManager().getTableStateManager().isTableState(
709           TableName.valueOf(snapshot.getTable()), ZooKeeperProtos.Table.State.ENABLED)) {
710         throw new UnsupportedOperationException("Table '" +
711             TableName.valueOf(snapshot.getTable()) + "' must be disabled in order to " +
712             "perform a restore operation" +
713             ".");
714       }
715 
716       // call coproc pre hook
717       if (cpHost != null) {
718         cpHost.preRestoreSnapshot(reqSnapshot, snapshotTableDesc);
719       }
720       restoreSnapshot(snapshot, snapshotTableDesc);
721       LOG.info("Restore snapshot=" + snapshot.getName() + " as table=" + tableName);
722 
723       if (cpHost != null) {
724         cpHost.postRestoreSnapshot(reqSnapshot, snapshotTableDesc);
725       }
726     } else {
727       HTableDescriptor htd = RestoreSnapshotHelper.cloneTableSchema(snapshotTableDesc, tableName);
728       if (cpHost != null) {
729         cpHost.preCloneSnapshot(reqSnapshot, htd);
730       }
731       cloneSnapshot(snapshot, htd);
732       LOG.info("Clone snapshot=" + snapshot.getName() + " as table=" + tableName);
733 
734       if (cpHost != null) {
735         cpHost.postCloneSnapshot(reqSnapshot, htd);
736       }
737     }
738   }
739 
740   /**
741    * Restore the specified snapshot.
742    * The restore will fail if the destination table has a snapshot or restore in progress.
743    *
744    * @param snapshot Snapshot Descriptor
745    * @param hTableDescriptor Table Descriptor
746    */
747   private synchronized void restoreSnapshot(final SnapshotDescription snapshot,
748       final HTableDescriptor hTableDescriptor) throws HBaseSnapshotException {
749     TableName tableName = hTableDescriptor.getTableName();
750 
751     // make sure we aren't running a snapshot on the same table
752     if (isTakingSnapshot(tableName)) {
753       throw new RestoreSnapshotException("Snapshot in progress on the restore table=" + tableName);
754     }
755 
756     // make sure we aren't running a restore on the same table
757     if (isRestoringTable(tableName)) {
758       throw new RestoreSnapshotException("Restore already in progress on the table=" + tableName);
759     }
760 
761     try {
762       RestoreSnapshotHandler handler =
763         new RestoreSnapshotHandler(master, snapshot, hTableDescriptor).prepare();
764       this.executorService.submit(handler);
765       restoreHandlers.put(tableName, handler);
766     } catch (Exception e) {
767       String msg = "Couldn't restore the snapshot=" + ClientSnapshotDescriptionUtils.toString(
768           snapshot)  +
769           " on table=" + tableName;
770       LOG.error(msg, e);
771       throw new RestoreSnapshotException(msg, e);
772     }
773   }
774 
775   /**
776    * Verify if the restore of the specified table is in progress.
777    *
778    * @param tableName table under restore
779    * @return <tt>true</tt> if there is a restore in progress of the specified table.
780    */
781   private synchronized boolean isRestoringTable(final TableName tableName) {
782     SnapshotSentinel sentinel = this.restoreHandlers.get(tableName);
783     return(sentinel != null && !sentinel.isFinished());
784   }
785 
786   /**
787    * Returns the status of a restore operation.
788    * If the in-progress restore is failed throws the exception that caused the failure.
789    *
790    * @param snapshot
791    * @return false if in progress, true if restore is completed or not requested.
792    * @throws IOException if there was a failure during the restore
793    */
794   public boolean isRestoreDone(final SnapshotDescription snapshot) throws IOException {
795     // check to see if the sentinel exists,
796     // and if the task is complete removes it from the in-progress restore map.
797     SnapshotSentinel sentinel = removeSentinelIfFinished(this.restoreHandlers, snapshot);
798 
799     // stop tracking "abandoned" handlers
800     cleanupSentinels();
801 
802     if (sentinel == null) {
803       // there is no sentinel so restore is not in progress.
804       return true;
805     }
806 
807     LOG.debug("Verify snapshot=" + snapshot.getName() + " against="
808         + sentinel.getSnapshot().getName() + " table=" +
809         TableName.valueOf(snapshot.getTable()));
810 
811     // If the restore is failed, rethrow the exception
812     sentinel.rethrowExceptionIfFailed();
813 
814     // check to see if we are done
815     if (sentinel.isFinished()) {
816       LOG.debug("Restore snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot) +
817           " has completed. Notifying the client.");
818       return true;
819     }
820 
821     if (LOG.isDebugEnabled()) {
822       LOG.debug("Sentinel is not yet finished with restoring snapshot=" +
823           ClientSnapshotDescriptionUtils.toString(snapshot));
824     }
825     return false;
826   }
827 
828   /**
829    * Return the handler if it is currently live and has the same snapshot target name.
830    * The handler is removed from the sentinels map if completed.
831    * @param sentinels live handlers
832    * @param snapshot snapshot description
833    * @return null if doesn't match, else a live handler.
834    */
835   private synchronized SnapshotSentinel removeSentinelIfFinished(
836       final Map<TableName, SnapshotSentinel> sentinels,
837       final SnapshotDescription snapshot) {
838     if (!snapshot.hasTable()) {
839       return null;
840     }
841 
842     TableName snapshotTable = TableName.valueOf(snapshot.getTable());
843     SnapshotSentinel h = sentinels.get(snapshotTable);
844     if (h == null) {
845       return null;
846     }
847 
848     if (!h.getSnapshot().getName().equals(snapshot.getName())) {
849       // specified snapshot is to the one currently running
850       return null;
851     }
852 
853     // Remove from the "in-progress" list once completed
854     if (h.isFinished()) {
855       sentinels.remove(snapshotTable);
856     }
857 
858     return h;
859   }
860 
861   /**
862    * Removes "abandoned" snapshot/restore requests.
863    * As part of the HBaseAdmin snapshot/restore API the operation status is checked until completed,
864    * and the in-progress maps are cleaned up when the status of a completed task is requested.
865    * To avoid having sentinels staying around for long time if something client side is failed,
866    * each operation tries to clean up the in-progress maps sentinels finished from a long time.
867    */
868   private void cleanupSentinels() {
869     cleanupSentinels(this.snapshotHandlers);
870     cleanupSentinels(this.restoreHandlers);
871   }
872 
873   /**
874    * Remove the sentinels that are marked as finished and the completion time
875    * has exceeded the removal timeout.
876    * @param sentinels map of sentinels to clean
877    */
878   private synchronized void cleanupSentinels(final Map<TableName, SnapshotSentinel> sentinels) {
879     long currentTime = EnvironmentEdgeManager.currentTime();
880     Iterator<Map.Entry<TableName, SnapshotSentinel>> it =
881         sentinels.entrySet().iterator();
882     while (it.hasNext()) {
883       Map.Entry<TableName, SnapshotSentinel> entry = it.next();
884       SnapshotSentinel sentinel = entry.getValue();
885       if (sentinel.isFinished() &&
886           (currentTime - sentinel.getCompletionTimestamp()) > SNAPSHOT_SENTINELS_CLEANUP_TIMEOUT)
887       {
888         it.remove();
889       }
890     }
891   }
892 
893   //
894   // Implementing Stoppable interface
895   //
896 
897   @Override
898   public void stop(String why) {
899     // short circuit
900     if (this.stopped) return;
901     // make sure we get stop
902     this.stopped = true;
903     // pass the stop onto take snapshot handlers
904     for (SnapshotSentinel snapshotHandler: this.snapshotHandlers.values()) {
905       snapshotHandler.cancel(why);
906     }
907 
908     // pass the stop onto all the restore handlers
909     for (SnapshotSentinel restoreHandler: this.restoreHandlers.values()) {
910       restoreHandler.cancel(why);
911     }
912     try {
913       coordinator.close();
914     } catch (IOException e) {
915       LOG.error("stop ProcedureCoordinator error", e);
916     }
917   }
918 
919   @Override
920   public boolean isStopped() {
921     return this.stopped;
922   }
923 
924   /**
925    * Throws an exception if snapshot operations (take a snapshot, restore, clone) are not supported.
926    * Called at the beginning of snapshot() and restoreSnapshot() methods.
927    * @throws UnsupportedOperationException if snapshot are not supported
928    */
929   public void checkSnapshotSupport() throws UnsupportedOperationException {
930     if (!this.isSnapshotSupported) {
931       throw new UnsupportedOperationException(
932         "To use snapshots, You must add to the hbase-site.xml of the HBase Master: '" +
933           HBASE_SNAPSHOT_ENABLED + "' property with value 'true'.");
934     }
935   }
936 
937   /**
938    * Called at startup, to verify if snapshot operation is supported, and to avoid
939    * starting the master if there're snapshots present but the cleaners needed are missing.
940    * Otherwise we can end up with snapshot data loss.
941    * @param conf The {@link Configuration} object to use
942    * @param mfs The MasterFileSystem to use
943    * @throws IOException in case of file-system operation failure
944    * @throws UnsupportedOperationException in case cleaners are missing and
945    *         there're snapshot in the system
946    */
947   private void checkSnapshotSupport(final Configuration conf, final MasterFileSystem mfs)
948       throws IOException, UnsupportedOperationException {
949     // Verify if snapshot is disabled by the user
950     String enabled = conf.get(HBASE_SNAPSHOT_ENABLED);
951     boolean snapshotEnabled = conf.getBoolean(HBASE_SNAPSHOT_ENABLED, false);
952     boolean userDisabled = (enabled != null && enabled.trim().length() > 0 && !snapshotEnabled);
953 
954     // Extract cleaners from conf
955     Set<String> hfileCleaners = new HashSet<String>();
956     String[] cleaners = conf.getStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS);
957     if (cleaners != null) Collections.addAll(hfileCleaners, cleaners);
958 
959     Set<String> logCleaners = new HashSet<String>();
960     cleaners = conf.getStrings(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS);
961     if (cleaners != null) Collections.addAll(logCleaners, cleaners);
962 
963     // check if an older version of snapshot directory was present
964     Path oldSnapshotDir = new Path(mfs.getRootDir(), HConstants.OLD_SNAPSHOT_DIR_NAME);
965     FileSystem fs = mfs.getFileSystem();
966     List<SnapshotDescription> ss = getCompletedSnapshots(new Path(rootDir, oldSnapshotDir));
967     if (ss != null && !ss.isEmpty()) {
968       LOG.error("Snapshots from an earlier release were found under: " + oldSnapshotDir);
969       LOG.error("Please rename the directory as " + HConstants.SNAPSHOT_DIR_NAME);
970     }
971 
972     // If the user has enabled the snapshot, we force the cleaners to be present
973     // otherwise we still need to check if cleaners are enabled or not and verify
974     // that there're no snapshot in the .snapshot folder.
975     if (snapshotEnabled) {
976       // Inject snapshot cleaners, if snapshot.enable is true
977       hfileCleaners.add(SnapshotHFileCleaner.class.getName());
978       hfileCleaners.add(HFileLinkCleaner.class.getName());
979       logCleaners.add(SnapshotLogCleaner.class.getName());
980 
981       // Set cleaners conf
982       conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS,
983         hfileCleaners.toArray(new String[hfileCleaners.size()]));
984       conf.setStrings(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS,
985         logCleaners.toArray(new String[logCleaners.size()]));
986     } else {
987       // Verify if cleaners are present
988       snapshotEnabled = logCleaners.contains(SnapshotLogCleaner.class.getName()) &&
989         hfileCleaners.contains(SnapshotHFileCleaner.class.getName()) &&
990         hfileCleaners.contains(HFileLinkCleaner.class.getName());
991 
992       // Warn if the cleaners are enabled but the snapshot.enabled property is false/not set.
993       if (snapshotEnabled) {
994         LOG.warn("Snapshot log and hfile cleaners are present in the configuration, " +
995           "but the '" + HBASE_SNAPSHOT_ENABLED + "' property " +
996           (userDisabled ? "is set to 'false'." : "is not set."));
997       }
998     }
999 
1000     // Mark snapshot feature as enabled if cleaners are present and user has not disabled it.
1001     this.isSnapshotSupported = snapshotEnabled && !userDisabled;
1002 
1003     // If cleaners are not enabled, verify that there're no snapshot in the .snapshot folder
1004     // otherwise we end up with snapshot data loss.
1005     if (!snapshotEnabled) {
1006       LOG.info("Snapshot feature is not enabled, missing log and hfile cleaners.");
1007       Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(mfs.getRootDir());
1008       if (fs.exists(snapshotDir)) {
1009         FileStatus[] snapshots = FSUtils.listStatus(fs, snapshotDir,
1010           new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
1011         if (snapshots != null) {
1012           LOG.error("Snapshots are present, but cleaners are not enabled.");
1013           checkSnapshotSupport();
1014         }
1015       }
1016     }
1017   }
1018 
1019   @Override
1020   public void initialize(MasterServices master, MetricsMaster metricsMaster) throws KeeperException,
1021       IOException, UnsupportedOperationException {
1022     this.master = master;
1023 
1024     this.rootDir = master.getMasterFileSystem().getRootDir();
1025     checkSnapshotSupport(master.getConfiguration(), master.getMasterFileSystem());
1026 
1027     // get the configuration for the coordinator
1028     Configuration conf = master.getConfiguration();
1029     long wakeFrequency = conf.getInt(SNAPSHOT_WAKE_MILLIS_KEY, SNAPSHOT_WAKE_MILLIS_DEFAULT);
1030     long timeoutMillis = conf.getLong(SNAPSHOT_TIMEOUT_MILLIS_KEY, SNAPSHOT_TIMEOUT_MILLIS_DEFAULT);
1031     int opThreads = conf.getInt(SNAPSHOT_POOL_THREADS_KEY, SNAPSHOT_POOL_THREADS_DEFAULT);
1032 
1033     // setup the default procedure coordinator
1034     String name = master.getServerName().toString();
1035     ThreadPoolExecutor tpool = ProcedureCoordinator.defaultPool(name, opThreads);
1036     ProcedureCoordinatorRpcs comms = new ZKProcedureCoordinatorRpcs(
1037         master.getZooKeeper(), SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION, name);
1038 
1039     this.coordinator = new ProcedureCoordinator(comms, tpool, timeoutMillis, wakeFrequency);
1040     this.executorService = master.getExecutorService();
1041     resetTempDir();
1042   }
1043 
1044   @Override
1045   public String getProcedureSignature() {
1046     return ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION;
1047   }
1048 
1049   @Override
1050   public void execProcedure(ProcedureDescription desc) throws IOException {
1051     takeSnapshot(toSnapshotDescription(desc));
1052   }
1053 
1054   @Override
1055   public boolean isProcedureDone(ProcedureDescription desc) throws IOException {
1056     return isSnapshotDone(toSnapshotDescription(desc));
1057   }
1058 
1059   private SnapshotDescription toSnapshotDescription(ProcedureDescription desc)
1060       throws IOException {
1061     SnapshotDescription.Builder builder = SnapshotDescription.newBuilder();
1062     if (!desc.hasInstance()) {
1063       throw new IOException("Snapshot name is not defined: " + desc.toString());
1064     }
1065     String snapshotName = desc.getInstance();
1066     List<NameStringPair> props = desc.getConfigurationList();
1067     String table = null;
1068     for (NameStringPair prop : props) {
1069       if ("table".equalsIgnoreCase(prop.getName())) {
1070         table = prop.getValue();
1071       }
1072     }
1073     if (table == null) {
1074       throw new IOException("Snapshot table is not defined: " + desc.toString());
1075     }
1076     TableName tableName = TableName.valueOf(table);
1077     builder.setTable(tableName.getNameAsString());
1078     builder.setName(snapshotName);
1079     builder.setType(SnapshotDescription.Type.FLUSH);
1080     return builder.build();
1081   }
1082 }