View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.master.snapshot;
19  
20  import java.io.FileNotFoundException;
21  import java.io.IOException;
22  import java.util.ArrayList;
23  import java.util.Collections;
24  import java.util.HashMap;
25  import java.util.HashSet;
26  import java.util.Iterator;
27  import java.util.List;
28  import java.util.Map;
29  import java.util.Set;
30  import java.util.concurrent.ThreadPoolExecutor;
31  
32  import org.apache.commons.logging.Log;
33  import org.apache.commons.logging.LogFactory;
34  import org.apache.hadoop.hbase.classification.InterfaceAudience;
35  import org.apache.hadoop.hbase.classification.InterfaceStability;
36  import org.apache.hadoop.conf.Configuration;
37  import org.apache.hadoop.fs.FSDataInputStream;
38  import org.apache.hadoop.fs.FileStatus;
39  import org.apache.hadoop.fs.FileSystem;
40  import org.apache.hadoop.fs.Path;
41  import org.apache.hadoop.hbase.TableName;
42  import org.apache.hadoop.hbase.HBaseInterfaceAudience;
43  import org.apache.hadoop.hbase.HConstants;
44  import org.apache.hadoop.hbase.HTableDescriptor;
45  import org.apache.hadoop.hbase.Stoppable;
46  import org.apache.hadoop.hbase.MetaTableAccessor;
47  import org.apache.hadoop.hbase.client.TableState;
48  import org.apache.hadoop.hbase.errorhandling.ForeignException;
49  import org.apache.hadoop.hbase.executor.ExecutorService;
50  import org.apache.hadoop.hbase.ipc.RequestContext;
51  import org.apache.hadoop.hbase.master.AssignmentManager;
52  import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
53  import org.apache.hadoop.hbase.master.MasterFileSystem;
54  import org.apache.hadoop.hbase.master.MasterServices;
55  import org.apache.hadoop.hbase.master.MetricsMaster;
56  import org.apache.hadoop.hbase.master.SnapshotSentinel;
57  import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
58  import org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner;
59  import org.apache.hadoop.hbase.procedure.MasterProcedureManager;
60  import org.apache.hadoop.hbase.procedure.Procedure;
61  import org.apache.hadoop.hbase.procedure.ProcedureCoordinator;
62  import org.apache.hadoop.hbase.procedure.ProcedureCoordinatorRpcs;
63  import org.apache.hadoop.hbase.procedure.ZKProcedureCoordinatorRpcs;
64  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameStringPair;
65  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ProcedureDescription;
66  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
67  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Type;
68  import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
69  import org.apache.hadoop.hbase.security.User;
70  import org.apache.hadoop.hbase.snapshot.ClientSnapshotDescriptionUtils;
71  import org.apache.hadoop.hbase.snapshot.HBaseSnapshotException;
72  import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
73  import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper;
74  import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
75  import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
76  import org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException;
77  import org.apache.hadoop.hbase.snapshot.SnapshotExistsException;
78  import org.apache.hadoop.hbase.snapshot.SnapshotManifest;
79  import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
80  import org.apache.hadoop.hbase.snapshot.TablePartiallyOpenException;
81  import org.apache.hadoop.hbase.snapshot.UnknownSnapshotException;
82  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
83  import org.apache.hadoop.hbase.util.FSUtils;
84  import org.apache.zookeeper.KeeperException;
85  
86  /**
87   * This class manages the procedure of taking and restoring snapshots. There is only one
88   * SnapshotManager for the master.
89   * <p>
90   * The class provides methods for monitoring in-progress snapshot actions.
91   * <p>
92   * Note: Currently there can only be one snapshot being taken at a time over the cluster. This is a
93   * simplification in the current implementation.
94   */
95  @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)
96  @InterfaceStability.Unstable
97  public class SnapshotManager extends MasterProcedureManager implements Stoppable {
98    private static final Log LOG = LogFactory.getLog(SnapshotManager.class);
99  
100   /** By default, check to see if the snapshot is complete every WAKE MILLIS (ms) */
101   private static final int SNAPSHOT_WAKE_MILLIS_DEFAULT = 500;
102 
103   /**
104    * Wait time before removing a finished sentinel from the in-progress map
105    *
106    * NOTE: This is used as a safety auto cleanup.
107    * The snapshot and restore handlers map entries are removed when a user asks if a snapshot or
108    * restore is completed. This operation is part of the HBaseAdmin snapshot/restore API flow.
109    * In case something fails on the client side and the snapshot/restore state is not reclaimed
110    * after a default timeout, the entry is removed from the in-progress map.
111    * At this point, if the user asks for the snapshot/restore status, the result will be
112    * snapshot done if exists or failed if it doesn't exists.
113    */
114   private static final int SNAPSHOT_SENTINELS_CLEANUP_TIMEOUT = 60 * 1000;
115 
116   /** Enable or disable snapshot support */
117   public static final String HBASE_SNAPSHOT_ENABLED = "hbase.snapshot.enabled";
118 
119   /**
120    * Conf key for # of ms elapsed between checks for snapshot errors while waiting for
121    * completion.
122    */
123   private static final String SNAPSHOT_WAKE_MILLIS_KEY = "hbase.snapshot.master.wakeMillis";
124 
125   /** By default, check to see if the snapshot is complete (ms) */
126   private static final int SNAPSHOT_TIMEOUT_MILLIS_DEFAULT = 60000;
127 
128   /**
129    * Conf key for # of ms elapsed before injecting a snapshot timeout error when waiting for
130    * completion.
131    */
132   private static final String SNAPSHOT_TIMEOUT_MILLIS_KEY = "hbase.snapshot.master.timeoutMillis";
133 
134   /** Name of the operation to use in the controller */
135   public static final String ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION = "online-snapshot";
136 
137   /** Conf key for # of threads used by the SnapshotManager thread pool */
138   private static final String SNAPSHOT_POOL_THREADS_KEY = "hbase.snapshot.master.threads";
139 
140   /** number of current operations running on the master */
141   private static final int SNAPSHOT_POOL_THREADS_DEFAULT = 1;
142 
143   private boolean stopped;
144   private MasterServices master;  // Needed by TableEventHandlers
145   private ProcedureCoordinator coordinator;
146 
147   // Is snapshot feature enabled?
148   private boolean isSnapshotSupported = false;
149 
150   // Snapshot handlers map, with table name as key.
151   // The map is always accessed and modified under the object lock using synchronized.
152   // snapshotTable() will insert an Handler in the table.
153   // isSnapshotDone() will remove the handler requested if the operation is finished.
154   private Map<TableName, SnapshotSentinel> snapshotHandlers =
155       new HashMap<TableName, SnapshotSentinel>();
156 
157   // Restore Sentinels map, with table name as key.
158   // The map is always accessed and modified under the object lock using synchronized.
159   // restoreSnapshot()/cloneSnapshot() will insert an Handler in the table.
160   // isRestoreDone() will remove the handler requested if the operation is finished.
161   private Map<TableName, SnapshotSentinel> restoreHandlers =
162       new HashMap<TableName, SnapshotSentinel>();
163 
164   private Path rootDir;
165   private ExecutorService executorService;
166 
167   public SnapshotManager() {}
168 
169   /**
170    * Fully specify all necessary components of a snapshot manager. Exposed for testing.
171    * @param master services for the master where the manager is running
172    * @param coordinator procedure coordinator instance.  exposed for testing.
173    * @param pool HBase ExecutorServcie instance, exposed for testing.
174    */
175   public SnapshotManager(final MasterServices master, final MetricsMaster metricsMaster,
176       ProcedureCoordinator coordinator, ExecutorService pool)
177       throws IOException, UnsupportedOperationException {
178     this.master = master;
179 
180     this.rootDir = master.getMasterFileSystem().getRootDir();
181     checkSnapshotSupport(master.getConfiguration(), master.getMasterFileSystem());
182 
183     this.coordinator = coordinator;
184     this.executorService = pool;
185     resetTempDir();
186   }
187 
188   /**
189    * Gets the list of all completed snapshots.
190    * @return list of SnapshotDescriptions
191    * @throws IOException File system exception
192    */
193   public List<SnapshotDescription> getCompletedSnapshots() throws IOException {
194     return getCompletedSnapshots(SnapshotDescriptionUtils.getSnapshotsDir(rootDir));
195   }
196 
197   /**
198    * Gets the list of all completed snapshots.
199    * @param snapshotDir snapshot directory
200    * @return list of SnapshotDescriptions
201    * @throws IOException File system exception
202    */
203   private List<SnapshotDescription> getCompletedSnapshots(Path snapshotDir) throws IOException {
204     List<SnapshotDescription> snapshotDescs = new ArrayList<SnapshotDescription>();
205     // first create the snapshot root path and check to see if it exists
206     FileSystem fs = master.getMasterFileSystem().getFileSystem();
207     if (snapshotDir == null) snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
208 
209     // if there are no snapshots, return an empty list
210     if (!fs.exists(snapshotDir)) {
211       return snapshotDescs;
212     }
213 
214     // ignore all the snapshots in progress
215     FileStatus[] snapshots = fs.listStatus(snapshotDir,
216       new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
217     // loop through all the completed snapshots
218     for (FileStatus snapshot : snapshots) {
219       Path info = new Path(snapshot.getPath(), SnapshotDescriptionUtils.SNAPSHOTINFO_FILE);
220       // if the snapshot is bad
221       if (!fs.exists(info)) {
222         LOG.error("Snapshot information for " + snapshot.getPath() + " doesn't exist");
223         continue;
224       }
225       FSDataInputStream in = null;
226       try {
227         in = fs.open(info);
228         SnapshotDescription desc = SnapshotDescription.parseFrom(in);
229         snapshotDescs.add(desc);
230       } catch (IOException e) {
231         LOG.warn("Found a corrupted snapshot " + snapshot.getPath(), e);
232       } finally {
233         if (in != null) {
234           in.close();
235         }
236       }
237     }
238     return snapshotDescs;
239   }
240 
241   /**
242    * Cleans up any snapshots in the snapshot/.tmp directory that were left from failed
243    * snapshot attempts.
244    *
245    * @throws IOException if we can't reach the filesystem
246    */
247   void resetTempDir() throws IOException {
248     // cleanup any existing snapshots.
249     Path tmpdir = SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir);
250     if (master.getMasterFileSystem().getFileSystem().exists(tmpdir)) {
251       if (!master.getMasterFileSystem().getFileSystem().delete(tmpdir, true)) {
252         LOG.warn("Couldn't delete working snapshot directory: " + tmpdir);
253       }
254     }
255   }
256 
257   /**
258    * Delete the specified snapshot
259    * @param snapshot
260    * @throws SnapshotDoesNotExistException If the specified snapshot does not exist.
261    * @throws IOException For filesystem IOExceptions
262    */
263   public void deleteSnapshot(SnapshotDescription snapshot) throws SnapshotDoesNotExistException, IOException {
264     // check to see if it is completed
265     if (!isSnapshotCompleted(snapshot)) {
266       throw new SnapshotDoesNotExistException(snapshot);
267     }
268 
269     String snapshotName = snapshot.getName();
270     // first create the snapshot description and check to see if it exists
271     FileSystem fs = master.getMasterFileSystem().getFileSystem();
272     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
273     // Get snapshot info from file system. The one passed as parameter is a "fake" snapshotInfo with
274     // just the "name" and it does not contains the "real" snapshot information
275     snapshot = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
276 
277     // call coproc pre hook
278     MasterCoprocessorHost cpHost = master.getMasterCoprocessorHost();
279     if (cpHost != null) {
280       cpHost.preDeleteSnapshot(snapshot);
281     }
282 
283     LOG.debug("Deleting snapshot: " + snapshotName);
284     // delete the existing snapshot
285     if (!fs.delete(snapshotDir, true)) {
286       throw new HBaseSnapshotException("Failed to delete snapshot directory: " + snapshotDir);
287     }
288 
289     // call coproc post hook
290     if (cpHost != null) {
291       cpHost.postDeleteSnapshot(snapshot);
292     }
293 
294   }
295 
296   /**
297    * Check if the specified snapshot is done
298    *
299    * @param expected
300    * @return true if snapshot is ready to be restored, false if it is still being taken.
301    * @throws IOException IOException if error from HDFS or RPC
302    * @throws UnknownSnapshotException if snapshot is invalid or does not exist.
303    */
304   public boolean isSnapshotDone(SnapshotDescription expected) throws IOException {
305     // check the request to make sure it has a snapshot
306     if (expected == null) {
307       throw new UnknownSnapshotException(
308          "No snapshot name passed in request, can't figure out which snapshot you want to check.");
309     }
310 
311     String ssString = ClientSnapshotDescriptionUtils.toString(expected);
312 
313     // check to see if the sentinel exists,
314     // and if the task is complete removes it from the in-progress snapshots map.
315     SnapshotSentinel handler = removeSentinelIfFinished(this.snapshotHandlers, expected);
316 
317     // stop tracking "abandoned" handlers
318     cleanupSentinels();
319 
320     if (handler == null) {
321       // If there's no handler in the in-progress map, it means one of the following:
322       //   - someone has already requested the snapshot state
323       //   - the requested snapshot was completed long time ago (cleanupSentinels() timeout)
324       //   - the snapshot was never requested
325       // In those cases returns to the user the "done state" if the snapshots exists on disk,
326       // otherwise raise an exception saying that the snapshot is not running and doesn't exist.
327       if (!isSnapshotCompleted(expected)) {
328         throw new UnknownSnapshotException("Snapshot " + ssString
329             + " is not currently running or one of the known completed snapshots.");
330       }
331       // was done, return true;
332       return true;
333     }
334 
335     // pass on any failure we find in the sentinel
336     try {
337       handler.rethrowExceptionIfFailed();
338     } catch (ForeignException e) {
339       // Give some procedure info on an exception.
340       String status;
341       Procedure p = coordinator.getProcedure(expected.getName());
342       if (p != null) {
343         status = p.getStatus();
344       } else {
345         status = expected.getName() + " not found in proclist " + coordinator.getProcedureNames();
346       }
347       throw new HBaseSnapshotException("Snapshot " + ssString +  " had an error.  " + status, e,
348           expected);
349     }
350 
351     // check to see if we are done
352     if (handler.isFinished()) {
353       LOG.debug("Snapshot '" + ssString + "' has completed, notifying client.");
354       return true;
355     } else if (LOG.isDebugEnabled()) {
356       LOG.debug("Snapshoting '" + ssString + "' is still in progress!");
357     }
358     return false;
359   }
360 
361   /**
362    * Check to see if there is a snapshot in progress with the same name or on the same table.
363    * Currently we have a limitation only allowing a single snapshot per table at a time. Also we
364    * don't allow snapshot with the same name.
365    * @param snapshot description of the snapshot being checked.
366    * @return <tt>true</tt> if there is a snapshot in progress with the same name or on the same
367    *         table.
368    */
369   synchronized boolean isTakingSnapshot(final SnapshotDescription snapshot) {
370     TableName snapshotTable = TableName.valueOf(snapshot.getTable());
371     if (isTakingSnapshot(snapshotTable)) {
372       return true;
373     }
374     Iterator<Map.Entry<TableName, SnapshotSentinel>> it = this.snapshotHandlers.entrySet().iterator();
375     while (it.hasNext()) {
376       Map.Entry<TableName, SnapshotSentinel> entry = it.next();
377       SnapshotSentinel sentinel = entry.getValue();
378       if (snapshot.getName().equals(sentinel.getSnapshot().getName()) && !sentinel.isFinished()) {
379         return true;
380       }
381     }
382     return false;
383   }
384 
385   /**
386    * Check to see if the specified table has a snapshot in progress.  Currently we have a
387    * limitation only allowing a single snapshot per table at a time.
388    * @param tableName name of the table being snapshotted.
389    * @return <tt>true</tt> if there is a snapshot in progress on the specified table.
390    */
391   synchronized boolean isTakingSnapshot(final TableName tableName) {
392     SnapshotSentinel handler = this.snapshotHandlers.get(tableName);
393     return handler != null && !handler.isFinished();
394   }
395 
396   /**
397    * Check to make sure that we are OK to run the passed snapshot. Checks to make sure that we
398    * aren't already running a snapshot or restore on the requested table.
399    * @param snapshot description of the snapshot we want to start
400    * @throws HBaseSnapshotException if the filesystem could not be prepared to start the snapshot
401    */
402   private synchronized void prepareToTakeSnapshot(SnapshotDescription snapshot)
403       throws HBaseSnapshotException {
404     FileSystem fs = master.getMasterFileSystem().getFileSystem();
405     Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
406     TableName snapshotTable =
407         TableName.valueOf(snapshot.getTable());
408 
409     // make sure we aren't already running a snapshot
410     if (isTakingSnapshot(snapshot)) {
411       SnapshotSentinel handler = this.snapshotHandlers.get(snapshotTable);
412       throw new SnapshotCreationException("Rejected taking "
413           + ClientSnapshotDescriptionUtils.toString(snapshot)
414           + " because we are already running another snapshot "
415           + (handler != null ? ("on the same table " +
416               ClientSnapshotDescriptionUtils.toString(handler.getSnapshot()))
417               : "with the same name"), snapshot);
418     }
419 
420     // make sure we aren't running a restore on the same table
421     if (isRestoringTable(snapshotTable)) {
422       SnapshotSentinel handler = restoreHandlers.get(snapshotTable);
423       throw new SnapshotCreationException("Rejected taking "
424           + ClientSnapshotDescriptionUtils.toString(snapshot)
425           + " because we are already have a restore in progress on the same snapshot "
426           + ClientSnapshotDescriptionUtils.toString(handler.getSnapshot()), snapshot);
427     }
428 
429     try {
430       // delete the working directory, since we aren't running the snapshot. Likely leftovers
431       // from a failed attempt.
432       fs.delete(workingDir, true);
433 
434       // recreate the working directory for the snapshot
435       if (!fs.mkdirs(workingDir)) {
436         throw new SnapshotCreationException("Couldn't create working directory (" + workingDir
437             + ") for snapshot" , snapshot);
438       }
439     } catch (HBaseSnapshotException e) {
440       throw e;
441     } catch (IOException e) {
442       throw new SnapshotCreationException(
443           "Exception while checking to see if snapshot could be started.", e, snapshot);
444     }
445   }
446 
447   /**
448    * Take a snapshot of a disabled table.
449    * @param snapshot description of the snapshot to take. Modified to be {@link Type#DISABLED}.
450    * @throws HBaseSnapshotException if the snapshot could not be started
451    */
452   private synchronized void snapshotDisabledTable(SnapshotDescription snapshot)
453       throws HBaseSnapshotException {
454     // setup the snapshot
455     prepareToTakeSnapshot(snapshot);
456 
457     // set the snapshot to be a disabled snapshot, since the client doesn't know about that
458     snapshot = snapshot.toBuilder().setType(Type.DISABLED).build();
459 
460     // Take the snapshot of the disabled table
461     DisabledTableSnapshotHandler handler =
462         new DisabledTableSnapshotHandler(snapshot, master);
463     snapshotTable(snapshot, handler);
464   }
465 
466   /**
467    * Take a snapshot of an enabled table.
468    * @param snapshot description of the snapshot to take.
469    * @throws HBaseSnapshotException if the snapshot could not be started
470    */
471   private synchronized void snapshotEnabledTable(SnapshotDescription snapshot)
472       throws HBaseSnapshotException {
473     // setup the snapshot
474     prepareToTakeSnapshot(snapshot);
475 
476     // Take the snapshot of the enabled table
477     EnabledTableSnapshotHandler handler =
478         new EnabledTableSnapshotHandler(snapshot, master, this);
479     snapshotTable(snapshot, handler);
480   }
481 
482   /**
483    * Take a snapshot using the specified handler.
484    * On failure the snapshot temporary working directory is removed.
485    * NOTE: prepareToTakeSnapshot() called before this one takes care of the rejecting the
486    *       snapshot request if the table is busy with another snapshot/restore operation.
487    * @param snapshot the snapshot description
488    * @param handler the snapshot handler
489    */
490   private synchronized void snapshotTable(SnapshotDescription snapshot,
491       final TakeSnapshotHandler handler) throws HBaseSnapshotException {
492     try {
493       handler.prepare();
494       this.executorService.submit(handler);
495       this.snapshotHandlers.put(TableName.valueOf(snapshot.getTable()), handler);
496     } catch (Exception e) {
497       // cleanup the working directory by trying to delete it from the fs.
498       Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
499       try {
500         if (!this.master.getMasterFileSystem().getFileSystem().delete(workingDir, true)) {
501           LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" +
502               ClientSnapshotDescriptionUtils.toString(snapshot));
503         }
504       } catch (IOException e1) {
505         LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" +
506             ClientSnapshotDescriptionUtils.toString(snapshot));
507       }
508       // fail the snapshot
509       throw new SnapshotCreationException("Could not build snapshot handler", e, snapshot);
510     }
511   }
512 
513   /**
514    * Take a snapshot based on the enabled/disabled state of the table.
515    *
516    * @param snapshot
517    * @throws HBaseSnapshotException when a snapshot specific exception occurs.
518    * @throws IOException when some sort of generic IO exception occurs.
519    */
520   public void takeSnapshot(SnapshotDescription snapshot) throws IOException {
521     // check to see if we already completed the snapshot
522     if (isSnapshotCompleted(snapshot)) {
523       throw new SnapshotExistsException("Snapshot '" + snapshot.getName()
524           + "' already stored on the filesystem.", snapshot);
525     }
526 
527     LOG.debug("No existing snapshot, attempting snapshot...");
528 
529     // stop tracking "abandoned" handlers
530     cleanupSentinels();
531 
532     // check to see if the table exists
533     HTableDescriptor desc = null;
534     try {
535       desc = master.getTableDescriptors().get(
536           TableName.valueOf(snapshot.getTable()));
537     } catch (FileNotFoundException e) {
538       String msg = "Table:" + snapshot.getTable() + " info doesn't exist!";
539       LOG.error(msg);
540       throw new SnapshotCreationException(msg, e, snapshot);
541     } catch (IOException e) {
542       throw new SnapshotCreationException("Error while geting table description for table "
543           + snapshot.getTable(), e, snapshot);
544     }
545     if (desc == null) {
546       throw new SnapshotCreationException("Table '" + snapshot.getTable()
547           + "' doesn't exist, can't take snapshot.", snapshot);
548     }
549     SnapshotDescription.Builder builder = snapshot.toBuilder();
550     // if not specified, set the snapshot format
551     if (!snapshot.hasVersion()) {
552       builder.setVersion(SnapshotDescriptionUtils.SNAPSHOT_LAYOUT_VERSION);
553     }
554     User user = RequestContext.getRequestUser();
555     if (User.isHBaseSecurityEnabled(master.getConfiguration()) && user != null) {
556       builder.setOwner(user.getShortName());
557     }
558     snapshot = builder.build();
559 
560     // call pre coproc hook
561     MasterCoprocessorHost cpHost = master.getMasterCoprocessorHost();
562     if (cpHost != null) {
563       cpHost.preSnapshot(snapshot, desc);
564     }
565 
566     // if the table is enabled, then have the RS run actually the snapshot work
567     TableName snapshotTable = TableName.valueOf(snapshot.getTable());
568     AssignmentManager assignmentMgr = master.getAssignmentManager();
569     if (assignmentMgr.getTableStateManager().isTableState(snapshotTable,
570         TableState.State.ENABLED)) {
571       LOG.debug("Table enabled, starting distributed snapshot.");
572       snapshotEnabledTable(snapshot);
573       LOG.debug("Started snapshot: " + ClientSnapshotDescriptionUtils.toString(snapshot));
574     }
575     // For disabled table, snapshot is created by the master
576     else if (assignmentMgr.getTableStateManager().isTableState(snapshotTable,
577         TableState.State.DISABLED)) {
578       LOG.debug("Table is disabled, running snapshot entirely on master.");
579       snapshotDisabledTable(snapshot);
580       LOG.debug("Started snapshot: " + ClientSnapshotDescriptionUtils.toString(snapshot));
581     } else {
582       LOG.error("Can't snapshot table '" + snapshot.getTable()
583           + "', isn't open or closed, we don't know what to do!");
584       TablePartiallyOpenException tpoe = new TablePartiallyOpenException(snapshot.getTable()
585           + " isn't fully open.");
586       throw new SnapshotCreationException("Table is not entirely open or closed", tpoe, snapshot);
587     }
588 
589     // call post coproc hook
590     if (cpHost != null) {
591       cpHost.postSnapshot(snapshot, desc);
592     }
593   }
594 
595   /**
596    * Set the handler for the current snapshot
597    * <p>
598    * Exposed for TESTING
599    * @param tableName
600    * @param handler handler the master should use
601    *
602    * TODO get rid of this if possible, repackaging, modify tests.
603    */
604   public synchronized void setSnapshotHandlerForTesting(
605       final TableName tableName,
606       final SnapshotSentinel handler) {
607     if (handler != null) {
608       this.snapshotHandlers.put(tableName, handler);
609     } else {
610       this.snapshotHandlers.remove(tableName);
611     }
612   }
613 
614   /**
615    * @return distributed commit coordinator for all running snapshots
616    */
617   ProcedureCoordinator getCoordinator() {
618     return coordinator;
619   }
620 
621   /**
622    * Check to see if the snapshot is one of the currently completed snapshots
623    * Returns true if the snapshot exists in the "completed snapshots folder".
624    *
625    * @param snapshot expected snapshot to check
626    * @return <tt>true</tt> if the snapshot is stored on the {@link FileSystem}, <tt>false</tt> if is
627    *         not stored
628    * @throws IOException if the filesystem throws an unexpected exception,
629    * @throws IllegalArgumentException if snapshot name is invalid.
630    */
631   private boolean isSnapshotCompleted(SnapshotDescription snapshot) throws IOException {
632     try {
633       final Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
634       FileSystem fs = master.getMasterFileSystem().getFileSystem();
635       // check to see if the snapshot already exists
636       return fs.exists(snapshotDir);
637     } catch (IllegalArgumentException iae) {
638       throw new UnknownSnapshotException("Unexpected exception thrown", iae);
639     }
640   }
641 
642   /**
643    * Clone the specified snapshot into a new table.
644    * The operation will fail if the destination table has a snapshot or restore in progress.
645    *
646    * @param snapshot Snapshot Descriptor
647    * @param hTableDescriptor Table Descriptor of the table to create
648    */
649   synchronized void cloneSnapshot(final SnapshotDescription snapshot,
650       final HTableDescriptor hTableDescriptor) throws HBaseSnapshotException {
651     TableName tableName = hTableDescriptor.getTableName();
652 
653     // make sure we aren't running a snapshot on the same table
654     if (isTakingSnapshot(tableName)) {
655       throw new RestoreSnapshotException("Snapshot in progress on the restore table=" + tableName);
656     }
657 
658     // make sure we aren't running a restore on the same table
659     if (isRestoringTable(tableName)) {
660       throw new RestoreSnapshotException("Restore already in progress on the table=" + tableName);
661     }
662 
663     try {
664       CloneSnapshotHandler handler =
665         new CloneSnapshotHandler(master, snapshot, hTableDescriptor).prepare();
666       this.executorService.submit(handler);
667       this.restoreHandlers.put(tableName, handler);
668     } catch (Exception e) {
669       String msg = "Couldn't clone the snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot) +
670         " on table=" + tableName;
671       LOG.error(msg, e);
672       throw new RestoreSnapshotException(msg, e);
673     }
674   }
675 
676   /**
677    * Restore the specified snapshot
678    * @param reqSnapshot
679    * @throws IOException
680    */
681   public void restoreSnapshot(SnapshotDescription reqSnapshot) throws IOException {
682     FileSystem fs = master.getMasterFileSystem().getFileSystem();
683     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(reqSnapshot, rootDir);
684     MasterCoprocessorHost cpHost = master.getMasterCoprocessorHost();
685 
686     // check if the snapshot exists
687     if (!fs.exists(snapshotDir)) {
688       LOG.error("A Snapshot named '" + reqSnapshot.getName() + "' does not exist.");
689       throw new SnapshotDoesNotExistException(reqSnapshot);
690     }
691 
692     // Get snapshot info from file system. The reqSnapshot is a "fake" snapshotInfo with
693     // just the snapshot "name" and table name to restore. It does not contains the "real" snapshot
694     // information.
695     SnapshotDescription snapshot = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
696     SnapshotManifest manifest = SnapshotManifest.open(master.getConfiguration(), fs,
697         snapshotDir, snapshot);
698     HTableDescriptor snapshotTableDesc = manifest.getTableDescriptor();
699     TableName tableName = TableName.valueOf(reqSnapshot.getTable());
700 
701     // stop tracking "abandoned" handlers
702     cleanupSentinels();
703 
704     // Verify snapshot validity
705     SnapshotReferenceUtil.verifySnapshot(master.getConfiguration(), fs, manifest);
706 
707     // Execute the restore/clone operation
708     if (MetaTableAccessor.tableExists(master.getShortCircuitConnection(), tableName)) {
709       if (master.getTableStateManager().isTableState(
710           TableName.valueOf(snapshot.getTable()), TableState.State.ENABLED)) {
711         throw new UnsupportedOperationException("Table '" +
712             TableName.valueOf(snapshot.getTable()) + "' must be disabled in order to " +
713             "perform a restore operation" +
714             ".");
715       }
716 
717       // call coproc pre hook
718       if (cpHost != null) {
719         cpHost.preRestoreSnapshot(reqSnapshot, snapshotTableDesc);
720       }
721       restoreSnapshot(snapshot, snapshotTableDesc);
722       LOG.info("Restore snapshot=" + snapshot.getName() + " as table=" + tableName);
723 
724       if (cpHost != null) {
725         cpHost.postRestoreSnapshot(reqSnapshot, snapshotTableDesc);
726       }
727     } else {
728       HTableDescriptor htd = RestoreSnapshotHelper.cloneTableSchema(snapshotTableDesc, tableName);
729       if (cpHost != null) {
730         cpHost.preCloneSnapshot(reqSnapshot, htd);
731       }
732       cloneSnapshot(snapshot, htd);
733       LOG.info("Clone snapshot=" + snapshot.getName() + " as table=" + tableName);
734 
735       if (cpHost != null) {
736         cpHost.postCloneSnapshot(reqSnapshot, htd);
737       }
738     }
739   }
740 
741   /**
742    * Restore the specified snapshot.
743    * The restore will fail if the destination table has a snapshot or restore in progress.
744    *
745    * @param snapshot Snapshot Descriptor
746    * @param hTableDescriptor Table Descriptor
747    */
748   private synchronized void restoreSnapshot(final SnapshotDescription snapshot,
749       final HTableDescriptor hTableDescriptor) throws HBaseSnapshotException {
750     TableName tableName = hTableDescriptor.getTableName();
751 
752     // make sure we aren't running a snapshot on the same table
753     if (isTakingSnapshot(tableName)) {
754       throw new RestoreSnapshotException("Snapshot in progress on the restore table=" + tableName);
755     }
756 
757     // make sure we aren't running a restore on the same table
758     if (isRestoringTable(tableName)) {
759       throw new RestoreSnapshotException("Restore already in progress on the table=" + tableName);
760     }
761 
762     try {
763       RestoreSnapshotHandler handler =
764         new RestoreSnapshotHandler(master, snapshot, hTableDescriptor).prepare();
765       this.executorService.submit(handler);
766       restoreHandlers.put(tableName, handler);
767     } catch (Exception e) {
768       String msg = "Couldn't restore the snapshot=" + ClientSnapshotDescriptionUtils.toString(
769           snapshot)  +
770           " on table=" + tableName;
771       LOG.error(msg, e);
772       throw new RestoreSnapshotException(msg, e);
773     }
774   }
775 
776   /**
777    * Verify if the restore of the specified table is in progress.
778    *
779    * @param tableName table under restore
780    * @return <tt>true</tt> if there is a restore in progress of the specified table.
781    */
782   private synchronized boolean isRestoringTable(final TableName tableName) {
783     SnapshotSentinel sentinel = this.restoreHandlers.get(tableName);
784     return(sentinel != null && !sentinel.isFinished());
785   }
786 
787   /**
788    * Returns the status of a restore operation.
789    * If the in-progress restore is failed throws the exception that caused the failure.
790    *
791    * @param snapshot
792    * @return false if in progress, true if restore is completed or not requested.
793    * @throws IOException if there was a failure during the restore
794    */
795   public boolean isRestoreDone(final SnapshotDescription snapshot) throws IOException {
796     // check to see if the sentinel exists,
797     // and if the task is complete removes it from the in-progress restore map.
798     SnapshotSentinel sentinel = removeSentinelIfFinished(this.restoreHandlers, snapshot);
799 
800     // stop tracking "abandoned" handlers
801     cleanupSentinels();
802 
803     if (sentinel == null) {
804       // there is no sentinel so restore is not in progress.
805       return true;
806     }
807 
808     LOG.debug("Verify snapshot=" + snapshot.getName() + " against="
809         + sentinel.getSnapshot().getName() + " table=" +
810         TableName.valueOf(snapshot.getTable()));
811 
812     // If the restore is failed, rethrow the exception
813     sentinel.rethrowExceptionIfFailed();
814 
815     // check to see if we are done
816     if (sentinel.isFinished()) {
817       LOG.debug("Restore snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot) +
818           " has completed. Notifying the client.");
819       return true;
820     }
821 
822     if (LOG.isDebugEnabled()) {
823       LOG.debug("Sentinel is not yet finished with restoring snapshot=" +
824           ClientSnapshotDescriptionUtils.toString(snapshot));
825     }
826     return false;
827   }
828 
829   /**
830    * Return the handler if it is currently live and has the same snapshot target name.
831    * The handler is removed from the sentinels map if completed.
832    * @param sentinels live handlers
833    * @param snapshot snapshot description
834    * @return null if doesn't match, else a live handler.
835    */
836   private synchronized SnapshotSentinel removeSentinelIfFinished(
837       final Map<TableName, SnapshotSentinel> sentinels,
838       final SnapshotDescription snapshot) {
839     if (!snapshot.hasTable()) {
840       return null;
841     }
842 
843     TableName snapshotTable = TableName.valueOf(snapshot.getTable());
844     SnapshotSentinel h = sentinels.get(snapshotTable);
845     if (h == null) {
846       return null;
847     }
848 
849     if (!h.getSnapshot().getName().equals(snapshot.getName())) {
850       // specified snapshot is to the one currently running
851       return null;
852     }
853 
854     // Remove from the "in-progress" list once completed
855     if (h.isFinished()) {
856       sentinels.remove(snapshotTable);
857     }
858 
859     return h;
860   }
861 
862   /**
863    * Removes "abandoned" snapshot/restore requests.
864    * As part of the HBaseAdmin snapshot/restore API the operation status is checked until completed,
865    * and the in-progress maps are cleaned up when the status of a completed task is requested.
866    * To avoid having sentinels staying around for long time if something client side is failed,
867    * each operation tries to clean up the in-progress maps sentinels finished from a long time.
868    */
869   private void cleanupSentinels() {
870     cleanupSentinels(this.snapshotHandlers);
871     cleanupSentinels(this.restoreHandlers);
872   }
873 
874   /**
875    * Remove the sentinels that are marked as finished and the completion time
876    * has exceeded the removal timeout.
877    * @param sentinels map of sentinels to clean
878    */
879   private synchronized void cleanupSentinels(final Map<TableName, SnapshotSentinel> sentinels) {
880     long currentTime = EnvironmentEdgeManager.currentTime();
881     Iterator<Map.Entry<TableName, SnapshotSentinel>> it =
882         sentinels.entrySet().iterator();
883     while (it.hasNext()) {
884       Map.Entry<TableName, SnapshotSentinel> entry = it.next();
885       SnapshotSentinel sentinel = entry.getValue();
886       if (sentinel.isFinished() &&
887           (currentTime - sentinel.getCompletionTimestamp()) > SNAPSHOT_SENTINELS_CLEANUP_TIMEOUT)
888       {
889         it.remove();
890       }
891     }
892   }
893 
894   //
895   // Implementing Stoppable interface
896   //
897 
898   @Override
899   public void stop(String why) {
900     // short circuit
901     if (this.stopped) return;
902     // make sure we get stop
903     this.stopped = true;
904     // pass the stop onto take snapshot handlers
905     for (SnapshotSentinel snapshotHandler: this.snapshotHandlers.values()) {
906       snapshotHandler.cancel(why);
907     }
908 
909     // pass the stop onto all the restore handlers
910     for (SnapshotSentinel restoreHandler: this.restoreHandlers.values()) {
911       restoreHandler.cancel(why);
912     }
913     try {
914       coordinator.close();
915     } catch (IOException e) {
916       LOG.error("stop ProcedureCoordinator error", e);
917     }
918   }
919 
920   @Override
921   public boolean isStopped() {
922     return this.stopped;
923   }
924 
925   /**
926    * Throws an exception if snapshot operations (take a snapshot, restore, clone) are not supported.
927    * Called at the beginning of snapshot() and restoreSnapshot() methods.
928    * @throws UnsupportedOperationException if snapshot are not supported
929    */
930   public void checkSnapshotSupport() throws UnsupportedOperationException {
931     if (!this.isSnapshotSupported) {
932       throw new UnsupportedOperationException(
933         "To use snapshots, You must add to the hbase-site.xml of the HBase Master: '" +
934           HBASE_SNAPSHOT_ENABLED + "' property with value 'true'.");
935     }
936   }
937 
938   /**
939    * Called at startup, to verify if snapshot operation is supported, and to avoid
940    * starting the master if there're snapshots present but the cleaners needed are missing.
941    * Otherwise we can end up with snapshot data loss.
942    * @param conf The {@link Configuration} object to use
943    * @param mfs The MasterFileSystem to use
944    * @throws IOException in case of file-system operation failure
945    * @throws UnsupportedOperationException in case cleaners are missing and
946    *         there're snapshot in the system
947    */
948   private void checkSnapshotSupport(final Configuration conf, final MasterFileSystem mfs)
949       throws IOException, UnsupportedOperationException {
950     // Verify if snapshot is disabled by the user
951     String enabled = conf.get(HBASE_SNAPSHOT_ENABLED);
952     boolean snapshotEnabled = conf.getBoolean(HBASE_SNAPSHOT_ENABLED, false);
953     boolean userDisabled = (enabled != null && enabled.trim().length() > 0 && !snapshotEnabled);
954 
955     // Extract cleaners from conf
956     Set<String> hfileCleaners = new HashSet<String>();
957     String[] cleaners = conf.getStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS);
958     if (cleaners != null) Collections.addAll(hfileCleaners, cleaners);
959 
960     Set<String> logCleaners = new HashSet<String>();
961     cleaners = conf.getStrings(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS);
962     if (cleaners != null) Collections.addAll(logCleaners, cleaners);
963 
964     // check if an older version of snapshot directory was present
965     Path oldSnapshotDir = new Path(mfs.getRootDir(), HConstants.OLD_SNAPSHOT_DIR_NAME);
966     FileSystem fs = mfs.getFileSystem();
967     List<SnapshotDescription> ss = getCompletedSnapshots(new Path(rootDir, oldSnapshotDir));
968     if (ss != null && !ss.isEmpty()) {
969       LOG.error("Snapshots from an earlier release were found under: " + oldSnapshotDir);
970       LOG.error("Please rename the directory as " + HConstants.SNAPSHOT_DIR_NAME);
971     }
972 
973     // If the user has enabled the snapshot, we force the cleaners to be present
974     // otherwise we still need to check if cleaners are enabled or not and verify
975     // that there're no snapshot in the .snapshot folder.
976     if (snapshotEnabled) {
977       // Inject snapshot cleaners, if snapshot.enable is true
978       hfileCleaners.add(SnapshotHFileCleaner.class.getName());
979       hfileCleaners.add(HFileLinkCleaner.class.getName());
980       logCleaners.add(SnapshotLogCleaner.class.getName());
981 
982       // Set cleaners conf
983       conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS,
984         hfileCleaners.toArray(new String[hfileCleaners.size()]));
985       conf.setStrings(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS,
986         logCleaners.toArray(new String[logCleaners.size()]));
987     } else {
988       // Verify if cleaners are present
989       snapshotEnabled = logCleaners.contains(SnapshotLogCleaner.class.getName()) &&
990         hfileCleaners.contains(SnapshotHFileCleaner.class.getName()) &&
991         hfileCleaners.contains(HFileLinkCleaner.class.getName());
992 
993       // Warn if the cleaners are enabled but the snapshot.enabled property is false/not set.
994       if (snapshotEnabled) {
995         LOG.warn("Snapshot log and hfile cleaners are present in the configuration, " +
996           "but the '" + HBASE_SNAPSHOT_ENABLED + "' property " +
997           (userDisabled ? "is set to 'false'." : "is not set."));
998       }
999     }
1000 
1001     // Mark snapshot feature as enabled if cleaners are present and user has not disabled it.
1002     this.isSnapshotSupported = snapshotEnabled && !userDisabled;
1003 
1004     // If cleaners are not enabled, verify that there're no snapshot in the .snapshot folder
1005     // otherwise we end up with snapshot data loss.
1006     if (!snapshotEnabled) {
1007       LOG.info("Snapshot feature is not enabled, missing log and hfile cleaners.");
1008       Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(mfs.getRootDir());
1009       if (fs.exists(snapshotDir)) {
1010         FileStatus[] snapshots = FSUtils.listStatus(fs, snapshotDir,
1011           new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
1012         if (snapshots != null) {
1013           LOG.error("Snapshots are present, but cleaners are not enabled.");
1014           checkSnapshotSupport();
1015         }
1016       }
1017     }
1018   }
1019 
1020   @Override
1021   public void initialize(MasterServices master, MetricsMaster metricsMaster) throws KeeperException,
1022       IOException, UnsupportedOperationException {
1023     this.master = master;
1024 
1025     this.rootDir = master.getMasterFileSystem().getRootDir();
1026     checkSnapshotSupport(master.getConfiguration(), master.getMasterFileSystem());
1027 
1028     // get the configuration for the coordinator
1029     Configuration conf = master.getConfiguration();
1030     long wakeFrequency = conf.getInt(SNAPSHOT_WAKE_MILLIS_KEY, SNAPSHOT_WAKE_MILLIS_DEFAULT);
1031     long timeoutMillis = conf.getLong(SNAPSHOT_TIMEOUT_MILLIS_KEY, SNAPSHOT_TIMEOUT_MILLIS_DEFAULT);
1032     int opThreads = conf.getInt(SNAPSHOT_POOL_THREADS_KEY, SNAPSHOT_POOL_THREADS_DEFAULT);
1033 
1034     // setup the default procedure coordinator
1035     String name = master.getServerName().toString();
1036     ThreadPoolExecutor tpool = ProcedureCoordinator.defaultPool(name, opThreads);
1037     ProcedureCoordinatorRpcs comms = new ZKProcedureCoordinatorRpcs(
1038         master.getZooKeeper(), SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION, name);
1039 
1040     this.coordinator = new ProcedureCoordinator(comms, tpool, timeoutMillis, wakeFrequency);
1041     this.executorService = master.getExecutorService();
1042     resetTempDir();
1043   }
1044 
1045   @Override
1046   public String getProcedureSignature() {
1047     return ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION;
1048   }
1049 
1050   @Override
1051   public void execProcedure(ProcedureDescription desc) throws IOException {
1052     takeSnapshot(toSnapshotDescription(desc));
1053   }
1054 
1055   @Override
1056   public boolean isProcedureDone(ProcedureDescription desc) throws IOException {
1057     return isSnapshotDone(toSnapshotDescription(desc));
1058   }
1059 
1060   private SnapshotDescription toSnapshotDescription(ProcedureDescription desc)
1061       throws IOException {
1062     SnapshotDescription.Builder builder = SnapshotDescription.newBuilder();
1063     if (!desc.hasInstance()) {
1064       throw new IOException("Snapshot name is not defined: " + desc.toString());
1065     }
1066     String snapshotName = desc.getInstance();
1067     List<NameStringPair> props = desc.getConfigurationList();
1068     String table = null;
1069     for (NameStringPair prop : props) {
1070       if ("table".equalsIgnoreCase(prop.getName())) {
1071         table = prop.getValue();
1072       }
1073     }
1074     if (table == null) {
1075       throw new IOException("Snapshot table is not defined: " + desc.toString());
1076     }
1077     TableName tableName = TableName.valueOf(table);
1078     builder.setTable(tableName.getNameAsString());
1079     builder.setName(snapshotName);
1080     builder.setType(SnapshotDescription.Type.FLUSH);
1081     return builder.build();
1082   }
1083 }