View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.snapshot;
20  
21  import java.io.IOException;
22  import java.io.InputStream;
23  import java.io.OutputStream;
24  import java.util.Arrays;
25  import java.util.HashMap;
26  import java.util.HashSet;
27  import java.util.LinkedList;
28  import java.util.List;
29  import java.util.Map;
30  import java.util.Set;
31  import java.util.TreeMap;
32  
33  import org.apache.commons.logging.Log;
34  import org.apache.commons.logging.LogFactory;
35  import org.apache.hadoop.classification.InterfaceAudience;
36  import org.apache.hadoop.conf.Configuration;
37  import org.apache.hadoop.fs.FileStatus;
38  import org.apache.hadoop.fs.FileSystem;
39  import org.apache.hadoop.fs.Path;
40  import org.apache.hadoop.hbase.HColumnDescriptor;
41  import org.apache.hadoop.hbase.HRegionInfo;
42  import org.apache.hadoop.hbase.HTableDescriptor;
43  import org.apache.hadoop.hbase.TableName;
44  import org.apache.hadoop.hbase.backup.HFileArchiver;
45  import org.apache.hadoop.hbase.catalog.CatalogTracker;
46  import org.apache.hadoop.hbase.catalog.MetaEditor;
47  import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
48  import org.apache.hadoop.hbase.io.HFileLink;
49  import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
50  import org.apache.hadoop.hbase.monitoring.MonitoredTask;
51  import org.apache.hadoop.hbase.monitoring.TaskMonitor;
52  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
53  import org.apache.hadoop.hbase.regionserver.HRegion;
54  import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
55  import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
56  import org.apache.hadoop.hbase.util.Bytes;
57  import org.apache.hadoop.hbase.util.FSTableDescriptors;
58  import org.apache.hadoop.hbase.util.FSUtils;
59  import org.apache.hadoop.hbase.util.FSVisitor;
60  import org.apache.hadoop.hbase.util.ModifyRegionUtils;
61  import org.apache.hadoop.hbase.util.Pair;
62  import org.apache.hadoop.io.IOUtils;
63  
64  /**
65   * Helper to Restore/Clone a Snapshot
66   *
67   * <p>The helper assumes that a table is already created, and by calling restore()
68   * the content present in the snapshot will be restored as the new content of the table.
69   *
70   * <p>Clone from Snapshot: If the target table is empty, the restore operation
71   * is just a "clone operation", where the only operations are:
72   * <ul>
73   *  <li>for each region in the snapshot create a new region
74   *    (note that the region will have a different name, since the encoding contains the table name)
75   *  <li>for each file in the region create a new HFileLink to point to the original file.
76   *  <li>restore the logs, if any
77   * </ul>
78   *
79   * <p>Restore from Snapshot:
80   * <ul>
81   *  <li>for each region in the table verify which are available in the snapshot and which are not
82   *    <ul>
83   *    <li>if the region is not present in the snapshot, remove it.
84   *    <li>if the region is present in the snapshot
85   *      <ul>
86   *      <li>for each file in the table region verify which are available in the snapshot
87   *        <ul>
88   *          <li>if the hfile is not present in the snapshot, remove it
89   *          <li>if the hfile is present, keep it (nothing to do)
90   *        </ul>
91   *      <li>for each file in the snapshot region but not in the table
92   *        <ul>
93   *          <li>create a new HFileLink that point to the original file
94   *        </ul>
95   *      </ul>
96   *    </ul>
97   *  <li>for each region in the snapshot not present in the current table state
98   *    <ul>
99   *    <li>create a new region and for each file in the region create a new HFileLink
100  *      (This is the same as the clone operation)
101  *    </ul>
102  *  <li>restore the logs, if any
103  * </ul>
104  */
105 @InterfaceAudience.Private
106 public class RestoreSnapshotHelper {
107   private static final Log LOG = LogFactory.getLog(RestoreSnapshotHelper.class);
108 
109   private final Map<byte[], byte[]> regionsMap =
110         new TreeMap<byte[], byte[]>(Bytes.BYTES_COMPARATOR);
111 
112   private final Map<String, Pair<String, String> > parentsMap =
113       new HashMap<String, Pair<String, String> >();
114 
115   private final ForeignExceptionDispatcher monitor;
116   private final MonitoredTask status;
117 
118   private final SnapshotDescription snapshotDesc;
119   private final TableName snapshotTable;
120   private final Path snapshotDir;
121 
122   private final HTableDescriptor tableDesc;
123   private final Path rootDir;
124   private final Path tableDir;
125 
126   private final Configuration conf;
127   private final FileSystem fs;
128 
129   public RestoreSnapshotHelper(final Configuration conf,
130       final FileSystem fs,
131       final SnapshotDescription snapshotDescription,
132       final Path snapshotDir,
133       final HTableDescriptor tableDescriptor,
134       final Path rootDir,
135       final ForeignExceptionDispatcher monitor,
136       final MonitoredTask status)
137   {
138     this.fs = fs;
139     this.conf = conf;
140     this.snapshotDesc = snapshotDescription;
141     this.snapshotTable = TableName.valueOf(snapshotDescription.getTable());
142     this.snapshotDir = snapshotDir;
143     this.tableDesc = tableDescriptor;
144     this.rootDir = rootDir;
145     this.tableDir = FSUtils.getTableDir(rootDir, tableDesc.getTableName());
146     this.monitor = monitor;
147     this.status = status;
148   }
149 
150   /**
151    * Restore the on-disk table to a specified snapshot state.
152    * @return the set of regions touched by the restore operation
153    */
154   public RestoreMetaChanges restoreHdfsRegions() throws IOException {
155     LOG.debug("starting restore");
156     Set<String> snapshotRegionNames = SnapshotReferenceUtil.getSnapshotRegionNames(fs, snapshotDir);
157     if (snapshotRegionNames == null) {
158       LOG.warn("Nothing to restore. Snapshot " + snapshotDesc + " looks empty");
159       return null;
160     }
161 
162     RestoreMetaChanges metaChanges = new RestoreMetaChanges(parentsMap);
163 
164     // Identify which region are still available and which not.
165     // NOTE: we rely upon the region name as: "table name, start key, end key"
166     List<HRegionInfo> tableRegions = getTableRegions();
167     if (tableRegions != null) {
168       monitor.rethrowException();
169       for (HRegionInfo regionInfo: tableRegions) {
170         String regionName = regionInfo.getEncodedName();
171         if (snapshotRegionNames.contains(regionName)) {
172           LOG.info("region to restore: " + regionName);
173           snapshotRegionNames.remove(regionName);
174           metaChanges.addRegionToRestore(regionInfo);
175         } else {
176           LOG.info("region to remove: " + regionName);
177           metaChanges.addRegionToRemove(regionInfo);
178         }
179       }
180 
181       // Restore regions using the snapshot data
182       monitor.rethrowException();
183       status.setStatus("Restoring table regions...");
184       restoreHdfsRegions(metaChanges.getRegionsToRestore());
185       status.setStatus("Finished restoring all table regions.");
186 
187       // Remove regions from the current table
188       monitor.rethrowException();
189       status.setStatus("Starting to delete excess regions from table");
190       removeHdfsRegions(metaChanges.getRegionsToRemove());
191       status.setStatus("Finished deleting excess regions from table.");
192     }
193 
194     // Regions to Add: present in the snapshot but not in the current table
195     if (snapshotRegionNames.size() > 0) {
196       List<HRegionInfo> regionsToAdd = new LinkedList<HRegionInfo>();
197 
198       monitor.rethrowException();
199       for (String regionName: snapshotRegionNames) {
200         LOG.info("region to add: " + regionName);
201         Path regionDir = new Path(snapshotDir, regionName);
202         regionsToAdd.add(HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir));
203       }
204 
205       // Create new regions cloning from the snapshot
206       monitor.rethrowException();
207       status.setStatus("Cloning regions...");
208       HRegionInfo[] clonedRegions = cloneHdfsRegions(regionsToAdd);
209       metaChanges.setNewRegions(clonedRegions);
210       status.setStatus("Finished cloning regions.");
211     }
212 
213     // Restore WALs
214     monitor.rethrowException();
215     status.setStatus("Restoring WALs to table...");
216     restoreWALs();
217     status.setStatus("Finished restoring WALs to table.");
218 
219     return metaChanges;
220   }
221 
222   /**
223    * Describe the set of operations needed to update hbase:meta after restore.
224    */
225   public static class RestoreMetaChanges {
226     private final Map<String, Pair<String, String> > parentsMap;
227 
228     private List<HRegionInfo> regionsToRestore = null;
229     private List<HRegionInfo> regionsToRemove = null;
230     private List<HRegionInfo> regionsToAdd = null;
231 
232     RestoreMetaChanges(final Map<String, Pair<String, String> > parentsMap) {
233       this.parentsMap = parentsMap;
234     }
235 
236     /**
237      * @return true if there're new regions
238      */
239     public boolean hasRegionsToAdd() {
240       return this.regionsToAdd != null && this.regionsToAdd.size() > 0;
241     }
242 
243     /**
244      * Returns the list of new regions added during the on-disk restore.
245      * The caller is responsible to add the regions to META.
246      * e.g MetaEditor.addRegionsToMeta(...)
247      * @return the list of regions to add to META
248      */
249     public List<HRegionInfo> getRegionsToAdd() {
250       return this.regionsToAdd;
251     }
252 
253     /**
254      * @return true if there're regions to restore
255      */
256     public boolean hasRegionsToRestore() {
257       return this.regionsToRestore != null && this.regionsToRestore.size() > 0;
258     }
259 
260     /**
261      * Returns the list of 'restored regions' during the on-disk restore.
262      * The caller is responsible to add the regions to hbase:meta if not present.
263      * @return the list of regions restored
264      */
265     public List<HRegionInfo> getRegionsToRestore() {
266       return this.regionsToRestore;
267     }
268 
269     /**
270      * @return true if there're regions to remove
271      */
272     public boolean hasRegionsToRemove() {
273       return this.regionsToRemove != null && this.regionsToRemove.size() > 0;
274     }
275 
276     /**
277      * Returns the list of regions removed during the on-disk restore.
278      * The caller is responsible to remove the regions from META.
279      * e.g. MetaEditor.deleteRegions(...)
280      * @return the list of regions to remove from META
281      */
282     public List<HRegionInfo> getRegionsToRemove() {
283       return this.regionsToRemove;
284     }
285 
286     void setNewRegions(final HRegionInfo[] hris) {
287       if (hris != null) {
288         regionsToAdd = Arrays.asList(hris);
289       } else {
290         regionsToAdd = null;
291       }
292     }
293 
294     void addRegionToRemove(final HRegionInfo hri) {
295       if (regionsToRemove == null) {
296         regionsToRemove = new LinkedList<HRegionInfo>();
297       }
298       regionsToRemove.add(hri);
299     }
300 
301     void addRegionToRestore(final HRegionInfo hri) {
302       if (regionsToRestore == null) {
303         regionsToRestore = new LinkedList<HRegionInfo>();
304       }
305       regionsToRestore.add(hri);
306     }
307 
308     public void updateMetaParentRegions(final CatalogTracker catalogTracker,
309         final List<HRegionInfo> regionInfos) throws IOException {
310       if (regionInfos == null || parentsMap.isEmpty()) return;
311 
312       // Extract region names and offlined regions
313       Map<String, HRegionInfo> regionsByName = new HashMap<String, HRegionInfo>(regionInfos.size());
314       List<HRegionInfo> parentRegions = new LinkedList();
315       for (HRegionInfo regionInfo: regionInfos) {
316         if (regionInfo.isSplitParent()) {
317           parentRegions.add(regionInfo);
318         } else {
319           regionsByName.put(regionInfo.getEncodedName(), regionInfo);
320         }
321       }
322 
323       // Update Offline parents
324       for (HRegionInfo regionInfo: parentRegions) {
325         Pair<String, String> daughters = parentsMap.get(regionInfo.getEncodedName());
326         if (daughters == null) {
327           // The snapshot contains an unreferenced region.
328           // It will be removed by the CatalogJanitor.
329           LOG.warn("Skip update of unreferenced offline parent: " + regionInfo);
330           continue;
331         }
332 
333         // One side of the split is already compacted
334         if (daughters.getSecond() == null) {
335           daughters.setSecond(daughters.getFirst());
336         }
337 
338         LOG.debug("Update splits parent " + regionInfo.getEncodedName() + " -> " + daughters);
339         MetaEditor.addRegionToMeta(catalogTracker, regionInfo,
340             regionsByName.get(daughters.getFirst()),
341             regionsByName.get(daughters.getSecond()));
342       }
343     }
344   }
345 
346   /**
347    * Remove specified regions from the file-system, using the archiver.
348    */
349   private void removeHdfsRegions(final List<HRegionInfo> regions) throws IOException {
350     if (regions != null && regions.size() > 0) {
351       for (HRegionInfo hri: regions) {
352         HFileArchiver.archiveRegion(conf, fs, hri);
353       }
354     }
355   }
356 
357   /**
358    * Restore specified regions by restoring content to the snapshot state.
359    */
360   private void restoreHdfsRegions(final List<HRegionInfo> regions) throws IOException {
361     if (regions == null || regions.size() == 0) return;
362     for (HRegionInfo hri: regions) restoreRegion(hri);
363   }
364 
365   /**
366    * Restore region by removing files not in the snapshot
367    * and adding the missing ones from the snapshot.
368    */
369   private void restoreRegion(HRegionInfo regionInfo) throws IOException {
370     Path snapshotRegionDir = new Path(snapshotDir, regionInfo.getEncodedName());
371     Map<String, List<String>> snapshotFiles =
372                 SnapshotReferenceUtil.getRegionHFileReferences(fs, snapshotRegionDir);
373     Path regionDir = new Path(tableDir, regionInfo.getEncodedName());
374     String tableName = tableDesc.getTableName().getNameAsString();
375 
376     // Restore families present in the table
377     for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) {
378       byte[] family = Bytes.toBytes(familyDir.getName());
379       Set<String> familyFiles = getTableRegionFamilyFiles(familyDir);
380       List<String> snapshotFamilyFiles = snapshotFiles.remove(familyDir.getName());
381       if (snapshotFamilyFiles != null) {
382         List<String> hfilesToAdd = new LinkedList<String>();
383         for (String hfileName: snapshotFamilyFiles) {
384           if (familyFiles.contains(hfileName)) {
385             // HFile already present
386             familyFiles.remove(hfileName);
387           } else {
388             // HFile missing
389             hfilesToAdd.add(hfileName);
390           }
391         }
392 
393         // Remove hfiles not present in the snapshot
394         for (String hfileName: familyFiles) {
395           Path hfile = new Path(familyDir, hfileName);
396           LOG.trace("Removing hfile=" + hfile +
397             " from region=" + regionInfo.getEncodedName() + " table=" + tableName);
398           HFileArchiver.archiveStoreFile(conf, fs, regionInfo, tableDir, family, hfile);
399         }
400 
401         // Restore Missing files
402         for (String hfileName: hfilesToAdd) {
403           LOG.trace("Adding HFileLink " + hfileName +
404             " to region=" + regionInfo.getEncodedName() + " table=" + tableName);
405           restoreStoreFile(familyDir, regionInfo, hfileName);
406         }
407       } else {
408         // Family doesn't exists in the snapshot
409         LOG.trace("Removing family=" + Bytes.toString(family) +
410           " from region=" + regionInfo.getEncodedName() + " table=" + tableName);
411         HFileArchiver.archiveFamily(fs, conf, regionInfo, tableDir, family);
412         fs.delete(familyDir, true);
413       }
414     }
415 
416     // Add families not present in the table
417     for (Map.Entry<String, List<String>> familyEntry: snapshotFiles.entrySet()) {
418       Path familyDir = new Path(regionDir, familyEntry.getKey());
419       if (!fs.mkdirs(familyDir)) {
420         throw new IOException("Unable to create familyDir=" + familyDir);
421       }
422 
423       for (String hfileName: familyEntry.getValue()) {
424         LOG.trace("Adding HFileLink " + hfileName + " to table=" + tableName);
425         restoreStoreFile(familyDir, regionInfo, hfileName);
426       }
427     }
428   }
429 
430   /**
431    * @return The set of files in the specified family directory.
432    */
433   private Set<String> getTableRegionFamilyFiles(final Path familyDir) throws IOException {
434     Set<String> familyFiles = new HashSet<String>();
435 
436     FileStatus[] hfiles = FSUtils.listStatus(fs, familyDir);
437     if (hfiles == null) return familyFiles;
438 
439     for (FileStatus hfileRef: hfiles) {
440       String hfileName = hfileRef.getPath().getName();
441       familyFiles.add(hfileName);
442     }
443 
444     return familyFiles;
445   }
446 
447   /**
448    * Clone specified regions. For each region create a new region
449    * and create a HFileLink for each hfile.
450    */
451   private HRegionInfo[] cloneHdfsRegions(final List<HRegionInfo> regions) throws IOException {
452     if (regions == null || regions.size() == 0) return null;
453 
454     final Map<String, HRegionInfo> snapshotRegions =
455       new HashMap<String, HRegionInfo>(regions.size());
456 
457     // clone region info (change embedded tableName with the new one)
458     HRegionInfo[] clonedRegionsInfo = new HRegionInfo[regions.size()];
459     for (int i = 0; i < clonedRegionsInfo.length; ++i) {
460       // clone the region info from the snapshot region info
461       HRegionInfo snapshotRegionInfo = regions.get(i);
462       clonedRegionsInfo[i] = cloneRegionInfo(snapshotRegionInfo);
463 
464       // add the region name mapping between snapshot and cloned
465       String snapshotRegionName = snapshotRegionInfo.getEncodedName();
466       String clonedRegionName = clonedRegionsInfo[i].getEncodedName();
467       regionsMap.put(Bytes.toBytes(snapshotRegionName), Bytes.toBytes(clonedRegionName));
468       LOG.info("clone region=" + snapshotRegionName + " as " + clonedRegionName);
469 
470       // Add mapping between cloned region name and snapshot region info
471       snapshotRegions.put(clonedRegionName, snapshotRegionInfo);
472     }
473 
474     // create the regions on disk
475     ModifyRegionUtils.createRegions(conf, rootDir, tableDir,
476       tableDesc, clonedRegionsInfo, new ModifyRegionUtils.RegionFillTask() {
477         @Override
478         public void fillRegion(final HRegion region) throws IOException {
479           cloneRegion(region, snapshotRegions.get(region.getRegionInfo().getEncodedName()));
480         }
481       });
482 
483     return clonedRegionsInfo;
484   }
485 
486   /**
487    * Clone region directory content from the snapshot info.
488    *
489    * Each region is encoded with the table name, so the cloned region will have
490    * a different region name.
491    *
492    * Instead of copying the hfiles a HFileLink is created.
493    *
494    * @param region {@link HRegion} cloned
495    * @param snapshotRegionInfo
496    */
497   private void cloneRegion(final HRegion region, final HRegionInfo snapshotRegionInfo)
498       throws IOException {
499     final Path snapshotRegionDir = new Path(snapshotDir, snapshotRegionInfo.getEncodedName());
500     final Path regionDir = new Path(tableDir, region.getRegionInfo().getEncodedName());
501     final String tableName = tableDesc.getTableName().getNameAsString();
502     SnapshotReferenceUtil.visitRegionStoreFiles(fs, snapshotRegionDir,
503       new FSVisitor.StoreFileVisitor() {
504         @Override
505         public void storeFile (final String region, final String family, final String hfile)
506             throws IOException {
507           LOG.info("Adding HFileLink " + hfile + " to table=" + tableName);
508           Path familyDir = new Path(regionDir, family);
509           restoreStoreFile(familyDir, snapshotRegionInfo, hfile);
510         }
511     });
512   }
513 
514   /**
515    * Create a new {@link HFileLink} to reference the store file.
516    * <p>The store file in the snapshot can be a simple hfile, an HFileLink or a reference.
517    * <ul>
518    *   <li>hfile: abc -> table=region-abc
519    *   <li>reference: abc.1234 -> table=region-abc.1234
520    *   <li>hfilelink: table=region-hfile -> table=region-hfile
521    * </ul>
522    * @param familyDir destination directory for the store file
523    * @param regionInfo destination region info for the table
524    * @param hfileName store file name (can be a Reference, HFileLink or simple HFile)
525    */
526   private void restoreStoreFile(final Path familyDir, final HRegionInfo regionInfo,
527       final String hfileName) throws IOException {
528     if (HFileLink.isHFileLink(hfileName)) {
529       HFileLink.createFromHFileLink(conf, fs, familyDir, hfileName);
530     } else if (StoreFileInfo.isReference(hfileName)) {
531       restoreReferenceFile(familyDir, regionInfo, hfileName);
532     } else {
533       HFileLink.create(conf, fs, familyDir, regionInfo, hfileName);
534     }
535   }
536 
537   /**
538    * Create a new {@link Reference} as copy of the source one.
539    * <p><blockquote><pre>
540    * The source table looks like:
541    *    1234/abc      (original file)
542    *    5678/abc.1234 (reference file)
543    *
544    * After the clone operation looks like:
545    *   wxyz/table=1234-abc
546    *   stuv/table=1234-abc.wxyz
547    *
548    * NOTE that the region name in the clone changes (md5 of regioninfo)
549    * and the reference should reflect that change.
550    * </pre></blockquote>
551    * @param familyDir destination directory for the store file
552    * @param regionInfo destination region info for the table
553    * @param hfileName reference file name
554    */
555   private void restoreReferenceFile(final Path familyDir, final HRegionInfo regionInfo,
556       final String hfileName) throws IOException {
557     // Extract the referred information (hfile name and parent region)
558     Path refPath = StoreFileInfo.getReferredToFile(new Path(new Path(new Path(
559         snapshotTable.getNameAsString(), regionInfo.getEncodedName()), familyDir.getName()),
560         hfileName));
561     String snapshotRegionName = refPath.getParent().getParent().getName();
562     String fileName = refPath.getName();
563 
564     // The new reference should have the cloned region name as parent, if it is a clone.
565     String clonedRegionName = Bytes.toString(regionsMap.get(Bytes.toBytes(snapshotRegionName)));
566     if (clonedRegionName == null) clonedRegionName = snapshotRegionName;
567 
568     // The output file should be a reference link table=snapshotRegion-fileName.clonedRegionName
569     Path linkPath = null;
570     String refLink = fileName;
571     if (!HFileLink.isHFileLink(fileName)) {
572       refLink = HFileLink.createHFileLinkName(snapshotTable, snapshotRegionName, fileName);
573       linkPath = new Path(familyDir,
574         HFileLink.createHFileLinkName(snapshotTable, regionInfo.getEncodedName(), hfileName));
575     }
576 
577     Path outPath = new Path(familyDir, refLink + '.' + clonedRegionName);
578 
579     // Create the new reference
580     InputStream in;
581     if (linkPath != null) {
582       in = new HFileLink(conf, linkPath).open(fs);
583     } else {
584       linkPath = new Path(new Path(HRegion.getRegionDir(snapshotDir, regionInfo.getEncodedName()),
585                       familyDir.getName()), hfileName);
586       in = fs.open(linkPath);
587     }
588     OutputStream out = fs.create(outPath);
589     IOUtils.copyBytes(in, out, conf);
590 
591     // Add the daughter region to the map
592     String regionName = Bytes.toString(regionsMap.get(regionInfo.getEncodedNameAsBytes()));
593     LOG.debug("Restore reference " + regionName + " to " + clonedRegionName);
594     synchronized (parentsMap) {
595       Pair<String, String> daughters = parentsMap.get(clonedRegionName);
596       if (daughters == null) {
597         daughters = new Pair<String, String>(regionName, null);
598         parentsMap.put(clonedRegionName, daughters);
599       } else if (!regionName.equals(daughters.getFirst())) {
600         daughters.setSecond(regionName);
601       }
602     }
603   }
604 
605   /**
606    * Create a new {@link HRegionInfo} from the snapshot region info.
607    * Keep the same startKey, endKey, regionId and split information but change
608    * the table name.
609    *
610    * @param snapshotRegionInfo Info for region to clone.
611    * @return the new HRegion instance
612    */
613   public HRegionInfo cloneRegionInfo(final HRegionInfo snapshotRegionInfo) {
614     HRegionInfo regionInfo = new HRegionInfo(tableDesc.getTableName(),
615                       snapshotRegionInfo.getStartKey(), snapshotRegionInfo.getEndKey(),
616                       snapshotRegionInfo.isSplit(), snapshotRegionInfo.getRegionId());
617     regionInfo.setOffline(snapshotRegionInfo.isOffline());
618     return regionInfo;
619   }
620 
621   /**
622    * Restore snapshot WALs.
623    *
624    * Global Snapshot keep a reference to region servers logs present during the snapshot.
625    * (/hbase/.snapshot/snapshotName/.logs/hostName/logName)
626    *
627    * Since each log contains different tables data, logs must be split to
628    * extract the table that we are interested in.
629    */
630   private void restoreWALs() throws IOException {
631     final SnapshotLogSplitter logSplitter = new SnapshotLogSplitter(conf, fs, tableDir,
632         snapshotTable, regionsMap);
633     // TODO: use executors to parallelize splitting
634     // TODO: once split, we do not need to split again for other restores
635     try {
636       // Recover.Edits
637       SnapshotReferenceUtil.visitRecoveredEdits(fs, snapshotDir,
638           new FSVisitor.RecoveredEditsVisitor() {
639         @Override
640         public void recoveredEdits (final String region, final String logfile) throws IOException {
641           Path path = SnapshotReferenceUtil.getRecoveredEdits(snapshotDir, region, logfile);
642           logSplitter.splitRecoveredEdit(path);
643         }
644       });
645 
646       // Region Server Logs
647       SnapshotReferenceUtil.visitLogFiles(fs, snapshotDir, new FSVisitor.LogFileVisitor() {
648         @Override
649         public void logFile (final String server, final String logfile) throws IOException {
650           logSplitter.splitLog(server, logfile);
651         }
652       });
653     } finally {
654       logSplitter.close();
655     }
656   }
657 
658   /**
659    * @return the set of the regions contained in the table
660    */
661   private List<HRegionInfo> getTableRegions() throws IOException {
662     LOG.debug("get table regions: " + tableDir);
663     FileStatus[] regionDirs = FSUtils.listStatus(fs, tableDir, new FSUtils.RegionDirFilter(fs));
664     if (regionDirs == null) return null;
665 
666     List<HRegionInfo> regions = new LinkedList<HRegionInfo>();
667     for (FileStatus regionDir: regionDirs) {
668       HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir.getPath());
669       regions.add(hri);
670     }
671     LOG.debug("found " + regions.size() + " regions for table=" +
672         tableDesc.getTableName().getNameAsString());
673     return regions;
674   }
675 
676   /**
677    * Create a new table descriptor cloning the snapshot table schema.
678    *
679    * @param snapshotTableDescriptor
680    * @param tableName
681    * @return cloned table descriptor
682    * @throws IOException
683    */
684   public static HTableDescriptor cloneTableSchema(final HTableDescriptor snapshotTableDescriptor,
685       final TableName tableName) throws IOException {
686     HTableDescriptor htd = new HTableDescriptor(tableName);
687     for (HColumnDescriptor hcd: snapshotTableDescriptor.getColumnFamilies()) {
688       htd.addFamily(hcd);
689     }
690     for (Map.Entry<ImmutableBytesWritable, ImmutableBytesWritable> e:
691         snapshotTableDescriptor.getValues().entrySet()) {
692       htd.setValue(e.getKey(), e.getValue());
693     }
694     for (Map.Entry<String, String> e: snapshotTableDescriptor.getConfiguration().entrySet()) {
695       htd.setConfiguration(e.getKey(), e.getValue());
696     }
697     return htd;
698   }
699 
700   /**
701    * Copy the snapshot files for a snapshot scanner, discards meta changes.
702    * @param conf
703    * @param fs
704    * @param rootDir
705    * @param restoreDir
706    * @param snapshotName
707    * @throws IOException
708    */
709   public static void copySnapshotForScanner(Configuration conf, FileSystem fs, Path rootDir,
710       Path restoreDir, String snapshotName) throws IOException {
711     // ensure that restore dir is not under root dir
712     if (!restoreDir.getFileSystem(conf).getUri().equals(rootDir.getFileSystem(conf).getUri())) {
713       throw new IllegalArgumentException("Filesystems for restore directory and HBase root directory " +
714           "should be the same");
715     }
716     if (restoreDir.toUri().getPath().startsWith(rootDir.toUri().getPath())) {
717       throw new IllegalArgumentException("Restore directory cannot be a sub directory of HBase " +
718           "root directory. RootDir: " + rootDir + ", restoreDir: " + restoreDir);
719     }
720 
721     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
722     SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
723 
724     //load table descriptor
725     HTableDescriptor htd = FSTableDescriptors.getTableDescriptorFromFs(fs, snapshotDir);
726 
727     MonitoredTask status = TaskMonitor.get().createStatus(
728         "Restoring  snapshot '" + snapshotName + "' to directory " + restoreDir);
729     ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher();
730 
731     RestoreSnapshotHelper helper = new RestoreSnapshotHelper(conf, fs, snapshotDesc,
732         snapshotDir, htd, restoreDir, monitor, status);
733     helper.restoreHdfsRegions(); // TODO: parallelize.
734 
735     if (LOG.isDebugEnabled()) {
736       LOG.debug("Restored table dir:" + restoreDir);
737       FSUtils.logFileSystemState(fs, restoreDir, LOG);
738     }
739   }
740 }