001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.snapshot;
019
020import java.io.IOException;
021import java.io.InputStream;
022import java.io.OutputStream;
023import java.util.ArrayList;
024import java.util.Arrays;
025import java.util.Collections;
026import java.util.HashMap;
027import java.util.HashSet;
028import java.util.LinkedList;
029import java.util.List;
030import java.util.Map;
031import java.util.Map.Entry;
032import java.util.Set;
033import java.util.TreeMap;
034import java.util.concurrent.ThreadPoolExecutor;
035import org.apache.hadoop.conf.Configuration;
036import org.apache.hadoop.fs.FileStatus;
037import org.apache.hadoop.fs.FileSystem;
038import org.apache.hadoop.fs.Path;
039import org.apache.hadoop.hbase.MetaTableAccessor;
040import org.apache.hadoop.hbase.TableName;
041import org.apache.hadoop.hbase.backup.HFileArchiver;
042import org.apache.hadoop.hbase.client.Connection;
043import org.apache.hadoop.hbase.client.ConnectionFactory;
044import org.apache.hadoop.hbase.client.RegionInfo;
045import org.apache.hadoop.hbase.client.RegionInfoBuilder;
046import org.apache.hadoop.hbase.client.TableDescriptor;
047import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
048import org.apache.hadoop.hbase.io.HFileLink;
049import org.apache.hadoop.hbase.io.Reference;
050import org.apache.hadoop.hbase.mob.MobUtils;
051import org.apache.hadoop.hbase.monitoring.MonitoredTask;
052import org.apache.hadoop.hbase.monitoring.TaskMonitor;
053import org.apache.hadoop.hbase.regionserver.HRegion;
054import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
055import org.apache.hadoop.hbase.regionserver.StoreContext;
056import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
057import org.apache.hadoop.hbase.regionserver.StoreUtils;
058import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker;
059import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
060import org.apache.hadoop.hbase.security.access.AccessControlClient;
061import org.apache.hadoop.hbase.security.access.Permission;
062import org.apache.hadoop.hbase.security.access.ShadedAccessControlUtil;
063import org.apache.hadoop.hbase.security.access.TablePermission;
064import org.apache.hadoop.hbase.util.Bytes;
065import org.apache.hadoop.hbase.util.CommonFSUtils;
066import org.apache.hadoop.hbase.util.FSUtils;
067import org.apache.hadoop.hbase.util.ModifyRegionUtils;
068import org.apache.hadoop.hbase.util.Pair;
069import org.apache.hadoop.io.IOUtils;
070import org.apache.yetus.audience.InterfaceAudience;
071import org.slf4j.Logger;
072import org.slf4j.LoggerFactory;
073
074import org.apache.hbase.thirdparty.com.google.common.collect.ListMultimap;
075
076import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
077import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription;
078import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
079
080/**
081 * Helper to Restore/Clone a Snapshot
082 * <p>
083 * The helper assumes that a table is already created, and by calling restore() the content present
084 * in the snapshot will be restored as the new content of the table.
085 * <p>
086 * Clone from Snapshot: If the target table is empty, the restore operation is just a "clone
087 * operation", where the only operations are:
088 * <ul>
089 * <li>for each region in the snapshot create a new region (note that the region will have a
090 * different name, since the encoding contains the table name)
091 * <li>for each file in the region create a new HFileLink to point to the original file.
092 * <li>restore the logs, if any
093 * </ul>
094 * <p>
095 * Restore from Snapshot:
096 * <ul>
097 * <li>for each region in the table verify which are available in the snapshot and which are not
098 * <ul>
099 * <li>if the region is not present in the snapshot, remove it.
100 * <li>if the region is present in the snapshot
101 * <ul>
102 * <li>for each file in the table region verify which are available in the snapshot
103 * <ul>
104 * <li>if the hfile is not present in the snapshot, remove it
105 * <li>if the hfile is present, keep it (nothing to do)
106 * </ul>
107 * <li>for each file in the snapshot region but not in the table
108 * <ul>
109 * <li>create a new HFileLink that point to the original file
110 * </ul>
111 * </ul>
112 * </ul>
113 * <li>for each region in the snapshot not present in the current table state
114 * <ul>
115 * <li>create a new region and for each file in the region create a new HFileLink (This is the same
116 * as the clone operation)
117 * </ul>
118 * <li>restore the logs, if any
119 * </ul>
120 */
121@InterfaceAudience.Private
122public class RestoreSnapshotHelper {
123  private static final Logger LOG = LoggerFactory.getLogger(RestoreSnapshotHelper.class);
124
125  private final Map<byte[], byte[]> regionsMap = new TreeMap<>(Bytes.BYTES_COMPARATOR);
126
127  private final Map<String, Pair<String, String>> parentsMap = new HashMap<>();
128
129  private final ForeignExceptionDispatcher monitor;
130  private final MonitoredTask status;
131
132  private final SnapshotManifest snapshotManifest;
133  private final SnapshotDescription snapshotDesc;
134  private final TableName snapshotTable;
135
136  private final TableDescriptor tableDesc;
137  private final Path rootDir;
138  private final Path tableDir;
139
140  private final Configuration conf;
141  private final FileSystem fs;
142  private final boolean createBackRefs;
143
144  public RestoreSnapshotHelper(final Configuration conf, final FileSystem fs,
145    final SnapshotManifest manifest, final TableDescriptor tableDescriptor, final Path rootDir,
146    final ForeignExceptionDispatcher monitor, final MonitoredTask status) {
147    this(conf, fs, manifest, tableDescriptor, rootDir, monitor, status, true);
148  }
149
150  public RestoreSnapshotHelper(final Configuration conf, final FileSystem fs,
151    final SnapshotManifest manifest, final TableDescriptor tableDescriptor, final Path rootDir,
152    final ForeignExceptionDispatcher monitor, final MonitoredTask status,
153    final boolean createBackRefs) {
154    this.fs = fs;
155    this.conf = conf;
156    this.snapshotManifest = manifest;
157    this.snapshotDesc = manifest.getSnapshotDescription();
158    this.snapshotTable = TableName.valueOf(snapshotDesc.getTable());
159    this.tableDesc = tableDescriptor;
160    this.rootDir = rootDir;
161    this.tableDir = CommonFSUtils.getTableDir(rootDir, tableDesc.getTableName());
162    this.monitor = monitor;
163    this.status = status;
164    this.createBackRefs = createBackRefs;
165  }
166
167  /**
168   * Restore the on-disk table to a specified snapshot state.
169   * @return the set of regions touched by the restore operation
170   */
171  public RestoreMetaChanges restoreHdfsRegions() throws IOException {
172    ThreadPoolExecutor exec = SnapshotManifest.createExecutor(conf, "RestoreSnapshot");
173    try {
174      return restoreHdfsRegions(exec);
175    } finally {
176      exec.shutdown();
177    }
178  }
179
180  private RestoreMetaChanges restoreHdfsRegions(final ThreadPoolExecutor exec) throws IOException {
181    LOG.info("starting restore table regions using snapshot=" + snapshotDesc);
182
183    Map<String, SnapshotRegionManifest> regionManifests = snapshotManifest.getRegionManifestsMap();
184    if (regionManifests == null) {
185      LOG.warn("Nothing to restore. Snapshot " + snapshotDesc + " looks empty");
186      return null;
187    }
188
189    RestoreMetaChanges metaChanges = new RestoreMetaChanges(tableDesc, parentsMap);
190
191    // Take a copy of the manifest.keySet() since we are going to modify
192    // this instance, by removing the regions already present in the restore dir.
193    Set<String> regionNames = new HashSet<>(regionManifests.keySet());
194
195    List<RegionInfo> tableRegions = getTableRegions();
196
197    RegionInfo mobRegion =
198      MobUtils.getMobRegionInfo(snapshotManifest.getTableDescriptor().getTableName());
199    if (tableRegions != null) {
200      // restore the mob region in case
201      if (regionNames.contains(mobRegion.getEncodedName())) {
202        monitor.rethrowException();
203        status.setStatus("Restoring mob region...");
204        List<RegionInfo> mobRegions = new ArrayList<>(1);
205        mobRegions.add(mobRegion);
206        restoreHdfsMobRegions(exec, regionManifests, mobRegions);
207        regionNames.remove(mobRegion.getEncodedName());
208        status.setStatus("Finished restoring mob region.");
209      }
210    }
211    if (regionNames.contains(mobRegion.getEncodedName())) {
212      // add the mob region
213      monitor.rethrowException();
214      status.setStatus("Cloning mob region...");
215      cloneHdfsMobRegion(regionManifests, mobRegion);
216      regionNames.remove(mobRegion.getEncodedName());
217      status.setStatus("Finished cloning mob region.");
218    }
219
220    // Identify which region are still available and which not.
221    // NOTE: we rely upon the region name as: "table name, start key, end key"
222    if (tableRegions != null) {
223      monitor.rethrowException();
224      for (RegionInfo regionInfo : tableRegions) {
225        String regionName = regionInfo.getEncodedName();
226        if (regionNames.contains(regionName)) {
227          LOG.info("region to restore: " + regionName);
228          regionNames.remove(regionName);
229          metaChanges.addRegionToRestore(
230            ProtobufUtil.toRegionInfo(regionManifests.get(regionName).getRegionInfo()));
231        } else {
232          LOG.info("region to remove: " + regionName);
233          metaChanges.addRegionToRemove(regionInfo);
234        }
235      }
236    }
237
238    // Regions to Add: present in the snapshot but not in the current table
239    List<RegionInfo> regionsToAdd = new ArrayList<>(regionNames.size());
240    if (regionNames.size() > 0) {
241      monitor.rethrowException();
242      for (String regionName : regionNames) {
243        LOG.info("region to add: " + regionName);
244        regionsToAdd
245          .add(ProtobufUtil.toRegionInfo(regionManifests.get(regionName).getRegionInfo()));
246      }
247    }
248
249    // Create new regions cloning from the snapshot
250    // HBASE-19980: We need to call cloneHdfsRegions() before restoreHdfsRegions() because
251    // regionsMap is constructed in cloneHdfsRegions() and it can be used in restoreHdfsRegions().
252    monitor.rethrowException();
253    status.setStatus("Cloning regions...");
254    RegionInfo[] clonedRegions = cloneHdfsRegions(exec, regionManifests, regionsToAdd);
255    metaChanges.setNewRegions(clonedRegions);
256    status.setStatus("Finished cloning regions.");
257
258    // Restore regions using the snapshot data
259    monitor.rethrowException();
260    status.setStatus("Restoring table regions...");
261    restoreHdfsRegions(exec, regionManifests, metaChanges.getRegionsToRestore());
262    status.setStatus("Finished restoring all table regions.");
263
264    // Remove regions from the current table
265    monitor.rethrowException();
266    status.setStatus("Starting to delete excess regions from table");
267    removeHdfsRegions(exec, metaChanges.getRegionsToRemove());
268    status.setStatus("Finished deleting excess regions from table.");
269
270    LOG.info("finishing restore table regions using snapshot=" + snapshotDesc);
271
272    return metaChanges;
273  }
274
275  /**
276   * Describe the set of operations needed to update hbase:meta after restore.
277   */
278  public static class RestoreMetaChanges {
279    private final Map<String, Pair<String, String>> parentsMap;
280    private final TableDescriptor htd;
281
282    private List<RegionInfo> regionsToRestore = null;
283    private List<RegionInfo> regionsToRemove = null;
284    private List<RegionInfo> regionsToAdd = null;
285
286    public RestoreMetaChanges(TableDescriptor htd, Map<String, Pair<String, String>> parentsMap) {
287      this.parentsMap = parentsMap;
288      this.htd = htd;
289    }
290
291    public TableDescriptor getTableDescriptor() {
292      return htd;
293    }
294
295    /**
296     * Returns the map of parent-children_pair.
297     * @return the map
298     */
299    public Map<String, Pair<String, String>> getParentToChildrenPairMap() {
300      return this.parentsMap;
301    }
302
303    /** Returns true if there're new regions */
304    public boolean hasRegionsToAdd() {
305      return this.regionsToAdd != null && this.regionsToAdd.size() > 0;
306    }
307
308    /**
309     * Returns the list of new regions added during the on-disk restore. The caller is responsible
310     * to add the regions to META. e.g MetaTableAccessor.addRegionsToMeta(...)
311     * @return the list of regions to add to META
312     */
313    public List<RegionInfo> getRegionsToAdd() {
314      return this.regionsToAdd;
315    }
316
317    /** Returns true if there're regions to restore */
318    public boolean hasRegionsToRestore() {
319      return this.regionsToRestore != null && this.regionsToRestore.size() > 0;
320    }
321
322    /**
323     * Returns the list of 'restored regions' during the on-disk restore. The caller is responsible
324     * to add the regions to hbase:meta if not present.
325     * @return the list of regions restored
326     */
327    public List<RegionInfo> getRegionsToRestore() {
328      return this.regionsToRestore;
329    }
330
331    /** Returns true if there're regions to remove */
332    public boolean hasRegionsToRemove() {
333      return this.regionsToRemove != null && this.regionsToRemove.size() > 0;
334    }
335
336    /**
337     * Returns the list of regions removed during the on-disk restore. The caller is responsible to
338     * remove the regions from META. e.g. MetaTableAccessor.deleteRegions(...)
339     * @return the list of regions to remove from META
340     */
341    public List<RegionInfo> getRegionsToRemove() {
342      return this.regionsToRemove;
343    }
344
345    void setNewRegions(final RegionInfo[] hris) {
346      if (hris != null) {
347        regionsToAdd = Arrays.asList(hris);
348      } else {
349        regionsToAdd = null;
350      }
351    }
352
353    void addRegionToRemove(final RegionInfo hri) {
354      if (regionsToRemove == null) {
355        regionsToRemove = new LinkedList<>();
356      }
357      regionsToRemove.add(hri);
358    }
359
360    void addRegionToRestore(final RegionInfo hri) {
361      if (regionsToRestore == null) {
362        regionsToRestore = new LinkedList<>();
363      }
364      regionsToRestore.add(hri);
365    }
366
367    public void updateMetaParentRegions(Connection connection, final List<RegionInfo> regionInfos)
368      throws IOException {
369      if (regionInfos == null || parentsMap.isEmpty()) return;
370
371      // Extract region names and offlined regions
372      Map<String, RegionInfo> regionsByName = new HashMap<>(regionInfos.size());
373      List<RegionInfo> parentRegions = new LinkedList<>();
374      for (RegionInfo regionInfo : regionInfos) {
375        if (regionInfo.isSplitParent()) {
376          parentRegions.add(regionInfo);
377        } else {
378          regionsByName.put(regionInfo.getEncodedName(), regionInfo);
379        }
380      }
381
382      // Update Offline parents
383      for (RegionInfo regionInfo : parentRegions) {
384        Pair<String, String> daughters = parentsMap.get(regionInfo.getEncodedName());
385        if (daughters == null) {
386          // The snapshot contains an unreferenced region.
387          // It will be removed by the CatalogJanitor.
388          LOG.warn("Skip update of unreferenced offline parent: " + regionInfo);
389          continue;
390        }
391
392        // One side of the split is already compacted
393        if (daughters.getSecond() == null) {
394          daughters.setSecond(daughters.getFirst());
395        }
396
397        LOG.debug("Update splits parent " + regionInfo.getEncodedName() + " -> " + daughters);
398        MetaTableAccessor.addSplitsToParent(connection, regionInfo,
399          regionsByName.get(daughters.getFirst()), regionsByName.get(daughters.getSecond()));
400      }
401    }
402  }
403
404  /**
405   * Remove specified regions from the file-system, using the archiver.
406   */
407  private void removeHdfsRegions(final ThreadPoolExecutor exec, final List<RegionInfo> regions)
408    throws IOException {
409    if (regions == null || regions.isEmpty()) return;
410    ModifyRegionUtils.editRegions(exec, regions, new ModifyRegionUtils.RegionEditTask() {
411      @Override
412      public void editRegion(final RegionInfo hri) throws IOException {
413        HFileArchiver.archiveRegion(conf, fs, hri);
414      }
415    });
416  }
417
418  /**
419   * Restore specified regions by restoring content to the snapshot state.
420   */
421  private void restoreHdfsRegions(final ThreadPoolExecutor exec,
422    final Map<String, SnapshotRegionManifest> regionManifests, final List<RegionInfo> regions)
423    throws IOException {
424    if (regions == null || regions.isEmpty()) return;
425    ModifyRegionUtils.editRegions(exec, regions, new ModifyRegionUtils.RegionEditTask() {
426      @Override
427      public void editRegion(final RegionInfo hri) throws IOException {
428        restoreRegion(hri, regionManifests.get(hri.getEncodedName()));
429      }
430    });
431  }
432
433  /**
434   * Restore specified mob regions by restoring content to the snapshot state.
435   */
436  private void restoreHdfsMobRegions(final ThreadPoolExecutor exec,
437    final Map<String, SnapshotRegionManifest> regionManifests, final List<RegionInfo> regions)
438    throws IOException {
439    if (regions == null || regions.isEmpty()) return;
440    ModifyRegionUtils.editRegions(exec, regions, new ModifyRegionUtils.RegionEditTask() {
441      @Override
442      public void editRegion(final RegionInfo hri) throws IOException {
443        restoreMobRegion(hri, regionManifests.get(hri.getEncodedName()));
444      }
445    });
446  }
447
448  private Map<String, List<SnapshotRegionManifest.StoreFile>>
449    getRegionHFileReferences(final SnapshotRegionManifest manifest) {
450    Map<String, List<SnapshotRegionManifest.StoreFile>> familyMap =
451      new HashMap<>(manifest.getFamilyFilesCount());
452    for (SnapshotRegionManifest.FamilyFiles familyFiles : manifest.getFamilyFilesList()) {
453      familyMap.put(familyFiles.getFamilyName().toStringUtf8(),
454        new ArrayList<>(familyFiles.getStoreFilesList()));
455    }
456    return familyMap;
457  }
458
459  /**
460   * Restore region by removing files not in the snapshot and adding the missing ones from the
461   * snapshot.
462   */
463  private void restoreRegion(final RegionInfo regionInfo,
464    final SnapshotRegionManifest regionManifest) throws IOException {
465    restoreRegion(regionInfo, regionManifest, new Path(tableDir, regionInfo.getEncodedName()));
466  }
467
468  /**
469   * Restore mob region by removing files not in the snapshot and adding the missing ones from the
470   * snapshot.
471   */
472  private void restoreMobRegion(final RegionInfo regionInfo,
473    final SnapshotRegionManifest regionManifest) throws IOException {
474    if (regionManifest == null) {
475      return;
476    }
477    restoreRegion(regionInfo, regionManifest,
478      MobUtils.getMobRegionPath(conf, tableDesc.getTableName()));
479  }
480
481  /**
482   * Restore region by removing files not in the snapshot and adding the missing ones from the
483   * snapshot.
484   */
485  private void restoreRegion(final RegionInfo regionInfo,
486    final SnapshotRegionManifest regionManifest, Path regionDir) throws IOException {
487    Map<String, List<SnapshotRegionManifest.StoreFile>> snapshotFiles =
488      getRegionHFileReferences(regionManifest);
489
490    String tableName = tableDesc.getTableName().getNameAsString();
491    final String snapshotName = snapshotDesc.getName();
492
493    Path regionPath = new Path(tableDir, regionInfo.getEncodedName());
494    HRegionFileSystem regionFS = (fs.exists(regionPath))
495      ? HRegionFileSystem.openRegionFromFileSystem(conf, fs, tableDir, regionInfo, false)
496      : HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, regionInfo);
497
498    // Restore families present in the table
499    for (Path familyDir : FSUtils.getFamilyDirs(fs, regionDir)) {
500      byte[] family = Bytes.toBytes(familyDir.getName());
501
502      Set<String> familyFiles = getTableRegionFamilyFiles(familyDir);
503      List<SnapshotRegionManifest.StoreFile> snapshotFamilyFiles =
504        snapshotFiles.remove(familyDir.getName());
505      List<StoreFileInfo> filesToTrack = new ArrayList<>();
506      if (snapshotFamilyFiles != null) {
507        List<SnapshotRegionManifest.StoreFile> hfilesToAdd = new ArrayList<>();
508        for (SnapshotRegionManifest.StoreFile storeFile : snapshotFamilyFiles) {
509          if (familyFiles.contains(storeFile.getName())) {
510            // HFile already present
511            familyFiles.remove(storeFile.getName());
512            // no need to restore already present files, but we need to add those to tracker
513            filesToTrack
514              .add(new StoreFileInfo(conf, fs, new Path(familyDir, storeFile.getName()), true));
515          } else {
516            // HFile missing
517            hfilesToAdd.add(storeFile);
518          }
519        }
520
521        // Remove hfiles not present in the snapshot
522        for (String hfileName : familyFiles) {
523          Path hfile = new Path(familyDir, hfileName);
524          if (!fs.getFileStatus(hfile).isDirectory()) {
525            LOG.trace("Removing HFile=" + hfileName + " not present in snapshot=" + snapshotName
526              + " from region=" + regionInfo.getEncodedName() + " table=" + tableName);
527            HFileArchiver.archiveStoreFile(conf, fs, regionInfo, tableDir, family, hfile);
528          }
529        }
530
531        // Restore Missing files
532        for (SnapshotRegionManifest.StoreFile storeFile : hfilesToAdd) {
533          LOG.debug("Restoring missing HFileLink " + storeFile.getName() + " of snapshot="
534            + snapshotName + " to region=" + regionInfo.getEncodedName() + " table=" + tableName);
535          String fileName = restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs);
536          // mark the reference file to be added to tracker
537          filesToTrack.add(new StoreFileInfo(conf, fs, new Path(familyDir, fileName), true));
538        }
539      } else {
540        // Family doesn't exists in the snapshot
541        LOG.trace("Removing family=" + Bytes.toString(family) + " in snapshot=" + snapshotName
542          + " from region=" + regionInfo.getEncodedName() + " table=" + tableName);
543        HFileArchiver.archiveFamilyByFamilyDir(fs, conf, regionInfo, familyDir, family);
544        fs.delete(familyDir, true);
545      }
546
547      StoreFileTracker tracker =
548        StoreFileTrackerFactory.create(conf, true, StoreContext.getBuilder()
549          .withFamilyStoreDirectoryPath(familyDir).withRegionFileSystem(regionFS).build());
550
551      // simply reset list of tracked files with the matching files
552      // and the extra one present in the snapshot
553      tracker.set(filesToTrack);
554    }
555
556    // Add families not present in the table
557    for (Map.Entry<String, List<SnapshotRegionManifest.StoreFile>> familyEntry : snapshotFiles
558      .entrySet()) {
559      Path familyDir = new Path(regionDir, familyEntry.getKey());
560      StoreFileTracker tracker =
561        StoreFileTrackerFactory.create(conf, true, StoreContext.getBuilder()
562          .withFamilyStoreDirectoryPath(familyDir).withRegionFileSystem(regionFS).build());
563      List<StoreFileInfo> files = new ArrayList<>();
564      if (!fs.mkdirs(familyDir)) {
565        throw new IOException("Unable to create familyDir=" + familyDir);
566      }
567
568      for (SnapshotRegionManifest.StoreFile storeFile : familyEntry.getValue()) {
569        LOG.trace("Adding HFileLink (Not present in the table) " + storeFile.getName()
570          + " of snapshot " + snapshotName + " to table=" + tableName);
571        String fileName = restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs);
572        files.add(new StoreFileInfo(conf, fs, new Path(familyDir, fileName), true));
573      }
574      tracker.set(files);
575    }
576  }
577
578  /** Returns The set of files in the specified family directory. */
579  private Set<String> getTableRegionFamilyFiles(final Path familyDir) throws IOException {
580    FileStatus[] hfiles = CommonFSUtils.listStatus(fs, familyDir);
581    if (hfiles == null) {
582      return Collections.emptySet();
583    }
584
585    Set<String> familyFiles = new HashSet<>(hfiles.length);
586    for (int i = 0; i < hfiles.length; ++i) {
587      String hfileName = hfiles[i].getPath().getName();
588      familyFiles.add(hfileName);
589    }
590
591    return familyFiles;
592  }
593
594  /**
595   * Clone specified regions. For each region create a new region and create a HFileLink for each
596   * hfile.
597   */
598  private RegionInfo[] cloneHdfsRegions(final ThreadPoolExecutor exec,
599    final Map<String, SnapshotRegionManifest> regionManifests, final List<RegionInfo> regions)
600    throws IOException {
601    if (regions == null || regions.isEmpty()) return null;
602
603    final Map<String, RegionInfo> snapshotRegions = new HashMap<>(regions.size());
604    final String snapshotName = snapshotDesc.getName();
605
606    // clone region info (change embedded tableName with the new one)
607    RegionInfo[] clonedRegionsInfo = new RegionInfo[regions.size()];
608    for (int i = 0; i < clonedRegionsInfo.length; ++i) {
609      // clone the region info from the snapshot region info
610      RegionInfo snapshotRegionInfo = regions.get(i);
611      clonedRegionsInfo[i] = cloneRegionInfo(snapshotRegionInfo);
612
613      // add the region name mapping between snapshot and cloned
614      String snapshotRegionName = snapshotRegionInfo.getEncodedName();
615      String clonedRegionName = clonedRegionsInfo[i].getEncodedName();
616      regionsMap.put(Bytes.toBytes(snapshotRegionName), Bytes.toBytes(clonedRegionName));
617      LOG.info("clone region=" + snapshotRegionName + " as " + clonedRegionName + " in snapshot "
618        + snapshotName);
619
620      // Add mapping between cloned region name and snapshot region info
621      snapshotRegions.put(clonedRegionName, snapshotRegionInfo);
622    }
623
624    // create the regions on disk
625    ModifyRegionUtils.createRegions(exec, conf, rootDir, tableDesc, clonedRegionsInfo,
626      new ModifyRegionUtils.RegionFillTask() {
627        @Override
628        public void fillRegion(final HRegion region) throws IOException {
629          RegionInfo snapshotHri = snapshotRegions.get(region.getRegionInfo().getEncodedName());
630          cloneRegion(region, snapshotHri, regionManifests.get(snapshotHri.getEncodedName()));
631        }
632      });
633
634    return clonedRegionsInfo;
635  }
636
637  /**
638   * Clone the mob region. For the region create a new region and create a HFileLink for each hfile.
639   */
640  private void cloneHdfsMobRegion(final Map<String, SnapshotRegionManifest> regionManifests,
641    final RegionInfo region) throws IOException {
642    // clone region info (change embedded tableName with the new one)
643    Path clonedRegionPath = MobUtils.getMobRegionPath(rootDir, tableDesc.getTableName());
644    cloneRegion(MobUtils.getMobRegionInfo(tableDesc.getTableName()), clonedRegionPath, region,
645      regionManifests.get(region.getEncodedName()));
646  }
647
648  /**
649   * Clone region directory content from the snapshot info. Each region is encoded with the table
650   * name, so the cloned region will have a different region name. Instead of copying the hfiles a
651   * HFileLink is created.
652   * @param regionDir {@link Path} cloned dir
653   */
654  private void cloneRegion(final RegionInfo newRegionInfo, final Path regionDir,
655    final RegionInfo snapshotRegionInfo, final SnapshotRegionManifest manifest) throws IOException {
656    final String tableName = tableDesc.getTableName().getNameAsString();
657    final String snapshotName = snapshotDesc.getName();
658    for (SnapshotRegionManifest.FamilyFiles familyFiles : manifest.getFamilyFilesList()) {
659      Path familyDir = new Path(regionDir, familyFiles.getFamilyName().toStringUtf8());
660      List<StoreFileInfo> clonedFiles = new ArrayList<>();
661      for (SnapshotRegionManifest.StoreFile storeFile : familyFiles.getStoreFilesList()) {
662        LOG.info("Adding HFileLink " + storeFile.getName() + " from cloned region " + "in snapshot "
663          + snapshotName + " to table=" + tableName);
664        if (MobUtils.isMobRegionInfo(newRegionInfo)) {
665          String mobFileName =
666            HFileLink.createHFileLinkName(snapshotRegionInfo, storeFile.getName());
667          Path mobPath = new Path(familyDir, mobFileName);
668          if (fs.exists(mobPath)) {
669            fs.delete(mobPath, true);
670          }
671          restoreStoreFile(familyDir, snapshotRegionInfo, storeFile, createBackRefs);
672        } else {
673          String file = restoreStoreFile(familyDir, snapshotRegionInfo, storeFile, createBackRefs);
674          clonedFiles.add(new StoreFileInfo(conf, fs, new Path(familyDir, file), true));
675        }
676      }
677      // we don't need to track files under mobdir
678      if (!MobUtils.isMobRegionInfo(newRegionInfo)) {
679        Path regionPath = new Path(tableDir, newRegionInfo.getEncodedName());
680        HRegionFileSystem regionFS = (fs.exists(regionPath))
681          ? HRegionFileSystem.openRegionFromFileSystem(conf, fs, tableDir, newRegionInfo, false)
682          : HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, newRegionInfo);
683
684        Configuration sftConf = StoreUtils.createStoreConfiguration(conf, tableDesc,
685          tableDesc.getColumnFamily(familyFiles.getFamilyName().toByteArray()));
686        StoreFileTracker tracker =
687          StoreFileTrackerFactory.create(sftConf, true, StoreContext.getBuilder()
688            .withFamilyStoreDirectoryPath(familyDir).withRegionFileSystem(regionFS).build());
689        tracker.set(clonedFiles);
690      }
691    }
692
693  }
694
695  /**
696   * Clone region directory content from the snapshot info. Each region is encoded with the table
697   * name, so the cloned region will have a different region name. Instead of copying the hfiles a
698   * HFileLink is created.
699   * @param region {@link HRegion} cloned
700   */
701  private void cloneRegion(final HRegion region, final RegionInfo snapshotRegionInfo,
702    final SnapshotRegionManifest manifest) throws IOException {
703    cloneRegion(region.getRegionInfo(), new Path(tableDir, region.getRegionInfo().getEncodedName()),
704      snapshotRegionInfo, manifest);
705  }
706
707  /**
708   * Create a new {@link HFileLink} to reference the store file.
709   * <p>
710   * The store file in the snapshot can be a simple hfile, an HFileLink or a reference.
711   * <ul>
712   * <li>hfile: abc -> table=region-abc
713   * <li>reference: abc.1234 -> table=region-abc.1234
714   * <li>hfilelink: table=region-hfile -> table=region-hfile
715   * </ul>
716   * @param familyDir     destination directory for the store file
717   * @param regionInfo    destination region info for the table
718   * @param createBackRef - Whether back reference should be created. Defaults to true.
719   * @param storeFile     store file name (can be a Reference, HFileLink or simple HFile)
720   */
721  private String restoreStoreFile(final Path familyDir, final RegionInfo regionInfo,
722    final SnapshotRegionManifest.StoreFile storeFile, final boolean createBackRef)
723    throws IOException {
724    String hfileName = storeFile.getName();
725    if (HFileLink.isHFileLink(hfileName)) {
726      return HFileLink.createFromHFileLink(conf, fs, familyDir, hfileName, createBackRef);
727    } else if (StoreFileInfo.isReference(hfileName)) {
728      return restoreReferenceFile(familyDir, regionInfo, storeFile);
729    } else {
730      return HFileLink.create(conf, fs, familyDir, regionInfo, hfileName, createBackRef);
731    }
732  }
733
734  /**
735   * Create a new {@link Reference} as copy of the source one.
736   * <p>
737   * <blockquote>
738   *
739   * <pre>
740   * The source table looks like:
741   *    1234/abc      (original file)
742   *    5678/abc.1234 (reference file)
743   *
744   * After the clone operation looks like:
745   *   wxyz/table=1234-abc
746   *   stuv/table=1234-abc.wxyz
747   *
748   * NOTE that the region name in the clone changes (md5 of regioninfo)
749   * and the reference should reflect that change.
750   * </pre>
751   *
752   * </blockquote>
753   * @param familyDir  destination directory for the store file
754   * @param regionInfo destination region info for the table
755   * @param storeFile  reference file name
756   */
757  private String restoreReferenceFile(final Path familyDir, final RegionInfo regionInfo,
758    final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
759    String hfileName = storeFile.getName();
760
761    // Extract the referred information (hfile name and parent region)
762    Path refPath =
763      StoreFileInfo
764        .getReferredToFile(
765          new Path(
766            new Path(
767              new Path(new Path(snapshotTable.getNamespaceAsString(),
768                snapshotTable.getQualifierAsString()), regionInfo.getEncodedName()),
769              familyDir.getName()),
770            hfileName));
771    String snapshotRegionName = refPath.getParent().getParent().getName();
772    String fileName = refPath.getName();
773
774    // The new reference should have the cloned region name as parent, if it is a clone.
775    String clonedRegionName = Bytes.toString(regionsMap.get(Bytes.toBytes(snapshotRegionName)));
776    if (clonedRegionName == null) clonedRegionName = snapshotRegionName;
777
778    // The output file should be a reference link table=snapshotRegion-fileName.clonedRegionName
779    Path linkPath = null;
780    String refLink = fileName;
781    if (!HFileLink.isHFileLink(fileName)) {
782      refLink = HFileLink.createHFileLinkName(snapshotTable, snapshotRegionName, fileName);
783      linkPath = new Path(familyDir,
784        HFileLink.createHFileLinkName(snapshotTable, regionInfo.getEncodedName(), hfileName));
785    }
786
787    Path outPath = new Path(familyDir, refLink + '.' + clonedRegionName);
788
789    // Create the new reference
790    if (storeFile.hasReference()) {
791      Reference reference = Reference.convert(storeFile.getReference());
792      reference.write(fs, outPath);
793    } else {
794      InputStream in;
795      if (linkPath != null) {
796        in = HFileLink.buildFromHFileLinkPattern(conf, linkPath).open(fs);
797      } else {
798        linkPath = new Path(new Path(
799          HRegion.getRegionDir(snapshotManifest.getSnapshotDir(), regionInfo.getEncodedName()),
800          familyDir.getName()), hfileName);
801        in = fs.open(linkPath);
802      }
803      OutputStream out = fs.create(outPath);
804      IOUtils.copyBytes(in, out, conf);
805    }
806
807    // Add the daughter region to the map
808    String regionName = Bytes.toString(regionsMap.get(regionInfo.getEncodedNameAsBytes()));
809    if (regionName == null) {
810      regionName = regionInfo.getEncodedName();
811    }
812    LOG.debug("Restore reference " + regionName + " to " + clonedRegionName);
813    synchronized (parentsMap) {
814      Pair<String, String> daughters = parentsMap.get(clonedRegionName);
815      if (daughters == null) {
816        // In case one side of the split is already compacted, regionName is put as both first and
817        // second of Pair
818        daughters = new Pair<>(regionName, regionName);
819        parentsMap.put(clonedRegionName, daughters);
820      } else if (!regionName.equals(daughters.getFirst())) {
821        daughters.setSecond(regionName);
822      }
823    }
824    return outPath.getName();
825  }
826
827  /**
828   * Create a new {@link RegionInfo} from the snapshot region info. Keep the same startKey, endKey,
829   * regionId and split information but change the table name.
830   * @param snapshotRegionInfo Info for region to clone.
831   * @return the new HRegion instance
832   */
833  public RegionInfo cloneRegionInfo(final RegionInfo snapshotRegionInfo) {
834    return cloneRegionInfo(tableDesc.getTableName(), snapshotRegionInfo);
835  }
836
837  public static RegionInfo cloneRegionInfo(TableName tableName, RegionInfo snapshotRegionInfo) {
838    return RegionInfoBuilder.newBuilder(tableName).setStartKey(snapshotRegionInfo.getStartKey())
839      .setEndKey(snapshotRegionInfo.getEndKey()).setSplit(snapshotRegionInfo.isSplit())
840      .setRegionId(snapshotRegionInfo.getRegionId()).setOffline(snapshotRegionInfo.isOffline())
841      .build();
842  }
843
844  /** Returns the set of the regions contained in the table */
845  private List<RegionInfo> getTableRegions() throws IOException {
846    LOG.debug("get table regions: " + tableDir);
847    FileStatus[] regionDirs =
848      CommonFSUtils.listStatus(fs, tableDir, new FSUtils.RegionDirFilter(fs));
849    if (regionDirs == null) {
850      return null;
851    }
852
853    List<RegionInfo> regions = new ArrayList<>(regionDirs.length);
854    for (int i = 0; i < regionDirs.length; ++i) {
855      RegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDirs[i].getPath());
856      regions.add(hri);
857    }
858    LOG.debug("found " + regions.size() + " regions for table="
859      + tableDesc.getTableName().getNameAsString());
860    return regions;
861  }
862
863  /**
864   * Copy the snapshot files for a snapshot scanner, discards meta changes.
865   */
866  public static RestoreMetaChanges copySnapshotForScanner(Configuration conf, FileSystem fs,
867    Path rootDir, Path restoreDir, String snapshotName) throws IOException {
868    // ensure that restore dir is not under root dir
869    if (!restoreDir.getFileSystem(conf).getUri().equals(rootDir.getFileSystem(conf).getUri())) {
870      throw new IllegalArgumentException(
871        "Filesystems for restore directory and HBase root " + "directory should be the same");
872    }
873    if (restoreDir.toUri().getPath().startsWith(rootDir.toUri().getPath() + "/")) {
874      throw new IllegalArgumentException("Restore directory cannot be a sub directory of HBase "
875        + "root directory. RootDir: " + rootDir + ", restoreDir: " + restoreDir);
876    }
877
878    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
879    SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
880    SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
881
882    MonitoredTask status = TaskMonitor.get()
883      .createStatus("Restoring  snapshot '" + snapshotName + "' to directory " + restoreDir);
884    ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher();
885
886    // we send createBackRefs=false so that restored hfiles do not create back reference links
887    // in the base hbase root dir.
888    RestoreSnapshotHelper helper = new RestoreSnapshotHelper(conf, fs, manifest,
889      manifest.getTableDescriptor(), restoreDir, monitor, status, false);
890    RestoreMetaChanges metaChanges = helper.restoreHdfsRegions(); // TODO: parallelize.
891
892    if (LOG.isDebugEnabled()) {
893      LOG.debug("Restored table dir:" + restoreDir);
894      CommonFSUtils.logFileSystemState(fs, restoreDir, LOG);
895    }
896    return metaChanges;
897  }
898
899  public static void restoreSnapshotAcl(SnapshotDescription snapshot, TableName newTableName,
900    Configuration conf) throws IOException {
901    if (snapshot.hasUsersAndPermissions() && snapshot.getUsersAndPermissions() != null) {
902      LOG.info("Restore snapshot acl to table. snapshot: " + snapshot + ", table: " + newTableName);
903      ListMultimap<String, Permission> perms =
904        ShadedAccessControlUtil.toUserTablePermissions(snapshot.getUsersAndPermissions());
905      try (Connection conn = ConnectionFactory.createConnection(conf)) {
906        for (Entry<String, Permission> e : perms.entries()) {
907          String user = e.getKey();
908          TablePermission tablePerm = (TablePermission) e.getValue();
909          AccessControlClient.grant(conn, newTableName, user, tablePerm.getFamily(),
910            tablePerm.getQualifier(), tablePerm.getActions());
911        }
912      } catch (Throwable e) {
913        throw new IOException("Grant acl into newly creatd table failed. snapshot: " + snapshot
914          + ", table: " + newTableName, e);
915      }
916    }
917  }
918}