001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.snapshot;
019
020import java.io.IOException;
021import java.io.InputStream;
022import java.io.OutputStream;
023import java.util.ArrayList;
024import java.util.Arrays;
025import java.util.Collections;
026import java.util.HashMap;
027import java.util.HashSet;
028import java.util.LinkedList;
029import java.util.List;
030import java.util.Map;
031import java.util.Map.Entry;
032import java.util.Set;
033import java.util.TreeMap;
034import java.util.concurrent.ThreadPoolExecutor;
035import org.apache.hadoop.conf.Configuration;
036import org.apache.hadoop.fs.FileStatus;
037import org.apache.hadoop.fs.FileSystem;
038import org.apache.hadoop.fs.Path;
039import org.apache.hadoop.hbase.MetaTableAccessor;
040import org.apache.hadoop.hbase.TableName;
041import org.apache.hadoop.hbase.backup.HFileArchiver;
042import org.apache.hadoop.hbase.client.Connection;
043import org.apache.hadoop.hbase.client.ConnectionFactory;
044import org.apache.hadoop.hbase.client.RegionInfo;
045import org.apache.hadoop.hbase.client.RegionInfoBuilder;
046import org.apache.hadoop.hbase.client.TableDescriptor;
047import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
048import org.apache.hadoop.hbase.io.HFileLink;
049import org.apache.hadoop.hbase.io.Reference;
050import org.apache.hadoop.hbase.mob.MobUtils;
051import org.apache.hadoop.hbase.monitoring.MonitoredTask;
052import org.apache.hadoop.hbase.monitoring.TaskMonitor;
053import org.apache.hadoop.hbase.regionserver.HRegion;
054import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
055import org.apache.hadoop.hbase.regionserver.StoreContext;
056import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
057import org.apache.hadoop.hbase.regionserver.StoreUtils;
058import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker;
059import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
060import org.apache.hadoop.hbase.security.access.AccessControlClient;
061import org.apache.hadoop.hbase.security.access.Permission;
062import org.apache.hadoop.hbase.security.access.ShadedAccessControlUtil;
063import org.apache.hadoop.hbase.security.access.TablePermission;
064import org.apache.hadoop.hbase.util.Bytes;
065import org.apache.hadoop.hbase.util.CommonFSUtils;
066import org.apache.hadoop.hbase.util.FSUtils;
067import org.apache.hadoop.hbase.util.ModifyRegionUtils;
068import org.apache.hadoop.hbase.util.Pair;
069import org.apache.hadoop.io.IOUtils;
070import org.apache.yetus.audience.InterfaceAudience;
071import org.slf4j.Logger;
072import org.slf4j.LoggerFactory;
073
074import org.apache.hbase.thirdparty.com.google.common.collect.ListMultimap;
075
076import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
077import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription;
078import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
079
080/**
081 * Helper to Restore/Clone a Snapshot
082 * <p>
083 * The helper assumes that a table is already created, and by calling restore() the content present
084 * in the snapshot will be restored as the new content of the table.
085 * <p>
086 * Clone from Snapshot: If the target table is empty, the restore operation is just a "clone
087 * operation", where the only operations are:
088 * <ul>
089 * <li>for each region in the snapshot create a new region (note that the region will have a
090 * different name, since the encoding contains the table name)
091 * <li>for each file in the region create a new HFileLink to point to the original file.
092 * <li>restore the logs, if any
093 * </ul>
094 * <p>
095 * Restore from Snapshot:
096 * <ul>
097 * <li>for each region in the table verify which are available in the snapshot and which are not
098 * <ul>
099 * <li>if the region is not present in the snapshot, remove it.
100 * <li>if the region is present in the snapshot
101 * <ul>
102 * <li>for each file in the table region verify which are available in the snapshot
103 * <ul>
104 * <li>if the hfile is not present in the snapshot, remove it
105 * <li>if the hfile is present, keep it (nothing to do)
106 * </ul>
107 * <li>for each file in the snapshot region but not in the table
108 * <ul>
109 * <li>create a new HFileLink that point to the original file
110 * </ul>
111 * </ul>
112 * </ul>
113 * <li>for each region in the snapshot not present in the current table state
114 * <ul>
115 * <li>create a new region and for each file in the region create a new HFileLink (This is the same
116 * as the clone operation)
117 * </ul>
118 * <li>restore the logs, if any
119 * </ul>
120 */
121@InterfaceAudience.Private
122public class RestoreSnapshotHelper {
123  private static final Logger LOG = LoggerFactory.getLogger(RestoreSnapshotHelper.class);
124
125  private final Map<byte[], byte[]> regionsMap = new TreeMap<>(Bytes.BYTES_COMPARATOR);
126
127  private final Map<String, Pair<String, String>> parentsMap = new HashMap<>();
128
129  private final ForeignExceptionDispatcher monitor;
130  private final MonitoredTask status;
131
132  private final SnapshotManifest snapshotManifest;
133  private final SnapshotDescription snapshotDesc;
134  private final TableName snapshotTable;
135
136  private final TableDescriptor tableDesc;
137  private final Path rootDir;
138  private final Path tableDir;
139
140  private final Configuration conf;
141  private final FileSystem fs;
142  private final boolean createBackRefs;
143
144  public RestoreSnapshotHelper(final Configuration conf, final FileSystem fs,
145    final SnapshotManifest manifest, final TableDescriptor tableDescriptor, final Path rootDir,
146    final ForeignExceptionDispatcher monitor, final MonitoredTask status) {
147    this(conf, fs, manifest, tableDescriptor, rootDir, monitor, status, true);
148  }
149
150  public RestoreSnapshotHelper(final Configuration conf, final FileSystem fs,
151    final SnapshotManifest manifest, final TableDescriptor tableDescriptor, final Path rootDir,
152    final ForeignExceptionDispatcher monitor, final MonitoredTask status,
153    final boolean createBackRefs) {
154    this.fs = fs;
155    this.conf = conf;
156    this.snapshotManifest = manifest;
157    this.snapshotDesc = manifest.getSnapshotDescription();
158    this.snapshotTable = TableName.valueOf(snapshotDesc.getTable());
159    this.tableDesc = tableDescriptor;
160    this.rootDir = rootDir;
161    this.tableDir = CommonFSUtils.getTableDir(rootDir, tableDesc.getTableName());
162    this.monitor = monitor;
163    this.status = status;
164    this.createBackRefs = createBackRefs;
165  }
166
167  /**
168   * Restore the on-disk table to a specified snapshot state.
169   * @return the set of regions touched by the restore operation
170   */
171  public RestoreMetaChanges restoreHdfsRegions() throws IOException {
172    ThreadPoolExecutor exec = SnapshotManifest.createExecutor(conf, "RestoreSnapshot");
173    try {
174      return restoreHdfsRegions(exec);
175    } finally {
176      exec.shutdown();
177    }
178  }
179
180  private RestoreMetaChanges restoreHdfsRegions(final ThreadPoolExecutor exec) throws IOException {
181    LOG.info("starting restore table regions using snapshot=" + snapshotDesc);
182
183    Map<String, SnapshotRegionManifest> regionManifests = snapshotManifest.getRegionManifestsMap();
184    if (regionManifests == null) {
185      LOG.warn("Nothing to restore. Snapshot " + snapshotDesc + " looks empty");
186      return null;
187    }
188
189    RestoreMetaChanges metaChanges = new RestoreMetaChanges(tableDesc, parentsMap);
190
191    // Take a copy of the manifest.keySet() since we are going to modify
192    // this instance, by removing the regions already present in the restore dir.
193    Set<String> regionNames = new HashSet<>(regionManifests.keySet());
194
195    List<RegionInfo> tableRegions = getTableRegions();
196
197    RegionInfo mobRegion =
198      MobUtils.getMobRegionInfo(snapshotManifest.getTableDescriptor().getTableName());
199    if (tableRegions != null) {
200      // restore the mob region in case
201      if (regionNames.contains(mobRegion.getEncodedName())) {
202        monitor.rethrowException();
203        status.setStatus("Restoring mob region...");
204        List<RegionInfo> mobRegions = new ArrayList<>(1);
205        mobRegions.add(mobRegion);
206        restoreHdfsMobRegions(exec, regionManifests, mobRegions);
207        regionNames.remove(mobRegion.getEncodedName());
208        status.setStatus("Finished restoring mob region.");
209      }
210    }
211    if (regionNames.contains(mobRegion.getEncodedName())) {
212      // add the mob region
213      monitor.rethrowException();
214      status.setStatus("Cloning mob region...");
215      cloneHdfsMobRegion(regionManifests, mobRegion);
216      regionNames.remove(mobRegion.getEncodedName());
217      status.setStatus("Finished cloning mob region.");
218    }
219
220    // Identify which region are still available and which not.
221    // NOTE: we rely upon the region name as: "table name, start key, end key"
222    if (tableRegions != null) {
223      monitor.rethrowException();
224      for (RegionInfo regionInfo : tableRegions) {
225        String regionName = regionInfo.getEncodedName();
226        if (regionNames.contains(regionName)) {
227          LOG.info("region to restore: " + regionName);
228          regionNames.remove(regionName);
229          metaChanges.addRegionToRestore(
230            ProtobufUtil.toRegionInfo(regionManifests.get(regionName).getRegionInfo()));
231        } else {
232          LOG.info("region to remove: " + regionName);
233          metaChanges.addRegionToRemove(regionInfo);
234        }
235      }
236    }
237
238    // Regions to Add: present in the snapshot but not in the current table
239    List<RegionInfo> regionsToAdd = new ArrayList<>(regionNames.size());
240    if (regionNames.size() > 0) {
241      monitor.rethrowException();
242      for (String regionName : regionNames) {
243        LOG.info("region to add: " + regionName);
244        regionsToAdd
245          .add(ProtobufUtil.toRegionInfo(regionManifests.get(regionName).getRegionInfo()));
246      }
247    }
248
249    // Create new regions cloning from the snapshot
250    // HBASE-19980: We need to call cloneHdfsRegions() before restoreHdfsRegions() because
251    // regionsMap is constructed in cloneHdfsRegions() and it can be used in restoreHdfsRegions().
252    monitor.rethrowException();
253    status.setStatus("Cloning regions...");
254    RegionInfo[] clonedRegions = cloneHdfsRegions(exec, regionManifests, regionsToAdd);
255    metaChanges.setNewRegions(clonedRegions);
256    status.setStatus("Finished cloning regions.");
257
258    // Restore regions using the snapshot data
259    monitor.rethrowException();
260    status.setStatus("Restoring table regions...");
261    restoreHdfsRegions(exec, regionManifests, metaChanges.getRegionsToRestore());
262    status.setStatus("Finished restoring all table regions.");
263
264    // Remove regions from the current table
265    monitor.rethrowException();
266    status.setStatus("Starting to delete excess regions from table");
267    removeHdfsRegions(exec, metaChanges.getRegionsToRemove());
268    status.setStatus("Finished deleting excess regions from table.");
269
270    LOG.info("finishing restore table regions using snapshot=" + snapshotDesc);
271
272    return metaChanges;
273  }
274
275  /**
276   * Describe the set of operations needed to update hbase:meta after restore.
277   */
278  public static class RestoreMetaChanges {
279    private final Map<String, Pair<String, String>> parentsMap;
280    private final TableDescriptor htd;
281
282    private List<RegionInfo> regionsToRestore = null;
283    private List<RegionInfo> regionsToRemove = null;
284    private List<RegionInfo> regionsToAdd = null;
285
286    public RestoreMetaChanges(TableDescriptor htd, Map<String, Pair<String, String>> parentsMap) {
287      this.parentsMap = parentsMap;
288      this.htd = htd;
289    }
290
291    public TableDescriptor getTableDescriptor() {
292      return htd;
293    }
294
295    /**
296     * Returns the map of parent-children_pair.
297     * @return the map
298     */
299    public Map<String, Pair<String, String>> getParentToChildrenPairMap() {
300      return this.parentsMap;
301    }
302
303    /**
304     * @return true if there're new regions
305     */
306    public boolean hasRegionsToAdd() {
307      return this.regionsToAdd != null && this.regionsToAdd.size() > 0;
308    }
309
310    /**
311     * Returns the list of new regions added during the on-disk restore. The caller is responsible
312     * to add the regions to META. e.g MetaTableAccessor.addRegionsToMeta(...)
313     * @return the list of regions to add to META
314     */
315    public List<RegionInfo> getRegionsToAdd() {
316      return this.regionsToAdd;
317    }
318
319    /**
320     * @return true if there're regions to restore
321     */
322    public boolean hasRegionsToRestore() {
323      return this.regionsToRestore != null && this.regionsToRestore.size() > 0;
324    }
325
326    /**
327     * Returns the list of 'restored regions' during the on-disk restore. The caller is responsible
328     * to add the regions to hbase:meta if not present.
329     * @return the list of regions restored
330     */
331    public List<RegionInfo> getRegionsToRestore() {
332      return this.regionsToRestore;
333    }
334
335    /**
336     * @return true if there're regions to remove
337     */
338    public boolean hasRegionsToRemove() {
339      return this.regionsToRemove != null && this.regionsToRemove.size() > 0;
340    }
341
342    /**
343     * Returns the list of regions removed during the on-disk restore. The caller is responsible to
344     * remove the regions from META. e.g. MetaTableAccessor.deleteRegions(...)
345     * @return the list of regions to remove from META
346     */
347    public List<RegionInfo> getRegionsToRemove() {
348      return this.regionsToRemove;
349    }
350
351    void setNewRegions(final RegionInfo[] hris) {
352      if (hris != null) {
353        regionsToAdd = Arrays.asList(hris);
354      } else {
355        regionsToAdd = null;
356      }
357    }
358
359    void addRegionToRemove(final RegionInfo hri) {
360      if (regionsToRemove == null) {
361        regionsToRemove = new LinkedList<>();
362      }
363      regionsToRemove.add(hri);
364    }
365
366    void addRegionToRestore(final RegionInfo hri) {
367      if (regionsToRestore == null) {
368        regionsToRestore = new LinkedList<>();
369      }
370      regionsToRestore.add(hri);
371    }
372
373    public void updateMetaParentRegions(Connection connection, final List<RegionInfo> regionInfos)
374      throws IOException {
375      if (regionInfos == null || parentsMap.isEmpty()) return;
376
377      // Extract region names and offlined regions
378      Map<String, RegionInfo> regionsByName = new HashMap<>(regionInfos.size());
379      List<RegionInfo> parentRegions = new LinkedList<>();
380      for (RegionInfo regionInfo : regionInfos) {
381        if (regionInfo.isSplitParent()) {
382          parentRegions.add(regionInfo);
383        } else {
384          regionsByName.put(regionInfo.getEncodedName(), regionInfo);
385        }
386      }
387
388      // Update Offline parents
389      for (RegionInfo regionInfo : parentRegions) {
390        Pair<String, String> daughters = parentsMap.get(regionInfo.getEncodedName());
391        if (daughters == null) {
392          // The snapshot contains an unreferenced region.
393          // It will be removed by the CatalogJanitor.
394          LOG.warn("Skip update of unreferenced offline parent: " + regionInfo);
395          continue;
396        }
397
398        // One side of the split is already compacted
399        if (daughters.getSecond() == null) {
400          daughters.setSecond(daughters.getFirst());
401        }
402
403        LOG.debug("Update splits parent " + regionInfo.getEncodedName() + " -> " + daughters);
404        MetaTableAccessor.addSplitsToParent(connection, regionInfo,
405          regionsByName.get(daughters.getFirst()), regionsByName.get(daughters.getSecond()));
406      }
407    }
408  }
409
410  /**
411   * Remove specified regions from the file-system, using the archiver.
412   */
413  private void removeHdfsRegions(final ThreadPoolExecutor exec, final List<RegionInfo> regions)
414    throws IOException {
415    if (regions == null || regions.isEmpty()) return;
416    ModifyRegionUtils.editRegions(exec, regions, new ModifyRegionUtils.RegionEditTask() {
417      @Override
418      public void editRegion(final RegionInfo hri) throws IOException {
419        HFileArchiver.archiveRegion(conf, fs, hri);
420      }
421    });
422  }
423
424  /**
425   * Restore specified regions by restoring content to the snapshot state.
426   */
427  private void restoreHdfsRegions(final ThreadPoolExecutor exec,
428    final Map<String, SnapshotRegionManifest> regionManifests, final List<RegionInfo> regions)
429    throws IOException {
430    if (regions == null || regions.isEmpty()) return;
431    ModifyRegionUtils.editRegions(exec, regions, new ModifyRegionUtils.RegionEditTask() {
432      @Override
433      public void editRegion(final RegionInfo hri) throws IOException {
434        restoreRegion(hri, regionManifests.get(hri.getEncodedName()));
435      }
436    });
437  }
438
439  /**
440   * Restore specified mob regions by restoring content to the snapshot state.
441   */
442  private void restoreHdfsMobRegions(final ThreadPoolExecutor exec,
443    final Map<String, SnapshotRegionManifest> regionManifests, final List<RegionInfo> regions)
444    throws IOException {
445    if (regions == null || regions.isEmpty()) return;
446    ModifyRegionUtils.editRegions(exec, regions, new ModifyRegionUtils.RegionEditTask() {
447      @Override
448      public void editRegion(final RegionInfo hri) throws IOException {
449        restoreMobRegion(hri, regionManifests.get(hri.getEncodedName()));
450      }
451    });
452  }
453
454  private Map<String, List<SnapshotRegionManifest.StoreFile>>
455    getRegionHFileReferences(final SnapshotRegionManifest manifest) {
456    Map<String, List<SnapshotRegionManifest.StoreFile>> familyMap =
457      new HashMap<>(manifest.getFamilyFilesCount());
458    for (SnapshotRegionManifest.FamilyFiles familyFiles : manifest.getFamilyFilesList()) {
459      familyMap.put(familyFiles.getFamilyName().toStringUtf8(),
460        new ArrayList<>(familyFiles.getStoreFilesList()));
461    }
462    return familyMap;
463  }
464
465  /**
466   * Restore region by removing files not in the snapshot and adding the missing ones from the
467   * snapshot.
468   */
469  private void restoreRegion(final RegionInfo regionInfo,
470    final SnapshotRegionManifest regionManifest) throws IOException {
471    restoreRegion(regionInfo, regionManifest, new Path(tableDir, regionInfo.getEncodedName()));
472  }
473
474  /**
475   * Restore mob region by removing files not in the snapshot and adding the missing ones from the
476   * snapshot.
477   */
478  private void restoreMobRegion(final RegionInfo regionInfo,
479    final SnapshotRegionManifest regionManifest) throws IOException {
480    if (regionManifest == null) {
481      return;
482    }
483    restoreRegion(regionInfo, regionManifest,
484      MobUtils.getMobRegionPath(conf, tableDesc.getTableName()));
485  }
486
487  /**
488   * Restore region by removing files not in the snapshot and adding the missing ones from the
489   * snapshot.
490   */
491  private void restoreRegion(final RegionInfo regionInfo,
492    final SnapshotRegionManifest regionManifest, Path regionDir) throws IOException {
493    Map<String, List<SnapshotRegionManifest.StoreFile>> snapshotFiles =
494      getRegionHFileReferences(regionManifest);
495
496    String tableName = tableDesc.getTableName().getNameAsString();
497    final String snapshotName = snapshotDesc.getName();
498
499    Path regionPath = new Path(tableDir, regionInfo.getEncodedName());
500    HRegionFileSystem regionFS = (fs.exists(regionPath))
501      ? HRegionFileSystem.openRegionFromFileSystem(conf, fs, tableDir, regionInfo, false)
502      : HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, regionInfo);
503
504    // Restore families present in the table
505    for (Path familyDir : FSUtils.getFamilyDirs(fs, regionDir)) {
506      byte[] family = Bytes.toBytes(familyDir.getName());
507
508      Set<String> familyFiles = getTableRegionFamilyFiles(familyDir);
509      List<SnapshotRegionManifest.StoreFile> snapshotFamilyFiles =
510        snapshotFiles.remove(familyDir.getName());
511      List<StoreFileInfo> filesToTrack = new ArrayList<>();
512      if (snapshotFamilyFiles != null) {
513        List<SnapshotRegionManifest.StoreFile> hfilesToAdd = new ArrayList<>();
514        for (SnapshotRegionManifest.StoreFile storeFile : snapshotFamilyFiles) {
515          if (familyFiles.contains(storeFile.getName())) {
516            // HFile already present
517            familyFiles.remove(storeFile.getName());
518            // no need to restore already present files, but we need to add those to tracker
519            filesToTrack
520              .add(new StoreFileInfo(conf, fs, new Path(familyDir, storeFile.getName()), true));
521          } else {
522            // HFile missing
523            hfilesToAdd.add(storeFile);
524          }
525        }
526
527        // Remove hfiles not present in the snapshot
528        for (String hfileName : familyFiles) {
529          Path hfile = new Path(familyDir, hfileName);
530          if (!fs.getFileStatus(hfile).isDirectory()) {
531            LOG.trace("Removing HFile=" + hfileName + " not present in snapshot=" + snapshotName
532              + " from region=" + regionInfo.getEncodedName() + " table=" + tableName);
533            HFileArchiver.archiveStoreFile(conf, fs, regionInfo, tableDir, family, hfile);
534          }
535        }
536
537        // Restore Missing files
538        for (SnapshotRegionManifest.StoreFile storeFile : hfilesToAdd) {
539          LOG.debug("Restoring missing HFileLink " + storeFile.getName() + " of snapshot="
540            + snapshotName + " to region=" + regionInfo.getEncodedName() + " table=" + tableName);
541          String fileName = restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs);
542          // mark the reference file to be added to tracker
543          filesToTrack.add(new StoreFileInfo(conf, fs, new Path(familyDir, fileName), true));
544        }
545      } else {
546        // Family doesn't exists in the snapshot
547        LOG.trace("Removing family=" + Bytes.toString(family) + " in snapshot=" + snapshotName
548          + " from region=" + regionInfo.getEncodedName() + " table=" + tableName);
549        HFileArchiver.archiveFamilyByFamilyDir(fs, conf, regionInfo, familyDir, family);
550        fs.delete(familyDir, true);
551      }
552
553      StoreFileTracker tracker =
554        StoreFileTrackerFactory.create(conf, true, StoreContext.getBuilder()
555          .withFamilyStoreDirectoryPath(familyDir).withRegionFileSystem(regionFS).build());
556
557      // simply reset list of tracked files with the matching files
558      // and the extra one present in the snapshot
559      tracker.set(filesToTrack);
560    }
561
562    // Add families not present in the table
563    for (Map.Entry<String, List<SnapshotRegionManifest.StoreFile>> familyEntry : snapshotFiles
564      .entrySet()) {
565      Path familyDir = new Path(regionDir, familyEntry.getKey());
566      StoreFileTracker tracker =
567        StoreFileTrackerFactory.create(conf, true, StoreContext.getBuilder()
568          .withFamilyStoreDirectoryPath(familyDir).withRegionFileSystem(regionFS).build());
569      List<StoreFileInfo> files = new ArrayList<>();
570      if (!fs.mkdirs(familyDir)) {
571        throw new IOException("Unable to create familyDir=" + familyDir);
572      }
573
574      for (SnapshotRegionManifest.StoreFile storeFile : familyEntry.getValue()) {
575        LOG.trace("Adding HFileLink (Not present in the table) " + storeFile.getName()
576          + " of snapshot " + snapshotName + " to table=" + tableName);
577        String fileName = restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs);
578        files.add(new StoreFileInfo(conf, fs, new Path(familyDir, fileName), true));
579      }
580      tracker.set(files);
581    }
582  }
583
584  /**
585   * @return The set of files in the specified family directory.
586   */
587  private Set<String> getTableRegionFamilyFiles(final Path familyDir) throws IOException {
588    FileStatus[] hfiles = CommonFSUtils.listStatus(fs, familyDir);
589    if (hfiles == null) {
590      return Collections.emptySet();
591    }
592
593    Set<String> familyFiles = new HashSet<>(hfiles.length);
594    for (int i = 0; i < hfiles.length; ++i) {
595      String hfileName = hfiles[i].getPath().getName();
596      familyFiles.add(hfileName);
597    }
598
599    return familyFiles;
600  }
601
602  /**
603   * Clone specified regions. For each region create a new region and create a HFileLink for each
604   * hfile.
605   */
606  private RegionInfo[] cloneHdfsRegions(final ThreadPoolExecutor exec,
607    final Map<String, SnapshotRegionManifest> regionManifests, final List<RegionInfo> regions)
608    throws IOException {
609    if (regions == null || regions.isEmpty()) return null;
610
611    final Map<String, RegionInfo> snapshotRegions = new HashMap<>(regions.size());
612    final String snapshotName = snapshotDesc.getName();
613
614    // clone region info (change embedded tableName with the new one)
615    RegionInfo[] clonedRegionsInfo = new RegionInfo[regions.size()];
616    for (int i = 0; i < clonedRegionsInfo.length; ++i) {
617      // clone the region info from the snapshot region info
618      RegionInfo snapshotRegionInfo = regions.get(i);
619      clonedRegionsInfo[i] = cloneRegionInfo(snapshotRegionInfo);
620
621      // add the region name mapping between snapshot and cloned
622      String snapshotRegionName = snapshotRegionInfo.getEncodedName();
623      String clonedRegionName = clonedRegionsInfo[i].getEncodedName();
624      regionsMap.put(Bytes.toBytes(snapshotRegionName), Bytes.toBytes(clonedRegionName));
625      LOG.info("clone region=" + snapshotRegionName + " as " + clonedRegionName + " in snapshot "
626        + snapshotName);
627
628      // Add mapping between cloned region name and snapshot region info
629      snapshotRegions.put(clonedRegionName, snapshotRegionInfo);
630    }
631
632    // create the regions on disk
633    ModifyRegionUtils.createRegions(exec, conf, rootDir, tableDesc, clonedRegionsInfo,
634      new ModifyRegionUtils.RegionFillTask() {
635        @Override
636        public void fillRegion(final HRegion region) throws IOException {
637          RegionInfo snapshotHri = snapshotRegions.get(region.getRegionInfo().getEncodedName());
638          cloneRegion(region, snapshotHri, regionManifests.get(snapshotHri.getEncodedName()));
639        }
640      });
641
642    return clonedRegionsInfo;
643  }
644
645  /**
646   * Clone the mob region. For the region create a new region and create a HFileLink for each hfile.
647   */
648  private void cloneHdfsMobRegion(final Map<String, SnapshotRegionManifest> regionManifests,
649    final RegionInfo region) throws IOException {
650    // clone region info (change embedded tableName with the new one)
651    Path clonedRegionPath = MobUtils.getMobRegionPath(rootDir, tableDesc.getTableName());
652    cloneRegion(MobUtils.getMobRegionInfo(tableDesc.getTableName()), clonedRegionPath, region,
653      regionManifests.get(region.getEncodedName()));
654  }
655
656  /**
657   * Clone region directory content from the snapshot info. Each region is encoded with the table
658   * name, so the cloned region will have a different region name. Instead of copying the hfiles a
659   * HFileLink is created.
660   * @param regionDir {@link Path} cloned dir n
661   */
662  private void cloneRegion(final RegionInfo newRegionInfo, final Path regionDir,
663    final RegionInfo snapshotRegionInfo, final SnapshotRegionManifest manifest) throws IOException {
664    final String tableName = tableDesc.getTableName().getNameAsString();
665    final String snapshotName = snapshotDesc.getName();
666    for (SnapshotRegionManifest.FamilyFiles familyFiles : manifest.getFamilyFilesList()) {
667      Path familyDir = new Path(regionDir, familyFiles.getFamilyName().toStringUtf8());
668      List<StoreFileInfo> clonedFiles = new ArrayList<>();
669      for (SnapshotRegionManifest.StoreFile storeFile : familyFiles.getStoreFilesList()) {
670        LOG.info("Adding HFileLink " + storeFile.getName() + " from cloned region " + "in snapshot "
671          + snapshotName + " to table=" + tableName);
672        if (MobUtils.isMobRegionInfo(newRegionInfo)) {
673          String mobFileName =
674            HFileLink.createHFileLinkName(snapshotRegionInfo, storeFile.getName());
675          Path mobPath = new Path(familyDir, mobFileName);
676          if (fs.exists(mobPath)) {
677            fs.delete(mobPath, true);
678          }
679          restoreStoreFile(familyDir, snapshotRegionInfo, storeFile, createBackRefs);
680        } else {
681          String file = restoreStoreFile(familyDir, snapshotRegionInfo, storeFile, createBackRefs);
682          clonedFiles.add(new StoreFileInfo(conf, fs, new Path(familyDir, file), true));
683        }
684      }
685      // we don't need to track files under mobdir
686      if (!MobUtils.isMobRegionInfo(newRegionInfo)) {
687        Path regionPath = new Path(tableDir, newRegionInfo.getEncodedName());
688        HRegionFileSystem regionFS = (fs.exists(regionPath))
689          ? HRegionFileSystem.openRegionFromFileSystem(conf, fs, tableDir, newRegionInfo, false)
690          : HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, newRegionInfo);
691
692        Configuration sftConf = StoreUtils.createStoreConfiguration(conf, tableDesc,
693          tableDesc.getColumnFamily(familyFiles.getFamilyName().toByteArray()));
694        StoreFileTracker tracker =
695          StoreFileTrackerFactory.create(sftConf, true, StoreContext.getBuilder()
696            .withFamilyStoreDirectoryPath(familyDir).withRegionFileSystem(regionFS).build());
697        tracker.set(clonedFiles);
698      }
699    }
700
701  }
702
703  /**
704   * Clone region directory content from the snapshot info. Each region is encoded with the table
705   * name, so the cloned region will have a different region name. Instead of copying the hfiles a
706   * HFileLink is created.
707   * @param region {@link HRegion} cloned n
708   */
709  private void cloneRegion(final HRegion region, final RegionInfo snapshotRegionInfo,
710    final SnapshotRegionManifest manifest) throws IOException {
711    cloneRegion(region.getRegionInfo(), new Path(tableDir, region.getRegionInfo().getEncodedName()),
712      snapshotRegionInfo, manifest);
713  }
714
715  /**
716   * Create a new {@link HFileLink} to reference the store file.
717   * <p>
718   * The store file in the snapshot can be a simple hfile, an HFileLink or a reference.
719   * <ul>
720   * <li>hfile: abc -> table=region-abc
721   * <li>reference: abc.1234 -> table=region-abc.1234
722   * <li>hfilelink: table=region-hfile -> table=region-hfile
723   * </ul>
724   * @param familyDir     destination directory for the store file
725   * @param regionInfo    destination region info for the table
726   * @param createBackRef - Whether back reference should be created. Defaults to true.
727   * @param storeFile     store file name (can be a Reference, HFileLink or simple HFile)
728   */
729  private String restoreStoreFile(final Path familyDir, final RegionInfo regionInfo,
730    final SnapshotRegionManifest.StoreFile storeFile, final boolean createBackRef)
731    throws IOException {
732    String hfileName = storeFile.getName();
733    if (HFileLink.isHFileLink(hfileName)) {
734      return HFileLink.createFromHFileLink(conf, fs, familyDir, hfileName, createBackRef);
735    } else if (StoreFileInfo.isReference(hfileName)) {
736      return restoreReferenceFile(familyDir, regionInfo, storeFile);
737    } else {
738      return HFileLink.create(conf, fs, familyDir, regionInfo, hfileName, createBackRef);
739    }
740  }
741
742  /**
743   * Create a new {@link Reference} as copy of the source one.
744   * <p>
745   * <blockquote>
746   *
747   * <pre>
748   * The source table looks like:
749   *    1234/abc      (original file)
750   *    5678/abc.1234 (reference file)
751   *
752   * After the clone operation looks like:
753   *   wxyz/table=1234-abc
754   *   stuv/table=1234-abc.wxyz
755   *
756   * NOTE that the region name in the clone changes (md5 of regioninfo)
757   * and the reference should reflect that change.
758   * </pre>
759   *
760   * </blockquote>
761   * @param familyDir  destination directory for the store file
762   * @param regionInfo destination region info for the table
763   * @param storeFile  reference file name
764   */
765  private String restoreReferenceFile(final Path familyDir, final RegionInfo regionInfo,
766    final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
767    String hfileName = storeFile.getName();
768
769    // Extract the referred information (hfile name and parent region)
770    Path refPath =
771      StoreFileInfo
772        .getReferredToFile(
773          new Path(
774            new Path(
775              new Path(new Path(snapshotTable.getNamespaceAsString(),
776                snapshotTable.getQualifierAsString()), regionInfo.getEncodedName()),
777              familyDir.getName()),
778            hfileName));
779    String snapshotRegionName = refPath.getParent().getParent().getName();
780    String fileName = refPath.getName();
781
782    // The new reference should have the cloned region name as parent, if it is a clone.
783    String clonedRegionName = Bytes.toString(regionsMap.get(Bytes.toBytes(snapshotRegionName)));
784    if (clonedRegionName == null) clonedRegionName = snapshotRegionName;
785
786    // The output file should be a reference link table=snapshotRegion-fileName.clonedRegionName
787    Path linkPath = null;
788    String refLink = fileName;
789    if (!HFileLink.isHFileLink(fileName)) {
790      refLink = HFileLink.createHFileLinkName(snapshotTable, snapshotRegionName, fileName);
791      linkPath = new Path(familyDir,
792        HFileLink.createHFileLinkName(snapshotTable, regionInfo.getEncodedName(), hfileName));
793    }
794
795    Path outPath = new Path(familyDir, refLink + '.' + clonedRegionName);
796
797    // Create the new reference
798    if (storeFile.hasReference()) {
799      Reference reference = Reference.convert(storeFile.getReference());
800      reference.write(fs, outPath);
801    } else {
802      InputStream in;
803      if (linkPath != null) {
804        in = HFileLink.buildFromHFileLinkPattern(conf, linkPath).open(fs);
805      } else {
806        linkPath = new Path(new Path(
807          HRegion.getRegionDir(snapshotManifest.getSnapshotDir(), regionInfo.getEncodedName()),
808          familyDir.getName()), hfileName);
809        in = fs.open(linkPath);
810      }
811      OutputStream out = fs.create(outPath);
812      IOUtils.copyBytes(in, out, conf);
813    }
814
815    // Add the daughter region to the map
816    String regionName = Bytes.toString(regionsMap.get(regionInfo.getEncodedNameAsBytes()));
817    if (regionName == null) {
818      regionName = regionInfo.getEncodedName();
819    }
820    LOG.debug("Restore reference " + regionName + " to " + clonedRegionName);
821    synchronized (parentsMap) {
822      Pair<String, String> daughters = parentsMap.get(clonedRegionName);
823      if (daughters == null) {
824        // In case one side of the split is already compacted, regionName is put as both first and
825        // second of Pair
826        daughters = new Pair<>(regionName, regionName);
827        parentsMap.put(clonedRegionName, daughters);
828      } else if (!regionName.equals(daughters.getFirst())) {
829        daughters.setSecond(regionName);
830      }
831    }
832    return outPath.getName();
833  }
834
835  /**
836   * Create a new {@link RegionInfo} from the snapshot region info. Keep the same startKey, endKey,
837   * regionId and split information but change the table name.
838   * @param snapshotRegionInfo Info for region to clone.
839   * @return the new HRegion instance
840   */
841  public RegionInfo cloneRegionInfo(final RegionInfo snapshotRegionInfo) {
842    return cloneRegionInfo(tableDesc.getTableName(), snapshotRegionInfo);
843  }
844
845  public static RegionInfo cloneRegionInfo(TableName tableName, RegionInfo snapshotRegionInfo) {
846    return RegionInfoBuilder.newBuilder(tableName).setStartKey(snapshotRegionInfo.getStartKey())
847      .setEndKey(snapshotRegionInfo.getEndKey()).setSplit(snapshotRegionInfo.isSplit())
848      .setRegionId(snapshotRegionInfo.getRegionId()).setOffline(snapshotRegionInfo.isOffline())
849      .build();
850  }
851
852  /**
853   * @return the set of the regions contained in the table
854   */
855  private List<RegionInfo> getTableRegions() throws IOException {
856    LOG.debug("get table regions: " + tableDir);
857    FileStatus[] regionDirs =
858      CommonFSUtils.listStatus(fs, tableDir, new FSUtils.RegionDirFilter(fs));
859    if (regionDirs == null) {
860      return null;
861    }
862
863    List<RegionInfo> regions = new ArrayList<>(regionDirs.length);
864    for (int i = 0; i < regionDirs.length; ++i) {
865      RegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDirs[i].getPath());
866      regions.add(hri);
867    }
868    LOG.debug("found " + regions.size() + " regions for table="
869      + tableDesc.getTableName().getNameAsString());
870    return regions;
871  }
872
873  /**
874   * Copy the snapshot files for a snapshot scanner, discards meta changes. nnnnnn
875   */
876  public static RestoreMetaChanges copySnapshotForScanner(Configuration conf, FileSystem fs,
877    Path rootDir, Path restoreDir, String snapshotName) throws IOException {
878    // ensure that restore dir is not under root dir
879    if (!restoreDir.getFileSystem(conf).getUri().equals(rootDir.getFileSystem(conf).getUri())) {
880      throw new IllegalArgumentException(
881        "Filesystems for restore directory and HBase root " + "directory should be the same");
882    }
883    if (restoreDir.toUri().getPath().startsWith(rootDir.toUri().getPath() + "/")) {
884      throw new IllegalArgumentException("Restore directory cannot be a sub directory of HBase "
885        + "root directory. RootDir: " + rootDir + ", restoreDir: " + restoreDir);
886    }
887
888    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
889    SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
890    SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
891
892    MonitoredTask status = TaskMonitor.get()
893      .createStatus("Restoring  snapshot '" + snapshotName + "' to directory " + restoreDir);
894    ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher();
895
896    // we send createBackRefs=false so that restored hfiles do not create back reference links
897    // in the base hbase root dir.
898    RestoreSnapshotHelper helper = new RestoreSnapshotHelper(conf, fs, manifest,
899      manifest.getTableDescriptor(), restoreDir, monitor, status, false);
900    RestoreMetaChanges metaChanges = helper.restoreHdfsRegions(); // TODO: parallelize.
901
902    if (LOG.isDebugEnabled()) {
903      LOG.debug("Restored table dir:" + restoreDir);
904      CommonFSUtils.logFileSystemState(fs, restoreDir, LOG);
905    }
906    return metaChanges;
907  }
908
909  public static void restoreSnapshotAcl(SnapshotDescription snapshot, TableName newTableName,
910    Configuration conf) throws IOException {
911    if (snapshot.hasUsersAndPermissions() && snapshot.getUsersAndPermissions() != null) {
912      LOG.info("Restore snapshot acl to table. snapshot: " + snapshot + ", table: " + newTableName);
913      ListMultimap<String, Permission> perms =
914        ShadedAccessControlUtil.toUserTablePermissions(snapshot.getUsersAndPermissions());
915      try (Connection conn = ConnectionFactory.createConnection(conf)) {
916        for (Entry<String, Permission> e : perms.entries()) {
917          String user = e.getKey();
918          TablePermission tablePerm = (TablePermission) e.getValue();
919          AccessControlClient.grant(conn, newTableName, user, tablePerm.getFamily(),
920            tablePerm.getQualifier(), tablePerm.getActions());
921        }
922      } catch (Throwable e) {
923        throw new IOException("Grant acl into newly creatd table failed. snapshot: " + snapshot
924          + ", table: " + newTableName, e);
925      }
926    }
927  }
928}