001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.snapshot; 019 020import java.io.IOException; 021import java.io.InputStream; 022import java.io.OutputStream; 023import java.util.ArrayList; 024import java.util.Arrays; 025import java.util.Collections; 026import java.util.HashMap; 027import java.util.HashSet; 028import java.util.LinkedList; 029import java.util.List; 030import java.util.Map; 031import java.util.Map.Entry; 032import java.util.Set; 033import java.util.TreeMap; 034import java.util.concurrent.ThreadPoolExecutor; 035import org.apache.hadoop.conf.Configuration; 036import org.apache.hadoop.fs.FileStatus; 037import org.apache.hadoop.fs.FileSystem; 038import org.apache.hadoop.fs.Path; 039import org.apache.hadoop.hbase.MetaTableAccessor; 040import org.apache.hadoop.hbase.TableName; 041import org.apache.hadoop.hbase.backup.HFileArchiver; 042import org.apache.hadoop.hbase.client.Connection; 043import org.apache.hadoop.hbase.client.ConnectionFactory; 044import org.apache.hadoop.hbase.client.RegionInfo; 045import org.apache.hadoop.hbase.client.RegionInfoBuilder; 046import org.apache.hadoop.hbase.client.TableDescriptor; 047import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher; 048import org.apache.hadoop.hbase.io.HFileLink; 049import org.apache.hadoop.hbase.io.Reference; 050import org.apache.hadoop.hbase.mob.MobUtils; 051import org.apache.hadoop.hbase.monitoring.MonitoredTask; 052import org.apache.hadoop.hbase.monitoring.TaskMonitor; 053import org.apache.hadoop.hbase.regionserver.HRegion; 054import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; 055import org.apache.hadoop.hbase.regionserver.StoreContext; 056import org.apache.hadoop.hbase.regionserver.StoreFileInfo; 057import org.apache.hadoop.hbase.regionserver.StoreUtils; 058import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; 059import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; 060import org.apache.hadoop.hbase.security.access.AccessControlClient; 061import org.apache.hadoop.hbase.security.access.Permission; 062import org.apache.hadoop.hbase.security.access.ShadedAccessControlUtil; 063import org.apache.hadoop.hbase.security.access.TablePermission; 064import org.apache.hadoop.hbase.util.Bytes; 065import org.apache.hadoop.hbase.util.CommonFSUtils; 066import org.apache.hadoop.hbase.util.FSUtils; 067import org.apache.hadoop.hbase.util.ModifyRegionUtils; 068import org.apache.hadoop.hbase.util.Pair; 069import org.apache.hadoop.io.IOUtils; 070import org.apache.yetus.audience.InterfaceAudience; 071import org.slf4j.Logger; 072import org.slf4j.LoggerFactory; 073 074import org.apache.hbase.thirdparty.com.google.common.collect.ListMultimap; 075 076import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; 077import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription; 078import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest; 079 080/** 081 * Helper to Restore/Clone a Snapshot 082 * <p> 083 * The helper assumes that a table is already created, and by calling restore() the content present 084 * in the snapshot will be restored as the new content of the table. 085 * <p> 086 * Clone from Snapshot: If the target table is empty, the restore operation is just a "clone 087 * operation", where the only operations are: 088 * <ul> 089 * <li>for each region in the snapshot create a new region (note that the region will have a 090 * different name, since the encoding contains the table name) 091 * <li>for each file in the region create a new HFileLink to point to the original file. 092 * <li>restore the logs, if any 093 * </ul> 094 * <p> 095 * Restore from Snapshot: 096 * <ul> 097 * <li>for each region in the table verify which are available in the snapshot and which are not 098 * <ul> 099 * <li>if the region is not present in the snapshot, remove it. 100 * <li>if the region is present in the snapshot 101 * <ul> 102 * <li>for each file in the table region verify which are available in the snapshot 103 * <ul> 104 * <li>if the hfile is not present in the snapshot, remove it 105 * <li>if the hfile is present, keep it (nothing to do) 106 * </ul> 107 * <li>for each file in the snapshot region but not in the table 108 * <ul> 109 * <li>create a new HFileLink that point to the original file 110 * </ul> 111 * </ul> 112 * </ul> 113 * <li>for each region in the snapshot not present in the current table state 114 * <ul> 115 * <li>create a new region and for each file in the region create a new HFileLink (This is the same 116 * as the clone operation) 117 * </ul> 118 * <li>restore the logs, if any 119 * </ul> 120 */ 121@InterfaceAudience.Private 122public class RestoreSnapshotHelper { 123 private static final Logger LOG = LoggerFactory.getLogger(RestoreSnapshotHelper.class); 124 125 private final Map<byte[], byte[]> regionsMap = new TreeMap<>(Bytes.BYTES_COMPARATOR); 126 127 private final Map<String, Pair<String, String>> parentsMap = new HashMap<>(); 128 129 private final ForeignExceptionDispatcher monitor; 130 private final MonitoredTask status; 131 132 private final SnapshotManifest snapshotManifest; 133 private final SnapshotDescription snapshotDesc; 134 private final TableName snapshotTable; 135 136 private final TableDescriptor tableDesc; 137 private final Path rootDir; 138 private final Path tableDir; 139 140 private final Configuration conf; 141 private final FileSystem fs; 142 private final boolean createBackRefs; 143 144 public RestoreSnapshotHelper(final Configuration conf, final FileSystem fs, 145 final SnapshotManifest manifest, final TableDescriptor tableDescriptor, final Path rootDir, 146 final ForeignExceptionDispatcher monitor, final MonitoredTask status) { 147 this(conf, fs, manifest, tableDescriptor, rootDir, monitor, status, true); 148 } 149 150 public RestoreSnapshotHelper(final Configuration conf, final FileSystem fs, 151 final SnapshotManifest manifest, final TableDescriptor tableDescriptor, final Path rootDir, 152 final ForeignExceptionDispatcher monitor, final MonitoredTask status, 153 final boolean createBackRefs) { 154 this.fs = fs; 155 this.conf = conf; 156 this.snapshotManifest = manifest; 157 this.snapshotDesc = manifest.getSnapshotDescription(); 158 this.snapshotTable = TableName.valueOf(snapshotDesc.getTable()); 159 this.tableDesc = tableDescriptor; 160 this.rootDir = rootDir; 161 this.tableDir = CommonFSUtils.getTableDir(rootDir, tableDesc.getTableName()); 162 this.monitor = monitor; 163 this.status = status; 164 this.createBackRefs = createBackRefs; 165 } 166 167 /** 168 * Restore the on-disk table to a specified snapshot state. 169 * @return the set of regions touched by the restore operation 170 */ 171 public RestoreMetaChanges restoreHdfsRegions() throws IOException { 172 ThreadPoolExecutor exec = SnapshotManifest.createExecutor(conf, "RestoreSnapshot"); 173 try { 174 return restoreHdfsRegions(exec); 175 } finally { 176 exec.shutdown(); 177 } 178 } 179 180 private RestoreMetaChanges restoreHdfsRegions(final ThreadPoolExecutor exec) throws IOException { 181 LOG.info("starting restore table regions using snapshot=" + snapshotDesc); 182 183 Map<String, SnapshotRegionManifest> regionManifests = snapshotManifest.getRegionManifestsMap(); 184 if (regionManifests == null) { 185 LOG.warn("Nothing to restore. Snapshot " + snapshotDesc + " looks empty"); 186 return null; 187 } 188 189 RestoreMetaChanges metaChanges = new RestoreMetaChanges(tableDesc, parentsMap); 190 191 // Take a copy of the manifest.keySet() since we are going to modify 192 // this instance, by removing the regions already present in the restore dir. 193 Set<String> regionNames = new HashSet<>(regionManifests.keySet()); 194 195 List<RegionInfo> tableRegions = getTableRegions(); 196 197 RegionInfo mobRegion = 198 MobUtils.getMobRegionInfo(snapshotManifest.getTableDescriptor().getTableName()); 199 if (tableRegions != null) { 200 // restore the mob region in case 201 if (regionNames.contains(mobRegion.getEncodedName())) { 202 monitor.rethrowException(); 203 status.setStatus("Restoring mob region..."); 204 List<RegionInfo> mobRegions = new ArrayList<>(1); 205 mobRegions.add(mobRegion); 206 restoreHdfsMobRegions(exec, regionManifests, mobRegions); 207 regionNames.remove(mobRegion.getEncodedName()); 208 status.setStatus("Finished restoring mob region."); 209 } 210 } 211 if (regionNames.contains(mobRegion.getEncodedName())) { 212 // add the mob region 213 monitor.rethrowException(); 214 status.setStatus("Cloning mob region..."); 215 cloneHdfsMobRegion(regionManifests, mobRegion); 216 regionNames.remove(mobRegion.getEncodedName()); 217 status.setStatus("Finished cloning mob region."); 218 } 219 220 // Identify which region are still available and which not. 221 // NOTE: we rely upon the region name as: "table name, start key, end key" 222 if (tableRegions != null) { 223 monitor.rethrowException(); 224 for (RegionInfo regionInfo : tableRegions) { 225 String regionName = regionInfo.getEncodedName(); 226 if (regionNames.contains(regionName)) { 227 LOG.info("region to restore: " + regionName); 228 regionNames.remove(regionName); 229 metaChanges.addRegionToRestore( 230 ProtobufUtil.toRegionInfo(regionManifests.get(regionName).getRegionInfo())); 231 } else { 232 LOG.info("region to remove: " + regionName); 233 metaChanges.addRegionToRemove(regionInfo); 234 } 235 } 236 } 237 238 // Regions to Add: present in the snapshot but not in the current table 239 List<RegionInfo> regionsToAdd = new ArrayList<>(regionNames.size()); 240 if (regionNames.size() > 0) { 241 monitor.rethrowException(); 242 for (String regionName : regionNames) { 243 LOG.info("region to add: " + regionName); 244 regionsToAdd 245 .add(ProtobufUtil.toRegionInfo(regionManifests.get(regionName).getRegionInfo())); 246 } 247 } 248 249 // Create new regions cloning from the snapshot 250 // HBASE-19980: We need to call cloneHdfsRegions() before restoreHdfsRegions() because 251 // regionsMap is constructed in cloneHdfsRegions() and it can be used in restoreHdfsRegions(). 252 monitor.rethrowException(); 253 status.setStatus("Cloning regions..."); 254 RegionInfo[] clonedRegions = cloneHdfsRegions(exec, regionManifests, regionsToAdd); 255 metaChanges.setNewRegions(clonedRegions); 256 status.setStatus("Finished cloning regions."); 257 258 // Restore regions using the snapshot data 259 monitor.rethrowException(); 260 status.setStatus("Restoring table regions..."); 261 restoreHdfsRegions(exec, regionManifests, metaChanges.getRegionsToRestore()); 262 status.setStatus("Finished restoring all table regions."); 263 264 // Remove regions from the current table 265 monitor.rethrowException(); 266 status.setStatus("Starting to delete excess regions from table"); 267 removeHdfsRegions(exec, metaChanges.getRegionsToRemove()); 268 status.setStatus("Finished deleting excess regions from table."); 269 270 LOG.info("finishing restore table regions using snapshot=" + snapshotDesc); 271 272 return metaChanges; 273 } 274 275 /** 276 * Describe the set of operations needed to update hbase:meta after restore. 277 */ 278 public static class RestoreMetaChanges { 279 private final Map<String, Pair<String, String>> parentsMap; 280 private final TableDescriptor htd; 281 282 private List<RegionInfo> regionsToRestore = null; 283 private List<RegionInfo> regionsToRemove = null; 284 private List<RegionInfo> regionsToAdd = null; 285 286 public RestoreMetaChanges(TableDescriptor htd, Map<String, Pair<String, String>> parentsMap) { 287 this.parentsMap = parentsMap; 288 this.htd = htd; 289 } 290 291 public TableDescriptor getTableDescriptor() { 292 return htd; 293 } 294 295 /** 296 * Returns the map of parent-children_pair. 297 * @return the map 298 */ 299 public Map<String, Pair<String, String>> getParentToChildrenPairMap() { 300 return this.parentsMap; 301 } 302 303 /** Returns true if there're new regions */ 304 public boolean hasRegionsToAdd() { 305 return this.regionsToAdd != null && this.regionsToAdd.size() > 0; 306 } 307 308 /** 309 * Returns the list of new regions added during the on-disk restore. The caller is responsible 310 * to add the regions to META. e.g MetaTableAccessor.addRegionsToMeta(...) 311 * @return the list of regions to add to META 312 */ 313 public List<RegionInfo> getRegionsToAdd() { 314 return this.regionsToAdd; 315 } 316 317 /** Returns true if there're regions to restore */ 318 public boolean hasRegionsToRestore() { 319 return this.regionsToRestore != null && this.regionsToRestore.size() > 0; 320 } 321 322 /** 323 * Returns the list of 'restored regions' during the on-disk restore. The caller is responsible 324 * to add the regions to hbase:meta if not present. 325 * @return the list of regions restored 326 */ 327 public List<RegionInfo> getRegionsToRestore() { 328 return this.regionsToRestore; 329 } 330 331 /** Returns true if there're regions to remove */ 332 public boolean hasRegionsToRemove() { 333 return this.regionsToRemove != null && this.regionsToRemove.size() > 0; 334 } 335 336 /** 337 * Returns the list of regions removed during the on-disk restore. The caller is responsible to 338 * remove the regions from META. e.g. MetaTableAccessor.deleteRegions(...) 339 * @return the list of regions to remove from META 340 */ 341 public List<RegionInfo> getRegionsToRemove() { 342 return this.regionsToRemove; 343 } 344 345 void setNewRegions(final RegionInfo[] hris) { 346 if (hris != null) { 347 regionsToAdd = Arrays.asList(hris); 348 } else { 349 regionsToAdd = null; 350 } 351 } 352 353 void addRegionToRemove(final RegionInfo hri) { 354 if (regionsToRemove == null) { 355 regionsToRemove = new LinkedList<>(); 356 } 357 regionsToRemove.add(hri); 358 } 359 360 void addRegionToRestore(final RegionInfo hri) { 361 if (regionsToRestore == null) { 362 regionsToRestore = new LinkedList<>(); 363 } 364 regionsToRestore.add(hri); 365 } 366 367 public void updateMetaParentRegions(Connection connection, final List<RegionInfo> regionInfos) 368 throws IOException { 369 if (regionInfos == null || parentsMap.isEmpty()) return; 370 371 // Extract region names and offlined regions 372 Map<String, RegionInfo> regionsByName = new HashMap<>(regionInfos.size()); 373 List<RegionInfo> parentRegions = new LinkedList<>(); 374 for (RegionInfo regionInfo : regionInfos) { 375 if (regionInfo.isSplitParent()) { 376 parentRegions.add(regionInfo); 377 } else { 378 regionsByName.put(regionInfo.getEncodedName(), regionInfo); 379 } 380 } 381 382 // Update Offline parents 383 for (RegionInfo regionInfo : parentRegions) { 384 Pair<String, String> daughters = parentsMap.get(regionInfo.getEncodedName()); 385 if (daughters == null) { 386 // The snapshot contains an unreferenced region. 387 // It will be removed by the CatalogJanitor. 388 LOG.warn("Skip update of unreferenced offline parent: " + regionInfo); 389 continue; 390 } 391 392 // One side of the split is already compacted 393 if (daughters.getSecond() == null) { 394 daughters.setSecond(daughters.getFirst()); 395 } 396 397 LOG.debug("Update splits parent " + regionInfo.getEncodedName() + " -> " + daughters); 398 MetaTableAccessor.addSplitsToParent(connection, regionInfo, 399 regionsByName.get(daughters.getFirst()), regionsByName.get(daughters.getSecond())); 400 } 401 } 402 } 403 404 /** 405 * Remove specified regions from the file-system, using the archiver. 406 */ 407 private void removeHdfsRegions(final ThreadPoolExecutor exec, final List<RegionInfo> regions) 408 throws IOException { 409 if (regions == null || regions.isEmpty()) return; 410 ModifyRegionUtils.editRegions(exec, regions, new ModifyRegionUtils.RegionEditTask() { 411 @Override 412 public void editRegion(final RegionInfo hri) throws IOException { 413 HFileArchiver.archiveRegion(conf, fs, hri); 414 } 415 }); 416 } 417 418 /** 419 * Restore specified regions by restoring content to the snapshot state. 420 */ 421 private void restoreHdfsRegions(final ThreadPoolExecutor exec, 422 final Map<String, SnapshotRegionManifest> regionManifests, final List<RegionInfo> regions) 423 throws IOException { 424 if (regions == null || regions.isEmpty()) return; 425 ModifyRegionUtils.editRegions(exec, regions, new ModifyRegionUtils.RegionEditTask() { 426 @Override 427 public void editRegion(final RegionInfo hri) throws IOException { 428 restoreRegion(hri, regionManifests.get(hri.getEncodedName())); 429 } 430 }); 431 } 432 433 /** 434 * Restore specified mob regions by restoring content to the snapshot state. 435 */ 436 private void restoreHdfsMobRegions(final ThreadPoolExecutor exec, 437 final Map<String, SnapshotRegionManifest> regionManifests, final List<RegionInfo> regions) 438 throws IOException { 439 if (regions == null || regions.isEmpty()) return; 440 ModifyRegionUtils.editRegions(exec, regions, new ModifyRegionUtils.RegionEditTask() { 441 @Override 442 public void editRegion(final RegionInfo hri) throws IOException { 443 restoreMobRegion(hri, regionManifests.get(hri.getEncodedName())); 444 } 445 }); 446 } 447 448 private Map<String, List<SnapshotRegionManifest.StoreFile>> 449 getRegionHFileReferences(final SnapshotRegionManifest manifest) { 450 Map<String, List<SnapshotRegionManifest.StoreFile>> familyMap = 451 new HashMap<>(manifest.getFamilyFilesCount()); 452 for (SnapshotRegionManifest.FamilyFiles familyFiles : manifest.getFamilyFilesList()) { 453 familyMap.put(familyFiles.getFamilyName().toStringUtf8(), 454 new ArrayList<>(familyFiles.getStoreFilesList())); 455 } 456 return familyMap; 457 } 458 459 /** 460 * Restore region by removing files not in the snapshot and adding the missing ones from the 461 * snapshot. 462 */ 463 private void restoreRegion(final RegionInfo regionInfo, 464 final SnapshotRegionManifest regionManifest) throws IOException { 465 restoreRegion(regionInfo, regionManifest, new Path(tableDir, regionInfo.getEncodedName())); 466 } 467 468 /** 469 * Restore mob region by removing files not in the snapshot and adding the missing ones from the 470 * snapshot. 471 */ 472 private void restoreMobRegion(final RegionInfo regionInfo, 473 final SnapshotRegionManifest regionManifest) throws IOException { 474 if (regionManifest == null) { 475 return; 476 } 477 restoreRegion(regionInfo, regionManifest, 478 MobUtils.getMobRegionPath(conf, tableDesc.getTableName())); 479 } 480 481 /** 482 * Restore region by removing files not in the snapshot and adding the missing ones from the 483 * snapshot. 484 */ 485 private void restoreRegion(final RegionInfo regionInfo, 486 final SnapshotRegionManifest regionManifest, Path regionDir) throws IOException { 487 Map<String, List<SnapshotRegionManifest.StoreFile>> snapshotFiles = 488 getRegionHFileReferences(regionManifest); 489 490 String tableName = tableDesc.getTableName().getNameAsString(); 491 final String snapshotName = snapshotDesc.getName(); 492 493 Path regionPath = new Path(tableDir, regionInfo.getEncodedName()); 494 HRegionFileSystem regionFS = (fs.exists(regionPath)) 495 ? HRegionFileSystem.openRegionFromFileSystem(conf, fs, tableDir, regionInfo, false) 496 : HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, regionInfo); 497 498 // Restore families present in the table 499 for (Path familyDir : FSUtils.getFamilyDirs(fs, regionDir)) { 500 byte[] family = Bytes.toBytes(familyDir.getName()); 501 502 Set<String> familyFiles = getTableRegionFamilyFiles(familyDir); 503 List<SnapshotRegionManifest.StoreFile> snapshotFamilyFiles = 504 snapshotFiles.remove(familyDir.getName()); 505 List<StoreFileInfo> filesToTrack = new ArrayList<>(); 506 if (snapshotFamilyFiles != null) { 507 List<SnapshotRegionManifest.StoreFile> hfilesToAdd = new ArrayList<>(); 508 for (SnapshotRegionManifest.StoreFile storeFile : snapshotFamilyFiles) { 509 if (familyFiles.contains(storeFile.getName())) { 510 // HFile already present 511 familyFiles.remove(storeFile.getName()); 512 // no need to restore already present files, but we need to add those to tracker 513 filesToTrack 514 .add(new StoreFileInfo(conf, fs, new Path(familyDir, storeFile.getName()), true)); 515 } else { 516 // HFile missing 517 hfilesToAdd.add(storeFile); 518 } 519 } 520 521 // Remove hfiles not present in the snapshot 522 for (String hfileName : familyFiles) { 523 Path hfile = new Path(familyDir, hfileName); 524 if (!fs.getFileStatus(hfile).isDirectory()) { 525 LOG.trace("Removing HFile=" + hfileName + " not present in snapshot=" + snapshotName 526 + " from region=" + regionInfo.getEncodedName() + " table=" + tableName); 527 HFileArchiver.archiveStoreFile(conf, fs, regionInfo, tableDir, family, hfile); 528 } 529 } 530 531 // Restore Missing files 532 for (SnapshotRegionManifest.StoreFile storeFile : hfilesToAdd) { 533 LOG.debug("Restoring missing HFileLink " + storeFile.getName() + " of snapshot=" 534 + snapshotName + " to region=" + regionInfo.getEncodedName() + " table=" + tableName); 535 String fileName = restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs); 536 // mark the reference file to be added to tracker 537 filesToTrack.add(new StoreFileInfo(conf, fs, new Path(familyDir, fileName), true)); 538 } 539 } else { 540 // Family doesn't exists in the snapshot 541 LOG.trace("Removing family=" + Bytes.toString(family) + " in snapshot=" + snapshotName 542 + " from region=" + regionInfo.getEncodedName() + " table=" + tableName); 543 HFileArchiver.archiveFamilyByFamilyDir(fs, conf, regionInfo, familyDir, family); 544 fs.delete(familyDir, true); 545 } 546 547 StoreFileTracker tracker = 548 StoreFileTrackerFactory.create(conf, true, StoreContext.getBuilder() 549 .withFamilyStoreDirectoryPath(familyDir).withRegionFileSystem(regionFS).build()); 550 551 // simply reset list of tracked files with the matching files 552 // and the extra one present in the snapshot 553 tracker.set(filesToTrack); 554 } 555 556 // Add families not present in the table 557 for (Map.Entry<String, List<SnapshotRegionManifest.StoreFile>> familyEntry : snapshotFiles 558 .entrySet()) { 559 Path familyDir = new Path(regionDir, familyEntry.getKey()); 560 StoreFileTracker tracker = 561 StoreFileTrackerFactory.create(conf, true, StoreContext.getBuilder() 562 .withFamilyStoreDirectoryPath(familyDir).withRegionFileSystem(regionFS).build()); 563 List<StoreFileInfo> files = new ArrayList<>(); 564 if (!fs.mkdirs(familyDir)) { 565 throw new IOException("Unable to create familyDir=" + familyDir); 566 } 567 568 for (SnapshotRegionManifest.StoreFile storeFile : familyEntry.getValue()) { 569 LOG.trace("Adding HFileLink (Not present in the table) " + storeFile.getName() 570 + " of snapshot " + snapshotName + " to table=" + tableName); 571 String fileName = restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs); 572 files.add(new StoreFileInfo(conf, fs, new Path(familyDir, fileName), true)); 573 } 574 tracker.set(files); 575 } 576 } 577 578 /** Returns The set of files in the specified family directory. */ 579 private Set<String> getTableRegionFamilyFiles(final Path familyDir) throws IOException { 580 FileStatus[] hfiles = CommonFSUtils.listStatus(fs, familyDir); 581 if (hfiles == null) { 582 return Collections.emptySet(); 583 } 584 585 Set<String> familyFiles = new HashSet<>(hfiles.length); 586 for (int i = 0; i < hfiles.length; ++i) { 587 String hfileName = hfiles[i].getPath().getName(); 588 familyFiles.add(hfileName); 589 } 590 591 return familyFiles; 592 } 593 594 /** 595 * Clone specified regions. For each region create a new region and create a HFileLink for each 596 * hfile. 597 */ 598 private RegionInfo[] cloneHdfsRegions(final ThreadPoolExecutor exec, 599 final Map<String, SnapshotRegionManifest> regionManifests, final List<RegionInfo> regions) 600 throws IOException { 601 if (regions == null || regions.isEmpty()) return null; 602 603 final Map<String, RegionInfo> snapshotRegions = new HashMap<>(regions.size()); 604 final String snapshotName = snapshotDesc.getName(); 605 606 // clone region info (change embedded tableName with the new one) 607 RegionInfo[] clonedRegionsInfo = new RegionInfo[regions.size()]; 608 for (int i = 0; i < clonedRegionsInfo.length; ++i) { 609 // clone the region info from the snapshot region info 610 RegionInfo snapshotRegionInfo = regions.get(i); 611 clonedRegionsInfo[i] = cloneRegionInfo(snapshotRegionInfo); 612 613 // add the region name mapping between snapshot and cloned 614 String snapshotRegionName = snapshotRegionInfo.getEncodedName(); 615 String clonedRegionName = clonedRegionsInfo[i].getEncodedName(); 616 regionsMap.put(Bytes.toBytes(snapshotRegionName), Bytes.toBytes(clonedRegionName)); 617 LOG.info("clone region=" + snapshotRegionName + " as " + clonedRegionName + " in snapshot " 618 + snapshotName); 619 620 // Add mapping between cloned region name and snapshot region info 621 snapshotRegions.put(clonedRegionName, snapshotRegionInfo); 622 } 623 624 // create the regions on disk 625 ModifyRegionUtils.createRegions(exec, conf, rootDir, tableDesc, clonedRegionsInfo, 626 new ModifyRegionUtils.RegionFillTask() { 627 @Override 628 public void fillRegion(final HRegion region) throws IOException { 629 RegionInfo snapshotHri = snapshotRegions.get(region.getRegionInfo().getEncodedName()); 630 cloneRegion(region, snapshotHri, regionManifests.get(snapshotHri.getEncodedName())); 631 } 632 }); 633 634 return clonedRegionsInfo; 635 } 636 637 /** 638 * Clone the mob region. For the region create a new region and create a HFileLink for each hfile. 639 */ 640 private void cloneHdfsMobRegion(final Map<String, SnapshotRegionManifest> regionManifests, 641 final RegionInfo region) throws IOException { 642 // clone region info (change embedded tableName with the new one) 643 Path clonedRegionPath = MobUtils.getMobRegionPath(rootDir, tableDesc.getTableName()); 644 cloneRegion(MobUtils.getMobRegionInfo(tableDesc.getTableName()), clonedRegionPath, region, 645 regionManifests.get(region.getEncodedName())); 646 } 647 648 /** 649 * Clone region directory content from the snapshot info. Each region is encoded with the table 650 * name, so the cloned region will have a different region name. Instead of copying the hfiles a 651 * HFileLink is created. 652 * @param regionDir {@link Path} cloned dir n 653 */ 654 private void cloneRegion(final RegionInfo newRegionInfo, final Path regionDir, 655 final RegionInfo snapshotRegionInfo, final SnapshotRegionManifest manifest) throws IOException { 656 final String tableName = tableDesc.getTableName().getNameAsString(); 657 final String snapshotName = snapshotDesc.getName(); 658 for (SnapshotRegionManifest.FamilyFiles familyFiles : manifest.getFamilyFilesList()) { 659 Path familyDir = new Path(regionDir, familyFiles.getFamilyName().toStringUtf8()); 660 List<StoreFileInfo> clonedFiles = new ArrayList<>(); 661 for (SnapshotRegionManifest.StoreFile storeFile : familyFiles.getStoreFilesList()) { 662 LOG.info("Adding HFileLink " + storeFile.getName() + " from cloned region " + "in snapshot " 663 + snapshotName + " to table=" + tableName); 664 if (MobUtils.isMobRegionInfo(newRegionInfo)) { 665 String mobFileName = 666 HFileLink.createHFileLinkName(snapshotRegionInfo, storeFile.getName()); 667 Path mobPath = new Path(familyDir, mobFileName); 668 if (fs.exists(mobPath)) { 669 fs.delete(mobPath, true); 670 } 671 restoreStoreFile(familyDir, snapshotRegionInfo, storeFile, createBackRefs); 672 } else { 673 String file = restoreStoreFile(familyDir, snapshotRegionInfo, storeFile, createBackRefs); 674 clonedFiles.add(new StoreFileInfo(conf, fs, new Path(familyDir, file), true)); 675 } 676 } 677 // we don't need to track files under mobdir 678 if (!MobUtils.isMobRegionInfo(newRegionInfo)) { 679 Path regionPath = new Path(tableDir, newRegionInfo.getEncodedName()); 680 HRegionFileSystem regionFS = (fs.exists(regionPath)) 681 ? HRegionFileSystem.openRegionFromFileSystem(conf, fs, tableDir, newRegionInfo, false) 682 : HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, newRegionInfo); 683 684 Configuration sftConf = StoreUtils.createStoreConfiguration(conf, tableDesc, 685 tableDesc.getColumnFamily(familyFiles.getFamilyName().toByteArray())); 686 StoreFileTracker tracker = 687 StoreFileTrackerFactory.create(sftConf, true, StoreContext.getBuilder() 688 .withFamilyStoreDirectoryPath(familyDir).withRegionFileSystem(regionFS).build()); 689 tracker.set(clonedFiles); 690 } 691 } 692 693 } 694 695 /** 696 * Clone region directory content from the snapshot info. Each region is encoded with the table 697 * name, so the cloned region will have a different region name. Instead of copying the hfiles a 698 * HFileLink is created. 699 * @param region {@link HRegion} cloned n 700 */ 701 private void cloneRegion(final HRegion region, final RegionInfo snapshotRegionInfo, 702 final SnapshotRegionManifest manifest) throws IOException { 703 cloneRegion(region.getRegionInfo(), new Path(tableDir, region.getRegionInfo().getEncodedName()), 704 snapshotRegionInfo, manifest); 705 } 706 707 /** 708 * Create a new {@link HFileLink} to reference the store file. 709 * <p> 710 * The store file in the snapshot can be a simple hfile, an HFileLink or a reference. 711 * <ul> 712 * <li>hfile: abc -> table=region-abc 713 * <li>reference: abc.1234 -> table=region-abc.1234 714 * <li>hfilelink: table=region-hfile -> table=region-hfile 715 * </ul> 716 * @param familyDir destination directory for the store file 717 * @param regionInfo destination region info for the table 718 * @param createBackRef - Whether back reference should be created. Defaults to true. 719 * @param storeFile store file name (can be a Reference, HFileLink or simple HFile) 720 */ 721 private String restoreStoreFile(final Path familyDir, final RegionInfo regionInfo, 722 final SnapshotRegionManifest.StoreFile storeFile, final boolean createBackRef) 723 throws IOException { 724 String hfileName = storeFile.getName(); 725 if (HFileLink.isHFileLink(hfileName)) { 726 return HFileLink.createFromHFileLink(conf, fs, familyDir, hfileName, createBackRef); 727 } else if (StoreFileInfo.isReference(hfileName)) { 728 return restoreReferenceFile(familyDir, regionInfo, storeFile); 729 } else { 730 return HFileLink.create(conf, fs, familyDir, regionInfo, hfileName, createBackRef); 731 } 732 } 733 734 /** 735 * Create a new {@link Reference} as copy of the source one. 736 * <p> 737 * <blockquote> 738 * 739 * <pre> 740 * The source table looks like: 741 * 1234/abc (original file) 742 * 5678/abc.1234 (reference file) 743 * 744 * After the clone operation looks like: 745 * wxyz/table=1234-abc 746 * stuv/table=1234-abc.wxyz 747 * 748 * NOTE that the region name in the clone changes (md5 of regioninfo) 749 * and the reference should reflect that change. 750 * </pre> 751 * 752 * </blockquote> 753 * @param familyDir destination directory for the store file 754 * @param regionInfo destination region info for the table 755 * @param storeFile reference file name 756 */ 757 private String restoreReferenceFile(final Path familyDir, final RegionInfo regionInfo, 758 final SnapshotRegionManifest.StoreFile storeFile) throws IOException { 759 String hfileName = storeFile.getName(); 760 761 // Extract the referred information (hfile name and parent region) 762 Path refPath = 763 StoreFileInfo 764 .getReferredToFile( 765 new Path( 766 new Path( 767 new Path(new Path(snapshotTable.getNamespaceAsString(), 768 snapshotTable.getQualifierAsString()), regionInfo.getEncodedName()), 769 familyDir.getName()), 770 hfileName)); 771 String snapshotRegionName = refPath.getParent().getParent().getName(); 772 String fileName = refPath.getName(); 773 774 // The new reference should have the cloned region name as parent, if it is a clone. 775 String clonedRegionName = Bytes.toString(regionsMap.get(Bytes.toBytes(snapshotRegionName))); 776 if (clonedRegionName == null) clonedRegionName = snapshotRegionName; 777 778 // The output file should be a reference link table=snapshotRegion-fileName.clonedRegionName 779 Path linkPath = null; 780 String refLink = fileName; 781 if (!HFileLink.isHFileLink(fileName)) { 782 refLink = HFileLink.createHFileLinkName(snapshotTable, snapshotRegionName, fileName); 783 linkPath = new Path(familyDir, 784 HFileLink.createHFileLinkName(snapshotTable, regionInfo.getEncodedName(), hfileName)); 785 } 786 787 Path outPath = new Path(familyDir, refLink + '.' + clonedRegionName); 788 789 // Create the new reference 790 if (storeFile.hasReference()) { 791 Reference reference = Reference.convert(storeFile.getReference()); 792 reference.write(fs, outPath); 793 } else { 794 InputStream in; 795 if (linkPath != null) { 796 in = HFileLink.buildFromHFileLinkPattern(conf, linkPath).open(fs); 797 } else { 798 linkPath = new Path(new Path( 799 HRegion.getRegionDir(snapshotManifest.getSnapshotDir(), regionInfo.getEncodedName()), 800 familyDir.getName()), hfileName); 801 in = fs.open(linkPath); 802 } 803 OutputStream out = fs.create(outPath); 804 IOUtils.copyBytes(in, out, conf); 805 } 806 807 // Add the daughter region to the map 808 String regionName = Bytes.toString(regionsMap.get(regionInfo.getEncodedNameAsBytes())); 809 if (regionName == null) { 810 regionName = regionInfo.getEncodedName(); 811 } 812 LOG.debug("Restore reference " + regionName + " to " + clonedRegionName); 813 synchronized (parentsMap) { 814 Pair<String, String> daughters = parentsMap.get(clonedRegionName); 815 if (daughters == null) { 816 // In case one side of the split is already compacted, regionName is put as both first and 817 // second of Pair 818 daughters = new Pair<>(regionName, regionName); 819 parentsMap.put(clonedRegionName, daughters); 820 } else if (!regionName.equals(daughters.getFirst())) { 821 daughters.setSecond(regionName); 822 } 823 } 824 return outPath.getName(); 825 } 826 827 /** 828 * Create a new {@link RegionInfo} from the snapshot region info. Keep the same startKey, endKey, 829 * regionId and split information but change the table name. 830 * @param snapshotRegionInfo Info for region to clone. 831 * @return the new HRegion instance 832 */ 833 public RegionInfo cloneRegionInfo(final RegionInfo snapshotRegionInfo) { 834 return cloneRegionInfo(tableDesc.getTableName(), snapshotRegionInfo); 835 } 836 837 public static RegionInfo cloneRegionInfo(TableName tableName, RegionInfo snapshotRegionInfo) { 838 return RegionInfoBuilder.newBuilder(tableName).setStartKey(snapshotRegionInfo.getStartKey()) 839 .setEndKey(snapshotRegionInfo.getEndKey()).setSplit(snapshotRegionInfo.isSplit()) 840 .setRegionId(snapshotRegionInfo.getRegionId()).setOffline(snapshotRegionInfo.isOffline()) 841 .build(); 842 } 843 844 /** Returns the set of the regions contained in the table */ 845 private List<RegionInfo> getTableRegions() throws IOException { 846 LOG.debug("get table regions: " + tableDir); 847 FileStatus[] regionDirs = 848 CommonFSUtils.listStatus(fs, tableDir, new FSUtils.RegionDirFilter(fs)); 849 if (regionDirs == null) { 850 return null; 851 } 852 853 List<RegionInfo> regions = new ArrayList<>(regionDirs.length); 854 for (int i = 0; i < regionDirs.length; ++i) { 855 RegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDirs[i].getPath()); 856 regions.add(hri); 857 } 858 LOG.debug("found " + regions.size() + " regions for table=" 859 + tableDesc.getTableName().getNameAsString()); 860 return regions; 861 } 862 863 /** 864 * Copy the snapshot files for a snapshot scanner, discards meta changes. nnnnnn 865 */ 866 public static RestoreMetaChanges copySnapshotForScanner(Configuration conf, FileSystem fs, 867 Path rootDir, Path restoreDir, String snapshotName) throws IOException { 868 // ensure that restore dir is not under root dir 869 if (!restoreDir.getFileSystem(conf).getUri().equals(rootDir.getFileSystem(conf).getUri())) { 870 throw new IllegalArgumentException( 871 "Filesystems for restore directory and HBase root " + "directory should be the same"); 872 } 873 if (restoreDir.toUri().getPath().startsWith(rootDir.toUri().getPath() + "/")) { 874 throw new IllegalArgumentException("Restore directory cannot be a sub directory of HBase " 875 + "root directory. RootDir: " + rootDir + ", restoreDir: " + restoreDir); 876 } 877 878 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); 879 SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); 880 SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc); 881 882 MonitoredTask status = TaskMonitor.get() 883 .createStatus("Restoring snapshot '" + snapshotName + "' to directory " + restoreDir); 884 ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(); 885 886 // we send createBackRefs=false so that restored hfiles do not create back reference links 887 // in the base hbase root dir. 888 RestoreSnapshotHelper helper = new RestoreSnapshotHelper(conf, fs, manifest, 889 manifest.getTableDescriptor(), restoreDir, monitor, status, false); 890 RestoreMetaChanges metaChanges = helper.restoreHdfsRegions(); // TODO: parallelize. 891 892 if (LOG.isDebugEnabled()) { 893 LOG.debug("Restored table dir:" + restoreDir); 894 CommonFSUtils.logFileSystemState(fs, restoreDir, LOG); 895 } 896 return metaChanges; 897 } 898 899 public static void restoreSnapshotAcl(SnapshotDescription snapshot, TableName newTableName, 900 Configuration conf) throws IOException { 901 if (snapshot.hasUsersAndPermissions() && snapshot.getUsersAndPermissions() != null) { 902 LOG.info("Restore snapshot acl to table. snapshot: " + snapshot + ", table: " + newTableName); 903 ListMultimap<String, Permission> perms = 904 ShadedAccessControlUtil.toUserTablePermissions(snapshot.getUsersAndPermissions()); 905 try (Connection conn = ConnectionFactory.createConnection(conf)) { 906 for (Entry<String, Permission> e : perms.entries()) { 907 String user = e.getKey(); 908 TablePermission tablePerm = (TablePermission) e.getValue(); 909 AccessControlClient.grant(conn, newTableName, user, tablePerm.getFamily(), 910 tablePerm.getQualifier(), tablePerm.getActions()); 911 } 912 } catch (Throwable e) { 913 throw new IOException("Grant acl into newly creatd table failed. snapshot: " + snapshot 914 + ", table: " + newTableName, e); 915 } 916 } 917 } 918}