001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 019package org.apache.hadoop.hbase.master.procedure; 020 021import java.io.IOException; 022import java.util.ArrayList; 023import java.util.HashMap; 024import java.util.Iterator; 025import java.util.List; 026import java.util.Map; 027import org.apache.hadoop.conf.Configuration; 028import org.apache.hadoop.fs.FileSystem; 029import org.apache.hadoop.fs.Path; 030import org.apache.hadoop.hbase.TableExistsException; 031import org.apache.hadoop.hbase.TableName; 032import org.apache.hadoop.hbase.client.RegionInfo; 033import org.apache.hadoop.hbase.client.TableDescriptor; 034import org.apache.hadoop.hbase.client.TableDescriptorBuilder; 035import org.apache.hadoop.hbase.errorhandling.ForeignException; 036import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher; 037import org.apache.hadoop.hbase.master.MasterCoprocessorHost; 038import org.apache.hadoop.hbase.master.MasterFileSystem; 039import org.apache.hadoop.hbase.master.MetricsSnapshot; 040import org.apache.hadoop.hbase.master.RegionState; 041import org.apache.hadoop.hbase.master.assignment.AssignmentManager; 042import org.apache.hadoop.hbase.master.procedure.CreateTableProcedure.CreateHdfsRegions; 043import org.apache.hadoop.hbase.mob.MobUtils; 044import org.apache.hadoop.hbase.monitoring.MonitoredTask; 045import org.apache.hadoop.hbase.monitoring.TaskMonitor; 046import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer; 047import org.apache.hadoop.hbase.snapshot.ClientSnapshotDescriptionUtils; 048import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException; 049import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper; 050import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; 051import org.apache.hadoop.hbase.snapshot.SnapshotManifest; 052import org.apache.hadoop.hbase.util.CommonFSUtils; 053import org.apache.hadoop.hbase.util.FSTableDescriptors; 054import org.apache.hadoop.hbase.util.Pair; 055import org.apache.yetus.audience.InterfaceAudience; 056import org.slf4j.Logger; 057import org.slf4j.LoggerFactory; 058 059import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; 060 061import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; 062import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; 063import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos; 064import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.CloneSnapshotState; 065import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription; 066 067@InterfaceAudience.Private 068public class CloneSnapshotProcedure 069 extends AbstractStateMachineTableProcedure<CloneSnapshotState> { 070 private static final Logger LOG = LoggerFactory.getLogger(CloneSnapshotProcedure.class); 071 072 private TableDescriptor tableDescriptor; 073 private SnapshotDescription snapshot; 074 private boolean restoreAcl; 075 private List<RegionInfo> newRegions = null; 076 private Map<String, Pair<String, String> > parentsToChildrenPairMap = new HashMap<>(); 077 078 // Monitor 079 private MonitoredTask monitorStatus = null; 080 081 /** 082 * Constructor (for failover) 083 */ 084 public CloneSnapshotProcedure() { 085 } 086 087 public CloneSnapshotProcedure(final MasterProcedureEnv env, 088 final TableDescriptor tableDescriptor, final SnapshotDescription snapshot) { 089 this(env, tableDescriptor, snapshot, false); 090 } 091 092 /** 093 * Constructor 094 * @param env MasterProcedureEnv 095 * @param tableDescriptor the table to operate on 096 * @param snapshot snapshot to clone from 097 */ 098 public CloneSnapshotProcedure(final MasterProcedureEnv env, 099 final TableDescriptor tableDescriptor, final SnapshotDescription snapshot, 100 final boolean restoreAcl) { 101 super(env); 102 this.tableDescriptor = tableDescriptor; 103 this.snapshot = snapshot; 104 this.restoreAcl = restoreAcl; 105 106 getMonitorStatus(); 107 } 108 109 /** 110 * Set up monitor status if it is not created. 111 */ 112 private MonitoredTask getMonitorStatus() { 113 if (monitorStatus == null) { 114 monitorStatus = TaskMonitor.get().createStatus("Cloning snapshot '" + snapshot.getName() + 115 "' to table " + getTableName()); 116 } 117 return monitorStatus; 118 } 119 120 private void restoreSnapshotAcl(MasterProcedureEnv env) throws IOException { 121 Configuration conf = env.getMasterServices().getConfiguration(); 122 if (restoreAcl && snapshot.hasUsersAndPermissions() && snapshot.getUsersAndPermissions() != null 123 && SnapshotDescriptionUtils.isSecurityAvailable(conf)) { 124 RestoreSnapshotHelper.restoreSnapshotAcl(snapshot, tableDescriptor.getTableName(), conf); 125 } 126 } 127 128 @Override 129 protected Flow executeFromState(final MasterProcedureEnv env, final CloneSnapshotState state) 130 throws InterruptedException { 131 LOG.trace("{} execute state={}", this, state); 132 try { 133 switch (state) { 134 case CLONE_SNAPSHOT_PRE_OPERATION: 135 // Verify if we can clone the table 136 prepareClone(env); 137 138 preCloneSnapshot(env); 139 setNextState(CloneSnapshotState.CLONE_SNAPSHOT_WRITE_FS_LAYOUT); 140 break; 141 case CLONE_SNAPSHOT_WRITE_FS_LAYOUT: 142 newRegions = createFilesystemLayout(env, tableDescriptor, newRegions); 143 env.getMasterServices().getTableDescriptors().update(tableDescriptor, true); 144 setNextState(CloneSnapshotState.CLONE_SNAPSHOT_ADD_TO_META); 145 break; 146 case CLONE_SNAPSHOT_ADD_TO_META: 147 addRegionsToMeta(env); 148 setNextState(CloneSnapshotState.CLONE_SNAPSHOT_ASSIGN_REGIONS); 149 break; 150 case CLONE_SNAPSHOT_ASSIGN_REGIONS: 151 CreateTableProcedure.setEnablingState(env, getTableName()); 152 153 // Separate newRegions to split regions and regions to assign 154 List<RegionInfo> splitRegions = new ArrayList<>(); 155 List<RegionInfo> regionsToAssign = new ArrayList<>(); 156 newRegions.forEach(ri -> { 157 if (ri.isOffline() && (ri.isSplit() || ri.isSplitParent())) { 158 splitRegions.add(ri); 159 } else { 160 regionsToAssign.add(ri); 161 } 162 }); 163 164 // For split regions, add them to RegionStates 165 AssignmentManager am = env.getAssignmentManager(); 166 splitRegions.forEach(ri -> 167 am.getRegionStates().updateRegionState(ri, RegionState.State.SPLIT) 168 ); 169 170 addChildProcedure(env.getAssignmentManager() 171 .createRoundRobinAssignProcedures(regionsToAssign)); 172 setNextState(CloneSnapshotState.CLONE_SNAPSHOT_UPDATE_DESC_CACHE); 173 break; 174 case CLONE_SNAPSHOT_UPDATE_DESC_CACHE: 175 // XXX: this stage should be named as set table enabled, as now we will cache the 176 // descriptor after writing fs layout. 177 CreateTableProcedure.setEnabledState(env, getTableName()); 178 setNextState(CloneSnapshotState.CLONE_SNAPHOST_RESTORE_ACL); 179 break; 180 case CLONE_SNAPHOST_RESTORE_ACL: 181 restoreSnapshotAcl(env); 182 setNextState(CloneSnapshotState.CLONE_SNAPSHOT_POST_OPERATION); 183 break; 184 case CLONE_SNAPSHOT_POST_OPERATION: 185 postCloneSnapshot(env); 186 187 MetricsSnapshot metricsSnapshot = new MetricsSnapshot(); 188 metricsSnapshot.addSnapshotClone( 189 getMonitorStatus().getCompletionTimestamp() - getMonitorStatus().getStartTime()); 190 getMonitorStatus().markComplete("Clone snapshot '"+ snapshot.getName() +"' completed!"); 191 return Flow.NO_MORE_STATE; 192 default: 193 throw new UnsupportedOperationException("unhandled state=" + state); 194 } 195 } catch (IOException e) { 196 if (isRollbackSupported(state)) { 197 setFailure("master-clone-snapshot", e); 198 } else { 199 LOG.warn("Retriable error trying to clone snapshot=" + snapshot.getName() + 200 " to table=" + getTableName() + " state=" + state, e); 201 } 202 } 203 return Flow.HAS_MORE_STATE; 204 } 205 206 @Override 207 protected void rollbackState(final MasterProcedureEnv env, final CloneSnapshotState state) 208 throws IOException { 209 if (state == CloneSnapshotState.CLONE_SNAPSHOT_PRE_OPERATION) { 210 DeleteTableProcedure.deleteTableStates(env, getTableName()); 211 // TODO-MAYBE: call the deleteTable coprocessor event? 212 return; 213 } 214 215 // The procedure doesn't have a rollback. The execution will succeed, at some point. 216 throw new UnsupportedOperationException("unhandled state=" + state); 217 } 218 219 @Override 220 protected boolean isRollbackSupported(final CloneSnapshotState state) { 221 switch (state) { 222 case CLONE_SNAPSHOT_PRE_OPERATION: 223 return true; 224 default: 225 return false; 226 } 227 } 228 229 @Override 230 protected CloneSnapshotState getState(final int stateId) { 231 return CloneSnapshotState.valueOf(stateId); 232 } 233 234 @Override 235 protected int getStateId(final CloneSnapshotState state) { 236 return state.getNumber(); 237 } 238 239 @Override 240 protected CloneSnapshotState getInitialState() { 241 return CloneSnapshotState.CLONE_SNAPSHOT_PRE_OPERATION; 242 } 243 244 @Override 245 public TableName getTableName() { 246 return tableDescriptor.getTableName(); 247 } 248 249 @Override 250 public TableOperationType getTableOperationType() { 251 return TableOperationType.CREATE; // Clone is creating a table 252 } 253 254 @Override 255 public void toStringClassDetails(StringBuilder sb) { 256 sb.append(getClass().getSimpleName()); 257 sb.append(" (table="); 258 sb.append(getTableName()); 259 sb.append(" snapshot="); 260 sb.append(snapshot); 261 sb.append(")"); 262 } 263 264 @Override 265 protected void serializeStateData(ProcedureStateSerializer serializer) 266 throws IOException { 267 super.serializeStateData(serializer); 268 269 MasterProcedureProtos.CloneSnapshotStateData.Builder cloneSnapshotMsg = 270 MasterProcedureProtos.CloneSnapshotStateData.newBuilder() 271 .setUserInfo(MasterProcedureUtil.toProtoUserInfo(getUser())) 272 .setSnapshot(this.snapshot) 273 .setTableSchema(ProtobufUtil.toTableSchema(tableDescriptor)); 274 if (newRegions != null) { 275 for (RegionInfo hri: newRegions) { 276 cloneSnapshotMsg.addRegionInfo(ProtobufUtil.toRegionInfo(hri)); 277 } 278 } 279 if (!parentsToChildrenPairMap.isEmpty()) { 280 final Iterator<Map.Entry<String, Pair<String, String>>> it = 281 parentsToChildrenPairMap.entrySet().iterator(); 282 while (it.hasNext()) { 283 final Map.Entry<String, Pair<String, String>> entry = it.next(); 284 285 MasterProcedureProtos.RestoreParentToChildRegionsPair.Builder parentToChildrenPair = 286 MasterProcedureProtos.RestoreParentToChildRegionsPair.newBuilder() 287 .setParentRegionName(entry.getKey()) 288 .setChild1RegionName(entry.getValue().getFirst()) 289 .setChild2RegionName(entry.getValue().getSecond()); 290 cloneSnapshotMsg.addParentToChildRegionsPairList(parentToChildrenPair); 291 } 292 } 293 serializer.serialize(cloneSnapshotMsg.build()); 294 } 295 296 @Override 297 protected void deserializeStateData(ProcedureStateSerializer serializer) 298 throws IOException { 299 super.deserializeStateData(serializer); 300 301 MasterProcedureProtos.CloneSnapshotStateData cloneSnapshotMsg = 302 serializer.deserialize(MasterProcedureProtos.CloneSnapshotStateData.class); 303 setUser(MasterProcedureUtil.toUserInfo(cloneSnapshotMsg.getUserInfo())); 304 snapshot = cloneSnapshotMsg.getSnapshot(); 305 tableDescriptor = ProtobufUtil.toTableDescriptor(cloneSnapshotMsg.getTableSchema()); 306 if (cloneSnapshotMsg.getRegionInfoCount() == 0) { 307 newRegions = null; 308 } else { 309 newRegions = new ArrayList<>(cloneSnapshotMsg.getRegionInfoCount()); 310 for (HBaseProtos.RegionInfo hri: cloneSnapshotMsg.getRegionInfoList()) { 311 newRegions.add(ProtobufUtil.toRegionInfo(hri)); 312 } 313 } 314 if (cloneSnapshotMsg.getParentToChildRegionsPairListCount() > 0) { 315 parentsToChildrenPairMap = new HashMap<>(); 316 for (MasterProcedureProtos.RestoreParentToChildRegionsPair parentToChildrenPair: 317 cloneSnapshotMsg.getParentToChildRegionsPairListList()) { 318 parentsToChildrenPairMap.put( 319 parentToChildrenPair.getParentRegionName(), 320 new Pair<>( 321 parentToChildrenPair.getChild1RegionName(), 322 parentToChildrenPair.getChild2RegionName())); 323 } 324 } 325 // Make sure that the monitor status is set up 326 getMonitorStatus(); 327 } 328 329 /** 330 * Action before any real action of cloning from snapshot. 331 * @param env MasterProcedureEnv 332 */ 333 private void prepareClone(final MasterProcedureEnv env) throws IOException { 334 final TableName tableName = getTableName(); 335 if (env.getMasterServices().getTableDescriptors().exists(tableName)) { 336 throw new TableExistsException(tableName); 337 } 338 } 339 340 /** 341 * Action before cloning from snapshot. 342 * @param env MasterProcedureEnv 343 * @throws IOException 344 * @throws InterruptedException 345 */ 346 private void preCloneSnapshot(final MasterProcedureEnv env) 347 throws IOException, InterruptedException { 348 if (!getTableName().isSystemTable()) { 349 // Check and update namespace quota 350 final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); 351 352 SnapshotManifest manifest = SnapshotManifest.open( 353 env.getMasterConfiguration(), 354 mfs.getFileSystem(), 355 SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, mfs.getRootDir()), 356 snapshot); 357 358 ProcedureSyncWait.getMasterQuotaManager(env) 359 .checkNamespaceTableAndRegionQuota(getTableName(), manifest.getRegionManifestsMap().size()); 360 } 361 362 final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost(); 363 if (cpHost != null) { 364 cpHost.preCreateTableAction(tableDescriptor, null, getUser()); 365 } 366 } 367 368 /** 369 * Action after cloning from snapshot. 370 * @param env MasterProcedureEnv 371 * @throws IOException 372 * @throws InterruptedException 373 */ 374 private void postCloneSnapshot(final MasterProcedureEnv env) 375 throws IOException, InterruptedException { 376 final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost(); 377 if (cpHost != null) { 378 final RegionInfo[] regions = (newRegions == null) ? null : 379 newRegions.toArray(new RegionInfo[newRegions.size()]); 380 cpHost.postCompletedCreateTableAction(tableDescriptor, regions, getUser()); 381 } 382 } 383 384 /** 385 * Create regions in file system. 386 * @param env MasterProcedureEnv 387 * @throws IOException 388 */ 389 private List<RegionInfo> createFilesystemLayout( 390 final MasterProcedureEnv env, 391 final TableDescriptor tableDescriptor, 392 final List<RegionInfo> newRegions) throws IOException { 393 return createFsLayout(env, tableDescriptor, newRegions, new CreateHdfsRegions() { 394 @Override 395 public List<RegionInfo> createHdfsRegions( 396 final MasterProcedureEnv env, 397 final Path tableRootDir, final TableName tableName, 398 final List<RegionInfo> newRegions) throws IOException { 399 400 final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); 401 final FileSystem fs = mfs.getFileSystem(); 402 final Path rootDir = mfs.getRootDir(); 403 final Configuration conf = env.getMasterConfiguration(); 404 final ForeignExceptionDispatcher monitorException = new ForeignExceptionDispatcher(); 405 406 getMonitorStatus().setStatus("Clone snapshot - creating regions for table: " + tableName); 407 408 try { 409 // 1. Execute the on-disk Clone 410 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir); 411 SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshot); 412 RestoreSnapshotHelper restoreHelper = new RestoreSnapshotHelper( 413 conf, fs, manifest, tableDescriptor, tableRootDir, monitorException, monitorStatus); 414 RestoreSnapshotHelper.RestoreMetaChanges metaChanges = restoreHelper.restoreHdfsRegions(); 415 416 // Clone operation should not have stuff to restore or remove 417 Preconditions.checkArgument( 418 !metaChanges.hasRegionsToRestore(), "A clone should not have regions to restore"); 419 Preconditions.checkArgument( 420 !metaChanges.hasRegionsToRemove(), "A clone should not have regions to remove"); 421 422 // At this point the clone is complete. Next step is enabling the table. 423 String msg = 424 "Clone snapshot="+ snapshot.getName() +" on table=" + tableName + " completed!"; 425 LOG.info(msg); 426 monitorStatus.setStatus(msg + " Waiting for table to be enabled..."); 427 428 // 2. Let the next step to add the regions to meta 429 return metaChanges.getRegionsToAdd(); 430 } catch (Exception e) { 431 String msg = "clone snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot) + 432 " failed because " + e.getMessage(); 433 LOG.error(msg, e); 434 IOException rse = new RestoreSnapshotException(msg, e, 435 ProtobufUtil.createSnapshotDesc(snapshot)); 436 437 // these handlers aren't futures so we need to register the error here. 438 monitorException.receive(new ForeignException("Master CloneSnapshotProcedure", rse)); 439 throw rse; 440 } 441 } 442 }); 443 } 444 445 /** 446 * Create region layout in file system. 447 * @param env MasterProcedureEnv 448 * @throws IOException 449 */ 450 private List<RegionInfo> createFsLayout( 451 final MasterProcedureEnv env, 452 final TableDescriptor tableDescriptor, 453 List<RegionInfo> newRegions, 454 final CreateHdfsRegions hdfsRegionHandler) throws IOException { 455 final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); 456 final Path tempdir = mfs.getTempDir(); 457 458 // 1. Create Table Descriptor 459 // using a copy of descriptor, table will be created enabling first 460 final Path tempTableDir = CommonFSUtils.getTableDir(tempdir, tableDescriptor.getTableName()); 461 if (CommonFSUtils.isExists(mfs.getFileSystem(), tempTableDir)) { 462 // if the region dirs exist, will cause exception and unlimited retry (see HBASE-24546) 463 LOG.warn("temp table dir already exists on disk: {}, will be deleted.", tempTableDir); 464 CommonFSUtils.deleteDirectory(mfs.getFileSystem(), tempTableDir); 465 } 466 ((FSTableDescriptors) (env.getMasterServices().getTableDescriptors())) 467 .createTableDescriptorForTableDirectory(tempTableDir, 468 TableDescriptorBuilder.newBuilder(tableDescriptor).build(), false); 469 470 // 2. Create Regions 471 newRegions = hdfsRegionHandler.createHdfsRegions( 472 env, tempdir, tableDescriptor.getTableName(), newRegions); 473 474 // 3. Move Table temp directory to the hbase root location 475 CreateTableProcedure.moveTempDirectoryToHBaseRoot(env, tableDescriptor, tempTableDir); 476 // Move Table temp mob directory to the hbase root location 477 Path tempMobTableDir = MobUtils.getMobTableDir(tempdir, tableDescriptor.getTableName()); 478 if (mfs.getFileSystem().exists(tempMobTableDir)) { 479 moveTempMobDirectoryToHBaseRoot(mfs, tableDescriptor, tempMobTableDir); 480 } 481 return newRegions; 482 } 483 484 /** 485 * Move table temp mob directory to the hbase root location 486 * @param mfs The master file system 487 * @param tableDescriptor The table to operate on 488 * @param tempMobTableDir The temp mob directory of table 489 * @throws IOException If failed to move temp mob dir to hbase root dir 490 */ 491 private void moveTempMobDirectoryToHBaseRoot(final MasterFileSystem mfs, 492 final TableDescriptor tableDescriptor, final Path tempMobTableDir) throws IOException { 493 FileSystem fs = mfs.getFileSystem(); 494 final Path tableMobDir = 495 MobUtils.getMobTableDir(mfs.getRootDir(), tableDescriptor.getTableName()); 496 if (!fs.delete(tableMobDir, true) && fs.exists(tableMobDir)) { 497 throw new IOException("Couldn't delete mob table " + tableMobDir); 498 } 499 if (!fs.exists(tableMobDir.getParent())) { 500 fs.mkdirs(tableMobDir.getParent()); 501 } 502 if (!fs.rename(tempMobTableDir, tableMobDir)) { 503 throw new IOException("Unable to move mob table from temp=" + tempMobTableDir 504 + " to hbase root=" + tableMobDir); 505 } 506 } 507 508 /** 509 * Add regions to hbase:meta table. 510 * @param env MasterProcedureEnv 511 * @throws IOException 512 */ 513 private void addRegionsToMeta(final MasterProcedureEnv env) throws IOException { 514 newRegions = CreateTableProcedure.addTableToMeta(env, tableDescriptor, newRegions); 515 516 // TODO: parentsToChildrenPairMap is always empty, which makes updateMetaParentRegions() 517 // a no-op. This part seems unnecessary. Figure out. - Appy 12/21/17 518 RestoreSnapshotHelper.RestoreMetaChanges metaChanges = 519 new RestoreSnapshotHelper.RestoreMetaChanges( 520 tableDescriptor, parentsToChildrenPairMap); 521 metaChanges.updateMetaParentRegions(env.getMasterServices().getConnection(), newRegions); 522 } 523 524}