001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 019package org.apache.hadoop.hbase.master.procedure; 020 021import java.io.IOException; 022import java.util.ArrayList; 023import java.util.HashMap; 024import java.util.Iterator; 025import java.util.List; 026import java.util.Map; 027 028import org.apache.hadoop.conf.Configuration; 029import org.apache.hadoop.fs.FileSystem; 030import org.apache.hadoop.fs.Path; 031import org.apache.hadoop.hbase.MetaTableAccessor; 032import org.apache.hadoop.hbase.TableExistsException; 033import org.apache.hadoop.hbase.TableName; 034import org.apache.hadoop.hbase.client.RegionInfo; 035import org.apache.hadoop.hbase.client.TableDescriptor; 036import org.apache.hadoop.hbase.client.TableDescriptorBuilder; 037import org.apache.hadoop.hbase.errorhandling.ForeignException; 038import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher; 039import org.apache.hadoop.hbase.master.MasterCoprocessorHost; 040import org.apache.hadoop.hbase.master.MasterFileSystem; 041import org.apache.hadoop.hbase.master.MetricsSnapshot; 042import org.apache.hadoop.hbase.master.procedure.CreateTableProcedure.CreateHdfsRegions; 043import org.apache.hadoop.hbase.mob.MobUtils; 044import org.apache.hadoop.hbase.monitoring.MonitoredTask; 045import org.apache.hadoop.hbase.monitoring.TaskMonitor; 046import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer; 047import org.apache.hadoop.hbase.snapshot.ClientSnapshotDescriptionUtils; 048import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException; 049import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper; 050import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; 051import org.apache.hadoop.hbase.snapshot.SnapshotManifest; 052import org.apache.hadoop.hbase.util.FSTableDescriptors; 053import org.apache.hadoop.hbase.util.FSUtils; 054import org.apache.hadoop.hbase.util.Pair; 055import org.apache.yetus.audience.InterfaceAudience; 056import org.slf4j.Logger; 057import org.slf4j.LoggerFactory; 058import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; 059import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; 060import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; 061import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos; 062import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.CloneSnapshotState; 063import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription; 064 065@InterfaceAudience.Private 066public class CloneSnapshotProcedure 067 extends AbstractStateMachineTableProcedure<CloneSnapshotState> { 068 private static final Logger LOG = LoggerFactory.getLogger(CloneSnapshotProcedure.class); 069 070 private TableDescriptor tableDescriptor; 071 private SnapshotDescription snapshot; 072 private boolean restoreAcl; 073 private List<RegionInfo> newRegions = null; 074 private Map<String, Pair<String, String> > parentsToChildrenPairMap = new HashMap<>(); 075 076 // Monitor 077 private MonitoredTask monitorStatus = null; 078 079 /** 080 * Constructor (for failover) 081 */ 082 public CloneSnapshotProcedure() { 083 } 084 085 public CloneSnapshotProcedure(final MasterProcedureEnv env, 086 final TableDescriptor tableDescriptor, final SnapshotDescription snapshot) { 087 this(env, tableDescriptor, snapshot, false); 088 } 089 090 /** 091 * Constructor 092 * @param env MasterProcedureEnv 093 * @param tableDescriptor the table to operate on 094 * @param snapshot snapshot to clone from 095 */ 096 public CloneSnapshotProcedure(final MasterProcedureEnv env, 097 final TableDescriptor tableDescriptor, final SnapshotDescription snapshot, 098 final boolean restoreAcl) { 099 super(env); 100 this.tableDescriptor = tableDescriptor; 101 this.snapshot = snapshot; 102 this.restoreAcl = restoreAcl; 103 104 getMonitorStatus(); 105 } 106 107 /** 108 * Set up monitor status if it is not created. 109 */ 110 private MonitoredTask getMonitorStatus() { 111 if (monitorStatus == null) { 112 monitorStatus = TaskMonitor.get().createStatus("Cloning snapshot '" + snapshot.getName() + 113 "' to table " + getTableName()); 114 } 115 return monitorStatus; 116 } 117 118 private void restoreSnapshotAcl(MasterProcedureEnv env) throws IOException { 119 Configuration conf = env.getMasterServices().getConfiguration(); 120 if (restoreAcl && snapshot.hasUsersAndPermissions() && snapshot.getUsersAndPermissions() != null 121 && SnapshotDescriptionUtils.isSecurityAvailable(conf)) { 122 RestoreSnapshotHelper.restoreSnapshotAcl(snapshot, tableDescriptor.getTableName(), conf); 123 } 124 } 125 126 @Override 127 protected Flow executeFromState(final MasterProcedureEnv env, final CloneSnapshotState state) 128 throws InterruptedException { 129 LOG.trace("{} execute state={}", this, state); 130 try { 131 switch (state) { 132 case CLONE_SNAPSHOT_PRE_OPERATION: 133 // Verify if we can clone the table 134 prepareClone(env); 135 136 preCloneSnapshot(env); 137 setNextState(CloneSnapshotState.CLONE_SNAPSHOT_WRITE_FS_LAYOUT); 138 break; 139 case CLONE_SNAPSHOT_WRITE_FS_LAYOUT: 140 newRegions = createFilesystemLayout(env, tableDescriptor, newRegions); 141 setNextState(CloneSnapshotState.CLONE_SNAPSHOT_ADD_TO_META); 142 break; 143 case CLONE_SNAPSHOT_ADD_TO_META: 144 addRegionsToMeta(env); 145 setNextState(CloneSnapshotState.CLONE_SNAPSHOT_ASSIGN_REGIONS); 146 break; 147 case CLONE_SNAPSHOT_ASSIGN_REGIONS: 148 CreateTableProcedure.setEnablingState(env, getTableName()); 149 addChildProcedure(env.getAssignmentManager().createRoundRobinAssignProcedures(newRegions)); 150 setNextState(CloneSnapshotState.CLONE_SNAPSHOT_UPDATE_DESC_CACHE); 151 break; 152 case CLONE_SNAPSHOT_UPDATE_DESC_CACHE: 153 CreateTableProcedure.setEnabledState(env, getTableName()); 154 CreateTableProcedure.updateTableDescCache(env, getTableName()); 155 setNextState(CloneSnapshotState.CLONE_SNAPHOST_RESTORE_ACL); 156 break; 157 case CLONE_SNAPHOST_RESTORE_ACL: 158 restoreSnapshotAcl(env); 159 setNextState(CloneSnapshotState.CLONE_SNAPSHOT_POST_OPERATION); 160 break; 161 case CLONE_SNAPSHOT_POST_OPERATION: 162 postCloneSnapshot(env); 163 164 MetricsSnapshot metricsSnapshot = new MetricsSnapshot(); 165 metricsSnapshot.addSnapshotClone( 166 getMonitorStatus().getCompletionTimestamp() - getMonitorStatus().getStartTime()); 167 getMonitorStatus().markComplete("Clone snapshot '"+ snapshot.getName() +"' completed!"); 168 return Flow.NO_MORE_STATE; 169 default: 170 throw new UnsupportedOperationException("unhandled state=" + state); 171 } 172 } catch (IOException e) { 173 if (isRollbackSupported(state)) { 174 setFailure("master-clone-snapshot", e); 175 } else { 176 LOG.warn("Retriable error trying to clone snapshot=" + snapshot.getName() + 177 " to table=" + getTableName() + " state=" + state, e); 178 } 179 } 180 return Flow.HAS_MORE_STATE; 181 } 182 183 @Override 184 protected void rollbackState(final MasterProcedureEnv env, final CloneSnapshotState state) 185 throws IOException { 186 if (state == CloneSnapshotState.CLONE_SNAPSHOT_PRE_OPERATION) { 187 DeleteTableProcedure.deleteTableStates(env, getTableName()); 188 // TODO-MAYBE: call the deleteTable coprocessor event? 189 return; 190 } 191 192 // The procedure doesn't have a rollback. The execution will succeed, at some point. 193 throw new UnsupportedOperationException("unhandled state=" + state); 194 } 195 196 @Override 197 protected boolean isRollbackSupported(final CloneSnapshotState state) { 198 switch (state) { 199 case CLONE_SNAPSHOT_PRE_OPERATION: 200 return true; 201 default: 202 return false; 203 } 204 } 205 206 @Override 207 protected CloneSnapshotState getState(final int stateId) { 208 return CloneSnapshotState.valueOf(stateId); 209 } 210 211 @Override 212 protected int getStateId(final CloneSnapshotState state) { 213 return state.getNumber(); 214 } 215 216 @Override 217 protected CloneSnapshotState getInitialState() { 218 return CloneSnapshotState.CLONE_SNAPSHOT_PRE_OPERATION; 219 } 220 221 @Override 222 public TableName getTableName() { 223 return tableDescriptor.getTableName(); 224 } 225 226 @Override 227 public TableOperationType getTableOperationType() { 228 return TableOperationType.CREATE; // Clone is creating a table 229 } 230 231 @Override 232 public void toStringClassDetails(StringBuilder sb) { 233 sb.append(getClass().getSimpleName()); 234 sb.append(" (table="); 235 sb.append(getTableName()); 236 sb.append(" snapshot="); 237 sb.append(snapshot); 238 sb.append(")"); 239 } 240 241 @Override 242 protected void serializeStateData(ProcedureStateSerializer serializer) 243 throws IOException { 244 super.serializeStateData(serializer); 245 246 MasterProcedureProtos.CloneSnapshotStateData.Builder cloneSnapshotMsg = 247 MasterProcedureProtos.CloneSnapshotStateData.newBuilder() 248 .setUserInfo(MasterProcedureUtil.toProtoUserInfo(getUser())) 249 .setSnapshot(this.snapshot) 250 .setTableSchema(ProtobufUtil.toTableSchema(tableDescriptor)); 251 if (newRegions != null) { 252 for (RegionInfo hri: newRegions) { 253 cloneSnapshotMsg.addRegionInfo(ProtobufUtil.toRegionInfo(hri)); 254 } 255 } 256 if (!parentsToChildrenPairMap.isEmpty()) { 257 final Iterator<Map.Entry<String, Pair<String, String>>> it = 258 parentsToChildrenPairMap.entrySet().iterator(); 259 while (it.hasNext()) { 260 final Map.Entry<String, Pair<String, String>> entry = it.next(); 261 262 MasterProcedureProtos.RestoreParentToChildRegionsPair.Builder parentToChildrenPair = 263 MasterProcedureProtos.RestoreParentToChildRegionsPair.newBuilder() 264 .setParentRegionName(entry.getKey()) 265 .setChild1RegionName(entry.getValue().getFirst()) 266 .setChild2RegionName(entry.getValue().getSecond()); 267 cloneSnapshotMsg.addParentToChildRegionsPairList(parentToChildrenPair); 268 } 269 } 270 serializer.serialize(cloneSnapshotMsg.build()); 271 } 272 273 @Override 274 protected void deserializeStateData(ProcedureStateSerializer serializer) 275 throws IOException { 276 super.deserializeStateData(serializer); 277 278 MasterProcedureProtos.CloneSnapshotStateData cloneSnapshotMsg = 279 serializer.deserialize(MasterProcedureProtos.CloneSnapshotStateData.class); 280 setUser(MasterProcedureUtil.toUserInfo(cloneSnapshotMsg.getUserInfo())); 281 snapshot = cloneSnapshotMsg.getSnapshot(); 282 tableDescriptor = ProtobufUtil.toTableDescriptor(cloneSnapshotMsg.getTableSchema()); 283 if (cloneSnapshotMsg.getRegionInfoCount() == 0) { 284 newRegions = null; 285 } else { 286 newRegions = new ArrayList<>(cloneSnapshotMsg.getRegionInfoCount()); 287 for (HBaseProtos.RegionInfo hri: cloneSnapshotMsg.getRegionInfoList()) { 288 newRegions.add(ProtobufUtil.toRegionInfo(hri)); 289 } 290 } 291 if (cloneSnapshotMsg.getParentToChildRegionsPairListCount() > 0) { 292 parentsToChildrenPairMap = new HashMap<>(); 293 for (MasterProcedureProtos.RestoreParentToChildRegionsPair parentToChildrenPair: 294 cloneSnapshotMsg.getParentToChildRegionsPairListList()) { 295 parentsToChildrenPairMap.put( 296 parentToChildrenPair.getParentRegionName(), 297 new Pair<>( 298 parentToChildrenPair.getChild1RegionName(), 299 parentToChildrenPair.getChild2RegionName())); 300 } 301 } 302 // Make sure that the monitor status is set up 303 getMonitorStatus(); 304 } 305 306 /** 307 * Action before any real action of cloning from snapshot. 308 * @param env MasterProcedureEnv 309 * @throws IOException 310 */ 311 private void prepareClone(final MasterProcedureEnv env) throws IOException { 312 final TableName tableName = getTableName(); 313 if (MetaTableAccessor.tableExists(env.getMasterServices().getConnection(), tableName)) { 314 throw new TableExistsException(getTableName()); 315 } 316 } 317 318 /** 319 * Action before cloning from snapshot. 320 * @param env MasterProcedureEnv 321 * @throws IOException 322 * @throws InterruptedException 323 */ 324 private void preCloneSnapshot(final MasterProcedureEnv env) 325 throws IOException, InterruptedException { 326 if (!getTableName().isSystemTable()) { 327 // Check and update namespace quota 328 final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); 329 330 SnapshotManifest manifest = SnapshotManifest.open( 331 env.getMasterConfiguration(), 332 mfs.getFileSystem(), 333 SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, mfs.getRootDir()), 334 snapshot); 335 336 ProcedureSyncWait.getMasterQuotaManager(env) 337 .checkNamespaceTableAndRegionQuota(getTableName(), manifest.getRegionManifestsMap().size()); 338 } 339 340 final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost(); 341 if (cpHost != null) { 342 cpHost.preCreateTableAction(tableDescriptor, null, getUser()); 343 } 344 } 345 346 /** 347 * Action after cloning from snapshot. 348 * @param env MasterProcedureEnv 349 * @throws IOException 350 * @throws InterruptedException 351 */ 352 private void postCloneSnapshot(final MasterProcedureEnv env) 353 throws IOException, InterruptedException { 354 final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost(); 355 if (cpHost != null) { 356 final RegionInfo[] regions = (newRegions == null) ? null : 357 newRegions.toArray(new RegionInfo[newRegions.size()]); 358 cpHost.postCompletedCreateTableAction(tableDescriptor, regions, getUser()); 359 } 360 } 361 362 /** 363 * Create regions in file system. 364 * @param env MasterProcedureEnv 365 * @throws IOException 366 */ 367 private List<RegionInfo> createFilesystemLayout( 368 final MasterProcedureEnv env, 369 final TableDescriptor tableDescriptor, 370 final List<RegionInfo> newRegions) throws IOException { 371 return createFsLayout(env, tableDescriptor, newRegions, new CreateHdfsRegions() { 372 @Override 373 public List<RegionInfo> createHdfsRegions( 374 final MasterProcedureEnv env, 375 final Path tableRootDir, final TableName tableName, 376 final List<RegionInfo> newRegions) throws IOException { 377 378 final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); 379 final FileSystem fs = mfs.getFileSystem(); 380 final Path rootDir = mfs.getRootDir(); 381 final Configuration conf = env.getMasterConfiguration(); 382 final ForeignExceptionDispatcher monitorException = new ForeignExceptionDispatcher(); 383 384 getMonitorStatus().setStatus("Clone snapshot - creating regions for table: " + tableName); 385 386 try { 387 // 1. Execute the on-disk Clone 388 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir); 389 SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshot); 390 RestoreSnapshotHelper restoreHelper = new RestoreSnapshotHelper( 391 conf, fs, manifest, tableDescriptor, tableRootDir, monitorException, monitorStatus); 392 RestoreSnapshotHelper.RestoreMetaChanges metaChanges = restoreHelper.restoreHdfsRegions(); 393 394 // Clone operation should not have stuff to restore or remove 395 Preconditions.checkArgument( 396 !metaChanges.hasRegionsToRestore(), "A clone should not have regions to restore"); 397 Preconditions.checkArgument( 398 !metaChanges.hasRegionsToRemove(), "A clone should not have regions to remove"); 399 400 // At this point the clone is complete. Next step is enabling the table. 401 String msg = 402 "Clone snapshot="+ snapshot.getName() +" on table=" + tableName + " completed!"; 403 LOG.info(msg); 404 monitorStatus.setStatus(msg + " Waiting for table to be enabled..."); 405 406 // 2. Let the next step to add the regions to meta 407 return metaChanges.getRegionsToAdd(); 408 } catch (Exception e) { 409 String msg = "clone snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot) + 410 " failed because " + e.getMessage(); 411 LOG.error(msg, e); 412 IOException rse = new RestoreSnapshotException(msg, e, 413 ProtobufUtil.createSnapshotDesc(snapshot)); 414 415 // these handlers aren't futures so we need to register the error here. 416 monitorException.receive(new ForeignException("Master CloneSnapshotProcedure", rse)); 417 throw rse; 418 } 419 } 420 }); 421 } 422 423 /** 424 * Create region layout in file system. 425 * @param env MasterProcedureEnv 426 * @throws IOException 427 */ 428 private List<RegionInfo> createFsLayout( 429 final MasterProcedureEnv env, 430 final TableDescriptor tableDescriptor, 431 List<RegionInfo> newRegions, 432 final CreateHdfsRegions hdfsRegionHandler) throws IOException { 433 final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); 434 final Path tempdir = mfs.getTempDir(); 435 436 // 1. Create Table Descriptor 437 // using a copy of descriptor, table will be created enabling first 438 final Path tempTableDir = FSUtils.getTableDir(tempdir, tableDescriptor.getTableName()); 439 ((FSTableDescriptors)(env.getMasterServices().getTableDescriptors())) 440 .createTableDescriptorForTableDirectory(tempTableDir, 441 TableDescriptorBuilder.newBuilder(tableDescriptor).build(), false); 442 443 // 2. Create Regions 444 newRegions = hdfsRegionHandler.createHdfsRegions( 445 env, tempdir, tableDescriptor.getTableName(), newRegions); 446 447 // 3. Move Table temp directory to the hbase root location 448 CreateTableProcedure.moveTempDirectoryToHBaseRoot(env, tableDescriptor, tempTableDir); 449 // Move Table temp mob directory to the hbase root location 450 Path tempMobTableDir = MobUtils.getMobTableDir(tempdir, tableDescriptor.getTableName()); 451 if (mfs.getFileSystem().exists(tempMobTableDir)) { 452 moveTempMobDirectoryToHBaseRoot(mfs, tableDescriptor, tempMobTableDir); 453 } 454 return newRegions; 455 } 456 457 /** 458 * Move table temp mob directory to the hbase root location 459 * @param mfs The master file system 460 * @param tableDescriptor The table to operate on 461 * @param tempMobTableDir The temp mob directory of table 462 * @throws IOException If failed to move temp mob dir to hbase root dir 463 */ 464 private void moveTempMobDirectoryToHBaseRoot(final MasterFileSystem mfs, 465 final TableDescriptor tableDescriptor, final Path tempMobTableDir) throws IOException { 466 FileSystem fs = mfs.getFileSystem(); 467 final Path tableMobDir = 468 MobUtils.getMobTableDir(mfs.getRootDir(), tableDescriptor.getTableName()); 469 if (!fs.delete(tableMobDir, true) && fs.exists(tableMobDir)) { 470 throw new IOException("Couldn't delete mob table " + tableMobDir); 471 } 472 if (!fs.exists(tableMobDir.getParent())) { 473 fs.mkdirs(tableMobDir.getParent()); 474 } 475 if (!fs.rename(tempMobTableDir, tableMobDir)) { 476 throw new IOException("Unable to move mob table from temp=" + tempMobTableDir 477 + " to hbase root=" + tableMobDir); 478 } 479 } 480 481 /** 482 * Add regions to hbase:meta table. 483 * @param env MasterProcedureEnv 484 * @throws IOException 485 */ 486 private void addRegionsToMeta(final MasterProcedureEnv env) throws IOException { 487 newRegions = CreateTableProcedure.addTableToMeta(env, tableDescriptor, newRegions); 488 489 // TODO: parentsToChildrenPairMap is always empty, which makes updateMetaParentRegions() 490 // a no-op. This part seems unnecessary. Figure out. - Appy 12/21/17 491 RestoreSnapshotHelper.RestoreMetaChanges metaChanges = 492 new RestoreSnapshotHelper.RestoreMetaChanges( 493 tableDescriptor, parentsToChildrenPairMap); 494 metaChanges.updateMetaParentRegions(env.getMasterServices().getConnection(), newRegions); 495 } 496 497}