001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.master.assignment; 019 020import java.io.IOException; 021import java.util.ArrayList; 022import java.util.Arrays; 023import java.util.Collection; 024import java.util.Collections; 025import java.util.List; 026import java.util.stream.Stream; 027import org.apache.hadoop.fs.FileSystem; 028import org.apache.hadoop.fs.Path; 029import org.apache.hadoop.hbase.MetaMutationAnnotation; 030import org.apache.hadoop.hbase.MetaTableAccessor; 031import org.apache.hadoop.hbase.ServerName; 032import org.apache.hadoop.hbase.TableName; 033import org.apache.hadoop.hbase.UnknownRegionException; 034import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; 035import org.apache.hadoop.hbase.client.DoNotRetryRegionException; 036import org.apache.hadoop.hbase.client.MasterSwitchType; 037import org.apache.hadoop.hbase.client.Mutation; 038import org.apache.hadoop.hbase.client.RegionInfo; 039import org.apache.hadoop.hbase.client.RegionInfoBuilder; 040import org.apache.hadoop.hbase.client.TableDescriptor; 041import org.apache.hadoop.hbase.exceptions.MergeRegionException; 042import org.apache.hadoop.hbase.io.hfile.CacheConfig; 043import org.apache.hadoop.hbase.master.MasterCoprocessorHost; 044import org.apache.hadoop.hbase.master.MasterFileSystem; 045import org.apache.hadoop.hbase.master.RegionState; 046import org.apache.hadoop.hbase.master.RegionState.State; 047import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan; 048import org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure; 049import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; 050import org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil; 051import org.apache.hadoop.hbase.procedure2.ProcedureMetrics; 052import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer; 053import org.apache.hadoop.hbase.quotas.QuotaExceededException; 054import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; 055import org.apache.hadoop.hbase.regionserver.HStoreFile; 056import org.apache.hadoop.hbase.regionserver.StoreFileInfo; 057import org.apache.hadoop.hbase.util.Bytes; 058import org.apache.hadoop.hbase.util.CommonFSUtils; 059import org.apache.hadoop.hbase.wal.WALSplitUtil; 060import org.apache.yetus.audience.InterfaceAudience; 061import org.slf4j.Logger; 062import org.slf4j.LoggerFactory; 063 064import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; 065import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoResponse; 066import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos; 067import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.MergeTableRegionsState; 068 069/** 070 * The procedure to Merge regions in a table. This procedure takes an exclusive table 071 * lock since it is working over multiple regions. It holds the lock for the life of the procedure. 072 * Throws exception on construction if determines context hostile to merge (cluster going down or 073 * master is shutting down or table is disabled). 074 */ 075@InterfaceAudience.Private 076public class MergeTableRegionsProcedure 077 extends AbstractStateMachineTableProcedure<MergeTableRegionsState> { 078 private static final Logger LOG = LoggerFactory.getLogger(MergeTableRegionsProcedure.class); 079 private ServerName regionLocation; 080 081 /** 082 * Two or more regions to merge, the 'merge parents'. 083 */ 084 private RegionInfo[] regionsToMerge; 085 086 /** 087 * The resulting merged region. 088 */ 089 private RegionInfo mergedRegion; 090 091 private boolean force; 092 093 public MergeTableRegionsProcedure() { 094 // Required by the Procedure framework to create the procedure on replay 095 } 096 097 public MergeTableRegionsProcedure(final MasterProcedureEnv env, 098 final RegionInfo[] regionsToMerge, final boolean force) 099 throws IOException { 100 super(env); 101 // Check parent regions. Make sure valid before starting work. 102 // This check calls the super method #checkOnline also. 103 checkRegionsToMerge(env, regionsToMerge, force); 104 // Sort the regions going into the merge. 105 Arrays.sort(regionsToMerge); 106 this.regionsToMerge = regionsToMerge; 107 this.mergedRegion = createMergedRegionInfo(regionsToMerge); 108 // Preflight depends on mergedRegion being set (at least). 109 preflightChecks(env, true); 110 this.force = force; 111 } 112 113 /** 114 * @throws MergeRegionException If unable to merge regions for whatever reasons. 115 */ 116 private static void checkRegionsToMerge(MasterProcedureEnv env, final RegionInfo[] regions, 117 final boolean force) throws MergeRegionException { 118 long count = Arrays.stream(regions).distinct().count(); 119 if (regions.length != count) { 120 throw new MergeRegionException("Duplicate regions specified; cannot merge a region to " + 121 "itself. Passed in " + regions.length + " but only " + count + " unique."); 122 } 123 if (count < 2) { 124 throw new MergeRegionException("Need two Regions at least to run a Merge"); 125 } 126 RegionInfo previous = null; 127 for (RegionInfo ri: regions) { 128 if (previous != null) { 129 if (!previous.getTable().equals(ri.getTable())) { 130 String msg = "Can't merge regions from different tables: " + previous + ", " + ri; 131 LOG.warn(msg); 132 throw new MergeRegionException(msg); 133 } 134 if (!force && !ri.isAdjacent(previous) && !ri.isOverlap(previous)) { 135 String msg = "Unable to merge non-adjacent or non-overlapping regions '" + 136 previous.getShortNameToLog() + "', '" + ri.getShortNameToLog() + "' when force=false"; 137 LOG.warn(msg); 138 throw new MergeRegionException(msg); 139 } 140 } 141 142 if (ri.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) { 143 throw new MergeRegionException("Can't merge non-default replicas; " + ri); 144 } 145 try { 146 checkOnline(env, ri); 147 } catch (DoNotRetryRegionException dnrre) { 148 throw new MergeRegionException(dnrre); 149 } 150 151 previous = ri; 152 } 153 } 154 155 /** 156 * Create merged region info by looking at passed in <code>regionsToMerge</code> 157 * to figure what extremes for start and end keys to use; merged region needs 158 * to have an extent sufficient to cover all regions-to-merge. 159 */ 160 private static RegionInfo createMergedRegionInfo(final RegionInfo[] regionsToMerge) { 161 byte [] lowestStartKey = null; 162 byte [] highestEndKey = null; 163 // Region Id is a timestamp. Merged region's id can't be less than that of 164 // merging regions else will insert at wrong location in hbase:meta (See HBASE-710). 165 long highestRegionId = -1; 166 for (RegionInfo ri: regionsToMerge) { 167 if (lowestStartKey == null) { 168 lowestStartKey = ri.getStartKey(); 169 } else if (Bytes.compareTo(ri.getStartKey(), lowestStartKey) < 0) { 170 lowestStartKey = ri.getStartKey(); 171 } 172 if (highestEndKey == null) { 173 highestEndKey = ri.getEndKey(); 174 } else if (ri.isLast() || Bytes.compareTo(ri.getEndKey(), highestEndKey) > 0) { 175 highestEndKey = ri.getEndKey(); 176 } 177 highestRegionId = ri.getRegionId() > highestRegionId? ri.getRegionId(): highestRegionId; 178 } 179 // Merged region is sorted between two merging regions in META 180 return RegionInfoBuilder.newBuilder(regionsToMerge[0].getTable()). 181 setStartKey(lowestStartKey). 182 setEndKey(highestEndKey). 183 setSplit(false). 184 setRegionId(highestRegionId + 1/*Add one so new merged region is highest*/). 185 build(); 186 } 187 188 @Override 189 protected Flow executeFromState(final MasterProcedureEnv env, 190 MergeTableRegionsState state) { 191 LOG.trace("{} execute state={}", this, state); 192 try { 193 switch (state) { 194 case MERGE_TABLE_REGIONS_PREPARE: 195 if (!prepareMergeRegion(env)) { 196 assert isFailed() : "Merge region should have an exception here"; 197 return Flow.NO_MORE_STATE; 198 } 199 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_PRE_MERGE_OPERATION); 200 break; 201 case MERGE_TABLE_REGIONS_PRE_MERGE_OPERATION: 202 preMergeRegions(env); 203 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_CLOSE_REGIONS); 204 break; 205 case MERGE_TABLE_REGIONS_CLOSE_REGIONS: 206 addChildProcedure(createUnassignProcedures(env)); 207 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_CHECK_CLOSED_REGIONS); 208 break; 209 case MERGE_TABLE_REGIONS_CHECK_CLOSED_REGIONS: 210 checkClosedRegions(env); 211 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_CREATE_MERGED_REGION); 212 break; 213 case MERGE_TABLE_REGIONS_CREATE_MERGED_REGION: 214 removeNonDefaultReplicas(env); 215 createMergedRegion(env); 216 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_WRITE_MAX_SEQUENCE_ID_FILE); 217 break; 218 case MERGE_TABLE_REGIONS_WRITE_MAX_SEQUENCE_ID_FILE: 219 writeMaxSequenceIdFile(env); 220 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_PRE_MERGE_COMMIT_OPERATION); 221 break; 222 case MERGE_TABLE_REGIONS_PRE_MERGE_COMMIT_OPERATION: 223 preMergeRegionsCommit(env); 224 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_UPDATE_META); 225 break; 226 case MERGE_TABLE_REGIONS_UPDATE_META: 227 updateMetaForMergedRegions(env); 228 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_POST_MERGE_COMMIT_OPERATION); 229 break; 230 case MERGE_TABLE_REGIONS_POST_MERGE_COMMIT_OPERATION: 231 postMergeRegionsCommit(env); 232 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_OPEN_MERGED_REGION); 233 break; 234 case MERGE_TABLE_REGIONS_OPEN_MERGED_REGION: 235 addChildProcedure(createAssignProcedures(env)); 236 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_POST_OPERATION); 237 break; 238 case MERGE_TABLE_REGIONS_POST_OPERATION: 239 postCompletedMergeRegions(env); 240 return Flow.NO_MORE_STATE; 241 default: 242 throw new UnsupportedOperationException(this + " unhandled state=" + state); 243 } 244 } catch (IOException e) { 245 String msg = "Error trying to merge " + RegionInfo.getShortNameToLog(regionsToMerge) + 246 " in " + getTableName() + " (in state=" + state + ")"; 247 if (!isRollbackSupported(state)) { 248 // We reach a state that cannot be rolled back. We just need to keep retrying. 249 LOG.warn(msg, e); 250 } else { 251 LOG.error(msg, e); 252 setFailure("master-merge-regions", e); 253 } 254 } 255 return Flow.HAS_MORE_STATE; 256 } 257 258 /** 259 * To rollback {@link MergeTableRegionsProcedure}, two AssignProcedures are asynchronously 260 * submitted for each region to be merged (rollback doesn't wait on the completion of the 261 * AssignProcedures) . This can be improved by changing rollback() to support sub-procedures. 262 * See HBASE-19851 for details. 263 */ 264 @Override 265 protected void rollbackState(final MasterProcedureEnv env, final MergeTableRegionsState state) 266 throws IOException { 267 LOG.trace("{} rollback state={}", this, state); 268 269 try { 270 switch (state) { 271 case MERGE_TABLE_REGIONS_POST_OPERATION: 272 case MERGE_TABLE_REGIONS_OPEN_MERGED_REGION: 273 case MERGE_TABLE_REGIONS_POST_MERGE_COMMIT_OPERATION: 274 case MERGE_TABLE_REGIONS_UPDATE_META: 275 String msg = this + " We are in the " + state + " state." + 276 " It is complicated to rollback the merge operation that region server is working on." + 277 " Rollback is not supported and we should let the merge operation to complete"; 278 LOG.warn(msg); 279 // PONR 280 throw new UnsupportedOperationException(this + " unhandled state=" + state); 281 case MERGE_TABLE_REGIONS_PRE_MERGE_COMMIT_OPERATION: 282 break; 283 case MERGE_TABLE_REGIONS_CREATE_MERGED_REGION: 284 case MERGE_TABLE_REGIONS_WRITE_MAX_SEQUENCE_ID_FILE: 285 cleanupMergedRegion(env); 286 break; 287 case MERGE_TABLE_REGIONS_CHECK_CLOSED_REGIONS: 288 break; 289 case MERGE_TABLE_REGIONS_CLOSE_REGIONS: 290 rollbackCloseRegionsForMerge(env); 291 break; 292 case MERGE_TABLE_REGIONS_PRE_MERGE_OPERATION: 293 postRollBackMergeRegions(env); 294 break; 295 case MERGE_TABLE_REGIONS_PREPARE: 296 break; 297 default: 298 throw new UnsupportedOperationException(this + " unhandled state=" + state); 299 } 300 } catch (Exception e) { 301 // This will be retried. Unless there is a bug in the code, 302 // this should be just a "temporary error" (e.g. network down) 303 LOG.warn("Failed rollback attempt step " + state + " for merging the regions " 304 + RegionInfo.getShortNameToLog(regionsToMerge) + " in table " + getTableName(), e); 305 throw e; 306 } 307 } 308 309 /* 310 * Check whether we are in the state that can be rolled back 311 */ 312 @Override 313 protected boolean isRollbackSupported(final MergeTableRegionsState state) { 314 switch (state) { 315 case MERGE_TABLE_REGIONS_POST_OPERATION: 316 case MERGE_TABLE_REGIONS_OPEN_MERGED_REGION: 317 case MERGE_TABLE_REGIONS_POST_MERGE_COMMIT_OPERATION: 318 case MERGE_TABLE_REGIONS_UPDATE_META: 319 // It is not safe to rollback in these states. 320 return false; 321 default: 322 break; 323 } 324 return true; 325 } 326 327 private void removeNonDefaultReplicas(MasterProcedureEnv env) throws IOException { 328 AssignmentManagerUtil.removeNonDefaultReplicas(env, Stream.of(regionsToMerge), 329 getRegionReplication(env)); 330 } 331 332 private void checkClosedRegions(MasterProcedureEnv env) throws IOException { 333 // Theoretically this should not happen any more after we use TRSP, but anyway 334 // let's add a check here 335 for (RegionInfo region : regionsToMerge) { 336 AssignmentManagerUtil.checkClosedRegion(env, region); 337 } 338 } 339 340 @Override 341 protected MergeTableRegionsState getState(final int stateId) { 342 return MergeTableRegionsState.forNumber(stateId); 343 } 344 345 @Override 346 protected int getStateId(final MergeTableRegionsState state) { 347 return state.getNumber(); 348 } 349 350 @Override 351 protected MergeTableRegionsState getInitialState() { 352 return MergeTableRegionsState.MERGE_TABLE_REGIONS_PREPARE; 353 } 354 355 @Override 356 protected void serializeStateData(ProcedureStateSerializer serializer) 357 throws IOException { 358 super.serializeStateData(serializer); 359 360 final MasterProcedureProtos.MergeTableRegionsStateData.Builder mergeTableRegionsMsg = 361 MasterProcedureProtos.MergeTableRegionsStateData.newBuilder() 362 .setUserInfo(MasterProcedureUtil.toProtoUserInfo(getUser())) 363 .setMergedRegionInfo(ProtobufUtil.toRegionInfo(mergedRegion)) 364 .setForcible(force); 365 for (RegionInfo ri: regionsToMerge) { 366 mergeTableRegionsMsg.addRegionInfo(ProtobufUtil.toRegionInfo(ri)); 367 } 368 serializer.serialize(mergeTableRegionsMsg.build()); 369 } 370 371 @Override 372 protected void deserializeStateData(ProcedureStateSerializer serializer) 373 throws IOException { 374 super.deserializeStateData(serializer); 375 376 final MasterProcedureProtos.MergeTableRegionsStateData mergeTableRegionsMsg = 377 serializer.deserialize(MasterProcedureProtos.MergeTableRegionsStateData.class); 378 setUser(MasterProcedureUtil.toUserInfo(mergeTableRegionsMsg.getUserInfo())); 379 380 assert(mergeTableRegionsMsg.getRegionInfoCount() == 2); 381 regionsToMerge = new RegionInfo[mergeTableRegionsMsg.getRegionInfoCount()]; 382 for (int i = 0; i < regionsToMerge.length; i++) { 383 regionsToMerge[i] = ProtobufUtil.toRegionInfo(mergeTableRegionsMsg.getRegionInfo(i)); 384 } 385 386 mergedRegion = ProtobufUtil.toRegionInfo(mergeTableRegionsMsg.getMergedRegionInfo()); 387 } 388 389 @Override 390 public void toStringClassDetails(StringBuilder sb) { 391 sb.append(getClass().getSimpleName()); 392 sb.append(" table="); 393 sb.append(getTableName()); 394 sb.append(", regions="); 395 sb.append(RegionInfo.getShortNameToLog(regionsToMerge)); 396 sb.append(", force="); 397 sb.append(force); 398 } 399 400 @Override 401 protected LockState acquireLock(final MasterProcedureEnv env) { 402 RegionInfo[] lockRegions = Arrays.copyOf(regionsToMerge, regionsToMerge.length + 1); 403 lockRegions[lockRegions.length - 1] = mergedRegion; 404 405 if (env.getProcedureScheduler().waitRegions(this, getTableName(), lockRegions)) { 406 try { 407 LOG.debug(LockState.LOCK_EVENT_WAIT + " " + env.getProcedureScheduler().dumpLocks()); 408 } catch (IOException e) { 409 // Ignore, just for logging 410 } 411 return LockState.LOCK_EVENT_WAIT; 412 } 413 return LockState.LOCK_ACQUIRED; 414 } 415 416 @Override 417 protected void releaseLock(final MasterProcedureEnv env) { 418 RegionInfo[] lockRegions = Arrays.copyOf(regionsToMerge, regionsToMerge.length + 1); 419 lockRegions[lockRegions.length - 1] = mergedRegion; 420 421 env.getProcedureScheduler().wakeRegions(this, getTableName(), lockRegions); 422 } 423 424 @Override 425 protected boolean holdLock(MasterProcedureEnv env) { 426 return true; 427 } 428 429 @Override 430 public TableName getTableName() { 431 return mergedRegion.getTable(); 432 } 433 434 @Override 435 public TableOperationType getTableOperationType() { 436 return TableOperationType.REGION_MERGE; 437 } 438 439 @Override 440 protected ProcedureMetrics getProcedureMetrics(MasterProcedureEnv env) { 441 return env.getAssignmentManager().getAssignmentManagerMetrics().getMergeProcMetrics(); 442 } 443 444 /** 445 * Prepare merge and do some check 446 */ 447 private boolean prepareMergeRegion(final MasterProcedureEnv env) throws IOException { 448 // Fail if we are taking snapshot for the given table 449 TableName tn = regionsToMerge[0].getTable(); 450 if (env.getMasterServices().getSnapshotManager().isTakingSnapshot(tn)) { 451 throw new MergeRegionException("Skip merging regions " + 452 RegionInfo.getShortNameToLog(regionsToMerge) + ", because we are snapshotting " + tn); 453 } 454 455 // Mostly this check is not used because we already check the switch before submit a merge 456 // procedure. Just for safe, check the switch again. This procedure can be rollbacked if 457 // the switch was set to false after submit. 458 if (!env.getMasterServices().isSplitOrMergeEnabled(MasterSwitchType.MERGE)) { 459 String regionsStr = Arrays.deepToString(this.regionsToMerge); 460 LOG.warn("Merge switch is off! skip merge of " + regionsStr); 461 setFailure(getClass().getSimpleName(), 462 new IOException("Merge of " + regionsStr + " failed because merge switch is off")); 463 return false; 464 } 465 466 if (!env.getMasterServices().getTableDescriptors().get(getTableName()).isMergeEnabled()) { 467 String regionsStr = Arrays.deepToString(regionsToMerge); 468 LOG.warn("Merge is disabled for the table! Skipping merge of {}", regionsStr); 469 setFailure(getClass().getSimpleName(), new IOException( 470 "Merge of " + regionsStr + " failed as region merge is disabled for the table")); 471 return false; 472 } 473 474 RegionStates regionStates = env.getAssignmentManager().getRegionStates(); 475 for (RegionInfo ri : this.regionsToMerge) { 476 if (MetaTableAccessor.hasMergeRegions(env.getMasterServices().getConnection(), 477 ri.getRegionName())) { 478 String msg = "Skip merging " + RegionInfo.getShortNameToLog(regionsToMerge) + 479 ", because a parent, " + RegionInfo.getShortNameToLog(ri) + ", has a merge qualifier " + 480 "(if a 'merge column' in parent, it was recently merged but still has outstanding " + 481 "references to its parents that must be cleared before it can participate in merge -- " + 482 "major compact it to hurry clearing of its references)"; 483 LOG.warn(msg); 484 throw new MergeRegionException(msg); 485 } 486 RegionState state = regionStates.getRegionState(ri.getEncodedName()); 487 if (state == null) { 488 throw new UnknownRegionException(RegionInfo.getShortNameToLog(ri) + 489 " UNKNOWN (Has it been garbage collected?)"); 490 } 491 if (!state.isOpened()) { 492 throw new MergeRegionException("Unable to merge regions that are NOT online: " + ri); 493 } 494 // Ask the remote regionserver if regions are mergeable. If we get an IOE, report it 495 // along with the failure, so we can see why regions are not mergeable at this time. 496 try { 497 if (!isMergeable(env, state)) { 498 setFailure(getClass().getSimpleName(), 499 new MergeRegionException( 500 "Skip merging " + RegionInfo.getShortNameToLog(regionsToMerge) + 501 ", because a parent, " + RegionInfo.getShortNameToLog(ri) + ", is not mergeable")); 502 return false; 503 } 504 } catch (IOException e) { 505 IOException ioe = new IOException(RegionInfo.getShortNameToLog(ri) + " NOT mergeable", e); 506 setFailure(getClass().getSimpleName(), ioe); 507 return false; 508 } 509 } 510 511 // Update region states to Merging 512 setRegionStateToMerging(env); 513 return true; 514 } 515 516 private boolean isMergeable(final MasterProcedureEnv env, final RegionState rs) 517 throws IOException { 518 GetRegionInfoResponse response = 519 AssignmentManagerUtil.getRegionInfoResponse(env, rs.getServerName(), rs.getRegion()); 520 return response.hasMergeable() && response.getMergeable(); 521 } 522 523 /** 524 * Pre merge region action 525 * @param env MasterProcedureEnv 526 **/ 527 private void preMergeRegions(final MasterProcedureEnv env) throws IOException { 528 final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost(); 529 if (cpHost != null) { 530 cpHost.preMergeRegionsAction(regionsToMerge, getUser()); 531 } 532 // TODO: Clean up split and merge. Currently all over the place. 533 try { 534 env.getMasterServices().getMasterQuotaManager().onRegionMerged(this.mergedRegion); 535 } catch (QuotaExceededException e) { 536 // TODO: why is this here? merge requests can be submitted by actors other than the normalizer 537 env.getMasterServices() 538 .getRegionNormalizerManager() 539 .planSkipped(NormalizationPlan.PlanType.MERGE); 540 throw e; 541 } 542 } 543 544 /** 545 * Action after rollback a merge table regions action. 546 */ 547 private void postRollBackMergeRegions(final MasterProcedureEnv env) throws IOException { 548 final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost(); 549 if (cpHost != null) { 550 cpHost.postRollBackMergeRegionsAction(regionsToMerge, getUser()); 551 } 552 } 553 554 /** 555 * Set the region states to MERGING state 556 */ 557 private void setRegionStateToMerging(final MasterProcedureEnv env) { 558 // Set State.MERGING to regions to be merged 559 RegionStates regionStates = env.getAssignmentManager().getRegionStates(); 560 for (RegionInfo ri: this.regionsToMerge) { 561 regionStates.getRegionStateNode(ri).setState(State.MERGING); 562 } 563 } 564 565 /** 566 * Create merged region. 567 * The way the merge works is that we make a 'merges' temporary 568 * directory in the FIRST parent region to merge (Do not change this without 569 * also changing the rollback where we look in this FIRST region for the 570 * merge dir). We then collect here references to all the store files in all 571 * the parent regions including those of the FIRST parent region into a 572 * subdirectory, named for the resultant merged region. We then call 573 * commitMergeRegion. It finds this subdirectory of storefile references 574 * and moves them under the new merge region (creating the region layout 575 * as side effect). After assign of the new merge region, we will run a 576 * compaction. This will undo the references but the reference files remain 577 * in place until the archiver runs (which it does on a period as a chore 578 * in the RegionServer that hosts the merge region -- see 579 * CompactedHFilesDischarger). Once the archiver has moved aside the 580 * no-longer used references, the merge region no longer has references. 581 * The catalog janitor will notice when it runs next and it will remove 582 * the old parent regions. 583 */ 584 private void createMergedRegion(final MasterProcedureEnv env) throws IOException { 585 final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); 586 final Path tabledir = CommonFSUtils.getTableDir(mfs.getRootDir(), regionsToMerge[0].getTable()); 587 final FileSystem fs = mfs.getFileSystem(); 588 HRegionFileSystem mergeRegionFs = null; 589 for (RegionInfo ri: this.regionsToMerge) { 590 HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem( 591 env.getMasterConfiguration(), fs, tabledir, ri, false); 592 if (mergeRegionFs == null) { 593 mergeRegionFs = regionFs; 594 mergeRegionFs.createMergesDir(); 595 } 596 mergeStoreFiles(env, regionFs, mergeRegionFs.getMergesDir()); 597 } 598 assert mergeRegionFs != null; 599 mergeRegionFs.commitMergedRegion(mergedRegion); 600 601 // Prepare to create merged regions 602 env.getAssignmentManager().getRegionStates(). 603 getOrCreateRegionStateNode(mergedRegion).setState(State.MERGING_NEW); 604 } 605 606 /** 607 * Create reference file(s) to parent region hfiles in the <code>mergeDir</code> 608 * @param regionFs merge parent region file system 609 * @param mergeDir the temp directory in which we are accumulating references. 610 */ 611 private void mergeStoreFiles(final MasterProcedureEnv env, final HRegionFileSystem regionFs, 612 final Path mergeDir) throws IOException { 613 final TableDescriptor htd = env.getMasterServices().getTableDescriptors().get(getTableName()); 614 for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) { 615 String family = hcd.getNameAsString(); 616 final Collection<StoreFileInfo> storeFiles = regionFs.getStoreFiles(family); 617 if (storeFiles != null && storeFiles.size() > 0) { 618 for (StoreFileInfo storeFileInfo : storeFiles) { 619 // Create reference file(s) to parent region file here in mergedDir. 620 // As this procedure is running on master, use CacheConfig.DISABLED means 621 // don't cache any block. 622 regionFs.mergeStoreFile(mergedRegion, family, new HStoreFile( 623 storeFileInfo, hcd.getBloomFilterType(), CacheConfig.DISABLED), mergeDir); 624 } 625 } 626 } 627 } 628 629 /** 630 * Clean up a merged region on rollback after failure. 631 */ 632 private void cleanupMergedRegion(final MasterProcedureEnv env) throws IOException { 633 final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); 634 TableName tn = this.regionsToMerge[0].getTable(); 635 final Path tabledir = CommonFSUtils.getTableDir(mfs.getRootDir(), tn); 636 final FileSystem fs = mfs.getFileSystem(); 637 // See createMergedRegion above where we specify the merge dir as being in the 638 // FIRST merge parent region. 639 HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem( 640 env.getMasterConfiguration(), fs, tabledir, regionsToMerge[0], false); 641 regionFs.cleanupMergedRegion(mergedRegion); 642 } 643 644 /** 645 * Rollback close regions 646 **/ 647 private void rollbackCloseRegionsForMerge(MasterProcedureEnv env) throws IOException { 648 AssignmentManagerUtil.reopenRegionsForRollback(env, Arrays.asList(regionsToMerge), 649 getRegionReplication(env), getServerName(env)); 650 } 651 652 private TransitRegionStateProcedure[] createUnassignProcedures(MasterProcedureEnv env) 653 throws IOException { 654 return AssignmentManagerUtil.createUnassignProceduresForSplitOrMerge(env, 655 Stream.of(regionsToMerge), getRegionReplication(env)); 656 } 657 658 private TransitRegionStateProcedure[] createAssignProcedures(MasterProcedureEnv env) 659 throws IOException { 660 return AssignmentManagerUtil.createAssignProceduresForOpeningNewRegions(env, 661 Collections.singletonList(mergedRegion), getRegionReplication(env), getServerName(env)); 662 } 663 664 private int getRegionReplication(final MasterProcedureEnv env) throws IOException { 665 return env.getMasterServices().getTableDescriptors().get(getTableName()). 666 getRegionReplication(); 667 } 668 669 /** 670 * Post merge region action 671 * @param env MasterProcedureEnv 672 **/ 673 private void preMergeRegionsCommit(final MasterProcedureEnv env) throws IOException { 674 final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost(); 675 if (cpHost != null) { 676 @MetaMutationAnnotation 677 final List<Mutation> metaEntries = new ArrayList<>(); 678 cpHost.preMergeRegionsCommit(regionsToMerge, metaEntries, getUser()); 679 try { 680 for (Mutation p : metaEntries) { 681 RegionInfo.parseRegionName(p.getRow()); 682 } 683 } catch (IOException e) { 684 LOG.error("Row key of mutation from coprocessor is not parsable as region name. " 685 + "Mutations from coprocessor should only be for hbase:meta table.", e); 686 throw e; 687 } 688 } 689 } 690 691 /** 692 * Add merged region to META and delete original regions. 693 */ 694 private void updateMetaForMergedRegions(final MasterProcedureEnv env) throws IOException { 695 env.getAssignmentManager().markRegionAsMerged(mergedRegion, getServerName(env), 696 this.regionsToMerge); 697 } 698 699 /** 700 * Post merge region action 701 * @param env MasterProcedureEnv 702 **/ 703 private void postMergeRegionsCommit(final MasterProcedureEnv env) throws IOException { 704 final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost(); 705 if (cpHost != null) { 706 cpHost.postMergeRegionsCommit(regionsToMerge, mergedRegion, getUser()); 707 } 708 } 709 710 /** 711 * Post merge region action 712 * @param env MasterProcedureEnv 713 **/ 714 private void postCompletedMergeRegions(final MasterProcedureEnv env) throws IOException { 715 final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost(); 716 if (cpHost != null) { 717 cpHost.postCompletedMergeRegionsAction(regionsToMerge, mergedRegion, getUser()); 718 } 719 } 720 721 /** 722 * The procedure could be restarted from a different machine. If the variable is null, we need to 723 * retrieve it. 724 * @param env MasterProcedureEnv 725 * @return serverName 726 */ 727 private ServerName getServerName(final MasterProcedureEnv env) { 728 if (regionLocation == null) { 729 regionLocation = env.getAssignmentManager().getRegionStates(). 730 getRegionServerOfRegion(regionsToMerge[0]); 731 // May still be null here but return null and let caller deal. 732 // Means we lost the in-memory-only location. We are in recovery 733 // or so. The caller should be able to deal w/ a null ServerName. 734 // Let them go to the Balancer to find one to use instead. 735 } 736 return regionLocation; 737 } 738 739 private void writeMaxSequenceIdFile(MasterProcedureEnv env) throws IOException { 740 MasterFileSystem fs = env.getMasterFileSystem(); 741 long maxSequenceId = -1L; 742 for (RegionInfo region : regionsToMerge) { 743 maxSequenceId = 744 Math.max(maxSequenceId, WALSplitUtil.getMaxRegionSequenceId(env.getMasterConfiguration(), 745 region, fs::getFileSystem, fs::getWALFileSystem)); 746 } 747 if (maxSequenceId > 0) { 748 WALSplitUtil.writeRegionSequenceIdFile(fs.getWALFileSystem(), 749 getWALRegionDir(env, mergedRegion), maxSequenceId); 750 } 751 } 752 753 /** 754 * @return The merged region. Maybe be null if called to early or we failed. 755 */ 756 RegionInfo getMergedRegion() { 757 return this.mergedRegion; 758 } 759 760 @Override 761 protected boolean abort(MasterProcedureEnv env) { 762 // Abort means rollback. We can't rollback all steps. HBASE-18018 added abort to all 763 // Procedures. Here is a Procedure that has a PONR and cannot be aborted once it enters this 764 // range of steps; what do we do for these should an operator want to cancel them? HBASE-20022. 765 return isRollbackSupported(getCurrentState()) && super.abort(env); 766 } 767}