001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.master.assignment; 019 020import java.io.IOException; 021import java.util.ArrayList; 022import java.util.Arrays; 023import java.util.Collection; 024import java.util.Collections; 025import java.util.List; 026import java.util.stream.Stream; 027import org.apache.hadoop.conf.Configuration; 028import org.apache.hadoop.fs.FileSystem; 029import org.apache.hadoop.fs.Path; 030import org.apache.hadoop.hbase.MetaMutationAnnotation; 031import org.apache.hadoop.hbase.ServerName; 032import org.apache.hadoop.hbase.TableName; 033import org.apache.hadoop.hbase.UnknownRegionException; 034import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; 035import org.apache.hadoop.hbase.client.DoNotRetryRegionException; 036import org.apache.hadoop.hbase.client.MasterSwitchType; 037import org.apache.hadoop.hbase.client.Mutation; 038import org.apache.hadoop.hbase.client.RegionInfo; 039import org.apache.hadoop.hbase.client.RegionInfoBuilder; 040import org.apache.hadoop.hbase.client.TableDescriptor; 041import org.apache.hadoop.hbase.exceptions.MergeRegionException; 042import org.apache.hadoop.hbase.io.hfile.CacheConfig; 043import org.apache.hadoop.hbase.master.MasterCoprocessorHost; 044import org.apache.hadoop.hbase.master.MasterFileSystem; 045import org.apache.hadoop.hbase.master.RegionState; 046import org.apache.hadoop.hbase.master.RegionState.State; 047import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan; 048import org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure; 049import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; 050import org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil; 051import org.apache.hadoop.hbase.procedure2.ProcedureMetrics; 052import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer; 053import org.apache.hadoop.hbase.quotas.QuotaExceededException; 054import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; 055import org.apache.hadoop.hbase.regionserver.HStoreFile; 056import org.apache.hadoop.hbase.regionserver.StoreFileInfo; 057import org.apache.hadoop.hbase.regionserver.StoreUtils; 058import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; 059import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; 060import org.apache.hadoop.hbase.util.Bytes; 061import org.apache.hadoop.hbase.util.CommonFSUtils; 062import org.apache.hadoop.hbase.wal.WALSplitUtil; 063import org.apache.yetus.audience.InterfaceAudience; 064import org.slf4j.Logger; 065import org.slf4j.LoggerFactory; 066 067import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; 068import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoResponse; 069import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos; 070import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.MergeTableRegionsState; 071 072/** 073 * The procedure to Merge regions in a table. This procedure takes an exclusive table lock since it 074 * is working over multiple regions. It holds the lock for the life of the procedure. Throws 075 * exception on construction if determines context hostile to merge (cluster going down or master is 076 * shutting down or table is disabled). 077 */ 078@InterfaceAudience.Private 079public class MergeTableRegionsProcedure 080 extends AbstractStateMachineTableProcedure<MergeTableRegionsState> { 081 private static final Logger LOG = LoggerFactory.getLogger(MergeTableRegionsProcedure.class); 082 private ServerName regionLocation; 083 084 /** 085 * Two or more regions to merge, the 'merge parents'. 086 */ 087 private RegionInfo[] regionsToMerge; 088 089 /** 090 * The resulting merged region. 091 */ 092 private RegionInfo mergedRegion; 093 094 private boolean force; 095 096 public MergeTableRegionsProcedure() { 097 // Required by the Procedure framework to create the procedure on replay 098 } 099 100 public MergeTableRegionsProcedure(final MasterProcedureEnv env, final RegionInfo[] regionsToMerge, 101 final boolean force) throws IOException { 102 super(env); 103 // Check parent regions. Make sure valid before starting work. 104 // This check calls the super method #checkOnline also. 105 checkRegionsToMerge(env, regionsToMerge, force); 106 // Sort the regions going into the merge. 107 Arrays.sort(regionsToMerge); 108 this.regionsToMerge = regionsToMerge; 109 this.mergedRegion = createMergedRegionInfo(regionsToMerge); 110 // Preflight depends on mergedRegion being set (at least). 111 preflightChecks(env, true); 112 this.force = force; 113 } 114 115 /** 116 * @throws MergeRegionException If unable to merge regions for whatever reasons. 117 */ 118 private static void checkRegionsToMerge(MasterProcedureEnv env, final RegionInfo[] regions, 119 final boolean force) throws MergeRegionException { 120 long count = Arrays.stream(regions).distinct().count(); 121 if (regions.length != count) { 122 throw new MergeRegionException("Duplicate regions specified; cannot merge a region to " 123 + "itself. Passed in " + regions.length + " but only " + count + " unique."); 124 } 125 if (count < 2) { 126 throw new MergeRegionException("Need two Regions at least to run a Merge"); 127 } 128 RegionInfo previous = null; 129 for (RegionInfo ri : regions) { 130 if (previous != null) { 131 if (!previous.getTable().equals(ri.getTable())) { 132 String msg = "Can't merge regions from different tables: " + previous + ", " + ri; 133 LOG.warn(msg); 134 throw new MergeRegionException(msg); 135 } 136 if (!force && !ri.isAdjacent(previous) && !ri.isOverlap(previous)) { 137 String msg = "Unable to merge non-adjacent or non-overlapping regions '" 138 + previous.getShortNameToLog() + "', '" + ri.getShortNameToLog() + "' when force=false"; 139 LOG.warn(msg); 140 throw new MergeRegionException(msg); 141 } 142 } 143 144 if (ri.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) { 145 throw new MergeRegionException("Can't merge non-default replicas; " + ri); 146 } 147 try { 148 checkOnline(env, ri); 149 } catch (DoNotRetryRegionException dnrre) { 150 throw new MergeRegionException(dnrre); 151 } 152 153 previous = ri; 154 } 155 } 156 157 /** 158 * Create merged region info by looking at passed in <code>regionsToMerge</code> to figure what 159 * extremes for start and end keys to use; merged region needs to have an extent sufficient to 160 * cover all regions-to-merge. 161 */ 162 private static RegionInfo createMergedRegionInfo(final RegionInfo[] regionsToMerge) { 163 byte[] lowestStartKey = null; 164 byte[] highestEndKey = null; 165 // Region Id is a timestamp. Merged region's id can't be less than that of 166 // merging regions else will insert at wrong location in hbase:meta (See HBASE-710). 167 long highestRegionId = -1; 168 for (RegionInfo ri : regionsToMerge) { 169 if (lowestStartKey == null) { 170 lowestStartKey = ri.getStartKey(); 171 } else if (Bytes.compareTo(ri.getStartKey(), lowestStartKey) < 0) { 172 lowestStartKey = ri.getStartKey(); 173 } 174 if (highestEndKey == null) { 175 highestEndKey = ri.getEndKey(); 176 } else if (ri.isLast() || Bytes.compareTo(ri.getEndKey(), highestEndKey) > 0) { 177 highestEndKey = ri.getEndKey(); 178 } 179 highestRegionId = ri.getRegionId() > highestRegionId ? ri.getRegionId() : highestRegionId; 180 } 181 // Merged region is sorted between two merging regions in META 182 return RegionInfoBuilder.newBuilder(regionsToMerge[0].getTable()).setStartKey(lowestStartKey) 183 .setEndKey(highestEndKey).setSplit(false) 184 .setRegionId(highestRegionId + 1/* Add one so new merged region is highest */).build(); 185 } 186 187 @Override 188 protected Flow executeFromState(final MasterProcedureEnv env, MergeTableRegionsState state) { 189 LOG.trace("{} execute state={}", this, state); 190 try { 191 switch (state) { 192 case MERGE_TABLE_REGIONS_PREPARE: 193 if (!prepareMergeRegion(env)) { 194 assert isFailed() : "Merge region should have an exception here"; 195 return Flow.NO_MORE_STATE; 196 } 197 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_PRE_MERGE_OPERATION); 198 break; 199 case MERGE_TABLE_REGIONS_PRE_MERGE_OPERATION: 200 preMergeRegions(env); 201 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_CLOSE_REGIONS); 202 break; 203 case MERGE_TABLE_REGIONS_CLOSE_REGIONS: 204 addChildProcedure(createUnassignProcedures(env)); 205 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_CHECK_CLOSED_REGIONS); 206 break; 207 case MERGE_TABLE_REGIONS_CHECK_CLOSED_REGIONS: 208 checkClosedRegions(env); 209 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_CREATE_MERGED_REGION); 210 break; 211 case MERGE_TABLE_REGIONS_CREATE_MERGED_REGION: 212 removeNonDefaultReplicas(env); 213 createMergedRegion(env); 214 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_WRITE_MAX_SEQUENCE_ID_FILE); 215 break; 216 case MERGE_TABLE_REGIONS_WRITE_MAX_SEQUENCE_ID_FILE: 217 writeMaxSequenceIdFile(env); 218 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_PRE_MERGE_COMMIT_OPERATION); 219 break; 220 case MERGE_TABLE_REGIONS_PRE_MERGE_COMMIT_OPERATION: 221 preMergeRegionsCommit(env); 222 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_UPDATE_META); 223 break; 224 case MERGE_TABLE_REGIONS_UPDATE_META: 225 updateMetaForMergedRegions(env); 226 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_POST_MERGE_COMMIT_OPERATION); 227 break; 228 case MERGE_TABLE_REGIONS_POST_MERGE_COMMIT_OPERATION: 229 postMergeRegionsCommit(env); 230 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_OPEN_MERGED_REGION); 231 break; 232 case MERGE_TABLE_REGIONS_OPEN_MERGED_REGION: 233 addChildProcedure(createAssignProcedures(env)); 234 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_POST_OPERATION); 235 break; 236 case MERGE_TABLE_REGIONS_POST_OPERATION: 237 postCompletedMergeRegions(env); 238 return Flow.NO_MORE_STATE; 239 default: 240 throw new UnsupportedOperationException(this + " unhandled state=" + state); 241 } 242 } catch (IOException e) { 243 String msg = "Error trying to merge " + RegionInfo.getShortNameToLog(regionsToMerge) + " in " 244 + getTableName() + " (in state=" + state + ")"; 245 if (!isRollbackSupported(state)) { 246 // We reach a state that cannot be rolled back. We just need to keep retrying. 247 LOG.warn(msg, e); 248 } else { 249 LOG.error(msg, e); 250 setFailure("master-merge-regions", e); 251 } 252 } 253 return Flow.HAS_MORE_STATE; 254 } 255 256 /** 257 * To rollback {@link MergeTableRegionsProcedure}, two AssignProcedures are asynchronously 258 * submitted for each region to be merged (rollback doesn't wait on the completion of the 259 * AssignProcedures) . This can be improved by changing rollback() to support sub-procedures. See 260 * HBASE-19851 for details. 261 */ 262 @Override 263 protected void rollbackState(final MasterProcedureEnv env, final MergeTableRegionsState state) 264 throws IOException { 265 LOG.trace("{} rollback state={}", this, state); 266 267 try { 268 switch (state) { 269 case MERGE_TABLE_REGIONS_POST_OPERATION: 270 case MERGE_TABLE_REGIONS_OPEN_MERGED_REGION: 271 case MERGE_TABLE_REGIONS_POST_MERGE_COMMIT_OPERATION: 272 case MERGE_TABLE_REGIONS_UPDATE_META: 273 String msg = this + " We are in the " + state + " state." 274 + " It is complicated to rollback the merge operation that region server is working on." 275 + " Rollback is not supported and we should let the merge operation to complete"; 276 LOG.warn(msg); 277 // PONR 278 throw new UnsupportedOperationException(this + " unhandled state=" + state); 279 case MERGE_TABLE_REGIONS_PRE_MERGE_COMMIT_OPERATION: 280 break; 281 case MERGE_TABLE_REGIONS_CREATE_MERGED_REGION: 282 case MERGE_TABLE_REGIONS_WRITE_MAX_SEQUENCE_ID_FILE: 283 cleanupMergedRegion(env); 284 break; 285 case MERGE_TABLE_REGIONS_CHECK_CLOSED_REGIONS: 286 break; 287 case MERGE_TABLE_REGIONS_CLOSE_REGIONS: 288 rollbackCloseRegionsForMerge(env); 289 break; 290 case MERGE_TABLE_REGIONS_PRE_MERGE_OPERATION: 291 postRollBackMergeRegions(env); 292 break; 293 case MERGE_TABLE_REGIONS_PREPARE: 294 rollbackPrepareMerge(env); 295 break; 296 default: 297 throw new UnsupportedOperationException(this + " unhandled state=" + state); 298 } 299 } catch (Exception e) { 300 // This will be retried. Unless there is a bug in the code, 301 // this should be just a "temporary error" (e.g. network down) 302 LOG.warn("Failed rollback attempt step " + state + " for merging the regions " 303 + RegionInfo.getShortNameToLog(regionsToMerge) + " in table " + getTableName(), e); 304 throw e; 305 } 306 } 307 308 /* 309 * Check whether we are in the state that can be rolled back 310 */ 311 @Override 312 protected boolean isRollbackSupported(final MergeTableRegionsState state) { 313 switch (state) { 314 case MERGE_TABLE_REGIONS_POST_OPERATION: 315 case MERGE_TABLE_REGIONS_OPEN_MERGED_REGION: 316 case MERGE_TABLE_REGIONS_POST_MERGE_COMMIT_OPERATION: 317 case MERGE_TABLE_REGIONS_UPDATE_META: 318 // It is not safe to rollback in these states. 319 return false; 320 default: 321 break; 322 } 323 return true; 324 } 325 326 private void removeNonDefaultReplicas(MasterProcedureEnv env) throws IOException { 327 AssignmentManagerUtil.removeNonDefaultReplicas(env, Stream.of(regionsToMerge), 328 getRegionReplication(env)); 329 } 330 331 private void checkClosedRegions(MasterProcedureEnv env) throws IOException { 332 // Theoretically this should not happen any more after we use TRSP, but anyway 333 // let's add a check here 334 for (RegionInfo region : regionsToMerge) { 335 AssignmentManagerUtil.checkClosedRegion(env, region); 336 } 337 } 338 339 @Override 340 protected MergeTableRegionsState getState(final int stateId) { 341 return MergeTableRegionsState.forNumber(stateId); 342 } 343 344 @Override 345 protected int getStateId(final MergeTableRegionsState state) { 346 return state.getNumber(); 347 } 348 349 @Override 350 protected MergeTableRegionsState getInitialState() { 351 return MergeTableRegionsState.MERGE_TABLE_REGIONS_PREPARE; 352 } 353 354 @Override 355 protected void serializeStateData(ProcedureStateSerializer serializer) throws IOException { 356 super.serializeStateData(serializer); 357 358 final MasterProcedureProtos.MergeTableRegionsStateData.Builder mergeTableRegionsMsg = 359 MasterProcedureProtos.MergeTableRegionsStateData.newBuilder() 360 .setUserInfo(MasterProcedureUtil.toProtoUserInfo(getUser())) 361 .setMergedRegionInfo(ProtobufUtil.toRegionInfo(mergedRegion)).setForcible(force); 362 for (RegionInfo ri : regionsToMerge) { 363 mergeTableRegionsMsg.addRegionInfo(ProtobufUtil.toRegionInfo(ri)); 364 } 365 serializer.serialize(mergeTableRegionsMsg.build()); 366 } 367 368 @Override 369 protected void deserializeStateData(ProcedureStateSerializer serializer) throws IOException { 370 super.deserializeStateData(serializer); 371 372 final MasterProcedureProtos.MergeTableRegionsStateData mergeTableRegionsMsg = 373 serializer.deserialize(MasterProcedureProtos.MergeTableRegionsStateData.class); 374 setUser(MasterProcedureUtil.toUserInfo(mergeTableRegionsMsg.getUserInfo())); 375 376 assert (mergeTableRegionsMsg.getRegionInfoCount() == 2); 377 regionsToMerge = new RegionInfo[mergeTableRegionsMsg.getRegionInfoCount()]; 378 for (int i = 0; i < regionsToMerge.length; i++) { 379 regionsToMerge[i] = ProtobufUtil.toRegionInfo(mergeTableRegionsMsg.getRegionInfo(i)); 380 } 381 382 mergedRegion = ProtobufUtil.toRegionInfo(mergeTableRegionsMsg.getMergedRegionInfo()); 383 } 384 385 @Override 386 public void toStringClassDetails(StringBuilder sb) { 387 sb.append(getClass().getSimpleName()); 388 sb.append(" table="); 389 sb.append(getTableName()); 390 sb.append(", regions="); 391 sb.append(RegionInfo.getShortNameToLog(regionsToMerge)); 392 sb.append(", force="); 393 sb.append(force); 394 } 395 396 @Override 397 protected LockState acquireLock(final MasterProcedureEnv env) { 398 RegionInfo[] lockRegions = Arrays.copyOf(regionsToMerge, regionsToMerge.length + 1); 399 lockRegions[lockRegions.length - 1] = mergedRegion; 400 401 if (env.getProcedureScheduler().waitRegions(this, getTableName(), lockRegions)) { 402 try { 403 LOG.debug(LockState.LOCK_EVENT_WAIT + " " + env.getProcedureScheduler().dumpLocks()); 404 } catch (IOException e) { 405 // Ignore, just for logging 406 } 407 return LockState.LOCK_EVENT_WAIT; 408 } 409 return LockState.LOCK_ACQUIRED; 410 } 411 412 @Override 413 protected void releaseLock(final MasterProcedureEnv env) { 414 RegionInfo[] lockRegions = Arrays.copyOf(regionsToMerge, regionsToMerge.length + 1); 415 lockRegions[lockRegions.length - 1] = mergedRegion; 416 417 env.getProcedureScheduler().wakeRegions(this, getTableName(), lockRegions); 418 } 419 420 @Override 421 protected boolean holdLock(MasterProcedureEnv env) { 422 return true; 423 } 424 425 @Override 426 public TableName getTableName() { 427 return mergedRegion.getTable(); 428 } 429 430 @Override 431 public TableOperationType getTableOperationType() { 432 return TableOperationType.REGION_MERGE; 433 } 434 435 @Override 436 protected ProcedureMetrics getProcedureMetrics(MasterProcedureEnv env) { 437 return env.getAssignmentManager().getAssignmentManagerMetrics().getMergeProcMetrics(); 438 } 439 440 /** 441 * Prepare merge and do some check 442 */ 443 private boolean prepareMergeRegion(final MasterProcedureEnv env) throws IOException { 444 // Fail if we are taking snapshot for the given table 445 TableName tn = regionsToMerge[0].getTable(); 446 if (env.getMasterServices().getSnapshotManager().isTableTakingAnySnapshot(tn)) { 447 throw new MergeRegionException("Skip merging regions " 448 + RegionInfo.getShortNameToLog(regionsToMerge) + ", because we are snapshotting " + tn); 449 } 450 451 // Mostly this check is not used because we already check the switch before submit a merge 452 // procedure. Just for safe, check the switch again. This procedure can be rollbacked if 453 // the switch was set to false after submit. 454 if (!env.getMasterServices().isSplitOrMergeEnabled(MasterSwitchType.MERGE)) { 455 String regionsStr = Arrays.deepToString(this.regionsToMerge); 456 LOG.warn("Merge switch is off! skip merge of " + regionsStr); 457 setFailure(getClass().getSimpleName(), 458 new IOException("Merge of " + regionsStr + " failed because merge switch is off")); 459 return false; 460 } 461 462 if (!env.getMasterServices().getTableDescriptors().get(getTableName()).isMergeEnabled()) { 463 String regionsStr = Arrays.deepToString(regionsToMerge); 464 LOG.warn("Merge is disabled for the table! Skipping merge of {}", regionsStr); 465 setFailure(getClass().getSimpleName(), new IOException( 466 "Merge of " + regionsStr + " failed as region merge is disabled for the table")); 467 return false; 468 } 469 470 RegionStates regionStates = env.getAssignmentManager().getRegionStates(); 471 RegionStateStore regionStateStore = env.getAssignmentManager().getRegionStateStore(); 472 for (RegionInfo ri : this.regionsToMerge) { 473 if (regionStateStore.hasMergeRegions(ri)) { 474 String msg = "Skip merging " + RegionInfo.getShortNameToLog(regionsToMerge) 475 + ", because a parent, " + RegionInfo.getShortNameToLog(ri) + ", has a merge qualifier " 476 + "(if a 'merge column' in parent, it was recently merged but still has outstanding " 477 + "references to its parents that must be cleared before it can participate in merge -- " 478 + "major compact it to hurry clearing of its references)"; 479 LOG.warn(msg); 480 throw new MergeRegionException(msg); 481 } 482 RegionState state = regionStates.getRegionState(ri.getEncodedName()); 483 if (state == null) { 484 throw new UnknownRegionException( 485 RegionInfo.getShortNameToLog(ri) + " UNKNOWN (Has it been garbage collected?)"); 486 } 487 if (!state.isOpened()) { 488 throw new MergeRegionException("Unable to merge regions that are NOT online: " + ri); 489 } 490 // Ask the remote regionserver if regions are mergeable. If we get an IOE, report it 491 // along with the failure, so we can see why regions are not mergeable at this time. 492 try { 493 if (!isMergeable(env, state)) { 494 setFailure(getClass().getSimpleName(), 495 new MergeRegionException("Skip merging " + RegionInfo.getShortNameToLog(regionsToMerge) 496 + ", because a parent, " + RegionInfo.getShortNameToLog(ri) + ", is not mergeable")); 497 return false; 498 } 499 } catch (IOException e) { 500 IOException ioe = new IOException(RegionInfo.getShortNameToLog(ri) + " NOT mergeable", e); 501 setFailure(getClass().getSimpleName(), ioe); 502 return false; 503 } 504 } 505 506 // Update region states to Merging 507 setRegionStateToMerging(env); 508 return true; 509 } 510 511 private boolean isMergeable(final MasterProcedureEnv env, final RegionState rs) 512 throws IOException { 513 GetRegionInfoResponse response = 514 AssignmentManagerUtil.getRegionInfoResponse(env, rs.getServerName(), rs.getRegion()); 515 return response.hasMergeable() && response.getMergeable(); 516 } 517 518 /** 519 * Action for rollback a merge table after prepare merge 520 */ 521 private void rollbackPrepareMerge(final MasterProcedureEnv env) throws IOException { 522 for (RegionInfo rinfo : regionsToMerge) { 523 RegionStateNode regionStateNode = 524 env.getAssignmentManager().getRegionStates().getRegionStateNode(rinfo); 525 if (regionStateNode.getState() == State.MERGING) { 526 regionStateNode.setState(State.OPEN); 527 } 528 } 529 } 530 531 /** 532 * Pre merge region action 533 * @param env MasterProcedureEnv 534 **/ 535 private void preMergeRegions(final MasterProcedureEnv env) throws IOException { 536 final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost(); 537 if (cpHost != null) { 538 cpHost.preMergeRegionsAction(regionsToMerge, getUser()); 539 } 540 // TODO: Clean up split and merge. Currently all over the place. 541 try { 542 env.getMasterServices().getMasterQuotaManager().onRegionMerged(this.mergedRegion); 543 } catch (QuotaExceededException e) { 544 // TODO: why is this here? merge requests can be submitted by actors other than the normalizer 545 env.getMasterServices().getRegionNormalizerManager() 546 .planSkipped(NormalizationPlan.PlanType.MERGE); 547 throw e; 548 } 549 } 550 551 /** 552 * Action after rollback a merge table regions action. 553 */ 554 private void postRollBackMergeRegions(final MasterProcedureEnv env) throws IOException { 555 final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost(); 556 if (cpHost != null) { 557 cpHost.postRollBackMergeRegionsAction(regionsToMerge, getUser()); 558 } 559 } 560 561 /** 562 * Set the region states to MERGING state 563 */ 564 private void setRegionStateToMerging(final MasterProcedureEnv env) { 565 // Set State.MERGING to regions to be merged 566 RegionStates regionStates = env.getAssignmentManager().getRegionStates(); 567 for (RegionInfo ri : this.regionsToMerge) { 568 regionStates.getRegionStateNode(ri).setState(State.MERGING); 569 } 570 } 571 572 /** 573 * Create merged region. The way the merge works is that we make a 'merges' temporary directory in 574 * the FIRST parent region to merge (Do not change this without also changing the rollback where 575 * we look in this FIRST region for the merge dir). We then collect here references to all the 576 * store files in all the parent regions including those of the FIRST parent region into a 577 * subdirectory, named for the resultant merged region. We then call commitMergeRegion. It finds 578 * this subdirectory of storefile references and moves them under the new merge region (creating 579 * the region layout as side effect). After assign of the new merge region, we will run a 580 * compaction. This will undo the references but the reference files remain in place until the 581 * archiver runs (which it does on a period as a chore in the RegionServer that hosts the merge 582 * region -- see CompactedHFilesDischarger). Once the archiver has moved aside the no-longer used 583 * references, the merge region no longer has references. The catalog janitor will notice when it 584 * runs next and it will remove the old parent regions. 585 */ 586 private void createMergedRegion(final MasterProcedureEnv env) throws IOException { 587 final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); 588 final Path tableDir = CommonFSUtils.getTableDir(mfs.getRootDir(), regionsToMerge[0].getTable()); 589 final FileSystem fs = mfs.getFileSystem(); 590 List<Path> mergedFiles = new ArrayList<>(); 591 HRegionFileSystem mergeRegionFs = HRegionFileSystem 592 .createRegionOnFileSystem(env.getMasterConfiguration(), fs, tableDir, mergedRegion); 593 594 for (RegionInfo ri : this.regionsToMerge) { 595 HRegionFileSystem regionFs = HRegionFileSystem 596 .openRegionFromFileSystem(env.getMasterConfiguration(), fs, tableDir, ri, false); 597 mergedFiles.addAll(mergeStoreFiles(env, regionFs, mergeRegionFs, mergedRegion)); 598 } 599 assert mergeRegionFs != null; 600 mergeRegionFs.commitMergedRegion(mergedFiles, env); 601 602 // Prepare to create merged regions 603 env.getAssignmentManager().getRegionStates().getOrCreateRegionStateNode(mergedRegion) 604 .setState(State.MERGING_NEW); 605 } 606 607 private List<Path> mergeStoreFiles(MasterProcedureEnv env, HRegionFileSystem regionFs, 608 HRegionFileSystem mergeRegionFs, RegionInfo mergedRegion) throws IOException { 609 final TableDescriptor htd = 610 env.getMasterServices().getTableDescriptors().get(mergedRegion.getTable()); 611 List<Path> mergedFiles = new ArrayList<>(); 612 for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) { 613 String family = hcd.getNameAsString(); 614 StoreFileTracker tracker = 615 StoreFileTrackerFactory.create(env.getMasterConfiguration(), htd, hcd, regionFs); 616 final Collection<StoreFileInfo> storeFiles = tracker.load(); 617 if (storeFiles != null && storeFiles.size() > 0) { 618 final Configuration storeConfiguration = 619 StoreUtils.createStoreConfiguration(env.getMasterConfiguration(), htd, hcd); 620 for (StoreFileInfo storeFileInfo : storeFiles) { 621 // Create reference file(s) to parent region file here in mergedDir. 622 // As this procedure is running on master, use CacheConfig.DISABLED means 623 // don't cache any block. 624 // We also need to pass through a suitable CompoundConfiguration as if this 625 // is running in a regionserver's Store context, or we might not be able 626 // to read the hfiles. 627 storeFileInfo.setConf(storeConfiguration); 628 Path refFile = mergeRegionFs.mergeStoreFile(regionFs.getRegionInfo(), family, 629 new HStoreFile(storeFileInfo, hcd.getBloomFilterType(), CacheConfig.DISABLED)); 630 mergedFiles.add(refFile); 631 } 632 } 633 } 634 return mergedFiles; 635 } 636 637 /** 638 * Clean up a merged region on rollback after failure. 639 */ 640 private void cleanupMergedRegion(final MasterProcedureEnv env) throws IOException { 641 final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); 642 TableName tn = this.regionsToMerge[0].getTable(); 643 final Path tabledir = CommonFSUtils.getTableDir(mfs.getRootDir(), tn); 644 final FileSystem fs = mfs.getFileSystem(); 645 // See createMergedRegion above where we specify the merge dir as being in the 646 // FIRST merge parent region. 647 HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem( 648 env.getMasterConfiguration(), fs, tabledir, regionsToMerge[0], false); 649 regionFs.cleanupMergedRegion(mergedRegion); 650 } 651 652 /** 653 * Rollback close regions 654 **/ 655 private void rollbackCloseRegionsForMerge(MasterProcedureEnv env) throws IOException { 656 // At this point we should check if region was actually closed. If it was not closed then we 657 // don't need to repoen the region and we can just change the regionNode state to OPEN. 658 // if it is alredy closed then we need to do a reopen of region 659 List<RegionInfo> toAssign = new ArrayList<>(); 660 for (RegionInfo rinfo : regionsToMerge) { 661 RegionStateNode regionStateNode = 662 env.getAssignmentManager().getRegionStates().getRegionStateNode(rinfo); 663 if (regionStateNode.getState() != State.MERGING) { 664 // same as before HBASE-28405 665 toAssign.add(rinfo); 666 } 667 } 668 AssignmentManagerUtil.reopenRegionsForRollback(env, toAssign, getRegionReplication(env), 669 getServerName(env)); 670 } 671 672 private TransitRegionStateProcedure[] createUnassignProcedures(MasterProcedureEnv env) 673 throws IOException { 674 return AssignmentManagerUtil.createUnassignProceduresForSplitOrMerge(env, 675 Stream.of(regionsToMerge), getRegionReplication(env)); 676 } 677 678 private TransitRegionStateProcedure[] createAssignProcedures(MasterProcedureEnv env) 679 throws IOException { 680 return AssignmentManagerUtil.createAssignProceduresForOpeningNewRegions(env, 681 Collections.singletonList(mergedRegion), getRegionReplication(env), getServerName(env)); 682 } 683 684 private int getRegionReplication(final MasterProcedureEnv env) throws IOException { 685 return env.getMasterServices().getTableDescriptors().get(getTableName()).getRegionReplication(); 686 } 687 688 /** 689 * Post merge region action 690 * @param env MasterProcedureEnv 691 **/ 692 private void preMergeRegionsCommit(final MasterProcedureEnv env) throws IOException { 693 final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost(); 694 if (cpHost != null) { 695 @MetaMutationAnnotation 696 final List<Mutation> metaEntries = new ArrayList<>(); 697 cpHost.preMergeRegionsCommit(regionsToMerge, metaEntries, getUser()); 698 try { 699 for (Mutation p : metaEntries) { 700 RegionInfo.parseRegionName(p.getRow()); 701 } 702 } catch (IOException e) { 703 LOG.error("Row key of mutation from coprocessor is not parsable as region name. " 704 + "Mutations from coprocessor should only be for hbase:meta table.", e); 705 throw e; 706 } 707 } 708 } 709 710 /** 711 * Add merged region to META and delete original regions. 712 */ 713 private void updateMetaForMergedRegions(final MasterProcedureEnv env) throws IOException { 714 env.getAssignmentManager().markRegionAsMerged(mergedRegion, getServerName(env), 715 this.regionsToMerge); 716 } 717 718 /** 719 * Post merge region action 720 * @param env MasterProcedureEnv 721 **/ 722 private void postMergeRegionsCommit(final MasterProcedureEnv env) throws IOException { 723 final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost(); 724 if (cpHost != null) { 725 cpHost.postMergeRegionsCommit(regionsToMerge, mergedRegion, getUser()); 726 } 727 } 728 729 /** 730 * Post merge region action 731 * @param env MasterProcedureEnv 732 **/ 733 private void postCompletedMergeRegions(final MasterProcedureEnv env) throws IOException { 734 final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost(); 735 if (cpHost != null) { 736 cpHost.postCompletedMergeRegionsAction(regionsToMerge, mergedRegion, getUser()); 737 } 738 } 739 740 /** 741 * The procedure could be restarted from a different machine. If the variable is null, we need to 742 * retrieve it. 743 * @param env MasterProcedureEnv 744 */ 745 private ServerName getServerName(final MasterProcedureEnv env) { 746 if (regionLocation == null) { 747 regionLocation = 748 env.getAssignmentManager().getRegionStates().getRegionServerOfRegion(regionsToMerge[0]); 749 // May still be null here but return null and let caller deal. 750 // Means we lost the in-memory-only location. We are in recovery 751 // or so. The caller should be able to deal w/ a null ServerName. 752 // Let them go to the Balancer to find one to use instead. 753 } 754 return regionLocation; 755 } 756 757 private void writeMaxSequenceIdFile(MasterProcedureEnv env) throws IOException { 758 MasterFileSystem fs = env.getMasterFileSystem(); 759 long maxSequenceId = -1L; 760 for (RegionInfo region : regionsToMerge) { 761 maxSequenceId = 762 Math.max(maxSequenceId, WALSplitUtil.getMaxRegionSequenceId(env.getMasterConfiguration(), 763 region, fs::getFileSystem, fs::getWALFileSystem)); 764 } 765 if (maxSequenceId > 0) { 766 WALSplitUtil.writeRegionSequenceIdFile(fs.getWALFileSystem(), 767 getWALRegionDir(env, mergedRegion), maxSequenceId); 768 } 769 } 770 771 /** Returns The merged region. Maybe be null if called to early or we failed. */ 772 RegionInfo getMergedRegion() { 773 return this.mergedRegion; 774 } 775 776 @Override 777 protected boolean abort(MasterProcedureEnv env) { 778 // Abort means rollback. We can't rollback all steps. HBASE-18018 added abort to all 779 // Procedures. Here is a Procedure that has a PONR and cannot be aborted once it enters this 780 // range of steps; what do we do for these should an operator want to cancel them? HBASE-20022. 781 return isRollbackSupported(getCurrentState()) && super.abort(env); 782 } 783}