001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.master.assignment; 019 020import java.io.IOException; 021import java.util.ArrayList; 022import java.util.Arrays; 023import java.util.Collection; 024import java.util.Collections; 025import java.util.List; 026import java.util.stream.Stream; 027import org.apache.hadoop.conf.Configuration; 028import org.apache.hadoop.fs.FileSystem; 029import org.apache.hadoop.fs.Path; 030import org.apache.hadoop.hbase.MetaMutationAnnotation; 031import org.apache.hadoop.hbase.ServerName; 032import org.apache.hadoop.hbase.TableName; 033import org.apache.hadoop.hbase.UnknownRegionException; 034import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; 035import org.apache.hadoop.hbase.client.DoNotRetryRegionException; 036import org.apache.hadoop.hbase.client.MasterSwitchType; 037import org.apache.hadoop.hbase.client.Mutation; 038import org.apache.hadoop.hbase.client.RegionInfo; 039import org.apache.hadoop.hbase.client.RegionInfoBuilder; 040import org.apache.hadoop.hbase.client.TableDescriptor; 041import org.apache.hadoop.hbase.exceptions.MergeRegionException; 042import org.apache.hadoop.hbase.io.hfile.CacheConfig; 043import org.apache.hadoop.hbase.master.MasterCoprocessorHost; 044import org.apache.hadoop.hbase.master.MasterFileSystem; 045import org.apache.hadoop.hbase.master.RegionState; 046import org.apache.hadoop.hbase.master.RegionState.State; 047import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan; 048import org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure; 049import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; 050import org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil; 051import org.apache.hadoop.hbase.procedure2.ProcedureMetrics; 052import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer; 053import org.apache.hadoop.hbase.quotas.MasterQuotaManager; 054import org.apache.hadoop.hbase.quotas.QuotaExceededException; 055import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; 056import org.apache.hadoop.hbase.regionserver.HStoreFile; 057import org.apache.hadoop.hbase.regionserver.StoreFileInfo; 058import org.apache.hadoop.hbase.regionserver.StoreUtils; 059import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; 060import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; 061import org.apache.hadoop.hbase.util.Bytes; 062import org.apache.hadoop.hbase.util.CommonFSUtils; 063import org.apache.hadoop.hbase.wal.WALSplitUtil; 064import org.apache.yetus.audience.InterfaceAudience; 065import org.slf4j.Logger; 066import org.slf4j.LoggerFactory; 067 068import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; 069import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoResponse; 070import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos; 071import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.MergeTableRegionsState; 072 073/** 074 * The procedure to Merge regions in a table. This procedure takes an exclusive table lock since it 075 * is working over multiple regions. It holds the lock for the life of the procedure. Throws 076 * exception on construction if determines context hostile to merge (cluster going down or master is 077 * shutting down or table is disabled). 078 */ 079@InterfaceAudience.Private 080public class MergeTableRegionsProcedure 081 extends AbstractStateMachineTableProcedure<MergeTableRegionsState> { 082 private static final Logger LOG = LoggerFactory.getLogger(MergeTableRegionsProcedure.class); 083 private ServerName regionLocation; 084 085 /** 086 * Two or more regions to merge, the 'merge parents'. 087 */ 088 private RegionInfo[] regionsToMerge; 089 090 /** 091 * The resulting merged region. 092 */ 093 private RegionInfo mergedRegion; 094 095 private boolean force; 096 097 public MergeTableRegionsProcedure() { 098 // Required by the Procedure framework to create the procedure on replay 099 } 100 101 public MergeTableRegionsProcedure(final MasterProcedureEnv env, final RegionInfo[] regionsToMerge, 102 final boolean force) throws IOException { 103 super(env); 104 // Check parent regions. Make sure valid before starting work. 105 // This check calls the super method #checkOnline also. 106 checkRegionsToMerge(env, regionsToMerge, force); 107 // Sort the regions going into the merge. 108 Arrays.sort(regionsToMerge); 109 this.regionsToMerge = regionsToMerge; 110 this.mergedRegion = createMergedRegionInfo(regionsToMerge); 111 // Preflight depends on mergedRegion being set (at least). 112 preflightChecks(env, true); 113 this.force = force; 114 } 115 116 /** 117 * @throws MergeRegionException If unable to merge regions for whatever reasons. 118 */ 119 private static void checkRegionsToMerge(MasterProcedureEnv env, final RegionInfo[] regions, 120 final boolean force) throws MergeRegionException { 121 long count = Arrays.stream(regions).distinct().count(); 122 if (regions.length != count) { 123 throw new MergeRegionException("Duplicate regions specified; cannot merge a region to " 124 + "itself. Passed in " + regions.length + " but only " + count + " unique."); 125 } 126 if (count < 2) { 127 throw new MergeRegionException("Need two Regions at least to run a Merge"); 128 } 129 RegionInfo previous = null; 130 for (RegionInfo ri : regions) { 131 if (previous != null) { 132 if (!previous.getTable().equals(ri.getTable())) { 133 String msg = "Can't merge regions from different tables: " + previous + ", " + ri; 134 LOG.warn(msg); 135 throw new MergeRegionException(msg); 136 } 137 if (!force && !ri.isAdjacent(previous) && !ri.isOverlap(previous)) { 138 String msg = "Unable to merge non-adjacent or non-overlapping regions '" 139 + previous.getShortNameToLog() + "', '" + ri.getShortNameToLog() + "' when force=false"; 140 LOG.warn(msg); 141 throw new MergeRegionException(msg); 142 } 143 } 144 145 if (ri.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) { 146 throw new MergeRegionException("Can't merge non-default replicas; " + ri); 147 } 148 try { 149 checkOnline(env, ri); 150 } catch (DoNotRetryRegionException dnrre) { 151 throw new MergeRegionException(dnrre); 152 } 153 154 previous = ri; 155 } 156 } 157 158 /** 159 * Create merged region info by looking at passed in <code>regionsToMerge</code> to figure what 160 * extremes for start and end keys to use; merged region needs to have an extent sufficient to 161 * cover all regions-to-merge. 162 */ 163 private static RegionInfo createMergedRegionInfo(final RegionInfo[] regionsToMerge) { 164 byte[] lowestStartKey = null; 165 byte[] highestEndKey = null; 166 // Region Id is a timestamp. Merged region's id can't be less than that of 167 // merging regions else will insert at wrong location in hbase:meta (See HBASE-710). 168 long highestRegionId = -1; 169 for (RegionInfo ri : regionsToMerge) { 170 if (lowestStartKey == null) { 171 lowestStartKey = ri.getStartKey(); 172 } else if (Bytes.compareTo(ri.getStartKey(), lowestStartKey) < 0) { 173 lowestStartKey = ri.getStartKey(); 174 } 175 if (highestEndKey == null) { 176 highestEndKey = ri.getEndKey(); 177 } else if (ri.isLast() || Bytes.compareTo(ri.getEndKey(), highestEndKey) > 0) { 178 highestEndKey = ri.getEndKey(); 179 } 180 highestRegionId = ri.getRegionId() > highestRegionId ? ri.getRegionId() : highestRegionId; 181 } 182 // Merged region is sorted between two merging regions in META 183 return RegionInfoBuilder.newBuilder(regionsToMerge[0].getTable()).setStartKey(lowestStartKey) 184 .setEndKey(highestEndKey).setSplit(false) 185 .setRegionId(highestRegionId + 1/* Add one so new merged region is highest */).build(); 186 } 187 188 @Override 189 protected Flow executeFromState(final MasterProcedureEnv env, MergeTableRegionsState state) { 190 LOG.trace("{} execute state={}", this, state); 191 try { 192 switch (state) { 193 case MERGE_TABLE_REGIONS_PREPARE: 194 if (!prepareMergeRegion(env)) { 195 assert isFailed() : "Merge region should have an exception here"; 196 return Flow.NO_MORE_STATE; 197 } 198 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_PRE_MERGE_OPERATION); 199 break; 200 case MERGE_TABLE_REGIONS_PRE_MERGE_OPERATION: 201 preMergeRegions(env); 202 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_CLOSE_REGIONS); 203 break; 204 case MERGE_TABLE_REGIONS_CLOSE_REGIONS: 205 addChildProcedure(createUnassignProcedures(env)); 206 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_CHECK_CLOSED_REGIONS); 207 break; 208 case MERGE_TABLE_REGIONS_CHECK_CLOSED_REGIONS: 209 checkClosedRegions(env); 210 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_CREATE_MERGED_REGION); 211 break; 212 case MERGE_TABLE_REGIONS_CREATE_MERGED_REGION: 213 removeNonDefaultReplicas(env); 214 createMergedRegion(env); 215 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_WRITE_MAX_SEQUENCE_ID_FILE); 216 break; 217 case MERGE_TABLE_REGIONS_WRITE_MAX_SEQUENCE_ID_FILE: 218 writeMaxSequenceIdFile(env); 219 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_PRE_MERGE_COMMIT_OPERATION); 220 break; 221 case MERGE_TABLE_REGIONS_PRE_MERGE_COMMIT_OPERATION: 222 preMergeRegionsCommit(env); 223 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_UPDATE_META); 224 break; 225 case MERGE_TABLE_REGIONS_UPDATE_META: 226 updateMetaForMergedRegions(env); 227 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_POST_MERGE_COMMIT_OPERATION); 228 break; 229 case MERGE_TABLE_REGIONS_POST_MERGE_COMMIT_OPERATION: 230 postMergeRegionsCommit(env); 231 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_OPEN_MERGED_REGION); 232 break; 233 case MERGE_TABLE_REGIONS_OPEN_MERGED_REGION: 234 addChildProcedure(createAssignProcedures(env)); 235 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_POST_OPERATION); 236 break; 237 case MERGE_TABLE_REGIONS_POST_OPERATION: 238 postCompletedMergeRegions(env); 239 return Flow.NO_MORE_STATE; 240 default: 241 throw new UnsupportedOperationException(this + " unhandled state=" + state); 242 } 243 } catch (IOException e) { 244 String msg = "Error trying to merge " + RegionInfo.getShortNameToLog(regionsToMerge) + " in " 245 + getTableName() + " (in state=" + state + ")"; 246 if (!isRollbackSupported(state)) { 247 // We reach a state that cannot be rolled back. We just need to keep retrying. 248 LOG.warn(msg, e); 249 } else { 250 LOG.error(msg, e); 251 setFailure("master-merge-regions", e); 252 } 253 } 254 return Flow.HAS_MORE_STATE; 255 } 256 257 /** 258 * To rollback {@link MergeTableRegionsProcedure}, two AssignProcedures are asynchronously 259 * submitted for each region to be merged (rollback doesn't wait on the completion of the 260 * AssignProcedures) . This can be improved by changing rollback() to support sub-procedures. See 261 * HBASE-19851 for details. 262 */ 263 @Override 264 protected void rollbackState(final MasterProcedureEnv env, final MergeTableRegionsState state) 265 throws IOException { 266 LOG.trace("{} rollback state={}", this, state); 267 268 try { 269 switch (state) { 270 case MERGE_TABLE_REGIONS_POST_OPERATION: 271 case MERGE_TABLE_REGIONS_OPEN_MERGED_REGION: 272 case MERGE_TABLE_REGIONS_POST_MERGE_COMMIT_OPERATION: 273 case MERGE_TABLE_REGIONS_UPDATE_META: 274 String msg = this + " We are in the " + state + " state." 275 + " It is complicated to rollback the merge operation that region server is working on." 276 + " Rollback is not supported and we should let the merge operation to complete"; 277 LOG.warn(msg); 278 // PONR 279 throw new UnsupportedOperationException(this + " unhandled state=" + state); 280 case MERGE_TABLE_REGIONS_PRE_MERGE_COMMIT_OPERATION: 281 break; 282 case MERGE_TABLE_REGIONS_CREATE_MERGED_REGION: 283 case MERGE_TABLE_REGIONS_WRITE_MAX_SEQUENCE_ID_FILE: 284 cleanupMergedRegion(env); 285 break; 286 case MERGE_TABLE_REGIONS_CHECK_CLOSED_REGIONS: 287 break; 288 case MERGE_TABLE_REGIONS_CLOSE_REGIONS: 289 rollbackCloseRegionsForMerge(env); 290 break; 291 case MERGE_TABLE_REGIONS_PRE_MERGE_OPERATION: 292 postRollBackMergeRegions(env); 293 break; 294 case MERGE_TABLE_REGIONS_PREPARE: 295 rollbackPrepareMerge(env); 296 break; 297 default: 298 throw new UnsupportedOperationException(this + " unhandled state=" + state); 299 } 300 } catch (Exception e) { 301 // This will be retried. Unless there is a bug in the code, 302 // this should be just a "temporary error" (e.g. network down) 303 LOG.warn("Failed rollback attempt step " + state + " for merging the regions " 304 + RegionInfo.getShortNameToLog(regionsToMerge) + " in table " + getTableName(), e); 305 throw e; 306 } 307 } 308 309 /* 310 * Check whether we are in the state that can be rolled back 311 */ 312 @Override 313 protected boolean isRollbackSupported(final MergeTableRegionsState state) { 314 switch (state) { 315 case MERGE_TABLE_REGIONS_POST_OPERATION: 316 case MERGE_TABLE_REGIONS_OPEN_MERGED_REGION: 317 case MERGE_TABLE_REGIONS_POST_MERGE_COMMIT_OPERATION: 318 case MERGE_TABLE_REGIONS_UPDATE_META: 319 // It is not safe to rollback in these states. 320 return false; 321 default: 322 break; 323 } 324 return true; 325 } 326 327 private void removeNonDefaultReplicas(MasterProcedureEnv env) throws IOException { 328 AssignmentManagerUtil.removeNonDefaultReplicas(env, Stream.of(regionsToMerge), 329 getRegionReplication(env)); 330 } 331 332 private void checkClosedRegions(MasterProcedureEnv env) throws IOException { 333 // Theoretically this should not happen any more after we use TRSP, but anyway 334 // let's add a check here 335 for (RegionInfo region : regionsToMerge) { 336 AssignmentManagerUtil.checkClosedRegion(env, region); 337 } 338 } 339 340 @Override 341 protected MergeTableRegionsState getState(final int stateId) { 342 return MergeTableRegionsState.forNumber(stateId); 343 } 344 345 @Override 346 protected int getStateId(final MergeTableRegionsState state) { 347 return state.getNumber(); 348 } 349 350 @Override 351 protected MergeTableRegionsState getInitialState() { 352 return MergeTableRegionsState.MERGE_TABLE_REGIONS_PREPARE; 353 } 354 355 @Override 356 protected void serializeStateData(ProcedureStateSerializer serializer) throws IOException { 357 super.serializeStateData(serializer); 358 359 final MasterProcedureProtos.MergeTableRegionsStateData.Builder mergeTableRegionsMsg = 360 MasterProcedureProtos.MergeTableRegionsStateData.newBuilder() 361 .setUserInfo(MasterProcedureUtil.toProtoUserInfo(getUser())) 362 .setMergedRegionInfo(ProtobufUtil.toRegionInfo(mergedRegion)).setForcible(force); 363 for (RegionInfo ri : regionsToMerge) { 364 mergeTableRegionsMsg.addRegionInfo(ProtobufUtil.toRegionInfo(ri)); 365 } 366 serializer.serialize(mergeTableRegionsMsg.build()); 367 } 368 369 @Override 370 protected void deserializeStateData(ProcedureStateSerializer serializer) throws IOException { 371 super.deserializeStateData(serializer); 372 373 final MasterProcedureProtos.MergeTableRegionsStateData mergeTableRegionsMsg = 374 serializer.deserialize(MasterProcedureProtos.MergeTableRegionsStateData.class); 375 setUser(MasterProcedureUtil.toUserInfo(mergeTableRegionsMsg.getUserInfo())); 376 377 assert (mergeTableRegionsMsg.getRegionInfoCount() == 2); 378 regionsToMerge = new RegionInfo[mergeTableRegionsMsg.getRegionInfoCount()]; 379 for (int i = 0; i < regionsToMerge.length; i++) { 380 regionsToMerge[i] = ProtobufUtil.toRegionInfo(mergeTableRegionsMsg.getRegionInfo(i)); 381 } 382 383 mergedRegion = ProtobufUtil.toRegionInfo(mergeTableRegionsMsg.getMergedRegionInfo()); 384 } 385 386 @Override 387 public void toStringClassDetails(StringBuilder sb) { 388 sb.append(getClass().getSimpleName()); 389 sb.append(" table="); 390 sb.append(getTableName()); 391 sb.append(", regions="); 392 sb.append(RegionInfo.getShortNameToLog(regionsToMerge)); 393 sb.append(", force="); 394 sb.append(force); 395 } 396 397 @Override 398 protected LockState acquireLock(final MasterProcedureEnv env) { 399 RegionInfo[] lockRegions = Arrays.copyOf(regionsToMerge, regionsToMerge.length + 1); 400 lockRegions[lockRegions.length - 1] = mergedRegion; 401 402 if (env.getProcedureScheduler().waitRegions(this, getTableName(), lockRegions)) { 403 try { 404 LOG.debug(LockState.LOCK_EVENT_WAIT + " " + env.getProcedureScheduler().dumpLocks()); 405 } catch (IOException e) { 406 // Ignore, just for logging 407 } 408 return LockState.LOCK_EVENT_WAIT; 409 } 410 return LockState.LOCK_ACQUIRED; 411 } 412 413 @Override 414 protected void releaseLock(final MasterProcedureEnv env) { 415 RegionInfo[] lockRegions = Arrays.copyOf(regionsToMerge, regionsToMerge.length + 1); 416 lockRegions[lockRegions.length - 1] = mergedRegion; 417 418 env.getProcedureScheduler().wakeRegions(this, getTableName(), lockRegions); 419 } 420 421 @Override 422 protected boolean holdLock(MasterProcedureEnv env) { 423 return true; 424 } 425 426 @Override 427 public TableName getTableName() { 428 return mergedRegion.getTable(); 429 } 430 431 @Override 432 public TableOperationType getTableOperationType() { 433 return TableOperationType.REGION_MERGE; 434 } 435 436 @Override 437 protected ProcedureMetrics getProcedureMetrics(MasterProcedureEnv env) { 438 return env.getAssignmentManager().getAssignmentManagerMetrics().getMergeProcMetrics(); 439 } 440 441 /** 442 * Prepare merge and do some check 443 */ 444 private boolean prepareMergeRegion(final MasterProcedureEnv env) throws IOException { 445 // Fail if we are taking snapshot for the given table 446 TableName tn = regionsToMerge[0].getTable(); 447 final String regionNamesToLog = RegionInfo.getShortNameToLog(regionsToMerge); 448 if (env.getMasterServices().getSnapshotManager().isTableTakingAnySnapshot(tn)) { 449 throw new MergeRegionException( 450 "Skip merging regions " + regionNamesToLog + ", because we are snapshotting " + tn); 451 } 452 453 /* 454 * Sometimes a ModifyTableProcedure has edited a table descriptor to change the number of region 455 * replicas for a table, but it has not yet opened/closed the new replicas. The 456 * ModifyTableProcedure assumes that nobody else will do the opening/closing of the new 457 * replicas, but a concurrent MergeTableRegionProcedure would violate that assumption. 458 */ 459 if (isTableModificationInProgress(env)) { 460 setFailure(getClass().getSimpleName(), 461 new IOException("Skip merging regions " + regionNamesToLog 462 + ", because there is an active procedure that is modifying the table " + tn)); 463 return false; 464 } 465 466 // Mostly the below two checks are not used because we already check the switches before 467 // submitting the merge procedure. Just for safety, we are checking the switch again here. 468 // Also, in case the switch was set to false after submission, this procedure can be rollbacked, 469 // thanks to this double check! 470 // case 1: check for cluster level switch 471 if (!env.getMasterServices().isSplitOrMergeEnabled(MasterSwitchType.MERGE)) { 472 LOG.warn("Merge switch is off! skip merge of " + regionNamesToLog); 473 setFailure(getClass().getSimpleName(), 474 new IOException("Merge of " + regionNamesToLog + " failed because merge switch is off")); 475 return false; 476 } 477 // case 2: check for table level switch 478 if (!env.getMasterServices().getTableDescriptors().get(getTableName()).isMergeEnabled()) { 479 LOG.warn("Merge is disabled for the table! Skipping merge of {}", regionNamesToLog); 480 setFailure(getClass().getSimpleName(), new IOException( 481 "Merge of " + regionNamesToLog + " failed as region merge is disabled for the table")); 482 return false; 483 } 484 485 RegionStates regionStates = env.getAssignmentManager().getRegionStates(); 486 RegionStateStore regionStateStore = env.getAssignmentManager().getRegionStateStore(); 487 for (RegionInfo ri : this.regionsToMerge) { 488 if (regionStateStore.hasMergeRegions(ri)) { 489 String msg = "Skip merging " + regionNamesToLog + ", because a parent, " 490 + RegionInfo.getShortNameToLog(ri) + ", has a merge qualifier " 491 + "(if a 'merge column' in parent, it was recently merged but still has outstanding " 492 + "references to its parents that must be cleared before it can participate in merge -- " 493 + "major compact it to hurry clearing of its references)"; 494 LOG.warn(msg); 495 throw new MergeRegionException(msg); 496 } 497 RegionState state = regionStates.getRegionState(ri.getEncodedName()); 498 if (state == null) { 499 throw new UnknownRegionException( 500 RegionInfo.getShortNameToLog(ri) + " UNKNOWN (Has it been garbage collected?)"); 501 } 502 if (!state.isOpened()) { 503 throw new MergeRegionException("Unable to merge regions that are NOT online: " + ri); 504 } 505 // Ask the remote regionserver if regions are mergeable. If we get an IOE, report it 506 // along with the failure, so we can see why regions are not mergeable at this time. 507 try { 508 if (!isMergeable(env, state)) { 509 setFailure(getClass().getSimpleName(), 510 new MergeRegionException("Skip merging " + regionNamesToLog + ", because a parent, " 511 + RegionInfo.getShortNameToLog(ri) + ", is not mergeable")); 512 return false; 513 } 514 } catch (IOException e) { 515 IOException ioe = new IOException(RegionInfo.getShortNameToLog(ri) + " NOT mergeable", e); 516 setFailure(getClass().getSimpleName(), ioe); 517 return false; 518 } 519 } 520 521 // Update region states to Merging 522 setRegionStateToMerging(env); 523 return true; 524 } 525 526 private boolean isMergeable(final MasterProcedureEnv env, final RegionState rs) 527 throws IOException { 528 GetRegionInfoResponse response = 529 AssignmentManagerUtil.getRegionInfoResponse(env, rs.getServerName(), rs.getRegion()); 530 return response.hasMergeable() && response.getMergeable(); 531 } 532 533 /** 534 * Action for rollback a merge table after prepare merge 535 */ 536 private void rollbackPrepareMerge(final MasterProcedureEnv env) throws IOException { 537 for (RegionInfo rinfo : regionsToMerge) { 538 RegionStateNode regionStateNode = 539 env.getAssignmentManager().getRegionStates().getRegionStateNode(rinfo); 540 if (regionStateNode.getState() == State.MERGING) { 541 regionStateNode.setState(State.OPEN); 542 } 543 } 544 } 545 546 /** 547 * Pre merge region action 548 * @param env MasterProcedureEnv 549 **/ 550 private void preMergeRegions(final MasterProcedureEnv env) throws IOException { 551 final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost(); 552 if (cpHost != null) { 553 cpHost.preMergeRegionsAction(regionsToMerge, getUser()); 554 } 555 // TODO: Clean up split and merge. Currently all over the place. 556 try { 557 MasterQuotaManager masterQuotaManager = env.getMasterServices().getMasterQuotaManager(); 558 if (masterQuotaManager != null) { 559 masterQuotaManager.onRegionMerged(this.mergedRegion); 560 } 561 } catch (QuotaExceededException e) { 562 // TODO: why is this here? merge requests can be submitted by actors other than the normalizer 563 env.getMasterServices().getRegionNormalizerManager() 564 .planSkipped(NormalizationPlan.PlanType.MERGE); 565 throw e; 566 } 567 } 568 569 /** 570 * Action after rollback a merge table regions action. 571 */ 572 private void postRollBackMergeRegions(final MasterProcedureEnv env) throws IOException { 573 final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost(); 574 if (cpHost != null) { 575 cpHost.postRollBackMergeRegionsAction(regionsToMerge, getUser()); 576 } 577 } 578 579 /** 580 * Set the region states to MERGING state 581 */ 582 private void setRegionStateToMerging(final MasterProcedureEnv env) { 583 // Set State.MERGING to regions to be merged 584 RegionStates regionStates = env.getAssignmentManager().getRegionStates(); 585 for (RegionInfo ri : this.regionsToMerge) { 586 regionStates.getRegionStateNode(ri).setState(State.MERGING); 587 } 588 } 589 590 /** 591 * Create merged region. The way the merge works is that we make a 'merges' temporary directory in 592 * the FIRST parent region to merge (Do not change this without also changing the rollback where 593 * we look in this FIRST region for the merge dir). We then collect here references to all the 594 * store files in all the parent regions including those of the FIRST parent region into a 595 * subdirectory, named for the resultant merged region. We then call commitMergeRegion. It finds 596 * this subdirectory of storefile references and moves them under the new merge region (creating 597 * the region layout as side effect). After assign of the new merge region, we will run a 598 * compaction. This will undo the references but the reference files remain in place until the 599 * archiver runs (which it does on a period as a chore in the RegionServer that hosts the merge 600 * region -- see CompactedHFilesDischarger). Once the archiver has moved aside the no-longer used 601 * references, the merge region no longer has references. The catalog janitor will notice when it 602 * runs next and it will remove the old parent regions. 603 */ 604 private void createMergedRegion(final MasterProcedureEnv env) throws IOException { 605 final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); 606 final Path tableDir = CommonFSUtils.getTableDir(mfs.getRootDir(), regionsToMerge[0].getTable()); 607 final FileSystem fs = mfs.getFileSystem(); 608 List<Path> mergedFiles = new ArrayList<>(); 609 HRegionFileSystem mergeRegionFs = HRegionFileSystem 610 .createRegionOnFileSystem(env.getMasterConfiguration(), fs, tableDir, mergedRegion); 611 612 for (RegionInfo ri : this.regionsToMerge) { 613 HRegionFileSystem regionFs = HRegionFileSystem 614 .openRegionFromFileSystem(env.getMasterConfiguration(), fs, tableDir, ri, false); 615 mergedFiles.addAll(mergeStoreFiles(env, regionFs, mergeRegionFs, mergedRegion)); 616 } 617 assert mergeRegionFs != null; 618 mergeRegionFs.commitMergedRegion(mergedFiles, env); 619 620 // Prepare to create merged regions 621 env.getAssignmentManager().getRegionStates().getOrCreateRegionStateNode(mergedRegion) 622 .setState(State.MERGING_NEW); 623 } 624 625 private List<Path> mergeStoreFiles(MasterProcedureEnv env, HRegionFileSystem regionFs, 626 HRegionFileSystem mergeRegionFs, RegionInfo mergedRegion) throws IOException { 627 final TableDescriptor htd = 628 env.getMasterServices().getTableDescriptors().get(mergedRegion.getTable()); 629 List<Path> mergedFiles = new ArrayList<>(); 630 for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) { 631 String family = hcd.getNameAsString(); 632 StoreFileTracker tracker = 633 StoreFileTrackerFactory.create(env.getMasterConfiguration(), htd, hcd, regionFs); 634 final Collection<StoreFileInfo> storeFiles = tracker.load(); 635 if (storeFiles != null && storeFiles.size() > 0) { 636 final Configuration storeConfiguration = 637 StoreUtils.createStoreConfiguration(env.getMasterConfiguration(), htd, hcd); 638 for (StoreFileInfo storeFileInfo : storeFiles) { 639 // Create reference file(s) to parent region file here in mergedDir. 640 // As this procedure is running on master, use CacheConfig.DISABLED means 641 // don't cache any block. 642 // We also need to pass through a suitable CompoundConfiguration as if this 643 // is running in a regionserver's Store context, or we might not be able 644 // to read the hfiles. 645 storeFileInfo.setConf(storeConfiguration); 646 Path refFile = mergeRegionFs.mergeStoreFile(regionFs.getRegionInfo(), family, 647 new HStoreFile(storeFileInfo, hcd.getBloomFilterType(), CacheConfig.DISABLED), tracker); 648 mergedFiles.add(refFile); 649 } 650 } 651 } 652 return mergedFiles; 653 } 654 655 /** 656 * Clean up a merged region on rollback after failure. 657 */ 658 private void cleanupMergedRegion(final MasterProcedureEnv env) throws IOException { 659 final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); 660 TableName tn = this.regionsToMerge[0].getTable(); 661 final Path tabledir = CommonFSUtils.getTableDir(mfs.getRootDir(), tn); 662 final FileSystem fs = mfs.getFileSystem(); 663 // See createMergedRegion above where we specify the merge dir as being in the 664 // FIRST merge parent region. 665 HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem( 666 env.getMasterConfiguration(), fs, tabledir, regionsToMerge[0], false); 667 regionFs.cleanupMergedRegion(mergedRegion); 668 } 669 670 /** 671 * Rollback close regions 672 **/ 673 private void rollbackCloseRegionsForMerge(MasterProcedureEnv env) throws IOException { 674 // At this point we should check if region was actually closed. If it was not closed then we 675 // don't need to repoen the region and we can just change the regionNode state to OPEN. 676 // if it is alredy closed then we need to do a reopen of region 677 List<RegionInfo> toAssign = new ArrayList<>(); 678 for (RegionInfo rinfo : regionsToMerge) { 679 RegionStateNode regionStateNode = 680 env.getAssignmentManager().getRegionStates().getRegionStateNode(rinfo); 681 if (regionStateNode.getState() != State.MERGING) { 682 // same as before HBASE-28405 683 toAssign.add(rinfo); 684 } 685 } 686 AssignmentManagerUtil.reopenRegionsForRollback(env, toAssign, getRegionReplication(env), 687 getServerName(env)); 688 } 689 690 private TransitRegionStateProcedure[] createUnassignProcedures(MasterProcedureEnv env) 691 throws IOException { 692 return AssignmentManagerUtil.createUnassignProceduresForSplitOrMerge(env, 693 Stream.of(regionsToMerge), getRegionReplication(env)); 694 } 695 696 private TransitRegionStateProcedure[] createAssignProcedures(MasterProcedureEnv env) 697 throws IOException { 698 return AssignmentManagerUtil.createAssignProceduresForOpeningNewRegions(env, 699 Collections.singletonList(mergedRegion), getRegionReplication(env), getServerName(env)); 700 } 701 702 private int getRegionReplication(final MasterProcedureEnv env) throws IOException { 703 return env.getMasterServices().getTableDescriptors().get(getTableName()).getRegionReplication(); 704 } 705 706 /** 707 * Post merge region action 708 * @param env MasterProcedureEnv 709 **/ 710 private void preMergeRegionsCommit(final MasterProcedureEnv env) throws IOException { 711 final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost(); 712 if (cpHost != null) { 713 @MetaMutationAnnotation 714 final List<Mutation> metaEntries = new ArrayList<>(); 715 cpHost.preMergeRegionsCommit(regionsToMerge, metaEntries, getUser()); 716 try { 717 for (Mutation p : metaEntries) { 718 RegionInfo.parseRegionName(p.getRow()); 719 } 720 } catch (IOException e) { 721 LOG.error("Row key of mutation from coprocessor is not parsable as region name. " 722 + "Mutations from coprocessor should only be for hbase:meta table.", e); 723 throw e; 724 } 725 } 726 } 727 728 /** 729 * Add merged region to META and delete original regions. 730 */ 731 private void updateMetaForMergedRegions(final MasterProcedureEnv env) throws IOException { 732 env.getAssignmentManager().markRegionAsMerged(mergedRegion, getServerName(env), 733 this.regionsToMerge); 734 } 735 736 /** 737 * Post merge region action 738 * @param env MasterProcedureEnv 739 **/ 740 private void postMergeRegionsCommit(final MasterProcedureEnv env) throws IOException { 741 final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost(); 742 if (cpHost != null) { 743 cpHost.postMergeRegionsCommit(regionsToMerge, mergedRegion, getUser()); 744 } 745 } 746 747 /** 748 * Post merge region action 749 * @param env MasterProcedureEnv 750 **/ 751 private void postCompletedMergeRegions(final MasterProcedureEnv env) throws IOException { 752 final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost(); 753 if (cpHost != null) { 754 cpHost.postCompletedMergeRegionsAction(regionsToMerge, mergedRegion, getUser()); 755 } 756 } 757 758 /** 759 * The procedure could be restarted from a different machine. If the variable is null, we need to 760 * retrieve it. 761 * @param env MasterProcedureEnv 762 */ 763 private ServerName getServerName(final MasterProcedureEnv env) { 764 if (regionLocation == null) { 765 regionLocation = 766 env.getAssignmentManager().getRegionStates().getRegionServerOfRegion(regionsToMerge[0]); 767 // May still be null here but return null and let caller deal. 768 // Means we lost the in-memory-only location. We are in recovery 769 // or so. The caller should be able to deal w/ a null ServerName. 770 // Let them go to the Balancer to find one to use instead. 771 } 772 return regionLocation; 773 } 774 775 private void writeMaxSequenceIdFile(MasterProcedureEnv env) throws IOException { 776 MasterFileSystem fs = env.getMasterFileSystem(); 777 long maxSequenceId = -1L; 778 for (RegionInfo region : regionsToMerge) { 779 maxSequenceId = 780 Math.max(maxSequenceId, WALSplitUtil.getMaxRegionSequenceId(env.getMasterConfiguration(), 781 region, fs::getFileSystem, fs::getWALFileSystem)); 782 } 783 if (maxSequenceId > 0) { 784 WALSplitUtil.writeRegionSequenceIdFile(fs.getWALFileSystem(), 785 getWALRegionDir(env, mergedRegion), maxSequenceId); 786 } 787 } 788 789 /** Returns The merged region. Maybe be null if called to early or we failed. */ 790 RegionInfo getMergedRegion() { 791 return this.mergedRegion; 792 } 793 794 @Override 795 protected boolean abort(MasterProcedureEnv env) { 796 // Abort means rollback. We can't rollback all steps. HBASE-18018 added abort to all 797 // Procedures. Here is a Procedure that has a PONR and cannot be aborted once it enters this 798 // range of steps; what do we do for these should an operator want to cancel them? HBASE-20022. 799 return isRollbackSupported(getCurrentState()) && super.abort(env); 800 } 801}