001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.master.assignment; 019 020import java.io.IOException; 021import java.util.ArrayList; 022import java.util.Arrays; 023import java.util.Collection; 024import java.util.Collections; 025import java.util.List; 026import java.util.stream.Stream; 027import org.apache.hadoop.conf.Configuration; 028import org.apache.hadoop.fs.FileSystem; 029import org.apache.hadoop.fs.Path; 030import org.apache.hadoop.hbase.MetaMutationAnnotation; 031import org.apache.hadoop.hbase.MetaTableAccessor; 032import org.apache.hadoop.hbase.ServerName; 033import org.apache.hadoop.hbase.TableName; 034import org.apache.hadoop.hbase.UnknownRegionException; 035import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; 036import org.apache.hadoop.hbase.client.DoNotRetryRegionException; 037import org.apache.hadoop.hbase.client.MasterSwitchType; 038import org.apache.hadoop.hbase.client.Mutation; 039import org.apache.hadoop.hbase.client.RegionInfo; 040import org.apache.hadoop.hbase.client.RegionInfoBuilder; 041import org.apache.hadoop.hbase.client.TableDescriptor; 042import org.apache.hadoop.hbase.exceptions.MergeRegionException; 043import org.apache.hadoop.hbase.io.hfile.CacheConfig; 044import org.apache.hadoop.hbase.master.MasterCoprocessorHost; 045import org.apache.hadoop.hbase.master.MasterFileSystem; 046import org.apache.hadoop.hbase.master.RegionState; 047import org.apache.hadoop.hbase.master.RegionState.State; 048import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan; 049import org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure; 050import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; 051import org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil; 052import org.apache.hadoop.hbase.procedure2.ProcedureMetrics; 053import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer; 054import org.apache.hadoop.hbase.quotas.QuotaExceededException; 055import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; 056import org.apache.hadoop.hbase.regionserver.HStoreFile; 057import org.apache.hadoop.hbase.regionserver.StoreFileInfo; 058import org.apache.hadoop.hbase.regionserver.StoreUtils; 059import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; 060import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; 061import org.apache.hadoop.hbase.util.Bytes; 062import org.apache.hadoop.hbase.util.CommonFSUtils; 063import org.apache.hadoop.hbase.wal.WALSplitUtil; 064import org.apache.yetus.audience.InterfaceAudience; 065import org.slf4j.Logger; 066import org.slf4j.LoggerFactory; 067 068import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; 069import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoResponse; 070import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos; 071import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.MergeTableRegionsState; 072 073/** 074 * The procedure to Merge regions in a table. This procedure takes an exclusive table lock since it 075 * is working over multiple regions. It holds the lock for the life of the procedure. Throws 076 * exception on construction if determines context hostile to merge (cluster going down or master is 077 * shutting down or table is disabled). 078 */ 079@InterfaceAudience.Private 080public class MergeTableRegionsProcedure 081 extends AbstractStateMachineTableProcedure<MergeTableRegionsState> { 082 private static final Logger LOG = LoggerFactory.getLogger(MergeTableRegionsProcedure.class); 083 private ServerName regionLocation; 084 085 /** 086 * Two or more regions to merge, the 'merge parents'. 087 */ 088 private RegionInfo[] regionsToMerge; 089 090 /** 091 * The resulting merged region. 092 */ 093 private RegionInfo mergedRegion; 094 095 private boolean force; 096 097 public MergeTableRegionsProcedure() { 098 // Required by the Procedure framework to create the procedure on replay 099 } 100 101 public MergeTableRegionsProcedure(final MasterProcedureEnv env, final RegionInfo[] regionsToMerge, 102 final boolean force) throws IOException { 103 super(env); 104 // Check parent regions. Make sure valid before starting work. 105 // This check calls the super method #checkOnline also. 106 checkRegionsToMerge(env, regionsToMerge, force); 107 // Sort the regions going into the merge. 108 Arrays.sort(regionsToMerge); 109 this.regionsToMerge = regionsToMerge; 110 this.mergedRegion = createMergedRegionInfo(regionsToMerge); 111 // Preflight depends on mergedRegion being set (at least). 112 preflightChecks(env, true); 113 this.force = force; 114 } 115 116 /** 117 * @throws MergeRegionException If unable to merge regions for whatever reasons. 118 */ 119 private static void checkRegionsToMerge(MasterProcedureEnv env, final RegionInfo[] regions, 120 final boolean force) throws MergeRegionException { 121 long count = Arrays.stream(regions).distinct().count(); 122 if (regions.length != count) { 123 throw new MergeRegionException("Duplicate regions specified; cannot merge a region to " 124 + "itself. Passed in " + regions.length + " but only " + count + " unique."); 125 } 126 if (count < 2) { 127 throw new MergeRegionException("Need two Regions at least to run a Merge"); 128 } 129 RegionInfo previous = null; 130 for (RegionInfo ri : regions) { 131 if (previous != null) { 132 if (!previous.getTable().equals(ri.getTable())) { 133 String msg = "Can't merge regions from different tables: " + previous + ", " + ri; 134 LOG.warn(msg); 135 throw new MergeRegionException(msg); 136 } 137 if (!force && !ri.isAdjacent(previous) && !ri.isOverlap(previous)) { 138 String msg = "Unable to merge non-adjacent or non-overlapping regions '" 139 + previous.getShortNameToLog() + "', '" + ri.getShortNameToLog() + "' when force=false"; 140 LOG.warn(msg); 141 throw new MergeRegionException(msg); 142 } 143 } 144 145 if (ri.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) { 146 throw new MergeRegionException("Can't merge non-default replicas; " + ri); 147 } 148 try { 149 checkOnline(env, ri); 150 } catch (DoNotRetryRegionException dnrre) { 151 throw new MergeRegionException(dnrre); 152 } 153 154 previous = ri; 155 } 156 } 157 158 /** 159 * Create merged region info by looking at passed in <code>regionsToMerge</code> to figure what 160 * extremes for start and end keys to use; merged region needs to have an extent sufficient to 161 * cover all regions-to-merge. 162 */ 163 private static RegionInfo createMergedRegionInfo(final RegionInfo[] regionsToMerge) { 164 byte[] lowestStartKey = null; 165 byte[] highestEndKey = null; 166 // Region Id is a timestamp. Merged region's id can't be less than that of 167 // merging regions else will insert at wrong location in hbase:meta (See HBASE-710). 168 long highestRegionId = -1; 169 for (RegionInfo ri : regionsToMerge) { 170 if (lowestStartKey == null) { 171 lowestStartKey = ri.getStartKey(); 172 } else if (Bytes.compareTo(ri.getStartKey(), lowestStartKey) < 0) { 173 lowestStartKey = ri.getStartKey(); 174 } 175 if (highestEndKey == null) { 176 highestEndKey = ri.getEndKey(); 177 } else if (ri.isLast() || Bytes.compareTo(ri.getEndKey(), highestEndKey) > 0) { 178 highestEndKey = ri.getEndKey(); 179 } 180 highestRegionId = ri.getRegionId() > highestRegionId ? ri.getRegionId() : highestRegionId; 181 } 182 // Merged region is sorted between two merging regions in META 183 return RegionInfoBuilder.newBuilder(regionsToMerge[0].getTable()).setStartKey(lowestStartKey) 184 .setEndKey(highestEndKey).setSplit(false) 185 .setRegionId(highestRegionId + 1/* Add one so new merged region is highest */).build(); 186 } 187 188 @Override 189 protected Flow executeFromState(final MasterProcedureEnv env, MergeTableRegionsState state) { 190 LOG.trace("{} execute state={}", this, state); 191 try { 192 switch (state) { 193 case MERGE_TABLE_REGIONS_PREPARE: 194 if (!prepareMergeRegion(env)) { 195 assert isFailed() : "Merge region should have an exception here"; 196 return Flow.NO_MORE_STATE; 197 } 198 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_PRE_MERGE_OPERATION); 199 break; 200 case MERGE_TABLE_REGIONS_PRE_MERGE_OPERATION: 201 preMergeRegions(env); 202 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_CLOSE_REGIONS); 203 break; 204 case MERGE_TABLE_REGIONS_CLOSE_REGIONS: 205 addChildProcedure(createUnassignProcedures(env)); 206 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_CHECK_CLOSED_REGIONS); 207 break; 208 case MERGE_TABLE_REGIONS_CHECK_CLOSED_REGIONS: 209 checkClosedRegions(env); 210 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_CREATE_MERGED_REGION); 211 break; 212 case MERGE_TABLE_REGIONS_CREATE_MERGED_REGION: 213 removeNonDefaultReplicas(env); 214 createMergedRegion(env); 215 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_WRITE_MAX_SEQUENCE_ID_FILE); 216 break; 217 case MERGE_TABLE_REGIONS_WRITE_MAX_SEQUENCE_ID_FILE: 218 writeMaxSequenceIdFile(env); 219 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_PRE_MERGE_COMMIT_OPERATION); 220 break; 221 case MERGE_TABLE_REGIONS_PRE_MERGE_COMMIT_OPERATION: 222 preMergeRegionsCommit(env); 223 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_UPDATE_META); 224 break; 225 case MERGE_TABLE_REGIONS_UPDATE_META: 226 updateMetaForMergedRegions(env); 227 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_POST_MERGE_COMMIT_OPERATION); 228 break; 229 case MERGE_TABLE_REGIONS_POST_MERGE_COMMIT_OPERATION: 230 postMergeRegionsCommit(env); 231 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_OPEN_MERGED_REGION); 232 break; 233 case MERGE_TABLE_REGIONS_OPEN_MERGED_REGION: 234 addChildProcedure(createAssignProcedures(env)); 235 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_POST_OPERATION); 236 break; 237 case MERGE_TABLE_REGIONS_POST_OPERATION: 238 postCompletedMergeRegions(env); 239 return Flow.NO_MORE_STATE; 240 default: 241 throw new UnsupportedOperationException(this + " unhandled state=" + state); 242 } 243 } catch (IOException e) { 244 String msg = "Error trying to merge " + RegionInfo.getShortNameToLog(regionsToMerge) + " in " 245 + getTableName() + " (in state=" + state + ")"; 246 if (!isRollbackSupported(state)) { 247 // We reach a state that cannot be rolled back. We just need to keep retrying. 248 LOG.warn(msg, e); 249 } else { 250 LOG.error(msg, e); 251 setFailure("master-merge-regions", e); 252 } 253 } 254 return Flow.HAS_MORE_STATE; 255 } 256 257 /** 258 * To rollback {@link MergeTableRegionsProcedure}, two AssignProcedures are asynchronously 259 * submitted for each region to be merged (rollback doesn't wait on the completion of the 260 * AssignProcedures) . This can be improved by changing rollback() to support sub-procedures. See 261 * HBASE-19851 for details. 262 */ 263 @Override 264 protected void rollbackState(final MasterProcedureEnv env, final MergeTableRegionsState state) 265 throws IOException { 266 LOG.trace("{} rollback state={}", this, state); 267 268 try { 269 switch (state) { 270 case MERGE_TABLE_REGIONS_POST_OPERATION: 271 case MERGE_TABLE_REGIONS_OPEN_MERGED_REGION: 272 case MERGE_TABLE_REGIONS_POST_MERGE_COMMIT_OPERATION: 273 case MERGE_TABLE_REGIONS_UPDATE_META: 274 String msg = this + " We are in the " + state + " state." 275 + " It is complicated to rollback the merge operation that region server is working on." 276 + " Rollback is not supported and we should let the merge operation to complete"; 277 LOG.warn(msg); 278 // PONR 279 throw new UnsupportedOperationException(this + " unhandled state=" + state); 280 case MERGE_TABLE_REGIONS_PRE_MERGE_COMMIT_OPERATION: 281 break; 282 case MERGE_TABLE_REGIONS_CREATE_MERGED_REGION: 283 case MERGE_TABLE_REGIONS_WRITE_MAX_SEQUENCE_ID_FILE: 284 cleanupMergedRegion(env); 285 break; 286 case MERGE_TABLE_REGIONS_CHECK_CLOSED_REGIONS: 287 break; 288 case MERGE_TABLE_REGIONS_CLOSE_REGIONS: 289 rollbackCloseRegionsForMerge(env); 290 break; 291 case MERGE_TABLE_REGIONS_PRE_MERGE_OPERATION: 292 postRollBackMergeRegions(env); 293 break; 294 case MERGE_TABLE_REGIONS_PREPARE: 295 break; 296 default: 297 throw new UnsupportedOperationException(this + " unhandled state=" + state); 298 } 299 } catch (Exception e) { 300 // This will be retried. Unless there is a bug in the code, 301 // this should be just a "temporary error" (e.g. network down) 302 LOG.warn("Failed rollback attempt step " + state + " for merging the regions " 303 + RegionInfo.getShortNameToLog(regionsToMerge) + " in table " + getTableName(), e); 304 throw e; 305 } 306 } 307 308 /* 309 * Check whether we are in the state that can be rolled back 310 */ 311 @Override 312 protected boolean isRollbackSupported(final MergeTableRegionsState state) { 313 switch (state) { 314 case MERGE_TABLE_REGIONS_POST_OPERATION: 315 case MERGE_TABLE_REGIONS_OPEN_MERGED_REGION: 316 case MERGE_TABLE_REGIONS_POST_MERGE_COMMIT_OPERATION: 317 case MERGE_TABLE_REGIONS_UPDATE_META: 318 // It is not safe to rollback in these states. 319 return false; 320 default: 321 break; 322 } 323 return true; 324 } 325 326 private void removeNonDefaultReplicas(MasterProcedureEnv env) throws IOException { 327 AssignmentManagerUtil.removeNonDefaultReplicas(env, Stream.of(regionsToMerge), 328 getRegionReplication(env)); 329 } 330 331 private void checkClosedRegions(MasterProcedureEnv env) throws IOException { 332 // Theoretically this should not happen any more after we use TRSP, but anyway 333 // let's add a check here 334 for (RegionInfo region : regionsToMerge) { 335 AssignmentManagerUtil.checkClosedRegion(env, region); 336 } 337 } 338 339 @Override 340 protected MergeTableRegionsState getState(final int stateId) { 341 return MergeTableRegionsState.forNumber(stateId); 342 } 343 344 @Override 345 protected int getStateId(final MergeTableRegionsState state) { 346 return state.getNumber(); 347 } 348 349 @Override 350 protected MergeTableRegionsState getInitialState() { 351 return MergeTableRegionsState.MERGE_TABLE_REGIONS_PREPARE; 352 } 353 354 @Override 355 protected void serializeStateData(ProcedureStateSerializer serializer) throws IOException { 356 super.serializeStateData(serializer); 357 358 final MasterProcedureProtos.MergeTableRegionsStateData.Builder mergeTableRegionsMsg = 359 MasterProcedureProtos.MergeTableRegionsStateData.newBuilder() 360 .setUserInfo(MasterProcedureUtil.toProtoUserInfo(getUser())) 361 .setMergedRegionInfo(ProtobufUtil.toRegionInfo(mergedRegion)).setForcible(force); 362 for (RegionInfo ri : regionsToMerge) { 363 mergeTableRegionsMsg.addRegionInfo(ProtobufUtil.toRegionInfo(ri)); 364 } 365 serializer.serialize(mergeTableRegionsMsg.build()); 366 } 367 368 @Override 369 protected void deserializeStateData(ProcedureStateSerializer serializer) throws IOException { 370 super.deserializeStateData(serializer); 371 372 final MasterProcedureProtos.MergeTableRegionsStateData mergeTableRegionsMsg = 373 serializer.deserialize(MasterProcedureProtos.MergeTableRegionsStateData.class); 374 setUser(MasterProcedureUtil.toUserInfo(mergeTableRegionsMsg.getUserInfo())); 375 376 assert (mergeTableRegionsMsg.getRegionInfoCount() == 2); 377 regionsToMerge = new RegionInfo[mergeTableRegionsMsg.getRegionInfoCount()]; 378 for (int i = 0; i < regionsToMerge.length; i++) { 379 regionsToMerge[i] = ProtobufUtil.toRegionInfo(mergeTableRegionsMsg.getRegionInfo(i)); 380 } 381 382 mergedRegion = ProtobufUtil.toRegionInfo(mergeTableRegionsMsg.getMergedRegionInfo()); 383 } 384 385 @Override 386 public void toStringClassDetails(StringBuilder sb) { 387 sb.append(getClass().getSimpleName()); 388 sb.append(" table="); 389 sb.append(getTableName()); 390 sb.append(", regions="); 391 sb.append(RegionInfo.getShortNameToLog(regionsToMerge)); 392 sb.append(", force="); 393 sb.append(force); 394 } 395 396 @Override 397 protected LockState acquireLock(final MasterProcedureEnv env) { 398 RegionInfo[] lockRegions = Arrays.copyOf(regionsToMerge, regionsToMerge.length + 1); 399 lockRegions[lockRegions.length - 1] = mergedRegion; 400 401 if (env.getProcedureScheduler().waitRegions(this, getTableName(), lockRegions)) { 402 try { 403 LOG.debug(LockState.LOCK_EVENT_WAIT + " " + env.getProcedureScheduler().dumpLocks()); 404 } catch (IOException e) { 405 // Ignore, just for logging 406 } 407 return LockState.LOCK_EVENT_WAIT; 408 } 409 return LockState.LOCK_ACQUIRED; 410 } 411 412 @Override 413 protected void releaseLock(final MasterProcedureEnv env) { 414 RegionInfo[] lockRegions = Arrays.copyOf(regionsToMerge, regionsToMerge.length + 1); 415 lockRegions[lockRegions.length - 1] = mergedRegion; 416 417 env.getProcedureScheduler().wakeRegions(this, getTableName(), lockRegions); 418 } 419 420 @Override 421 protected boolean holdLock(MasterProcedureEnv env) { 422 return true; 423 } 424 425 @Override 426 public TableName getTableName() { 427 return mergedRegion.getTable(); 428 } 429 430 @Override 431 public TableOperationType getTableOperationType() { 432 return TableOperationType.REGION_MERGE; 433 } 434 435 @Override 436 protected ProcedureMetrics getProcedureMetrics(MasterProcedureEnv env) { 437 return env.getAssignmentManager().getAssignmentManagerMetrics().getMergeProcMetrics(); 438 } 439 440 /** 441 * Prepare merge and do some check 442 */ 443 private boolean prepareMergeRegion(final MasterProcedureEnv env) throws IOException { 444 // Fail if we are taking snapshot for the given table 445 TableName tn = regionsToMerge[0].getTable(); 446 if (env.getMasterServices().getSnapshotManager().isTakingSnapshot(tn)) { 447 throw new MergeRegionException("Skip merging regions " 448 + RegionInfo.getShortNameToLog(regionsToMerge) + ", because we are snapshotting " + tn); 449 } 450 451 // Mostly this check is not used because we already check the switch before submit a merge 452 // procedure. Just for safe, check the switch again. This procedure can be rollbacked if 453 // the switch was set to false after submit. 454 if (!env.getMasterServices().isSplitOrMergeEnabled(MasterSwitchType.MERGE)) { 455 String regionsStr = Arrays.deepToString(this.regionsToMerge); 456 LOG.warn("Merge switch is off! skip merge of " + regionsStr); 457 setFailure(getClass().getSimpleName(), 458 new IOException("Merge of " + regionsStr + " failed because merge switch is off")); 459 return false; 460 } 461 462 if (!env.getMasterServices().getTableDescriptors().get(getTableName()).isMergeEnabled()) { 463 String regionsStr = Arrays.deepToString(regionsToMerge); 464 LOG.warn("Merge is disabled for the table! Skipping merge of {}", regionsStr); 465 setFailure(getClass().getSimpleName(), new IOException( 466 "Merge of " + regionsStr + " failed as region merge is disabled for the table")); 467 return false; 468 } 469 470 RegionStates regionStates = env.getAssignmentManager().getRegionStates(); 471 for (RegionInfo ri : this.regionsToMerge) { 472 if ( 473 MetaTableAccessor.hasMergeRegions(env.getMasterServices().getConnection(), 474 ri.getRegionName()) 475 ) { 476 String msg = "Skip merging " + RegionInfo.getShortNameToLog(regionsToMerge) 477 + ", because a parent, " + RegionInfo.getShortNameToLog(ri) + ", has a merge qualifier " 478 + "(if a 'merge column' in parent, it was recently merged but still has outstanding " 479 + "references to its parents that must be cleared before it can participate in merge -- " 480 + "major compact it to hurry clearing of its references)"; 481 LOG.warn(msg); 482 throw new MergeRegionException(msg); 483 } 484 RegionState state = regionStates.getRegionState(ri.getEncodedName()); 485 if (state == null) { 486 throw new UnknownRegionException( 487 RegionInfo.getShortNameToLog(ri) + " UNKNOWN (Has it been garbage collected?)"); 488 } 489 if (!state.isOpened()) { 490 throw new MergeRegionException("Unable to merge regions that are NOT online: " + ri); 491 } 492 // Ask the remote regionserver if regions are mergeable. If we get an IOE, report it 493 // along with the failure, so we can see why regions are not mergeable at this time. 494 try { 495 if (!isMergeable(env, state)) { 496 setFailure(getClass().getSimpleName(), 497 new MergeRegionException("Skip merging " + RegionInfo.getShortNameToLog(regionsToMerge) 498 + ", because a parent, " + RegionInfo.getShortNameToLog(ri) + ", is not mergeable")); 499 return false; 500 } 501 } catch (IOException e) { 502 IOException ioe = new IOException(RegionInfo.getShortNameToLog(ri) + " NOT mergeable", e); 503 setFailure(getClass().getSimpleName(), ioe); 504 return false; 505 } 506 } 507 508 // Update region states to Merging 509 setRegionStateToMerging(env); 510 return true; 511 } 512 513 private boolean isMergeable(final MasterProcedureEnv env, final RegionState rs) 514 throws IOException { 515 GetRegionInfoResponse response = 516 AssignmentManagerUtil.getRegionInfoResponse(env, rs.getServerName(), rs.getRegion()); 517 return response.hasMergeable() && response.getMergeable(); 518 } 519 520 /** 521 * Pre merge region action 522 * @param env MasterProcedureEnv 523 **/ 524 private void preMergeRegions(final MasterProcedureEnv env) throws IOException { 525 final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost(); 526 if (cpHost != null) { 527 cpHost.preMergeRegionsAction(regionsToMerge, getUser()); 528 } 529 // TODO: Clean up split and merge. Currently all over the place. 530 try { 531 env.getMasterServices().getMasterQuotaManager().onRegionMerged(this.mergedRegion); 532 } catch (QuotaExceededException e) { 533 // TODO: why is this here? merge requests can be submitted by actors other than the normalizer 534 env.getMasterServices().getRegionNormalizerManager() 535 .planSkipped(NormalizationPlan.PlanType.MERGE); 536 throw e; 537 } 538 } 539 540 /** 541 * Action after rollback a merge table regions action. 542 */ 543 private void postRollBackMergeRegions(final MasterProcedureEnv env) throws IOException { 544 final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost(); 545 if (cpHost != null) { 546 cpHost.postRollBackMergeRegionsAction(regionsToMerge, getUser()); 547 } 548 } 549 550 /** 551 * Set the region states to MERGING state 552 */ 553 private void setRegionStateToMerging(final MasterProcedureEnv env) { 554 // Set State.MERGING to regions to be merged 555 RegionStates regionStates = env.getAssignmentManager().getRegionStates(); 556 for (RegionInfo ri : this.regionsToMerge) { 557 regionStates.getRegionStateNode(ri).setState(State.MERGING); 558 } 559 } 560 561 /** 562 * Create merged region. The way the merge works is that we make a 'merges' temporary directory in 563 * the FIRST parent region to merge (Do not change this without also changing the rollback where 564 * we look in this FIRST region for the merge dir). We then collect here references to all the 565 * store files in all the parent regions including those of the FIRST parent region into a 566 * subdirectory, named for the resultant merged region. We then call commitMergeRegion. It finds 567 * this subdirectory of storefile references and moves them under the new merge region (creating 568 * the region layout as side effect). After assign of the new merge region, we will run a 569 * compaction. This will undo the references but the reference files remain in place until the 570 * archiver runs (which it does on a period as a chore in the RegionServer that hosts the merge 571 * region -- see CompactedHFilesDischarger). Once the archiver has moved aside the no-longer used 572 * references, the merge region no longer has references. The catalog janitor will notice when it 573 * runs next and it will remove the old parent regions. 574 */ 575 private void createMergedRegion(final MasterProcedureEnv env) throws IOException { 576 final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); 577 final Path tableDir = CommonFSUtils.getTableDir(mfs.getRootDir(), regionsToMerge[0].getTable()); 578 final FileSystem fs = mfs.getFileSystem(); 579 List<Path> mergedFiles = new ArrayList<>(); 580 HRegionFileSystem mergeRegionFs = HRegionFileSystem 581 .createRegionOnFileSystem(env.getMasterConfiguration(), fs, tableDir, mergedRegion); 582 583 for (RegionInfo ri : this.regionsToMerge) { 584 HRegionFileSystem regionFs = HRegionFileSystem 585 .openRegionFromFileSystem(env.getMasterConfiguration(), fs, tableDir, ri, false); 586 mergedFiles.addAll(mergeStoreFiles(env, regionFs, mergeRegionFs, mergedRegion)); 587 } 588 assert mergeRegionFs != null; 589 mergeRegionFs.commitMergedRegion(mergedFiles, env); 590 591 // Prepare to create merged regions 592 env.getAssignmentManager().getRegionStates().getOrCreateRegionStateNode(mergedRegion) 593 .setState(State.MERGING_NEW); 594 } 595 596 private List<Path> mergeStoreFiles(MasterProcedureEnv env, HRegionFileSystem regionFs, 597 HRegionFileSystem mergeRegionFs, RegionInfo mergedRegion) throws IOException { 598 final TableDescriptor htd = 599 env.getMasterServices().getTableDescriptors().get(mergedRegion.getTable()); 600 List<Path> mergedFiles = new ArrayList<>(); 601 for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) { 602 String family = hcd.getNameAsString(); 603 StoreFileTracker tracker = 604 StoreFileTrackerFactory.create(env.getMasterConfiguration(), htd, hcd, regionFs); 605 final Collection<StoreFileInfo> storeFiles = tracker.load(); 606 if (storeFiles != null && storeFiles.size() > 0) { 607 final Configuration storeConfiguration = 608 StoreUtils.createStoreConfiguration(env.getMasterConfiguration(), htd, hcd); 609 for (StoreFileInfo storeFileInfo : storeFiles) { 610 // Create reference file(s) to parent region file here in mergedDir. 611 // As this procedure is running on master, use CacheConfig.DISABLED means 612 // don't cache any block. 613 // We also need to pass through a suitable CompoundConfiguration as if this 614 // is running in a regionserver's Store context, or we might not be able 615 // to read the hfiles. 616 storeFileInfo.setConf(storeConfiguration); 617 Path refFile = mergeRegionFs.mergeStoreFile(regionFs.getRegionInfo(), family, 618 new HStoreFile(storeFileInfo, hcd.getBloomFilterType(), CacheConfig.DISABLED)); 619 mergedFiles.add(refFile); 620 } 621 } 622 } 623 return mergedFiles; 624 } 625 626 /** 627 * Clean up a merged region on rollback after failure. 628 */ 629 private void cleanupMergedRegion(final MasterProcedureEnv env) throws IOException { 630 final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); 631 TableName tn = this.regionsToMerge[0].getTable(); 632 final Path tabledir = CommonFSUtils.getTableDir(mfs.getRootDir(), tn); 633 final FileSystem fs = mfs.getFileSystem(); 634 // See createMergedRegion above where we specify the merge dir as being in the 635 // FIRST merge parent region. 636 HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem( 637 env.getMasterConfiguration(), fs, tabledir, regionsToMerge[0], false); 638 regionFs.cleanupMergedRegion(mergedRegion); 639 } 640 641 /** 642 * Rollback close regions 643 **/ 644 private void rollbackCloseRegionsForMerge(MasterProcedureEnv env) throws IOException { 645 AssignmentManagerUtil.reopenRegionsForRollback(env, Arrays.asList(regionsToMerge), 646 getRegionReplication(env), getServerName(env)); 647 } 648 649 private TransitRegionStateProcedure[] createUnassignProcedures(MasterProcedureEnv env) 650 throws IOException { 651 return AssignmentManagerUtil.createUnassignProceduresForSplitOrMerge(env, 652 Stream.of(regionsToMerge), getRegionReplication(env)); 653 } 654 655 private TransitRegionStateProcedure[] createAssignProcedures(MasterProcedureEnv env) 656 throws IOException { 657 return AssignmentManagerUtil.createAssignProceduresForOpeningNewRegions(env, 658 Collections.singletonList(mergedRegion), getRegionReplication(env), getServerName(env)); 659 } 660 661 private int getRegionReplication(final MasterProcedureEnv env) throws IOException { 662 return env.getMasterServices().getTableDescriptors().get(getTableName()).getRegionReplication(); 663 } 664 665 /** 666 * Post merge region action 667 * @param env MasterProcedureEnv 668 **/ 669 private void preMergeRegionsCommit(final MasterProcedureEnv env) throws IOException { 670 final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost(); 671 if (cpHost != null) { 672 @MetaMutationAnnotation 673 final List<Mutation> metaEntries = new ArrayList<>(); 674 cpHost.preMergeRegionsCommit(regionsToMerge, metaEntries, getUser()); 675 try { 676 for (Mutation p : metaEntries) { 677 RegionInfo.parseRegionName(p.getRow()); 678 } 679 } catch (IOException e) { 680 LOG.error("Row key of mutation from coprocessor is not parsable as region name. " 681 + "Mutations from coprocessor should only be for hbase:meta table.", e); 682 throw e; 683 } 684 } 685 } 686 687 /** 688 * Add merged region to META and delete original regions. 689 */ 690 private void updateMetaForMergedRegions(final MasterProcedureEnv env) throws IOException { 691 env.getAssignmentManager().markRegionAsMerged(mergedRegion, getServerName(env), 692 this.regionsToMerge); 693 } 694 695 /** 696 * Post merge region action 697 * @param env MasterProcedureEnv 698 **/ 699 private void postMergeRegionsCommit(final MasterProcedureEnv env) throws IOException { 700 final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost(); 701 if (cpHost != null) { 702 cpHost.postMergeRegionsCommit(regionsToMerge, mergedRegion, getUser()); 703 } 704 } 705 706 /** 707 * Post merge region action 708 * @param env MasterProcedureEnv 709 **/ 710 private void postCompletedMergeRegions(final MasterProcedureEnv env) throws IOException { 711 final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost(); 712 if (cpHost != null) { 713 cpHost.postCompletedMergeRegionsAction(regionsToMerge, mergedRegion, getUser()); 714 } 715 } 716 717 /** 718 * The procedure could be restarted from a different machine. If the variable is null, we need to 719 * retrieve it. 720 * @param env MasterProcedureEnv n 721 */ 722 private ServerName getServerName(final MasterProcedureEnv env) { 723 if (regionLocation == null) { 724 regionLocation = 725 env.getAssignmentManager().getRegionStates().getRegionServerOfRegion(regionsToMerge[0]); 726 // May still be null here but return null and let caller deal. 727 // Means we lost the in-memory-only location. We are in recovery 728 // or so. The caller should be able to deal w/ a null ServerName. 729 // Let them go to the Balancer to find one to use instead. 730 } 731 return regionLocation; 732 } 733 734 private void writeMaxSequenceIdFile(MasterProcedureEnv env) throws IOException { 735 MasterFileSystem fs = env.getMasterFileSystem(); 736 long maxSequenceId = -1L; 737 for (RegionInfo region : regionsToMerge) { 738 maxSequenceId = 739 Math.max(maxSequenceId, WALSplitUtil.getMaxRegionSequenceId(env.getMasterConfiguration(), 740 region, fs::getFileSystem, fs::getWALFileSystem)); 741 } 742 if (maxSequenceId > 0) { 743 WALSplitUtil.writeRegionSequenceIdFile(fs.getWALFileSystem(), 744 getWALRegionDir(env, mergedRegion), maxSequenceId); 745 } 746 } 747 748 /** Returns The merged region. Maybe be null if called to early or we failed. */ 749 RegionInfo getMergedRegion() { 750 return this.mergedRegion; 751 } 752 753 @Override 754 protected boolean abort(MasterProcedureEnv env) { 755 // Abort means rollback. We can't rollback all steps. HBASE-18018 added abort to all 756 // Procedures. Here is a Procedure that has a PONR and cannot be aborted once it enters this 757 // range of steps; what do we do for these should an operator want to cancel them? HBASE-20022. 758 return isRollbackSupported(getCurrentState()) && super.abort(env); 759 } 760}