001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.master.assignment; 019 020import java.io.IOException; 021import java.util.ArrayList; 022import java.util.Arrays; 023import java.util.Collection; 024import java.util.Collections; 025import java.util.List; 026import java.util.stream.Stream; 027import org.apache.hadoop.conf.Configuration; 028import org.apache.hadoop.fs.FileSystem; 029import org.apache.hadoop.fs.Path; 030import org.apache.hadoop.hbase.MetaMutationAnnotation; 031import org.apache.hadoop.hbase.ServerName; 032import org.apache.hadoop.hbase.TableName; 033import org.apache.hadoop.hbase.UnknownRegionException; 034import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; 035import org.apache.hadoop.hbase.client.DoNotRetryRegionException; 036import org.apache.hadoop.hbase.client.MasterSwitchType; 037import org.apache.hadoop.hbase.client.Mutation; 038import org.apache.hadoop.hbase.client.RegionInfo; 039import org.apache.hadoop.hbase.client.RegionInfoBuilder; 040import org.apache.hadoop.hbase.client.TableDescriptor; 041import org.apache.hadoop.hbase.exceptions.MergeRegionException; 042import org.apache.hadoop.hbase.io.hfile.CacheConfig; 043import org.apache.hadoop.hbase.master.MasterCoprocessorHost; 044import org.apache.hadoop.hbase.master.MasterFileSystem; 045import org.apache.hadoop.hbase.master.RegionState; 046import org.apache.hadoop.hbase.master.RegionState.State; 047import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan; 048import org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure; 049import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; 050import org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil; 051import org.apache.hadoop.hbase.procedure2.ProcedureMetrics; 052import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer; 053import org.apache.hadoop.hbase.quotas.QuotaExceededException; 054import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; 055import org.apache.hadoop.hbase.regionserver.HStoreFile; 056import org.apache.hadoop.hbase.regionserver.StoreFileInfo; 057import org.apache.hadoop.hbase.regionserver.StoreUtils; 058import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; 059import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; 060import org.apache.hadoop.hbase.util.Bytes; 061import org.apache.hadoop.hbase.util.CommonFSUtils; 062import org.apache.hadoop.hbase.wal.WALSplitUtil; 063import org.apache.yetus.audience.InterfaceAudience; 064import org.slf4j.Logger; 065import org.slf4j.LoggerFactory; 066 067import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; 068import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoResponse; 069import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos; 070import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.MergeTableRegionsState; 071 072/** 073 * The procedure to Merge regions in a table. This procedure takes an exclusive table lock since it 074 * is working over multiple regions. It holds the lock for the life of the procedure. Throws 075 * exception on construction if determines context hostile to merge (cluster going down or master is 076 * shutting down or table is disabled). 077 */ 078@InterfaceAudience.Private 079public class MergeTableRegionsProcedure 080 extends AbstractStateMachineTableProcedure<MergeTableRegionsState> { 081 private static final Logger LOG = LoggerFactory.getLogger(MergeTableRegionsProcedure.class); 082 private ServerName regionLocation; 083 084 /** 085 * Two or more regions to merge, the 'merge parents'. 086 */ 087 private RegionInfo[] regionsToMerge; 088 089 /** 090 * The resulting merged region. 091 */ 092 private RegionInfo mergedRegion; 093 094 private boolean force; 095 096 public MergeTableRegionsProcedure() { 097 // Required by the Procedure framework to create the procedure on replay 098 } 099 100 public MergeTableRegionsProcedure(final MasterProcedureEnv env, final RegionInfo[] regionsToMerge, 101 final boolean force) throws IOException { 102 super(env); 103 // Check parent regions. Make sure valid before starting work. 104 // This check calls the super method #checkOnline also. 105 checkRegionsToMerge(env, regionsToMerge, force); 106 // Sort the regions going into the merge. 107 Arrays.sort(regionsToMerge); 108 this.regionsToMerge = regionsToMerge; 109 this.mergedRegion = createMergedRegionInfo(regionsToMerge); 110 // Preflight depends on mergedRegion being set (at least). 111 preflightChecks(env, true); 112 this.force = force; 113 } 114 115 /** 116 * @throws MergeRegionException If unable to merge regions for whatever reasons. 117 */ 118 private static void checkRegionsToMerge(MasterProcedureEnv env, final RegionInfo[] regions, 119 final boolean force) throws MergeRegionException { 120 long count = Arrays.stream(regions).distinct().count(); 121 if (regions.length != count) { 122 throw new MergeRegionException("Duplicate regions specified; cannot merge a region to " 123 + "itself. Passed in " + regions.length + " but only " + count + " unique."); 124 } 125 if (count < 2) { 126 throw new MergeRegionException("Need two Regions at least to run a Merge"); 127 } 128 RegionInfo previous = null; 129 for (RegionInfo ri : regions) { 130 if (previous != null) { 131 if (!previous.getTable().equals(ri.getTable())) { 132 String msg = "Can't merge regions from different tables: " + previous + ", " + ri; 133 LOG.warn(msg); 134 throw new MergeRegionException(msg); 135 } 136 if (!force && !ri.isAdjacent(previous) && !ri.isOverlap(previous)) { 137 String msg = "Unable to merge non-adjacent or non-overlapping regions '" 138 + previous.getShortNameToLog() + "', '" + ri.getShortNameToLog() + "' when force=false"; 139 LOG.warn(msg); 140 throw new MergeRegionException(msg); 141 } 142 } 143 144 if (ri.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) { 145 throw new MergeRegionException("Can't merge non-default replicas; " + ri); 146 } 147 try { 148 checkOnline(env, ri); 149 } catch (DoNotRetryRegionException dnrre) { 150 throw new MergeRegionException(dnrre); 151 } 152 153 previous = ri; 154 } 155 } 156 157 /** 158 * Create merged region info by looking at passed in <code>regionsToMerge</code> to figure what 159 * extremes for start and end keys to use; merged region needs to have an extent sufficient to 160 * cover all regions-to-merge. 161 */ 162 private static RegionInfo createMergedRegionInfo(final RegionInfo[] regionsToMerge) { 163 byte[] lowestStartKey = null; 164 byte[] highestEndKey = null; 165 // Region Id is a timestamp. Merged region's id can't be less than that of 166 // merging regions else will insert at wrong location in hbase:meta (See HBASE-710). 167 long highestRegionId = -1; 168 for (RegionInfo ri : regionsToMerge) { 169 if (lowestStartKey == null) { 170 lowestStartKey = ri.getStartKey(); 171 } else if (Bytes.compareTo(ri.getStartKey(), lowestStartKey) < 0) { 172 lowestStartKey = ri.getStartKey(); 173 } 174 if (highestEndKey == null) { 175 highestEndKey = ri.getEndKey(); 176 } else if (ri.isLast() || Bytes.compareTo(ri.getEndKey(), highestEndKey) > 0) { 177 highestEndKey = ri.getEndKey(); 178 } 179 highestRegionId = ri.getRegionId() > highestRegionId ? ri.getRegionId() : highestRegionId; 180 } 181 // Merged region is sorted between two merging regions in META 182 return RegionInfoBuilder.newBuilder(regionsToMerge[0].getTable()).setStartKey(lowestStartKey) 183 .setEndKey(highestEndKey).setSplit(false) 184 .setRegionId(highestRegionId + 1/* Add one so new merged region is highest */).build(); 185 } 186 187 @Override 188 protected Flow executeFromState(final MasterProcedureEnv env, MergeTableRegionsState state) { 189 LOG.trace("{} execute state={}", this, state); 190 try { 191 switch (state) { 192 case MERGE_TABLE_REGIONS_PREPARE: 193 if (!prepareMergeRegion(env)) { 194 assert isFailed() : "Merge region should have an exception here"; 195 return Flow.NO_MORE_STATE; 196 } 197 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_PRE_MERGE_OPERATION); 198 break; 199 case MERGE_TABLE_REGIONS_PRE_MERGE_OPERATION: 200 preMergeRegions(env); 201 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_CLOSE_REGIONS); 202 break; 203 case MERGE_TABLE_REGIONS_CLOSE_REGIONS: 204 addChildProcedure(createUnassignProcedures(env)); 205 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_CHECK_CLOSED_REGIONS); 206 break; 207 case MERGE_TABLE_REGIONS_CHECK_CLOSED_REGIONS: 208 checkClosedRegions(env); 209 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_CREATE_MERGED_REGION); 210 break; 211 case MERGE_TABLE_REGIONS_CREATE_MERGED_REGION: 212 removeNonDefaultReplicas(env); 213 createMergedRegion(env); 214 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_WRITE_MAX_SEQUENCE_ID_FILE); 215 break; 216 case MERGE_TABLE_REGIONS_WRITE_MAX_SEQUENCE_ID_FILE: 217 writeMaxSequenceIdFile(env); 218 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_PRE_MERGE_COMMIT_OPERATION); 219 break; 220 case MERGE_TABLE_REGIONS_PRE_MERGE_COMMIT_OPERATION: 221 preMergeRegionsCommit(env); 222 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_UPDATE_META); 223 break; 224 case MERGE_TABLE_REGIONS_UPDATE_META: 225 updateMetaForMergedRegions(env); 226 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_POST_MERGE_COMMIT_OPERATION); 227 break; 228 case MERGE_TABLE_REGIONS_POST_MERGE_COMMIT_OPERATION: 229 postMergeRegionsCommit(env); 230 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_OPEN_MERGED_REGION); 231 break; 232 case MERGE_TABLE_REGIONS_OPEN_MERGED_REGION: 233 addChildProcedure(createAssignProcedures(env)); 234 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_POST_OPERATION); 235 break; 236 case MERGE_TABLE_REGIONS_POST_OPERATION: 237 postCompletedMergeRegions(env); 238 return Flow.NO_MORE_STATE; 239 default: 240 throw new UnsupportedOperationException(this + " unhandled state=" + state); 241 } 242 } catch (IOException e) { 243 String msg = "Error trying to merge " + RegionInfo.getShortNameToLog(regionsToMerge) + " in " 244 + getTableName() + " (in state=" + state + ")"; 245 if (!isRollbackSupported(state)) { 246 // We reach a state that cannot be rolled back. We just need to keep retrying. 247 LOG.warn(msg, e); 248 } else { 249 LOG.error(msg, e); 250 setFailure("master-merge-regions", e); 251 } 252 } 253 return Flow.HAS_MORE_STATE; 254 } 255 256 /** 257 * To rollback {@link MergeTableRegionsProcedure}, two AssignProcedures are asynchronously 258 * submitted for each region to be merged (rollback doesn't wait on the completion of the 259 * AssignProcedures) . This can be improved by changing rollback() to support sub-procedures. See 260 * HBASE-19851 for details. 261 */ 262 @Override 263 protected void rollbackState(final MasterProcedureEnv env, final MergeTableRegionsState state) 264 throws IOException { 265 LOG.trace("{} rollback state={}", this, state); 266 267 try { 268 switch (state) { 269 case MERGE_TABLE_REGIONS_POST_OPERATION: 270 case MERGE_TABLE_REGIONS_OPEN_MERGED_REGION: 271 case MERGE_TABLE_REGIONS_POST_MERGE_COMMIT_OPERATION: 272 case MERGE_TABLE_REGIONS_UPDATE_META: 273 String msg = this + " We are in the " + state + " state." 274 + " It is complicated to rollback the merge operation that region server is working on." 275 + " Rollback is not supported and we should let the merge operation to complete"; 276 LOG.warn(msg); 277 // PONR 278 throw new UnsupportedOperationException(this + " unhandled state=" + state); 279 case MERGE_TABLE_REGIONS_PRE_MERGE_COMMIT_OPERATION: 280 break; 281 case MERGE_TABLE_REGIONS_CREATE_MERGED_REGION: 282 case MERGE_TABLE_REGIONS_WRITE_MAX_SEQUENCE_ID_FILE: 283 cleanupMergedRegion(env); 284 break; 285 case MERGE_TABLE_REGIONS_CHECK_CLOSED_REGIONS: 286 break; 287 case MERGE_TABLE_REGIONS_CLOSE_REGIONS: 288 rollbackCloseRegionsForMerge(env); 289 break; 290 case MERGE_TABLE_REGIONS_PRE_MERGE_OPERATION: 291 postRollBackMergeRegions(env); 292 break; 293 case MERGE_TABLE_REGIONS_PREPARE: 294 break; 295 default: 296 throw new UnsupportedOperationException(this + " unhandled state=" + state); 297 } 298 } catch (Exception e) { 299 // This will be retried. Unless there is a bug in the code, 300 // this should be just a "temporary error" (e.g. network down) 301 LOG.warn("Failed rollback attempt step " + state + " for merging the regions " 302 + RegionInfo.getShortNameToLog(regionsToMerge) + " in table " + getTableName(), e); 303 throw e; 304 } 305 } 306 307 /* 308 * Check whether we are in the state that can be rolled back 309 */ 310 @Override 311 protected boolean isRollbackSupported(final MergeTableRegionsState state) { 312 switch (state) { 313 case MERGE_TABLE_REGIONS_POST_OPERATION: 314 case MERGE_TABLE_REGIONS_OPEN_MERGED_REGION: 315 case MERGE_TABLE_REGIONS_POST_MERGE_COMMIT_OPERATION: 316 case MERGE_TABLE_REGIONS_UPDATE_META: 317 // It is not safe to rollback in these states. 318 return false; 319 default: 320 break; 321 } 322 return true; 323 } 324 325 private void removeNonDefaultReplicas(MasterProcedureEnv env) throws IOException { 326 AssignmentManagerUtil.removeNonDefaultReplicas(env, Stream.of(regionsToMerge), 327 getRegionReplication(env)); 328 } 329 330 private void checkClosedRegions(MasterProcedureEnv env) throws IOException { 331 // Theoretically this should not happen any more after we use TRSP, but anyway 332 // let's add a check here 333 for (RegionInfo region : regionsToMerge) { 334 AssignmentManagerUtil.checkClosedRegion(env, region); 335 } 336 } 337 338 @Override 339 protected MergeTableRegionsState getState(final int stateId) { 340 return MergeTableRegionsState.forNumber(stateId); 341 } 342 343 @Override 344 protected int getStateId(final MergeTableRegionsState state) { 345 return state.getNumber(); 346 } 347 348 @Override 349 protected MergeTableRegionsState getInitialState() { 350 return MergeTableRegionsState.MERGE_TABLE_REGIONS_PREPARE; 351 } 352 353 @Override 354 protected void serializeStateData(ProcedureStateSerializer serializer) throws IOException { 355 super.serializeStateData(serializer); 356 357 final MasterProcedureProtos.MergeTableRegionsStateData.Builder mergeTableRegionsMsg = 358 MasterProcedureProtos.MergeTableRegionsStateData.newBuilder() 359 .setUserInfo(MasterProcedureUtil.toProtoUserInfo(getUser())) 360 .setMergedRegionInfo(ProtobufUtil.toRegionInfo(mergedRegion)).setForcible(force); 361 for (RegionInfo ri : regionsToMerge) { 362 mergeTableRegionsMsg.addRegionInfo(ProtobufUtil.toRegionInfo(ri)); 363 } 364 serializer.serialize(mergeTableRegionsMsg.build()); 365 } 366 367 @Override 368 protected void deserializeStateData(ProcedureStateSerializer serializer) throws IOException { 369 super.deserializeStateData(serializer); 370 371 final MasterProcedureProtos.MergeTableRegionsStateData mergeTableRegionsMsg = 372 serializer.deserialize(MasterProcedureProtos.MergeTableRegionsStateData.class); 373 setUser(MasterProcedureUtil.toUserInfo(mergeTableRegionsMsg.getUserInfo())); 374 375 assert (mergeTableRegionsMsg.getRegionInfoCount() == 2); 376 regionsToMerge = new RegionInfo[mergeTableRegionsMsg.getRegionInfoCount()]; 377 for (int i = 0; i < regionsToMerge.length; i++) { 378 regionsToMerge[i] = ProtobufUtil.toRegionInfo(mergeTableRegionsMsg.getRegionInfo(i)); 379 } 380 381 mergedRegion = ProtobufUtil.toRegionInfo(mergeTableRegionsMsg.getMergedRegionInfo()); 382 } 383 384 @Override 385 public void toStringClassDetails(StringBuilder sb) { 386 sb.append(getClass().getSimpleName()); 387 sb.append(" table="); 388 sb.append(getTableName()); 389 sb.append(", regions="); 390 sb.append(RegionInfo.getShortNameToLog(regionsToMerge)); 391 sb.append(", force="); 392 sb.append(force); 393 } 394 395 @Override 396 protected LockState acquireLock(final MasterProcedureEnv env) { 397 RegionInfo[] lockRegions = Arrays.copyOf(regionsToMerge, regionsToMerge.length + 1); 398 lockRegions[lockRegions.length - 1] = mergedRegion; 399 400 if (env.getProcedureScheduler().waitRegions(this, getTableName(), lockRegions)) { 401 try { 402 LOG.debug(LockState.LOCK_EVENT_WAIT + " " + env.getProcedureScheduler().dumpLocks()); 403 } catch (IOException e) { 404 // Ignore, just for logging 405 } 406 return LockState.LOCK_EVENT_WAIT; 407 } 408 return LockState.LOCK_ACQUIRED; 409 } 410 411 @Override 412 protected void releaseLock(final MasterProcedureEnv env) { 413 RegionInfo[] lockRegions = Arrays.copyOf(regionsToMerge, regionsToMerge.length + 1); 414 lockRegions[lockRegions.length - 1] = mergedRegion; 415 416 env.getProcedureScheduler().wakeRegions(this, getTableName(), lockRegions); 417 } 418 419 @Override 420 protected boolean holdLock(MasterProcedureEnv env) { 421 return true; 422 } 423 424 @Override 425 public TableName getTableName() { 426 return mergedRegion.getTable(); 427 } 428 429 @Override 430 public TableOperationType getTableOperationType() { 431 return TableOperationType.REGION_MERGE; 432 } 433 434 @Override 435 protected ProcedureMetrics getProcedureMetrics(MasterProcedureEnv env) { 436 return env.getAssignmentManager().getAssignmentManagerMetrics().getMergeProcMetrics(); 437 } 438 439 /** 440 * Prepare merge and do some check 441 */ 442 private boolean prepareMergeRegion(final MasterProcedureEnv env) throws IOException { 443 // Fail if we are taking snapshot for the given table 444 TableName tn = regionsToMerge[0].getTable(); 445 if (env.getMasterServices().getSnapshotManager().isTableTakingAnySnapshot(tn)) { 446 throw new MergeRegionException("Skip merging regions " 447 + RegionInfo.getShortNameToLog(regionsToMerge) + ", because we are snapshotting " + tn); 448 } 449 450 // Mostly this check is not used because we already check the switch before submit a merge 451 // procedure. Just for safe, check the switch again. This procedure can be rollbacked if 452 // the switch was set to false after submit. 453 if (!env.getMasterServices().isSplitOrMergeEnabled(MasterSwitchType.MERGE)) { 454 String regionsStr = Arrays.deepToString(this.regionsToMerge); 455 LOG.warn("Merge switch is off! skip merge of " + regionsStr); 456 setFailure(getClass().getSimpleName(), 457 new IOException("Merge of " + regionsStr + " failed because merge switch is off")); 458 return false; 459 } 460 461 if (!env.getMasterServices().getTableDescriptors().get(getTableName()).isMergeEnabled()) { 462 String regionsStr = Arrays.deepToString(regionsToMerge); 463 LOG.warn("Merge is disabled for the table! Skipping merge of {}", regionsStr); 464 setFailure(getClass().getSimpleName(), new IOException( 465 "Merge of " + regionsStr + " failed as region merge is disabled for the table")); 466 return false; 467 } 468 469 RegionStates regionStates = env.getAssignmentManager().getRegionStates(); 470 RegionStateStore regionStateStore = env.getAssignmentManager().getRegionStateStore(); 471 for (RegionInfo ri : this.regionsToMerge) { 472 if (regionStateStore.hasMergeRegions(ri)) { 473 String msg = "Skip merging " + RegionInfo.getShortNameToLog(regionsToMerge) 474 + ", because a parent, " + RegionInfo.getShortNameToLog(ri) + ", has a merge qualifier " 475 + "(if a 'merge column' in parent, it was recently merged but still has outstanding " 476 + "references to its parents that must be cleared before it can participate in merge -- " 477 + "major compact it to hurry clearing of its references)"; 478 LOG.warn(msg); 479 throw new MergeRegionException(msg); 480 } 481 RegionState state = regionStates.getRegionState(ri.getEncodedName()); 482 if (state == null) { 483 throw new UnknownRegionException( 484 RegionInfo.getShortNameToLog(ri) + " UNKNOWN (Has it been garbage collected?)"); 485 } 486 if (!state.isOpened()) { 487 throw new MergeRegionException("Unable to merge regions that are NOT online: " + ri); 488 } 489 // Ask the remote regionserver if regions are mergeable. If we get an IOE, report it 490 // along with the failure, so we can see why regions are not mergeable at this time. 491 try { 492 if (!isMergeable(env, state)) { 493 setFailure(getClass().getSimpleName(), 494 new MergeRegionException("Skip merging " + RegionInfo.getShortNameToLog(regionsToMerge) 495 + ", because a parent, " + RegionInfo.getShortNameToLog(ri) + ", is not mergeable")); 496 return false; 497 } 498 } catch (IOException e) { 499 IOException ioe = new IOException(RegionInfo.getShortNameToLog(ri) + " NOT mergeable", e); 500 setFailure(getClass().getSimpleName(), ioe); 501 return false; 502 } 503 } 504 505 // Update region states to Merging 506 setRegionStateToMerging(env); 507 return true; 508 } 509 510 private boolean isMergeable(final MasterProcedureEnv env, final RegionState rs) 511 throws IOException { 512 GetRegionInfoResponse response = 513 AssignmentManagerUtil.getRegionInfoResponse(env, rs.getServerName(), rs.getRegion()); 514 return response.hasMergeable() && response.getMergeable(); 515 } 516 517 /** 518 * Pre merge region action 519 * @param env MasterProcedureEnv 520 **/ 521 private void preMergeRegions(final MasterProcedureEnv env) throws IOException { 522 final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost(); 523 if (cpHost != null) { 524 cpHost.preMergeRegionsAction(regionsToMerge, getUser()); 525 } 526 // TODO: Clean up split and merge. Currently all over the place. 527 try { 528 env.getMasterServices().getMasterQuotaManager().onRegionMerged(this.mergedRegion); 529 } catch (QuotaExceededException e) { 530 // TODO: why is this here? merge requests can be submitted by actors other than the normalizer 531 env.getMasterServices().getRegionNormalizerManager() 532 .planSkipped(NormalizationPlan.PlanType.MERGE); 533 throw e; 534 } 535 } 536 537 /** 538 * Action after rollback a merge table regions action. 539 */ 540 private void postRollBackMergeRegions(final MasterProcedureEnv env) throws IOException { 541 final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost(); 542 if (cpHost != null) { 543 cpHost.postRollBackMergeRegionsAction(regionsToMerge, getUser()); 544 } 545 } 546 547 /** 548 * Set the region states to MERGING state 549 */ 550 private void setRegionStateToMerging(final MasterProcedureEnv env) { 551 // Set State.MERGING to regions to be merged 552 RegionStates regionStates = env.getAssignmentManager().getRegionStates(); 553 for (RegionInfo ri : this.regionsToMerge) { 554 regionStates.getRegionStateNode(ri).setState(State.MERGING); 555 } 556 } 557 558 /** 559 * Create merged region. The way the merge works is that we make a 'merges' temporary directory in 560 * the FIRST parent region to merge (Do not change this without also changing the rollback where 561 * we look in this FIRST region for the merge dir). We then collect here references to all the 562 * store files in all the parent regions including those of the FIRST parent region into a 563 * subdirectory, named for the resultant merged region. We then call commitMergeRegion. It finds 564 * this subdirectory of storefile references and moves them under the new merge region (creating 565 * the region layout as side effect). After assign of the new merge region, we will run a 566 * compaction. This will undo the references but the reference files remain in place until the 567 * archiver runs (which it does on a period as a chore in the RegionServer that hosts the merge 568 * region -- see CompactedHFilesDischarger). Once the archiver has moved aside the no-longer used 569 * references, the merge region no longer has references. The catalog janitor will notice when it 570 * runs next and it will remove the old parent regions. 571 */ 572 private void createMergedRegion(final MasterProcedureEnv env) throws IOException { 573 final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); 574 final Path tableDir = CommonFSUtils.getTableDir(mfs.getRootDir(), regionsToMerge[0].getTable()); 575 final FileSystem fs = mfs.getFileSystem(); 576 List<Path> mergedFiles = new ArrayList<>(); 577 HRegionFileSystem mergeRegionFs = HRegionFileSystem 578 .createRegionOnFileSystem(env.getMasterConfiguration(), fs, tableDir, mergedRegion); 579 580 for (RegionInfo ri : this.regionsToMerge) { 581 HRegionFileSystem regionFs = HRegionFileSystem 582 .openRegionFromFileSystem(env.getMasterConfiguration(), fs, tableDir, ri, false); 583 mergedFiles.addAll(mergeStoreFiles(env, regionFs, mergeRegionFs, mergedRegion)); 584 } 585 assert mergeRegionFs != null; 586 mergeRegionFs.commitMergedRegion(mergedFiles, env); 587 588 // Prepare to create merged regions 589 env.getAssignmentManager().getRegionStates().getOrCreateRegionStateNode(mergedRegion) 590 .setState(State.MERGING_NEW); 591 } 592 593 private List<Path> mergeStoreFiles(MasterProcedureEnv env, HRegionFileSystem regionFs, 594 HRegionFileSystem mergeRegionFs, RegionInfo mergedRegion) throws IOException { 595 final TableDescriptor htd = 596 env.getMasterServices().getTableDescriptors().get(mergedRegion.getTable()); 597 List<Path> mergedFiles = new ArrayList<>(); 598 for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) { 599 String family = hcd.getNameAsString(); 600 StoreFileTracker tracker = 601 StoreFileTrackerFactory.create(env.getMasterConfiguration(), htd, hcd, regionFs); 602 final Collection<StoreFileInfo> storeFiles = tracker.load(); 603 if (storeFiles != null && storeFiles.size() > 0) { 604 final Configuration storeConfiguration = 605 StoreUtils.createStoreConfiguration(env.getMasterConfiguration(), htd, hcd); 606 for (StoreFileInfo storeFileInfo : storeFiles) { 607 // Create reference file(s) to parent region file here in mergedDir. 608 // As this procedure is running on master, use CacheConfig.DISABLED means 609 // don't cache any block. 610 // We also need to pass through a suitable CompoundConfiguration as if this 611 // is running in a regionserver's Store context, or we might not be able 612 // to read the hfiles. 613 storeFileInfo.setConf(storeConfiguration); 614 Path refFile = mergeRegionFs.mergeStoreFile(regionFs.getRegionInfo(), family, 615 new HStoreFile(storeFileInfo, hcd.getBloomFilterType(), CacheConfig.DISABLED)); 616 mergedFiles.add(refFile); 617 } 618 } 619 } 620 return mergedFiles; 621 } 622 623 /** 624 * Clean up a merged region on rollback after failure. 625 */ 626 private void cleanupMergedRegion(final MasterProcedureEnv env) throws IOException { 627 final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); 628 TableName tn = this.regionsToMerge[0].getTable(); 629 final Path tabledir = CommonFSUtils.getTableDir(mfs.getRootDir(), tn); 630 final FileSystem fs = mfs.getFileSystem(); 631 // See createMergedRegion above where we specify the merge dir as being in the 632 // FIRST merge parent region. 633 HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem( 634 env.getMasterConfiguration(), fs, tabledir, regionsToMerge[0], false); 635 regionFs.cleanupMergedRegion(mergedRegion); 636 } 637 638 /** 639 * Rollback close regions 640 **/ 641 private void rollbackCloseRegionsForMerge(MasterProcedureEnv env) throws IOException { 642 AssignmentManagerUtil.reopenRegionsForRollback(env, Arrays.asList(regionsToMerge), 643 getRegionReplication(env), getServerName(env)); 644 } 645 646 private TransitRegionStateProcedure[] createUnassignProcedures(MasterProcedureEnv env) 647 throws IOException { 648 return AssignmentManagerUtil.createUnassignProceduresForSplitOrMerge(env, 649 Stream.of(regionsToMerge), getRegionReplication(env)); 650 } 651 652 private TransitRegionStateProcedure[] createAssignProcedures(MasterProcedureEnv env) 653 throws IOException { 654 return AssignmentManagerUtil.createAssignProceduresForOpeningNewRegions(env, 655 Collections.singletonList(mergedRegion), getRegionReplication(env), getServerName(env)); 656 } 657 658 private int getRegionReplication(final MasterProcedureEnv env) throws IOException { 659 return env.getMasterServices().getTableDescriptors().get(getTableName()).getRegionReplication(); 660 } 661 662 /** 663 * Post merge region action 664 * @param env MasterProcedureEnv 665 **/ 666 private void preMergeRegionsCommit(final MasterProcedureEnv env) throws IOException { 667 final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost(); 668 if (cpHost != null) { 669 @MetaMutationAnnotation 670 final List<Mutation> metaEntries = new ArrayList<>(); 671 cpHost.preMergeRegionsCommit(regionsToMerge, metaEntries, getUser()); 672 try { 673 for (Mutation p : metaEntries) { 674 RegionInfo.parseRegionName(p.getRow()); 675 } 676 } catch (IOException e) { 677 LOG.error("Row key of mutation from coprocessor is not parsable as region name. " 678 + "Mutations from coprocessor should only be for hbase:meta table.", e); 679 throw e; 680 } 681 } 682 } 683 684 /** 685 * Add merged region to META and delete original regions. 686 */ 687 private void updateMetaForMergedRegions(final MasterProcedureEnv env) throws IOException { 688 env.getAssignmentManager().markRegionAsMerged(mergedRegion, getServerName(env), 689 this.regionsToMerge); 690 } 691 692 /** 693 * Post merge region action 694 * @param env MasterProcedureEnv 695 **/ 696 private void postMergeRegionsCommit(final MasterProcedureEnv env) throws IOException { 697 final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost(); 698 if (cpHost != null) { 699 cpHost.postMergeRegionsCommit(regionsToMerge, mergedRegion, getUser()); 700 } 701 } 702 703 /** 704 * Post merge region action 705 * @param env MasterProcedureEnv 706 **/ 707 private void postCompletedMergeRegions(final MasterProcedureEnv env) throws IOException { 708 final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost(); 709 if (cpHost != null) { 710 cpHost.postCompletedMergeRegionsAction(regionsToMerge, mergedRegion, getUser()); 711 } 712 } 713 714 /** 715 * The procedure could be restarted from a different machine. If the variable is null, we need to 716 * retrieve it. 717 * @param env MasterProcedureEnv 718 */ 719 private ServerName getServerName(final MasterProcedureEnv env) { 720 if (regionLocation == null) { 721 regionLocation = 722 env.getAssignmentManager().getRegionStates().getRegionServerOfRegion(regionsToMerge[0]); 723 // May still be null here but return null and let caller deal. 724 // Means we lost the in-memory-only location. We are in recovery 725 // or so. The caller should be able to deal w/ a null ServerName. 726 // Let them go to the Balancer to find one to use instead. 727 } 728 return regionLocation; 729 } 730 731 private void writeMaxSequenceIdFile(MasterProcedureEnv env) throws IOException { 732 MasterFileSystem fs = env.getMasterFileSystem(); 733 long maxSequenceId = -1L; 734 for (RegionInfo region : regionsToMerge) { 735 maxSequenceId = 736 Math.max(maxSequenceId, WALSplitUtil.getMaxRegionSequenceId(env.getMasterConfiguration(), 737 region, fs::getFileSystem, fs::getWALFileSystem)); 738 } 739 if (maxSequenceId > 0) { 740 WALSplitUtil.writeRegionSequenceIdFile(fs.getWALFileSystem(), 741 getWALRegionDir(env, mergedRegion), maxSequenceId); 742 } 743 } 744 745 /** Returns The merged region. Maybe be null if called to early or we failed. */ 746 RegionInfo getMergedRegion() { 747 return this.mergedRegion; 748 } 749 750 @Override 751 protected boolean abort(MasterProcedureEnv env) { 752 // Abort means rollback. We can't rollback all steps. HBASE-18018 added abort to all 753 // Procedures. Here is a Procedure that has a PONR and cannot be aborted once it enters this 754 // range of steps; what do we do for these should an operator want to cancel them? HBASE-20022. 755 return isRollbackSupported(getCurrentState()) && super.abort(env); 756 } 757}