001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 019package org.apache.hadoop.hbase.master.assignment; 020 021import java.io.IOException; 022import java.util.ArrayList; 023import java.util.Arrays; 024import java.util.Collection; 025import java.util.Collections; 026import java.util.List; 027import java.util.stream.Collectors; 028import org.apache.hadoop.conf.Configuration; 029import org.apache.hadoop.fs.FileSystem; 030import org.apache.hadoop.fs.Path; 031import org.apache.hadoop.hbase.MetaMutationAnnotation; 032import org.apache.hadoop.hbase.ServerName; 033import org.apache.hadoop.hbase.TableName; 034import org.apache.hadoop.hbase.UnknownRegionException; 035import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; 036import org.apache.hadoop.hbase.client.DoNotRetryRegionException; 037import org.apache.hadoop.hbase.client.MasterSwitchType; 038import org.apache.hadoop.hbase.client.Mutation; 039import org.apache.hadoop.hbase.client.RegionInfo; 040import org.apache.hadoop.hbase.client.RegionInfoBuilder; 041import org.apache.hadoop.hbase.client.RegionReplicaUtil; 042import org.apache.hadoop.hbase.client.TableDescriptor; 043import org.apache.hadoop.hbase.exceptions.MergeRegionException; 044import org.apache.hadoop.hbase.io.hfile.CacheConfig; 045import org.apache.hadoop.hbase.master.CatalogJanitor; 046import org.apache.hadoop.hbase.master.MasterCoprocessorHost; 047import org.apache.hadoop.hbase.master.MasterFileSystem; 048import org.apache.hadoop.hbase.master.RegionState; 049import org.apache.hadoop.hbase.master.RegionState.State; 050import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan; 051import org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure; 052import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; 053import org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil; 054import org.apache.hadoop.hbase.master.procedure.TableQueue; 055import org.apache.hadoop.hbase.procedure2.ProcedureMetrics; 056import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer; 057import org.apache.hadoop.hbase.quotas.QuotaExceededException; 058import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; 059import org.apache.hadoop.hbase.regionserver.HStoreFile; 060import org.apache.hadoop.hbase.regionserver.StoreFileInfo; 061import org.apache.hadoop.hbase.util.Bytes; 062import org.apache.hadoop.hbase.util.FSUtils; 063import org.apache.hadoop.hbase.wal.WALSplitter; 064import org.apache.yetus.audience.InterfaceAudience; 065import org.slf4j.Logger; 066import org.slf4j.LoggerFactory; 067 068import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting; 069 070import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; 071import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoResponse; 072import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos; 073import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.MergeTableRegionsState; 074 075/** 076 * The procedure to Merge regions in a table. This procedure takes an exclusive table 077 * lock since it is working over multiple regions. It holds the lock for the life of the procedure. 078 * Throws exception on construction if determines context hostile to merge (cluster going down or 079 * master is shutting down or table is disabled). 080 */ 081@InterfaceAudience.Private 082public class MergeTableRegionsProcedure 083 extends AbstractStateMachineTableProcedure<MergeTableRegionsState> { 084 private static final Logger LOG = LoggerFactory.getLogger(MergeTableRegionsProcedure.class); 085 private ServerName regionLocation; 086 087 /** 088 * Two or more regions to merge, the 'merge parents'. 089 */ 090 private RegionInfo[] regionsToMerge; 091 092 /** 093 * The resulting merged region. 094 */ 095 private RegionInfo mergedRegion; 096 097 private boolean force; 098 099 public MergeTableRegionsProcedure() { 100 // Required by the Procedure framework to create the procedure on replay 101 } 102 103 public MergeTableRegionsProcedure(final MasterProcedureEnv env, 104 final RegionInfo[] regionsToMerge, final boolean force) 105 throws IOException { 106 super(env); 107 // Check parent regions. Make sure valid before starting work. 108 // This check calls the super method #checkOnline also. 109 checkRegionsToMerge(env, regionsToMerge, force); 110 // Sort the regions going into the merge. 111 Arrays.sort(regionsToMerge); 112 this.regionsToMerge = regionsToMerge; 113 this.mergedRegion = createMergedRegionInfo(regionsToMerge); 114 // Preflight depends on mergedRegion being set (at least). 115 preflightChecks(env, true); 116 this.force = force; 117 } 118 119 /** 120 * @throws MergeRegionException If unable to merge regions for whatever reasons. 121 */ 122 private static void checkRegionsToMerge(MasterProcedureEnv env, final RegionInfo[] regions, 123 final boolean force) throws MergeRegionException { 124 long count = Arrays.stream(regions).distinct().count(); 125 if (regions.length != count) { 126 throw new MergeRegionException("Duplicate regions specified; cannot merge a region to " + 127 "itself. Passed in " + regions.length + " but only " + count + " unique."); 128 } 129 if (count < 2) { 130 throw new MergeRegionException("Need two Regions at least to run a Merge"); 131 } 132 RegionInfo previous = null; 133 for (RegionInfo ri: regions) { 134 if (previous != null) { 135 if (!previous.getTable().equals(ri.getTable())) { 136 String msg = "Can't merge regions from different tables: " + previous + ", " + ri; 137 LOG.warn(msg); 138 throw new MergeRegionException(msg); 139 } 140 if (!force && !ri.isAdjacent(previous) && !ri.isOverlap(previous)) { 141 String msg = "Unable to merge non-adjacent or non-overlapping regions " + 142 previous.getShortNameToLog() + ", " + ri.getShortNameToLog() + " when force=false"; 143 LOG.warn(msg); 144 throw new MergeRegionException(msg); 145 } 146 } 147 148 if (ri.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) { 149 throw new MergeRegionException("Can't merge non-default replicas; " + ri); 150 } 151 try { 152 checkOnline(env, ri); 153 } catch (DoNotRetryRegionException dnrre) { 154 throw new MergeRegionException(dnrre); 155 } 156 157 previous = ri; 158 } 159 } 160 161 /** 162 * Create merged region info by looking at passed in <code>regionsToMerge</code> 163 * to figure what extremes for start and end keys to use; merged region needs 164 * to have an extent sufficient to cover all regions-to-merge. 165 */ 166 private static RegionInfo createMergedRegionInfo(final RegionInfo[] regionsToMerge) { 167 byte [] lowestStartKey = null; 168 byte [] highestEndKey = null; 169 long highestRegionId = -1; 170 for (RegionInfo ri: regionsToMerge) { 171 if (lowestStartKey == null) { 172 lowestStartKey = ri.getStartKey(); 173 } else if (Bytes.compareTo(ri.getStartKey(), lowestStartKey) < 0) { 174 lowestStartKey = ri.getStartKey(); 175 } 176 if (highestEndKey == null) { 177 highestEndKey = ri.getEndKey(); 178 } else if (ri.isLast() || Bytes.compareTo(ri.getEndKey(), highestEndKey) > 0) { 179 highestEndKey = ri.getEndKey(); 180 } 181 highestRegionId = ri.getRegionId() > highestRegionId? ri.getRegionId(): highestRegionId; 182 } 183 // Merged region is sorted between two merging regions in META 184 return RegionInfoBuilder.newBuilder(regionsToMerge[0].getTable()). 185 setStartKey(lowestStartKey). 186 setEndKey(highestEndKey). 187 setSplit(false). 188 setRegionId(highestRegionId + 1/*Add one so new merged region is highest*/). 189 build(); 190 } 191 192 @Override 193 protected Flow executeFromState(final MasterProcedureEnv env, 194 MergeTableRegionsState state) { 195 LOG.trace("{} execute state={}", this, state); 196 try { 197 switch (state) { 198 case MERGE_TABLE_REGIONS_PREPARE: 199 if (!prepareMergeRegion(env)) { 200 assert isFailed() : "Merge region should have an exception here"; 201 return Flow.NO_MORE_STATE; 202 } 203 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_PRE_MERGE_OPERATION); 204 break; 205 case MERGE_TABLE_REGIONS_PRE_MERGE_OPERATION: 206 preMergeRegions(env); 207 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_CLOSE_REGIONS); 208 break; 209 case MERGE_TABLE_REGIONS_CLOSE_REGIONS: 210 addChildProcedure(createUnassignProcedures(env, getRegionReplication(env))); 211 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_CHECK_CLOSED_REGIONS); 212 break; 213 case MERGE_TABLE_REGIONS_CHECK_CLOSED_REGIONS: 214 List<RegionInfo> ris = hasRecoveredEdits(env); 215 if (ris.isEmpty()) { 216 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_CREATE_MERGED_REGION); 217 } else { 218 // Need to reopen parent regions to pickup missed recovered.edits. Do it by creating 219 // child assigns and then stepping back to MERGE_TABLE_REGIONS_CLOSE_REGIONS. 220 // Just assign the primary regions recovering the missed recovered.edits -- no replicas. 221 // May need to cycle here a few times if heavy writes. 222 // TODO: Add an assign read-only. 223 for (RegionInfo ri: ris) { 224 LOG.info("Found recovered.edits under {}, reopen to pickup missed edits!", ri); 225 addChildProcedure(env.getAssignmentManager().createAssignProcedure(ri)); 226 } 227 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_CLOSE_REGIONS); 228 } 229 break; 230 case MERGE_TABLE_REGIONS_CREATE_MERGED_REGION: 231 createMergedRegion(env); 232 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_WRITE_MAX_SEQUENCE_ID_FILE); 233 break; 234 case MERGE_TABLE_REGIONS_WRITE_MAX_SEQUENCE_ID_FILE: 235 writeMaxSequenceIdFile(env); 236 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_PRE_MERGE_COMMIT_OPERATION); 237 break; 238 case MERGE_TABLE_REGIONS_PRE_MERGE_COMMIT_OPERATION: 239 preMergeRegionsCommit(env); 240 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_UPDATE_META); 241 break; 242 case MERGE_TABLE_REGIONS_UPDATE_META: 243 updateMetaForMergedRegions(env); 244 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_POST_MERGE_COMMIT_OPERATION); 245 break; 246 case MERGE_TABLE_REGIONS_POST_MERGE_COMMIT_OPERATION: 247 postMergeRegionsCommit(env); 248 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_OPEN_MERGED_REGION); 249 break; 250 case MERGE_TABLE_REGIONS_OPEN_MERGED_REGION: 251 addChildProcedure(createAssignProcedures(env, getRegionReplication(env))); 252 setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_POST_OPERATION); 253 break; 254 case MERGE_TABLE_REGIONS_POST_OPERATION: 255 postCompletedMergeRegions(env); 256 return Flow.NO_MORE_STATE; 257 default: 258 throw new UnsupportedOperationException(this + " unhandled state=" + state); 259 } 260 } catch (IOException e) { 261 String msg = "Error trying to merge " + RegionInfo.getShortNameToLog(regionsToMerge) + 262 " in " + getTableName() + " (in state=" + state + ")"; 263 if (!isRollbackSupported(state)) { 264 // We reach a state that cannot be rolled back. We just need to keep retrying. 265 LOG.warn(msg, e); 266 } else { 267 LOG.error(msg, e); 268 setFailure("master-merge-regions", e); 269 } 270 } 271 return Flow.HAS_MORE_STATE; 272 } 273 274 /** 275 * To rollback {@link MergeTableRegionsProcedure}, two AssignProcedures are asynchronously 276 * submitted for each region to be merged (rollback doesn't wait on the completion of the 277 * AssignProcedures) . This can be improved by changing rollback() to support sub-procedures. 278 * See HBASE-19851 for details. 279 */ 280 @Override 281 protected void rollbackState(final MasterProcedureEnv env, final MergeTableRegionsState state) 282 throws IOException { 283 LOG.trace("{} rollback state={}", this, state); 284 285 try { 286 switch (state) { 287 case MERGE_TABLE_REGIONS_POST_OPERATION: 288 case MERGE_TABLE_REGIONS_OPEN_MERGED_REGION: 289 case MERGE_TABLE_REGIONS_POST_MERGE_COMMIT_OPERATION: 290 case MERGE_TABLE_REGIONS_UPDATE_META: 291 String msg = this + " We are in the " + state + " state." + 292 " It is complicated to rollback the merge operation that region server is working on." + 293 " Rollback is not supported and we should let the merge operation to complete"; 294 LOG.warn(msg); 295 // PONR 296 throw new UnsupportedOperationException(this + " unhandled state=" + state); 297 case MERGE_TABLE_REGIONS_PRE_MERGE_COMMIT_OPERATION: 298 break; 299 case MERGE_TABLE_REGIONS_CREATE_MERGED_REGION: 300 case MERGE_TABLE_REGIONS_WRITE_MAX_SEQUENCE_ID_FILE: 301 cleanupMergedRegion(env); 302 break; 303 case MERGE_TABLE_REGIONS_CHECK_CLOSED_REGIONS: 304 break; 305 case MERGE_TABLE_REGIONS_CLOSE_REGIONS: 306 rollbackCloseRegionsForMerge(env); 307 break; 308 case MERGE_TABLE_REGIONS_PRE_MERGE_OPERATION: 309 postRollBackMergeRegions(env); 310 break; 311 case MERGE_TABLE_REGIONS_PREPARE: 312 break; 313 default: 314 throw new UnsupportedOperationException(this + " unhandled state=" + state); 315 } 316 } catch (Exception e) { 317 // This will be retried. Unless there is a bug in the code, 318 // this should be just a "temporary error" (e.g. network down) 319 LOG.warn("Failed rollback attempt step " + state + " for merging the regions " 320 + RegionInfo.getShortNameToLog(regionsToMerge) + " in table " + getTableName(), e); 321 throw e; 322 } 323 } 324 325 /* 326 * Check whether we are in the state that can be rolled back 327 */ 328 @Override 329 protected boolean isRollbackSupported(final MergeTableRegionsState state) { 330 switch (state) { 331 case MERGE_TABLE_REGIONS_POST_OPERATION: 332 case MERGE_TABLE_REGIONS_OPEN_MERGED_REGION: 333 case MERGE_TABLE_REGIONS_POST_MERGE_COMMIT_OPERATION: 334 case MERGE_TABLE_REGIONS_UPDATE_META: 335 // It is not safe to rollback in these states. 336 return false; 337 default: 338 break; 339 } 340 return true; 341 } 342 343 @Override 344 protected MergeTableRegionsState getState(final int stateId) { 345 return MergeTableRegionsState.forNumber(stateId); 346 } 347 348 @Override 349 protected int getStateId(final MergeTableRegionsState state) { 350 return state.getNumber(); 351 } 352 353 @Override 354 protected MergeTableRegionsState getInitialState() { 355 return MergeTableRegionsState.MERGE_TABLE_REGIONS_PREPARE; 356 } 357 358 @Override 359 protected void serializeStateData(ProcedureStateSerializer serializer) 360 throws IOException { 361 super.serializeStateData(serializer); 362 363 final MasterProcedureProtos.MergeTableRegionsStateData.Builder mergeTableRegionsMsg = 364 MasterProcedureProtos.MergeTableRegionsStateData.newBuilder() 365 .setUserInfo(MasterProcedureUtil.toProtoUserInfo(getUser())) 366 .setMergedRegionInfo(ProtobufUtil.toRegionInfo(mergedRegion)) 367 .setForcible(force); 368 for (RegionInfo ri: regionsToMerge) { 369 mergeTableRegionsMsg.addRegionInfo(ProtobufUtil.toRegionInfo(ri)); 370 } 371 serializer.serialize(mergeTableRegionsMsg.build()); 372 } 373 374 @Override 375 protected void deserializeStateData(ProcedureStateSerializer serializer) 376 throws IOException { 377 super.deserializeStateData(serializer); 378 379 final MasterProcedureProtos.MergeTableRegionsStateData mergeTableRegionsMsg = 380 serializer.deserialize(MasterProcedureProtos.MergeTableRegionsStateData.class); 381 setUser(MasterProcedureUtil.toUserInfo(mergeTableRegionsMsg.getUserInfo())); 382 383 assert(mergeTableRegionsMsg.getRegionInfoCount() == 2); 384 regionsToMerge = new RegionInfo[mergeTableRegionsMsg.getRegionInfoCount()]; 385 for (int i = 0; i < regionsToMerge.length; i++) { 386 regionsToMerge[i] = ProtobufUtil.toRegionInfo(mergeTableRegionsMsg.getRegionInfo(i)); 387 } 388 389 mergedRegion = ProtobufUtil.toRegionInfo(mergeTableRegionsMsg.getMergedRegionInfo()); 390 } 391 392 @Override 393 public void toStringClassDetails(StringBuilder sb) { 394 sb.append(getClass().getSimpleName()); 395 sb.append(" table="); 396 sb.append(getTableName()); 397 sb.append(", regions="); 398 sb.append(RegionInfo.getShortNameToLog(regionsToMerge)); 399 sb.append(", force="); 400 sb.append(force); 401 } 402 403 @Override 404 protected LockState acquireLock(final MasterProcedureEnv env) { 405 RegionInfo[] lockRegions = Arrays.copyOf(regionsToMerge, regionsToMerge.length + 1); 406 lockRegions[lockRegions.length - 1] = mergedRegion; 407 408 if (env.getProcedureScheduler().waitRegions(this, getTableName(), lockRegions)) { 409 try { 410 LOG.debug(LockState.LOCK_EVENT_WAIT + " " + env.getProcedureScheduler().dumpLocks()); 411 } catch (IOException e) { 412 // Ignore, just for logging 413 } 414 return LockState.LOCK_EVENT_WAIT; 415 } 416 return LockState.LOCK_ACQUIRED; 417 } 418 419 @Override 420 protected void releaseLock(final MasterProcedureEnv env) { 421 RegionInfo[] lockRegions = Arrays.copyOf(regionsToMerge, regionsToMerge.length + 1); 422 lockRegions[lockRegions.length - 1] = mergedRegion; 423 424 env.getProcedureScheduler().wakeRegions(this, getTableName(), lockRegions); 425 } 426 427 @Override 428 protected boolean holdLock(MasterProcedureEnv env) { 429 return true; 430 } 431 432 @Override 433 public TableName getTableName() { 434 return mergedRegion.getTable(); 435 } 436 437 @Override 438 public TableOperationType getTableOperationType() { 439 return TableOperationType.REGION_MERGE; 440 } 441 442 @Override 443 protected ProcedureMetrics getProcedureMetrics(MasterProcedureEnv env) { 444 return env.getAssignmentManager().getAssignmentManagerMetrics().getMergeProcMetrics(); 445 } 446 447 /** 448 * Return list of regions that have recovered.edits... usually its an empty list. 449 * @param env the master env 450 * @throws IOException IOException 451 */ 452 private List<RegionInfo> hasRecoveredEdits(final MasterProcedureEnv env) throws IOException { 453 List<RegionInfo> ris = new ArrayList<RegionInfo>(regionsToMerge.length); 454 for (int i = 0; i < regionsToMerge.length; i++) { 455 RegionInfo ri = regionsToMerge[i]; 456 if (SplitTableRegionProcedure.hasRecoveredEdits(env, ri)) { 457 ris.add(ri); 458 } 459 } 460 return ris; 461 } 462 463 /** 464 * Prepare merge and do some check 465 * @param env MasterProcedureEnv 466 * @throws IOException 467 */ 468 private boolean prepareMergeRegion(final MasterProcedureEnv env) throws IOException { 469 // Fail if we are taking snapshot for the given table 470 TableName tn = regionsToMerge[0].getTable(); 471 if (env.getMasterServices().getSnapshotManager().isTakingSnapshot(tn)) { 472 throw new MergeRegionException("Skip merging regions " + 473 RegionInfo.getShortNameToLog(regionsToMerge) + ", because we are snapshotting " + tn); 474 } 475 if (!env.getMasterServices().isSplitOrMergeEnabled(MasterSwitchType.MERGE)) { 476 String regionsStr = Arrays.deepToString(this.regionsToMerge); 477 LOG.warn("Merge switch is off! skip merge of " + regionsStr); 478 super.setFailure(getClass().getSimpleName(), 479 new IOException("Merge of " + regionsStr + " failed because merge switch is off")); 480 return false; 481 } 482 // See HBASE-21395, for 2.0.x and 2.1.x only. 483 // A safe fence here, if there is a table procedure going on, abort the merge. 484 // There some cases that may lead to table procedure roll back (more serious 485 // than roll back the merge procedure here), or the merged regions was brought online 486 // by the table procedure because of the race between merge procedure and table procedure 487 List<AbstractStateMachineTableProcedure> tableProcedures = env 488 .getMasterServices().getProcedures().stream() 489 .filter(p -> p instanceof AbstractStateMachineTableProcedure) 490 .map(p -> (AbstractStateMachineTableProcedure) p) 491 .filter(p -> p.getProcId() != this.getProcId() && p.getTableName() 492 .equals(regionsToMerge[0].getTable()) && !p.isFinished() 493 && TableQueue.requireTableExclusiveLock(p)) 494 .collect(Collectors.toList()); 495 if (tableProcedures != null && tableProcedures.size() > 0) { 496 throw new MergeRegionException(tableProcedures.get(0).toString() 497 + " is going on against the same table, abort the merge of " + this 498 .toString()); 499 } 500 501 CatalogJanitor catalogJanitor = env.getMasterServices().getCatalogJanitor(); 502 RegionStates regionStates = env.getAssignmentManager().getRegionStates(); 503 for (RegionInfo ri: this.regionsToMerge) { 504 if (!catalogJanitor.cleanMergeQualifier(ri)) { 505 String msg = "Skip merging " + RegionInfo.getShortNameToLog(regionsToMerge) + 506 ", because parent " + RegionInfo.getShortNameToLog(ri) + " has a merge qualifier"; 507 LOG.warn(msg); 508 throw new MergeRegionException(msg); 509 } 510 RegionState state = regionStates.getRegionState(ri.getEncodedName()); 511 if (state == null) { 512 throw new UnknownRegionException("No state for " + RegionInfo.getShortNameToLog(ri)); 513 } 514 if (!state.isOpened()) { 515 throw new MergeRegionException("Unable to merge regions that are not online: " + ri); 516 } 517 // Ask the remote regionserver if regions are mergeable. If we get an IOE, report it 518 // along with the failure, so we can see why regions are not mergeable at this time. 519 try { 520 if (!isMergeable(env, state)) { 521 return false; 522 } 523 } catch (IOException e) { 524 IOException ioe = new IOException(RegionInfo.getShortNameToLog(ri) + " NOT mergeable", e); 525 super.setFailure(getClass().getSimpleName(), ioe); 526 return false; 527 } 528 } 529 530 // Update region states to Merging 531 setRegionStateToMerging(env); 532 return true; 533 } 534 535 private boolean isMergeable(final MasterProcedureEnv env, final RegionState rs) 536 throws IOException { 537 GetRegionInfoResponse response = 538 Util.getRegionInfoResponse(env, rs.getServerName(), rs.getRegion()); 539 return response.hasMergeable() && response.getMergeable(); 540 } 541 542 /** 543 * Pre merge region action 544 * @param env MasterProcedureEnv 545 **/ 546 private void preMergeRegions(final MasterProcedureEnv env) throws IOException { 547 final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost(); 548 if (cpHost != null) { 549 cpHost.preMergeRegionsAction(regionsToMerge, getUser()); 550 } 551 // TODO: Clean up split and merge. Currently all over the place. 552 try { 553 env.getMasterServices().getMasterQuotaManager().onRegionMerged(this.mergedRegion); 554 } catch (QuotaExceededException e) { 555 env.getMasterServices().getRegionNormalizer().planSkipped(this.mergedRegion, 556 NormalizationPlan.PlanType.MERGE); 557 throw e; 558 } 559 } 560 561 /** 562 * Action after rollback a merge table regions action. 563 */ 564 private void postRollBackMergeRegions(final MasterProcedureEnv env) throws IOException { 565 final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost(); 566 if (cpHost != null) { 567 cpHost.postRollBackMergeRegionsAction(regionsToMerge, getUser()); 568 } 569 } 570 571 /** 572 * Set the region states to MERGING state 573 * @param env MasterProcedureEnv 574 */ 575 public void setRegionStateToMerging(final MasterProcedureEnv env) { 576 // Set State.MERGING to regions to be merged 577 RegionStates regionStates = env.getAssignmentManager().getRegionStates(); 578 for (RegionInfo ri: this.regionsToMerge) { 579 regionStates.getRegionStateNode(ri).setState(State.MERGING); 580 } 581 } 582 583 /** 584 * Create merged region. 585 * The way the merge works is that we make a 'merges' temporary 586 * directory in the FIRST parent region to merge (Do not change this without 587 * also changing the rollback where we look in this FIRST region for the 588 * merge dir). We then collect here references to all the store files in all 589 * the parent regions including those of the FIRST parent region into a 590 * subdirectory, named for the resultant merged region. We then call 591 * commitMergeRegion. It finds this subdirectory of storefile references 592 * and moves them under the new merge region (creating the region layout 593 * as side effect). After assign of the new merge region, we will run a 594 * compaction. This will undo the references but the reference files remain 595 * in place until the archiver runs (which it does on a period as a chore 596 * in the RegionServer that hosts the merge region -- see 597 * CompactedHFilesDischarger). Once the archiver has moved aside the 598 * no-longer used references, the merge region no longer has references. 599 * The catalog janitor will notice when it runs next and it will remove 600 * the old parent regions. 601 */ 602 private void createMergedRegion(final MasterProcedureEnv env) throws IOException { 603 final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); 604 final Path tabledir = FSUtils.getTableDir(mfs.getRootDir(), regionsToMerge[0].getTable()); 605 final FileSystem fs = mfs.getFileSystem(); 606 HRegionFileSystem mergeRegionFs = null; 607 for (RegionInfo ri: this.regionsToMerge) { 608 HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem( 609 env.getMasterConfiguration(), fs, tabledir, ri, false); 610 if (mergeRegionFs == null) { 611 mergeRegionFs = regionFs; 612 mergeRegionFs.createMergesDir(); 613 } 614 mergeStoreFiles(env, regionFs, mergeRegionFs.getMergesDir()); 615 } 616 assert mergeRegionFs != null; 617 mergeRegionFs.commitMergedRegion(mergedRegion); 618 619 // Prepare to create merged regions 620 env.getAssignmentManager().getRegionStates(). 621 getOrCreateRegionStateNode(mergedRegion).setState(State.MERGING_NEW); 622 } 623 624 /** 625 * Create reference file(s) to parent region hfiles in the <code>mergeDir</code> 626 * @param regionFs merge parent region file system 627 * @param mergeDir the temp directory in which we are accumulating references. 628 */ 629 private void mergeStoreFiles(final MasterProcedureEnv env, final HRegionFileSystem regionFs, 630 final Path mergeDir) throws IOException { 631 final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); 632 final Configuration conf = env.getMasterConfiguration(); 633 final TableDescriptor htd = env.getMasterServices().getTableDescriptors().get(getTableName()); 634 for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) { 635 String family = hcd.getNameAsString(); 636 final Collection<StoreFileInfo> storeFiles = regionFs.getStoreFiles(family); 637 if (storeFiles != null && storeFiles.size() > 0) { 638 for (StoreFileInfo storeFileInfo : storeFiles) { 639 // Create reference file(s) to parent region file here in mergedDir. 640 // As this procedure is running on master, use CacheConfig.DISABLED means 641 // don't cache any block. 642 regionFs.mergeStoreFile(mergedRegion, family, new HStoreFile(mfs.getFileSystem(), 643 storeFileInfo, conf, CacheConfig.DISABLED, hcd.getBloomFilterType(), true), 644 mergeDir); 645 } 646 } 647 } 648 } 649 650 /** 651 * Clean up a merged region on rollback after failure. 652 */ 653 private void cleanupMergedRegion(final MasterProcedureEnv env) throws IOException { 654 final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); 655 TableName tn = this.regionsToMerge[0].getTable(); 656 final Path tabledir = FSUtils.getTableDir(mfs.getRootDir(), tn); 657 final FileSystem fs = mfs.getFileSystem(); 658 // See createMergedRegion above where we specify the merge dir as being in the 659 // FIRST merge parent region. 660 HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem( 661 env.getMasterConfiguration(), fs, tabledir, regionsToMerge[0], false); 662 regionFs.cleanupMergedRegion(mergedRegion); 663 } 664 665 /** 666 * Rollback close regions 667 * @param env MasterProcedureEnv 668 **/ 669 private void rollbackCloseRegionsForMerge(final MasterProcedureEnv env) throws IOException { 670 // Check whether the region is closed; if so, open it in the same server 671 final int regionReplication = getRegionReplication(env); 672 final ServerName serverName = getServerName(env); 673 674 AssignProcedure[] procs = 675 createAssignProcedures(regionReplication, env, Arrays.asList(regionsToMerge), serverName); 676 env.getMasterServices().getMasterProcedureExecutor().submitProcedures(procs); 677 } 678 679 private AssignProcedure[] createAssignProcedures(final int regionReplication, 680 final MasterProcedureEnv env, final List<RegionInfo> hris, final ServerName serverName) { 681 final AssignProcedure[] procs = new AssignProcedure[hris.size() * regionReplication]; 682 int procsIdx = 0; 683 for (int i = 0; i < hris.size(); ++i) { 684 // create procs for the primary region with the target server. 685 final RegionInfo hri = RegionReplicaUtil.getRegionInfoForReplica(hris.get(i), 0); 686 procs[procsIdx++] = env.getAssignmentManager().createAssignProcedure(hri, serverName); 687 } 688 if (regionReplication > 1) { 689 List<RegionInfo> regionReplicas = 690 new ArrayList<RegionInfo>(hris.size() * (regionReplication - 1)); 691 for (int i = 0; i < hris.size(); ++i) { 692 // We don't include primary replica here 693 for (int j = 1; j < regionReplication; ++j) { 694 regionReplicas.add(RegionReplicaUtil.getRegionInfoForReplica(hris.get(i), j)); 695 } 696 } 697 // for the replica regions exclude the primary region's server and call LB's roundRobin 698 // assignment 699 AssignProcedure[] replicaAssignProcs = env.getAssignmentManager() 700 .createRoundRobinAssignProcedures(regionReplicas, Collections.singletonList(serverName)); 701 for (AssignProcedure proc : replicaAssignProcs) { 702 procs[procsIdx++] = proc; 703 } 704 } 705 return procs; 706 } 707 708 private UnassignProcedure[] createUnassignProcedures(final MasterProcedureEnv env, 709 final int regionReplication) { 710 final UnassignProcedure[] procs = 711 new UnassignProcedure[regionsToMerge.length * regionReplication]; 712 int procsIdx = 0; 713 for (int i = 0; i < regionsToMerge.length; ++i) { 714 for (int j = 0; j < regionReplication; ++j) { 715 final RegionInfo hri = RegionReplicaUtil.getRegionInfoForReplica(regionsToMerge[i], j); 716 procs[procsIdx++] = env.getAssignmentManager(). 717 createUnassignProcedure(hri, null, true, !RegionReplicaUtil.isDefaultReplica(hri)); 718 } 719 } 720 return procs; 721 } 722 723 private AssignProcedure[] createAssignProcedures(final MasterProcedureEnv env, 724 final int regionReplication) { 725 final ServerName targetServer = getServerName(env); 726 return createAssignProcedures(regionReplication, env, Collections.singletonList(mergedRegion), 727 targetServer); 728 } 729 730 private int getRegionReplication(final MasterProcedureEnv env) throws IOException { 731 return env.getMasterServices().getTableDescriptors().get(getTableName()). 732 getRegionReplication(); 733 } 734 735 /** 736 * Post merge region action 737 * @param env MasterProcedureEnv 738 **/ 739 private void preMergeRegionsCommit(final MasterProcedureEnv env) throws IOException { 740 final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost(); 741 if (cpHost != null) { 742 @MetaMutationAnnotation 743 final List<Mutation> metaEntries = new ArrayList<>(); 744 cpHost.preMergeRegionsCommit(regionsToMerge, metaEntries, getUser()); 745 try { 746 for (Mutation p : metaEntries) { 747 RegionInfo.parseRegionName(p.getRow()); 748 } 749 } catch (IOException e) { 750 LOG.error("Row key of mutation from coprocessor is not parsable as region name. " 751 + "Mutations from coprocessor should only be for hbase:meta table.", e); 752 throw e; 753 } 754 } 755 } 756 757 /** 758 * Add merged region to META and delete original regions. 759 */ 760 private void updateMetaForMergedRegions(final MasterProcedureEnv env) throws IOException { 761 env.getAssignmentManager().markRegionAsMerged(mergedRegion, getServerName(env), 762 this.regionsToMerge); 763 } 764 765 /** 766 * Post merge region action 767 * @param env MasterProcedureEnv 768 **/ 769 private void postMergeRegionsCommit(final MasterProcedureEnv env) throws IOException { 770 final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost(); 771 if (cpHost != null) { 772 cpHost.postMergeRegionsCommit(regionsToMerge, mergedRegion, getUser()); 773 } 774 } 775 776 /** 777 * Post merge region action 778 * @param env MasterProcedureEnv 779 **/ 780 private void postCompletedMergeRegions(final MasterProcedureEnv env) throws IOException { 781 final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost(); 782 if (cpHost != null) { 783 cpHost.postCompletedMergeRegionsAction(regionsToMerge, mergedRegion, getUser()); 784 } 785 } 786 787 /** 788 * The procedure could be restarted from a different machine. If the variable is null, we need to 789 * retrieve it. 790 * @param env MasterProcedureEnv 791 * @return serverName 792 */ 793 private ServerName getServerName(final MasterProcedureEnv env) { 794 if (regionLocation == null) { 795 regionLocation = env.getAssignmentManager().getRegionStates(). 796 getRegionServerOfRegion(regionsToMerge[0]); 797 // May still be null here but return null and let caller deal. 798 // Means we lost the in-memory-only location. We are in recovery 799 // or so. The caller should be able to deal w/ a null ServerName. 800 // Let them go to the Balancer to find one to use instead. 801 } 802 return regionLocation; 803 } 804 805 private void writeMaxSequenceIdFile(MasterProcedureEnv env) throws IOException { 806 MasterFileSystem fs = env.getMasterFileSystem(); 807 long maxSequenceId = -1L; 808 for (RegionInfo region : regionsToMerge) { 809 maxSequenceId = 810 Math.max(maxSequenceId, WALSplitter.getMaxRegionSequenceId(env.getMasterConfiguration(), 811 region, fs::getFileSystem, fs::getWALFileSystem)); 812 } 813 if (maxSequenceId > 0) { 814 WALSplitter.writeRegionSequenceIdFile(fs.getWALFileSystem(), 815 getWALRegionDir(env, mergedRegion), maxSequenceId); 816 } 817 } 818 819 /** 820 * @return The merged region. Maybe be null if called to early or we failed. 821 */ 822 @VisibleForTesting 823 RegionInfo getMergedRegion() { 824 return this.mergedRegion; 825 } 826 827 @Override 828 protected boolean abort(MasterProcedureEnv env) { 829 // Abort means rollback. We can't rollback all steps. HBASE-18018 added abort to all 830 // Procedures. Here is a Procedure that has a PONR and cannot be aborted once it enters this 831 // range of steps; what do we do for these should an operator want to cancel them? HBASE-20022. 832 return isRollbackSupported(getCurrentState()) && super.abort(env); 833 } 834}