001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.assignment;
019
020import java.io.IOException;
021import java.util.ArrayList;
022import java.util.Arrays;
023import java.util.Collection;
024import java.util.Collections;
025import java.util.List;
026import java.util.stream.Stream;
027import org.apache.hadoop.conf.Configuration;
028import org.apache.hadoop.fs.FileSystem;
029import org.apache.hadoop.fs.Path;
030import org.apache.hadoop.hbase.MetaMutationAnnotation;
031import org.apache.hadoop.hbase.ServerName;
032import org.apache.hadoop.hbase.TableName;
033import org.apache.hadoop.hbase.UnknownRegionException;
034import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
035import org.apache.hadoop.hbase.client.DoNotRetryRegionException;
036import org.apache.hadoop.hbase.client.MasterSwitchType;
037import org.apache.hadoop.hbase.client.Mutation;
038import org.apache.hadoop.hbase.client.RegionInfo;
039import org.apache.hadoop.hbase.client.RegionInfoBuilder;
040import org.apache.hadoop.hbase.client.TableDescriptor;
041import org.apache.hadoop.hbase.exceptions.MergeRegionException;
042import org.apache.hadoop.hbase.io.hfile.CacheConfig;
043import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
044import org.apache.hadoop.hbase.master.MasterFileSystem;
045import org.apache.hadoop.hbase.master.RegionState;
046import org.apache.hadoop.hbase.master.RegionState.State;
047import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan;
048import org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure;
049import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
050import org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil;
051import org.apache.hadoop.hbase.procedure2.ProcedureMetrics;
052import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
053import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
054import org.apache.hadoop.hbase.quotas.QuotaExceededException;
055import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
056import org.apache.hadoop.hbase.regionserver.HStoreFile;
057import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
058import org.apache.hadoop.hbase.regionserver.StoreUtils;
059import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker;
060import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
061import org.apache.hadoop.hbase.util.Bytes;
062import org.apache.hadoop.hbase.util.CommonFSUtils;
063import org.apache.hadoop.hbase.wal.WALSplitUtil;
064import org.apache.yetus.audience.InterfaceAudience;
065import org.slf4j.Logger;
066import org.slf4j.LoggerFactory;
067
068import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
069import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoResponse;
070import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos;
071import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.MergeTableRegionsState;
072
073/**
074 * The procedure to Merge regions in a table. This procedure takes an exclusive table lock since it
075 * is working over multiple regions. It holds the lock for the life of the procedure. Throws
076 * exception on construction if determines context hostile to merge (cluster going down or master is
077 * shutting down or table is disabled).
078 */
079@InterfaceAudience.Private
080public class MergeTableRegionsProcedure
081  extends AbstractStateMachineTableProcedure<MergeTableRegionsState> {
082  private static final Logger LOG = LoggerFactory.getLogger(MergeTableRegionsProcedure.class);
083  private ServerName regionLocation;
084
085  /**
086   * Two or more regions to merge, the 'merge parents'.
087   */
088  private RegionInfo[] regionsToMerge;
089
090  /**
091   * The resulting merged region.
092   */
093  private RegionInfo mergedRegion;
094
095  private boolean force;
096
097  public MergeTableRegionsProcedure() {
098    // Required by the Procedure framework to create the procedure on replay
099  }
100
101  public MergeTableRegionsProcedure(final MasterProcedureEnv env, final RegionInfo[] regionsToMerge,
102    final boolean force) throws IOException {
103    super(env);
104    // Check parent regions. Make sure valid before starting work.
105    // This check calls the super method #checkOnline also.
106    checkRegionsToMerge(env, regionsToMerge, force);
107    // Sort the regions going into the merge.
108    Arrays.sort(regionsToMerge);
109    this.regionsToMerge = regionsToMerge;
110    this.mergedRegion = createMergedRegionInfo(regionsToMerge);
111    // Preflight depends on mergedRegion being set (at least).
112    preflightChecks(env, true);
113    this.force = force;
114  }
115
116  /**
117   * @throws MergeRegionException If unable to merge regions for whatever reasons.
118   */
119  private static void checkRegionsToMerge(MasterProcedureEnv env, final RegionInfo[] regions,
120    final boolean force) throws MergeRegionException {
121    long count = Arrays.stream(regions).distinct().count();
122    if (regions.length != count) {
123      throw new MergeRegionException("Duplicate regions specified; cannot merge a region to "
124        + "itself. Passed in " + regions.length + " but only " + count + " unique.");
125    }
126    if (count < 2) {
127      throw new MergeRegionException("Need two Regions at least to run a Merge");
128    }
129    RegionInfo previous = null;
130    for (RegionInfo ri : regions) {
131      if (previous != null) {
132        if (!previous.getTable().equals(ri.getTable())) {
133          String msg = "Can't merge regions from different tables: " + previous + ", " + ri;
134          LOG.warn(msg);
135          throw new MergeRegionException(msg);
136        }
137        if (!force && !ri.isAdjacent(previous) && !ri.isOverlap(previous)) {
138          String msg = "Unable to merge non-adjacent or non-overlapping regions '"
139            + previous.getShortNameToLog() + "', '" + ri.getShortNameToLog() + "' when force=false";
140          LOG.warn(msg);
141          throw new MergeRegionException(msg);
142        }
143      }
144
145      if (ri.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) {
146        throw new MergeRegionException("Can't merge non-default replicas; " + ri);
147      }
148      try {
149        checkOnline(env, ri);
150      } catch (DoNotRetryRegionException dnrre) {
151        throw new MergeRegionException(dnrre);
152      }
153
154      previous = ri;
155    }
156  }
157
158  /**
159   * Create merged region info by looking at passed in <code>regionsToMerge</code> to figure what
160   * extremes for start and end keys to use; merged region needs to have an extent sufficient to
161   * cover all regions-to-merge.
162   */
163  private static RegionInfo createMergedRegionInfo(final RegionInfo[] regionsToMerge) {
164    byte[] lowestStartKey = null;
165    byte[] highestEndKey = null;
166    // Region Id is a timestamp. Merged region's id can't be less than that of
167    // merging regions else will insert at wrong location in hbase:meta (See HBASE-710).
168    long highestRegionId = -1;
169    for (RegionInfo ri : regionsToMerge) {
170      if (lowestStartKey == null) {
171        lowestStartKey = ri.getStartKey();
172      } else if (Bytes.compareTo(ri.getStartKey(), lowestStartKey) < 0) {
173        lowestStartKey = ri.getStartKey();
174      }
175      if (highestEndKey == null) {
176        highestEndKey = ri.getEndKey();
177      } else if (ri.isLast() || Bytes.compareTo(ri.getEndKey(), highestEndKey) > 0) {
178        highestEndKey = ri.getEndKey();
179      }
180      highestRegionId = ri.getRegionId() > highestRegionId ? ri.getRegionId() : highestRegionId;
181    }
182    // Merged region is sorted between two merging regions in META
183    return RegionInfoBuilder.newBuilder(regionsToMerge[0].getTable()).setStartKey(lowestStartKey)
184      .setEndKey(highestEndKey).setSplit(false)
185      .setRegionId(highestRegionId + 1/* Add one so new merged region is highest */).build();
186  }
187
188  @Override
189  protected Flow executeFromState(final MasterProcedureEnv env, MergeTableRegionsState state) {
190    LOG.trace("{} execute state={}", this, state);
191    try {
192      switch (state) {
193        case MERGE_TABLE_REGIONS_PREPARE:
194          if (!prepareMergeRegion(env)) {
195            assert isFailed() : "Merge region should have an exception here";
196            return Flow.NO_MORE_STATE;
197          }
198          setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_PRE_MERGE_OPERATION);
199          break;
200        case MERGE_TABLE_REGIONS_PRE_MERGE_OPERATION:
201          preMergeRegions(env);
202          setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_CLOSE_REGIONS);
203          break;
204        case MERGE_TABLE_REGIONS_CLOSE_REGIONS:
205          addChildProcedure(createUnassignProcedures(env));
206          setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_CHECK_CLOSED_REGIONS);
207          break;
208        case MERGE_TABLE_REGIONS_CHECK_CLOSED_REGIONS:
209          checkClosedRegions(env);
210          setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_CREATE_MERGED_REGION);
211          break;
212        case MERGE_TABLE_REGIONS_CREATE_MERGED_REGION:
213          removeNonDefaultReplicas(env);
214          createMergedRegion(env);
215          setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_WRITE_MAX_SEQUENCE_ID_FILE);
216          break;
217        case MERGE_TABLE_REGIONS_WRITE_MAX_SEQUENCE_ID_FILE:
218          writeMaxSequenceIdFile(env);
219          setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_PRE_MERGE_COMMIT_OPERATION);
220          break;
221        case MERGE_TABLE_REGIONS_PRE_MERGE_COMMIT_OPERATION:
222          preMergeRegionsCommit(env);
223          setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_UPDATE_META);
224          break;
225        case MERGE_TABLE_REGIONS_UPDATE_META:
226          updateMetaForMergedRegions(env);
227          setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_POST_MERGE_COMMIT_OPERATION);
228          break;
229        case MERGE_TABLE_REGIONS_POST_MERGE_COMMIT_OPERATION:
230          postMergeRegionsCommit(env);
231          setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_OPEN_MERGED_REGION);
232          break;
233        case MERGE_TABLE_REGIONS_OPEN_MERGED_REGION:
234          addChildProcedure(createAssignProcedures(env));
235          setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_POST_OPERATION);
236          break;
237        case MERGE_TABLE_REGIONS_POST_OPERATION:
238          postCompletedMergeRegions(env);
239          return Flow.NO_MORE_STATE;
240        default:
241          throw new UnsupportedOperationException(this + " unhandled state=" + state);
242      }
243    } catch (IOException e) {
244      String msg = "Error trying to merge " + RegionInfo.getShortNameToLog(regionsToMerge) + " in "
245        + getTableName() + " (in state=" + state + ")";
246      if (!isRollbackSupported(state)) {
247        // We reach a state that cannot be rolled back. We just need to keep retrying.
248        LOG.warn(msg, e);
249      } else {
250        LOG.error(msg, e);
251        setFailure("master-merge-regions", e);
252      }
253    }
254    return Flow.HAS_MORE_STATE;
255  }
256
257  /**
258   * To rollback {@link MergeTableRegionsProcedure}, two AssignProcedures are asynchronously
259   * submitted for each region to be merged (rollback doesn't wait on the completion of the
260   * AssignProcedures) . This can be improved by changing rollback() to support sub-procedures. See
261   * HBASE-19851 for details.
262   */
263  @Override
264  protected void rollbackState(final MasterProcedureEnv env, final MergeTableRegionsState state)
265    throws IOException {
266    LOG.trace("{} rollback state={}", this, state);
267
268    try {
269      switch (state) {
270        case MERGE_TABLE_REGIONS_POST_OPERATION:
271        case MERGE_TABLE_REGIONS_OPEN_MERGED_REGION:
272        case MERGE_TABLE_REGIONS_POST_MERGE_COMMIT_OPERATION:
273        case MERGE_TABLE_REGIONS_UPDATE_META:
274          String msg = this + " We are in the " + state + " state."
275            + " It is complicated to rollback the merge operation that region server is working on."
276            + " Rollback is not supported and we should let the merge operation to complete";
277          LOG.warn(msg);
278          // PONR
279          throw new UnsupportedOperationException(this + " unhandled state=" + state);
280        case MERGE_TABLE_REGIONS_PRE_MERGE_COMMIT_OPERATION:
281          break;
282        case MERGE_TABLE_REGIONS_CREATE_MERGED_REGION:
283        case MERGE_TABLE_REGIONS_WRITE_MAX_SEQUENCE_ID_FILE:
284          cleanupMergedRegion(env);
285          break;
286        case MERGE_TABLE_REGIONS_CHECK_CLOSED_REGIONS:
287          break;
288        case MERGE_TABLE_REGIONS_CLOSE_REGIONS:
289          rollbackCloseRegionsForMerge(env);
290          break;
291        case MERGE_TABLE_REGIONS_PRE_MERGE_OPERATION:
292          postRollBackMergeRegions(env);
293          break;
294        case MERGE_TABLE_REGIONS_PREPARE:
295          rollbackPrepareMerge(env);
296          break;
297        default:
298          throw new UnsupportedOperationException(this + " unhandled state=" + state);
299      }
300    } catch (Exception e) {
301      // This will be retried. Unless there is a bug in the code,
302      // this should be just a "temporary error" (e.g. network down)
303      LOG.warn("Failed rollback attempt step " + state + " for merging the regions "
304        + RegionInfo.getShortNameToLog(regionsToMerge) + " in table " + getTableName(), e);
305      throw e;
306    }
307  }
308
309  /*
310   * Check whether we are in the state that can be rolled back
311   */
312  @Override
313  protected boolean isRollbackSupported(final MergeTableRegionsState state) {
314    switch (state) {
315      case MERGE_TABLE_REGIONS_POST_OPERATION:
316      case MERGE_TABLE_REGIONS_OPEN_MERGED_REGION:
317      case MERGE_TABLE_REGIONS_POST_MERGE_COMMIT_OPERATION:
318      case MERGE_TABLE_REGIONS_UPDATE_META:
319        // It is not safe to rollback in these states.
320        return false;
321      default:
322        break;
323    }
324    return true;
325  }
326
327  private void removeNonDefaultReplicas(MasterProcedureEnv env) throws IOException {
328    AssignmentManagerUtil.removeNonDefaultReplicas(env, Stream.of(regionsToMerge),
329      getRegionReplication(env));
330  }
331
332  private void checkClosedRegions(MasterProcedureEnv env) throws IOException {
333    // Theoretically this should not happen any more after we use TRSP, but anyway
334    // let's add a check here
335    for (RegionInfo region : regionsToMerge) {
336      AssignmentManagerUtil.checkClosedRegion(env, region);
337    }
338  }
339
340  @Override
341  protected MergeTableRegionsState getState(final int stateId) {
342    return MergeTableRegionsState.forNumber(stateId);
343  }
344
345  @Override
346  protected int getStateId(final MergeTableRegionsState state) {
347    return state.getNumber();
348  }
349
350  @Override
351  protected MergeTableRegionsState getInitialState() {
352    return MergeTableRegionsState.MERGE_TABLE_REGIONS_PREPARE;
353  }
354
355  @Override
356  protected void serializeStateData(ProcedureStateSerializer serializer) throws IOException {
357    super.serializeStateData(serializer);
358
359    final MasterProcedureProtos.MergeTableRegionsStateData.Builder mergeTableRegionsMsg =
360      MasterProcedureProtos.MergeTableRegionsStateData.newBuilder()
361        .setUserInfo(MasterProcedureUtil.toProtoUserInfo(getUser()))
362        .setMergedRegionInfo(ProtobufUtil.toRegionInfo(mergedRegion)).setForcible(force);
363    for (RegionInfo ri : regionsToMerge) {
364      mergeTableRegionsMsg.addRegionInfo(ProtobufUtil.toRegionInfo(ri));
365    }
366    serializer.serialize(mergeTableRegionsMsg.build());
367  }
368
369  @Override
370  protected void deserializeStateData(ProcedureStateSerializer serializer) throws IOException {
371    super.deserializeStateData(serializer);
372
373    final MasterProcedureProtos.MergeTableRegionsStateData mergeTableRegionsMsg =
374      serializer.deserialize(MasterProcedureProtos.MergeTableRegionsStateData.class);
375    setUser(MasterProcedureUtil.toUserInfo(mergeTableRegionsMsg.getUserInfo()));
376
377    assert (mergeTableRegionsMsg.getRegionInfoCount() == 2);
378    regionsToMerge = new RegionInfo[mergeTableRegionsMsg.getRegionInfoCount()];
379    for (int i = 0; i < regionsToMerge.length; i++) {
380      regionsToMerge[i] = ProtobufUtil.toRegionInfo(mergeTableRegionsMsg.getRegionInfo(i));
381    }
382
383    mergedRegion = ProtobufUtil.toRegionInfo(mergeTableRegionsMsg.getMergedRegionInfo());
384  }
385
386  @Override
387  public void toStringClassDetails(StringBuilder sb) {
388    sb.append(getClass().getSimpleName());
389    sb.append(" table=");
390    sb.append(getTableName());
391    sb.append(", regions=");
392    sb.append(RegionInfo.getShortNameToLog(regionsToMerge));
393    sb.append(", force=");
394    sb.append(force);
395  }
396
397  @Override
398  protected LockState acquireLock(final MasterProcedureEnv env) {
399    RegionInfo[] lockRegions = Arrays.copyOf(regionsToMerge, regionsToMerge.length + 1);
400    lockRegions[lockRegions.length - 1] = mergedRegion;
401
402    if (env.getProcedureScheduler().waitRegions(this, getTableName(), lockRegions)) {
403      try {
404        LOG.debug(LockState.LOCK_EVENT_WAIT + " " + env.getProcedureScheduler().dumpLocks());
405      } catch (IOException e) {
406        // Ignore, just for logging
407      }
408      return LockState.LOCK_EVENT_WAIT;
409    }
410    return LockState.LOCK_ACQUIRED;
411  }
412
413  @Override
414  protected void releaseLock(final MasterProcedureEnv env) {
415    RegionInfo[] lockRegions = Arrays.copyOf(regionsToMerge, regionsToMerge.length + 1);
416    lockRegions[lockRegions.length - 1] = mergedRegion;
417
418    env.getProcedureScheduler().wakeRegions(this, getTableName(), lockRegions);
419  }
420
421  @Override
422  protected boolean holdLock(MasterProcedureEnv env) {
423    return true;
424  }
425
426  @Override
427  public TableName getTableName() {
428    return mergedRegion.getTable();
429  }
430
431  @Override
432  public TableOperationType getTableOperationType() {
433    return TableOperationType.REGION_MERGE;
434  }
435
436  @Override
437  protected ProcedureMetrics getProcedureMetrics(MasterProcedureEnv env) {
438    return env.getAssignmentManager().getAssignmentManagerMetrics().getMergeProcMetrics();
439  }
440
441  /**
442   * Prepare merge and do some check
443   */
444  private boolean prepareMergeRegion(final MasterProcedureEnv env) throws IOException {
445    // Fail if we are taking snapshot for the given table
446    TableName tn = regionsToMerge[0].getTable();
447    final String regionNamesToLog = RegionInfo.getShortNameToLog(regionsToMerge);
448    if (env.getMasterServices().getSnapshotManager().isTableTakingAnySnapshot(tn)) {
449      throw new MergeRegionException(
450        "Skip merging regions " + regionNamesToLog + ", because we are snapshotting " + tn);
451    }
452
453    /*
454     * Sometimes a ModifyTableProcedure has edited a table descriptor to change the number of region
455     * replicas for a table, but it has not yet opened/closed the new replicas. The
456     * ModifyTableProcedure assumes that nobody else will do the opening/closing of the new
457     * replicas, but a concurrent MergeTableRegionProcedure would violate that assumption.
458     */
459    if (isTableModificationInProgress(env)) {
460      setFailure(getClass().getSimpleName(),
461        new IOException("Skip merging regions " + regionNamesToLog
462          + ", because there is an active procedure that is modifying the table " + tn));
463      return false;
464    }
465
466    // Mostly the below two checks are not used because we already check the switches before
467    // submitting the merge procedure. Just for safety, we are checking the switch again here.
468    // Also, in case the switch was set to false after submission, this procedure can be rollbacked,
469    // thanks to this double check!
470    // case 1: check for cluster level switch
471    if (!env.getMasterServices().isSplitOrMergeEnabled(MasterSwitchType.MERGE)) {
472      LOG.warn("Merge switch is off! skip merge of " + regionNamesToLog);
473      setFailure(getClass().getSimpleName(),
474        new IOException("Merge of " + regionNamesToLog + " failed because merge switch is off"));
475      return false;
476    }
477    // case 2: check for table level switch
478    if (!env.getMasterServices().getTableDescriptors().get(getTableName()).isMergeEnabled()) {
479      LOG.warn("Merge is disabled for the table! Skipping merge of {}", regionNamesToLog);
480      setFailure(getClass().getSimpleName(), new IOException(
481        "Merge of " + regionNamesToLog + " failed as region merge is disabled for the table"));
482      return false;
483    }
484
485    RegionStates regionStates = env.getAssignmentManager().getRegionStates();
486    RegionStateStore regionStateStore = env.getAssignmentManager().getRegionStateStore();
487    for (RegionInfo ri : this.regionsToMerge) {
488      if (regionStateStore.hasMergeRegions(ri)) {
489        String msg = "Skip merging " + regionNamesToLog + ", because a parent, "
490          + RegionInfo.getShortNameToLog(ri) + ", has a merge qualifier "
491          + "(if a 'merge column' in parent, it was recently merged but still has outstanding "
492          + "references to its parents that must be cleared before it can participate in merge -- "
493          + "major compact it to hurry clearing of its references)";
494        LOG.warn(msg);
495        throw new MergeRegionException(msg);
496      }
497      RegionState state = regionStates.getRegionState(ri.getEncodedName());
498      if (state == null) {
499        throw new UnknownRegionException(
500          RegionInfo.getShortNameToLog(ri) + " UNKNOWN (Has it been garbage collected?)");
501      }
502      if (!state.isOpened()) {
503        throw new MergeRegionException("Unable to merge regions that are NOT online: " + ri);
504      }
505      // Ask the remote regionserver if regions are mergeable. If we get an IOE, report it
506      // along with the failure, so we can see why regions are not mergeable at this time.
507      try {
508        if (!isMergeable(env, state)) {
509          setFailure(getClass().getSimpleName(),
510            new MergeRegionException("Skip merging " + regionNamesToLog + ", because a parent, "
511              + RegionInfo.getShortNameToLog(ri) + ", is not mergeable"));
512          return false;
513        }
514      } catch (IOException e) {
515        IOException ioe = new IOException(RegionInfo.getShortNameToLog(ri) + " NOT mergeable", e);
516        setFailure(getClass().getSimpleName(), ioe);
517        return false;
518      }
519    }
520
521    // Update region states to Merging
522    setRegionStateToMerging(env);
523    return true;
524  }
525
526  private boolean isMergeable(final MasterProcedureEnv env, final RegionState rs)
527    throws IOException {
528    GetRegionInfoResponse response =
529      AssignmentManagerUtil.getRegionInfoResponse(env, rs.getServerName(), rs.getRegion());
530    return response.hasMergeable() && response.getMergeable();
531  }
532
533  /**
534   * Action for rollback a merge table after prepare merge
535   */
536  private void rollbackPrepareMerge(final MasterProcedureEnv env) throws IOException {
537    for (RegionInfo rinfo : regionsToMerge) {
538      RegionStateNode regionStateNode =
539        env.getAssignmentManager().getRegionStates().getRegionStateNode(rinfo);
540      if (regionStateNode.getState() == State.MERGING) {
541        regionStateNode.setState(State.OPEN);
542      }
543    }
544  }
545
546  /**
547   * Pre merge region action
548   * @param env MasterProcedureEnv
549   **/
550  private void preMergeRegions(final MasterProcedureEnv env) throws IOException {
551    final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
552    if (cpHost != null) {
553      cpHost.preMergeRegionsAction(regionsToMerge, getUser());
554    }
555    // TODO: Clean up split and merge. Currently all over the place.
556    try {
557      MasterQuotaManager masterQuotaManager = env.getMasterServices().getMasterQuotaManager();
558      if (masterQuotaManager != null) {
559        masterQuotaManager.onRegionMerged(this.mergedRegion);
560      }
561    } catch (QuotaExceededException e) {
562      // TODO: why is this here? merge requests can be submitted by actors other than the normalizer
563      env.getMasterServices().getRegionNormalizerManager()
564        .planSkipped(NormalizationPlan.PlanType.MERGE);
565      throw e;
566    }
567  }
568
569  /**
570   * Action after rollback a merge table regions action.
571   */
572  private void postRollBackMergeRegions(final MasterProcedureEnv env) throws IOException {
573    final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
574    if (cpHost != null) {
575      cpHost.postRollBackMergeRegionsAction(regionsToMerge, getUser());
576    }
577  }
578
579  /**
580   * Set the region states to MERGING state
581   */
582  private void setRegionStateToMerging(final MasterProcedureEnv env) {
583    // Set State.MERGING to regions to be merged
584    RegionStates regionStates = env.getAssignmentManager().getRegionStates();
585    for (RegionInfo ri : this.regionsToMerge) {
586      regionStates.getRegionStateNode(ri).setState(State.MERGING);
587    }
588  }
589
590  /**
591   * Create merged region. The way the merge works is that we make a 'merges' temporary directory in
592   * the FIRST parent region to merge (Do not change this without also changing the rollback where
593   * we look in this FIRST region for the merge dir). We then collect here references to all the
594   * store files in all the parent regions including those of the FIRST parent region into a
595   * subdirectory, named for the resultant merged region. We then call commitMergeRegion. It finds
596   * this subdirectory of storefile references and moves them under the new merge region (creating
597   * the region layout as side effect). After assign of the new merge region, we will run a
598   * compaction. This will undo the references but the reference files remain in place until the
599   * archiver runs (which it does on a period as a chore in the RegionServer that hosts the merge
600   * region -- see CompactedHFilesDischarger). Once the archiver has moved aside the no-longer used
601   * references, the merge region no longer has references. The catalog janitor will notice when it
602   * runs next and it will remove the old parent regions.
603   */
604  private void createMergedRegion(final MasterProcedureEnv env) throws IOException {
605    final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
606    final Path tableDir = CommonFSUtils.getTableDir(mfs.getRootDir(), regionsToMerge[0].getTable());
607    final FileSystem fs = mfs.getFileSystem();
608    List<Path> mergedFiles = new ArrayList<>();
609    HRegionFileSystem mergeRegionFs = HRegionFileSystem
610      .createRegionOnFileSystem(env.getMasterConfiguration(), fs, tableDir, mergedRegion);
611
612    for (RegionInfo ri : this.regionsToMerge) {
613      HRegionFileSystem regionFs = HRegionFileSystem
614        .openRegionFromFileSystem(env.getMasterConfiguration(), fs, tableDir, ri, false);
615      mergedFiles.addAll(mergeStoreFiles(env, regionFs, mergeRegionFs, mergedRegion));
616    }
617    assert mergeRegionFs != null;
618    mergeRegionFs.commitMergedRegion(mergedFiles, env);
619
620    // Prepare to create merged regions
621    env.getAssignmentManager().getRegionStates().getOrCreateRegionStateNode(mergedRegion)
622      .setState(State.MERGING_NEW);
623  }
624
625  private List<Path> mergeStoreFiles(MasterProcedureEnv env, HRegionFileSystem regionFs,
626    HRegionFileSystem mergeRegionFs, RegionInfo mergedRegion) throws IOException {
627    final TableDescriptor htd =
628      env.getMasterServices().getTableDescriptors().get(mergedRegion.getTable());
629    List<Path> mergedFiles = new ArrayList<>();
630    for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) {
631      String family = hcd.getNameAsString();
632      StoreFileTracker tracker =
633        StoreFileTrackerFactory.create(env.getMasterConfiguration(), htd, hcd, regionFs);
634      final Collection<StoreFileInfo> storeFiles = tracker.load();
635      if (storeFiles != null && storeFiles.size() > 0) {
636        final Configuration storeConfiguration =
637          StoreUtils.createStoreConfiguration(env.getMasterConfiguration(), htd, hcd);
638        for (StoreFileInfo storeFileInfo : storeFiles) {
639          // Create reference file(s) to parent region file here in mergedDir.
640          // As this procedure is running on master, use CacheConfig.DISABLED means
641          // don't cache any block.
642          // We also need to pass through a suitable CompoundConfiguration as if this
643          // is running in a regionserver's Store context, or we might not be able
644          // to read the hfiles.
645          storeFileInfo.setConf(storeConfiguration);
646          Path refFile = mergeRegionFs.mergeStoreFile(regionFs.getRegionInfo(), family,
647            new HStoreFile(storeFileInfo, hcd.getBloomFilterType(), CacheConfig.DISABLED), tracker);
648          mergedFiles.add(refFile);
649        }
650      }
651    }
652    return mergedFiles;
653  }
654
655  /**
656   * Clean up a merged region on rollback after failure.
657   */
658  private void cleanupMergedRegion(final MasterProcedureEnv env) throws IOException {
659    final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
660    TableName tn = this.regionsToMerge[0].getTable();
661    final Path tabledir = CommonFSUtils.getTableDir(mfs.getRootDir(), tn);
662    final FileSystem fs = mfs.getFileSystem();
663    // See createMergedRegion above where we specify the merge dir as being in the
664    // FIRST merge parent region.
665    HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(
666      env.getMasterConfiguration(), fs, tabledir, regionsToMerge[0], false);
667    regionFs.cleanupMergedRegion(mergedRegion);
668  }
669
670  /**
671   * Rollback close regions
672   **/
673  private void rollbackCloseRegionsForMerge(MasterProcedureEnv env) throws IOException {
674    // At this point we should check if region was actually closed. If it was not closed then we
675    // don't need to repoen the region and we can just change the regionNode state to OPEN.
676    // if it is alredy closed then we need to do a reopen of region
677    List<RegionInfo> toAssign = new ArrayList<>();
678    for (RegionInfo rinfo : regionsToMerge) {
679      RegionStateNode regionStateNode =
680        env.getAssignmentManager().getRegionStates().getRegionStateNode(rinfo);
681      if (regionStateNode.getState() != State.MERGING) {
682        // same as before HBASE-28405
683        toAssign.add(rinfo);
684      }
685    }
686    AssignmentManagerUtil.reopenRegionsForRollback(env, toAssign, getRegionReplication(env),
687      getServerName(env));
688  }
689
690  private TransitRegionStateProcedure[] createUnassignProcedures(MasterProcedureEnv env)
691    throws IOException {
692    return AssignmentManagerUtil.createUnassignProceduresForSplitOrMerge(env,
693      Stream.of(regionsToMerge), getRegionReplication(env));
694  }
695
696  private TransitRegionStateProcedure[] createAssignProcedures(MasterProcedureEnv env)
697    throws IOException {
698    return AssignmentManagerUtil.createAssignProceduresForOpeningNewRegions(env,
699      Collections.singletonList(mergedRegion), getRegionReplication(env), getServerName(env));
700  }
701
702  private int getRegionReplication(final MasterProcedureEnv env) throws IOException {
703    return env.getMasterServices().getTableDescriptors().get(getTableName()).getRegionReplication();
704  }
705
706  /**
707   * Post merge region action
708   * @param env MasterProcedureEnv
709   **/
710  private void preMergeRegionsCommit(final MasterProcedureEnv env) throws IOException {
711    final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
712    if (cpHost != null) {
713      @MetaMutationAnnotation
714      final List<Mutation> metaEntries = new ArrayList<>();
715      cpHost.preMergeRegionsCommit(regionsToMerge, metaEntries, getUser());
716      try {
717        for (Mutation p : metaEntries) {
718          RegionInfo.parseRegionName(p.getRow());
719        }
720      } catch (IOException e) {
721        LOG.error("Row key of mutation from coprocessor is not parsable as region name. "
722          + "Mutations from coprocessor should only be for hbase:meta table.", e);
723        throw e;
724      }
725    }
726  }
727
728  /**
729   * Add merged region to META and delete original regions.
730   */
731  private void updateMetaForMergedRegions(final MasterProcedureEnv env) throws IOException {
732    env.getAssignmentManager().markRegionAsMerged(mergedRegion, getServerName(env),
733      this.regionsToMerge);
734  }
735
736  /**
737   * Post merge region action
738   * @param env MasterProcedureEnv
739   **/
740  private void postMergeRegionsCommit(final MasterProcedureEnv env) throws IOException {
741    final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
742    if (cpHost != null) {
743      cpHost.postMergeRegionsCommit(regionsToMerge, mergedRegion, getUser());
744    }
745  }
746
747  /**
748   * Post merge region action
749   * @param env MasterProcedureEnv
750   **/
751  private void postCompletedMergeRegions(final MasterProcedureEnv env) throws IOException {
752    final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
753    if (cpHost != null) {
754      cpHost.postCompletedMergeRegionsAction(regionsToMerge, mergedRegion, getUser());
755    }
756  }
757
758  /**
759   * The procedure could be restarted from a different machine. If the variable is null, we need to
760   * retrieve it.
761   * @param env MasterProcedureEnv
762   */
763  private ServerName getServerName(final MasterProcedureEnv env) {
764    if (regionLocation == null) {
765      regionLocation =
766        env.getAssignmentManager().getRegionStates().getRegionServerOfRegion(regionsToMerge[0]);
767      // May still be null here but return null and let caller deal.
768      // Means we lost the in-memory-only location. We are in recovery
769      // or so. The caller should be able to deal w/ a null ServerName.
770      // Let them go to the Balancer to find one to use instead.
771    }
772    return regionLocation;
773  }
774
775  private void writeMaxSequenceIdFile(MasterProcedureEnv env) throws IOException {
776    MasterFileSystem fs = env.getMasterFileSystem();
777    long maxSequenceId = -1L;
778    for (RegionInfo region : regionsToMerge) {
779      maxSequenceId =
780        Math.max(maxSequenceId, WALSplitUtil.getMaxRegionSequenceId(env.getMasterConfiguration(),
781          region, fs::getFileSystem, fs::getWALFileSystem));
782    }
783    if (maxSequenceId > 0) {
784      WALSplitUtil.writeRegionSequenceIdFile(fs.getWALFileSystem(),
785        getWALRegionDir(env, mergedRegion), maxSequenceId);
786    }
787  }
788
789  /** Returns The merged region. Maybe be null if called to early or we failed. */
790  RegionInfo getMergedRegion() {
791    return this.mergedRegion;
792  }
793
794  @Override
795  protected boolean abort(MasterProcedureEnv env) {
796    // Abort means rollback. We can't rollback all steps. HBASE-18018 added abort to all
797    // Procedures. Here is a Procedure that has a PONR and cannot be aborted once it enters this
798    // range of steps; what do we do for these should an operator want to cancel them? HBASE-20022.
799    return isRollbackSupported(getCurrentState()) && super.abort(env);
800  }
801}