001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.assignment;
019
020import java.io.IOException;
021import java.util.ArrayList;
022import java.util.Arrays;
023import java.util.Collection;
024import java.util.Collections;
025import java.util.List;
026import java.util.stream.Stream;
027import org.apache.hadoop.conf.Configuration;
028import org.apache.hadoop.fs.FileSystem;
029import org.apache.hadoop.fs.Path;
030import org.apache.hadoop.hbase.MetaMutationAnnotation;
031import org.apache.hadoop.hbase.ServerName;
032import org.apache.hadoop.hbase.TableName;
033import org.apache.hadoop.hbase.UnknownRegionException;
034import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
035import org.apache.hadoop.hbase.client.DoNotRetryRegionException;
036import org.apache.hadoop.hbase.client.MasterSwitchType;
037import org.apache.hadoop.hbase.client.Mutation;
038import org.apache.hadoop.hbase.client.RegionInfo;
039import org.apache.hadoop.hbase.client.RegionInfoBuilder;
040import org.apache.hadoop.hbase.client.TableDescriptor;
041import org.apache.hadoop.hbase.exceptions.MergeRegionException;
042import org.apache.hadoop.hbase.io.hfile.CacheConfig;
043import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
044import org.apache.hadoop.hbase.master.MasterFileSystem;
045import org.apache.hadoop.hbase.master.RegionState;
046import org.apache.hadoop.hbase.master.RegionState.State;
047import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan;
048import org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure;
049import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
050import org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil;
051import org.apache.hadoop.hbase.procedure2.ProcedureMetrics;
052import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
053import org.apache.hadoop.hbase.quotas.QuotaExceededException;
054import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
055import org.apache.hadoop.hbase.regionserver.HStoreFile;
056import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
057import org.apache.hadoop.hbase.regionserver.StoreUtils;
058import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker;
059import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
060import org.apache.hadoop.hbase.util.Bytes;
061import org.apache.hadoop.hbase.util.CommonFSUtils;
062import org.apache.hadoop.hbase.wal.WALSplitUtil;
063import org.apache.yetus.audience.InterfaceAudience;
064import org.slf4j.Logger;
065import org.slf4j.LoggerFactory;
066
067import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
068import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoResponse;
069import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos;
070import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.MergeTableRegionsState;
071
072/**
073 * The procedure to Merge regions in a table. This procedure takes an exclusive table lock since it
074 * is working over multiple regions. It holds the lock for the life of the procedure. Throws
075 * exception on construction if determines context hostile to merge (cluster going down or master is
076 * shutting down or table is disabled).
077 */
078@InterfaceAudience.Private
079public class MergeTableRegionsProcedure
080  extends AbstractStateMachineTableProcedure<MergeTableRegionsState> {
081  private static final Logger LOG = LoggerFactory.getLogger(MergeTableRegionsProcedure.class);
082  private ServerName regionLocation;
083
084  /**
085   * Two or more regions to merge, the 'merge parents'.
086   */
087  private RegionInfo[] regionsToMerge;
088
089  /**
090   * The resulting merged region.
091   */
092  private RegionInfo mergedRegion;
093
094  private boolean force;
095
096  public MergeTableRegionsProcedure() {
097    // Required by the Procedure framework to create the procedure on replay
098  }
099
100  public MergeTableRegionsProcedure(final MasterProcedureEnv env, final RegionInfo[] regionsToMerge,
101    final boolean force) throws IOException {
102    super(env);
103    // Check parent regions. Make sure valid before starting work.
104    // This check calls the super method #checkOnline also.
105    checkRegionsToMerge(env, regionsToMerge, force);
106    // Sort the regions going into the merge.
107    Arrays.sort(regionsToMerge);
108    this.regionsToMerge = regionsToMerge;
109    this.mergedRegion = createMergedRegionInfo(regionsToMerge);
110    // Preflight depends on mergedRegion being set (at least).
111    preflightChecks(env, true);
112    this.force = force;
113  }
114
115  /**
116   * @throws MergeRegionException If unable to merge regions for whatever reasons.
117   */
118  private static void checkRegionsToMerge(MasterProcedureEnv env, final RegionInfo[] regions,
119    final boolean force) throws MergeRegionException {
120    long count = Arrays.stream(regions).distinct().count();
121    if (regions.length != count) {
122      throw new MergeRegionException("Duplicate regions specified; cannot merge a region to "
123        + "itself. Passed in " + regions.length + " but only " + count + " unique.");
124    }
125    if (count < 2) {
126      throw new MergeRegionException("Need two Regions at least to run a Merge");
127    }
128    RegionInfo previous = null;
129    for (RegionInfo ri : regions) {
130      if (previous != null) {
131        if (!previous.getTable().equals(ri.getTable())) {
132          String msg = "Can't merge regions from different tables: " + previous + ", " + ri;
133          LOG.warn(msg);
134          throw new MergeRegionException(msg);
135        }
136        if (!force && !ri.isAdjacent(previous) && !ri.isOverlap(previous)) {
137          String msg = "Unable to merge non-adjacent or non-overlapping regions '"
138            + previous.getShortNameToLog() + "', '" + ri.getShortNameToLog() + "' when force=false";
139          LOG.warn(msg);
140          throw new MergeRegionException(msg);
141        }
142      }
143
144      if (ri.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) {
145        throw new MergeRegionException("Can't merge non-default replicas; " + ri);
146      }
147      try {
148        checkOnline(env, ri);
149      } catch (DoNotRetryRegionException dnrre) {
150        throw new MergeRegionException(dnrre);
151      }
152
153      previous = ri;
154    }
155  }
156
157  /**
158   * Create merged region info by looking at passed in <code>regionsToMerge</code> to figure what
159   * extremes for start and end keys to use; merged region needs to have an extent sufficient to
160   * cover all regions-to-merge.
161   */
162  private static RegionInfo createMergedRegionInfo(final RegionInfo[] regionsToMerge) {
163    byte[] lowestStartKey = null;
164    byte[] highestEndKey = null;
165    // Region Id is a timestamp. Merged region's id can't be less than that of
166    // merging regions else will insert at wrong location in hbase:meta (See HBASE-710).
167    long highestRegionId = -1;
168    for (RegionInfo ri : regionsToMerge) {
169      if (lowestStartKey == null) {
170        lowestStartKey = ri.getStartKey();
171      } else if (Bytes.compareTo(ri.getStartKey(), lowestStartKey) < 0) {
172        lowestStartKey = ri.getStartKey();
173      }
174      if (highestEndKey == null) {
175        highestEndKey = ri.getEndKey();
176      } else if (ri.isLast() || Bytes.compareTo(ri.getEndKey(), highestEndKey) > 0) {
177        highestEndKey = ri.getEndKey();
178      }
179      highestRegionId = ri.getRegionId() > highestRegionId ? ri.getRegionId() : highestRegionId;
180    }
181    // Merged region is sorted between two merging regions in META
182    return RegionInfoBuilder.newBuilder(regionsToMerge[0].getTable()).setStartKey(lowestStartKey)
183      .setEndKey(highestEndKey).setSplit(false)
184      .setRegionId(highestRegionId + 1/* Add one so new merged region is highest */).build();
185  }
186
187  @Override
188  protected Flow executeFromState(final MasterProcedureEnv env, MergeTableRegionsState state) {
189    LOG.trace("{} execute state={}", this, state);
190    try {
191      switch (state) {
192        case MERGE_TABLE_REGIONS_PREPARE:
193          if (!prepareMergeRegion(env)) {
194            assert isFailed() : "Merge region should have an exception here";
195            return Flow.NO_MORE_STATE;
196          }
197          setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_PRE_MERGE_OPERATION);
198          break;
199        case MERGE_TABLE_REGIONS_PRE_MERGE_OPERATION:
200          preMergeRegions(env);
201          setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_CLOSE_REGIONS);
202          break;
203        case MERGE_TABLE_REGIONS_CLOSE_REGIONS:
204          addChildProcedure(createUnassignProcedures(env));
205          setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_CHECK_CLOSED_REGIONS);
206          break;
207        case MERGE_TABLE_REGIONS_CHECK_CLOSED_REGIONS:
208          checkClosedRegions(env);
209          setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_CREATE_MERGED_REGION);
210          break;
211        case MERGE_TABLE_REGIONS_CREATE_MERGED_REGION:
212          removeNonDefaultReplicas(env);
213          createMergedRegion(env);
214          setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_WRITE_MAX_SEQUENCE_ID_FILE);
215          break;
216        case MERGE_TABLE_REGIONS_WRITE_MAX_SEQUENCE_ID_FILE:
217          writeMaxSequenceIdFile(env);
218          setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_PRE_MERGE_COMMIT_OPERATION);
219          break;
220        case MERGE_TABLE_REGIONS_PRE_MERGE_COMMIT_OPERATION:
221          preMergeRegionsCommit(env);
222          setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_UPDATE_META);
223          break;
224        case MERGE_TABLE_REGIONS_UPDATE_META:
225          updateMetaForMergedRegions(env);
226          setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_POST_MERGE_COMMIT_OPERATION);
227          break;
228        case MERGE_TABLE_REGIONS_POST_MERGE_COMMIT_OPERATION:
229          postMergeRegionsCommit(env);
230          setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_OPEN_MERGED_REGION);
231          break;
232        case MERGE_TABLE_REGIONS_OPEN_MERGED_REGION:
233          addChildProcedure(createAssignProcedures(env));
234          setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_POST_OPERATION);
235          break;
236        case MERGE_TABLE_REGIONS_POST_OPERATION:
237          postCompletedMergeRegions(env);
238          return Flow.NO_MORE_STATE;
239        default:
240          throw new UnsupportedOperationException(this + " unhandled state=" + state);
241      }
242    } catch (IOException e) {
243      String msg = "Error trying to merge " + RegionInfo.getShortNameToLog(regionsToMerge) + " in "
244        + getTableName() + " (in state=" + state + ")";
245      if (!isRollbackSupported(state)) {
246        // We reach a state that cannot be rolled back. We just need to keep retrying.
247        LOG.warn(msg, e);
248      } else {
249        LOG.error(msg, e);
250        setFailure("master-merge-regions", e);
251      }
252    }
253    return Flow.HAS_MORE_STATE;
254  }
255
256  /**
257   * To rollback {@link MergeTableRegionsProcedure}, two AssignProcedures are asynchronously
258   * submitted for each region to be merged (rollback doesn't wait on the completion of the
259   * AssignProcedures) . This can be improved by changing rollback() to support sub-procedures. See
260   * HBASE-19851 for details.
261   */
262  @Override
263  protected void rollbackState(final MasterProcedureEnv env, final MergeTableRegionsState state)
264    throws IOException {
265    LOG.trace("{} rollback state={}", this, state);
266
267    try {
268      switch (state) {
269        case MERGE_TABLE_REGIONS_POST_OPERATION:
270        case MERGE_TABLE_REGIONS_OPEN_MERGED_REGION:
271        case MERGE_TABLE_REGIONS_POST_MERGE_COMMIT_OPERATION:
272        case MERGE_TABLE_REGIONS_UPDATE_META:
273          String msg = this + " We are in the " + state + " state."
274            + " It is complicated to rollback the merge operation that region server is working on."
275            + " Rollback is not supported and we should let the merge operation to complete";
276          LOG.warn(msg);
277          // PONR
278          throw new UnsupportedOperationException(this + " unhandled state=" + state);
279        case MERGE_TABLE_REGIONS_PRE_MERGE_COMMIT_OPERATION:
280          break;
281        case MERGE_TABLE_REGIONS_CREATE_MERGED_REGION:
282        case MERGE_TABLE_REGIONS_WRITE_MAX_SEQUENCE_ID_FILE:
283          cleanupMergedRegion(env);
284          break;
285        case MERGE_TABLE_REGIONS_CHECK_CLOSED_REGIONS:
286          break;
287        case MERGE_TABLE_REGIONS_CLOSE_REGIONS:
288          rollbackCloseRegionsForMerge(env);
289          break;
290        case MERGE_TABLE_REGIONS_PRE_MERGE_OPERATION:
291          postRollBackMergeRegions(env);
292          break;
293        case MERGE_TABLE_REGIONS_PREPARE:
294          break;
295        default:
296          throw new UnsupportedOperationException(this + " unhandled state=" + state);
297      }
298    } catch (Exception e) {
299      // This will be retried. Unless there is a bug in the code,
300      // this should be just a "temporary error" (e.g. network down)
301      LOG.warn("Failed rollback attempt step " + state + " for merging the regions "
302        + RegionInfo.getShortNameToLog(regionsToMerge) + " in table " + getTableName(), e);
303      throw e;
304    }
305  }
306
307  /*
308   * Check whether we are in the state that can be rolled back
309   */
310  @Override
311  protected boolean isRollbackSupported(final MergeTableRegionsState state) {
312    switch (state) {
313      case MERGE_TABLE_REGIONS_POST_OPERATION:
314      case MERGE_TABLE_REGIONS_OPEN_MERGED_REGION:
315      case MERGE_TABLE_REGIONS_POST_MERGE_COMMIT_OPERATION:
316      case MERGE_TABLE_REGIONS_UPDATE_META:
317        // It is not safe to rollback in these states.
318        return false;
319      default:
320        break;
321    }
322    return true;
323  }
324
325  private void removeNonDefaultReplicas(MasterProcedureEnv env) throws IOException {
326    AssignmentManagerUtil.removeNonDefaultReplicas(env, Stream.of(regionsToMerge),
327      getRegionReplication(env));
328  }
329
330  private void checkClosedRegions(MasterProcedureEnv env) throws IOException {
331    // Theoretically this should not happen any more after we use TRSP, but anyway
332    // let's add a check here
333    for (RegionInfo region : regionsToMerge) {
334      AssignmentManagerUtil.checkClosedRegion(env, region);
335    }
336  }
337
338  @Override
339  protected MergeTableRegionsState getState(final int stateId) {
340    return MergeTableRegionsState.forNumber(stateId);
341  }
342
343  @Override
344  protected int getStateId(final MergeTableRegionsState state) {
345    return state.getNumber();
346  }
347
348  @Override
349  protected MergeTableRegionsState getInitialState() {
350    return MergeTableRegionsState.MERGE_TABLE_REGIONS_PREPARE;
351  }
352
353  @Override
354  protected void serializeStateData(ProcedureStateSerializer serializer) throws IOException {
355    super.serializeStateData(serializer);
356
357    final MasterProcedureProtos.MergeTableRegionsStateData.Builder mergeTableRegionsMsg =
358      MasterProcedureProtos.MergeTableRegionsStateData.newBuilder()
359        .setUserInfo(MasterProcedureUtil.toProtoUserInfo(getUser()))
360        .setMergedRegionInfo(ProtobufUtil.toRegionInfo(mergedRegion)).setForcible(force);
361    for (RegionInfo ri : regionsToMerge) {
362      mergeTableRegionsMsg.addRegionInfo(ProtobufUtil.toRegionInfo(ri));
363    }
364    serializer.serialize(mergeTableRegionsMsg.build());
365  }
366
367  @Override
368  protected void deserializeStateData(ProcedureStateSerializer serializer) throws IOException {
369    super.deserializeStateData(serializer);
370
371    final MasterProcedureProtos.MergeTableRegionsStateData mergeTableRegionsMsg =
372      serializer.deserialize(MasterProcedureProtos.MergeTableRegionsStateData.class);
373    setUser(MasterProcedureUtil.toUserInfo(mergeTableRegionsMsg.getUserInfo()));
374
375    assert (mergeTableRegionsMsg.getRegionInfoCount() == 2);
376    regionsToMerge = new RegionInfo[mergeTableRegionsMsg.getRegionInfoCount()];
377    for (int i = 0; i < regionsToMerge.length; i++) {
378      regionsToMerge[i] = ProtobufUtil.toRegionInfo(mergeTableRegionsMsg.getRegionInfo(i));
379    }
380
381    mergedRegion = ProtobufUtil.toRegionInfo(mergeTableRegionsMsg.getMergedRegionInfo());
382  }
383
384  @Override
385  public void toStringClassDetails(StringBuilder sb) {
386    sb.append(getClass().getSimpleName());
387    sb.append(" table=");
388    sb.append(getTableName());
389    sb.append(", regions=");
390    sb.append(RegionInfo.getShortNameToLog(regionsToMerge));
391    sb.append(", force=");
392    sb.append(force);
393  }
394
395  @Override
396  protected LockState acquireLock(final MasterProcedureEnv env) {
397    RegionInfo[] lockRegions = Arrays.copyOf(regionsToMerge, regionsToMerge.length + 1);
398    lockRegions[lockRegions.length - 1] = mergedRegion;
399
400    if (env.getProcedureScheduler().waitRegions(this, getTableName(), lockRegions)) {
401      try {
402        LOG.debug(LockState.LOCK_EVENT_WAIT + " " + env.getProcedureScheduler().dumpLocks());
403      } catch (IOException e) {
404        // Ignore, just for logging
405      }
406      return LockState.LOCK_EVENT_WAIT;
407    }
408    return LockState.LOCK_ACQUIRED;
409  }
410
411  @Override
412  protected void releaseLock(final MasterProcedureEnv env) {
413    RegionInfo[] lockRegions = Arrays.copyOf(regionsToMerge, regionsToMerge.length + 1);
414    lockRegions[lockRegions.length - 1] = mergedRegion;
415
416    env.getProcedureScheduler().wakeRegions(this, getTableName(), lockRegions);
417  }
418
419  @Override
420  protected boolean holdLock(MasterProcedureEnv env) {
421    return true;
422  }
423
424  @Override
425  public TableName getTableName() {
426    return mergedRegion.getTable();
427  }
428
429  @Override
430  public TableOperationType getTableOperationType() {
431    return TableOperationType.REGION_MERGE;
432  }
433
434  @Override
435  protected ProcedureMetrics getProcedureMetrics(MasterProcedureEnv env) {
436    return env.getAssignmentManager().getAssignmentManagerMetrics().getMergeProcMetrics();
437  }
438
439  /**
440   * Prepare merge and do some check
441   */
442  private boolean prepareMergeRegion(final MasterProcedureEnv env) throws IOException {
443    // Fail if we are taking snapshot for the given table
444    TableName tn = regionsToMerge[0].getTable();
445    if (env.getMasterServices().getSnapshotManager().isTakingSnapshot(tn)) {
446      throw new MergeRegionException("Skip merging regions "
447        + RegionInfo.getShortNameToLog(regionsToMerge) + ", because we are snapshotting " + tn);
448    }
449
450    // Mostly this check is not used because we already check the switch before submit a merge
451    // procedure. Just for safe, check the switch again. This procedure can be rollbacked if
452    // the switch was set to false after submit.
453    if (!env.getMasterServices().isSplitOrMergeEnabled(MasterSwitchType.MERGE)) {
454      String regionsStr = Arrays.deepToString(this.regionsToMerge);
455      LOG.warn("Merge switch is off! skip merge of " + regionsStr);
456      setFailure(getClass().getSimpleName(),
457        new IOException("Merge of " + regionsStr + " failed because merge switch is off"));
458      return false;
459    }
460
461    if (!env.getMasterServices().getTableDescriptors().get(getTableName()).isMergeEnabled()) {
462      String regionsStr = Arrays.deepToString(regionsToMerge);
463      LOG.warn("Merge is disabled for the table! Skipping merge of {}", regionsStr);
464      setFailure(getClass().getSimpleName(), new IOException(
465        "Merge of " + regionsStr + " failed as region merge is disabled for the table"));
466      return false;
467    }
468
469    RegionStates regionStates = env.getAssignmentManager().getRegionStates();
470    RegionStateStore regionStateStore = env.getAssignmentManager().getRegionStateStore();
471    for (RegionInfo ri : this.regionsToMerge) {
472      if (regionStateStore.hasMergeRegions(ri)) {
473        String msg = "Skip merging " + RegionInfo.getShortNameToLog(regionsToMerge)
474          + ", because a parent, " + RegionInfo.getShortNameToLog(ri) + ", has a merge qualifier "
475          + "(if a 'merge column' in parent, it was recently merged but still has outstanding "
476          + "references to its parents that must be cleared before it can participate in merge -- "
477          + "major compact it to hurry clearing of its references)";
478        LOG.warn(msg);
479        throw new MergeRegionException(msg);
480      }
481      RegionState state = regionStates.getRegionState(ri.getEncodedName());
482      if (state == null) {
483        throw new UnknownRegionException(
484          RegionInfo.getShortNameToLog(ri) + " UNKNOWN (Has it been garbage collected?)");
485      }
486      if (!state.isOpened()) {
487        throw new MergeRegionException("Unable to merge regions that are NOT online: " + ri);
488      }
489      // Ask the remote regionserver if regions are mergeable. If we get an IOE, report it
490      // along with the failure, so we can see why regions are not mergeable at this time.
491      try {
492        if (!isMergeable(env, state)) {
493          setFailure(getClass().getSimpleName(),
494            new MergeRegionException("Skip merging " + RegionInfo.getShortNameToLog(regionsToMerge)
495              + ", because a parent, " + RegionInfo.getShortNameToLog(ri) + ", is not mergeable"));
496          return false;
497        }
498      } catch (IOException e) {
499        IOException ioe = new IOException(RegionInfo.getShortNameToLog(ri) + " NOT mergeable", e);
500        setFailure(getClass().getSimpleName(), ioe);
501        return false;
502      }
503    }
504
505    // Update region states to Merging
506    setRegionStateToMerging(env);
507    return true;
508  }
509
510  private boolean isMergeable(final MasterProcedureEnv env, final RegionState rs)
511    throws IOException {
512    GetRegionInfoResponse response =
513      AssignmentManagerUtil.getRegionInfoResponse(env, rs.getServerName(), rs.getRegion());
514    return response.hasMergeable() && response.getMergeable();
515  }
516
517  /**
518   * Pre merge region action
519   * @param env MasterProcedureEnv
520   **/
521  private void preMergeRegions(final MasterProcedureEnv env) throws IOException {
522    final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
523    if (cpHost != null) {
524      cpHost.preMergeRegionsAction(regionsToMerge, getUser());
525    }
526    // TODO: Clean up split and merge. Currently all over the place.
527    try {
528      env.getMasterServices().getMasterQuotaManager().onRegionMerged(this.mergedRegion);
529    } catch (QuotaExceededException e) {
530      // TODO: why is this here? merge requests can be submitted by actors other than the normalizer
531      env.getMasterServices().getRegionNormalizerManager()
532        .planSkipped(NormalizationPlan.PlanType.MERGE);
533      throw e;
534    }
535  }
536
537  /**
538   * Action after rollback a merge table regions action.
539   */
540  private void postRollBackMergeRegions(final MasterProcedureEnv env) throws IOException {
541    final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
542    if (cpHost != null) {
543      cpHost.postRollBackMergeRegionsAction(regionsToMerge, getUser());
544    }
545  }
546
547  /**
548   * Set the region states to MERGING state
549   */
550  private void setRegionStateToMerging(final MasterProcedureEnv env) {
551    // Set State.MERGING to regions to be merged
552    RegionStates regionStates = env.getAssignmentManager().getRegionStates();
553    for (RegionInfo ri : this.regionsToMerge) {
554      regionStates.getRegionStateNode(ri).setState(State.MERGING);
555    }
556  }
557
558  /**
559   * Create merged region. The way the merge works is that we make a 'merges' temporary directory in
560   * the FIRST parent region to merge (Do not change this without also changing the rollback where
561   * we look in this FIRST region for the merge dir). We then collect here references to all the
562   * store files in all the parent regions including those of the FIRST parent region into a
563   * subdirectory, named for the resultant merged region. We then call commitMergeRegion. It finds
564   * this subdirectory of storefile references and moves them under the new merge region (creating
565   * the region layout as side effect). After assign of the new merge region, we will run a
566   * compaction. This will undo the references but the reference files remain in place until the
567   * archiver runs (which it does on a period as a chore in the RegionServer that hosts the merge
568   * region -- see CompactedHFilesDischarger). Once the archiver has moved aside the no-longer used
569   * references, the merge region no longer has references. The catalog janitor will notice when it
570   * runs next and it will remove the old parent regions.
571   */
572  private void createMergedRegion(final MasterProcedureEnv env) throws IOException {
573    final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
574    final Path tableDir = CommonFSUtils.getTableDir(mfs.getRootDir(), regionsToMerge[0].getTable());
575    final FileSystem fs = mfs.getFileSystem();
576    List<Path> mergedFiles = new ArrayList<>();
577    HRegionFileSystem mergeRegionFs = HRegionFileSystem
578      .createRegionOnFileSystem(env.getMasterConfiguration(), fs, tableDir, mergedRegion);
579
580    for (RegionInfo ri : this.regionsToMerge) {
581      HRegionFileSystem regionFs = HRegionFileSystem
582        .openRegionFromFileSystem(env.getMasterConfiguration(), fs, tableDir, ri, false);
583      mergedFiles.addAll(mergeStoreFiles(env, regionFs, mergeRegionFs, mergedRegion));
584    }
585    assert mergeRegionFs != null;
586    mergeRegionFs.commitMergedRegion(mergedFiles, env);
587
588    // Prepare to create merged regions
589    env.getAssignmentManager().getRegionStates().getOrCreateRegionStateNode(mergedRegion)
590      .setState(State.MERGING_NEW);
591  }
592
593  private List<Path> mergeStoreFiles(MasterProcedureEnv env, HRegionFileSystem regionFs,
594    HRegionFileSystem mergeRegionFs, RegionInfo mergedRegion) throws IOException {
595    final TableDescriptor htd =
596      env.getMasterServices().getTableDescriptors().get(mergedRegion.getTable());
597    List<Path> mergedFiles = new ArrayList<>();
598    for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) {
599      String family = hcd.getNameAsString();
600      StoreFileTracker tracker =
601        StoreFileTrackerFactory.create(env.getMasterConfiguration(), htd, hcd, regionFs);
602      final Collection<StoreFileInfo> storeFiles = tracker.load();
603      if (storeFiles != null && storeFiles.size() > 0) {
604        final Configuration storeConfiguration =
605          StoreUtils.createStoreConfiguration(env.getMasterConfiguration(), htd, hcd);
606        for (StoreFileInfo storeFileInfo : storeFiles) {
607          // Create reference file(s) to parent region file here in mergedDir.
608          // As this procedure is running on master, use CacheConfig.DISABLED means
609          // don't cache any block.
610          // We also need to pass through a suitable CompoundConfiguration as if this
611          // is running in a regionserver's Store context, or we might not be able
612          // to read the hfiles.
613          storeFileInfo.setConf(storeConfiguration);
614          Path refFile = mergeRegionFs.mergeStoreFile(regionFs.getRegionInfo(), family,
615            new HStoreFile(storeFileInfo, hcd.getBloomFilterType(), CacheConfig.DISABLED));
616          mergedFiles.add(refFile);
617        }
618      }
619    }
620    return mergedFiles;
621  }
622
623  /**
624   * Clean up a merged region on rollback after failure.
625   */
626  private void cleanupMergedRegion(final MasterProcedureEnv env) throws IOException {
627    final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
628    TableName tn = this.regionsToMerge[0].getTable();
629    final Path tabledir = CommonFSUtils.getTableDir(mfs.getRootDir(), tn);
630    final FileSystem fs = mfs.getFileSystem();
631    // See createMergedRegion above where we specify the merge dir as being in the
632    // FIRST merge parent region.
633    HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(
634      env.getMasterConfiguration(), fs, tabledir, regionsToMerge[0], false);
635    regionFs.cleanupMergedRegion(mergedRegion);
636  }
637
638  /**
639   * Rollback close regions
640   **/
641  private void rollbackCloseRegionsForMerge(MasterProcedureEnv env) throws IOException {
642    AssignmentManagerUtil.reopenRegionsForRollback(env, Arrays.asList(regionsToMerge),
643      getRegionReplication(env), getServerName(env));
644  }
645
646  private TransitRegionStateProcedure[] createUnassignProcedures(MasterProcedureEnv env)
647    throws IOException {
648    return AssignmentManagerUtil.createUnassignProceduresForSplitOrMerge(env,
649      Stream.of(regionsToMerge), getRegionReplication(env));
650  }
651
652  private TransitRegionStateProcedure[] createAssignProcedures(MasterProcedureEnv env)
653    throws IOException {
654    return AssignmentManagerUtil.createAssignProceduresForOpeningNewRegions(env,
655      Collections.singletonList(mergedRegion), getRegionReplication(env), getServerName(env));
656  }
657
658  private int getRegionReplication(final MasterProcedureEnv env) throws IOException {
659    return env.getMasterServices().getTableDescriptors().get(getTableName()).getRegionReplication();
660  }
661
662  /**
663   * Post merge region action
664   * @param env MasterProcedureEnv
665   **/
666  private void preMergeRegionsCommit(final MasterProcedureEnv env) throws IOException {
667    final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
668    if (cpHost != null) {
669      @MetaMutationAnnotation
670      final List<Mutation> metaEntries = new ArrayList<>();
671      cpHost.preMergeRegionsCommit(regionsToMerge, metaEntries, getUser());
672      try {
673        for (Mutation p : metaEntries) {
674          RegionInfo.parseRegionName(p.getRow());
675        }
676      } catch (IOException e) {
677        LOG.error("Row key of mutation from coprocessor is not parsable as region name. "
678          + "Mutations from coprocessor should only be for hbase:meta table.", e);
679        throw e;
680      }
681    }
682  }
683
684  /**
685   * Add merged region to META and delete original regions.
686   */
687  private void updateMetaForMergedRegions(final MasterProcedureEnv env) throws IOException {
688    env.getAssignmentManager().markRegionAsMerged(mergedRegion, getServerName(env),
689      this.regionsToMerge);
690  }
691
692  /**
693   * Post merge region action
694   * @param env MasterProcedureEnv
695   **/
696  private void postMergeRegionsCommit(final MasterProcedureEnv env) throws IOException {
697    final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
698    if (cpHost != null) {
699      cpHost.postMergeRegionsCommit(regionsToMerge, mergedRegion, getUser());
700    }
701  }
702
703  /**
704   * Post merge region action
705   * @param env MasterProcedureEnv
706   **/
707  private void postCompletedMergeRegions(final MasterProcedureEnv env) throws IOException {
708    final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
709    if (cpHost != null) {
710      cpHost.postCompletedMergeRegionsAction(regionsToMerge, mergedRegion, getUser());
711    }
712  }
713
714  /**
715   * The procedure could be restarted from a different machine. If the variable is null, we need to
716   * retrieve it.
717   * @param env MasterProcedureEnv
718   */
719  private ServerName getServerName(final MasterProcedureEnv env) {
720    if (regionLocation == null) {
721      regionLocation =
722        env.getAssignmentManager().getRegionStates().getRegionServerOfRegion(regionsToMerge[0]);
723      // May still be null here but return null and let caller deal.
724      // Means we lost the in-memory-only location. We are in recovery
725      // or so. The caller should be able to deal w/ a null ServerName.
726      // Let them go to the Balancer to find one to use instead.
727    }
728    return regionLocation;
729  }
730
731  private void writeMaxSequenceIdFile(MasterProcedureEnv env) throws IOException {
732    MasterFileSystem fs = env.getMasterFileSystem();
733    long maxSequenceId = -1L;
734    for (RegionInfo region : regionsToMerge) {
735      maxSequenceId =
736        Math.max(maxSequenceId, WALSplitUtil.getMaxRegionSequenceId(env.getMasterConfiguration(),
737          region, fs::getFileSystem, fs::getWALFileSystem));
738    }
739    if (maxSequenceId > 0) {
740      WALSplitUtil.writeRegionSequenceIdFile(fs.getWALFileSystem(),
741        getWALRegionDir(env, mergedRegion), maxSequenceId);
742    }
743  }
744
745  /** Returns The merged region. Maybe be null if called to early or we failed. */
746  RegionInfo getMergedRegion() {
747    return this.mergedRegion;
748  }
749
750  @Override
751  protected boolean abort(MasterProcedureEnv env) {
752    // Abort means rollback. We can't rollback all steps. HBASE-18018 added abort to all
753    // Procedures. Here is a Procedure that has a PONR and cannot be aborted once it enters this
754    // range of steps; what do we do for these should an operator want to cancel them? HBASE-20022.
755    return isRollbackSupported(getCurrentState()) && super.abort(env);
756  }
757}