001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.assignment;
019
020import java.io.IOException;
021import java.util.ArrayList;
022import java.util.Arrays;
023import java.util.Collection;
024import java.util.Collections;
025import java.util.List;
026import java.util.stream.Stream;
027import org.apache.hadoop.conf.Configuration;
028import org.apache.hadoop.fs.FileSystem;
029import org.apache.hadoop.fs.Path;
030import org.apache.hadoop.hbase.MetaMutationAnnotation;
031import org.apache.hadoop.hbase.MetaTableAccessor;
032import org.apache.hadoop.hbase.ServerName;
033import org.apache.hadoop.hbase.TableName;
034import org.apache.hadoop.hbase.UnknownRegionException;
035import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
036import org.apache.hadoop.hbase.client.DoNotRetryRegionException;
037import org.apache.hadoop.hbase.client.MasterSwitchType;
038import org.apache.hadoop.hbase.client.Mutation;
039import org.apache.hadoop.hbase.client.RegionInfo;
040import org.apache.hadoop.hbase.client.RegionInfoBuilder;
041import org.apache.hadoop.hbase.client.TableDescriptor;
042import org.apache.hadoop.hbase.exceptions.MergeRegionException;
043import org.apache.hadoop.hbase.io.hfile.CacheConfig;
044import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
045import org.apache.hadoop.hbase.master.MasterFileSystem;
046import org.apache.hadoop.hbase.master.RegionState;
047import org.apache.hadoop.hbase.master.RegionState.State;
048import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan;
049import org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure;
050import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
051import org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil;
052import org.apache.hadoop.hbase.procedure2.ProcedureMetrics;
053import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
054import org.apache.hadoop.hbase.quotas.QuotaExceededException;
055import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
056import org.apache.hadoop.hbase.regionserver.HStoreFile;
057import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
058import org.apache.hadoop.hbase.regionserver.StoreUtils;
059import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker;
060import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
061import org.apache.hadoop.hbase.util.Bytes;
062import org.apache.hadoop.hbase.util.CommonFSUtils;
063import org.apache.hadoop.hbase.wal.WALSplitUtil;
064import org.apache.yetus.audience.InterfaceAudience;
065import org.slf4j.Logger;
066import org.slf4j.LoggerFactory;
067
068import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
069import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoResponse;
070import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos;
071import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.MergeTableRegionsState;
072
073/**
074 * The procedure to Merge regions in a table. This procedure takes an exclusive table lock since it
075 * is working over multiple regions. It holds the lock for the life of the procedure. Throws
076 * exception on construction if determines context hostile to merge (cluster going down or master is
077 * shutting down or table is disabled).
078 */
079@InterfaceAudience.Private
080public class MergeTableRegionsProcedure
081  extends AbstractStateMachineTableProcedure<MergeTableRegionsState> {
082  private static final Logger LOG = LoggerFactory.getLogger(MergeTableRegionsProcedure.class);
083  private ServerName regionLocation;
084
085  /**
086   * Two or more regions to merge, the 'merge parents'.
087   */
088  private RegionInfo[] regionsToMerge;
089
090  /**
091   * The resulting merged region.
092   */
093  private RegionInfo mergedRegion;
094
095  private boolean force;
096
097  public MergeTableRegionsProcedure() {
098    // Required by the Procedure framework to create the procedure on replay
099  }
100
101  public MergeTableRegionsProcedure(final MasterProcedureEnv env, final RegionInfo[] regionsToMerge,
102    final boolean force) throws IOException {
103    super(env);
104    // Check parent regions. Make sure valid before starting work.
105    // This check calls the super method #checkOnline also.
106    checkRegionsToMerge(env, regionsToMerge, force);
107    // Sort the regions going into the merge.
108    Arrays.sort(regionsToMerge);
109    this.regionsToMerge = regionsToMerge;
110    this.mergedRegion = createMergedRegionInfo(regionsToMerge);
111    // Preflight depends on mergedRegion being set (at least).
112    preflightChecks(env, true);
113    this.force = force;
114  }
115
116  /**
117   * @throws MergeRegionException If unable to merge regions for whatever reasons.
118   */
119  private static void checkRegionsToMerge(MasterProcedureEnv env, final RegionInfo[] regions,
120    final boolean force) throws MergeRegionException {
121    long count = Arrays.stream(regions).distinct().count();
122    if (regions.length != count) {
123      throw new MergeRegionException("Duplicate regions specified; cannot merge a region to "
124        + "itself. Passed in " + regions.length + " but only " + count + " unique.");
125    }
126    if (count < 2) {
127      throw new MergeRegionException("Need two Regions at least to run a Merge");
128    }
129    RegionInfo previous = null;
130    for (RegionInfo ri : regions) {
131      if (previous != null) {
132        if (!previous.getTable().equals(ri.getTable())) {
133          String msg = "Can't merge regions from different tables: " + previous + ", " + ri;
134          LOG.warn(msg);
135          throw new MergeRegionException(msg);
136        }
137        if (!force && !ri.isAdjacent(previous) && !ri.isOverlap(previous)) {
138          String msg = "Unable to merge non-adjacent or non-overlapping regions '"
139            + previous.getShortNameToLog() + "', '" + ri.getShortNameToLog() + "' when force=false";
140          LOG.warn(msg);
141          throw new MergeRegionException(msg);
142        }
143      }
144
145      if (ri.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) {
146        throw new MergeRegionException("Can't merge non-default replicas; " + ri);
147      }
148      try {
149        checkOnline(env, ri);
150      } catch (DoNotRetryRegionException dnrre) {
151        throw new MergeRegionException(dnrre);
152      }
153
154      previous = ri;
155    }
156  }
157
158  /**
159   * Create merged region info by looking at passed in <code>regionsToMerge</code> to figure what
160   * extremes for start and end keys to use; merged region needs to have an extent sufficient to
161   * cover all regions-to-merge.
162   */
163  private static RegionInfo createMergedRegionInfo(final RegionInfo[] regionsToMerge) {
164    byte[] lowestStartKey = null;
165    byte[] highestEndKey = null;
166    // Region Id is a timestamp. Merged region's id can't be less than that of
167    // merging regions else will insert at wrong location in hbase:meta (See HBASE-710).
168    long highestRegionId = -1;
169    for (RegionInfo ri : regionsToMerge) {
170      if (lowestStartKey == null) {
171        lowestStartKey = ri.getStartKey();
172      } else if (Bytes.compareTo(ri.getStartKey(), lowestStartKey) < 0) {
173        lowestStartKey = ri.getStartKey();
174      }
175      if (highestEndKey == null) {
176        highestEndKey = ri.getEndKey();
177      } else if (ri.isLast() || Bytes.compareTo(ri.getEndKey(), highestEndKey) > 0) {
178        highestEndKey = ri.getEndKey();
179      }
180      highestRegionId = ri.getRegionId() > highestRegionId ? ri.getRegionId() : highestRegionId;
181    }
182    // Merged region is sorted between two merging regions in META
183    return RegionInfoBuilder.newBuilder(regionsToMerge[0].getTable()).setStartKey(lowestStartKey)
184      .setEndKey(highestEndKey).setSplit(false)
185      .setRegionId(highestRegionId + 1/* Add one so new merged region is highest */).build();
186  }
187
188  @Override
189  protected Flow executeFromState(final MasterProcedureEnv env, MergeTableRegionsState state) {
190    LOG.trace("{} execute state={}", this, state);
191    try {
192      switch (state) {
193        case MERGE_TABLE_REGIONS_PREPARE:
194          if (!prepareMergeRegion(env)) {
195            assert isFailed() : "Merge region should have an exception here";
196            return Flow.NO_MORE_STATE;
197          }
198          setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_PRE_MERGE_OPERATION);
199          break;
200        case MERGE_TABLE_REGIONS_PRE_MERGE_OPERATION:
201          preMergeRegions(env);
202          setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_CLOSE_REGIONS);
203          break;
204        case MERGE_TABLE_REGIONS_CLOSE_REGIONS:
205          addChildProcedure(createUnassignProcedures(env));
206          setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_CHECK_CLOSED_REGIONS);
207          break;
208        case MERGE_TABLE_REGIONS_CHECK_CLOSED_REGIONS:
209          checkClosedRegions(env);
210          setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_CREATE_MERGED_REGION);
211          break;
212        case MERGE_TABLE_REGIONS_CREATE_MERGED_REGION:
213          removeNonDefaultReplicas(env);
214          createMergedRegion(env);
215          setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_WRITE_MAX_SEQUENCE_ID_FILE);
216          break;
217        case MERGE_TABLE_REGIONS_WRITE_MAX_SEQUENCE_ID_FILE:
218          writeMaxSequenceIdFile(env);
219          setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_PRE_MERGE_COMMIT_OPERATION);
220          break;
221        case MERGE_TABLE_REGIONS_PRE_MERGE_COMMIT_OPERATION:
222          preMergeRegionsCommit(env);
223          setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_UPDATE_META);
224          break;
225        case MERGE_TABLE_REGIONS_UPDATE_META:
226          updateMetaForMergedRegions(env);
227          setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_POST_MERGE_COMMIT_OPERATION);
228          break;
229        case MERGE_TABLE_REGIONS_POST_MERGE_COMMIT_OPERATION:
230          postMergeRegionsCommit(env);
231          setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_OPEN_MERGED_REGION);
232          break;
233        case MERGE_TABLE_REGIONS_OPEN_MERGED_REGION:
234          addChildProcedure(createAssignProcedures(env));
235          setNextState(MergeTableRegionsState.MERGE_TABLE_REGIONS_POST_OPERATION);
236          break;
237        case MERGE_TABLE_REGIONS_POST_OPERATION:
238          postCompletedMergeRegions(env);
239          return Flow.NO_MORE_STATE;
240        default:
241          throw new UnsupportedOperationException(this + " unhandled state=" + state);
242      }
243    } catch (IOException e) {
244      String msg = "Error trying to merge " + RegionInfo.getShortNameToLog(regionsToMerge) + " in "
245        + getTableName() + " (in state=" + state + ")";
246      if (!isRollbackSupported(state)) {
247        // We reach a state that cannot be rolled back. We just need to keep retrying.
248        LOG.warn(msg, e);
249      } else {
250        LOG.error(msg, e);
251        setFailure("master-merge-regions", e);
252      }
253    }
254    return Flow.HAS_MORE_STATE;
255  }
256
257  /**
258   * To rollback {@link MergeTableRegionsProcedure}, two AssignProcedures are asynchronously
259   * submitted for each region to be merged (rollback doesn't wait on the completion of the
260   * AssignProcedures) . This can be improved by changing rollback() to support sub-procedures. See
261   * HBASE-19851 for details.
262   */
263  @Override
264  protected void rollbackState(final MasterProcedureEnv env, final MergeTableRegionsState state)
265    throws IOException {
266    LOG.trace("{} rollback state={}", this, state);
267
268    try {
269      switch (state) {
270        case MERGE_TABLE_REGIONS_POST_OPERATION:
271        case MERGE_TABLE_REGIONS_OPEN_MERGED_REGION:
272        case MERGE_TABLE_REGIONS_POST_MERGE_COMMIT_OPERATION:
273        case MERGE_TABLE_REGIONS_UPDATE_META:
274          String msg = this + " We are in the " + state + " state."
275            + " It is complicated to rollback the merge operation that region server is working on."
276            + " Rollback is not supported and we should let the merge operation to complete";
277          LOG.warn(msg);
278          // PONR
279          throw new UnsupportedOperationException(this + " unhandled state=" + state);
280        case MERGE_TABLE_REGIONS_PRE_MERGE_COMMIT_OPERATION:
281          break;
282        case MERGE_TABLE_REGIONS_CREATE_MERGED_REGION:
283        case MERGE_TABLE_REGIONS_WRITE_MAX_SEQUENCE_ID_FILE:
284          cleanupMergedRegion(env);
285          break;
286        case MERGE_TABLE_REGIONS_CHECK_CLOSED_REGIONS:
287          break;
288        case MERGE_TABLE_REGIONS_CLOSE_REGIONS:
289          rollbackCloseRegionsForMerge(env);
290          break;
291        case MERGE_TABLE_REGIONS_PRE_MERGE_OPERATION:
292          postRollBackMergeRegions(env);
293          break;
294        case MERGE_TABLE_REGIONS_PREPARE:
295          break;
296        default:
297          throw new UnsupportedOperationException(this + " unhandled state=" + state);
298      }
299    } catch (Exception e) {
300      // This will be retried. Unless there is a bug in the code,
301      // this should be just a "temporary error" (e.g. network down)
302      LOG.warn("Failed rollback attempt step " + state + " for merging the regions "
303        + RegionInfo.getShortNameToLog(regionsToMerge) + " in table " + getTableName(), e);
304      throw e;
305    }
306  }
307
308  /*
309   * Check whether we are in the state that can be rolled back
310   */
311  @Override
312  protected boolean isRollbackSupported(final MergeTableRegionsState state) {
313    switch (state) {
314      case MERGE_TABLE_REGIONS_POST_OPERATION:
315      case MERGE_TABLE_REGIONS_OPEN_MERGED_REGION:
316      case MERGE_TABLE_REGIONS_POST_MERGE_COMMIT_OPERATION:
317      case MERGE_TABLE_REGIONS_UPDATE_META:
318        // It is not safe to rollback in these states.
319        return false;
320      default:
321        break;
322    }
323    return true;
324  }
325
326  private void removeNonDefaultReplicas(MasterProcedureEnv env) throws IOException {
327    AssignmentManagerUtil.removeNonDefaultReplicas(env, Stream.of(regionsToMerge),
328      getRegionReplication(env));
329  }
330
331  private void checkClosedRegions(MasterProcedureEnv env) throws IOException {
332    // Theoretically this should not happen any more after we use TRSP, but anyway
333    // let's add a check here
334    for (RegionInfo region : regionsToMerge) {
335      AssignmentManagerUtil.checkClosedRegion(env, region);
336    }
337  }
338
339  @Override
340  protected MergeTableRegionsState getState(final int stateId) {
341    return MergeTableRegionsState.forNumber(stateId);
342  }
343
344  @Override
345  protected int getStateId(final MergeTableRegionsState state) {
346    return state.getNumber();
347  }
348
349  @Override
350  protected MergeTableRegionsState getInitialState() {
351    return MergeTableRegionsState.MERGE_TABLE_REGIONS_PREPARE;
352  }
353
354  @Override
355  protected void serializeStateData(ProcedureStateSerializer serializer) throws IOException {
356    super.serializeStateData(serializer);
357
358    final MasterProcedureProtos.MergeTableRegionsStateData.Builder mergeTableRegionsMsg =
359      MasterProcedureProtos.MergeTableRegionsStateData.newBuilder()
360        .setUserInfo(MasterProcedureUtil.toProtoUserInfo(getUser()))
361        .setMergedRegionInfo(ProtobufUtil.toRegionInfo(mergedRegion)).setForcible(force);
362    for (RegionInfo ri : regionsToMerge) {
363      mergeTableRegionsMsg.addRegionInfo(ProtobufUtil.toRegionInfo(ri));
364    }
365    serializer.serialize(mergeTableRegionsMsg.build());
366  }
367
368  @Override
369  protected void deserializeStateData(ProcedureStateSerializer serializer) throws IOException {
370    super.deserializeStateData(serializer);
371
372    final MasterProcedureProtos.MergeTableRegionsStateData mergeTableRegionsMsg =
373      serializer.deserialize(MasterProcedureProtos.MergeTableRegionsStateData.class);
374    setUser(MasterProcedureUtil.toUserInfo(mergeTableRegionsMsg.getUserInfo()));
375
376    assert (mergeTableRegionsMsg.getRegionInfoCount() == 2);
377    regionsToMerge = new RegionInfo[mergeTableRegionsMsg.getRegionInfoCount()];
378    for (int i = 0; i < regionsToMerge.length; i++) {
379      regionsToMerge[i] = ProtobufUtil.toRegionInfo(mergeTableRegionsMsg.getRegionInfo(i));
380    }
381
382    mergedRegion = ProtobufUtil.toRegionInfo(mergeTableRegionsMsg.getMergedRegionInfo());
383  }
384
385  @Override
386  public void toStringClassDetails(StringBuilder sb) {
387    sb.append(getClass().getSimpleName());
388    sb.append(" table=");
389    sb.append(getTableName());
390    sb.append(", regions=");
391    sb.append(RegionInfo.getShortNameToLog(regionsToMerge));
392    sb.append(", force=");
393    sb.append(force);
394  }
395
396  @Override
397  protected LockState acquireLock(final MasterProcedureEnv env) {
398    RegionInfo[] lockRegions = Arrays.copyOf(regionsToMerge, regionsToMerge.length + 1);
399    lockRegions[lockRegions.length - 1] = mergedRegion;
400
401    if (env.getProcedureScheduler().waitRegions(this, getTableName(), lockRegions)) {
402      try {
403        LOG.debug(LockState.LOCK_EVENT_WAIT + " " + env.getProcedureScheduler().dumpLocks());
404      } catch (IOException e) {
405        // Ignore, just for logging
406      }
407      return LockState.LOCK_EVENT_WAIT;
408    }
409    return LockState.LOCK_ACQUIRED;
410  }
411
412  @Override
413  protected void releaseLock(final MasterProcedureEnv env) {
414    RegionInfo[] lockRegions = Arrays.copyOf(regionsToMerge, regionsToMerge.length + 1);
415    lockRegions[lockRegions.length - 1] = mergedRegion;
416
417    env.getProcedureScheduler().wakeRegions(this, getTableName(), lockRegions);
418  }
419
420  @Override
421  protected boolean holdLock(MasterProcedureEnv env) {
422    return true;
423  }
424
425  @Override
426  public TableName getTableName() {
427    return mergedRegion.getTable();
428  }
429
430  @Override
431  public TableOperationType getTableOperationType() {
432    return TableOperationType.REGION_MERGE;
433  }
434
435  @Override
436  protected ProcedureMetrics getProcedureMetrics(MasterProcedureEnv env) {
437    return env.getAssignmentManager().getAssignmentManagerMetrics().getMergeProcMetrics();
438  }
439
440  /**
441   * Prepare merge and do some check
442   */
443  private boolean prepareMergeRegion(final MasterProcedureEnv env) throws IOException {
444    // Fail if we are taking snapshot for the given table
445    TableName tn = regionsToMerge[0].getTable();
446    if (env.getMasterServices().getSnapshotManager().isTakingSnapshot(tn)) {
447      throw new MergeRegionException("Skip merging regions "
448        + RegionInfo.getShortNameToLog(regionsToMerge) + ", because we are snapshotting " + tn);
449    }
450
451    // Mostly this check is not used because we already check the switch before submit a merge
452    // procedure. Just for safe, check the switch again. This procedure can be rollbacked if
453    // the switch was set to false after submit.
454    if (!env.getMasterServices().isSplitOrMergeEnabled(MasterSwitchType.MERGE)) {
455      String regionsStr = Arrays.deepToString(this.regionsToMerge);
456      LOG.warn("Merge switch is off! skip merge of " + regionsStr);
457      setFailure(getClass().getSimpleName(),
458        new IOException("Merge of " + regionsStr + " failed because merge switch is off"));
459      return false;
460    }
461
462    if (!env.getMasterServices().getTableDescriptors().get(getTableName()).isMergeEnabled()) {
463      String regionsStr = Arrays.deepToString(regionsToMerge);
464      LOG.warn("Merge is disabled for the table! Skipping merge of {}", regionsStr);
465      setFailure(getClass().getSimpleName(), new IOException(
466        "Merge of " + regionsStr + " failed as region merge is disabled for the table"));
467      return false;
468    }
469
470    RegionStates regionStates = env.getAssignmentManager().getRegionStates();
471    for (RegionInfo ri : this.regionsToMerge) {
472      if (
473        MetaTableAccessor.hasMergeRegions(env.getMasterServices().getConnection(),
474          ri.getRegionName())
475      ) {
476        String msg = "Skip merging " + RegionInfo.getShortNameToLog(regionsToMerge)
477          + ", because a parent, " + RegionInfo.getShortNameToLog(ri) + ", has a merge qualifier "
478          + "(if a 'merge column' in parent, it was recently merged but still has outstanding "
479          + "references to its parents that must be cleared before it can participate in merge -- "
480          + "major compact it to hurry clearing of its references)";
481        LOG.warn(msg);
482        throw new MergeRegionException(msg);
483      }
484      RegionState state = regionStates.getRegionState(ri.getEncodedName());
485      if (state == null) {
486        throw new UnknownRegionException(
487          RegionInfo.getShortNameToLog(ri) + " UNKNOWN (Has it been garbage collected?)");
488      }
489      if (!state.isOpened()) {
490        throw new MergeRegionException("Unable to merge regions that are NOT online: " + ri);
491      }
492      // Ask the remote regionserver if regions are mergeable. If we get an IOE, report it
493      // along with the failure, so we can see why regions are not mergeable at this time.
494      try {
495        if (!isMergeable(env, state)) {
496          setFailure(getClass().getSimpleName(),
497            new MergeRegionException("Skip merging " + RegionInfo.getShortNameToLog(regionsToMerge)
498              + ", because a parent, " + RegionInfo.getShortNameToLog(ri) + ", is not mergeable"));
499          return false;
500        }
501      } catch (IOException e) {
502        IOException ioe = new IOException(RegionInfo.getShortNameToLog(ri) + " NOT mergeable", e);
503        setFailure(getClass().getSimpleName(), ioe);
504        return false;
505      }
506    }
507
508    // Update region states to Merging
509    setRegionStateToMerging(env);
510    return true;
511  }
512
513  private boolean isMergeable(final MasterProcedureEnv env, final RegionState rs)
514    throws IOException {
515    GetRegionInfoResponse response =
516      AssignmentManagerUtil.getRegionInfoResponse(env, rs.getServerName(), rs.getRegion());
517    return response.hasMergeable() && response.getMergeable();
518  }
519
520  /**
521   * Pre merge region action
522   * @param env MasterProcedureEnv
523   **/
524  private void preMergeRegions(final MasterProcedureEnv env) throws IOException {
525    final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
526    if (cpHost != null) {
527      cpHost.preMergeRegionsAction(regionsToMerge, getUser());
528    }
529    // TODO: Clean up split and merge. Currently all over the place.
530    try {
531      env.getMasterServices().getMasterQuotaManager().onRegionMerged(this.mergedRegion);
532    } catch (QuotaExceededException e) {
533      // TODO: why is this here? merge requests can be submitted by actors other than the normalizer
534      env.getMasterServices().getRegionNormalizerManager()
535        .planSkipped(NormalizationPlan.PlanType.MERGE);
536      throw e;
537    }
538  }
539
540  /**
541   * Action after rollback a merge table regions action.
542   */
543  private void postRollBackMergeRegions(final MasterProcedureEnv env) throws IOException {
544    final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
545    if (cpHost != null) {
546      cpHost.postRollBackMergeRegionsAction(regionsToMerge, getUser());
547    }
548  }
549
550  /**
551   * Set the region states to MERGING state
552   */
553  private void setRegionStateToMerging(final MasterProcedureEnv env) {
554    // Set State.MERGING to regions to be merged
555    RegionStates regionStates = env.getAssignmentManager().getRegionStates();
556    for (RegionInfo ri : this.regionsToMerge) {
557      regionStates.getRegionStateNode(ri).setState(State.MERGING);
558    }
559  }
560
561  /**
562   * Create merged region. The way the merge works is that we make a 'merges' temporary directory in
563   * the FIRST parent region to merge (Do not change this without also changing the rollback where
564   * we look in this FIRST region for the merge dir). We then collect here references to all the
565   * store files in all the parent regions including those of the FIRST parent region into a
566   * subdirectory, named for the resultant merged region. We then call commitMergeRegion. It finds
567   * this subdirectory of storefile references and moves them under the new merge region (creating
568   * the region layout as side effect). After assign of the new merge region, we will run a
569   * compaction. This will undo the references but the reference files remain in place until the
570   * archiver runs (which it does on a period as a chore in the RegionServer that hosts the merge
571   * region -- see CompactedHFilesDischarger). Once the archiver has moved aside the no-longer used
572   * references, the merge region no longer has references. The catalog janitor will notice when it
573   * runs next and it will remove the old parent regions.
574   */
575  private void createMergedRegion(final MasterProcedureEnv env) throws IOException {
576    final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
577    final Path tableDir = CommonFSUtils.getTableDir(mfs.getRootDir(), regionsToMerge[0].getTable());
578    final FileSystem fs = mfs.getFileSystem();
579    List<Path> mergedFiles = new ArrayList<>();
580    HRegionFileSystem mergeRegionFs = HRegionFileSystem
581      .createRegionOnFileSystem(env.getMasterConfiguration(), fs, tableDir, mergedRegion);
582
583    for (RegionInfo ri : this.regionsToMerge) {
584      HRegionFileSystem regionFs = HRegionFileSystem
585        .openRegionFromFileSystem(env.getMasterConfiguration(), fs, tableDir, ri, false);
586      mergedFiles.addAll(mergeStoreFiles(env, regionFs, mergeRegionFs, mergedRegion));
587    }
588    assert mergeRegionFs != null;
589    mergeRegionFs.commitMergedRegion(mergedFiles, env);
590
591    // Prepare to create merged regions
592    env.getAssignmentManager().getRegionStates().getOrCreateRegionStateNode(mergedRegion)
593      .setState(State.MERGING_NEW);
594  }
595
596  private List<Path> mergeStoreFiles(MasterProcedureEnv env, HRegionFileSystem regionFs,
597    HRegionFileSystem mergeRegionFs, RegionInfo mergedRegion) throws IOException {
598    final TableDescriptor htd =
599      env.getMasterServices().getTableDescriptors().get(mergedRegion.getTable());
600    List<Path> mergedFiles = new ArrayList<>();
601    for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) {
602      String family = hcd.getNameAsString();
603      StoreFileTracker tracker =
604        StoreFileTrackerFactory.create(env.getMasterConfiguration(), htd, hcd, regionFs);
605      final Collection<StoreFileInfo> storeFiles = tracker.load();
606      if (storeFiles != null && storeFiles.size() > 0) {
607        final Configuration storeConfiguration =
608          StoreUtils.createStoreConfiguration(env.getMasterConfiguration(), htd, hcd);
609        for (StoreFileInfo storeFileInfo : storeFiles) {
610          // Create reference file(s) to parent region file here in mergedDir.
611          // As this procedure is running on master, use CacheConfig.DISABLED means
612          // don't cache any block.
613          // We also need to pass through a suitable CompoundConfiguration as if this
614          // is running in a regionserver's Store context, or we might not be able
615          // to read the hfiles.
616          storeFileInfo.setConf(storeConfiguration);
617          Path refFile = mergeRegionFs.mergeStoreFile(regionFs.getRegionInfo(), family,
618            new HStoreFile(storeFileInfo, hcd.getBloomFilterType(), CacheConfig.DISABLED));
619          mergedFiles.add(refFile);
620        }
621      }
622    }
623    return mergedFiles;
624  }
625
626  /**
627   * Clean up a merged region on rollback after failure.
628   */
629  private void cleanupMergedRegion(final MasterProcedureEnv env) throws IOException {
630    final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
631    TableName tn = this.regionsToMerge[0].getTable();
632    final Path tabledir = CommonFSUtils.getTableDir(mfs.getRootDir(), tn);
633    final FileSystem fs = mfs.getFileSystem();
634    // See createMergedRegion above where we specify the merge dir as being in the
635    // FIRST merge parent region.
636    HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(
637      env.getMasterConfiguration(), fs, tabledir, regionsToMerge[0], false);
638    regionFs.cleanupMergedRegion(mergedRegion);
639  }
640
641  /**
642   * Rollback close regions
643   **/
644  private void rollbackCloseRegionsForMerge(MasterProcedureEnv env) throws IOException {
645    AssignmentManagerUtil.reopenRegionsForRollback(env, Arrays.asList(regionsToMerge),
646      getRegionReplication(env), getServerName(env));
647  }
648
649  private TransitRegionStateProcedure[] createUnassignProcedures(MasterProcedureEnv env)
650    throws IOException {
651    return AssignmentManagerUtil.createUnassignProceduresForSplitOrMerge(env,
652      Stream.of(regionsToMerge), getRegionReplication(env));
653  }
654
655  private TransitRegionStateProcedure[] createAssignProcedures(MasterProcedureEnv env)
656    throws IOException {
657    return AssignmentManagerUtil.createAssignProceduresForOpeningNewRegions(env,
658      Collections.singletonList(mergedRegion), getRegionReplication(env), getServerName(env));
659  }
660
661  private int getRegionReplication(final MasterProcedureEnv env) throws IOException {
662    return env.getMasterServices().getTableDescriptors().get(getTableName()).getRegionReplication();
663  }
664
665  /**
666   * Post merge region action
667   * @param env MasterProcedureEnv
668   **/
669  private void preMergeRegionsCommit(final MasterProcedureEnv env) throws IOException {
670    final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
671    if (cpHost != null) {
672      @MetaMutationAnnotation
673      final List<Mutation> metaEntries = new ArrayList<>();
674      cpHost.preMergeRegionsCommit(regionsToMerge, metaEntries, getUser());
675      try {
676        for (Mutation p : metaEntries) {
677          RegionInfo.parseRegionName(p.getRow());
678        }
679      } catch (IOException e) {
680        LOG.error("Row key of mutation from coprocessor is not parsable as region name. "
681          + "Mutations from coprocessor should only be for hbase:meta table.", e);
682        throw e;
683      }
684    }
685  }
686
687  /**
688   * Add merged region to META and delete original regions.
689   */
690  private void updateMetaForMergedRegions(final MasterProcedureEnv env) throws IOException {
691    env.getAssignmentManager().markRegionAsMerged(mergedRegion, getServerName(env),
692      this.regionsToMerge);
693  }
694
695  /**
696   * Post merge region action
697   * @param env MasterProcedureEnv
698   **/
699  private void postMergeRegionsCommit(final MasterProcedureEnv env) throws IOException {
700    final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
701    if (cpHost != null) {
702      cpHost.postMergeRegionsCommit(regionsToMerge, mergedRegion, getUser());
703    }
704  }
705
706  /**
707   * Post merge region action
708   * @param env MasterProcedureEnv
709   **/
710  private void postCompletedMergeRegions(final MasterProcedureEnv env) throws IOException {
711    final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
712    if (cpHost != null) {
713      cpHost.postCompletedMergeRegionsAction(regionsToMerge, mergedRegion, getUser());
714    }
715  }
716
717  /**
718   * The procedure could be restarted from a different machine. If the variable is null, we need to
719   * retrieve it.
720   * @param env MasterProcedureEnv n
721   */
722  private ServerName getServerName(final MasterProcedureEnv env) {
723    if (regionLocation == null) {
724      regionLocation =
725        env.getAssignmentManager().getRegionStates().getRegionServerOfRegion(regionsToMerge[0]);
726      // May still be null here but return null and let caller deal.
727      // Means we lost the in-memory-only location. We are in recovery
728      // or so. The caller should be able to deal w/ a null ServerName.
729      // Let them go to the Balancer to find one to use instead.
730    }
731    return regionLocation;
732  }
733
734  private void writeMaxSequenceIdFile(MasterProcedureEnv env) throws IOException {
735    MasterFileSystem fs = env.getMasterFileSystem();
736    long maxSequenceId = -1L;
737    for (RegionInfo region : regionsToMerge) {
738      maxSequenceId =
739        Math.max(maxSequenceId, WALSplitUtil.getMaxRegionSequenceId(env.getMasterConfiguration(),
740          region, fs::getFileSystem, fs::getWALFileSystem));
741    }
742    if (maxSequenceId > 0) {
743      WALSplitUtil.writeRegionSequenceIdFile(fs.getWALFileSystem(),
744        getWALRegionDir(env, mergedRegion), maxSequenceId);
745    }
746  }
747
748  /** Returns The merged region. Maybe be null if called to early or we failed. */
749  RegionInfo getMergedRegion() {
750    return this.mergedRegion;
751  }
752
753  @Override
754  protected boolean abort(MasterProcedureEnv env) {
755    // Abort means rollback. We can't rollback all steps. HBASE-18018 added abort to all
756    // Procedures. Here is a Procedure that has a PONR and cannot be aborted once it enters this
757    // range of steps; what do we do for these should an operator want to cancel them? HBASE-20022.
758    return isRollbackSupported(getCurrentState()) && super.abort(env);
759  }
760}