001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.procedure;
019
020import static org.apache.hadoop.hbase.HConstants.DEFAULT_HBASE_SPLIT_COORDINATED_BY_ZK;
021import static org.apache.hadoop.hbase.HConstants.HBASE_SPLIT_WAL_COORDINATED_BY_ZK;
022
023import java.io.IOException;
024import java.util.ArrayList;
025import java.util.Arrays;
026import java.util.List;
027import org.apache.hadoop.hbase.DoNotRetryIOException;
028import org.apache.hadoop.hbase.ServerName;
029import org.apache.hadoop.hbase.client.RegionInfo;
030import org.apache.hadoop.hbase.client.RegionInfoBuilder;
031import org.apache.hadoop.hbase.client.RegionReplicaUtil;
032import org.apache.hadoop.hbase.client.TableState;
033import org.apache.hadoop.hbase.master.MasterServices;
034import org.apache.hadoop.hbase.master.MasterWalManager;
035import org.apache.hadoop.hbase.master.SplitWALManager;
036import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
037import org.apache.hadoop.hbase.master.assignment.RegionStateNode;
038import org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure;
039import org.apache.hadoop.hbase.master.replication.ClaimReplicationQueuesProcedure;
040import org.apache.hadoop.hbase.monitoring.MonitoredTask;
041import org.apache.hadoop.hbase.monitoring.TaskMonitor;
042import org.apache.hadoop.hbase.procedure2.Procedure;
043import org.apache.hadoop.hbase.procedure2.ProcedureMetrics;
044import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
045import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException;
046import org.apache.hadoop.hbase.procedure2.ProcedureYieldException;
047import org.apache.hadoop.hbase.procedure2.StateMachineProcedure;
048import org.apache.yetus.audience.InterfaceAudience;
049import org.slf4j.Logger;
050import org.slf4j.LoggerFactory;
051
052import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
053import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos;
054import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.ServerCrashState;
055
056/**
057 * Handle crashed server. This is a port to ProcedureV2 of what used to be euphemistically called
058 * ServerShutdownHandler.
059 * <p>
060 * The procedure flow varies dependent on whether meta is assigned and if we are to split logs.
061 * <p>
062 * We come in here after ServerManager has noticed a server has expired. Procedures queued on the
063 * rpc should have been notified about fail and should be concurrently getting themselves ready to
064 * assign elsewhere.
065 */
066@InterfaceAudience.Private
067public class ServerCrashProcedure extends
068  StateMachineProcedure<MasterProcedureEnv, ServerCrashState> implements ServerProcedureInterface {
069  private static final Logger LOG = LoggerFactory.getLogger(ServerCrashProcedure.class);
070
071  /**
072   * Configuration parameter to enable/disable the retain region assignment during
073   * ServerCrashProcedure.
074   * <p>
075   * By default retain assignment is disabled which makes the failover faster and improve the
076   * availability; useful for cloud scenario where region block locality is not important. Enable
077   * this when RegionServers are deployed on same host where Datanode are running, this will improve
078   * read performance due to local read.
079   * <p>
080   * see HBASE-24900 for more details.
081   */
082  public static final String MASTER_SCP_RETAIN_ASSIGNMENT = "hbase.master.scp.retain.assignment";
083  /** Default value of {@link #MASTER_SCP_RETAIN_ASSIGNMENT} */
084  public static final boolean DEFAULT_MASTER_SCP_RETAIN_ASSIGNMENT = false;
085
086  /**
087   * Name of the crashed server to process.
088   */
089  private ServerName serverName;
090
091  /**
092   * Whether DeadServer knows that we are processing it.
093   */
094  private boolean notifiedDeadServer = false;
095
096  /**
097   * Regions that were on the crashed server.
098   */
099  private List<RegionInfo> regionsOnCrashedServer;
100
101  private boolean carryingMeta = false;
102  private boolean shouldSplitWal;
103  private MonitoredTask status;
104  // currentRunningState is updated when ServerCrashProcedure get scheduled, child procedures update
105  // progress will not update the state because the actual state is overwritten by its next state
106  private ServerCrashState currentRunningState = getInitialState();
107
108  /**
109   * Call this constructor queuing up a Procedure.
110   * @param serverName     Name of the crashed server.
111   * @param shouldSplitWal True if we should split WALs as part of crashed server processing.
112   * @param carryingMeta   True if carrying hbase:meta table region.
113   */
114  public ServerCrashProcedure(final MasterProcedureEnv env, final ServerName serverName,
115    final boolean shouldSplitWal, final boolean carryingMeta) {
116    this.serverName = serverName;
117    this.shouldSplitWal = shouldSplitWal;
118    this.carryingMeta = carryingMeta;
119    this.setOwner(env.getRequestUser());
120  }
121
122  /**
123   * Used when deserializing from a procedure store; we'll construct one of these then call
124   * #deserializeStateData(InputStream). Do not use directly.
125   */
126  public ServerCrashProcedure() {
127  }
128
129  public boolean isInRecoverMetaState() {
130    return getCurrentState() == ServerCrashState.SERVER_CRASH_PROCESS_META;
131  }
132
133  @Override
134  protected Flow executeFromState(MasterProcedureEnv env, ServerCrashState state)
135    throws ProcedureSuspendedException, ProcedureYieldException {
136    final MasterServices services = env.getMasterServices();
137    final AssignmentManager am = env.getAssignmentManager();
138    updateProgress(true);
139    // HBASE-14802 If we have not yet notified that we are processing a dead server, do so now.
140    // This adds server to the DeadServer processing list but not to the DeadServers list.
141    // Server gets removed from processing list below on procedure successful finish.
142    if (!notifiedDeadServer) {
143      notifiedDeadServer = true;
144    }
145
146    switch (state) {
147      case SERVER_CRASH_START:
148      case SERVER_CRASH_SPLIT_META_LOGS:
149      case SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR:
150      case SERVER_CRASH_ASSIGN_META:
151        break;
152      default:
153        // If hbase:meta is not assigned, yield.
154        if (env.getAssignmentManager().waitMetaLoaded(this)) {
155          throw new ProcedureSuspendedException();
156        }
157    }
158    try {
159      switch (state) {
160        case SERVER_CRASH_START:
161          LOG.info("Start " + this);
162          // If carrying meta, process it first. Else, get list of regions on crashed server.
163          if (this.carryingMeta) {
164            setNextState(ServerCrashState.SERVER_CRASH_SPLIT_META_LOGS);
165          } else {
166            setNextState(ServerCrashState.SERVER_CRASH_GET_REGIONS);
167          }
168          break;
169        case SERVER_CRASH_SPLIT_META_LOGS:
170          if (
171            env.getMasterConfiguration().getBoolean(HBASE_SPLIT_WAL_COORDINATED_BY_ZK,
172              DEFAULT_HBASE_SPLIT_COORDINATED_BY_ZK)
173          ) {
174            zkCoordinatedSplitMetaLogs(env);
175            setNextState(ServerCrashState.SERVER_CRASH_ASSIGN_META);
176          } else {
177            am.getRegionStates().metaLogSplitting(serverName);
178            addChildProcedure(createSplittingWalProcedures(env, true));
179            setNextState(ServerCrashState.SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR);
180          }
181          break;
182        case SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR:
183          if (isSplittingDone(env, true)) {
184            setNextState(ServerCrashState.SERVER_CRASH_ASSIGN_META);
185            am.getRegionStates().metaLogSplit(serverName);
186          } else {
187            setNextState(ServerCrashState.SERVER_CRASH_SPLIT_META_LOGS);
188          }
189          break;
190        case SERVER_CRASH_ASSIGN_META:
191          assignRegions(env, Arrays.asList(RegionInfoBuilder.FIRST_META_REGIONINFO));
192          setNextState(ServerCrashState.SERVER_CRASH_GET_REGIONS);
193          break;
194        case SERVER_CRASH_GET_REGIONS:
195          this.regionsOnCrashedServer = getRegionsOnCrashedServer(env);
196          // Where to go next? Depends on whether we should split logs at all or
197          // if we should do distributed log splitting.
198          if (regionsOnCrashedServer != null) {
199            LOG.info("{} had {} regions", serverName, regionsOnCrashedServer.size());
200            if (LOG.isTraceEnabled()) {
201              this.regionsOnCrashedServer.stream().forEach(ri -> LOG.trace(ri.getShortNameToLog()));
202            }
203          }
204          if (!this.shouldSplitWal) {
205            setNextState(ServerCrashState.SERVER_CRASH_ASSIGN);
206          } else {
207            setNextState(ServerCrashState.SERVER_CRASH_SPLIT_LOGS);
208          }
209          break;
210        case SERVER_CRASH_SPLIT_LOGS:
211          if (
212            env.getMasterConfiguration().getBoolean(HBASE_SPLIT_WAL_COORDINATED_BY_ZK,
213              DEFAULT_HBASE_SPLIT_COORDINATED_BY_ZK)
214          ) {
215            zkCoordinatedSplitLogs(env);
216            setNextState(ServerCrashState.SERVER_CRASH_ASSIGN);
217          } else {
218            am.getRegionStates().logSplitting(this.serverName);
219            addChildProcedure(createSplittingWalProcedures(env, false));
220            setNextState(ServerCrashState.SERVER_CRASH_DELETE_SPLIT_WALS_DIR);
221          }
222          break;
223        case SERVER_CRASH_DELETE_SPLIT_WALS_DIR:
224          if (isSplittingDone(env, false)) {
225            cleanupSplitDir(env);
226            setNextState(ServerCrashState.SERVER_CRASH_ASSIGN);
227            am.getRegionStates().logSplit(this.serverName);
228          } else {
229            setNextState(ServerCrashState.SERVER_CRASH_SPLIT_LOGS);
230          }
231          break;
232        case SERVER_CRASH_ASSIGN:
233          // If no regions to assign, skip assign and skip to the finish.
234          // Filter out meta regions. Those are handled elsewhere in this procedure.
235          // Filter changes this.regionsOnCrashedServer.
236          if (filterDefaultMetaRegions()) {
237            if (LOG.isTraceEnabled()) {
238              LOG.trace("Assigning regions " + RegionInfo.getShortNameToLog(regionsOnCrashedServer)
239                + ", " + this + "; cycles=" + getCycles());
240            }
241            assignRegions(env, regionsOnCrashedServer);
242          }
243          setNextState(ServerCrashState.SERVER_CRASH_CLAIM_REPLICATION_QUEUES);
244          break;
245        case SERVER_CRASH_HANDLE_RIT2:
246          // Noop. Left in place because we used to call handleRIT here for a second time
247          // but no longer necessary since HBASE-20634.
248          setNextState(ServerCrashState.SERVER_CRASH_CLAIM_REPLICATION_QUEUES);
249          break;
250        case SERVER_CRASH_CLAIM_REPLICATION_QUEUES:
251          addChildProcedure(new ClaimReplicationQueuesProcedure(serverName));
252          setNextState(ServerCrashState.SERVER_CRASH_FINISH);
253          break;
254        case SERVER_CRASH_FINISH:
255          LOG.info("removed crashed server {} after splitting done", serverName);
256          services.getAssignmentManager().getRegionStates().removeServer(serverName);
257          updateProgress(true);
258          return Flow.NO_MORE_STATE;
259        default:
260          throw new UnsupportedOperationException("unhandled state=" + state);
261      }
262    } catch (IOException e) {
263      LOG.warn("Failed state=" + state + ", retry " + this + "; cycles=" + getCycles(), e);
264    }
265    return Flow.HAS_MORE_STATE;
266  }
267
268  /** Returns List of Regions on crashed server. */
269  List<RegionInfo> getRegionsOnCrashedServer(MasterProcedureEnv env) {
270    return env.getMasterServices().getAssignmentManager().getRegionsOnServer(serverName);
271  }
272
273  private void cleanupSplitDir(MasterProcedureEnv env) {
274    SplitWALManager splitWALManager = env.getMasterServices().getSplitWALManager();
275    try {
276      if (!this.carryingMeta) {
277        // If we are NOT carrying hbase:meta, check if any left-over hbase:meta WAL files from an
278        // old hbase:meta tenancy on this server; clean these up if any before trying to remove the
279        // WAL directory of this server or we will fail. See archiveMetaLog comment for more details
280        // on this condition.
281        env.getMasterServices().getMasterWalManager().archiveMetaLog(this.serverName);
282      }
283      splitWALManager.deleteWALDir(serverName);
284    } catch (IOException e) {
285      LOG.info("Remove WAL directory for {} failed, ignore...{}", serverName, e.getMessage());
286    }
287  }
288
289  private boolean isSplittingDone(MasterProcedureEnv env, boolean splitMeta) {
290    SplitWALManager splitWALManager = env.getMasterServices().getSplitWALManager();
291    try {
292      int wals = splitWALManager.getWALsToSplit(serverName, splitMeta).size();
293      LOG.debug("Check if {} WAL splitting is done? wals={}, meta={}", serverName, wals, splitMeta);
294      return wals == 0;
295    } catch (IOException e) {
296      LOG.warn("Get WALs of {} failed, retry...", serverName, e);
297      return false;
298    }
299  }
300
301  private Procedure[] createSplittingWalProcedures(MasterProcedureEnv env, boolean splitMeta)
302    throws IOException {
303    LOG.info("Splitting WALs {}, isMeta: {}", this, splitMeta);
304    SplitWALManager splitWALManager = env.getMasterServices().getSplitWALManager();
305    List<Procedure> procedures = splitWALManager.splitWALs(serverName, splitMeta);
306    return procedures.toArray(new Procedure[procedures.size()]);
307  }
308
309  private boolean filterDefaultMetaRegions() {
310    if (regionsOnCrashedServer == null) {
311      return false;
312    }
313    regionsOnCrashedServer.removeIf(this::isDefaultMetaRegion);
314    return !regionsOnCrashedServer.isEmpty();
315  }
316
317  private boolean isDefaultMetaRegion(RegionInfo hri) {
318    return hri.isMetaRegion() && RegionReplicaUtil.isDefaultReplica(hri);
319  }
320
321  /**
322   * Split hbase:meta logs using 'classic' zk-based coordination. Superceded by procedure-based WAL
323   * splitting.
324   * @see #createSplittingWalProcedures(MasterProcedureEnv, boolean)
325   */
326  private void zkCoordinatedSplitMetaLogs(MasterProcedureEnv env) throws IOException {
327    LOG.debug("Splitting meta WALs {}", this);
328    MasterWalManager mwm = env.getMasterServices().getMasterWalManager();
329    AssignmentManager am = env.getMasterServices().getAssignmentManager();
330    am.getRegionStates().metaLogSplitting(serverName);
331    mwm.splitMetaLog(serverName);
332    am.getRegionStates().metaLogSplit(serverName);
333    LOG.debug("Done splitting meta WALs {}", this);
334  }
335
336  /**
337   * Split logs using 'classic' zk-based coordination. Superceded by procedure-based WAL splitting.
338   * @see #createSplittingWalProcedures(MasterProcedureEnv, boolean)
339   */
340  private void zkCoordinatedSplitLogs(final MasterProcedureEnv env) throws IOException {
341    LOG.debug("Splitting WALs {}", this);
342    MasterWalManager mwm = env.getMasterServices().getMasterWalManager();
343    AssignmentManager am = env.getMasterServices().getAssignmentManager();
344    // TODO: For Matteo. Below BLOCKs!!!! Redo so can relinquish executor while it is running.
345    // PROBLEM!!! WE BLOCK HERE. Can block for hours if hundreds of WALs to split and hundreds
346    // of SCPs running because big cluster crashed down.
347    am.getRegionStates().logSplitting(this.serverName);
348    mwm.splitLog(this.serverName);
349    if (!carryingMeta) {
350      mwm.archiveMetaLog(this.serverName);
351    }
352    am.getRegionStates().logSplit(this.serverName);
353    LOG.debug("Done splitting WALs {}", this);
354  }
355
356  void updateProgress(boolean updateState) {
357    String msg = "Processing ServerCrashProcedure of " + serverName;
358    if (status == null) {
359      status = TaskMonitor.get().createStatus(msg);
360      return;
361    }
362    if (currentRunningState == ServerCrashState.SERVER_CRASH_FINISH) {
363      status.markComplete(msg + " done");
364      return;
365    }
366    if (updateState) {
367      currentRunningState = getCurrentState();
368    }
369    int childrenLatch = getChildrenLatch();
370    status.setStatus(msg + " current State " + currentRunningState
371      + (childrenLatch > 0
372        ? "; remaining num of running child procedures = " + childrenLatch
373        : ""));
374  }
375
376  @Override
377  protected void rollbackState(MasterProcedureEnv env, ServerCrashState state) throws IOException {
378    // Can't rollback.
379    throw new UnsupportedOperationException("unhandled state=" + state);
380  }
381
382  @Override
383  protected ServerCrashState getState(int stateId) {
384    return ServerCrashState.forNumber(stateId);
385  }
386
387  @Override
388  protected int getStateId(ServerCrashState state) {
389    return state.getNumber();
390  }
391
392  @Override
393  protected ServerCrashState getInitialState() {
394    return ServerCrashState.SERVER_CRASH_START;
395  }
396
397  @Override
398  protected boolean abort(MasterProcedureEnv env) {
399    // TODO
400    return false;
401  }
402
403  @Override
404  protected LockState acquireLock(final MasterProcedureEnv env) {
405    if (env.getProcedureScheduler().waitServerExclusiveLock(this, getServerName())) {
406      return LockState.LOCK_EVENT_WAIT;
407    }
408    return LockState.LOCK_ACQUIRED;
409  }
410
411  @Override
412  protected void releaseLock(final MasterProcedureEnv env) {
413    env.getProcedureScheduler().wakeServerExclusiveLock(this, getServerName());
414  }
415
416  @Override
417  public void toStringClassDetails(StringBuilder sb) {
418    sb.append(getProcName());
419    sb.append(", splitWal=");
420    sb.append(shouldSplitWal);
421    sb.append(", meta=");
422    sb.append(carryingMeta);
423  }
424
425  @Override
426  public String getProcName() {
427    return getClass().getSimpleName() + " " + this.serverName;
428  }
429
430  @Override
431  protected void serializeStateData(ProcedureStateSerializer serializer) throws IOException {
432    super.serializeStateData(serializer);
433
434    MasterProcedureProtos.ServerCrashStateData.Builder state =
435      MasterProcedureProtos.ServerCrashStateData.newBuilder()
436        .setServerName(ProtobufUtil.toServerName(this.serverName))
437        .setCarryingMeta(this.carryingMeta).setShouldSplitWal(this.shouldSplitWal);
438    if (this.regionsOnCrashedServer != null && !this.regionsOnCrashedServer.isEmpty()) {
439      for (RegionInfo hri : this.regionsOnCrashedServer) {
440        state.addRegionsOnCrashedServer(ProtobufUtil.toRegionInfo(hri));
441      }
442    }
443    serializer.serialize(state.build());
444  }
445
446  @Override
447  protected void deserializeStateData(ProcedureStateSerializer serializer) throws IOException {
448    super.deserializeStateData(serializer);
449
450    MasterProcedureProtos.ServerCrashStateData state =
451      serializer.deserialize(MasterProcedureProtos.ServerCrashStateData.class);
452    this.serverName = ProtobufUtil.toServerName(state.getServerName());
453    this.carryingMeta = state.hasCarryingMeta() ? state.getCarryingMeta() : false;
454    // shouldSplitWAL has a default over in pb so this invocation will always work.
455    this.shouldSplitWal = state.getShouldSplitWal();
456    int size = state.getRegionsOnCrashedServerCount();
457    if (size > 0) {
458      this.regionsOnCrashedServer = new ArrayList<>(size);
459      for (org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfo ri : state
460        .getRegionsOnCrashedServerList()) {
461        this.regionsOnCrashedServer.add(ProtobufUtil.toRegionInfo(ri));
462      }
463    }
464    updateProgress(false);
465  }
466
467  @Override
468  public ServerName getServerName() {
469    return this.serverName;
470  }
471
472  @Override
473  public boolean hasMetaTableRegion() {
474    return this.carryingMeta;
475  }
476
477  @Override
478  public ServerOperationType getServerOperationType() {
479    return ServerOperationType.CRASH_HANDLER;
480  }
481
482  @Override
483  protected boolean shouldWaitClientAck(MasterProcedureEnv env) {
484    // The operation is triggered internally on the server
485    // the client does not know about this procedure.
486    return false;
487  }
488
489  /**
490   * Moved out here so can be overridden by the HBCK fix-up SCP to be less strict about what it will
491   * tolerate as a 'match'.
492   * @return True if the region location in <code>rsn</code> matches that of this crashed server.
493   */
494  protected boolean isMatchingRegionLocation(RegionStateNode rsn) {
495    return this.serverName.equals(rsn.getRegionLocation());
496  }
497
498  /**
499   * Assign the regions on the crashed RS to other Rses.
500   * <p/>
501   * In this method we will go through all the RegionStateNodes of the give regions to find out
502   * whether there is already an TRSP for the region, if so we interrupt it and let it retry on
503   * other server, otherwise we will schedule a TRSP to bring the region online.
504   * <p/>
505   * We will also check whether the table for a region is enabled, if not, we will skip assigning
506   * it.
507   */
508  private void assignRegions(MasterProcedureEnv env, List<RegionInfo> regions) throws IOException {
509    AssignmentManager am = env.getMasterServices().getAssignmentManager();
510    boolean retainAssignment = env.getMasterConfiguration().getBoolean(MASTER_SCP_RETAIN_ASSIGNMENT,
511      DEFAULT_MASTER_SCP_RETAIN_ASSIGNMENT);
512    for (RegionInfo region : regions) {
513      RegionStateNode regionNode = am.getRegionStates().getOrCreateRegionStateNode(region);
514      regionNode.lock();
515      try {
516        // This is possible, as when a server is dead, TRSP will fail to schedule a RemoteProcedure
517        // and then try to assign the region to a new RS. And before it has updated the region
518        // location to the new RS, we may have already called the am.getRegionsOnServer so we will
519        // consider the region is still on this crashed server. Then before we arrive here, the
520        // TRSP could have updated the region location, or even finished itself, so the region is
521        // no longer on this crashed server any more. We should not try to assign it again. Please
522        // see HBASE-23594 for more details.
523        // UPDATE: HBCKServerCrashProcedure overrides isMatchingRegionLocation; this check can get
524        // in the way of our clearing out 'Unknown Servers'.
525        if (!isMatchingRegionLocation(regionNode)) {
526          // See HBASE-24117, though we have already changed the shutdown order, it is still worth
527          // double checking here to confirm that we do not skip assignment incorrectly.
528          if (!am.isRunning()) {
529            throw new DoNotRetryIOException(
530              "AssignmentManager has been stopped, can not process assignment any more");
531          }
532          LOG.info("{} found {} whose regionLocation no longer matches {}, skipping assign...",
533            this, regionNode, serverName);
534          continue;
535        }
536        if (regionNode.getProcedure() != null) {
537          LOG.info("{} found RIT {}; {}", this, regionNode.getProcedure(), regionNode);
538          regionNode.getProcedure().serverCrashed(env, regionNode, getServerName(),
539            !retainAssignment);
540          continue;
541        }
542        if (
543          env.getMasterServices().getTableStateManager().isTableState(regionNode.getTable(),
544            TableState.State.DISABLING)
545        ) {
546          // We need to change the state here otherwise the TRSP scheduled by DTP will try to
547          // close the region from a dead server and will never succeed. Please see HBASE-23636
548          // for more details.
549          env.getAssignmentManager().regionClosedAbnormally(regionNode);
550          LOG.info("{} found table disabling for region {}, set it state to ABNORMALLY_CLOSED.",
551            this, regionNode);
552          continue;
553        }
554        if (
555          env.getMasterServices().getTableStateManager().isTableState(regionNode.getTable(),
556            TableState.State.DISABLED)
557        ) {
558          // This should not happen, table disabled but has regions on server.
559          LOG.warn("Found table disabled for region {}, procDetails: {}", regionNode, this);
560          continue;
561        }
562        TransitRegionStateProcedure proc =
563          TransitRegionStateProcedure.assign(env, region, !retainAssignment, null);
564        regionNode.setProcedure(proc);
565        addChildProcedure(proc);
566      } finally {
567        regionNode.unlock();
568      }
569    }
570  }
571
572  @Override
573  protected ProcedureMetrics getProcedureMetrics(MasterProcedureEnv env) {
574    return env.getMasterServices().getMasterMetrics().getServerCrashProcMetrics();
575  }
576
577  @Override
578  protected boolean holdLock(MasterProcedureEnv env) {
579    return true;
580  }
581
582  public static void updateProgress(MasterProcedureEnv env, long parentId) {
583    if (parentId == NO_PROC_ID) {
584      return;
585    }
586    Procedure parentProcedure =
587      env.getMasterServices().getMasterProcedureExecutor().getProcedure(parentId);
588    if (parentProcedure != null && parentProcedure instanceof ServerCrashProcedure) {
589      ((ServerCrashProcedure) parentProcedure).updateProgress(false);
590    }
591  }
592}