001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.assignment;
019
020import edu.umd.cs.findbugs.annotations.NonNull;
021import java.io.IOException;
022import java.util.ArrayList;
023import java.util.Collection;
024import java.util.Collections;
025import java.util.HashMap;
026import java.util.HashSet;
027import java.util.List;
028import java.util.Map;
029import java.util.Set;
030import java.util.concurrent.Future;
031import java.util.concurrent.TimeUnit;
032import java.util.concurrent.atomic.AtomicBoolean;
033import java.util.concurrent.locks.Condition;
034import java.util.concurrent.locks.ReentrantLock;
035import java.util.stream.Collectors;
036import java.util.stream.Stream;
037import org.apache.hadoop.conf.Configuration;
038import org.apache.hadoop.hbase.CatalogFamilyFormat;
039import org.apache.hadoop.hbase.DoNotRetryIOException;
040import org.apache.hadoop.hbase.HBaseIOException;
041import org.apache.hadoop.hbase.HConstants;
042import org.apache.hadoop.hbase.PleaseHoldException;
043import org.apache.hadoop.hbase.ServerName;
044import org.apache.hadoop.hbase.TableName;
045import org.apache.hadoop.hbase.UnknownRegionException;
046import org.apache.hadoop.hbase.client.DoNotRetryRegionException;
047import org.apache.hadoop.hbase.client.MasterSwitchType;
048import org.apache.hadoop.hbase.client.RegionInfo;
049import org.apache.hadoop.hbase.client.RegionInfoBuilder;
050import org.apache.hadoop.hbase.client.RegionReplicaUtil;
051import org.apache.hadoop.hbase.client.RegionStatesCount;
052import org.apache.hadoop.hbase.client.Result;
053import org.apache.hadoop.hbase.client.ResultScanner;
054import org.apache.hadoop.hbase.client.Scan;
055import org.apache.hadoop.hbase.client.TableDescriptor;
056import org.apache.hadoop.hbase.client.TableState;
057import org.apache.hadoop.hbase.exceptions.UnexpectedStateException;
058import org.apache.hadoop.hbase.favored.FavoredNodesManager;
059import org.apache.hadoop.hbase.favored.FavoredNodesPromoter;
060import org.apache.hadoop.hbase.master.LoadBalancer;
061import org.apache.hadoop.hbase.master.MasterServices;
062import org.apache.hadoop.hbase.master.MetricsAssignmentManager;
063import org.apache.hadoop.hbase.master.RegionPlan;
064import org.apache.hadoop.hbase.master.RegionState;
065import org.apache.hadoop.hbase.master.RegionState.State;
066import org.apache.hadoop.hbase.master.ServerManager;
067import org.apache.hadoop.hbase.master.TableStateManager;
068import org.apache.hadoop.hbase.master.balancer.FavoredStochasticBalancer;
069import org.apache.hadoop.hbase.master.procedure.HBCKServerCrashProcedure;
070import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
071import org.apache.hadoop.hbase.master.procedure.MasterProcedureScheduler;
072import org.apache.hadoop.hbase.master.procedure.ProcedureSyncWait;
073import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure;
074import org.apache.hadoop.hbase.master.region.MasterRegion;
075import org.apache.hadoop.hbase.procedure2.Procedure;
076import org.apache.hadoop.hbase.procedure2.ProcedureEvent;
077import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
078import org.apache.hadoop.hbase.procedure2.ProcedureInMemoryChore;
079import org.apache.hadoop.hbase.procedure2.util.StringUtils;
080import org.apache.hadoop.hbase.regionserver.SequenceId;
081import org.apache.hadoop.hbase.rsgroup.RSGroupBasedLoadBalancer;
082import org.apache.hadoop.hbase.util.Bytes;
083import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
084import org.apache.hadoop.hbase.util.Pair;
085import org.apache.hadoop.hbase.util.Threads;
086import org.apache.hadoop.hbase.util.VersionInfo;
087import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
088import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
089import org.apache.yetus.audience.InterfaceAudience;
090import org.apache.zookeeper.KeeperException;
091import org.slf4j.Logger;
092import org.slf4j.LoggerFactory;
093
094import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
095import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition;
096import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode;
097import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionRequest;
098import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionResponse;
099
100/**
101 * The AssignmentManager is the coordinator for region assign/unassign operations.
102 * <ul>
103 * <li>In-memory states of regions and servers are stored in {@link RegionStates}.</li>
104 * <li>hbase:meta state updates are handled by {@link RegionStateStore}.</li>
105 * </ul>
106 * Regions are created by CreateTable, Split, Merge. Regions are deleted by DeleteTable, Split,
107 * Merge. Assigns are triggered by CreateTable, EnableTable, Split, Merge, ServerCrash. Unassigns
108 * are triggered by DisableTable, Split, Merge
109 */
110@InterfaceAudience.Private
111public class AssignmentManager {
112  private static final Logger LOG = LoggerFactory.getLogger(AssignmentManager.class);
113
114  // TODO: AMv2
115  // - handle region migration from hbase1 to hbase2.
116  // - handle sys table assignment first (e.g. acl, namespace)
117  // - handle table priorities
118  // - If ServerBusyException trying to update hbase:meta, we abort the Master
119  // See updateRegionLocation in RegionStateStore.
120  //
121  // See also
122  // https://docs.google.com/document/d/1eVKa7FHdeoJ1-9o8yZcOTAQbv0u0bblBlCCzVSIn69g/edit#heading=h.ystjyrkbtoq5
123  // for other TODOs.
124
125  public static final String BOOTSTRAP_THREAD_POOL_SIZE_CONF_KEY =
126    "hbase.assignment.bootstrap.thread.pool.size";
127
128  public static final String ASSIGN_DISPATCH_WAIT_MSEC_CONF_KEY =
129    "hbase.assignment.dispatch.wait.msec";
130  private static final int DEFAULT_ASSIGN_DISPATCH_WAIT_MSEC = 150;
131
132  public static final String ASSIGN_DISPATCH_WAITQ_MAX_CONF_KEY =
133    "hbase.assignment.dispatch.wait.queue.max.size";
134  private static final int DEFAULT_ASSIGN_DISPATCH_WAITQ_MAX = 100;
135
136  public static final String RIT_CHORE_INTERVAL_MSEC_CONF_KEY =
137    "hbase.assignment.rit.chore.interval.msec";
138  private static final int DEFAULT_RIT_CHORE_INTERVAL_MSEC = 60 * 1000;
139
140  public static final String DEAD_REGION_METRIC_CHORE_INTERVAL_MSEC_CONF_KEY =
141    "hbase.assignment.dead.region.metric.chore.interval.msec";
142  private static final int DEFAULT_DEAD_REGION_METRIC_CHORE_INTERVAL_MSEC = 120 * 1000;
143
144  public static final String ASSIGN_MAX_ATTEMPTS = "hbase.assignment.maximum.attempts";
145  private static final int DEFAULT_ASSIGN_MAX_ATTEMPTS = Integer.MAX_VALUE;
146
147  public static final String ASSIGN_RETRY_IMMEDIATELY_MAX_ATTEMPTS =
148    "hbase.assignment.retry.immediately.maximum.attempts";
149  private static final int DEFAULT_ASSIGN_RETRY_IMMEDIATELY_MAX_ATTEMPTS = 3;
150
151  /** Region in Transition metrics threshold time */
152  public static final String METRICS_RIT_STUCK_WARNING_THRESHOLD =
153    "hbase.metrics.rit.stuck.warning.threshold";
154  private static final int DEFAULT_RIT_STUCK_WARNING_THRESHOLD = 60 * 1000;
155  public static final String UNEXPECTED_STATE_REGION = "Unexpected state for ";
156
157  public static final String FORCE_REGION_RETAINMENT = "hbase.master.scp.retain.assignment.force";
158
159  public static final boolean DEFAULT_FORCE_REGION_RETAINMENT = false;
160
161  /** The wait time in millis before checking again if the region's previous RS is back online */
162  public static final String FORCE_REGION_RETAINMENT_WAIT_INTERVAL =
163    "hbase.master.scp.retain.assignment.force.wait-interval";
164
165  public static final long DEFAULT_FORCE_REGION_RETAINMENT_WAIT_INTERVAL = 50;
166
167  /**
168   * The number of times to check if the region's previous RS is back online, before giving up and
169   * proceeding with assignment on a new RS
170   */
171  public static final String FORCE_REGION_RETAINMENT_RETRIES =
172    "hbase.master.scp.retain.assignment.force.retries";
173
174  public static final int DEFAULT_FORCE_REGION_RETAINMENT_RETRIES = 600;
175
176  private final ProcedureEvent<?> metaAssignEvent = new ProcedureEvent<>("meta assign");
177  private final ProcedureEvent<?> metaLoadEvent = new ProcedureEvent<>("meta load");
178
179  private final MetricsAssignmentManager metrics;
180  private final RegionInTransitionChore ritChore;
181  private final DeadServerMetricRegionChore deadMetricChore;
182  private final MasterServices master;
183
184  private final AtomicBoolean running = new AtomicBoolean(false);
185  private final RegionStates regionStates = new RegionStates();
186  private final RegionStateStore regionStateStore;
187
188  /**
189   * When the operator uses this configuration option, any version between the current cluster
190   * version and the value of "hbase.min.version.move.system.tables" does not trigger any
191   * auto-region movement. Auto-region movement here refers to auto-migration of system table
192   * regions to newer server versions. It is assumed that the configured range of versions does not
193   * require special handling of moving system table regions to higher versioned RegionServer. This
194   * auto-migration is done by {@link #checkIfShouldMoveSystemRegionAsync()}. Example: Let's assume
195   * the cluster is on version 1.4.0 and we have set "hbase.min.version.move.system.tables" as
196   * "2.0.0". Now if we upgrade one RegionServer on 1.4.0 cluster to 1.6.0 (< 2.0.0), then
197   * AssignmentManager will not move hbase:meta, hbase:namespace and other system table regions to
198   * newly brought up RegionServer 1.6.0 as part of auto-migration. However, if we upgrade one
199   * RegionServer on 1.4.0 cluster to 2.2.0 (> 2.0.0), then AssignmentManager will move all system
200   * table regions to newly brought up RegionServer 2.2.0 as part of auto-migration done by
201   * {@link #checkIfShouldMoveSystemRegionAsync()}. "hbase.min.version.move.system.tables" is
202   * introduced as part of HBASE-22923.
203   */
204  private final String minVersionToMoveSysTables;
205
206  private static final String MIN_VERSION_MOVE_SYS_TABLES_CONFIG =
207    "hbase.min.version.move.system.tables";
208  private static final String DEFAULT_MIN_VERSION_MOVE_SYS_TABLES_CONFIG = "";
209
210  private final Map<ServerName, Set<byte[]>> rsReports = new HashMap<>();
211
212  private final boolean shouldAssignRegionsWithFavoredNodes;
213  private final int assignDispatchWaitQueueMaxSize;
214  private final int assignDispatchWaitMillis;
215  private final int assignMaxAttempts;
216  private final int assignRetryImmediatelyMaxAttempts;
217
218  private final MasterRegion masterRegion;
219
220  private final Object checkIfShouldMoveSystemRegionLock = new Object();
221
222  private Thread assignThread;
223
224  private final boolean forceRegionRetainment;
225
226  private final long forceRegionRetainmentWaitInterval;
227
228  private final int forceRegionRetainmentRetries;
229
230  public AssignmentManager(MasterServices master, MasterRegion masterRegion) {
231    this(master, masterRegion, new RegionStateStore(master, masterRegion));
232  }
233
234  AssignmentManager(MasterServices master, MasterRegion masterRegion, RegionStateStore stateStore) {
235    this.master = master;
236    this.regionStateStore = stateStore;
237    this.metrics = new MetricsAssignmentManager();
238    this.masterRegion = masterRegion;
239
240    final Configuration conf = master.getConfiguration();
241
242    // Only read favored nodes if using the favored nodes load balancer.
243    this.shouldAssignRegionsWithFavoredNodes = FavoredStochasticBalancer.class
244      .isAssignableFrom(conf.getClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, Object.class));
245
246    this.assignDispatchWaitMillis =
247      conf.getInt(ASSIGN_DISPATCH_WAIT_MSEC_CONF_KEY, DEFAULT_ASSIGN_DISPATCH_WAIT_MSEC);
248    this.assignDispatchWaitQueueMaxSize =
249      conf.getInt(ASSIGN_DISPATCH_WAITQ_MAX_CONF_KEY, DEFAULT_ASSIGN_DISPATCH_WAITQ_MAX);
250
251    this.assignMaxAttempts =
252      Math.max(1, conf.getInt(ASSIGN_MAX_ATTEMPTS, DEFAULT_ASSIGN_MAX_ATTEMPTS));
253    this.assignRetryImmediatelyMaxAttempts = conf.getInt(ASSIGN_RETRY_IMMEDIATELY_MAX_ATTEMPTS,
254      DEFAULT_ASSIGN_RETRY_IMMEDIATELY_MAX_ATTEMPTS);
255
256    int ritChoreInterval =
257      conf.getInt(RIT_CHORE_INTERVAL_MSEC_CONF_KEY, DEFAULT_RIT_CHORE_INTERVAL_MSEC);
258    this.ritChore = new RegionInTransitionChore(ritChoreInterval);
259
260    int deadRegionChoreInterval = conf.getInt(DEAD_REGION_METRIC_CHORE_INTERVAL_MSEC_CONF_KEY,
261      DEFAULT_DEAD_REGION_METRIC_CHORE_INTERVAL_MSEC);
262    if (deadRegionChoreInterval > 0) {
263      this.deadMetricChore = new DeadServerMetricRegionChore(deadRegionChoreInterval);
264    } else {
265      this.deadMetricChore = null;
266    }
267    minVersionToMoveSysTables =
268      conf.get(MIN_VERSION_MOVE_SYS_TABLES_CONFIG, DEFAULT_MIN_VERSION_MOVE_SYS_TABLES_CONFIG);
269
270    forceRegionRetainment =
271      conf.getBoolean(FORCE_REGION_RETAINMENT, DEFAULT_FORCE_REGION_RETAINMENT);
272    forceRegionRetainmentWaitInterval = conf.getLong(FORCE_REGION_RETAINMENT_WAIT_INTERVAL,
273      DEFAULT_FORCE_REGION_RETAINMENT_WAIT_INTERVAL);
274    forceRegionRetainmentRetries =
275      conf.getInt(FORCE_REGION_RETAINMENT_RETRIES, DEFAULT_FORCE_REGION_RETAINMENT_RETRIES);
276  }
277
278  private void mirrorMetaLocations() throws IOException, KeeperException {
279    // For compatibility, mirror the meta region state to zookeeper
280    // And we still need to use zookeeper to publish the meta region locations to region
281    // server, so they can serve as ClientMetaService
282    ZKWatcher zk = master.getZooKeeper();
283    if (zk == null || !zk.getRecoverableZooKeeper().getState().isAlive()) {
284      // this is possible in tests, we do not provide a zk watcher or the zk watcher has been closed
285      return;
286    }
287    Collection<RegionStateNode> metaStates = regionStates.getRegionStateNodes();
288    for (RegionStateNode metaState : metaStates) {
289      MetaTableLocator.setMetaLocation(zk, metaState.getRegionLocation(),
290        metaState.getRegionInfo().getReplicaId(), metaState.getState());
291    }
292    int replicaCount = metaStates.size();
293    // remove extra mirror locations
294    for (String znode : zk.getMetaReplicaNodes()) {
295      int replicaId = zk.getZNodePaths().getMetaReplicaIdFromZNode(znode);
296      if (replicaId >= replicaCount) {
297        MetaTableLocator.deleteMetaLocation(zk, replicaId);
298      }
299    }
300  }
301
302  public void start() throws IOException, KeeperException {
303    if (!running.compareAndSet(false, true)) {
304      return;
305    }
306
307    LOG.trace("Starting assignment manager");
308
309    // Start the Assignment Thread
310    startAssignmentThread();
311    // load meta region states.
312    // here we are still in the early steps of active master startup. There is only one thread(us)
313    // can access AssignmentManager and create region node, so here we do not need to lock the
314    // region node.
315    try (ResultScanner scanner =
316      masterRegion.getScanner(new Scan().addFamily(HConstants.CATALOG_FAMILY))) {
317      for (;;) {
318        Result result = scanner.next();
319        if (result == null) {
320          break;
321        }
322        RegionStateStore
323          .visitMetaEntry((r, regionInfo, state, regionLocation, lastHost, openSeqNum) -> {
324            RegionStateNode regionNode = regionStates.getOrCreateRegionStateNode(regionInfo);
325            regionNode.setState(state);
326            regionNode.setLastHost(lastHost);
327            regionNode.setRegionLocation(regionLocation);
328            regionNode.setOpenSeqNum(openSeqNum);
329            if (regionNode.getProcedure() != null) {
330              regionNode.getProcedure().stateLoaded(this, regionNode);
331            }
332            if (regionLocation != null) {
333              regionStates.addRegionToServer(regionNode);
334            }
335            if (RegionReplicaUtil.isDefaultReplica(regionInfo.getReplicaId())) {
336              setMetaAssigned(regionInfo, state == State.OPEN);
337            }
338            LOG.debug("Loaded hbase:meta {}", regionNode);
339          }, result);
340      }
341    }
342    mirrorMetaLocations();
343  }
344
345  /**
346   * Create RegionStateNode based on the TRSP list, and attach the TRSP to the RegionStateNode.
347   * <p>
348   * This is used to restore the RIT region list, so we do not need to restore it in the loadingMeta
349   * method below. And it is also very important as now before submitting a TRSP, we need to attach
350   * it to the RegionStateNode, which acts like a guard, so we need to restore this information at
351   * the very beginning, before we start processing any procedures.
352   */
353  public void setupRIT(List<TransitRegionStateProcedure> procs) {
354    procs.forEach(proc -> {
355      RegionInfo regionInfo = proc.getRegion();
356      RegionStateNode regionNode = regionStates.getOrCreateRegionStateNode(regionInfo);
357      TransitRegionStateProcedure existingProc = regionNode.getProcedure();
358      if (existingProc != null) {
359        // This is possible, as we will detach the procedure from the RSN before we
360        // actually finish the procedure. This is because that, we will detach the TRSP from the RSN
361        // during execution, at that time, the procedure has not been marked as done in the pv2
362        // framework yet, so it is possible that we schedule a new TRSP immediately and when
363        // arriving here, we will find out that there are multiple TRSPs for the region. But we can
364        // make sure that, only the last one can take the charge, the previous ones should have all
365        // been finished already. So here we will compare the proc id, the greater one will win.
366        if (existingProc.getProcId() < proc.getProcId()) {
367          // the new one wins, unset and set it to the new one below
368          regionNode.unsetProcedure(existingProc);
369        } else {
370          // the old one wins, skip
371          return;
372        }
373      }
374      LOG.info("Attach {} to {} to restore RIT", proc, regionNode);
375      regionNode.setProcedure(proc);
376    });
377  }
378
379  public void stop() {
380    if (!running.compareAndSet(true, false)) {
381      return;
382    }
383
384    LOG.info("Stopping assignment manager");
385
386    // The AM is started before the procedure executor,
387    // but the actual work will be loaded/submitted only once we have the executor
388    final boolean hasProcExecutor = master.getMasterProcedureExecutor() != null;
389
390    // Remove the RIT chore
391    if (hasProcExecutor) {
392      master.getMasterProcedureExecutor().removeChore(this.ritChore);
393      if (this.deadMetricChore != null) {
394        master.getMasterProcedureExecutor().removeChore(this.deadMetricChore);
395      }
396    }
397
398    // Stop the Assignment Thread
399    stopAssignmentThread();
400
401    // Stop the RegionStateStore
402    regionStates.clear();
403
404    // Update meta events (for testing)
405    if (hasProcExecutor) {
406      metaLoadEvent.suspend();
407      for (RegionInfo hri : getMetaRegionSet()) {
408        setMetaAssigned(hri, false);
409      }
410    }
411  }
412
413  public boolean isRunning() {
414    return running.get();
415  }
416
417  public Configuration getConfiguration() {
418    return master.getConfiguration();
419  }
420
421  public MetricsAssignmentManager getAssignmentManagerMetrics() {
422    return metrics;
423  }
424
425  private LoadBalancer getBalancer() {
426    return master.getLoadBalancer();
427  }
428
429  private FavoredNodesPromoter getFavoredNodePromoter() {
430    return (FavoredNodesPromoter) ((RSGroupBasedLoadBalancer) master.getLoadBalancer())
431      .getInternalBalancer();
432  }
433
434  private MasterProcedureEnv getProcedureEnvironment() {
435    return master.getMasterProcedureExecutor().getEnvironment();
436  }
437
438  private MasterProcedureScheduler getProcedureScheduler() {
439    return getProcedureEnvironment().getProcedureScheduler();
440  }
441
442  int getAssignMaxAttempts() {
443    return assignMaxAttempts;
444  }
445
446  public boolean isForceRegionRetainment() {
447    return forceRegionRetainment;
448  }
449
450  public long getForceRegionRetainmentWaitInterval() {
451    return forceRegionRetainmentWaitInterval;
452  }
453
454  public int getForceRegionRetainmentRetries() {
455    return forceRegionRetainmentRetries;
456  }
457
458  int getAssignRetryImmediatelyMaxAttempts() {
459    return assignRetryImmediatelyMaxAttempts;
460  }
461
462  public RegionStates getRegionStates() {
463    return regionStates;
464  }
465
466  /**
467   * Returns the regions hosted by the specified server.
468   * <p/>
469   * Notice that, for SCP, after we submit the SCP, no one can change the region list for the
470   * ServerStateNode so we do not need any locks here. And for other usage, this can only give you a
471   * snapshot of the current region list for this server, which means, right after you get the
472   * region list, new regions may be moved to this server or some regions may be moved out from this
473   * server, so you should not use it critically if you need strong consistency.
474   */
475  public List<RegionInfo> getRegionsOnServer(ServerName serverName) {
476    ServerStateNode serverInfo = regionStates.getServerNode(serverName);
477    if (serverInfo == null) {
478      return Collections.emptyList();
479    }
480    return serverInfo.getRegionInfoList();
481  }
482
483  private RegionInfo getRegionInfo(RegionStateNode rsn) {
484    if (rsn.isSplit() && !rsn.getRegionInfo().isSplit()) {
485      // see the comments in markRegionAsSplit on why we need to do this converting.
486      return RegionInfoBuilder.newBuilder(rsn.getRegionInfo()).setSplit(true).setOffline(true)
487        .build();
488    } else {
489      return rsn.getRegionInfo();
490    }
491  }
492
493  private Stream<RegionStateNode> getRegionStateNodes(TableName tableName,
494    boolean excludeOfflinedSplitParents) {
495    Stream<RegionStateNode> stream = regionStates.getTableRegionStateNodes(tableName).stream();
496    if (excludeOfflinedSplitParents) {
497      return stream.filter(rsn -> !rsn.isSplit());
498    } else {
499      return stream;
500    }
501  }
502
503  public List<RegionInfo> getTableRegions(TableName tableName,
504    boolean excludeOfflinedSplitParents) {
505    return getRegionStateNodes(tableName, excludeOfflinedSplitParents).map(this::getRegionInfo)
506      .collect(Collectors.toList());
507  }
508
509  public List<Pair<RegionInfo, ServerName>> getTableRegionsAndLocations(TableName tableName,
510    boolean excludeOfflinedSplitParents) {
511    return getRegionStateNodes(tableName, excludeOfflinedSplitParents)
512      .map(rsn -> Pair.newPair(getRegionInfo(rsn), rsn.getRegionLocation()))
513      .collect(Collectors.toList());
514  }
515
516  public RegionStateStore getRegionStateStore() {
517    return regionStateStore;
518  }
519
520  public List<ServerName> getFavoredNodes(final RegionInfo regionInfo) {
521    return this.shouldAssignRegionsWithFavoredNodes
522      ? getFavoredNodePromoter().getFavoredNodes(regionInfo)
523      : ServerName.EMPTY_SERVER_LIST;
524  }
525
526  // ============================================================================================
527  // Table State Manager helpers
528  // ============================================================================================
529  private TableStateManager getTableStateManager() {
530    return master.getTableStateManager();
531  }
532
533  private boolean isTableEnabled(final TableName tableName) {
534    return getTableStateManager().isTableState(tableName, TableState.State.ENABLED);
535  }
536
537  private boolean isTableDisabled(final TableName tableName) {
538    return getTableStateManager().isTableState(tableName, TableState.State.DISABLED,
539      TableState.State.DISABLING);
540  }
541
542  // ============================================================================================
543  // META Helpers
544  // ============================================================================================
545  private boolean isMetaRegion(final RegionInfo regionInfo) {
546    return regionInfo.isMetaRegion();
547  }
548
549  public boolean isMetaRegion(final byte[] regionName) {
550    return getMetaRegionFromName(regionName) != null;
551  }
552
553  public RegionInfo getMetaRegionFromName(final byte[] regionName) {
554    for (RegionInfo hri : getMetaRegionSet()) {
555      if (Bytes.equals(hri.getRegionName(), regionName)) {
556        return hri;
557      }
558    }
559    return null;
560  }
561
562  public boolean isCarryingMeta(final ServerName serverName) {
563    // TODO: handle multiple meta
564    return isCarryingRegion(serverName, RegionInfoBuilder.FIRST_META_REGIONINFO);
565  }
566
567  private boolean isCarryingRegion(final ServerName serverName, final RegionInfo regionInfo) {
568    // TODO: check for state?
569    final RegionStateNode node = regionStates.getRegionStateNode(regionInfo);
570    return (node != null && serverName.equals(node.getRegionLocation()));
571  }
572
573  private RegionInfo getMetaForRegion(final RegionInfo regionInfo) {
574    // if (regionInfo.isMetaRegion()) return regionInfo;
575    // TODO: handle multiple meta. if the region provided is not meta lookup
576    // which meta the region belongs to.
577    return RegionInfoBuilder.FIRST_META_REGIONINFO;
578  }
579
580  // TODO: handle multiple meta.
581  private static final Set<RegionInfo> META_REGION_SET =
582    Collections.singleton(RegionInfoBuilder.FIRST_META_REGIONINFO);
583
584  public Set<RegionInfo> getMetaRegionSet() {
585    return META_REGION_SET;
586  }
587
588  // ============================================================================================
589  // META Event(s) helpers
590  // ============================================================================================
591  /**
592   * Notice that, this only means the meta region is available on a RS, but the AM may still be
593   * loading the region states from meta, so usually you need to check {@link #isMetaLoaded()} first
594   * before checking this method, unless you can make sure that your piece of code can only be
595   * executed after AM builds the region states.
596   * @see #isMetaLoaded()
597   */
598  public boolean isMetaAssigned() {
599    return metaAssignEvent.isReady();
600  }
601
602  public boolean isMetaRegionInTransition() {
603    return !isMetaAssigned();
604  }
605
606  /**
607   * Notice that this event does not mean the AM has already finished region state rebuilding. See
608   * the comment of {@link #isMetaAssigned()} for more details.
609   * @see #isMetaAssigned()
610   */
611  public boolean waitMetaAssigned(Procedure<?> proc, RegionInfo regionInfo) {
612    return getMetaAssignEvent(getMetaForRegion(regionInfo)).suspendIfNotReady(proc);
613  }
614
615  private void setMetaAssigned(RegionInfo metaRegionInfo, boolean assigned) {
616    assert isMetaRegion(metaRegionInfo) : "unexpected non-meta region " + metaRegionInfo;
617    ProcedureEvent<?> metaAssignEvent = getMetaAssignEvent(metaRegionInfo);
618    if (assigned) {
619      metaAssignEvent.wake(getProcedureScheduler());
620    } else {
621      metaAssignEvent.suspend();
622    }
623  }
624
625  private ProcedureEvent<?> getMetaAssignEvent(RegionInfo metaRegionInfo) {
626    assert isMetaRegion(metaRegionInfo) : "unexpected non-meta region " + metaRegionInfo;
627    // TODO: handle multiple meta.
628    return metaAssignEvent;
629  }
630
631  /**
632   * Wait until AM finishes the meta loading, i.e, the region states rebuilding.
633   * @see #isMetaLoaded()
634   * @see #waitMetaAssigned(Procedure, RegionInfo)
635   */
636  public boolean waitMetaLoaded(Procedure<?> proc) {
637    return metaLoadEvent.suspendIfNotReady(proc);
638  }
639
640  /**
641   * This method will be called in master initialization method after calling
642   * {@link #processOfflineRegions()}, as in processOfflineRegions we will generate assign
643   * procedures for offline regions, which may be conflict with creating table.
644   * <p/>
645   * This is a bit dirty, should be reconsidered after we decide whether to keep the
646   * {@link #processOfflineRegions()} method.
647   */
648  public void wakeMetaLoadedEvent() {
649    metaLoadEvent.wake(getProcedureScheduler());
650    assert isMetaLoaded() : "expected meta to be loaded";
651  }
652
653  /**
654   * Return whether AM finishes the meta loading, i.e, the region states rebuilding.
655   * @see #isMetaAssigned()
656   * @see #waitMetaLoaded(Procedure)
657   */
658  public boolean isMetaLoaded() {
659    return metaLoadEvent.isReady();
660  }
661
662  /**
663   * Start a new thread to check if there are region servers whose versions are higher than others.
664   * If so, move all system table regions to RS with the highest version to keep compatibility. The
665   * reason is, RS in new version may not be able to access RS in old version when there are some
666   * incompatible changes.
667   * <p>
668   * This method is called when a new RegionServer is added to cluster only.
669   * </p>
670   */
671  public void checkIfShouldMoveSystemRegionAsync() {
672    // TODO: Fix this thread. If a server is killed and a new one started, this thread thinks that
673    // it should 'move' the system tables from the old server to the new server but
674    // ServerCrashProcedure is on it; and it will take care of the assign without dataloss.
675    if (this.master.getServerManager().countOfRegionServers() <= 1) {
676      return;
677    }
678    // This thread used to run whenever there was a change in the cluster. The ZooKeeper
679    // childrenChanged notification came in before the nodeDeleted message and so this method
680    // cold run before a ServerCrashProcedure could run. That meant that this thread could see
681    // a Crashed Server before ServerCrashProcedure and it could find system regions on the
682    // crashed server and go move them before ServerCrashProcedure had a chance; could be
683    // dataloss too if WALs were not recovered.
684    new Thread(() -> {
685      try {
686        synchronized (checkIfShouldMoveSystemRegionLock) {
687          List<RegionPlan> plans = new ArrayList<>();
688          // TODO: I don't think this code does a good job if all servers in cluster have same
689          // version. It looks like it will schedule unnecessary moves.
690          for (ServerName server : getExcludedServersForSystemTable()) {
691            if (master.getServerManager().isServerDead(server)) {
692              // TODO: See HBASE-18494 and HBASE-18495. Though getExcludedServersForSystemTable()
693              // considers only online servers, the server could be queued for dead server
694              // processing. As region assignments for crashed server is handled by
695              // ServerCrashProcedure, do NOT handle them here. The goal is to handle this through
696              // regular flow of LoadBalancer as a favored node and not to have this special
697              // handling.
698              continue;
699            }
700            List<RegionInfo> regionsShouldMove = getSystemTables(server);
701            if (!regionsShouldMove.isEmpty()) {
702              for (RegionInfo regionInfo : regionsShouldMove) {
703                // null value for dest forces destination server to be selected by balancer
704                RegionPlan plan = new RegionPlan(regionInfo, server, null);
705                if (regionInfo.isMetaRegion()) {
706                  // Must move meta region first.
707                  LOG.info("Async MOVE of {} to newer Server={}", regionInfo.getEncodedName(),
708                    server);
709                  moveAsync(plan);
710                } else {
711                  plans.add(plan);
712                }
713              }
714            }
715            for (RegionPlan plan : plans) {
716              LOG.info("Async MOVE of {} to newer Server={}", plan.getRegionInfo().getEncodedName(),
717                server);
718              moveAsync(plan);
719            }
720          }
721        }
722      } catch (Throwable t) {
723        LOG.error(t.toString(), t);
724      }
725    }).start();
726  }
727
728  private List<RegionInfo> getSystemTables(ServerName serverName) {
729    ServerStateNode serverNode = regionStates.getServerNode(serverName);
730    if (serverNode == null) {
731      return Collections.emptyList();
732    }
733    return serverNode.getSystemRegionInfoList();
734  }
735
736  private void preTransitCheck(RegionStateNode regionNode, RegionState.State[] expectedStates)
737    throws HBaseIOException {
738    if (regionNode.getProcedure() != null) {
739      throw new HBaseIOException(
740        regionNode + " is currently in transition; pid=" + regionNode.getProcedure().getProcId());
741    }
742    if (!regionNode.isInState(expectedStates)) {
743      throw new DoNotRetryRegionException(UNEXPECTED_STATE_REGION + regionNode);
744    }
745    if (isTableDisabled(regionNode.getTable())) {
746      throw new DoNotRetryIOException(regionNode.getTable() + " is disabled for " + regionNode);
747    }
748  }
749
750  /**
751   * Create an assign TransitRegionStateProcedure. Makes sure of RegionState. Throws exception if
752   * not appropriate UNLESS override is set. Used by hbck2 but also by straightline
753   * {@link #assign(RegionInfo, ServerName)} and {@link #assignAsync(RegionInfo, ServerName)}.
754   * @see #createAssignProcedure(RegionStateNode, ServerName) for a version that does NO checking
755   *      used when only when no checking needed.
756   * @param override If false, check RegionState is appropriate for assign; if not throw exception.
757   */
758  private TransitRegionStateProcedure createAssignProcedure(RegionInfo regionInfo, ServerName sn,
759    boolean override) throws IOException {
760    RegionStateNode regionNode = regionStates.getOrCreateRegionStateNode(regionInfo);
761    regionNode.lock();
762    try {
763      if (override) {
764        if (regionNode.getProcedure() != null) {
765          regionNode.unsetProcedure(regionNode.getProcedure());
766        }
767      } else {
768        preTransitCheck(regionNode, STATES_EXPECTED_ON_ASSIGN);
769      }
770      assert regionNode.getProcedure() == null;
771      return regionNode.setProcedure(
772        TransitRegionStateProcedure.assign(getProcedureEnvironment(), regionInfo, sn));
773    } finally {
774      regionNode.unlock();
775    }
776  }
777
778  /**
779   * Create an assign TransitRegionStateProcedure. Does NO checking of RegionState. Presumes
780   * appriopriate state ripe for assign.
781   * @see #createAssignProcedure(RegionInfo, ServerName, boolean)
782   */
783  private TransitRegionStateProcedure createAssignProcedure(RegionStateNode regionNode,
784    ServerName targetServer) {
785    regionNode.lock();
786    try {
787      return regionNode.setProcedure(TransitRegionStateProcedure.assign(getProcedureEnvironment(),
788        regionNode.getRegionInfo(), targetServer));
789    } finally {
790      regionNode.unlock();
791    }
792  }
793
794  public long assign(RegionInfo regionInfo, ServerName sn) throws IOException {
795    TransitRegionStateProcedure proc = createAssignProcedure(regionInfo, sn, false);
796    ProcedureSyncWait.submitAndWaitProcedure(master.getMasterProcedureExecutor(), proc);
797    return proc.getProcId();
798  }
799
800  public long assign(RegionInfo regionInfo) throws IOException {
801    return assign(regionInfo, null);
802  }
803
804  /**
805   * Submits a procedure that assigns a region to a target server without waiting for it to finish
806   * @param regionInfo the region we would like to assign
807   * @param sn         target server name
808   */
809  public Future<byte[]> assignAsync(RegionInfo regionInfo, ServerName sn) throws IOException {
810    return ProcedureSyncWait.submitProcedure(master.getMasterProcedureExecutor(),
811      createAssignProcedure(regionInfo, sn, false));
812  }
813
814  /**
815   * Submits a procedure that assigns a region without waiting for it to finish
816   * @param regionInfo the region we would like to assign
817   */
818  public Future<byte[]> assignAsync(RegionInfo regionInfo) throws IOException {
819    return assignAsync(regionInfo, null);
820  }
821
822  public long unassign(RegionInfo regionInfo) throws IOException {
823    RegionStateNode regionNode = regionStates.getRegionStateNode(regionInfo);
824    if (regionNode == null) {
825      throw new UnknownRegionException("No RegionState found for " + regionInfo.getEncodedName());
826    }
827    TransitRegionStateProcedure proc;
828    regionNode.lock();
829    try {
830      preTransitCheck(regionNode, STATES_EXPECTED_ON_UNASSIGN_OR_MOVE);
831      proc = TransitRegionStateProcedure.unassign(getProcedureEnvironment(), regionInfo);
832      regionNode.setProcedure(proc);
833    } finally {
834      regionNode.unlock();
835    }
836    ProcedureSyncWait.submitAndWaitProcedure(master.getMasterProcedureExecutor(), proc);
837    return proc.getProcId();
838  }
839
840  public TransitRegionStateProcedure createMoveRegionProcedure(RegionInfo regionInfo,
841    ServerName targetServer) throws HBaseIOException {
842    RegionStateNode regionNode = this.regionStates.getRegionStateNode(regionInfo);
843    if (regionNode == null) {
844      throw new UnknownRegionException(
845        "No RegionStateNode found for " + regionInfo.getEncodedName() + "(Closed/Deleted?)");
846    }
847    TransitRegionStateProcedure proc;
848    regionNode.lock();
849    try {
850      preTransitCheck(regionNode, STATES_EXPECTED_ON_UNASSIGN_OR_MOVE);
851      regionNode.checkOnline();
852      proc = TransitRegionStateProcedure.move(getProcedureEnvironment(), regionInfo, targetServer);
853      regionNode.setProcedure(proc);
854    } finally {
855      regionNode.unlock();
856    }
857    return proc;
858  }
859
860  public void move(RegionInfo regionInfo) throws IOException {
861    TransitRegionStateProcedure proc = createMoveRegionProcedure(regionInfo, null);
862    ProcedureSyncWait.submitAndWaitProcedure(master.getMasterProcedureExecutor(), proc);
863  }
864
865  public Future<byte[]> moveAsync(RegionPlan regionPlan) throws HBaseIOException {
866    TransitRegionStateProcedure proc =
867      createMoveRegionProcedure(regionPlan.getRegionInfo(), regionPlan.getDestination());
868    return ProcedureSyncWait.submitProcedure(master.getMasterProcedureExecutor(), proc);
869  }
870
871  public Future<byte[]> balance(RegionPlan regionPlan) throws HBaseIOException {
872    ServerName current =
873      this.getRegionStates().getRegionAssignments().get(regionPlan.getRegionInfo());
874    if (current == null || !current.equals(regionPlan.getSource())) {
875      LOG.debug("Skip region plan {}, source server not match, current region location is {}",
876        regionPlan, current == null ? "(null)" : current);
877      return null;
878    }
879    return moveAsync(regionPlan);
880  }
881
882  // ============================================================================================
883  // RegionTransition procedures helpers
884  // ============================================================================================
885
886  /**
887   * Create round-robin assigns. Use on table creation to distribute out regions across cluster.
888   * @return AssignProcedures made out of the passed in <code>hris</code> and a call to the balancer
889   *         to populate the assigns with targets chosen using round-robin (default balancer
890   *         scheme). If at assign-time, the target chosen is no longer up, thats fine, the
891   *         AssignProcedure will ask the balancer for a new target, and so on.
892   */
893  public TransitRegionStateProcedure[] createRoundRobinAssignProcedures(List<RegionInfo> hris,
894    List<ServerName> serversToExclude) {
895    if (hris.isEmpty()) {
896      return new TransitRegionStateProcedure[0];
897    }
898
899    if (
900      serversToExclude != null && this.master.getServerManager().getOnlineServersList().size() == 1
901    ) {
902      LOG.debug("Only one region server found and hence going ahead with the assignment");
903      serversToExclude = null;
904    }
905    try {
906      // Ask the balancer to assign our regions. Pass the regions en masse. The balancer can do
907      // a better job if it has all the assignments in the one lump.
908      Map<ServerName, List<RegionInfo>> assignments = getBalancer().roundRobinAssignment(hris,
909        this.master.getServerManager().createDestinationServersList(serversToExclude));
910      // Return mid-method!
911      return createAssignProcedures(assignments);
912    } catch (IOException hioe) {
913      LOG.warn("Failed roundRobinAssignment", hioe);
914    }
915    // If an error above, fall-through to this simpler assign. Last resort.
916    return createAssignProcedures(hris);
917  }
918
919  /**
920   * Create round-robin assigns. Use on table creation to distribute out regions across cluster.
921   * @return AssignProcedures made out of the passed in <code>hris</code> and a call to the balancer
922   *         to populate the assigns with targets chosen using round-robin (default balancer
923   *         scheme). If at assign-time, the target chosen is no longer up, thats fine, the
924   *         AssignProcedure will ask the balancer for a new target, and so on.
925   */
926  public TransitRegionStateProcedure[] createRoundRobinAssignProcedures(List<RegionInfo> hris) {
927    return createRoundRobinAssignProcedures(hris, null);
928  }
929
930  static int compare(TransitRegionStateProcedure left, TransitRegionStateProcedure right) {
931    if (left.getRegion().isMetaRegion()) {
932      if (right.getRegion().isMetaRegion()) {
933        return RegionInfo.COMPARATOR.compare(left.getRegion(), right.getRegion());
934      }
935      return -1;
936    } else if (right.getRegion().isMetaRegion()) {
937      return +1;
938    }
939    if (left.getRegion().getTable().isSystemTable()) {
940      if (right.getRegion().getTable().isSystemTable()) {
941        return RegionInfo.COMPARATOR.compare(left.getRegion(), right.getRegion());
942      }
943      return -1;
944    } else if (right.getRegion().getTable().isSystemTable()) {
945      return +1;
946    }
947    return RegionInfo.COMPARATOR.compare(left.getRegion(), right.getRegion());
948  }
949
950  /**
951   * Create one TransitRegionStateProcedure to assign a region w/o specifying a target server. This
952   * method is called from HBCK2.
953   * @return an assign or null
954   */
955  public TransitRegionStateProcedure createOneAssignProcedure(RegionInfo ri, boolean override) {
956    TransitRegionStateProcedure trsp = null;
957    try {
958      trsp = createAssignProcedure(ri, null, override);
959    } catch (IOException ioe) {
960      LOG.info(
961        "Failed {} assign, override={}"
962          + (override ? "" : "; set override to by-pass state checks."),
963        ri.getEncodedName(), override, ioe);
964    }
965    return trsp;
966  }
967
968  /**
969   * Create one TransitRegionStateProcedure to unassign a region. This method is called from HBCK2.
970   * @return an unassign or null
971   */
972  public TransitRegionStateProcedure createOneUnassignProcedure(RegionInfo ri, boolean override) {
973    RegionStateNode regionNode = regionStates.getOrCreateRegionStateNode(ri);
974    TransitRegionStateProcedure trsp = null;
975    regionNode.lock();
976    try {
977      if (override) {
978        if (regionNode.getProcedure() != null) {
979          regionNode.unsetProcedure(regionNode.getProcedure());
980        }
981      } else {
982        // This is where we could throw an exception; i.e. override is false.
983        preTransitCheck(regionNode, STATES_EXPECTED_ON_UNASSIGN_OR_MOVE);
984      }
985      assert regionNode.getProcedure() == null;
986      trsp =
987        TransitRegionStateProcedure.unassign(getProcedureEnvironment(), regionNode.getRegionInfo());
988      regionNode.setProcedure(trsp);
989    } catch (IOException ioe) {
990      // 'override' must be false here.
991      LOG.info("Failed {} unassign, override=false; set override to by-pass state checks.",
992        ri.getEncodedName(), ioe);
993    } finally {
994      regionNode.unlock();
995    }
996    return trsp;
997  }
998
999  /**
1000   * Create an array of TransitRegionStateProcedure w/o specifying a target server. Used as fallback
1001   * of caller is unable to do {@link #createAssignProcedures(Map)}.
1002   * <p/>
1003   * If no target server, at assign time, we will try to use the former location of the region if
1004   * one exists. This is how we 'retain' the old location across a server restart.
1005   * <p/>
1006   * Should only be called when you can make sure that no one can touch these regions other than
1007   * you. For example, when you are creating or enabling table. Presumes all Regions are in
1008   * appropriate state ripe for assign; no checking of Region state is done in here.
1009   * @see #createAssignProcedures(Map)
1010   */
1011  public TransitRegionStateProcedure[] createAssignProcedures(List<RegionInfo> hris) {
1012    return hris.stream().map(hri -> regionStates.getOrCreateRegionStateNode(hri))
1013      .map(regionNode -> createAssignProcedure(regionNode, null)).sorted(AssignmentManager::compare)
1014      .toArray(TransitRegionStateProcedure[]::new);
1015  }
1016
1017  /**
1018   * Tied to {@link #createAssignProcedures(List)} in that it is called if caller is unable to run
1019   * this method. Presumes all Regions are in appropriate state ripe for assign; no checking of
1020   * Region state is done in here.
1021   * @param assignments Map of assignments from which we produce an array of AssignProcedures.
1022   * @return Assignments made from the passed in <code>assignments</code>
1023   * @see #createAssignProcedures(List)
1024   */
1025  private TransitRegionStateProcedure[]
1026    createAssignProcedures(Map<ServerName, List<RegionInfo>> assignments) {
1027    return assignments.entrySet().stream()
1028      .flatMap(e -> e.getValue().stream().map(hri -> regionStates.getOrCreateRegionStateNode(hri))
1029        .map(regionNode -> createAssignProcedure(regionNode, e.getKey())))
1030      .sorted(AssignmentManager::compare).toArray(TransitRegionStateProcedure[]::new);
1031  }
1032
1033  // for creating unassign TRSP when disabling a table or closing excess region replicas
1034  private TransitRegionStateProcedure forceCreateUnssignProcedure(RegionStateNode regionNode) {
1035    regionNode.lock();
1036    try {
1037      if (regionNode.isInState(State.OFFLINE, State.CLOSED, State.SPLIT)) {
1038        return null;
1039      }
1040      // in general, a split parent should be in CLOSED or SPLIT state, but anyway, let's check it
1041      // here for safety
1042      if (regionNode.getRegionInfo().isSplit()) {
1043        LOG.warn("{} is a split parent but not in CLOSED or SPLIT state", regionNode);
1044        return null;
1045      }
1046      // As in DisableTableProcedure or ModifyTableProcedure, we will hold the xlock for table, so
1047      // we can make sure that this procedure has not been executed yet, as TRSP will hold the
1048      // shared lock for table all the time. So here we will unset it and when it is actually
1049      // executed, it will find that the attach procedure is not itself and quit immediately.
1050      if (regionNode.getProcedure() != null) {
1051        regionNode.unsetProcedure(regionNode.getProcedure());
1052      }
1053      return regionNode.setProcedure(TransitRegionStateProcedure.unassign(getProcedureEnvironment(),
1054        regionNode.getRegionInfo()));
1055    } finally {
1056      regionNode.unlock();
1057    }
1058  }
1059
1060  /**
1061   * Called by DisableTableProcedure to unassign all the regions for a table.
1062   */
1063  public TransitRegionStateProcedure[] createUnassignProceduresForDisabling(TableName tableName) {
1064    return regionStates.getTableRegionStateNodes(tableName).stream()
1065      .map(this::forceCreateUnssignProcedure).filter(p -> p != null)
1066      .toArray(TransitRegionStateProcedure[]::new);
1067  }
1068
1069  /**
1070   * Called by ModifyTableProcedures to unassign all the excess region replicas for a table.
1071   */
1072  public TransitRegionStateProcedure[] createUnassignProceduresForClosingExcessRegionReplicas(
1073    TableName tableName, int newReplicaCount) {
1074    return regionStates.getTableRegionStateNodes(tableName).stream()
1075      .filter(regionNode -> regionNode.getRegionInfo().getReplicaId() >= newReplicaCount)
1076      .map(this::forceCreateUnssignProcedure).filter(p -> p != null)
1077      .toArray(TransitRegionStateProcedure[]::new);
1078  }
1079
1080  public SplitTableRegionProcedure createSplitProcedure(final RegionInfo regionToSplit,
1081    final byte[] splitKey) throws IOException {
1082    return new SplitTableRegionProcedure(getProcedureEnvironment(), regionToSplit, splitKey);
1083  }
1084
1085  public MergeTableRegionsProcedure createMergeProcedure(RegionInfo... ris) throws IOException {
1086    return new MergeTableRegionsProcedure(getProcedureEnvironment(), ris, false);
1087  }
1088
1089  /**
1090   * Delete the region states. This is called by "DeleteTable"
1091   */
1092  public void deleteTable(final TableName tableName) throws IOException {
1093    final ArrayList<RegionInfo> regions = regionStates.getTableRegionsInfo(tableName);
1094    regionStateStore.deleteRegions(regions);
1095    for (int i = 0; i < regions.size(); ++i) {
1096      final RegionInfo regionInfo = regions.get(i);
1097      regionStates.deleteRegion(regionInfo);
1098    }
1099  }
1100
1101  // ============================================================================================
1102  // RS Region Transition Report helpers
1103  // ============================================================================================
1104  private void reportRegionStateTransition(ReportRegionStateTransitionResponse.Builder builder,
1105    ServerName serverName, List<RegionStateTransition> transitionList) throws IOException {
1106    for (RegionStateTransition transition : transitionList) {
1107      switch (transition.getTransitionCode()) {
1108        case OPENED:
1109        case FAILED_OPEN:
1110        case CLOSED:
1111          assert transition.getRegionInfoCount() == 1 : transition;
1112          final RegionInfo hri = ProtobufUtil.toRegionInfo(transition.getRegionInfo(0));
1113          long procId =
1114            transition.getProcIdCount() > 0 ? transition.getProcId(0) : Procedure.NO_PROC_ID;
1115          updateRegionTransition(serverName, transition.getTransitionCode(), hri,
1116            transition.hasOpenSeqNum() ? transition.getOpenSeqNum() : HConstants.NO_SEQNUM, procId);
1117          break;
1118        case READY_TO_SPLIT:
1119        case SPLIT:
1120        case SPLIT_REVERTED:
1121          assert transition.getRegionInfoCount() == 3 : transition;
1122          final RegionInfo parent = ProtobufUtil.toRegionInfo(transition.getRegionInfo(0));
1123          final RegionInfo splitA = ProtobufUtil.toRegionInfo(transition.getRegionInfo(1));
1124          final RegionInfo splitB = ProtobufUtil.toRegionInfo(transition.getRegionInfo(2));
1125          updateRegionSplitTransition(serverName, transition.getTransitionCode(), parent, splitA,
1126            splitB);
1127          break;
1128        case READY_TO_MERGE:
1129        case MERGED:
1130        case MERGE_REVERTED:
1131          assert transition.getRegionInfoCount() == 3 : transition;
1132          final RegionInfo merged = ProtobufUtil.toRegionInfo(transition.getRegionInfo(0));
1133          final RegionInfo mergeA = ProtobufUtil.toRegionInfo(transition.getRegionInfo(1));
1134          final RegionInfo mergeB = ProtobufUtil.toRegionInfo(transition.getRegionInfo(2));
1135          updateRegionMergeTransition(serverName, transition.getTransitionCode(), merged, mergeA,
1136            mergeB);
1137          break;
1138      }
1139    }
1140  }
1141
1142  public ReportRegionStateTransitionResponse reportRegionStateTransition(
1143    final ReportRegionStateTransitionRequest req) throws PleaseHoldException {
1144    ReportRegionStateTransitionResponse.Builder builder =
1145      ReportRegionStateTransitionResponse.newBuilder();
1146    ServerName serverName = ProtobufUtil.toServerName(req.getServer());
1147    ServerStateNode serverNode = regionStates.getOrCreateServer(serverName);
1148    // here we have to acquire a read lock instead of a simple exclusive lock. This is because that
1149    // we should not block other reportRegionStateTransition call from the same region server. This
1150    // is not only about performance, but also to prevent dead lock. Think of the meta region is
1151    // also on the same region server and you hold the lock which blocks the
1152    // reportRegionStateTransition for meta, and since meta is not online, you will block inside the
1153    // lock protection to wait for meta online...
1154    serverNode.readLock().lock();
1155    try {
1156      // we only accept reportRegionStateTransition if the region server is online, see the comment
1157      // above in submitServerCrash method and HBASE-21508 for more details.
1158      if (serverNode.isInState(ServerState.ONLINE)) {
1159        try {
1160          reportRegionStateTransition(builder, serverName, req.getTransitionList());
1161        } catch (PleaseHoldException e) {
1162          LOG.trace("Failed transition ", e);
1163          throw e;
1164        } catch (UnsupportedOperationException | IOException e) {
1165          // TODO: at the moment we have a single error message and the RS will abort
1166          // if the master says that one of the region transitions failed.
1167          LOG.warn("Failed transition", e);
1168          builder.setErrorMessage("Failed transition " + e.getMessage());
1169        }
1170      } else {
1171        LOG.warn("The region server {} is already dead, skip reportRegionStateTransition call",
1172          serverName);
1173        builder.setErrorMessage("You are dead");
1174      }
1175    } finally {
1176      serverNode.readLock().unlock();
1177    }
1178
1179    return builder.build();
1180  }
1181
1182  private void updateRegionTransition(ServerName serverName, TransitionCode state,
1183    RegionInfo regionInfo, long seqId, long procId) throws IOException {
1184    checkMetaLoaded(regionInfo);
1185
1186    RegionStateNode regionNode = regionStates.getRegionStateNode(regionInfo);
1187    if (regionNode == null) {
1188      // the table/region is gone. maybe a delete, split, merge
1189      throw new UnexpectedStateException(String.format(
1190        "Server %s was trying to transition region %s to %s. but Region is not known.", serverName,
1191        regionInfo, state));
1192    }
1193    LOG.trace("Update region transition serverName={} region={} regionState={}", serverName,
1194      regionNode, state);
1195
1196    ServerStateNode serverNode = regionStates.getOrCreateServer(serverName);
1197    regionNode.lock();
1198    try {
1199      if (!reportTransition(regionNode, serverNode, state, seqId, procId)) {
1200        // Don't log WARN if shutting down cluster; during shutdown. Avoid the below messages:
1201        // 2018-08-13 10:45:10,551 WARN ...AssignmentManager: No matching procedure found for
1202        // rit=OPEN, location=ve0538.halxg.cloudera.com,16020,1533493000958,
1203        // table=IntegrationTestBigLinkedList, region=65ab289e2fc1530df65f6c3d7cde7aa5 transition
1204        // to CLOSED
1205        // These happen because on cluster shutdown, we currently let the RegionServers close
1206        // regions. This is the only time that region close is not run by the Master (so cluster
1207        // goes down fast). Consider changing it so Master runs all shutdowns.
1208        if (
1209          this.master.getServerManager().isClusterShutdown() && state.equals(TransitionCode.CLOSED)
1210        ) {
1211          LOG.info("RegionServer {} {}", state, regionNode.getRegionInfo().getEncodedName());
1212        } else {
1213          LOG.warn("No matching procedure found for {} transition on {} to {}", serverName,
1214            regionNode, state);
1215        }
1216      }
1217    } finally {
1218      regionNode.unlock();
1219    }
1220  }
1221
1222  private boolean reportTransition(RegionStateNode regionNode, ServerStateNode serverNode,
1223    TransitionCode state, long seqId, long procId) throws IOException {
1224    ServerName serverName = serverNode.getServerName();
1225    TransitRegionStateProcedure proc = regionNode.getProcedure();
1226    if (proc == null) {
1227      return false;
1228    }
1229    proc.reportTransition(master.getMasterProcedureExecutor().getEnvironment(), regionNode,
1230      serverName, state, seqId, procId);
1231    return true;
1232  }
1233
1234  private void updateRegionSplitTransition(final ServerName serverName, final TransitionCode state,
1235    final RegionInfo parent, final RegionInfo hriA, final RegionInfo hriB) throws IOException {
1236    checkMetaLoaded(parent);
1237
1238    if (state != TransitionCode.READY_TO_SPLIT) {
1239      throw new UnexpectedStateException(
1240        "unsupported split regionState=" + state + " for parent region " + parent
1241          + " maybe an old RS (< 2.0) had the operation in progress");
1242    }
1243
1244    // sanity check on the request
1245    if (!Bytes.equals(hriA.getEndKey(), hriB.getStartKey())) {
1246      throw new UnsupportedOperationException("unsupported split request with bad keys: parent="
1247        + parent + " hriA=" + hriA + " hriB=" + hriB);
1248    }
1249
1250    if (!master.isSplitOrMergeEnabled(MasterSwitchType.SPLIT)) {
1251      LOG.warn("Split switch is off! skip split of " + parent);
1252      throw new DoNotRetryIOException(
1253        "Split region " + parent.getRegionNameAsString() + " failed due to split switch off");
1254    }
1255
1256    // Submit the Split procedure
1257    final byte[] splitKey = hriB.getStartKey();
1258    if (LOG.isDebugEnabled()) {
1259      LOG.debug("Split request from " + serverName + ", parent=" + parent + " splitKey="
1260        + Bytes.toStringBinary(splitKey));
1261    }
1262    // Processing this report happens asynchronously from other activities which can mutate
1263    // the region state. For example, a split procedure may already be running for this parent.
1264    // A split procedure cannot succeed if the parent region is no longer open, so we can
1265    // ignore it in that case.
1266    // Note that submitting more than one split procedure for a given region is
1267    // harmless -- the split is fenced in the procedure handling -- but it would be noisy in
1268    // the logs. Only one procedure can succeed. The other procedure(s) would abort during
1269    // initialization and report failure with WARN level logging.
1270    RegionState parentState = regionStates.getRegionState(parent);
1271    if (parentState != null && parentState.isOpened()) {
1272      master.getMasterProcedureExecutor().submitProcedure(createSplitProcedure(parent, splitKey));
1273    } else {
1274      LOG.info("Ignoring split request from " + serverName + ", parent=" + parent
1275        + " because parent is unknown or not open");
1276      return;
1277    }
1278
1279    // If the RS is < 2.0 throw an exception to abort the operation, we are handling the split
1280    if (master.getServerManager().getVersionNumber(serverName) < 0x0200000) {
1281      throw new UnsupportedOperationException(
1282        String.format("Split handled by the master: " + "parent=%s hriA=%s hriB=%s",
1283          parent.getShortNameToLog(), hriA, hriB));
1284    }
1285  }
1286
1287  private void updateRegionMergeTransition(final ServerName serverName, final TransitionCode state,
1288    final RegionInfo merged, final RegionInfo hriA, final RegionInfo hriB) throws IOException {
1289    checkMetaLoaded(merged);
1290
1291    if (state != TransitionCode.READY_TO_MERGE) {
1292      throw new UnexpectedStateException(
1293        "Unsupported merge regionState=" + state + " for regionA=" + hriA + " regionB=" + hriB
1294          + " merged=" + merged + " maybe an old RS (< 2.0) had the operation in progress");
1295    }
1296
1297    if (!master.isSplitOrMergeEnabled(MasterSwitchType.MERGE)) {
1298      LOG.warn("Merge switch is off! skip merge of regionA=" + hriA + " regionB=" + hriB);
1299      throw new DoNotRetryIOException(
1300        "Merge of regionA=" + hriA + " regionB=" + hriB + " failed because merge switch is off");
1301    }
1302
1303    // Submit the Merge procedure
1304    if (LOG.isDebugEnabled()) {
1305      LOG.debug("Handling merge request from RS=" + merged + ", merged=" + merged);
1306    }
1307    master.getMasterProcedureExecutor().submitProcedure(createMergeProcedure(hriA, hriB));
1308
1309    // If the RS is < 2.0 throw an exception to abort the operation, we are handling the merge
1310    if (master.getServerManager().getVersionNumber(serverName) < 0x0200000) {
1311      throw new UnsupportedOperationException(
1312        String.format("Merge not handled yet: regionState=%s merged=%s hriA=%s hriB=%s", state,
1313          merged, hriA, hriB));
1314    }
1315  }
1316
1317  // ============================================================================================
1318  // RS Status update (report online regions) helpers
1319  // ============================================================================================
1320  /**
1321   * The master will call this method when the RS send the regionServerReport(). The report will
1322   * contains the "online regions". This method will check the the online regions against the
1323   * in-memory state of the AM, and we will log a warn message if there is a mismatch. This is
1324   * because that there is no fencing between the reportRegionStateTransition method and
1325   * regionServerReport method, so there could be race and introduce inconsistency here, but
1326   * actually there is no problem.
1327   * <p/>
1328   * Please see HBASE-21421 and HBASE-21463 for more details.
1329   */
1330  public void reportOnlineRegions(ServerName serverName, Set<byte[]> regionNames) {
1331    if (!isRunning()) {
1332      return;
1333    }
1334    if (LOG.isTraceEnabled()) {
1335      LOG.trace("ReportOnlineRegions {} regionCount={}, metaLoaded={} {}", serverName,
1336        regionNames.size(), isMetaLoaded(),
1337        regionNames.stream().map(Bytes::toStringBinary).collect(Collectors.toList()));
1338    }
1339
1340    ServerStateNode serverNode = regionStates.getOrCreateServer(serverName);
1341    synchronized (serverNode) {
1342      if (!serverNode.isInState(ServerState.ONLINE)) {
1343        LOG.warn("Got a report from a server result in state " + serverNode.getState());
1344        return;
1345      }
1346    }
1347
1348    // Track the regionserver reported online regions in memory.
1349    synchronized (rsReports) {
1350      rsReports.put(serverName, regionNames);
1351    }
1352
1353    if (regionNames.isEmpty()) {
1354      // nothing to do if we don't have regions
1355      LOG.trace("no online region found on {}", serverName);
1356      return;
1357    }
1358    if (!isMetaLoaded()) {
1359      // we are still on startup, skip checking
1360      return;
1361    }
1362    // The Heartbeat tells us of what regions are on the region serve, check the state.
1363    checkOnlineRegionsReport(serverNode, regionNames);
1364  }
1365
1366  /**
1367   * Close <code>regionName</code> on <code>sn</code> silently and immediately without using a
1368   * Procedure or going via hbase:meta. For case where a RegionServer's hosting of a Region is not
1369   * aligned w/ the Master's accounting of Region state. This is for cleaning up an error in
1370   * accounting.
1371   */
1372  private void closeRegionSilently(ServerName sn, byte[] regionName) {
1373    try {
1374      RegionInfo ri = CatalogFamilyFormat.parseRegionInfoFromRegionName(regionName);
1375      // Pass -1 for timeout. Means do not wait.
1376      ServerManager.closeRegionSilentlyAndWait(this.master.getAsyncClusterConnection(), sn, ri, -1);
1377    } catch (Exception e) {
1378      LOG.error("Failed trying to close {} on {}", Bytes.toStringBinary(regionName), sn, e);
1379    }
1380  }
1381
1382  /**
1383   * Check that what the RegionServer reports aligns with the Master's image. If disagreement, we
1384   * will tell the RegionServer to expediently close a Region we do not think it should have.
1385   */
1386  private void checkOnlineRegionsReport(ServerStateNode serverNode, Set<byte[]> regionNames) {
1387    ServerName serverName = serverNode.getServerName();
1388    for (byte[] regionName : regionNames) {
1389      if (!isRunning()) {
1390        return;
1391      }
1392      RegionStateNode regionNode = regionStates.getRegionStateNodeFromName(regionName);
1393      if (regionNode == null) {
1394        String regionNameAsStr = Bytes.toStringBinary(regionName);
1395        LOG.warn("No RegionStateNode for {} but reported as up on {}; closing...", regionNameAsStr,
1396          serverName);
1397        closeRegionSilently(serverNode.getServerName(), regionName);
1398        continue;
1399      }
1400      final long lag = 1000;
1401      regionNode.lock();
1402      try {
1403        long diff = EnvironmentEdgeManager.currentTime() - regionNode.getLastUpdate();
1404        if (regionNode.isInState(State.OPENING, State.OPEN)) {
1405          // This is possible as a region server has just closed a region but the region server
1406          // report is generated before the closing, but arrive after the closing. Make sure there
1407          // is some elapsed time so less false alarms.
1408          if (!regionNode.getRegionLocation().equals(serverName) && diff > lag) {
1409            LOG.warn("Reporting {} server does not match {} (time since last "
1410              + "update={}ms); closing...", serverName, regionNode, diff);
1411            closeRegionSilently(serverNode.getServerName(), regionName);
1412          }
1413        } else if (!regionNode.isInState(State.CLOSING, State.SPLITTING)) {
1414          // So, we can get report that a region is CLOSED or SPLIT because a heartbeat
1415          // came in at about same time as a region transition. Make sure there is some
1416          // elapsed time so less false alarms.
1417          if (diff > lag) {
1418            LOG.warn("Reporting {} state does not match {} (time since last update={}ms)",
1419              serverName, regionNode, diff);
1420          }
1421        }
1422      } finally {
1423        regionNode.unlock();
1424      }
1425    }
1426  }
1427
1428  // ============================================================================================
1429  // RIT chore
1430  // ============================================================================================
1431  private static class RegionInTransitionChore extends ProcedureInMemoryChore<MasterProcedureEnv> {
1432    public RegionInTransitionChore(final int timeoutMsec) {
1433      super(timeoutMsec);
1434    }
1435
1436    @Override
1437    protected void periodicExecute(final MasterProcedureEnv env) {
1438      final AssignmentManager am = env.getAssignmentManager();
1439
1440      final RegionInTransitionStat ritStat = am.computeRegionInTransitionStat();
1441      if (ritStat.hasRegionsOverThreshold()) {
1442        for (RegionState hri : ritStat.getRegionOverThreshold()) {
1443          am.handleRegionOverStuckWarningThreshold(hri.getRegion());
1444        }
1445      }
1446
1447      // update metrics
1448      am.updateRegionsInTransitionMetrics(ritStat);
1449    }
1450  }
1451
1452  private static class DeadServerMetricRegionChore
1453    extends ProcedureInMemoryChore<MasterProcedureEnv> {
1454    public DeadServerMetricRegionChore(final int timeoutMsec) {
1455      super(timeoutMsec);
1456    }
1457
1458    @Override
1459    protected void periodicExecute(final MasterProcedureEnv env) {
1460      final ServerManager sm = env.getMasterServices().getServerManager();
1461      final AssignmentManager am = env.getAssignmentManager();
1462      // To minimize inconsistencies we are not going to snapshot live servers in advance in case
1463      // new servers are added; OTOH we don't want to add heavy sync for a consistent view since
1464      // this is for metrics. Instead, we're going to check each regions as we go; to avoid making
1465      // too many checks, we maintain a local lists of server, limiting us to false negatives. If
1466      // we miss some recently-dead server, we'll just see it next time.
1467      Set<ServerName> recentlyLiveServers = new HashSet<>();
1468      int deadRegions = 0, unknownRegions = 0;
1469      for (RegionStateNode rsn : am.getRegionStates().getRegionStateNodes()) {
1470        if (rsn.getState() != State.OPEN) {
1471          continue; // Opportunistic check, should quickly skip RITs, offline tables, etc.
1472        }
1473        // Do not need to acquire region state lock as this is only for showing metrics.
1474        ServerName sn = rsn.getRegionLocation();
1475        State state = rsn.getState();
1476        if (state != State.OPEN) {
1477          continue; // Mostly skipping RITs that are already being take care of.
1478        }
1479        if (sn == null) {
1480          ++unknownRegions; // Opened on null?
1481          continue;
1482        }
1483        if (recentlyLiveServers.contains(sn)) {
1484          continue;
1485        }
1486        ServerManager.ServerLiveState sls = sm.isServerKnownAndOnline(sn);
1487        switch (sls) {
1488          case LIVE:
1489            recentlyLiveServers.add(sn);
1490            break;
1491          case DEAD:
1492            ++deadRegions;
1493            break;
1494          case UNKNOWN:
1495            ++unknownRegions;
1496            break;
1497          default:
1498            throw new AssertionError("Unexpected " + sls);
1499        }
1500      }
1501      if (deadRegions > 0 || unknownRegions > 0) {
1502        LOG.info("Found {} OPEN regions on dead servers and {} OPEN regions on unknown servers",
1503          deadRegions, unknownRegions);
1504      }
1505
1506      am.updateDeadServerRegionMetrics(deadRegions, unknownRegions);
1507    }
1508  }
1509
1510  public RegionInTransitionStat computeRegionInTransitionStat() {
1511    final RegionInTransitionStat rit = new RegionInTransitionStat(getConfiguration());
1512    rit.update(this);
1513    return rit;
1514  }
1515
1516  public static class RegionInTransitionStat {
1517    private final int ritThreshold;
1518
1519    private HashMap<String, RegionState> ritsOverThreshold = null;
1520    private long statTimestamp;
1521    private long oldestRITTime = 0;
1522    private int totalRITsTwiceThreshold = 0;
1523    private int totalRITs = 0;
1524
1525    public RegionInTransitionStat(final Configuration conf) {
1526      this.ritThreshold =
1527        conf.getInt(METRICS_RIT_STUCK_WARNING_THRESHOLD, DEFAULT_RIT_STUCK_WARNING_THRESHOLD);
1528    }
1529
1530    public int getRITThreshold() {
1531      return ritThreshold;
1532    }
1533
1534    public long getTimestamp() {
1535      return statTimestamp;
1536    }
1537
1538    public int getTotalRITs() {
1539      return totalRITs;
1540    }
1541
1542    public long getOldestRITTime() {
1543      return oldestRITTime;
1544    }
1545
1546    public int getTotalRITsOverThreshold() {
1547      Map<String, RegionState> m = this.ritsOverThreshold;
1548      return m != null ? m.size() : 0;
1549    }
1550
1551    public boolean hasRegionsTwiceOverThreshold() {
1552      return totalRITsTwiceThreshold > 0;
1553    }
1554
1555    public boolean hasRegionsOverThreshold() {
1556      Map<String, RegionState> m = this.ritsOverThreshold;
1557      return m != null && !m.isEmpty();
1558    }
1559
1560    public Collection<RegionState> getRegionOverThreshold() {
1561      Map<String, RegionState> m = this.ritsOverThreshold;
1562      return m != null ? m.values() : Collections.emptySet();
1563    }
1564
1565    public boolean isRegionOverThreshold(final RegionInfo regionInfo) {
1566      Map<String, RegionState> m = this.ritsOverThreshold;
1567      return m != null && m.containsKey(regionInfo.getEncodedName());
1568    }
1569
1570    public boolean isRegionTwiceOverThreshold(final RegionInfo regionInfo) {
1571      Map<String, RegionState> m = this.ritsOverThreshold;
1572      if (m == null) {
1573        return false;
1574      }
1575      final RegionState state = m.get(regionInfo.getEncodedName());
1576      if (state == null) {
1577        return false;
1578      }
1579      return (statTimestamp - state.getStamp()) > (ritThreshold * 2);
1580    }
1581
1582    protected void update(final AssignmentManager am) {
1583      final RegionStates regionStates = am.getRegionStates();
1584      this.statTimestamp = EnvironmentEdgeManager.currentTime();
1585      update(regionStates.getRegionsStateInTransition(), statTimestamp);
1586      update(regionStates.getRegionFailedOpen(), statTimestamp);
1587
1588      if (LOG.isDebugEnabled() && ritsOverThreshold != null && !ritsOverThreshold.isEmpty()) {
1589        LOG.debug("RITs over threshold: {}",
1590          ritsOverThreshold.entrySet().stream()
1591            .map(e -> e.getKey() + ":" + e.getValue().getState().name())
1592            .collect(Collectors.joining("\n")));
1593      }
1594    }
1595
1596    private void update(final Collection<RegionState> regions, final long currentTime) {
1597      for (RegionState state : regions) {
1598        totalRITs++;
1599        final long ritStartedMs = state.getStamp();
1600        if (ritStartedMs == 0) {
1601          // Don't output bogus values to metrics if they accidentally make it here.
1602          LOG.warn("The RIT {} has no start time", state.getRegion());
1603          continue;
1604        }
1605        final long ritTime = currentTime - ritStartedMs;
1606        if (ritTime > ritThreshold) {
1607          if (ritsOverThreshold == null) {
1608            ritsOverThreshold = new HashMap<String, RegionState>();
1609          }
1610          ritsOverThreshold.put(state.getRegion().getEncodedName(), state);
1611          totalRITsTwiceThreshold += (ritTime > (ritThreshold * 2)) ? 1 : 0;
1612        }
1613        if (oldestRITTime < ritTime) {
1614          oldestRITTime = ritTime;
1615        }
1616      }
1617    }
1618  }
1619
1620  private void updateRegionsInTransitionMetrics(final RegionInTransitionStat ritStat) {
1621    metrics.updateRITOldestAge(ritStat.getOldestRITTime());
1622    metrics.updateRITCount(ritStat.getTotalRITs());
1623    metrics.updateRITCountOverThreshold(ritStat.getTotalRITsOverThreshold());
1624  }
1625
1626  private void updateDeadServerRegionMetrics(int deadRegions, int unknownRegions) {
1627    metrics.updateDeadServerOpenRegions(deadRegions);
1628    metrics.updateUnknownServerOpenRegions(unknownRegions);
1629  }
1630
1631  private void handleRegionOverStuckWarningThreshold(final RegionInfo regionInfo) {
1632    final RegionStateNode regionNode = regionStates.getRegionStateNode(regionInfo);
1633    // if (regionNode.isStuck()) {
1634    LOG.warn("STUCK Region-In-Transition {}", regionNode);
1635  }
1636
1637  // ============================================================================================
1638  // TODO: Master load/bootstrap
1639  // ============================================================================================
1640  public void joinCluster() throws IOException {
1641    long startTime = System.nanoTime();
1642    LOG.debug("Joining cluster...");
1643
1644    // Scan hbase:meta to build list of existing regions, servers, and assignment.
1645    // hbase:meta is online now or will be. Inside loadMeta, we keep trying. Can't make progress
1646    // w/o meta.
1647    loadMeta();
1648
1649    while (master.getServerManager().countOfRegionServers() < 1) {
1650      LOG.info("Waiting for RegionServers to join; current count={}",
1651        master.getServerManager().countOfRegionServers());
1652      Threads.sleep(250);
1653    }
1654    LOG.info("Number of RegionServers={}", master.getServerManager().countOfRegionServers());
1655
1656    // Start the chores
1657    master.getMasterProcedureExecutor().addChore(this.ritChore);
1658    master.getMasterProcedureExecutor().addChore(this.deadMetricChore);
1659
1660    long costMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime);
1661    LOG.info("Joined the cluster in {}", StringUtils.humanTimeDiff(costMs));
1662  }
1663
1664  /**
1665   * Create assign procedure for offline regions. Just follow the old
1666   * processofflineServersWithOnlineRegions method. Since now we do not need to deal with dead
1667   * server any more, we only deal with the regions in OFFLINE state in this method. And this is a
1668   * bit strange, that for new regions, we will add it in CLOSED state instead of OFFLINE state, and
1669   * usually there will be a procedure to track them. The processofflineServersWithOnlineRegions is
1670   * a legacy from long ago, as things are going really different now, maybe we do not need this
1671   * method any more. Need to revisit later.
1672   */
1673  // Public so can be run by the Master as part of the startup. Needs hbase:meta to be online.
1674  // Needs to be done after the table state manager has been started.
1675  public void processOfflineRegions() {
1676    TransitRegionStateProcedure[] procs =
1677      regionStates.getRegionStateNodes().stream().filter(rsn -> rsn.isInState(State.OFFLINE))
1678        .filter(rsn -> isTableEnabled(rsn.getRegionInfo().getTable())).map(rsn -> {
1679          rsn.lock();
1680          try {
1681            if (rsn.getProcedure() != null) {
1682              return null;
1683            } else {
1684              return rsn.setProcedure(TransitRegionStateProcedure.assign(getProcedureEnvironment(),
1685                rsn.getRegionInfo(), null));
1686            }
1687          } finally {
1688            rsn.unlock();
1689          }
1690        }).filter(p -> p != null).toArray(TransitRegionStateProcedure[]::new);
1691    if (procs.length > 0) {
1692      master.getMasterProcedureExecutor().submitProcedures(procs);
1693    }
1694  }
1695
1696  /*
1697   * AM internal RegionStateStore.RegionStateVisitor implementation. To be used when scanning META
1698   * table for region rows, using RegionStateStore utility methods. RegionStateStore methods will
1699   * convert Result into proper RegionInfo instances, but those would still need to be added into
1700   * AssignmentManager.regionStates in-memory cache. RegionMetaLoadingVisitor.visitRegionState
1701   * method provides the logic for adding RegionInfo instances as loaded from latest META scan into
1702   * AssignmentManager.regionStates.
1703   */
1704  private class RegionMetaLoadingVisitor implements RegionStateStore.RegionStateVisitor {
1705
1706    @Override
1707    public void visitRegionState(Result result, final RegionInfo regionInfo, final State state,
1708      final ServerName regionLocation, final ServerName lastHost, final long openSeqNum) {
1709      if (
1710        state == null && regionLocation == null && lastHost == null
1711          && openSeqNum == SequenceId.NO_SEQUENCE_ID
1712      ) {
1713        // This is a row with nothing in it.
1714        LOG.warn("Skipping empty row={}", result);
1715        return;
1716      }
1717      State localState = state;
1718      if (localState == null) {
1719        // No region state column data in hbase:meta table! Are I doing a rolling upgrade from
1720        // hbase1 to hbase2? Am I restoring a SNAPSHOT or otherwise adding a region to hbase:meta?
1721        // In any of these cases, state is empty. For now, presume OFFLINE but there are probably
1722        // cases where we need to probe more to be sure this correct; TODO informed by experience.
1723        LOG.info(regionInfo.getEncodedName() + " regionState=null; presuming " + State.OFFLINE);
1724        localState = State.OFFLINE;
1725      }
1726      RegionStateNode regionNode = regionStates.getOrCreateRegionStateNode(regionInfo);
1727      // Do not need to lock on regionNode, as we can make sure that before we finish loading
1728      // meta, all the related procedures can not be executed. The only exception is for meta
1729      // region related operations, but here we do not load the informations for meta region.
1730      regionNode.setState(localState);
1731      regionNode.setLastHost(lastHost);
1732      regionNode.setRegionLocation(regionLocation);
1733      regionNode.setOpenSeqNum(openSeqNum);
1734
1735      // Note: keep consistent with other methods, see region(Opening|Opened|Closing)
1736      // RIT/ServerCrash handling should take care of the transiting regions.
1737      if (
1738        localState.matches(State.OPEN, State.OPENING, State.CLOSING, State.SPLITTING, State.MERGING)
1739      ) {
1740        assert regionLocation != null : "found null region location for " + regionNode;
1741        regionStates.addRegionToServer(regionNode);
1742      } else if (localState == State.OFFLINE || regionInfo.isOffline()) {
1743        regionStates.addToOfflineRegions(regionNode);
1744      }
1745      if (regionNode.getProcedure() != null) {
1746        regionNode.getProcedure().stateLoaded(AssignmentManager.this, regionNode);
1747      }
1748    }
1749  };
1750
1751  /**
1752   * Attempt to load {@code regionInfo} from META, adding any results to the
1753   * {@link #regionStateStore} Is NOT aware of replica regions.
1754   * @param regionInfo the region to be loaded from META.
1755   * @throws IOException If some error occurs while querying META or parsing results.
1756   */
1757  public void populateRegionStatesFromMeta(@NonNull final RegionInfo regionInfo)
1758    throws IOException {
1759    final String regionEncodedName = RegionInfo.DEFAULT_REPLICA_ID == regionInfo.getReplicaId()
1760      ? regionInfo.getEncodedName()
1761      : RegionInfoBuilder.newBuilder(regionInfo).setReplicaId(RegionInfo.DEFAULT_REPLICA_ID).build()
1762        .getEncodedName();
1763    populateRegionStatesFromMeta(regionEncodedName);
1764  }
1765
1766  /**
1767   * Attempt to load {@code regionEncodedName} from META, adding any results to the
1768   * {@link #regionStateStore} Is NOT aware of replica regions.
1769   * @param regionEncodedName encoded name for the region to be loaded from META.
1770   * @throws IOException If some error occurs while querying META or parsing results.
1771   */
1772  public void populateRegionStatesFromMeta(@NonNull String regionEncodedName) throws IOException {
1773    final RegionMetaLoadingVisitor visitor = new RegionMetaLoadingVisitor();
1774    regionStateStore.visitMetaForRegion(regionEncodedName, visitor);
1775  }
1776
1777  private void loadMeta() throws IOException {
1778    // TODO: use a thread pool
1779    regionStateStore.visitMeta(new RegionMetaLoadingVisitor());
1780  }
1781
1782  /**
1783   * Used to check if the meta loading is done.
1784   * <p/>
1785   * if not we throw PleaseHoldException since we are rebuilding the RegionStates
1786   * @param hri region to check if it is already rebuild
1787   * @throws PleaseHoldException if meta has not been loaded yet
1788   */
1789  private void checkMetaLoaded(RegionInfo hri) throws PleaseHoldException {
1790    if (!isRunning()) {
1791      throw new PleaseHoldException("AssignmentManager not running");
1792    }
1793    boolean meta = isMetaRegion(hri);
1794    boolean metaLoaded = isMetaLoaded();
1795    if (!meta && !metaLoaded) {
1796      throw new PleaseHoldException(
1797        "Master not fully online; hbase:meta=" + meta + ", metaLoaded=" + metaLoaded);
1798    }
1799  }
1800
1801  // ============================================================================================
1802  // TODO: Metrics
1803  // ============================================================================================
1804  public int getNumRegionsOpened() {
1805    // TODO: Used by TestRegionPlacement.java and assume monotonically increasing value
1806    return 0;
1807  }
1808
1809  /**
1810   * Usually run by the Master in reaction to server crash during normal processing. Can also be
1811   * invoked via external RPC to effect repair; in the latter case, the 'force' flag is set so we
1812   * push through the SCP though context may indicate already-running-SCP (An old SCP may have
1813   * exited abnormally, or damaged cluster may still have references in hbase:meta to 'Unknown
1814   * Servers' -- servers that are not online or in dead servers list, etc.)
1815   * @param force Set if the request came in externally over RPC (via hbck2). Force means run the
1816   *              SCP even if it seems as though there might be an outstanding SCP running.
1817   * @return pid of scheduled SCP or {@link Procedure#NO_PROC_ID} if none scheduled.
1818   */
1819  public long submitServerCrash(ServerName serverName, boolean shouldSplitWal, boolean force) {
1820    // May be an 'Unknown Server' so handle case where serverNode is null.
1821    ServerStateNode serverNode = regionStates.getServerNode(serverName);
1822    // Remove the in-memory rsReports result
1823    synchronized (rsReports) {
1824      rsReports.remove(serverName);
1825    }
1826
1827    // We hold the write lock here for fencing on reportRegionStateTransition. Once we set the
1828    // server state to CRASHED, we will no longer accept the reportRegionStateTransition call from
1829    // this server. This is used to simplify the implementation for TRSP and SCP, where we can make
1830    // sure that, the region list fetched by SCP will not be changed any more.
1831    if (serverNode != null) {
1832      serverNode.writeLock().lock();
1833    }
1834    boolean carryingMeta;
1835    long pid;
1836    try {
1837      ProcedureExecutor<MasterProcedureEnv> procExec = this.master.getMasterProcedureExecutor();
1838      carryingMeta = isCarryingMeta(serverName);
1839      if (!force && serverNode != null && !serverNode.isInState(ServerState.ONLINE)) {
1840        LOG.info("Skip adding ServerCrashProcedure for {} (meta={}) -- running?", serverNode,
1841          carryingMeta);
1842        return Procedure.NO_PROC_ID;
1843      } else {
1844        MasterProcedureEnv mpe = procExec.getEnvironment();
1845        // If serverNode == null, then 'Unknown Server'. Schedule HBCKSCP instead.
1846        // HBCKSCP scours Master in-memory state AND hbase;meta for references to
1847        // serverName just-in-case. An SCP that is scheduled when the server is
1848        // 'Unknown' probably originated externally with HBCK2 fix-it tool.
1849        ServerState oldState = null;
1850        if (serverNode != null) {
1851          oldState = serverNode.getState();
1852          serverNode.setState(ServerState.CRASHED);
1853        }
1854
1855        if (force) {
1856          pid = procExec.submitProcedure(
1857            new HBCKServerCrashProcedure(mpe, serverName, shouldSplitWal, carryingMeta));
1858        } else {
1859          pid = procExec.submitProcedure(
1860            new ServerCrashProcedure(mpe, serverName, shouldSplitWal, carryingMeta));
1861        }
1862        LOG.info("Scheduled ServerCrashProcedure pid={} for {} (carryingMeta={}){}.", pid,
1863          serverName, carryingMeta,
1864          serverNode == null ? "" : " " + serverNode.toString() + ", oldState=" + oldState);
1865      }
1866    } finally {
1867      if (serverNode != null) {
1868        serverNode.writeLock().unlock();
1869      }
1870    }
1871    return pid;
1872  }
1873
1874  public void offlineRegion(final RegionInfo regionInfo) {
1875    // TODO used by MasterRpcServices
1876    RegionStateNode node = regionStates.getRegionStateNode(regionInfo);
1877    if (node != null) {
1878      node.offline();
1879    }
1880  }
1881
1882  public void onlineRegion(final RegionInfo regionInfo, final ServerName serverName) {
1883    // TODO used by TestSplitTransactionOnCluster.java
1884  }
1885
1886  public Map<ServerName, List<RegionInfo>>
1887    getSnapShotOfAssignment(final Collection<RegionInfo> regions) {
1888    return regionStates.getSnapShotOfAssignment(regions);
1889  }
1890
1891  // ============================================================================================
1892  // TODO: UTILS/HELPERS?
1893  // ============================================================================================
1894  /**
1895   * Used by the client (via master) to identify if all regions have the schema updates
1896   * @return Pair indicating the status of the alter command (pending/total)
1897   */
1898  public Pair<Integer, Integer> getReopenStatus(TableName tableName) {
1899    if (isTableDisabled(tableName)) {
1900      return new Pair<Integer, Integer>(0, 0);
1901    }
1902
1903    final List<RegionState> states = regionStates.getTableRegionStates(tableName);
1904    int ritCount = 0;
1905    for (RegionState regionState : states) {
1906      if (!regionState.isOpened() && !regionState.isSplit()) {
1907        ritCount++;
1908      }
1909    }
1910    return new Pair<Integer, Integer>(ritCount, states.size());
1911  }
1912
1913  // ============================================================================================
1914  // TODO: Region State In Transition
1915  // ============================================================================================
1916  public boolean hasRegionsInTransition() {
1917    return regionStates.hasRegionsInTransition();
1918  }
1919
1920  public List<RegionStateNode> getRegionsInTransition() {
1921    return regionStates.getRegionsInTransition();
1922  }
1923
1924  public List<RegionInfo> getAssignedRegions() {
1925    return regionStates.getAssignedRegions();
1926  }
1927
1928  /**
1929   * Resolve a cached {@link RegionInfo} from the region name as a {@code byte[]}.
1930   */
1931  public RegionInfo getRegionInfo(final byte[] regionName) {
1932    final RegionStateNode regionState = regionStates.getRegionStateNodeFromName(regionName);
1933    return regionState != null ? regionState.getRegionInfo() : null;
1934  }
1935
1936  /**
1937   * Resolve a cached {@link RegionInfo} from the encoded region name as a {@code String}.
1938   */
1939  public RegionInfo getRegionInfo(final String encodedRegionName) {
1940    final RegionStateNode regionState =
1941      regionStates.getRegionStateNodeFromEncodedRegionName(encodedRegionName);
1942    return regionState != null ? regionState.getRegionInfo() : null;
1943  }
1944
1945  // ============================================================================================
1946  // Expected states on region state transition.
1947  // Notice that there is expected states for transiting to OPENING state, this is because SCP.
1948  // See the comments in regionOpening method for more details.
1949  // ============================================================================================
1950  private static final State[] STATES_EXPECTED_ON_OPEN = { State.OPENING, // Normal case
1951    State.OPEN // Retrying
1952  };
1953
1954  private static final State[] STATES_EXPECTED_ON_CLOSING = { State.OPEN, // Normal case
1955    State.CLOSING, // Retrying
1956    State.SPLITTING, // Offline the split parent
1957    State.MERGING // Offline the merge parents
1958  };
1959
1960  private static final State[] STATES_EXPECTED_ON_CLOSED = { State.CLOSING, // Normal case
1961    State.CLOSED // Retrying
1962  };
1963
1964  // This is for manually scheduled region assign, can add other states later if we find out other
1965  // usages
1966  private static final State[] STATES_EXPECTED_ON_ASSIGN = { State.CLOSED, State.OFFLINE };
1967
1968  // We only allow unassign or move a region which is in OPEN state.
1969  private static final State[] STATES_EXPECTED_ON_UNASSIGN_OR_MOVE = { State.OPEN };
1970
1971  // ============================================================================================
1972  // Region Status update
1973  // Should only be called in TransitRegionStateProcedure(and related procedures), as the locking
1974  // and pre-assumptions are very tricky.
1975  // ============================================================================================
1976  private void transitStateAndUpdate(RegionStateNode regionNode, RegionState.State newState,
1977    RegionState.State... expectedStates) throws IOException {
1978    RegionState.State state = regionNode.getState();
1979    regionNode.transitionState(newState, expectedStates);
1980    boolean succ = false;
1981    try {
1982      regionStateStore.updateRegionLocation(regionNode);
1983      succ = true;
1984    } finally {
1985      if (!succ) {
1986        // revert
1987        regionNode.setState(state);
1988      }
1989    }
1990  }
1991
1992  // should be called within the synchronized block of RegionStateNode
1993  void regionOpening(RegionStateNode regionNode) throws IOException {
1994    // As in SCP, for performance reason, there is no TRSP attached with this region, we will not
1995    // update the region state, which means that the region could be in any state when we want to
1996    // assign it after a RS crash. So here we do not pass the expectedStates parameter.
1997    transitStateAndUpdate(regionNode, State.OPENING);
1998    regionStates.addRegionToServer(regionNode);
1999    // update the operation count metrics
2000    metrics.incrementOperationCounter();
2001  }
2002
2003  // should be called under the RegionStateNode lock
2004  // The parameter 'giveUp' means whether we will try to open the region again, if it is true, then
2005  // we will persist the FAILED_OPEN state into hbase:meta.
2006  void regionFailedOpen(RegionStateNode regionNode, boolean giveUp) throws IOException {
2007    RegionState.State state = regionNode.getState();
2008    ServerName regionLocation = regionNode.getRegionLocation();
2009    if (giveUp) {
2010      regionNode.setState(State.FAILED_OPEN);
2011      regionNode.setRegionLocation(null);
2012      boolean succ = false;
2013      try {
2014        regionStateStore.updateRegionLocation(regionNode);
2015        succ = true;
2016      } finally {
2017        if (!succ) {
2018          // revert
2019          regionNode.setState(state);
2020          regionNode.setRegionLocation(regionLocation);
2021        }
2022      }
2023    }
2024    if (regionLocation != null) {
2025      regionStates.removeRegionFromServer(regionLocation, regionNode);
2026    }
2027  }
2028
2029  // should be called under the RegionStateNode lock
2030  void regionClosing(RegionStateNode regionNode) throws IOException {
2031    transitStateAndUpdate(regionNode, State.CLOSING, STATES_EXPECTED_ON_CLOSING);
2032
2033    RegionInfo hri = regionNode.getRegionInfo();
2034    // Set meta has not initialized early. so people trying to create/edit tables will wait
2035    if (isMetaRegion(hri)) {
2036      setMetaAssigned(hri, false);
2037    }
2038    regionStates.addRegionToServer(regionNode);
2039    // update the operation count metrics
2040    metrics.incrementOperationCounter();
2041  }
2042
2043  // for open and close, they will first be persist to the procedure store in
2044  // RegionRemoteProcedureBase. So here we will first change the in memory state as it is considered
2045  // as succeeded if the persistence to procedure store is succeeded, and then when the
2046  // RegionRemoteProcedureBase is woken up, we will persist the RegionStateNode to hbase:meta.
2047
2048  // should be called under the RegionStateNode lock
2049  void regionOpenedWithoutPersistingToMeta(RegionStateNode regionNode) throws IOException {
2050    regionNode.transitionState(State.OPEN, STATES_EXPECTED_ON_OPEN);
2051    RegionInfo regionInfo = regionNode.getRegionInfo();
2052    regionStates.addRegionToServer(regionNode);
2053    regionStates.removeFromFailedOpen(regionInfo);
2054  }
2055
2056  // should be called under the RegionStateNode lock
2057  void regionClosedWithoutPersistingToMeta(RegionStateNode regionNode) throws IOException {
2058    ServerName regionLocation = regionNode.getRegionLocation();
2059    regionNode.transitionState(State.CLOSED, STATES_EXPECTED_ON_CLOSED);
2060    regionNode.setRegionLocation(null);
2061    if (regionLocation != null) {
2062      regionNode.setLastHost(regionLocation);
2063      regionStates.removeRegionFromServer(regionLocation, regionNode);
2064    }
2065  }
2066
2067  // should be called under the RegionStateNode lock
2068  // for SCP
2069  public void regionClosedAbnormally(RegionStateNode regionNode) throws IOException {
2070    RegionState.State state = regionNode.getState();
2071    ServerName regionLocation = regionNode.getRegionLocation();
2072    regionNode.transitionState(State.ABNORMALLY_CLOSED);
2073    regionNode.setRegionLocation(null);
2074    boolean succ = false;
2075    try {
2076      regionStateStore.updateRegionLocation(regionNode);
2077      succ = true;
2078    } finally {
2079      if (!succ) {
2080        // revert
2081        regionNode.setState(state);
2082        regionNode.setRegionLocation(regionLocation);
2083      }
2084    }
2085    if (regionLocation != null) {
2086      regionNode.setLastHost(regionLocation);
2087      regionStates.removeRegionFromServer(regionLocation, regionNode);
2088    }
2089  }
2090
2091  void persistToMeta(RegionStateNode regionNode) throws IOException {
2092    regionStateStore.updateRegionLocation(regionNode);
2093    RegionInfo regionInfo = regionNode.getRegionInfo();
2094    if (isMetaRegion(regionInfo) && regionNode.getState() == State.OPEN) {
2095      // Usually we'd set a table ENABLED at this stage but hbase:meta is ALWAYs enabled, it
2096      // can't be disabled -- so skip the RPC (besides... enabled is managed by TableStateManager
2097      // which is backed by hbase:meta... Avoid setting ENABLED to avoid having to update state
2098      // on table that contains state.
2099      setMetaAssigned(regionInfo, true);
2100    }
2101  }
2102
2103  // ============================================================================================
2104  // The above methods can only be called in TransitRegionStateProcedure(and related procedures)
2105  // ============================================================================================
2106
2107  public void markRegionAsSplit(final RegionInfo parent, final ServerName serverName,
2108    final RegionInfo daughterA, final RegionInfo daughterB) throws IOException {
2109    // Update hbase:meta. Parent will be marked offline and split up in hbase:meta.
2110    // The parent stays in regionStates until cleared when removed by CatalogJanitor.
2111    // Update its state in regionStates to it shows as offline and split when read
2112    // later figuring what regions are in a table and what are not: see
2113    // regionStates#getRegionsOfTable
2114    final RegionStateNode node = regionStates.getOrCreateRegionStateNode(parent);
2115    node.setState(State.SPLIT);
2116    final RegionStateNode nodeA = regionStates.getOrCreateRegionStateNode(daughterA);
2117    nodeA.setState(State.SPLITTING_NEW);
2118    final RegionStateNode nodeB = regionStates.getOrCreateRegionStateNode(daughterB);
2119    nodeB.setState(State.SPLITTING_NEW);
2120
2121    TableDescriptor td = master.getTableDescriptors().get(parent.getTable());
2122    // TODO: here we just update the parent region info in meta, to set split and offline to true,
2123    // without changing the one in the region node. This is a bit confusing but the region info
2124    // field in RegionStateNode is not expected to be changed in the current design. Need to find a
2125    // possible way to address this problem, or at least adding more comments about the trick to
2126    // deal with this problem, that when you want to filter out split parent, you need to check both
2127    // the RegionState on whether it is split, and also the region info. If one of them matches then
2128    // it is a split parent. And usually only one of them can match, as after restart, the region
2129    // state will be changed from SPLIT to CLOSED.
2130    regionStateStore.splitRegion(parent, daughterA, daughterB, serverName, td);
2131    if (shouldAssignFavoredNodes(parent)) {
2132      List<ServerName> onlineServers = this.master.getServerManager().getOnlineServersList();
2133      getFavoredNodePromoter().generateFavoredNodesForDaughter(onlineServers, parent, daughterA,
2134        daughterB);
2135    }
2136  }
2137
2138  /**
2139   * When called here, the merge has happened. The merged regions have been unassigned and the above
2140   * markRegionClosed has been called on each so they have been disassociated from a hosting Server.
2141   * The merged region will be open after this call. The merged regions are removed from hbase:meta
2142   * below. Later they are deleted from the filesystem by the catalog janitor running against
2143   * hbase:meta. It notices when the merged region no longer holds references to the old regions
2144   * (References are deleted after a compaction rewrites what the Reference points at but not until
2145   * the archiver chore runs, are the References removed).
2146   */
2147  public void markRegionAsMerged(final RegionInfo child, final ServerName serverName,
2148    RegionInfo[] mergeParents) throws IOException {
2149    final RegionStateNode node = regionStates.getOrCreateRegionStateNode(child);
2150    node.setState(State.MERGED);
2151    for (RegionInfo ri : mergeParents) {
2152      regionStates.deleteRegion(ri);
2153    }
2154    TableDescriptor td = master.getTableDescriptors().get(child.getTable());
2155    regionStateStore.mergeRegions(child, mergeParents, serverName, td);
2156    if (shouldAssignFavoredNodes(child)) {
2157      getFavoredNodePromoter().generateFavoredNodesForMergedRegion(child, mergeParents);
2158    }
2159  }
2160
2161  /*
2162   * Favored nodes should be applied only when FavoredNodes balancer is configured and the region
2163   * belongs to a non-system table.
2164   */
2165  private boolean shouldAssignFavoredNodes(RegionInfo region) {
2166    return this.shouldAssignRegionsWithFavoredNodes
2167      && FavoredNodesManager.isFavoredNodeApplicable(region);
2168  }
2169
2170  // ============================================================================================
2171  // Assign Queue (Assign/Balance)
2172  // ============================================================================================
2173  private final ArrayList<RegionStateNode> pendingAssignQueue = new ArrayList<RegionStateNode>();
2174  private final ReentrantLock assignQueueLock = new ReentrantLock();
2175  private final Condition assignQueueFullCond = assignQueueLock.newCondition();
2176
2177  /**
2178   * Add the assign operation to the assignment queue. The pending assignment operation will be
2179   * processed, and each region will be assigned by a server using the balancer.
2180   */
2181  protected void queueAssign(final RegionStateNode regionNode) {
2182    regionNode.getProcedureEvent().suspend();
2183
2184    // TODO: quick-start for meta and the other sys-tables?
2185    assignQueueLock.lock();
2186    try {
2187      pendingAssignQueue.add(regionNode);
2188      if (
2189        regionNode.isSystemTable() || pendingAssignQueue.size() == 1
2190          || pendingAssignQueue.size() >= assignDispatchWaitQueueMaxSize
2191      ) {
2192        assignQueueFullCond.signal();
2193      }
2194    } finally {
2195      assignQueueLock.unlock();
2196    }
2197  }
2198
2199  private void startAssignmentThread() {
2200    assignThread = new Thread(master.getServerName().toShortString()) {
2201      @Override
2202      public void run() {
2203        while (isRunning()) {
2204          processAssignQueue();
2205        }
2206        pendingAssignQueue.clear();
2207      }
2208    };
2209    assignThread.setDaemon(true);
2210    assignThread.start();
2211  }
2212
2213  private void stopAssignmentThread() {
2214    assignQueueSignal();
2215    try {
2216      while (assignThread.isAlive()) {
2217        assignQueueSignal();
2218        assignThread.join(250);
2219      }
2220    } catch (InterruptedException e) {
2221      LOG.warn("join interrupted", e);
2222      Thread.currentThread().interrupt();
2223    }
2224  }
2225
2226  private void assignQueueSignal() {
2227    assignQueueLock.lock();
2228    try {
2229      assignQueueFullCond.signal();
2230    } finally {
2231      assignQueueLock.unlock();
2232    }
2233  }
2234
2235  @edu.umd.cs.findbugs.annotations.SuppressWarnings("WA_AWAIT_NOT_IN_LOOP")
2236  private HashMap<RegionInfo, RegionStateNode> waitOnAssignQueue() {
2237    HashMap<RegionInfo, RegionStateNode> regions = null;
2238
2239    assignQueueLock.lock();
2240    try {
2241      if (pendingAssignQueue.isEmpty() && isRunning()) {
2242        assignQueueFullCond.await();
2243      }
2244
2245      if (!isRunning()) {
2246        return null;
2247      }
2248      assignQueueFullCond.await(assignDispatchWaitMillis, TimeUnit.MILLISECONDS);
2249      regions = new HashMap<RegionInfo, RegionStateNode>(pendingAssignQueue.size());
2250      for (RegionStateNode regionNode : pendingAssignQueue) {
2251        regions.put(regionNode.getRegionInfo(), regionNode);
2252      }
2253      pendingAssignQueue.clear();
2254    } catch (InterruptedException e) {
2255      LOG.warn("got interrupted ", e);
2256      Thread.currentThread().interrupt();
2257    } finally {
2258      assignQueueLock.unlock();
2259    }
2260    return regions;
2261  }
2262
2263  private void processAssignQueue() {
2264    final HashMap<RegionInfo, RegionStateNode> regions = waitOnAssignQueue();
2265    if (regions == null || regions.size() == 0 || !isRunning()) {
2266      return;
2267    }
2268
2269    if (LOG.isTraceEnabled()) {
2270      LOG.trace("PROCESS ASSIGN QUEUE regionCount=" + regions.size());
2271    }
2272
2273    // TODO: Optimize balancer. pass a RegionPlan?
2274    final HashMap<RegionInfo, ServerName> retainMap = new HashMap<>();
2275    final List<RegionInfo> userHRIs = new ArrayList<>(regions.size());
2276    // Regions for system tables requiring reassignment
2277    final List<RegionInfo> systemHRIs = new ArrayList<>();
2278    for (RegionStateNode regionStateNode : regions.values()) {
2279      boolean sysTable = regionStateNode.isSystemTable();
2280      final List<RegionInfo> hris = sysTable ? systemHRIs : userHRIs;
2281      if (regionStateNode.getRegionLocation() != null) {
2282        retainMap.put(regionStateNode.getRegionInfo(), regionStateNode.getRegionLocation());
2283      } else {
2284        hris.add(regionStateNode.getRegionInfo());
2285      }
2286    }
2287
2288    // TODO: connect with the listener to invalidate the cache
2289
2290    // TODO use events
2291    List<ServerName> servers = master.getServerManager().createDestinationServersList();
2292    for (int i = 0; servers.size() < 1; ++i) {
2293      // Report every fourth time around this loop; try not to flood log.
2294      if (i % 4 == 0) {
2295        LOG.warn("No servers available; cannot place " + regions.size() + " unassigned regions.");
2296      }
2297
2298      if (!isRunning()) {
2299        LOG.debug("Stopped! Dropping assign of " + regions.size() + " queued regions.");
2300        return;
2301      }
2302      Threads.sleep(250);
2303      servers = master.getServerManager().createDestinationServersList();
2304    }
2305
2306    if (!systemHRIs.isEmpty()) {
2307      // System table regions requiring reassignment are present, get region servers
2308      // not available for system table regions
2309      final List<ServerName> excludeServers = getExcludedServersForSystemTable();
2310      List<ServerName> serversForSysTables =
2311        servers.stream().filter(s -> !excludeServers.contains(s)).collect(Collectors.toList());
2312      if (serversForSysTables.isEmpty()) {
2313        LOG.warn("Filtering old server versions and the excluded produced an empty set; "
2314          + "instead considering all candidate servers!");
2315      }
2316      LOG.debug("Processing assignQueue; systemServersCount=" + serversForSysTables.size()
2317        + ", allServersCount=" + servers.size());
2318      processAssignmentPlans(regions, null, systemHRIs,
2319        serversForSysTables.isEmpty() && !containsBogusAssignments(regions, systemHRIs)
2320          ? servers
2321          : serversForSysTables);
2322    }
2323
2324    processAssignmentPlans(regions, retainMap, userHRIs, servers);
2325  }
2326
2327  private boolean containsBogusAssignments(Map<RegionInfo, RegionStateNode> regions,
2328    List<RegionInfo> hirs) {
2329    for (RegionInfo ri : hirs) {
2330      if (
2331        regions.get(ri).getRegionLocation() != null
2332          && regions.get(ri).getRegionLocation().equals(LoadBalancer.BOGUS_SERVER_NAME)
2333      ) {
2334        return true;
2335      }
2336    }
2337    return false;
2338  }
2339
2340  private void processAssignmentPlans(final HashMap<RegionInfo, RegionStateNode> regions,
2341    final HashMap<RegionInfo, ServerName> retainMap, final List<RegionInfo> hris,
2342    final List<ServerName> servers) {
2343    boolean isTraceEnabled = LOG.isTraceEnabled();
2344    if (isTraceEnabled) {
2345      LOG.trace("Available servers count=" + servers.size() + ": " + servers);
2346    }
2347
2348    final LoadBalancer balancer = getBalancer();
2349    // ask the balancer where to place regions
2350    if (retainMap != null && !retainMap.isEmpty()) {
2351      if (isTraceEnabled) {
2352        LOG.trace("retain assign regions=" + retainMap);
2353      }
2354      try {
2355        acceptPlan(regions, balancer.retainAssignment(retainMap, servers));
2356      } catch (IOException e) {
2357        LOG.warn("unable to retain assignment", e);
2358        addToPendingAssignment(regions, retainMap.keySet());
2359      }
2360    }
2361
2362    // TODO: Do we need to split retain and round-robin?
2363    // the retain seems to fallback to round-robin/random if the region is not in the map.
2364    if (!hris.isEmpty()) {
2365      Collections.sort(hris, RegionInfo.COMPARATOR);
2366      if (isTraceEnabled) {
2367        LOG.trace("round robin regions=" + hris);
2368      }
2369      try {
2370        acceptPlan(regions, balancer.roundRobinAssignment(hris, servers));
2371      } catch (IOException e) {
2372        LOG.warn("unable to round-robin assignment", e);
2373        addToPendingAssignment(regions, hris);
2374      }
2375    }
2376  }
2377
2378  private void acceptPlan(final HashMap<RegionInfo, RegionStateNode> regions,
2379    final Map<ServerName, List<RegionInfo>> plan) throws HBaseIOException {
2380    final ProcedureEvent<?>[] events = new ProcedureEvent[regions.size()];
2381    final long st = EnvironmentEdgeManager.currentTime();
2382
2383    if (plan.isEmpty()) {
2384      throw new HBaseIOException("unable to compute plans for regions=" + regions.size());
2385    }
2386
2387    int evcount = 0;
2388    for (Map.Entry<ServerName, List<RegionInfo>> entry : plan.entrySet()) {
2389      final ServerName server = entry.getKey();
2390      for (RegionInfo hri : entry.getValue()) {
2391        final RegionStateNode regionNode = regions.get(hri);
2392        regionNode.setRegionLocation(server);
2393        if (server.equals(LoadBalancer.BOGUS_SERVER_NAME) && regionNode.isSystemTable()) {
2394          assignQueueLock.lock();
2395          try {
2396            pendingAssignQueue.add(regionNode);
2397          } finally {
2398            assignQueueLock.unlock();
2399          }
2400        } else {
2401          events[evcount++] = regionNode.getProcedureEvent();
2402        }
2403      }
2404    }
2405    ProcedureEvent.wakeEvents(getProcedureScheduler(), events);
2406
2407    final long et = EnvironmentEdgeManager.currentTime();
2408    if (LOG.isTraceEnabled()) {
2409      LOG.trace("ASSIGN ACCEPT " + events.length + " -> " + StringUtils.humanTimeDiff(et - st));
2410    }
2411  }
2412
2413  private void addToPendingAssignment(final HashMap<RegionInfo, RegionStateNode> regions,
2414    final Collection<RegionInfo> pendingRegions) {
2415    assignQueueLock.lock();
2416    try {
2417      for (RegionInfo hri : pendingRegions) {
2418        pendingAssignQueue.add(regions.get(hri));
2419      }
2420    } finally {
2421      assignQueueLock.unlock();
2422    }
2423  }
2424
2425  /**
2426   * For a given cluster with mixed versions of servers, get a list of servers with lower versions,
2427   * where system table regions should not be assigned to. For system table, we must assign regions
2428   * to a server with highest version. However, we can disable this exclusion using config:
2429   * "hbase.min.version.move.system.tables" if checkForMinVersion is true. Detailed explanation
2430   * available with definition of minVersionToMoveSysTables.
2431   * @return List of Excluded servers for System table regions.
2432   */
2433  public List<ServerName> getExcludedServersForSystemTable() {
2434    // TODO: This should be a cached list kept by the ServerManager rather than calculated on each
2435    // move or system region assign. The RegionServerTracker keeps list of online Servers with
2436    // RegionServerInfo that includes Version.
2437    List<Pair<ServerName, String>> serverList =
2438      master.getServerManager().getOnlineServersList().stream()
2439        .map(s -> new Pair<>(s, master.getRegionServerVersion(s))).collect(Collectors.toList());
2440    if (serverList.isEmpty()) {
2441      return new ArrayList<>();
2442    }
2443    String highestVersion = Collections
2444      .max(serverList, (o1, o2) -> VersionInfo.compareVersion(o1.getSecond(), o2.getSecond()))
2445      .getSecond();
2446    if (!DEFAULT_MIN_VERSION_MOVE_SYS_TABLES_CONFIG.equals(minVersionToMoveSysTables)) {
2447      int comparedValue = VersionInfo.compareVersion(minVersionToMoveSysTables, highestVersion);
2448      if (comparedValue > 0) {
2449        return new ArrayList<>();
2450      }
2451    }
2452    return serverList.stream().filter(pair -> !pair.getSecond().equals(highestVersion))
2453      .map(Pair::getFirst).collect(Collectors.toList());
2454  }
2455
2456  MasterServices getMaster() {
2457    return master;
2458  }
2459
2460  /** Returns a snapshot of rsReports */
2461  public Map<ServerName, Set<byte[]>> getRSReports() {
2462    Map<ServerName, Set<byte[]>> rsReportsSnapshot = new HashMap<>();
2463    synchronized (rsReports) {
2464      rsReports.entrySet().forEach(e -> rsReportsSnapshot.put(e.getKey(), e.getValue()));
2465    }
2466    return rsReportsSnapshot;
2467  }
2468
2469  /**
2470   * Provide regions state count for given table. e.g howmany regions of give table are
2471   * opened/closed/rit etc
2472   * @param tableName TableName
2473   * @return region states count
2474   */
2475  public RegionStatesCount getRegionStatesCount(TableName tableName) {
2476    int openRegionsCount = 0;
2477    int closedRegionCount = 0;
2478    int ritCount = 0;
2479    int splitRegionCount = 0;
2480    int totalRegionCount = 0;
2481    if (!isTableDisabled(tableName)) {
2482      final List<RegionState> states = regionStates.getTableRegionStates(tableName);
2483      for (RegionState regionState : states) {
2484        if (regionState.isOpened()) {
2485          openRegionsCount++;
2486        } else if (regionState.isClosed()) {
2487          closedRegionCount++;
2488        } else if (regionState.isSplit()) {
2489          splitRegionCount++;
2490        }
2491      }
2492      totalRegionCount = states.size();
2493      ritCount = totalRegionCount - openRegionsCount - splitRegionCount;
2494    }
2495    return new RegionStatesCount.RegionStatesCountBuilder().setOpenRegions(openRegionsCount)
2496      .setClosedRegions(closedRegionCount).setSplitRegions(splitRegionCount)
2497      .setRegionsInTransition(ritCount).setTotalRegions(totalRegionCount).build();
2498  }
2499
2500}