001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master;
019
020import static org.apache.hadoop.hbase.HConstants.DEFAULT_HBASE_SPLIT_COORDINATED_BY_ZK;
021import static org.apache.hadoop.hbase.HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS;
022import static org.apache.hadoop.hbase.HConstants.HBASE_SPLIT_WAL_COORDINATED_BY_ZK;
023import static org.apache.hadoop.hbase.util.DNS.MASTER_HOSTNAME_KEY;
024
025import java.io.IOException;
026import java.io.InterruptedIOException;
027import java.lang.reflect.Constructor;
028import java.lang.reflect.InvocationTargetException;
029import java.net.InetAddress;
030import java.net.InetSocketAddress;
031import java.net.UnknownHostException;
032import java.time.Duration;
033import java.util.ArrayList;
034import java.util.Arrays;
035import java.util.Collection;
036import java.util.Collections;
037import java.util.Comparator;
038import java.util.EnumSet;
039import java.util.HashMap;
040import java.util.Iterator;
041import java.util.LinkedList;
042import java.util.List;
043import java.util.Map;
044import java.util.Objects;
045import java.util.Optional;
046import java.util.Set;
047import java.util.concurrent.ExecutionException;
048import java.util.concurrent.Future;
049import java.util.concurrent.TimeUnit;
050import java.util.concurrent.TimeoutException;
051import java.util.concurrent.atomic.AtomicInteger;
052import java.util.concurrent.locks.ReentrantLock;
053import java.util.regex.Pattern;
054import java.util.stream.Collectors;
055import javax.servlet.ServletException;
056import javax.servlet.http.HttpServlet;
057import javax.servlet.http.HttpServletRequest;
058import javax.servlet.http.HttpServletResponse;
059import org.apache.commons.io.IOUtils;
060import org.apache.commons.lang3.StringUtils;
061import org.apache.hadoop.conf.Configuration;
062import org.apache.hadoop.fs.FSDataOutputStream;
063import org.apache.hadoop.fs.Path;
064import org.apache.hadoop.hbase.ChoreService;
065import org.apache.hadoop.hbase.ClusterId;
066import org.apache.hadoop.hbase.ClusterMetrics;
067import org.apache.hadoop.hbase.ClusterMetrics.Option;
068import org.apache.hadoop.hbase.ClusterMetricsBuilder;
069import org.apache.hadoop.hbase.DoNotRetryIOException;
070import org.apache.hadoop.hbase.HBaseIOException;
071import org.apache.hadoop.hbase.HBaseInterfaceAudience;
072import org.apache.hadoop.hbase.HConstants;
073import org.apache.hadoop.hbase.InvalidFamilyOperationException;
074import org.apache.hadoop.hbase.MasterNotRunningException;
075import org.apache.hadoop.hbase.MetaTableAccessor;
076import org.apache.hadoop.hbase.NamespaceDescriptor;
077import org.apache.hadoop.hbase.PleaseHoldException;
078import org.apache.hadoop.hbase.ReplicationPeerNotFoundException;
079import org.apache.hadoop.hbase.ServerName;
080import org.apache.hadoop.hbase.TableName;
081import org.apache.hadoop.hbase.TableNotDisabledException;
082import org.apache.hadoop.hbase.TableNotFoundException;
083import org.apache.hadoop.hbase.UnknownRegionException;
084import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
085import org.apache.hadoop.hbase.client.MasterSwitchType;
086import org.apache.hadoop.hbase.client.NormalizeTableFilterParams;
087import org.apache.hadoop.hbase.client.RegionInfo;
088import org.apache.hadoop.hbase.client.RegionInfoBuilder;
089import org.apache.hadoop.hbase.client.RegionStatesCount;
090import org.apache.hadoop.hbase.client.TableDescriptor;
091import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
092import org.apache.hadoop.hbase.client.TableState;
093import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
094import org.apache.hadoop.hbase.executor.ExecutorType;
095import org.apache.hadoop.hbase.favored.FavoredNodesManager;
096import org.apache.hadoop.hbase.http.InfoServer;
097import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils;
098import org.apache.hadoop.hbase.ipc.RpcServer;
099import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
100import org.apache.hadoop.hbase.log.HBaseMarkers;
101import org.apache.hadoop.hbase.master.MasterRpcServices.BalanceSwitchMode;
102import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
103import org.apache.hadoop.hbase.master.assignment.MergeTableRegionsProcedure;
104import org.apache.hadoop.hbase.master.assignment.RegionStateNode;
105import org.apache.hadoop.hbase.master.assignment.RegionStates;
106import org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure;
107import org.apache.hadoop.hbase.master.balancer.BalancerChore;
108import org.apache.hadoop.hbase.master.balancer.ClusterStatusChore;
109import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory;
110import org.apache.hadoop.hbase.master.cleaner.DirScanPool;
111import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
112import org.apache.hadoop.hbase.master.cleaner.LogCleaner;
113import org.apache.hadoop.hbase.master.cleaner.ReplicationBarrierCleaner;
114import org.apache.hadoop.hbase.master.cleaner.SnapshotCleanerChore;
115import org.apache.hadoop.hbase.master.janitor.CatalogJanitor;
116import org.apache.hadoop.hbase.master.locking.LockManager;
117import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan;
118import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan.PlanType;
119import org.apache.hadoop.hbase.master.normalizer.RegionNormalizer;
120import org.apache.hadoop.hbase.master.normalizer.RegionNormalizerChore;
121import org.apache.hadoop.hbase.master.normalizer.RegionNormalizerFactory;
122import org.apache.hadoop.hbase.master.procedure.CreateTableProcedure;
123import org.apache.hadoop.hbase.master.procedure.DeleteNamespaceProcedure;
124import org.apache.hadoop.hbase.master.procedure.DeleteTableProcedure;
125import org.apache.hadoop.hbase.master.procedure.DisableTableProcedure;
126import org.apache.hadoop.hbase.master.procedure.EnableTableProcedure;
127import org.apache.hadoop.hbase.master.procedure.InitMetaProcedure;
128import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants;
129import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
130import org.apache.hadoop.hbase.master.procedure.MasterProcedureScheduler;
131import org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil;
132import org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.NonceProcedureRunnable;
133import org.apache.hadoop.hbase.master.procedure.ModifyTableProcedure;
134import org.apache.hadoop.hbase.master.procedure.ProcedurePrepareLatch;
135import org.apache.hadoop.hbase.master.procedure.ProcedureSyncWait;
136import org.apache.hadoop.hbase.master.procedure.ReopenTableRegionsProcedure;
137import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure;
138import org.apache.hadoop.hbase.master.procedure.TruncateTableProcedure;
139import org.apache.hadoop.hbase.master.region.MasterRegion;
140import org.apache.hadoop.hbase.master.region.MasterRegionFactory;
141import org.apache.hadoop.hbase.master.replication.AbstractPeerProcedure;
142import org.apache.hadoop.hbase.master.replication.AddPeerProcedure;
143import org.apache.hadoop.hbase.master.replication.DisablePeerProcedure;
144import org.apache.hadoop.hbase.master.replication.EnablePeerProcedure;
145import org.apache.hadoop.hbase.master.replication.RemovePeerProcedure;
146import org.apache.hadoop.hbase.master.replication.ReplicationPeerManager;
147import org.apache.hadoop.hbase.master.replication.SyncReplicationReplayWALManager;
148import org.apache.hadoop.hbase.master.replication.TransitPeerSyncReplicationStateProcedure;
149import org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure;
150import org.apache.hadoop.hbase.master.slowlog.SlowLogMasterService;
151import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
152import org.apache.hadoop.hbase.master.zksyncer.MasterAddressSyncer;
153import org.apache.hadoop.hbase.master.zksyncer.MetaLocationSyncer;
154import org.apache.hadoop.hbase.mob.MobFileCleanerChore;
155import org.apache.hadoop.hbase.mob.MobFileCompactionChore;
156import org.apache.hadoop.hbase.monitoring.MemoryBoundedLogMessageBuffer;
157import org.apache.hadoop.hbase.monitoring.MonitoredTask;
158import org.apache.hadoop.hbase.monitoring.TaskMonitor;
159import org.apache.hadoop.hbase.procedure.MasterProcedureManagerHost;
160import org.apache.hadoop.hbase.procedure.flush.MasterFlushTableProcedureManager;
161import org.apache.hadoop.hbase.procedure2.LockedResource;
162import org.apache.hadoop.hbase.procedure2.Procedure;
163import org.apache.hadoop.hbase.procedure2.ProcedureEvent;
164import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
165import org.apache.hadoop.hbase.procedure2.RemoteProcedureDispatcher.RemoteProcedure;
166import org.apache.hadoop.hbase.procedure2.RemoteProcedureException;
167import org.apache.hadoop.hbase.procedure2.store.ProcedureStore;
168import org.apache.hadoop.hbase.procedure2.store.ProcedureStore.ProcedureStoreListener;
169import org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore;
170import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
171import org.apache.hadoop.hbase.quotas.MasterQuotasObserver;
172import org.apache.hadoop.hbase.quotas.QuotaObserverChore;
173import org.apache.hadoop.hbase.quotas.QuotaTableUtil;
174import org.apache.hadoop.hbase.quotas.QuotaUtil;
175import org.apache.hadoop.hbase.quotas.SnapshotQuotaObserverChore;
176import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot;
177import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus;
178import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshotNotifier;
179import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshotNotifierFactory;
180import org.apache.hadoop.hbase.quotas.SpaceViolationPolicy;
181import org.apache.hadoop.hbase.regionserver.HRegionServer;
182import org.apache.hadoop.hbase.regionserver.RSRpcServices;
183import org.apache.hadoop.hbase.replication.ReplicationException;
184import org.apache.hadoop.hbase.replication.ReplicationLoadSource;
185import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
186import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
187import org.apache.hadoop.hbase.replication.ReplicationUtils;
188import org.apache.hadoop.hbase.replication.SyncReplicationState;
189import org.apache.hadoop.hbase.replication.master.ReplicationHFileCleaner;
190import org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner;
191import org.apache.hadoop.hbase.replication.regionserver.ReplicationStatus;
192import org.apache.hadoop.hbase.rsgroup.RSGroupAdminEndpoint;
193import org.apache.hadoop.hbase.rsgroup.RSGroupBasedLoadBalancer;
194import org.apache.hadoop.hbase.rsgroup.RSGroupInfoManager;
195import org.apache.hadoop.hbase.rsgroup.RSGroupUtil;
196import org.apache.hadoop.hbase.security.AccessDeniedException;
197import org.apache.hadoop.hbase.security.SecurityConstants;
198import org.apache.hadoop.hbase.security.UserProvider;
199import org.apache.hadoop.hbase.trace.TraceUtil;
200import org.apache.hadoop.hbase.util.Addressing;
201import org.apache.hadoop.hbase.util.Bytes;
202import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
203import org.apache.hadoop.hbase.util.FutureUtils;
204import org.apache.hadoop.hbase.util.HBaseFsck;
205import org.apache.hadoop.hbase.util.HFileArchiveUtil;
206import org.apache.hadoop.hbase.util.IdLock;
207import org.apache.hadoop.hbase.util.ModifyRegionUtils;
208import org.apache.hadoop.hbase.util.Pair;
209import org.apache.hadoop.hbase.util.RetryCounter;
210import org.apache.hadoop.hbase.util.RetryCounterFactory;
211import org.apache.hadoop.hbase.util.TableDescriptorChecker;
212import org.apache.hadoop.hbase.util.Threads;
213import org.apache.hadoop.hbase.util.VersionInfo;
214import org.apache.hadoop.hbase.zookeeper.LoadBalancerTracker;
215import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker;
216import org.apache.hadoop.hbase.zookeeper.RegionNormalizerTracker;
217import org.apache.hadoop.hbase.zookeeper.SnapshotCleanupTracker;
218import org.apache.hadoop.hbase.zookeeper.ZKClusterId;
219import org.apache.hadoop.hbase.zookeeper.ZKUtil;
220import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
221import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
222import org.apache.yetus.audience.InterfaceAudience;
223import org.apache.zookeeper.KeeperException;
224import org.slf4j.Logger;
225import org.slf4j.LoggerFactory;
226
227import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
228import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
229import org.apache.hbase.thirdparty.com.google.common.collect.Maps;
230import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
231import org.apache.hbase.thirdparty.com.google.protobuf.Descriptors;
232import org.apache.hbase.thirdparty.com.google.protobuf.Service;
233import org.apache.hbase.thirdparty.org.apache.commons.collections4.CollectionUtils;
234import org.apache.hbase.thirdparty.org.eclipse.jetty.server.Server;
235import org.apache.hbase.thirdparty.org.eclipse.jetty.server.ServerConnector;
236import org.apache.hbase.thirdparty.org.eclipse.jetty.servlet.ServletHolder;
237import org.apache.hbase.thirdparty.org.eclipse.jetty.webapp.WebAppContext;
238
239import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter;
240import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState;
241import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription;
242
243/**
244 * HMaster is the "master server" for HBase. An HBase cluster has one active
245 * master.  If many masters are started, all compete.  Whichever wins goes on to
246 * run the cluster.  All others park themselves in their constructor until
247 * master or cluster shutdown or until the active master loses its lease in
248 * zookeeper.  Thereafter, all running master jostle to take over master role.
249 *
250 * <p>The Master can be asked shutdown the cluster. See {@link #shutdown()}.  In
251 * this case it will tell all regionservers to go down and then wait on them
252 * all reporting in that they are down.  This master will then shut itself down.
253 *
254 * <p>You can also shutdown just this master.  Call {@link #stopMaster()}.
255 *
256 * @see org.apache.zookeeper.Watcher
257 */
258@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
259@SuppressWarnings("deprecation")
260public class HMaster extends HRegionServer implements MasterServices {
261  private static Logger LOG = LoggerFactory.getLogger(HMaster.class);
262
263  /**
264   * Protection against zombie master. Started once Master accepts active responsibility and
265   * starts taking over responsibilities. Allows a finite time window before giving up ownership.
266   */
267  private static class InitializationMonitor extends Thread {
268    /** The amount of time in milliseconds to sleep before checking initialization status. */
269    public static final String TIMEOUT_KEY = "hbase.master.initializationmonitor.timeout";
270    public static final long TIMEOUT_DEFAULT = TimeUnit.MILLISECONDS.convert(15, TimeUnit.MINUTES);
271
272    /**
273     * When timeout expired and initialization has not complete, call {@link System#exit(int)} when
274     * true, do nothing otherwise.
275     */
276    public static final String HALT_KEY = "hbase.master.initializationmonitor.haltontimeout";
277    public static final boolean HALT_DEFAULT = false;
278
279    private final HMaster master;
280    private final long timeout;
281    private final boolean haltOnTimeout;
282
283    /** Creates a Thread that monitors the {@link #isInitialized()} state. */
284    InitializationMonitor(HMaster master) {
285      super("MasterInitializationMonitor");
286      this.master = master;
287      this.timeout = master.getConfiguration().getLong(TIMEOUT_KEY, TIMEOUT_DEFAULT);
288      this.haltOnTimeout = master.getConfiguration().getBoolean(HALT_KEY, HALT_DEFAULT);
289      this.setDaemon(true);
290    }
291
292    @Override
293    public void run() {
294      try {
295        while (!master.isStopped() && master.isActiveMaster()) {
296          Thread.sleep(timeout);
297          if (master.isInitialized()) {
298            LOG.debug("Initialization completed within allotted tolerance. Monitor exiting.");
299          } else {
300            LOG.error("Master failed to complete initialization after " + timeout + "ms. Please"
301                + " consider submitting a bug report including a thread dump of this process.");
302            if (haltOnTimeout) {
303              LOG.error("Zombie Master exiting. Thread dump to stdout");
304              Threads.printThreadInfo(System.out, "Zombie HMaster");
305              System.exit(-1);
306            }
307          }
308        }
309      } catch (InterruptedException ie) {
310        LOG.trace("InitMonitor thread interrupted. Existing.");
311      }
312    }
313  }
314
315  // MASTER is name of the webapp and the attribute name used stuffing this
316  //instance into web context.
317  public static final String MASTER = "master";
318
319  // Manager and zk listener for master election
320  private final ActiveMasterManager activeMasterManager;
321  // Region server tracker
322  private RegionServerTracker regionServerTracker;
323  // Draining region server tracker
324  private DrainingServerTracker drainingServerTracker;
325  // Tracker for load balancer state
326  LoadBalancerTracker loadBalancerTracker;
327  // Tracker for meta location, if any client ZK quorum specified
328  MetaLocationSyncer metaLocationSyncer;
329  // Tracker for active master location, if any client ZK quorum specified
330  MasterAddressSyncer masterAddressSyncer;
331  // Tracker for auto snapshot cleanup state
332  SnapshotCleanupTracker snapshotCleanupTracker;
333
334  // Tracker for split and merge state
335  private SplitOrMergeTracker splitOrMergeTracker;
336
337  // Tracker for region normalizer state
338  private RegionNormalizerTracker regionNormalizerTracker;
339
340  private ClusterSchemaService clusterSchemaService;
341
342  public static final String HBASE_MASTER_WAIT_ON_SERVICE_IN_SECONDS =
343    "hbase.master.wait.on.service.seconds";
344  public static final int DEFAULT_HBASE_MASTER_WAIT_ON_SERVICE_IN_SECONDS = 5 * 60;
345
346  public static final String HBASE_MASTER_CLEANER_INTERVAL = "hbase.master.cleaner.interval";
347
348  public static final int DEFAULT_HBASE_MASTER_CLEANER_INTERVAL = 600 * 1000;
349
350  // Metrics for the HMaster
351  final MetricsMaster metricsMaster;
352  // file system manager for the master FS operations
353  private MasterFileSystem fileSystemManager;
354  private MasterWalManager walManager;
355
356  // manager to manage procedure-based WAL splitting, can be null if current
357  // is zk-based WAL splitting. SplitWALManager will replace SplitLogManager
358  // and MasterWalManager, which means zk-based WAL splitting code will be
359  // useless after we switch to the procedure-based one. our eventual goal
360  // is to remove all the zk-based WAL splitting code.
361  private SplitWALManager splitWALManager;
362
363  // server manager to deal with region server info
364  private volatile ServerManager serverManager;
365
366  // manager of assignment nodes in zookeeper
367  private AssignmentManager assignmentManager;
368
369
370  /**
371   * Cache for the meta region replica's locations. Also tracks their changes to avoid stale
372   * cache entries.
373   */
374  private final MetaRegionLocationCache metaRegionLocationCache;
375
376  private RSGroupInfoManager rsGroupInfoManager;
377
378  // manager of replication
379  private ReplicationPeerManager replicationPeerManager;
380
381  private SyncReplicationReplayWALManager syncReplicationReplayWALManager;
382
383  // buffer for "fatal error" notices from region servers
384  // in the cluster. This is only used for assisting
385  // operations/debugging.
386  MemoryBoundedLogMessageBuffer rsFatals;
387
388  // flag set after we become the active master (used for testing)
389  private volatile boolean activeMaster = false;
390
391  // flag set after we complete initialization once active
392  private final ProcedureEvent<?> initialized = new ProcedureEvent<>("master initialized");
393
394  // flag set after master services are started,
395  // initialization may have not completed yet.
396  volatile boolean serviceStarted = false;
397
398  // Maximum time we should run balancer for
399  private final int maxBalancingTime;
400  // Maximum percent of regions in transition when balancing
401  private final double maxRitPercent;
402
403  private final LockManager lockManager = new LockManager(this);
404
405  private RSGroupBasedLoadBalancer balancer;
406  // a lock to prevent concurrent normalization actions.
407  private final ReentrantLock normalizationInProgressLock = new ReentrantLock();
408  private RegionNormalizer normalizer;
409  private BalancerChore balancerChore;
410  private RegionNormalizerChore normalizerChore;
411  private ClusterStatusChore clusterStatusChore;
412  private ClusterStatusPublisher clusterStatusPublisherChore = null;
413  private SnapshotCleanerChore snapshotCleanerChore = null;
414
415  private HbckChore hbckChore;
416  CatalogJanitor catalogJanitorChore;
417  private DirScanPool cleanerPool;
418  private LogCleaner logCleaner;
419  private HFileCleaner hfileCleaner;
420  private ReplicationBarrierCleaner replicationBarrierCleaner;
421  private MobFileCleanerChore mobFileCleanerChore;
422  private MobFileCompactionChore mobFileCompactionChore;
423  // used to synchronize the mobCompactionStates
424  private final IdLock mobCompactionLock = new IdLock();
425  // save the information of mob compactions in tables.
426  // the key is table name, the value is the number of compactions in that table.
427  private Map<TableName, AtomicInteger> mobCompactionStates = Maps.newConcurrentMap();
428
429  MasterCoprocessorHost cpHost;
430
431  private final boolean preLoadTableDescriptors;
432
433  // Time stamps for when a hmaster became active
434  private long masterActiveTime;
435
436  // Time stamp for when HMaster finishes becoming Active Master
437  private long masterFinishedInitializationTime;
438
439  Map<String, Service> coprocessorServiceHandlers = Maps.newHashMap();
440
441  // monitor for snapshot of hbase tables
442  SnapshotManager snapshotManager;
443  // monitor for distributed procedures
444  private MasterProcedureManagerHost mpmHost;
445
446  private RegionsRecoveryChore regionsRecoveryChore = null;
447
448  private RegionsRecoveryConfigManager regionsRecoveryConfigManager = null;
449  // it is assigned after 'initialized' guard set to true, so should be volatile
450  private volatile MasterQuotaManager quotaManager;
451  private SpaceQuotaSnapshotNotifier spaceQuotaSnapshotNotifier;
452  private QuotaObserverChore quotaObserverChore;
453  private SnapshotQuotaObserverChore snapshotQuotaChore;
454
455  private ProcedureExecutor<MasterProcedureEnv> procedureExecutor;
456  private ProcedureStore procedureStore;
457
458  // the master local storage to store procedure data, etc.
459  private MasterRegion masterRegion;
460
461  // handle table states
462  private TableStateManager tableStateManager;
463
464  private long splitPlanCount;
465  private long mergePlanCount;
466
467  /** jetty server for master to redirect requests to regionserver infoServer */
468  private Server masterJettyServer;
469
470  // Determine if we should do normal startup or minimal "single-user" mode with no region
471  // servers and no user tables. Useful for repair and recovery of hbase:meta
472  private final boolean maintenanceMode;
473  static final String MAINTENANCE_MODE = "hbase.master.maintenance_mode";
474
475  // Cached clusterId on stand by masters to serve clusterID requests from clients.
476  private final CachedClusterId cachedClusterId;
477
478  public static class RedirectServlet extends HttpServlet {
479    private static final long serialVersionUID = 2894774810058302473L;
480    private final int regionServerInfoPort;
481    private final String regionServerHostname;
482
483    /**
484     * @param infoServer that we're trying to send all requests to
485     * @param hostname may be null. if given, will be used for redirects instead of host from client.
486     */
487    public RedirectServlet(InfoServer infoServer, String hostname) {
488       regionServerInfoPort = infoServer.getPort();
489       regionServerHostname = hostname;
490    }
491
492    @Override
493    public void doGet(HttpServletRequest request,
494        HttpServletResponse response) throws ServletException, IOException {
495      String redirectHost = regionServerHostname;
496      if(redirectHost == null) {
497        redirectHost = request.getServerName();
498        if(!Addressing.isLocalAddress(InetAddress.getByName(redirectHost))) {
499          LOG.warn("Couldn't resolve '" + redirectHost + "' as an address local to this node and '" +
500              MASTER_HOSTNAME_KEY + "' is not set; client will get an HTTP 400 response. If " +
501              "your HBase deployment relies on client accessible names that the region server process " +
502              "can't resolve locally, then you should set the previously mentioned configuration variable " +
503              "to an appropriate hostname.");
504          // no sending client provided input back to the client, so the goal host is just in the logs.
505          response.sendError(400, "Request was to a host that I can't resolve for any of the network interfaces on " +
506              "this node. If this is due to an intermediary such as an HTTP load balancer or other proxy, your HBase " +
507              "administrator can set '" + MASTER_HOSTNAME_KEY + "' to point to the correct hostname.");
508          return;
509        }
510      }
511      // TODO this scheme should come from looking at the scheme registered in the infoserver's http server for the
512      // host and port we're using, but it's buried way too deep to do that ATM.
513      String redirectUrl = request.getScheme() + "://"
514        + redirectHost + ":" + regionServerInfoPort
515        + request.getRequestURI();
516      response.sendRedirect(redirectUrl);
517    }
518  }
519
520  /**
521   * Initializes the HMaster. The steps are as follows:
522   * <p>
523   * <ol>
524   * <li>Initialize the local HRegionServer
525   * <li>Start the ActiveMasterManager.
526   * </ol>
527   * <p>
528   * Remaining steps of initialization occur in
529   * {@link #finishActiveMasterInitialization(MonitoredTask)} after the master becomes the
530   * active one.
531   */
532  public HMaster(final Configuration conf) throws IOException {
533    super(conf);
534    TraceUtil.initTracer(conf);
535    try {
536      if (conf.getBoolean(MAINTENANCE_MODE, false)) {
537        LOG.info("Detected {}=true via configuration.", MAINTENANCE_MODE);
538        maintenanceMode = true;
539      } else if (Boolean.getBoolean(MAINTENANCE_MODE)) {
540        LOG.info("Detected {}=true via environment variables.", MAINTENANCE_MODE);
541        maintenanceMode = true;
542      } else {
543        maintenanceMode = false;
544      }
545      this.rsFatals = new MemoryBoundedLogMessageBuffer(
546          conf.getLong("hbase.master.buffer.for.rs.fatals", 1 * 1024 * 1024));
547      LOG.info("hbase.rootdir={}, hbase.cluster.distributed={}", getDataRootDir(),
548          this.conf.getBoolean(HConstants.CLUSTER_DISTRIBUTED, false));
549
550      // Disable usage of meta replicas in the master
551      this.conf.setBoolean(HConstants.USE_META_REPLICAS, false);
552
553      decorateMasterConfiguration(this.conf);
554
555      // Hack! Maps DFSClient => Master for logs.  HDFS made this
556      // config param for task trackers, but we can piggyback off of it.
557      if (this.conf.get("mapreduce.task.attempt.id") == null) {
558        this.conf.set("mapreduce.task.attempt.id", "hb_m_" + this.serverName.toString());
559      }
560
561      this.metricsMaster = new MetricsMaster(new MetricsMasterWrapperImpl(this));
562
563      // preload table descriptor at startup
564      this.preLoadTableDescriptors = conf.getBoolean("hbase.master.preload.tabledescriptors", true);
565
566      this.maxBalancingTime = getMaxBalancingTime();
567      this.maxRitPercent = conf.getDouble(HConstants.HBASE_MASTER_BALANCER_MAX_RIT_PERCENT,
568          HConstants.DEFAULT_HBASE_MASTER_BALANCER_MAX_RIT_PERCENT);
569
570      // Do we publish the status?
571
572      boolean shouldPublish = conf.getBoolean(HConstants.STATUS_PUBLISHED,
573          HConstants.STATUS_PUBLISHED_DEFAULT);
574      Class<? extends ClusterStatusPublisher.Publisher> publisherClass =
575          conf.getClass(ClusterStatusPublisher.STATUS_PUBLISHER_CLASS,
576              ClusterStatusPublisher.DEFAULT_STATUS_PUBLISHER_CLASS,
577              ClusterStatusPublisher.Publisher.class);
578
579      if (shouldPublish) {
580        if (publisherClass == null) {
581          LOG.warn(HConstants.STATUS_PUBLISHED + " is true, but " +
582              ClusterStatusPublisher.DEFAULT_STATUS_PUBLISHER_CLASS +
583              " is not set - not publishing status");
584        } else {
585          clusterStatusPublisherChore = new ClusterStatusPublisher(this, conf, publisherClass);
586          LOG.debug("Created {}", this.clusterStatusPublisherChore);
587          getChoreService().scheduleChore(clusterStatusPublisherChore);
588        }
589      }
590
591      this.metaRegionLocationCache = new MetaRegionLocationCache(this.zooKeeper);
592      this.activeMasterManager = createActiveMasterManager(zooKeeper, serverName, this);
593
594      cachedClusterId = new CachedClusterId(this, conf);
595    } catch (Throwable t) {
596      // Make sure we log the exception. HMaster is often started via reflection and the
597      // cause of failed startup is lost.
598      LOG.error("Failed construction of Master", t);
599      throw t;
600    }
601  }
602
603  /**
604   * Protected to have custom implementations in tests override the default ActiveMaster
605   * implementation.
606   */
607  protected ActiveMasterManager createActiveMasterManager(ZKWatcher zk, ServerName sn,
608      org.apache.hadoop.hbase.Server server) throws InterruptedIOException {
609    return new ActiveMasterManager(zk, sn, server);
610  }
611
612  @Override
613  protected String getUseThisHostnameInstead(Configuration conf) {
614    return conf.get(MASTER_HOSTNAME_KEY);
615  }
616
617  // Main run loop. Calls through to the regionserver run loop AFTER becoming active Master; will
618  // block in here until then.
619  @Override
620  public void run() {
621    try {
622      Threads.setDaemonThreadRunning(new Thread(() -> {
623        try {
624          int infoPort = putUpJettyServer();
625          startActiveMasterManager(infoPort);
626        } catch (Throwable t) {
627          // Make sure we log the exception.
628          String error = "Failed to become Active Master";
629          LOG.error(error, t);
630          // Abort should have been called already.
631          if (!isAborted()) {
632            abort(error, t);
633          }
634        }
635      }), getName() + ":becomeActiveMaster");
636      // Fall in here even if we have been aborted. Need to run the shutdown services and
637      // the super run call will do this for us.
638      super.run();
639    } finally {
640      if (this.clusterSchemaService != null) {
641        // If on way out, then we are no longer active master.
642        this.clusterSchemaService.stopAsync();
643        try {
644          this.clusterSchemaService.awaitTerminated(
645              getConfiguration().getInt(HBASE_MASTER_WAIT_ON_SERVICE_IN_SECONDS,
646              DEFAULT_HBASE_MASTER_WAIT_ON_SERVICE_IN_SECONDS), TimeUnit.SECONDS);
647        } catch (TimeoutException te) {
648          LOG.warn("Failed shutdown of clusterSchemaService", te);
649        }
650      }
651      this.activeMaster = false;
652    }
653  }
654
655  // return the actual infoPort, -1 means disable info server.
656  private int putUpJettyServer() throws IOException {
657    if (!conf.getBoolean("hbase.master.infoserver.redirect", true)) {
658      return -1;
659    }
660    final int infoPort = conf.getInt("hbase.master.info.port.orig",
661      HConstants.DEFAULT_MASTER_INFOPORT);
662    // -1 is for disabling info server, so no redirecting
663    if (infoPort < 0 || infoServer == null) {
664      return -1;
665    }
666    if(infoPort == infoServer.getPort()) {
667      return infoPort;
668    }
669    final String addr = conf.get("hbase.master.info.bindAddress", "0.0.0.0");
670    if (!Addressing.isLocalAddress(InetAddress.getByName(addr))) {
671      String msg =
672          "Failed to start redirecting jetty server. Address " + addr
673              + " does not belong to this host. Correct configuration parameter: "
674              + "hbase.master.info.bindAddress";
675      LOG.error(msg);
676      throw new IOException(msg);
677    }
678
679    // TODO I'm pretty sure we could just add another binding to the InfoServer run by
680    // the RegionServer and have it run the RedirectServlet instead of standing up
681    // a second entire stack here.
682    masterJettyServer = new Server();
683    final ServerConnector connector = new ServerConnector(masterJettyServer);
684    connector.setHost(addr);
685    connector.setPort(infoPort);
686    masterJettyServer.addConnector(connector);
687    masterJettyServer.setStopAtShutdown(true);
688
689    final String redirectHostname =
690        StringUtils.isBlank(useThisHostnameInstead) ? null : useThisHostnameInstead;
691
692    final RedirectServlet redirect = new RedirectServlet(infoServer, redirectHostname);
693    final WebAppContext context = new WebAppContext(null, "/", null, null, null, null, WebAppContext.NO_SESSIONS);
694    context.addServlet(new ServletHolder(redirect), "/*");
695    context.setServer(masterJettyServer);
696
697    try {
698      masterJettyServer.start();
699    } catch (Exception e) {
700      throw new IOException("Failed to start redirecting jetty server", e);
701    }
702    return connector.getLocalPort();
703  }
704
705  /**
706   * For compatibility, if failed with regionserver credentials, try the master one
707   */
708  @Override
709  protected void login(UserProvider user, String host) throws IOException {
710    try {
711      super.login(user, host);
712    } catch (IOException ie) {
713      user.login(SecurityConstants.MASTER_KRB_KEYTAB_FILE,
714              SecurityConstants.MASTER_KRB_PRINCIPAL, host);
715    }
716  }
717
718  /**
719   * If configured to put regions on active master,
720   * wait till a backup master becomes active.
721   * Otherwise, loop till the server is stopped or aborted.
722   */
723  @Override
724  protected void waitForMasterActive(){
725    if (maintenanceMode) {
726      return;
727    }
728    boolean tablesOnMaster = LoadBalancer.isTablesOnMaster(conf);
729    while (!(tablesOnMaster && activeMaster) && !isStopped() && !isAborted()) {
730      sleeper.sleep();
731    }
732  }
733
734  @VisibleForTesting
735  public MasterRpcServices getMasterRpcServices() {
736    return (MasterRpcServices)rpcServices;
737  }
738
739  public boolean balanceSwitch(final boolean b) throws IOException {
740    return getMasterRpcServices().switchBalancer(b, BalanceSwitchMode.ASYNC);
741  }
742
743  @Override
744  protected String getProcessName() {
745    return MASTER;
746  }
747
748  @Override
749  protected boolean canCreateBaseZNode() {
750    return true;
751  }
752
753  @Override
754  protected boolean canUpdateTableDescriptor() {
755    return true;
756  }
757
758  @Override
759  protected boolean cacheTableDescriptor() {
760    return true;
761  }
762
763  @Override
764  protected RSRpcServices createRpcServices() throws IOException {
765    return new MasterRpcServices(this);
766  }
767
768  @Override
769  protected void configureInfoServer() {
770    infoServer.addUnprivilegedServlet("master-status", "/master-status", MasterStatusServlet.class);
771    infoServer.setAttribute(MASTER, this);
772    if (LoadBalancer.isTablesOnMaster(conf)) {
773      super.configureInfoServer();
774    }
775  }
776
777  @Override
778  protected Class<? extends HttpServlet> getDumpServlet() {
779    return MasterDumpServlet.class;
780  }
781
782  @Override
783  public MetricsMaster getMasterMetrics() {
784    return metricsMaster;
785  }
786
787  /**
788   * <p>
789   * Initialize all ZK based system trackers. But do not include {@link RegionServerTracker}, it
790   * should have already been initialized along with {@link ServerManager}.
791   * </p>
792   * <p>
793   * Will be overridden in tests.
794   * </p>
795   */
796  @VisibleForTesting
797  protected void initializeZKBasedSystemTrackers()
798      throws IOException, InterruptedException, KeeperException, ReplicationException {
799    this.balancer = new RSGroupBasedLoadBalancer();
800    this.balancer.setConf(conf);
801    this.loadBalancerTracker = new LoadBalancerTracker(zooKeeper, this);
802    this.loadBalancerTracker.start();
803
804    this.normalizer = RegionNormalizerFactory.getRegionNormalizer(conf);
805    this.normalizer.setMasterServices(this);
806    this.regionNormalizerTracker = new RegionNormalizerTracker(zooKeeper, this);
807    this.regionNormalizerTracker.start();
808
809    this.splitOrMergeTracker = new SplitOrMergeTracker(zooKeeper, conf, this);
810    this.splitOrMergeTracker.start();
811
812    // This is for backwards compatible. We do not need the CP for rs group now but if user want to
813    // load it, we need to enable rs group.
814    String[] cpClasses = conf.getStrings(MasterCoprocessorHost.MASTER_COPROCESSOR_CONF_KEY);
815    if (cpClasses != null) {
816      for (String cpClass : cpClasses) {
817        if (RSGroupAdminEndpoint.class.getName().equals(cpClass)) {
818          RSGroupUtil.enableRSGroup(conf);
819          break;
820        }
821      }
822    }
823    this.rsGroupInfoManager = RSGroupInfoManager.create(this);
824
825    this.replicationPeerManager = ReplicationPeerManager.create(zooKeeper, conf, clusterId);
826
827    this.drainingServerTracker = new DrainingServerTracker(zooKeeper, this, this.serverManager);
828    this.drainingServerTracker.start();
829
830    this.snapshotCleanupTracker = new SnapshotCleanupTracker(zooKeeper, this);
831    this.snapshotCleanupTracker.start();
832
833    String clientQuorumServers = conf.get(HConstants.CLIENT_ZOOKEEPER_QUORUM);
834    boolean clientZkObserverMode = conf.getBoolean(HConstants.CLIENT_ZOOKEEPER_OBSERVER_MODE,
835      HConstants.DEFAULT_CLIENT_ZOOKEEPER_OBSERVER_MODE);
836    if (clientQuorumServers != null && !clientZkObserverMode) {
837      // we need to take care of the ZK information synchronization
838      // if given client ZK are not observer nodes
839      ZKWatcher clientZkWatcher = new ZKWatcher(conf,
840          getProcessName() + ":" + rpcServices.getSocketAddress().getPort() + "-clientZK", this,
841          false, true);
842      this.metaLocationSyncer = new MetaLocationSyncer(zooKeeper, clientZkWatcher, this);
843      this.metaLocationSyncer.start();
844      this.masterAddressSyncer = new MasterAddressSyncer(zooKeeper, clientZkWatcher, this);
845      this.masterAddressSyncer.start();
846      // set cluster id is a one-go effort
847      ZKClusterId.setClusterId(clientZkWatcher, fileSystemManager.getClusterId());
848    }
849
850    // Set the cluster as up.  If new RSs, they'll be waiting on this before
851    // going ahead with their startup.
852    boolean wasUp = this.clusterStatusTracker.isClusterUp();
853    if (!wasUp) this.clusterStatusTracker.setClusterUp();
854
855    LOG.info("Active/primary master=" + this.serverName +
856        ", sessionid=0x" +
857        Long.toHexString(this.zooKeeper.getRecoverableZooKeeper().getSessionId()) +
858        ", setting cluster-up flag (Was=" + wasUp + ")");
859
860    // create/initialize the snapshot manager and other procedure managers
861    this.snapshotManager = new SnapshotManager();
862    this.mpmHost = new MasterProcedureManagerHost();
863    this.mpmHost.register(this.snapshotManager);
864    this.mpmHost.register(new MasterFlushTableProcedureManager());
865    this.mpmHost.loadProcedures(conf);
866    this.mpmHost.initialize(this, this.metricsMaster);
867  }
868
869  // Will be overriden in test to inject customized AssignmentManager
870  @VisibleForTesting
871  protected AssignmentManager createAssignmentManager(MasterServices master) {
872    return new AssignmentManager(master);
873  }
874
875  /**
876   * Finish initialization of HMaster after becoming the primary master.
877   * <p/>
878   * The startup order is a bit complicated but very important, do not change it unless you know
879   * what you are doing.
880   * <ol>
881   * <li>Initialize file system based components - file system manager, wal manager, table
882   * descriptors, etc</li>
883   * <li>Publish cluster id</li>
884   * <li>Here comes the most complicated part - initialize server manager, assignment manager and
885   * region server tracker
886   * <ol type='i'>
887   * <li>Create server manager</li>
888   * <li>Create procedure executor, load the procedures, but do not start workers. We will start it
889   * later after we finish scheduling SCPs to avoid scheduling duplicated SCPs for the same
890   * server</li>
891   * <li>Create assignment manager and start it, load the meta region state, but do not load data
892   * from meta region</li>
893   * <li>Start region server tracker, construct the online servers set and find out dead servers and
894   * schedule SCP for them. The online servers will be constructed by scanning zk, and we will also
895   * scan the wal directory to find out possible live region servers, and the differences between
896   * these two sets are the dead servers</li>
897   * </ol>
898   * </li>
899   * <li>If this is a new deploy, schedule a InitMetaProcedure to initialize meta</li>
900   * <li>Start necessary service threads - balancer, catalog janior, executor services, and also the
901   * procedure executor, etc. Notice that the balancer must be created first as assignment manager
902   * may use it when assigning regions.</li>
903   * <li>Wait for meta to be initialized if necesssary, start table state manager.</li>
904   * <li>Wait for enough region servers to check-in</li>
905   * <li>Let assignment manager load data from meta and construct region states</li>
906   * <li>Start all other things such as chore services, etc</li>
907   * </ol>
908   * <p/>
909   * Notice that now we will not schedule a special procedure to make meta online(unless the first
910   * time where meta has not been created yet), we will rely on SCP to bring meta online.
911   */
912  private void finishActiveMasterInitialization(MonitoredTask status) throws IOException,
913          InterruptedException, KeeperException, ReplicationException {
914    /*
915     * We are active master now... go initialize components we need to run.
916     */
917    status.setStatus("Initializing Master file system");
918
919    this.masterActiveTime = System.currentTimeMillis();
920    // TODO: Do this using Dependency Injection, using PicoContainer, Guice or Spring.
921
922    // always initialize the MemStoreLAB as we use a region to store data in master now, see
923    // localStore.
924    initializeMemStoreChunkCreator();
925    this.fileSystemManager = new MasterFileSystem(conf);
926    this.walManager = new MasterWalManager(this);
927
928    // warm-up HTDs cache on master initialization
929    if (preLoadTableDescriptors) {
930      status.setStatus("Pre-loading table descriptors");
931      this.tableDescriptors.getAll();
932    }
933
934    // Publish cluster ID; set it in Master too. The superclass RegionServer does this later but
935    // only after it has checked in with the Master. At least a few tests ask Master for clusterId
936    // before it has called its run method and before RegionServer has done the reportForDuty.
937    ClusterId clusterId = fileSystemManager.getClusterId();
938    status.setStatus("Publishing Cluster ID " + clusterId + " in ZooKeeper");
939    ZKClusterId.setClusterId(this.zooKeeper, fileSystemManager.getClusterId());
940    this.clusterId = clusterId.toString();
941
942    // Precaution. Put in place the old hbck1 lock file to fence out old hbase1s running their
943    // hbck1s against an hbase2 cluster; it could do damage. To skip this behavior, set
944    // hbase.write.hbck1.lock.file to false.
945    if (this.conf.getBoolean("hbase.write.hbck1.lock.file", true)) {
946      Pair<Path, FSDataOutputStream> result = null;
947      try {
948        result = HBaseFsck.checkAndMarkRunningHbck(this.conf,
949            HBaseFsck.createLockRetryCounterFactory(this.conf).create());
950      } finally {
951        if (result != null) {
952          IOUtils.closeQuietly(result.getSecond());
953        }
954      }
955    }
956
957    status.setStatus("Initialize ServerManager and schedule SCP for crash servers");
958    // The below two managers must be created before loading procedures, as they will be used during
959    // loading.
960    this.serverManager = createServerManager(this);
961    this.syncReplicationReplayWALManager = new SyncReplicationReplayWALManager(this);
962    if (!conf.getBoolean(HBASE_SPLIT_WAL_COORDINATED_BY_ZK,
963      DEFAULT_HBASE_SPLIT_COORDINATED_BY_ZK)) {
964      this.splitWALManager = new SplitWALManager(this);
965    }
966
967    // initialize master local region
968    masterRegion = MasterRegionFactory.create(this);
969    createProcedureExecutor();
970    Map<Class<?>, List<Procedure<MasterProcedureEnv>>> procsByType =
971      procedureExecutor.getActiveProceduresNoCopy().stream()
972        .collect(Collectors.groupingBy(p -> p.getClass()));
973
974    // Create Assignment Manager
975    this.assignmentManager = createAssignmentManager(this);
976    this.assignmentManager.start();
977    // TODO: TRSP can perform as the sub procedure for other procedures, so even if it is marked as
978    // completed, it could still be in the procedure list. This is a bit strange but is another
979    // story, need to verify the implementation for ProcedureExecutor and ProcedureStore.
980    List<TransitRegionStateProcedure> ritList =
981      procsByType.getOrDefault(TransitRegionStateProcedure.class, Collections.emptyList()).stream()
982        .filter(p -> !p.isFinished()).map(p -> (TransitRegionStateProcedure) p)
983        .collect(Collectors.toList());
984    this.assignmentManager.setupRIT(ritList);
985
986    // Start RegionServerTracker with listing of servers found with exiting SCPs -- these should
987    // be registered in the deadServers set -- and with the list of servernames out on the
988    // filesystem that COULD BE 'alive' (we'll schedule SCPs for each and let SCP figure it out).
989    // We also pass dirs that are already 'splitting'... so we can do some checks down in tracker.
990    // TODO: Generate the splitting and live Set in one pass instead of two as we currently do.
991    this.regionServerTracker = new RegionServerTracker(zooKeeper, this, this.serverManager);
992    this.regionServerTracker.start(
993      procsByType.getOrDefault(ServerCrashProcedure.class, Collections.emptyList()).stream()
994        .map(p -> (ServerCrashProcedure) p).map(p -> p.getServerName()).collect(Collectors.toSet()),
995      walManager.getLiveServersFromWALDir(), walManager.getSplittingServersFromWALDir());
996    // This manager will be started AFTER hbase:meta is confirmed on line.
997    // hbase.mirror.table.state.to.zookeeper is so hbase1 clients can connect. They read table
998    // state from zookeeper while hbase2 reads it from hbase:meta. Disable if no hbase1 clients.
999    this.tableStateManager =
1000      this.conf.getBoolean(MirroringTableStateManager.MIRROR_TABLE_STATE_TO_ZK_KEY, true)
1001        ?
1002        new MirroringTableStateManager(this):
1003        new TableStateManager(this);
1004
1005    status.setStatus("Initializing ZK system trackers");
1006    initializeZKBasedSystemTrackers();
1007    status.setStatus("Loading last flushed sequence id of regions");
1008    try {
1009      this.serverManager.loadLastFlushedSequenceIds();
1010    } catch (IOException e) {
1011      LOG.info("Failed to load last flushed sequence id of regions"
1012          + " from file system", e);
1013    }
1014    // Set ourselves as active Master now our claim has succeeded up in zk.
1015    this.activeMaster = true;
1016
1017    // Start the Zombie master detector after setting master as active, see HBASE-21535
1018    Thread zombieDetector = new Thread(new InitializationMonitor(this),
1019        "ActiveMasterInitializationMonitor-" + System.currentTimeMillis());
1020    zombieDetector.setDaemon(true);
1021    zombieDetector.start();
1022
1023    if (!maintenanceMode) {
1024      // Add the Observer to delete quotas on table deletion before starting all CPs by
1025      // default with quota support, avoiding if user specifically asks to not load this Observer.
1026      if (QuotaUtil.isQuotaEnabled(conf)) {
1027        updateConfigurationForQuotasObserver(conf);
1028      }
1029      // initialize master side coprocessors before we start handling requests
1030      status.setStatus("Initializing master coprocessors");
1031      this.cpHost = new MasterCoprocessorHost(this, this.conf);
1032    }
1033
1034    // Checking if meta needs initializing.
1035    status.setStatus("Initializing meta table if this is a new deploy");
1036    InitMetaProcedure initMetaProc = null;
1037    // Print out state of hbase:meta on startup; helps debugging.
1038    RegionState rs = this.assignmentManager.getRegionStates().
1039        getRegionState(RegionInfoBuilder.FIRST_META_REGIONINFO);
1040    LOG.info("hbase:meta {}", rs);
1041    if (rs != null && rs.isOffline()) {
1042      Optional<InitMetaProcedure> optProc = procedureExecutor.getProcedures().stream()
1043        .filter(p -> p instanceof InitMetaProcedure).map(o -> (InitMetaProcedure) o).findAny();
1044      initMetaProc = optProc.orElseGet(() -> {
1045        // schedule an init meta procedure if meta has not been deployed yet
1046        InitMetaProcedure temp = new InitMetaProcedure();
1047        procedureExecutor.submitProcedure(temp);
1048        return temp;
1049      });
1050    }
1051
1052    // initialize load balancer
1053    this.balancer.setMasterServices(this);
1054    this.balancer.setClusterMetrics(getClusterMetricsWithoutCoprocessor());
1055    this.balancer.initialize();
1056
1057    // start up all service threads.
1058    status.setStatus("Initializing master service threads");
1059    startServiceThreads();
1060    // wait meta to be initialized after we start procedure executor
1061    if (initMetaProc != null) {
1062      initMetaProc.await();
1063    }
1064    // Wake up this server to check in
1065    sleeper.skipSleepCycle();
1066
1067    // Wait for region servers to report in.
1068    // With this as part of master initialization, it precludes our being able to start a single
1069    // server that is both Master and RegionServer. Needs more thought. TODO.
1070    String statusStr = "Wait for region servers to report in";
1071    status.setStatus(statusStr);
1072    LOG.info(Objects.toString(status));
1073    waitForRegionServers(status);
1074
1075    // Check if master is shutting down because issue initializing regionservers or balancer.
1076    if (isStopped()) {
1077      return;
1078    }
1079
1080    status.setStatus("Starting assignment manager");
1081    // FIRST HBASE:META READ!!!!
1082    // The below cannot make progress w/o hbase:meta being online.
1083    // This is the FIRST attempt at going to hbase:meta. Meta on-lining is going on in background
1084    // as procedures run -- in particular SCPs for crashed servers... One should put up hbase:meta
1085    // if it is down. It may take a while to come online. So, wait here until meta if for sure
1086    // available. That's what waitForMetaOnline does.
1087    if (!waitForMetaOnline()) {
1088      return;
1089    }
1090    this.assignmentManager.joinCluster();
1091    // The below depends on hbase:meta being online.
1092    this.tableStateManager.start();
1093    // Below has to happen after tablestatemanager has started in the case where this hbase-2.x
1094    // is being started over an hbase-1.x dataset. tablestatemanager runs a migration as part
1095    // of its 'start' moving table state from zookeeper to hbase:meta. This migration needs to
1096    // complete before we do this next step processing offline regions else it fails reading
1097    // table states messing up master launch (namespace table, etc., are not assigned).
1098    this.assignmentManager.processOfflineRegions();
1099    // Initialize after meta is up as below scans meta
1100    if (getFavoredNodesManager() != null && !maintenanceMode) {
1101      SnapshotOfRegionAssignmentFromMeta snapshotOfRegionAssignment =
1102          new SnapshotOfRegionAssignmentFromMeta(getConnection());
1103      snapshotOfRegionAssignment.initialize();
1104      getFavoredNodesManager().initialize(snapshotOfRegionAssignment);
1105    }
1106
1107    // set cluster status again after user regions are assigned
1108    this.balancer.setClusterMetrics(getClusterMetricsWithoutCoprocessor());
1109
1110    // Start balancer and meta catalog janitor after meta and regions have been assigned.
1111    status.setStatus("Starting balancer and catalog janitor");
1112    this.clusterStatusChore = new ClusterStatusChore(this, balancer);
1113    getChoreService().scheduleChore(clusterStatusChore);
1114    this.balancerChore = new BalancerChore(this);
1115    getChoreService().scheduleChore(balancerChore);
1116    this.normalizerChore = new RegionNormalizerChore(this);
1117    getChoreService().scheduleChore(normalizerChore);
1118    this.catalogJanitorChore = new CatalogJanitor(this);
1119    getChoreService().scheduleChore(catalogJanitorChore);
1120    this.hbckChore = new HbckChore(this);
1121    getChoreService().scheduleChore(hbckChore);
1122    this.serverManager.startChore();
1123
1124    // Only for rolling upgrade, where we need to migrate the data in namespace table to meta table.
1125    if (!waitForNamespaceOnline()) {
1126      return;
1127    }
1128    status.setStatus("Starting cluster schema service");
1129    initClusterSchemaService();
1130
1131    if (this.cpHost != null) {
1132      try {
1133        this.cpHost.preMasterInitialization();
1134      } catch (IOException e) {
1135        LOG.error("Coprocessor preMasterInitialization() hook failed", e);
1136      }
1137    }
1138
1139    status.markComplete("Initialization successful");
1140    LOG.info(String.format("Master has completed initialization %.3fsec",
1141       (System.currentTimeMillis() - masterActiveTime) / 1000.0f));
1142    this.masterFinishedInitializationTime = System.currentTimeMillis();
1143    configurationManager.registerObserver(this.balancer);
1144    configurationManager.registerObserver(this.cleanerPool);
1145    configurationManager.registerObserver(this.hfileCleaner);
1146    configurationManager.registerObserver(this.logCleaner);
1147    configurationManager.registerObserver(this.regionsRecoveryConfigManager);
1148    // Set master as 'initialized'.
1149    setInitialized(true);
1150
1151    if (maintenanceMode) {
1152      LOG.info("Detected repair mode, skipping final initialization steps.");
1153      return;
1154    }
1155
1156    assignmentManager.checkIfShouldMoveSystemRegionAsync();
1157    status.setStatus("Assign meta replicas");
1158    MasterMetaBootstrap metaBootstrap = createMetaBootstrap();
1159    try {
1160      metaBootstrap.assignMetaReplicas();
1161    } catch (IOException | KeeperException e){
1162      LOG.error("Assigning meta replica failed: ", e);
1163    }
1164    status.setStatus("Starting quota manager");
1165    initQuotaManager();
1166    if (QuotaUtil.isQuotaEnabled(conf)) {
1167      // Create the quota snapshot notifier
1168      spaceQuotaSnapshotNotifier = createQuotaSnapshotNotifier();
1169      spaceQuotaSnapshotNotifier.initialize(getConnection());
1170      this.quotaObserverChore = new QuotaObserverChore(this, getMasterMetrics());
1171      // Start the chore to read the region FS space reports and act on them
1172      getChoreService().scheduleChore(quotaObserverChore);
1173
1174      this.snapshotQuotaChore = new SnapshotQuotaObserverChore(this, getMasterMetrics());
1175      // Start the chore to read snapshots and add their usage to table/NS quotas
1176      getChoreService().scheduleChore(snapshotQuotaChore);
1177    }
1178    final SlowLogMasterService slowLogMasterService = new SlowLogMasterService(conf, this);
1179    slowLogMasterService.init();
1180
1181    // clear the dead servers with same host name and port of online server because we are not
1182    // removing dead server with same hostname and port of rs which is trying to check in before
1183    // master initialization. See HBASE-5916.
1184    this.serverManager.clearDeadServersWithSameHostNameAndPortOfOnlineServer();
1185
1186    // Check and set the znode ACLs if needed in case we are overtaking a non-secure configuration
1187    status.setStatus("Checking ZNode ACLs");
1188    zooKeeper.checkAndSetZNodeAcls();
1189
1190    status.setStatus("Initializing MOB Cleaner");
1191    initMobCleaner();
1192
1193    status.setStatus("Calling postStartMaster coprocessors");
1194    if (this.cpHost != null) {
1195      // don't let cp initialization errors kill the master
1196      try {
1197        this.cpHost.postStartMaster();
1198      } catch (IOException ioe) {
1199        LOG.error("Coprocessor postStartMaster() hook failed", ioe);
1200      }
1201    }
1202
1203    zombieDetector.interrupt();
1204
1205    /*
1206     * After master has started up, lets do balancer post startup initialization. Since this runs
1207     * in activeMasterManager thread, it should be fine.
1208     */
1209    long start = System.currentTimeMillis();
1210    this.balancer.postMasterStartupInitialize();
1211    if (LOG.isDebugEnabled()) {
1212      LOG.debug("Balancer post startup initialization complete, took " + (
1213          (System.currentTimeMillis() - start) / 1000) + " seconds");
1214    }
1215  }
1216
1217  /**
1218   * Check hbase:meta is up and ready for reading. For use during Master startup only.
1219   * @return True if meta is UP and online and startup can progress. Otherwise, meta is not online
1220   *   and we will hold here until operator intervention.
1221   */
1222  @VisibleForTesting
1223  public boolean waitForMetaOnline() {
1224    return isRegionOnline(RegionInfoBuilder.FIRST_META_REGIONINFO);
1225  }
1226
1227  /**
1228   * @return True if region is online and scannable else false if an error or shutdown (Otherwise
1229   *   we just block in here holding up all forward-progess).
1230   */
1231  private boolean isRegionOnline(RegionInfo ri) {
1232    RetryCounter rc = null;
1233    while (!isStopped()) {
1234      RegionState rs = this.assignmentManager.getRegionStates().getRegionState(ri);
1235      if (rs.isOpened()) {
1236        if (this.getServerManager().isServerOnline(rs.getServerName())) {
1237          return true;
1238        }
1239      }
1240      // Region is not OPEN.
1241      Optional<Procedure<MasterProcedureEnv>> optProc = this.procedureExecutor.getProcedures().
1242          stream().filter(p -> p instanceof ServerCrashProcedure).findAny();
1243      // TODO: Add a page to refguide on how to do repair. Have this log message point to it.
1244      // Page will talk about loss of edits, how to schedule at least the meta WAL recovery, and
1245      // then how to assign including how to break region lock if one held.
1246      LOG.warn("{} is NOT online; state={}; ServerCrashProcedures={}. Master startup cannot " +
1247          "progress, in holding-pattern until region onlined.",
1248          ri.getRegionNameAsString(), rs, optProc.isPresent());
1249      // Check once-a-minute.
1250      if (rc == null) {
1251        rc = new RetryCounterFactory(1000).create();
1252      }
1253      Threads.sleep(rc.getBackoffTimeAndIncrementAttempts());
1254    }
1255    return false;
1256  }
1257
1258  /**
1259   * Check hbase:namespace table is assigned. If not, startup will hang looking for the ns table
1260   * <p/>
1261   * This is for rolling upgrading, later we will migrate the data in ns table to the ns family of
1262   * meta table. And if this is a new cluster, this method will return immediately as there will be
1263   * no namespace table/region.
1264   * @return True if namespace table is up/online.
1265   */
1266  private boolean waitForNamespaceOnline() throws IOException {
1267    TableState nsTableState =
1268      MetaTableAccessor.getTableState(getConnection(), TableName.NAMESPACE_TABLE_NAME);
1269    if (nsTableState == null || nsTableState.isDisabled()) {
1270      // this means we have already migrated the data and disabled or deleted the namespace table,
1271      // or this is a new deploy which does not have a namespace table from the beginning.
1272      return true;
1273    }
1274    List<RegionInfo> ris =
1275      this.assignmentManager.getRegionStates().getRegionsOfTable(TableName.NAMESPACE_TABLE_NAME);
1276    if (ris.isEmpty()) {
1277      // maybe this will not happen any more, but anyway, no harm to add a check here...
1278      return true;
1279    }
1280    // Else there are namespace regions up in meta. Ensure they are assigned before we go on.
1281    for (RegionInfo ri : ris) {
1282      if (!isRegionOnline(ri)) {
1283        return false;
1284      }
1285    }
1286    return true;
1287  }
1288
1289  /**
1290   * Adds the {@code MasterQuotasObserver} to the list of configured Master observers to
1291   * automatically remove quotas for a table when that table is deleted.
1292   */
1293  @VisibleForTesting
1294  public void updateConfigurationForQuotasObserver(Configuration conf) {
1295    // We're configured to not delete quotas on table deletion, so we don't need to add the obs.
1296    if (!conf.getBoolean(
1297          MasterQuotasObserver.REMOVE_QUOTA_ON_TABLE_DELETE,
1298          MasterQuotasObserver.REMOVE_QUOTA_ON_TABLE_DELETE_DEFAULT)) {
1299      return;
1300    }
1301    String[] masterCoprocs = conf.getStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY);
1302    final int length = null == masterCoprocs ? 0 : masterCoprocs.length;
1303    String[] updatedCoprocs = new String[length + 1];
1304    if (length > 0) {
1305      System.arraycopy(masterCoprocs, 0, updatedCoprocs, 0, masterCoprocs.length);
1306    }
1307    updatedCoprocs[length] = MasterQuotasObserver.class.getName();
1308    conf.setStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, updatedCoprocs);
1309  }
1310
1311  private void initMobCleaner() {
1312    this.mobFileCleanerChore = new MobFileCleanerChore(this);
1313    getChoreService().scheduleChore(mobFileCleanerChore);
1314    this.mobFileCompactionChore = new MobFileCompactionChore(this);
1315    getChoreService().scheduleChore(mobFileCompactionChore);
1316  }
1317
1318  /**
1319   * <p>
1320   * Create a {@link MasterMetaBootstrap} instance.
1321   * </p>
1322   * <p>
1323   * Will be overridden in tests.
1324   * </p>
1325   */
1326  @VisibleForTesting
1327  protected MasterMetaBootstrap createMetaBootstrap() {
1328    // We put this out here in a method so can do a Mockito.spy and stub it out
1329    // w/ a mocked up MasterMetaBootstrap.
1330    return new MasterMetaBootstrap(this);
1331  }
1332
1333  /**
1334   * <p>
1335   * Create a {@link ServerManager} instance.
1336   * </p>
1337   * <p>
1338   * Will be overridden in tests.
1339   * </p>
1340   */
1341  @VisibleForTesting
1342  protected ServerManager createServerManager(final MasterServices master) throws IOException {
1343    // We put this out here in a method so can do a Mockito.spy and stub it out
1344    // w/ a mocked up ServerManager.
1345    setupClusterConnection();
1346    return new ServerManager(master);
1347  }
1348
1349  private void waitForRegionServers(final MonitoredTask status)
1350      throws IOException, InterruptedException {
1351    this.serverManager.waitForRegionServers(status);
1352  }
1353
1354  // Will be overridden in tests
1355  @VisibleForTesting
1356  protected void initClusterSchemaService() throws IOException, InterruptedException {
1357    this.clusterSchemaService = new ClusterSchemaServiceImpl(this);
1358    this.clusterSchemaService.startAsync();
1359    try {
1360      this.clusterSchemaService.awaitRunning(getConfiguration().getInt(
1361        HBASE_MASTER_WAIT_ON_SERVICE_IN_SECONDS,
1362        DEFAULT_HBASE_MASTER_WAIT_ON_SERVICE_IN_SECONDS), TimeUnit.SECONDS);
1363    } catch (TimeoutException toe) {
1364      throw new IOException("Timedout starting ClusterSchemaService", toe);
1365    }
1366  }
1367
1368  private void initQuotaManager() throws IOException {
1369    MasterQuotaManager quotaManager = new MasterQuotaManager(this);
1370    quotaManager.start();
1371    this.quotaManager = quotaManager;
1372  }
1373
1374  private SpaceQuotaSnapshotNotifier createQuotaSnapshotNotifier() {
1375    SpaceQuotaSnapshotNotifier notifier =
1376        SpaceQuotaSnapshotNotifierFactory.getInstance().create(getConfiguration());
1377    return notifier;
1378  }
1379
1380  boolean isCatalogJanitorEnabled() {
1381    return catalogJanitorChore != null ? catalogJanitorChore.getEnabled() : false;
1382  }
1383
1384  boolean isCleanerChoreEnabled() {
1385    boolean hfileCleanerFlag = true, logCleanerFlag = true;
1386
1387    if (hfileCleaner != null) {
1388      hfileCleanerFlag = hfileCleaner.getEnabled();
1389    }
1390
1391    if (logCleaner != null) {
1392      logCleanerFlag = logCleaner.getEnabled();
1393    }
1394
1395    return (hfileCleanerFlag && logCleanerFlag);
1396  }
1397
1398  @Override
1399  public ServerManager getServerManager() {
1400    return this.serverManager;
1401  }
1402
1403  @Override
1404  public MasterFileSystem getMasterFileSystem() {
1405    return this.fileSystemManager;
1406  }
1407
1408  @Override
1409  public MasterWalManager getMasterWalManager() {
1410    return this.walManager;
1411  }
1412
1413  @Override
1414  public SplitWALManager getSplitWALManager() {
1415    return splitWALManager;
1416  }
1417
1418  @Override
1419  public TableStateManager getTableStateManager() {
1420    return tableStateManager;
1421  }
1422
1423  /*
1424   * Start up all services. If any of these threads gets an unhandled exception
1425   * then they just die with a logged message.  This should be fine because
1426   * in general, we do not expect the master to get such unhandled exceptions
1427   *  as OOMEs; it should be lightly loaded. See what HRegionServer does if
1428   *  need to install an unexpected exception handler.
1429   */
1430  private void startServiceThreads() throws IOException {
1431    // Start the executor service pools
1432    this.executorService.startExecutorService(ExecutorType.MASTER_OPEN_REGION, conf.getInt(
1433      HConstants.MASTER_OPEN_REGION_THREADS, HConstants.MASTER_OPEN_REGION_THREADS_DEFAULT));
1434    this.executorService.startExecutorService(ExecutorType.MASTER_CLOSE_REGION, conf.getInt(
1435      HConstants.MASTER_CLOSE_REGION_THREADS, HConstants.MASTER_CLOSE_REGION_THREADS_DEFAULT));
1436    this.executorService.startExecutorService(ExecutorType.MASTER_SERVER_OPERATIONS,
1437      conf.getInt(HConstants.MASTER_SERVER_OPERATIONS_THREADS,
1438        HConstants.MASTER_SERVER_OPERATIONS_THREADS_DEFAULT));
1439    this.executorService.startExecutorService(ExecutorType.MASTER_META_SERVER_OPERATIONS,
1440      conf.getInt(HConstants.MASTER_META_SERVER_OPERATIONS_THREADS,
1441        HConstants.MASTER_META_SERVER_OPERATIONS_THREADS_DEFAULT));
1442    this.executorService.startExecutorService(ExecutorType.M_LOG_REPLAY_OPS, conf.getInt(
1443      HConstants.MASTER_LOG_REPLAY_OPS_THREADS, HConstants.MASTER_LOG_REPLAY_OPS_THREADS_DEFAULT));
1444    this.executorService.startExecutorService(ExecutorType.MASTER_SNAPSHOT_OPERATIONS, conf.getInt(
1445      SnapshotManager.SNAPSHOT_POOL_THREADS_KEY, SnapshotManager.SNAPSHOT_POOL_THREADS_DEFAULT));
1446
1447    // We depend on there being only one instance of this executor running
1448    // at a time. To do concurrency, would need fencing of enable/disable of
1449    // tables.
1450    // Any time changing this maxThreads to > 1, pls see the comment at
1451    // AccessController#postCompletedCreateTableAction
1452    this.executorService.startExecutorService(ExecutorType.MASTER_TABLE_OPERATIONS, 1);
1453    startProcedureExecutor();
1454
1455    // Create cleaner thread pool
1456    cleanerPool = new DirScanPool(conf);
1457    // Start log cleaner thread
1458    int cleanerInterval =
1459      conf.getInt(HBASE_MASTER_CLEANER_INTERVAL, DEFAULT_HBASE_MASTER_CLEANER_INTERVAL);
1460    this.logCleaner = new LogCleaner(cleanerInterval, this, conf,
1461      getMasterWalManager().getFileSystem(), getMasterWalManager().getOldLogDir(), cleanerPool);
1462    getChoreService().scheduleChore(logCleaner);
1463
1464    // start the hfile archive cleaner thread
1465    Path archiveDir = HFileArchiveUtil.getArchivePath(conf);
1466    Map<String, Object> params = new HashMap<>();
1467    params.put(MASTER, this);
1468    this.hfileCleaner = new HFileCleaner(cleanerInterval, this, conf,
1469      getMasterFileSystem().getFileSystem(), archiveDir, cleanerPool, params);
1470    getChoreService().scheduleChore(hfileCleaner);
1471
1472    // Regions Reopen based on very high storeFileRefCount is considered enabled
1473    // only if hbase.regions.recovery.store.file.ref.count has value > 0
1474    final int maxStoreFileRefCount = conf.getInt(
1475      HConstants.STORE_FILE_REF_COUNT_THRESHOLD,
1476      HConstants.DEFAULT_STORE_FILE_REF_COUNT_THRESHOLD);
1477    if (maxStoreFileRefCount > 0) {
1478      this.regionsRecoveryChore = new RegionsRecoveryChore(this, conf, this);
1479      getChoreService().scheduleChore(this.regionsRecoveryChore);
1480    } else {
1481      LOG.info("Reopening regions with very high storeFileRefCount is disabled. " +
1482          "Provide threshold value > 0 for {} to enable it.",
1483        HConstants.STORE_FILE_REF_COUNT_THRESHOLD);
1484    }
1485
1486    this.regionsRecoveryConfigManager = new RegionsRecoveryConfigManager(this);
1487
1488    replicationBarrierCleaner = new ReplicationBarrierCleaner(conf, this, getConnection(),
1489      replicationPeerManager);
1490    getChoreService().scheduleChore(replicationBarrierCleaner);
1491
1492    final boolean isSnapshotChoreEnabled = this.snapshotCleanupTracker
1493        .isSnapshotCleanupEnabled();
1494    this.snapshotCleanerChore = new SnapshotCleanerChore(this, conf, getSnapshotManager());
1495    if (isSnapshotChoreEnabled) {
1496      getChoreService().scheduleChore(this.snapshotCleanerChore);
1497    } else {
1498      if (LOG.isTraceEnabled()) {
1499        LOG.trace("Snapshot Cleaner Chore is disabled. Not starting up the chore..");
1500      }
1501    }
1502    serviceStarted = true;
1503    if (LOG.isTraceEnabled()) {
1504      LOG.trace("Started service threads");
1505    }
1506  }
1507
1508  @Override
1509  protected void stopServiceThreads() {
1510    if (masterJettyServer != null) {
1511      LOG.info("Stopping master jetty server");
1512      try {
1513        masterJettyServer.stop();
1514      } catch (Exception e) {
1515        LOG.error("Failed to stop master jetty server", e);
1516      }
1517    }
1518    stopChores();
1519
1520    super.stopServiceThreads();
1521    if (cleanerPool != null) {
1522      cleanerPool.shutdownNow();
1523      cleanerPool = null;
1524    }
1525
1526    LOG.debug("Stopping service threads");
1527
1528    // stop procedure executor prior to other services such as server manager and assignment
1529    // manager, as these services are important for some running procedures. See HBASE-24117 for
1530    // example.
1531    stopProcedureExecutor();
1532
1533    if (this.quotaManager != null) {
1534      this.quotaManager.stop();
1535    }
1536
1537    if (this.activeMasterManager != null) {
1538      this.activeMasterManager.stop();
1539    }
1540    if (this.serverManager != null) {
1541      this.serverManager.stop();
1542    }
1543    if (this.assignmentManager != null) {
1544      this.assignmentManager.stop();
1545    }
1546
1547    if (masterRegion != null) {
1548      masterRegion.close(isAborted());
1549    }
1550    if (this.walManager != null) {
1551      this.walManager.stop();
1552    }
1553    if (this.fileSystemManager != null) {
1554      this.fileSystemManager.stop();
1555    }
1556    if (this.mpmHost != null) {
1557      this.mpmHost.stop("server shutting down.");
1558    }
1559    if (this.regionServerTracker != null) {
1560      this.regionServerTracker.stop();
1561    }
1562  }
1563
1564  private void createProcedureExecutor() throws IOException {
1565    MasterProcedureEnv procEnv = new MasterProcedureEnv(this);
1566    procedureStore =
1567      new RegionProcedureStore(this, masterRegion, new MasterProcedureEnv.FsUtilsLeaseRecovery(this));
1568    procedureStore.registerListener(new ProcedureStoreListener() {
1569
1570      @Override
1571      public void abortProcess() {
1572        abort("The Procedure Store lost the lease", null);
1573      }
1574    });
1575    MasterProcedureScheduler procedureScheduler = procEnv.getProcedureScheduler();
1576    procedureExecutor = new ProcedureExecutor<>(conf, procEnv, procedureStore, procedureScheduler);
1577    configurationManager.registerObserver(procEnv);
1578
1579    int cpus = Runtime.getRuntime().availableProcessors();
1580    final int numThreads = conf.getInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, Math.max(
1581      (cpus > 0 ? cpus / 4 : 0), MasterProcedureConstants.DEFAULT_MIN_MASTER_PROCEDURE_THREADS));
1582    final boolean abortOnCorruption =
1583      conf.getBoolean(MasterProcedureConstants.EXECUTOR_ABORT_ON_CORRUPTION,
1584        MasterProcedureConstants.DEFAULT_EXECUTOR_ABORT_ON_CORRUPTION);
1585    procedureStore.start(numThreads);
1586    // Just initialize it but do not start the workers, we will start the workers later by calling
1587    // startProcedureExecutor. See the javadoc for finishActiveMasterInitialization for more
1588    // details.
1589    procedureExecutor.init(numThreads, abortOnCorruption);
1590    if (!procEnv.getRemoteDispatcher().start()) {
1591      throw new HBaseIOException("Failed start of remote dispatcher");
1592    }
1593  }
1594
1595  private void startProcedureExecutor() throws IOException {
1596    procedureExecutor.startWorkers();
1597  }
1598
1599  /**
1600   * Turn on/off Snapshot Cleanup Chore
1601   *
1602   * @param on indicates whether Snapshot Cleanup Chore is to be run
1603   */
1604  void switchSnapshotCleanup(final boolean on, final boolean synchronous) {
1605    if (synchronous) {
1606      synchronized (this.snapshotCleanerChore) {
1607        switchSnapshotCleanup(on);
1608      }
1609    } else {
1610      switchSnapshotCleanup(on);
1611    }
1612  }
1613
1614  private void switchSnapshotCleanup(final boolean on) {
1615    try {
1616      snapshotCleanupTracker.setSnapshotCleanupEnabled(on);
1617      if (on) {
1618        if (!getChoreService().isChoreScheduled(this.snapshotCleanerChore)) {
1619          getChoreService().scheduleChore(this.snapshotCleanerChore);
1620        }
1621      } else {
1622        getChoreService().cancelChore(this.snapshotCleanerChore);
1623      }
1624    } catch (KeeperException e) {
1625      LOG.error("Error updating snapshot cleanup mode to {}", on, e);
1626    }
1627  }
1628
1629
1630  private void stopProcedureExecutor() {
1631    if (procedureExecutor != null) {
1632      configurationManager.deregisterObserver(procedureExecutor.getEnvironment());
1633      procedureExecutor.getEnvironment().getRemoteDispatcher().stop();
1634      procedureExecutor.stop();
1635      procedureExecutor.join();
1636      procedureExecutor = null;
1637    }
1638
1639    if (procedureStore != null) {
1640      procedureStore.stop(isAborted());
1641      procedureStore = null;
1642    }
1643  }
1644
1645  private void stopChores() {
1646    ChoreService choreService = getChoreService();
1647    if (choreService != null) {
1648      choreService.cancelChore(this.mobFileCleanerChore);
1649      choreService.cancelChore(this.mobFileCompactionChore);
1650      choreService.cancelChore(this.balancerChore);
1651      choreService.cancelChore(this.normalizerChore);
1652      choreService.cancelChore(this.clusterStatusChore);
1653      choreService.cancelChore(this.catalogJanitorChore);
1654      choreService.cancelChore(this.clusterStatusPublisherChore);
1655      choreService.cancelChore(this.snapshotQuotaChore);
1656      choreService.cancelChore(this.logCleaner);
1657      choreService.cancelChore(this.hfileCleaner);
1658      choreService.cancelChore(this.replicationBarrierCleaner);
1659      choreService.cancelChore(this.snapshotCleanerChore);
1660      choreService.cancelChore(this.hbckChore);
1661      choreService.cancelChore(this.regionsRecoveryChore);
1662    }
1663  }
1664
1665  /**
1666   * @return Get remote side's InetAddress
1667   */
1668  InetAddress getRemoteInetAddress(final int port,
1669      final long serverStartCode) throws UnknownHostException {
1670    // Do it out here in its own little method so can fake an address when
1671    // mocking up in tests.
1672    InetAddress ia = RpcServer.getRemoteIp();
1673
1674    // The call could be from the local regionserver,
1675    // in which case, there is no remote address.
1676    if (ia == null && serverStartCode == startcode) {
1677      InetSocketAddress isa = rpcServices.getSocketAddress();
1678      if (isa != null && isa.getPort() == port) {
1679        ia = isa.getAddress();
1680      }
1681    }
1682    return ia;
1683  }
1684
1685  /**
1686   * @return Maximum time we should run balancer for
1687   */
1688  private int getMaxBalancingTime() {
1689    // if max balancing time isn't set, defaulting it to period time
1690    int maxBalancingTime = getConfiguration().getInt(HConstants.HBASE_BALANCER_MAX_BALANCING,
1691      getConfiguration()
1692        .getInt(HConstants.HBASE_BALANCER_PERIOD, HConstants.DEFAULT_HBASE_BALANCER_PERIOD));
1693    return maxBalancingTime;
1694  }
1695
1696  /**
1697   * @return Maximum number of regions in transition
1698   */
1699  private int getMaxRegionsInTransition() {
1700    int numRegions = this.assignmentManager.getRegionStates().getRegionAssignments().size();
1701    return Math.max((int) Math.floor(numRegions * this.maxRitPercent), 1);
1702  }
1703
1704  /**
1705   * It first sleep to the next balance plan start time. Meanwhile, throttling by the max
1706   * number regions in transition to protect availability.
1707   * @param nextBalanceStartTime The next balance plan start time
1708   * @param maxRegionsInTransition max number of regions in transition
1709   * @param cutoffTime when to exit balancer
1710   */
1711  private void balanceThrottling(long nextBalanceStartTime, int maxRegionsInTransition,
1712      long cutoffTime) {
1713    boolean interrupted = false;
1714
1715    // Sleep to next balance plan start time
1716    // But if there are zero regions in transition, it can skip sleep to speed up.
1717    while (!interrupted && System.currentTimeMillis() < nextBalanceStartTime
1718        && this.assignmentManager.getRegionStates().hasRegionsInTransition()) {
1719      try {
1720        Thread.sleep(100);
1721      } catch (InterruptedException ie) {
1722        interrupted = true;
1723      }
1724    }
1725
1726    // Throttling by max number regions in transition
1727    while (!interrupted
1728        && maxRegionsInTransition > 0
1729        && this.assignmentManager.getRegionStates().getRegionsInTransitionCount()
1730        >= maxRegionsInTransition && System.currentTimeMillis() <= cutoffTime) {
1731      try {
1732        // sleep if the number of regions in transition exceeds the limit
1733        Thread.sleep(100);
1734      } catch (InterruptedException ie) {
1735        interrupted = true;
1736      }
1737    }
1738
1739    if (interrupted) Thread.currentThread().interrupt();
1740  }
1741
1742  public boolean balance() throws IOException {
1743    return balance(false);
1744  }
1745
1746  /**
1747   * Checks master state before initiating action over region topology.
1748   * @param action the name of the action under consideration, for logging.
1749   * @return {@code true} when the caller should exit early, {@code false} otherwise.
1750   */
1751  private boolean skipRegionManagementAction(final String action) {
1752    if (!isInitialized()) {
1753      LOG.debug("Master has not been initialized, don't run {}.", action);
1754      return true;
1755    }
1756    if (this.getServerManager().isClusterShutdown()) {
1757      LOG.info("Cluster is shutting down, don't run {}.", action);
1758      return true;
1759    }
1760    if (isInMaintenanceMode()) {
1761      LOG.info("Master is in maintenance mode, don't run {}.", action);
1762      return true;
1763    }
1764    return false;
1765  }
1766
1767  public boolean balance(boolean force) throws IOException {
1768    if (loadBalancerTracker == null || !loadBalancerTracker.isBalancerOn()) {
1769      return false;
1770    }
1771    if (skipRegionManagementAction("balancer")) {
1772      return false;
1773    }
1774
1775    synchronized (this.balancer) {
1776        // Only allow one balance run at at time.
1777      if (this.assignmentManager.hasRegionsInTransition()) {
1778        List<RegionStateNode> regionsInTransition = assignmentManager.getRegionsInTransition();
1779        // if hbase:meta region is in transition, result of assignment cannot be recorded
1780        // ignore the force flag in that case
1781        boolean metaInTransition = assignmentManager.isMetaRegionInTransition();
1782        String prefix = force && !metaInTransition ? "R" : "Not r";
1783        List<RegionStateNode> toPrint = regionsInTransition;
1784        int max = 5;
1785        boolean truncated = false;
1786        if (regionsInTransition.size() > max) {
1787          toPrint = regionsInTransition.subList(0, max);
1788          truncated = true;
1789        }
1790        LOG.info(prefix + " not running balancer because " + regionsInTransition.size() +
1791          " region(s) in transition: " + toPrint + (truncated? "(truncated list)": ""));
1792        if (!force || metaInTransition) return false;
1793      }
1794      if (this.serverManager.areDeadServersInProgress()) {
1795        LOG.info("Not running balancer because processing dead regionserver(s): " +
1796          this.serverManager.getDeadServers());
1797        return false;
1798      }
1799
1800      if (this.cpHost != null) {
1801        try {
1802          if (this.cpHost.preBalance()) {
1803            LOG.debug("Coprocessor bypassing balancer request");
1804            return false;
1805          }
1806        } catch (IOException ioe) {
1807          LOG.error("Error invoking master coprocessor preBalance()", ioe);
1808          return false;
1809        }
1810      }
1811
1812      Map<TableName, Map<ServerName, List<RegionInfo>>> assignments =
1813        this.assignmentManager.getRegionStates()
1814          .getAssignmentsForBalancer(tableStateManager, this.serverManager.getOnlineServersList());
1815      for (Map<ServerName, List<RegionInfo>> serverMap : assignments.values()) {
1816        serverMap.keySet().removeAll(this.serverManager.getDrainingServersList());
1817      }
1818
1819      //Give the balancer the current cluster state.
1820      this.balancer.setClusterMetrics(getClusterMetricsWithoutCoprocessor());
1821
1822      List<RegionPlan> plans = this.balancer.balanceCluster(assignments);
1823
1824      if (skipRegionManagementAction("balancer")) {
1825        // make one last check that the cluster isn't shutting down before proceeding.
1826        return false;
1827      }
1828
1829      List<RegionPlan> sucRPs = executeRegionPlansWithThrottling(plans);
1830
1831      if (this.cpHost != null) {
1832        try {
1833          this.cpHost.postBalance(sucRPs);
1834        } catch (IOException ioe) {
1835          // balancing already succeeded so don't change the result
1836          LOG.error("Error invoking master coprocessor postBalance()", ioe);
1837        }
1838      }
1839    }
1840    // If LoadBalancer did not generate any plans, it means the cluster is already balanced.
1841    // Return true indicating a success.
1842    return true;
1843  }
1844
1845  /**
1846   * Execute region plans with throttling
1847   * @param plans to execute
1848   * @return succeeded plans
1849   */
1850  public List<RegionPlan> executeRegionPlansWithThrottling(List<RegionPlan> plans) {
1851    List<RegionPlan> successRegionPlans = new ArrayList<>();
1852    int maxRegionsInTransition = getMaxRegionsInTransition();
1853    long balanceStartTime = System.currentTimeMillis();
1854    long cutoffTime = balanceStartTime + this.maxBalancingTime;
1855    int rpCount = 0;  // number of RegionPlans balanced so far
1856    if (plans != null && !plans.isEmpty()) {
1857      int balanceInterval = this.maxBalancingTime / plans.size();
1858      LOG.info("Balancer plans size is " + plans.size() + ", the balance interval is "
1859          + balanceInterval + " ms, and the max number regions in transition is "
1860          + maxRegionsInTransition);
1861
1862      for (RegionPlan plan: plans) {
1863        LOG.info("balance " + plan);
1864        //TODO: bulk assign
1865        try {
1866          this.assignmentManager.moveAsync(plan);
1867        } catch (HBaseIOException hioe) {
1868          //should ignore failed plans here, avoiding the whole balance plans be aborted
1869          //later calls of balance() can fetch up the failed and skipped plans
1870          LOG.warn("Failed balance plan {}, skipping...", plan, hioe);
1871        }
1872        //rpCount records balance plans processed, does not care if a plan succeeds
1873        rpCount++;
1874        successRegionPlans.add(plan);
1875
1876        if (this.maxBalancingTime > 0) {
1877          balanceThrottling(balanceStartTime + rpCount * balanceInterval, maxRegionsInTransition,
1878            cutoffTime);
1879        }
1880
1881        // if performing next balance exceeds cutoff time, exit the loop
1882        if (this.maxBalancingTime > 0 && rpCount < plans.size()
1883          && System.currentTimeMillis() > cutoffTime) {
1884          // TODO: After balance, there should not be a cutoff time (keeping it as
1885          // a security net for now)
1886          LOG.debug("No more balancing till next balance run; maxBalanceTime="
1887              + this.maxBalancingTime);
1888          break;
1889        }
1890      }
1891    }
1892    return successRegionPlans;
1893  }
1894
1895  @Override
1896  public RegionNormalizer getRegionNormalizer() {
1897    return this.normalizer;
1898  }
1899
1900  public boolean normalizeRegions() throws IOException {
1901    return normalizeRegions(new NormalizeTableFilterParams.Builder().build());
1902  }
1903
1904  /**
1905   * Perform normalization of cluster.
1906   *
1907   * @return true if an existing normalization was already in progress, or if a new normalization
1908   *   was performed successfully; false otherwise (specifically, if HMaster finished initializing
1909   *   or normalization is globally disabled).
1910   */
1911  public boolean normalizeRegions(final NormalizeTableFilterParams ntfp) throws IOException {
1912    final long startTime = EnvironmentEdgeManager.currentTime();
1913    if (regionNormalizerTracker == null || !regionNormalizerTracker.isNormalizerOn()) {
1914      LOG.debug("Region normalization is disabled, don't run region normalizer.");
1915      return false;
1916    }
1917    if (skipRegionManagementAction("region normalizer")) {
1918      return false;
1919    }
1920    if (assignmentManager.hasRegionsInTransition()) {
1921      return false;
1922    }
1923
1924    if (!normalizationInProgressLock.tryLock()) {
1925      // Don't run the normalizer concurrently
1926      LOG.info("Normalization already in progress. Skipping request.");
1927      return true;
1928    }
1929
1930    int affectedTables = 0;
1931    try {
1932      final Set<TableName> matchingTables = getTableDescriptors(new LinkedList<>(),
1933        ntfp.getNamespace(), ntfp.getRegex(), ntfp.getTableNames(), false)
1934        .stream()
1935        .map(TableDescriptor::getTableName)
1936        .collect(Collectors.toSet());
1937      final Set<TableName> allEnabledTables =
1938        tableStateManager.getTablesInStates(TableState.State.ENABLED);
1939      final List<TableName> targetTables =
1940        new ArrayList<>(Sets.intersection(matchingTables, allEnabledTables));
1941      Collections.shuffle(targetTables);
1942
1943      final List<Long> submittedPlanProcIds = new ArrayList<>();
1944      for (TableName table : targetTables) {
1945        if (table.isSystemTable()) {
1946          continue;
1947        }
1948        final TableDescriptor tblDesc = getTableDescriptors().get(table);
1949        if (tblDesc != null && !tblDesc.isNormalizationEnabled()) {
1950          LOG.debug(
1951            "Skipping table {} because normalization is disabled in its table properties.", table);
1952          continue;
1953        }
1954
1955        // make one last check that the cluster isn't shutting down before proceeding.
1956        if (skipRegionManagementAction("region normalizer")) {
1957          return false;
1958        }
1959
1960        final List<NormalizationPlan> plans = normalizer.computePlansForTable(table);
1961        if (CollectionUtils.isEmpty(plans)) {
1962          LOG.debug("No normalization required for table {}.", table);
1963          continue;
1964        }
1965
1966        affectedTables++;
1967        // as of this writing, `plan.submit()` is non-blocking and uses Async Admin APIs to
1968        // submit task , so there's no artificial rate-
1969        // limiting of merge/split requests due to this serial loop.
1970        for (NormalizationPlan plan : plans) {
1971          long procId = plan.submit(this);
1972          submittedPlanProcIds.add(procId);
1973          if (plan.getType() == PlanType.SPLIT) {
1974            splitPlanCount++;
1975          } else if (plan.getType() == PlanType.MERGE) {
1976            mergePlanCount++;
1977          }
1978        }
1979      }
1980      final long endTime = EnvironmentEdgeManager.currentTime();
1981      LOG.info("Normalizer ran successfully in {}. Submitted {} plans, affecting {} tables.",
1982        Duration.ofMillis(endTime - startTime), submittedPlanProcIds.size(), affectedTables);
1983      LOG.debug("Normalizer submitted procID list: {}", submittedPlanProcIds);
1984    } finally {
1985      normalizationInProgressLock.unlock();
1986    }
1987    return true;
1988  }
1989
1990  /**
1991   * @return Client info for use as prefix on an audit log string; who did an action
1992   */
1993  @Override
1994  public String getClientIdAuditPrefix() {
1995    return "Client=" + RpcServer.getRequestUserName().orElse(null)
1996        + "/" + RpcServer.getRemoteAddress().orElse(null);
1997  }
1998
1999  /**
2000   * Switch for the background CatalogJanitor thread.
2001   * Used for testing.  The thread will continue to run.  It will just be a noop
2002   * if disabled.
2003   * @param b If false, the catalog janitor won't do anything.
2004   */
2005  public void setCatalogJanitorEnabled(final boolean b) {
2006    this.catalogJanitorChore.setEnabled(b);
2007  }
2008
2009  @Override
2010  public long mergeRegions(
2011      final RegionInfo[] regionsToMerge,
2012      final boolean forcible,
2013      final long ng,
2014      final long nonce) throws IOException {
2015    checkInitialized();
2016
2017    if (!isSplitOrMergeEnabled(MasterSwitchType.MERGE)) {
2018      String regionsStr = Arrays.deepToString(regionsToMerge);
2019      LOG.warn("Merge switch is off! skip merge of " + regionsStr);
2020      throw new DoNotRetryIOException("Merge of " + regionsStr +
2021          " failed because merge switch is off");
2022    }
2023
2024    final String mergeRegionsStr = Arrays.stream(regionsToMerge).map(RegionInfo::getEncodedName)
2025      .collect(Collectors.joining(", "));
2026    return MasterProcedureUtil.submitProcedure(new NonceProcedureRunnable(this, ng, nonce) {
2027      @Override
2028      protected void run() throws IOException {
2029        getMaster().getMasterCoprocessorHost().preMergeRegions(regionsToMerge);
2030        String aid = getClientIdAuditPrefix();
2031        LOG.info("{} merge regions {}", aid, mergeRegionsStr);
2032        submitProcedure(new MergeTableRegionsProcedure(procedureExecutor.getEnvironment(),
2033            regionsToMerge, forcible));
2034        getMaster().getMasterCoprocessorHost().postMergeRegions(regionsToMerge);
2035      }
2036
2037      @Override
2038      protected String getDescription() {
2039        return "MergeTableProcedure";
2040      }
2041    });
2042  }
2043
2044  @Override
2045  public long splitRegion(final RegionInfo regionInfo, final byte[] splitRow,
2046      final long nonceGroup, final long nonce)
2047  throws IOException {
2048    checkInitialized();
2049
2050    if (!isSplitOrMergeEnabled(MasterSwitchType.SPLIT)) {
2051      LOG.warn("Split switch is off! skip split of " + regionInfo);
2052      throw new DoNotRetryIOException("Split region " + regionInfo.getRegionNameAsString() +
2053          " failed due to split switch off");
2054    }
2055
2056    return MasterProcedureUtil.submitProcedure(
2057        new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {
2058      @Override
2059      protected void run() throws IOException {
2060        getMaster().getMasterCoprocessorHost().preSplitRegion(regionInfo.getTable(), splitRow);
2061        LOG.info(getClientIdAuditPrefix() + " split " + regionInfo.getRegionNameAsString());
2062
2063        // Execute the operation asynchronously
2064        submitProcedure(getAssignmentManager().createSplitProcedure(regionInfo, splitRow));
2065      }
2066
2067      @Override
2068      protected String getDescription() {
2069        return "SplitTableProcedure";
2070      }
2071    });
2072  }
2073
2074  private void warmUpRegion(ServerName server, RegionInfo region) {
2075    FutureUtils.addListener(asyncClusterConnection.getRegionServerAdmin(server)
2076      .warmupRegion(RequestConverter.buildWarmupRegionRequest(region)), (r, e) -> {
2077        if (e != null) {
2078          LOG.warn("Failed to warm up region {} on server {}", region, server, e);
2079        }
2080      });
2081  }
2082
2083  // Public so can be accessed by tests. Blocks until move is done.
2084  // Replace with an async implementation from which you can get
2085  // a success/failure result.
2086  @VisibleForTesting
2087  public void move(final byte[] encodedRegionName, byte[] destServerName) throws IOException {
2088    RegionState regionState = assignmentManager.getRegionStates().
2089      getRegionState(Bytes.toString(encodedRegionName));
2090
2091    RegionInfo hri;
2092    if (regionState != null) {
2093      hri = regionState.getRegion();
2094    } else {
2095      throw new UnknownRegionException(Bytes.toStringBinary(encodedRegionName));
2096    }
2097
2098    ServerName dest;
2099    List<ServerName> exclude = hri.getTable().isSystemTable() ? assignmentManager.getExcludedServersForSystemTable()
2100        : new ArrayList<>(1);
2101    if (destServerName != null && exclude.contains(ServerName.valueOf(Bytes.toString(destServerName)))) {
2102      LOG.info(
2103          Bytes.toString(encodedRegionName) + " can not move to " + Bytes.toString(destServerName)
2104              + " because the server is in exclude list");
2105      destServerName = null;
2106    }
2107    if (destServerName == null || destServerName.length == 0) {
2108      LOG.info("Passed destination servername is null/empty so " +
2109        "choosing a server at random");
2110      exclude.add(regionState.getServerName());
2111      final List<ServerName> destServers = this.serverManager.createDestinationServersList(exclude);
2112      dest = balancer.randomAssignment(hri, destServers);
2113      if (dest == null) {
2114        LOG.debug("Unable to determine a plan to assign " + hri);
2115        return;
2116      }
2117    } else {
2118      ServerName candidate = ServerName.valueOf(Bytes.toString(destServerName));
2119      dest = balancer.randomAssignment(hri, Lists.newArrayList(candidate));
2120      if (dest == null) {
2121        LOG.debug("Unable to determine a plan to assign " + hri);
2122        return;
2123      }
2124      // TODO: deal with table on master for rs group.
2125      if (dest.equals(serverName)) {
2126        // To avoid unnecessary region moving later by balancer. Don't put user
2127        // regions on master.
2128        LOG.debug("Skipping move of region " + hri.getRegionNameAsString() +
2129          " to avoid unnecessary region moving later by load balancer," +
2130          " because it should not be on master");
2131        return;
2132      }
2133    }
2134
2135    if (dest.equals(regionState.getServerName())) {
2136      LOG.debug("Skipping move of region " + hri.getRegionNameAsString()
2137        + " because region already assigned to the same server " + dest + ".");
2138      return;
2139    }
2140
2141    // Now we can do the move
2142    RegionPlan rp = new RegionPlan(hri, regionState.getServerName(), dest);
2143    assert rp.getDestination() != null: rp.toString() + " " + dest;
2144
2145    try {
2146      checkInitialized();
2147      if (this.cpHost != null) {
2148        this.cpHost.preMove(hri, rp.getSource(), rp.getDestination());
2149      }
2150
2151      TransitRegionStateProcedure proc =
2152        this.assignmentManager.createMoveRegionProcedure(rp.getRegionInfo(), rp.getDestination());
2153      // Warmup the region on the destination before initiating the move.
2154      // A region server could reject the close request because it either does not
2155      // have the specified region or the region is being split.
2156      warmUpRegion(rp.getDestination(), hri);
2157
2158      LOG.info(getClientIdAuditPrefix() + " move " + rp + ", running balancer");
2159      Future<byte[]> future = ProcedureSyncWait.submitProcedure(this.procedureExecutor, proc);
2160      try {
2161        // Is this going to work? Will we throw exception on error?
2162        // TODO: CompletableFuture rather than this stunted Future.
2163        future.get();
2164      } catch (InterruptedException | ExecutionException e) {
2165        throw new HBaseIOException(e);
2166      }
2167      if (this.cpHost != null) {
2168        this.cpHost.postMove(hri, rp.getSource(), rp.getDestination());
2169      }
2170    } catch (IOException ioe) {
2171      if (ioe instanceof HBaseIOException) {
2172        throw (HBaseIOException)ioe;
2173      }
2174      throw new HBaseIOException(ioe);
2175    }
2176  }
2177
2178  @Override
2179  public long createTable(final TableDescriptor tableDescriptor, final byte[][] splitKeys,
2180      final long nonceGroup, final long nonce) throws IOException {
2181    checkInitialized();
2182    TableDescriptor desc = getMasterCoprocessorHost().preCreateTableRegionsInfos(tableDescriptor);
2183    if (desc == null) {
2184      throw new IOException("Creation for " + tableDescriptor + " is canceled by CP");
2185    }
2186    String namespace = desc.getTableName().getNamespaceAsString();
2187    this.clusterSchemaService.getNamespace(namespace);
2188
2189    RegionInfo[] newRegions = ModifyRegionUtils.createRegionInfos(desc, splitKeys);
2190    TableDescriptorChecker.sanityCheck(conf, desc);
2191
2192    return MasterProcedureUtil
2193      .submitProcedure(new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {
2194        @Override
2195        protected void run() throws IOException {
2196          getMaster().getMasterCoprocessorHost().preCreateTable(desc, newRegions);
2197
2198          LOG.info(getClientIdAuditPrefix() + " create " + desc);
2199
2200          // TODO: We can handle/merge duplicate requests, and differentiate the case of
2201          // TableExistsException by saying if the schema is the same or not.
2202          //
2203          // We need to wait for the procedure to potentially fail due to "prepare" sanity
2204          // checks. This will block only the beginning of the procedure. See HBASE-19953.
2205          ProcedurePrepareLatch latch = ProcedurePrepareLatch.createBlockingLatch();
2206          submitProcedure(
2207            new CreateTableProcedure(procedureExecutor.getEnvironment(), desc, newRegions, latch));
2208          latch.await();
2209
2210          getMaster().getMasterCoprocessorHost().postCreateTable(desc, newRegions);
2211        }
2212
2213        @Override
2214        protected String getDescription() {
2215          return "CreateTableProcedure";
2216        }
2217      });
2218  }
2219
2220  @Override
2221  public long createSystemTable(final TableDescriptor tableDescriptor) throws IOException {
2222    if (isStopped()) {
2223      throw new MasterNotRunningException();
2224    }
2225
2226    TableName tableName = tableDescriptor.getTableName();
2227    if (!(tableName.isSystemTable())) {
2228      throw new IllegalArgumentException(
2229        "Only system table creation can use this createSystemTable API");
2230    }
2231
2232    RegionInfo[] newRegions = ModifyRegionUtils.createRegionInfos(tableDescriptor, null);
2233
2234    LOG.info(getClientIdAuditPrefix() + " create " + tableDescriptor);
2235
2236    // This special create table is called locally to master.  Therefore, no RPC means no need
2237    // to use nonce to detect duplicated RPC call.
2238    long procId = this.procedureExecutor.submitProcedure(
2239      new CreateTableProcedure(procedureExecutor.getEnvironment(), tableDescriptor, newRegions));
2240
2241    return procId;
2242  }
2243
2244  private void startActiveMasterManager(int infoPort) throws KeeperException {
2245    String backupZNode = ZNodePaths.joinZNode(
2246      zooKeeper.getZNodePaths().backupMasterAddressesZNode, serverName.toString());
2247    /*
2248    * Add a ZNode for ourselves in the backup master directory since we
2249    * may not become the active master. If so, we want the actual active
2250    * master to know we are backup masters, so that it won't assign
2251    * regions to us if so configured.
2252    *
2253    * If we become the active master later, ActiveMasterManager will delete
2254    * this node explicitly.  If we crash before then, ZooKeeper will delete
2255    * this node for us since it is ephemeral.
2256    */
2257    LOG.info("Adding backup master ZNode " + backupZNode);
2258    if (!MasterAddressTracker.setMasterAddress(zooKeeper, backupZNode, serverName, infoPort)) {
2259      LOG.warn("Failed create of " + backupZNode + " by " + serverName);
2260    }
2261    this.activeMasterManager.setInfoPort(infoPort);
2262    int timeout = conf.getInt(HConstants.ZK_SESSION_TIMEOUT, HConstants.DEFAULT_ZK_SESSION_TIMEOUT);
2263    // If we're a backup master, stall until a primary to write this address
2264    if (conf.getBoolean(HConstants.MASTER_TYPE_BACKUP, HConstants.DEFAULT_MASTER_TYPE_BACKUP)) {
2265      LOG.debug("HMaster started in backup mode. Stalling until master znode is written.");
2266      // This will only be a minute or so while the cluster starts up,
2267      // so don't worry about setting watches on the parent znode
2268      while (!activeMasterManager.hasActiveMaster()) {
2269        LOG.debug("Waiting for master address and cluster state znode to be written.");
2270        Threads.sleep(timeout);
2271      }
2272    }
2273    MonitoredTask status = TaskMonitor.get().createStatus("Master startup");
2274    status.setDescription("Master startup");
2275    try {
2276      if (activeMasterManager.blockUntilBecomingActiveMaster(timeout, status)) {
2277        finishActiveMasterInitialization(status);
2278      }
2279    } catch (Throwable t) {
2280      status.setStatus("Failed to become active: " + t.getMessage());
2281      LOG.error(HBaseMarkers.FATAL, "Failed to become active master", t);
2282      // HBASE-5680: Likely hadoop23 vs hadoop 20.x/1.x incompatibility
2283      if (t instanceof NoClassDefFoundError && t.getMessage().
2284          contains("org/apache/hadoop/hdfs/protocol/HdfsConstants$SafeModeAction")) {
2285        // improved error message for this special case
2286        abort("HBase is having a problem with its Hadoop jars.  You may need to recompile " +
2287          "HBase against Hadoop version " + org.apache.hadoop.util.VersionInfo.getVersion() +
2288          " or change your hadoop jars to start properly", t);
2289      } else {
2290        abort("Unhandled exception. Starting shutdown.", t);
2291      }
2292    } finally {
2293      status.cleanup();
2294    }
2295  }
2296
2297  private static boolean isCatalogTable(final TableName tableName) {
2298    return tableName.equals(TableName.META_TABLE_NAME);
2299  }
2300
2301  @Override
2302  public long deleteTable(
2303      final TableName tableName,
2304      final long nonceGroup,
2305      final long nonce) throws IOException {
2306    checkInitialized();
2307
2308    return MasterProcedureUtil.submitProcedure(
2309        new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {
2310      @Override
2311      protected void run() throws IOException {
2312        getMaster().getMasterCoprocessorHost().preDeleteTable(tableName);
2313
2314        LOG.info(getClientIdAuditPrefix() + " delete " + tableName);
2315
2316        // TODO: We can handle/merge duplicate request
2317        //
2318        // We need to wait for the procedure to potentially fail due to "prepare" sanity
2319        // checks. This will block only the beginning of the procedure. See HBASE-19953.
2320        ProcedurePrepareLatch latch = ProcedurePrepareLatch.createBlockingLatch();
2321        submitProcedure(new DeleteTableProcedure(procedureExecutor.getEnvironment(),
2322            tableName, latch));
2323        latch.await();
2324
2325        getMaster().getMasterCoprocessorHost().postDeleteTable(tableName);
2326      }
2327
2328      @Override
2329      protected String getDescription() {
2330        return "DeleteTableProcedure";
2331      }
2332    });
2333  }
2334
2335  @Override
2336  public long truncateTable(
2337      final TableName tableName,
2338      final boolean preserveSplits,
2339      final long nonceGroup,
2340      final long nonce) throws IOException {
2341    checkInitialized();
2342
2343    return MasterProcedureUtil.submitProcedure(
2344        new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {
2345      @Override
2346      protected void run() throws IOException {
2347        getMaster().getMasterCoprocessorHost().preTruncateTable(tableName);
2348
2349        LOG.info(getClientIdAuditPrefix() + " truncate " + tableName);
2350        ProcedurePrepareLatch latch = ProcedurePrepareLatch.createLatch(2, 0);
2351        submitProcedure(new TruncateTableProcedure(procedureExecutor.getEnvironment(),
2352            tableName, preserveSplits, latch));
2353        latch.await();
2354
2355        getMaster().getMasterCoprocessorHost().postTruncateTable(tableName);
2356      }
2357
2358      @Override
2359      protected String getDescription() {
2360        return "TruncateTableProcedure";
2361      }
2362    });
2363  }
2364
2365  @Override
2366  public long addColumn(final TableName tableName, final ColumnFamilyDescriptor column,
2367      final long nonceGroup, final long nonce) throws IOException {
2368    checkInitialized();
2369    checkTableExists(tableName);
2370
2371    return modifyTable(tableName, new TableDescriptorGetter() {
2372
2373      @Override
2374      public TableDescriptor get() throws IOException {
2375        TableDescriptor old = getTableDescriptors().get(tableName);
2376        if (old.hasColumnFamily(column.getName())) {
2377          throw new InvalidFamilyOperationException("Column family '" + column.getNameAsString()
2378              + "' in table '" + tableName + "' already exists so cannot be added");
2379        }
2380
2381        return TableDescriptorBuilder.newBuilder(old).setColumnFamily(column).build();
2382      }
2383    }, nonceGroup, nonce, true);
2384  }
2385
2386  /**
2387   * Implement to return TableDescriptor after pre-checks
2388   */
2389  protected interface TableDescriptorGetter {
2390    TableDescriptor get() throws IOException;
2391  }
2392
2393  @Override
2394  public long modifyColumn(final TableName tableName, final ColumnFamilyDescriptor descriptor,
2395      final long nonceGroup, final long nonce) throws IOException {
2396    checkInitialized();
2397    checkTableExists(tableName);
2398    return modifyTable(tableName, new TableDescriptorGetter() {
2399
2400      @Override
2401      public TableDescriptor get() throws IOException {
2402        TableDescriptor old = getTableDescriptors().get(tableName);
2403        if (!old.hasColumnFamily(descriptor.getName())) {
2404          throw new InvalidFamilyOperationException("Family '" + descriptor.getNameAsString()
2405              + "' does not exist, so it cannot be modified");
2406        }
2407
2408        return TableDescriptorBuilder.newBuilder(old).modifyColumnFamily(descriptor).build();
2409      }
2410    }, nonceGroup, nonce, true);
2411  }
2412
2413  @Override
2414  public long deleteColumn(final TableName tableName, final byte[] columnName,
2415      final long nonceGroup, final long nonce) throws IOException {
2416    checkInitialized();
2417    checkTableExists(tableName);
2418
2419    return modifyTable(tableName, new TableDescriptorGetter() {
2420
2421      @Override
2422      public TableDescriptor get() throws IOException {
2423        TableDescriptor old = getTableDescriptors().get(tableName);
2424
2425        if (!old.hasColumnFamily(columnName)) {
2426          throw new InvalidFamilyOperationException("Family '" + Bytes.toString(columnName)
2427              + "' does not exist, so it cannot be deleted");
2428        }
2429        if (old.getColumnFamilyCount() == 1) {
2430          throw new InvalidFamilyOperationException("Family '" + Bytes.toString(columnName)
2431              + "' is the only column family in the table, so it cannot be deleted");
2432        }
2433        return TableDescriptorBuilder.newBuilder(old).removeColumnFamily(columnName).build();
2434      }
2435    }, nonceGroup, nonce, true);
2436  }
2437
2438  @Override
2439  public long enableTable(final TableName tableName, final long nonceGroup, final long nonce)
2440      throws IOException {
2441    checkInitialized();
2442
2443    return MasterProcedureUtil.submitProcedure(
2444        new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {
2445      @Override
2446      protected void run() throws IOException {
2447        getMaster().getMasterCoprocessorHost().preEnableTable(tableName);
2448
2449        // Normally, it would make sense for this authorization check to exist inside
2450        // AccessController, but because the authorization check is done based on internal state
2451        // (rather than explicit permissions) we'll do the check here instead of in the
2452        // coprocessor.
2453        MasterQuotaManager quotaManager = getMasterQuotaManager();
2454        if (quotaManager != null) {
2455          if (quotaManager.isQuotaInitialized()) {
2456              SpaceQuotaSnapshot currSnapshotOfTable =
2457                  QuotaTableUtil.getCurrentSnapshotFromQuotaTable(getConnection(), tableName);
2458              if (currSnapshotOfTable != null) {
2459                SpaceQuotaStatus quotaStatus = currSnapshotOfTable.getQuotaStatus();
2460                if (quotaStatus.isInViolation()
2461                    && SpaceViolationPolicy.DISABLE == quotaStatus.getPolicy().orElse(null)) {
2462                throw new AccessDeniedException("Enabling the table '" + tableName
2463                    + "' is disallowed due to a violated space quota.");
2464              }
2465            }
2466          } else if (LOG.isTraceEnabled()) {
2467            LOG.trace("Unable to check for space quotas as the MasterQuotaManager is not enabled");
2468          }
2469        }
2470
2471        LOG.info(getClientIdAuditPrefix() + " enable " + tableName);
2472
2473        // Execute the operation asynchronously - client will check the progress of the operation
2474        // In case the request is from a <1.1 client before returning,
2475        // we want to make sure that the table is prepared to be
2476        // enabled (the table is locked and the table state is set).
2477        // Note: if the procedure throws exception, we will catch it and rethrow.
2478        final ProcedurePrepareLatch prepareLatch = ProcedurePrepareLatch.createLatch();
2479        submitProcedure(new EnableTableProcedure(procedureExecutor.getEnvironment(),
2480            tableName, prepareLatch));
2481        prepareLatch.await();
2482
2483        getMaster().getMasterCoprocessorHost().postEnableTable(tableName);
2484      }
2485
2486      @Override
2487      protected String getDescription() {
2488        return "EnableTableProcedure";
2489      }
2490    });
2491  }
2492
2493  @Override
2494  public long disableTable(final TableName tableName, final long nonceGroup, final long nonce)
2495      throws IOException {
2496    checkInitialized();
2497
2498    return MasterProcedureUtil.submitProcedure(
2499        new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {
2500      @Override
2501      protected void run() throws IOException {
2502        getMaster().getMasterCoprocessorHost().preDisableTable(tableName);
2503
2504        LOG.info(getClientIdAuditPrefix() + " disable " + tableName);
2505
2506        // Execute the operation asynchronously - client will check the progress of the operation
2507        // In case the request is from a <1.1 client before returning,
2508        // we want to make sure that the table is prepared to be
2509        // enabled (the table is locked and the table state is set).
2510        // Note: if the procedure throws exception, we will catch it and rethrow.
2511        //
2512        // We need to wait for the procedure to potentially fail due to "prepare" sanity
2513        // checks. This will block only the beginning of the procedure. See HBASE-19953.
2514        final ProcedurePrepareLatch prepareLatch = ProcedurePrepareLatch.createBlockingLatch();
2515        submitProcedure(new DisableTableProcedure(procedureExecutor.getEnvironment(),
2516            tableName, false, prepareLatch));
2517        prepareLatch.await();
2518
2519        getMaster().getMasterCoprocessorHost().postDisableTable(tableName);
2520      }
2521
2522      @Override
2523      protected String getDescription() {
2524        return "DisableTableProcedure";
2525      }
2526    });
2527  }
2528
2529  private long modifyTable(final TableName tableName,
2530      final TableDescriptorGetter newDescriptorGetter, final long nonceGroup, final long nonce,
2531      final boolean shouldCheckDescriptor) throws IOException {
2532    return MasterProcedureUtil
2533        .submitProcedure(new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {
2534          @Override
2535          protected void run() throws IOException {
2536            TableDescriptor oldDescriptor = getMaster().getTableDescriptors().get(tableName);
2537            TableDescriptor newDescriptor = getMaster().getMasterCoprocessorHost()
2538                .preModifyTable(tableName, oldDescriptor, newDescriptorGetter.get());
2539            TableDescriptorChecker.sanityCheck(conf, newDescriptor);
2540            LOG.info("{} modify table {} from {} to {}", getClientIdAuditPrefix(), tableName,
2541                oldDescriptor, newDescriptor);
2542
2543            // Execute the operation synchronously - wait for the operation completes before
2544            // continuing.
2545            //
2546            // We need to wait for the procedure to potentially fail due to "prepare" sanity
2547            // checks. This will block only the beginning of the procedure. See HBASE-19953.
2548            ProcedurePrepareLatch latch = ProcedurePrepareLatch.createBlockingLatch();
2549            submitProcedure(new ModifyTableProcedure(procedureExecutor.getEnvironment(),
2550                newDescriptor, latch, oldDescriptor, shouldCheckDescriptor));
2551            latch.await();
2552
2553            getMaster().getMasterCoprocessorHost().postModifyTable(tableName, oldDescriptor,
2554              newDescriptor);
2555          }
2556
2557          @Override
2558          protected String getDescription() {
2559            return "ModifyTableProcedure";
2560          }
2561        });
2562
2563  }
2564
2565  @Override
2566  public long modifyTable(final TableName tableName, final TableDescriptor newDescriptor,
2567      final long nonceGroup, final long nonce) throws IOException {
2568    checkInitialized();
2569    return modifyTable(tableName, new TableDescriptorGetter() {
2570      @Override
2571      public TableDescriptor get() throws IOException {
2572        return newDescriptor;
2573      }
2574    }, nonceGroup, nonce, false);
2575
2576  }
2577
2578  public long restoreSnapshot(final SnapshotDescription snapshotDesc,
2579      final long nonceGroup, final long nonce, final boolean restoreAcl) throws IOException {
2580    checkInitialized();
2581    getSnapshotManager().checkSnapshotSupport();
2582
2583    // Ensure namespace exists. Will throw exception if non-known NS.
2584    final TableName dstTable = TableName.valueOf(snapshotDesc.getTable());
2585    getClusterSchema().getNamespace(dstTable.getNamespaceAsString());
2586
2587    return MasterProcedureUtil.submitProcedure(
2588        new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {
2589      @Override
2590      protected void run() throws IOException {
2591          setProcId(
2592            getSnapshotManager().restoreOrCloneSnapshot(snapshotDesc, getNonceKey(), restoreAcl));
2593      }
2594
2595      @Override
2596      protected String getDescription() {
2597        return "RestoreSnapshotProcedure";
2598      }
2599    });
2600  }
2601
2602  private void checkTableExists(final TableName tableName)
2603    throws IOException, TableNotFoundException {
2604    if (!tableDescriptors.exists(tableName)) {
2605      throw new TableNotFoundException(tableName);
2606    }
2607  }
2608
2609  @Override
2610  public void checkTableModifiable(final TableName tableName)
2611      throws IOException, TableNotFoundException, TableNotDisabledException {
2612    if (isCatalogTable(tableName)) {
2613      throw new IOException("Can't modify catalog tables");
2614    }
2615    checkTableExists(tableName);
2616    TableState ts = getTableStateManager().getTableState(tableName);
2617    if (!ts.isDisabled()) {
2618      throw new TableNotDisabledException("Not DISABLED; " + ts);
2619    }
2620  }
2621
2622  public ClusterMetrics getClusterMetricsWithoutCoprocessor() throws InterruptedIOException {
2623    return getClusterMetricsWithoutCoprocessor(EnumSet.allOf(Option.class));
2624  }
2625
2626  public ClusterMetrics getClusterMetricsWithoutCoprocessor(EnumSet<Option> options)
2627      throws InterruptedIOException {
2628    ClusterMetricsBuilder builder = ClusterMetricsBuilder.newBuilder();
2629    // given that hbase1 can't submit the request with Option,
2630    // we return all information to client if the list of Option is empty.
2631    if (options.isEmpty()) {
2632      options = EnumSet.allOf(Option.class);
2633    }
2634
2635    for (Option opt : options) {
2636      switch (opt) {
2637        case HBASE_VERSION: builder.setHBaseVersion(VersionInfo.getVersion()); break;
2638        case CLUSTER_ID: builder.setClusterId(getClusterId()); break;
2639        case MASTER: builder.setMasterName(getServerName()); break;
2640        case BACKUP_MASTERS: builder.setBackerMasterNames(getBackupMasters()); break;
2641        case LIVE_SERVERS: {
2642          if (serverManager != null) {
2643            builder.setLiveServerMetrics(serverManager.getOnlineServers().entrySet().stream()
2644              .collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue())));
2645          }
2646          break;
2647        }
2648        case DEAD_SERVERS: {
2649          if (serverManager != null) {
2650            builder.setDeadServerNames(new ArrayList<>(
2651              serverManager.getDeadServers().copyServerNames()));
2652          }
2653          break;
2654        }
2655        case MASTER_COPROCESSORS: {
2656          if (cpHost != null) {
2657            builder.setMasterCoprocessorNames(Arrays.asList(getMasterCoprocessors()));
2658          }
2659          break;
2660        }
2661        case REGIONS_IN_TRANSITION: {
2662          if (assignmentManager != null) {
2663            builder.setRegionsInTransition(assignmentManager.getRegionStates()
2664                .getRegionsStateInTransition());
2665          }
2666          break;
2667        }
2668        case BALANCER_ON: {
2669          if (loadBalancerTracker != null) {
2670            builder.setBalancerOn(loadBalancerTracker.isBalancerOn());
2671          }
2672          break;
2673        }
2674        case MASTER_INFO_PORT: {
2675          if (infoServer != null) {
2676            builder.setMasterInfoPort(infoServer.getPort());
2677          }
2678          break;
2679        }
2680        case SERVERS_NAME: {
2681          if (serverManager != null) {
2682            builder.setServerNames(serverManager.getOnlineServersList());
2683          }
2684          break;
2685        }
2686        case TABLE_TO_REGIONS_COUNT: {
2687          if (isActiveMaster() && isInitialized() && assignmentManager != null) {
2688            try {
2689              Map<TableName, RegionStatesCount> tableRegionStatesCountMap = new HashMap<>();
2690              Map<String, TableDescriptor> tableDescriptorMap = getTableDescriptors().getAll();
2691              for (TableDescriptor tableDescriptor : tableDescriptorMap.values()) {
2692                TableName tableName = tableDescriptor.getTableName();
2693                RegionStatesCount regionStatesCount = assignmentManager
2694                  .getRegionStatesCount(tableName);
2695                tableRegionStatesCountMap.put(tableName, regionStatesCount);
2696              }
2697              builder.setTableRegionStatesCount(tableRegionStatesCountMap);
2698            } catch (IOException e) {
2699              LOG.error("Error while populating TABLE_TO_REGIONS_COUNT for Cluster Metrics..", e);
2700            }
2701          }
2702          break;
2703        }
2704      }
2705    }
2706    return builder.build();
2707  }
2708
2709  /**
2710   * @return cluster status
2711   */
2712  public ClusterMetrics getClusterMetrics() throws IOException {
2713    return getClusterMetrics(EnumSet.allOf(Option.class));
2714  }
2715
2716  public ClusterMetrics getClusterMetrics(EnumSet<Option> options) throws IOException {
2717    if (cpHost != null) {
2718      cpHost.preGetClusterMetrics();
2719    }
2720    ClusterMetrics status = getClusterMetricsWithoutCoprocessor(options);
2721    if (cpHost != null) {
2722      cpHost.postGetClusterMetrics(status);
2723    }
2724    return status;
2725  }
2726
2727  List<ServerName> getBackupMasters() {
2728    return activeMasterManager.getBackupMasters();
2729  }
2730
2731  /**
2732   * The set of loaded coprocessors is stored in a static set. Since it's
2733   * statically allocated, it does not require that HMaster's cpHost be
2734   * initialized prior to accessing it.
2735   * @return a String representation of the set of names of the loaded coprocessors.
2736   */
2737  public static String getLoadedCoprocessors() {
2738    return CoprocessorHost.getLoadedCoprocessors().toString();
2739  }
2740
2741  /**
2742   * @return timestamp in millis when HMaster was started.
2743   */
2744  public long getMasterStartTime() {
2745    return startcode;
2746  }
2747
2748  /**
2749   * @return timestamp in millis when HMaster became the active master.
2750   */
2751  public long getMasterActiveTime() {
2752    return masterActiveTime;
2753  }
2754
2755  /**
2756   * @return timestamp in millis when HMaster finished becoming the active master
2757   */
2758  public long getMasterFinishedInitializationTime() {
2759    return masterFinishedInitializationTime;
2760  }
2761
2762  public int getNumWALFiles() {
2763    return 0;
2764  }
2765
2766  public ProcedureStore getProcedureStore() {
2767    return procedureStore;
2768  }
2769
2770  public int getRegionServerInfoPort(final ServerName sn) {
2771    int port = this.serverManager.getInfoPort(sn);
2772    return port == 0 ? conf.getInt(HConstants.REGIONSERVER_INFO_PORT,
2773      HConstants.DEFAULT_REGIONSERVER_INFOPORT) : port;
2774  }
2775
2776  @Override
2777  public String getRegionServerVersion(ServerName sn) {
2778    // Will return "0.0.0" if the server is not online to prevent move system region to unknown
2779    // version RS.
2780    return this.serverManager.getVersion(sn);
2781  }
2782
2783  @Override
2784  public void checkIfShouldMoveSystemRegionAsync() {
2785    assignmentManager.checkIfShouldMoveSystemRegionAsync();
2786  }
2787
2788  /**
2789   * @return array of coprocessor SimpleNames.
2790   */
2791  public String[] getMasterCoprocessors() {
2792    Set<String> masterCoprocessors = getMasterCoprocessorHost().getCoprocessors();
2793    return masterCoprocessors.toArray(new String[masterCoprocessors.size()]);
2794  }
2795
2796  @Override
2797  public void abort(String reason, Throwable cause) {
2798    if (!setAbortRequested() || isStopped()) {
2799      LOG.debug("Abort called but aborted={}, stopped={}", isAborted(), isStopped());
2800      return;
2801    }
2802    if (cpHost != null) {
2803      // HBASE-4014: dump a list of loaded coprocessors.
2804      LOG.error(HBaseMarkers.FATAL, "Master server abort: loaded coprocessors are: " +
2805          getLoadedCoprocessors());
2806    }
2807    String msg = "***** ABORTING master " + this + ": " + reason + " *****";
2808    if (cause != null) {
2809      LOG.error(HBaseMarkers.FATAL, msg, cause);
2810    } else {
2811      LOG.error(HBaseMarkers.FATAL, msg);
2812    }
2813
2814    try {
2815      stopMaster();
2816    } catch (IOException e) {
2817      LOG.error("Exception occurred while stopping master", e);
2818    }
2819  }
2820
2821  @Override
2822  public ZKWatcher getZooKeeper() {
2823    return zooKeeper;
2824  }
2825
2826  @Override
2827  public MasterCoprocessorHost getMasterCoprocessorHost() {
2828    return cpHost;
2829  }
2830
2831  @Override
2832  public MasterQuotaManager getMasterQuotaManager() {
2833    return quotaManager;
2834  }
2835
2836  @Override
2837  public ProcedureExecutor<MasterProcedureEnv> getMasterProcedureExecutor() {
2838    return procedureExecutor;
2839  }
2840
2841  @Override
2842  public ServerName getServerName() {
2843    return this.serverName;
2844  }
2845
2846  @Override
2847  public AssignmentManager getAssignmentManager() {
2848    return this.assignmentManager;
2849  }
2850
2851  @Override
2852  public CatalogJanitor getCatalogJanitor() {
2853    return this.catalogJanitorChore;
2854  }
2855
2856  public MemoryBoundedLogMessageBuffer getRegionServerFatalLogBuffer() {
2857    return rsFatals;
2858  }
2859
2860  /**
2861   * Shutdown the cluster.
2862   * Master runs a coordinated stop of all RegionServers and then itself.
2863   */
2864  public void shutdown() throws IOException {
2865    if (cpHost != null) {
2866      cpHost.preShutdown();
2867    }
2868
2869    // Tell the servermanager cluster shutdown has been called. This makes it so when Master is
2870    // last running server, it'll stop itself. Next, we broadcast the cluster shutdown by setting
2871    // the cluster status as down. RegionServers will notice this change in state and will start
2872    // shutting themselves down. When last has exited, Master can go down.
2873    if (this.serverManager != null) {
2874      this.serverManager.shutdownCluster();
2875    }
2876    if (this.clusterStatusTracker != null) {
2877      try {
2878        this.clusterStatusTracker.setClusterDown();
2879      } catch (KeeperException e) {
2880        LOG.error("ZooKeeper exception trying to set cluster as down in ZK", e);
2881      }
2882    }
2883    // Stop the procedure executor. Will stop any ongoing assign, unassign, server crash etc.,
2884    // processing so we can go down.
2885    if (this.procedureExecutor != null) {
2886      this.procedureExecutor.stop();
2887    }
2888    // Shutdown our cluster connection. This will kill any hosted RPCs that might be going on;
2889    // this is what we want especially if the Master is in startup phase doing call outs to
2890    // hbase:meta, etc. when cluster is down. Without ths connection close, we'd have to wait on
2891    // the rpc to timeout.
2892    if (this.asyncClusterConnection != null) {
2893      this.asyncClusterConnection.close();
2894    }
2895  }
2896
2897  public void stopMaster() throws IOException {
2898    if (cpHost != null) {
2899      cpHost.preStopMaster();
2900    }
2901    stop("Stopped by " + Thread.currentThread().getName());
2902  }
2903
2904  @Override
2905  public void stop(String msg) {
2906    if (!isStopped()) {
2907      super.stop(msg);
2908      if (this.activeMasterManager != null) {
2909        this.activeMasterManager.stop();
2910      }
2911    }
2912  }
2913
2914  @VisibleForTesting
2915  protected void checkServiceStarted() throws ServerNotRunningYetException {
2916    if (!serviceStarted) {
2917      throw new ServerNotRunningYetException("Server is not running yet");
2918    }
2919  }
2920
2921  public static class MasterStoppedException extends DoNotRetryIOException {
2922    MasterStoppedException() {
2923      super();
2924    }
2925  }
2926
2927  void checkInitialized() throws PleaseHoldException, ServerNotRunningYetException,
2928      MasterNotRunningException, MasterStoppedException {
2929    checkServiceStarted();
2930    if (!isInitialized()) {
2931      throw new PleaseHoldException("Master is initializing");
2932    }
2933    if (isStopped()) {
2934      throw new MasterStoppedException();
2935    }
2936  }
2937
2938  /**
2939   * Report whether this master is currently the active master or not.
2940   * If not active master, we are parked on ZK waiting to become active.
2941   *
2942   * This method is used for testing.
2943   *
2944   * @return true if active master, false if not.
2945   */
2946  @Override
2947  public boolean isActiveMaster() {
2948    return activeMaster;
2949  }
2950
2951  /**
2952   * Report whether this master has completed with its initialization and is
2953   * ready.  If ready, the master is also the active master.  A standby master
2954   * is never ready.
2955   *
2956   * This method is used for testing.
2957   *
2958   * @return true if master is ready to go, false if not.
2959   */
2960  @Override
2961  public boolean isInitialized() {
2962    return initialized.isReady();
2963  }
2964
2965  /**
2966   * Report whether this master is in maintenance mode.
2967   *
2968   * @return true if master is in maintenanceMode
2969   */
2970  @Override
2971  public boolean isInMaintenanceMode() {
2972    return maintenanceMode;
2973  }
2974
2975  @VisibleForTesting
2976  public void setInitialized(boolean isInitialized) {
2977    procedureExecutor.getEnvironment().setEventReady(initialized, isInitialized);
2978  }
2979
2980  @Override
2981  public ProcedureEvent<?> getInitializedEvent() {
2982    return initialized;
2983  }
2984
2985  /**
2986   * Compute the average load across all region servers.
2987   * Currently, this uses a very naive computation - just uses the number of
2988   * regions being served, ignoring stats about number of requests.
2989   * @return the average load
2990   */
2991  public double getAverageLoad() {
2992    if (this.assignmentManager == null) {
2993      return 0;
2994    }
2995
2996    RegionStates regionStates = this.assignmentManager.getRegionStates();
2997    if (regionStates == null) {
2998      return 0;
2999    }
3000    return regionStates.getAverageLoad();
3001  }
3002
3003  /*
3004   * @return the count of region split plans executed
3005   */
3006  public long getSplitPlanCount() {
3007    return splitPlanCount;
3008  }
3009
3010  /*
3011   * @return the count of region merge plans executed
3012   */
3013  public long getMergePlanCount() {
3014    return mergePlanCount;
3015  }
3016
3017  @Override
3018  public boolean registerService(Service instance) {
3019    /*
3020     * No stacking of instances is allowed for a single service name
3021     */
3022    Descriptors.ServiceDescriptor serviceDesc = instance.getDescriptorForType();
3023    String serviceName = CoprocessorRpcUtils.getServiceName(serviceDesc);
3024    if (coprocessorServiceHandlers.containsKey(serviceName)) {
3025      LOG.error("Coprocessor service "+serviceName+
3026          " already registered, rejecting request from "+instance
3027      );
3028      return false;
3029    }
3030
3031    coprocessorServiceHandlers.put(serviceName, instance);
3032    if (LOG.isDebugEnabled()) {
3033      LOG.debug("Registered master coprocessor service: service="+serviceName);
3034    }
3035    return true;
3036  }
3037
3038  /**
3039   * Utility for constructing an instance of the passed HMaster class.
3040   * @param masterClass
3041   * @return HMaster instance.
3042   */
3043  public static HMaster constructMaster(Class<? extends HMaster> masterClass,
3044      final Configuration conf)  {
3045    try {
3046      Constructor<? extends HMaster> c = masterClass.getConstructor(Configuration.class);
3047      return c.newInstance(conf);
3048    } catch(Exception e) {
3049      Throwable error = e;
3050      if (e instanceof InvocationTargetException &&
3051          ((InvocationTargetException)e).getTargetException() != null) {
3052        error = ((InvocationTargetException)e).getTargetException();
3053      }
3054      throw new RuntimeException("Failed construction of Master: " + masterClass.toString() + ". "
3055        , error);
3056    }
3057  }
3058
3059  /**
3060   * @see org.apache.hadoop.hbase.master.HMasterCommandLine
3061   */
3062  public static void main(String [] args) {
3063    LOG.info("STARTING service " + HMaster.class.getSimpleName());
3064    VersionInfo.logVersion();
3065    new HMasterCommandLine(HMaster.class).doMain(args);
3066  }
3067
3068  public HFileCleaner getHFileCleaner() {
3069    return this.hfileCleaner;
3070  }
3071
3072  public LogCleaner getLogCleaner() {
3073    return this.logCleaner;
3074  }
3075
3076  /**
3077   * @return the underlying snapshot manager
3078   */
3079  @Override
3080  public SnapshotManager getSnapshotManager() {
3081    return this.snapshotManager;
3082  }
3083
3084  /**
3085   * @return the underlying MasterProcedureManagerHost
3086   */
3087  @Override
3088  public MasterProcedureManagerHost getMasterProcedureManagerHost() {
3089    return mpmHost;
3090  }
3091
3092  @Override
3093  public ClusterSchema getClusterSchema() {
3094    return this.clusterSchemaService;
3095  }
3096
3097  /**
3098   * Create a new Namespace.
3099   * @param namespaceDescriptor descriptor for new Namespace
3100   * @param nonceGroup Identifier for the source of the request, a client or process.
3101   * @param nonce A unique identifier for this operation from the client or process identified by
3102   * <code>nonceGroup</code> (the source must ensure each operation gets a unique id).
3103   * @return procedure id
3104   */
3105  long createNamespace(final NamespaceDescriptor namespaceDescriptor, final long nonceGroup,
3106      final long nonce) throws IOException {
3107    checkInitialized();
3108
3109    TableName.isLegalNamespaceName(Bytes.toBytes(namespaceDescriptor.getName()));
3110
3111    return MasterProcedureUtil.submitProcedure(new MasterProcedureUtil.NonceProcedureRunnable(this,
3112          nonceGroup, nonce) {
3113      @Override
3114      protected void run() throws IOException {
3115        getMaster().getMasterCoprocessorHost().preCreateNamespace(namespaceDescriptor);
3116        // We need to wait for the procedure to potentially fail due to "prepare" sanity
3117        // checks. This will block only the beginning of the procedure. See HBASE-19953.
3118        ProcedurePrepareLatch latch = ProcedurePrepareLatch.createBlockingLatch();
3119        LOG.info(getClientIdAuditPrefix() + " creating " + namespaceDescriptor);
3120        // Execute the operation synchronously - wait for the operation to complete before
3121        // continuing.
3122        setProcId(getClusterSchema().createNamespace(namespaceDescriptor, getNonceKey(), latch));
3123        latch.await();
3124        getMaster().getMasterCoprocessorHost().postCreateNamespace(namespaceDescriptor);
3125      }
3126
3127      @Override
3128      protected String getDescription() {
3129        return "CreateNamespaceProcedure";
3130      }
3131    });
3132  }
3133
3134  /**
3135   * Modify an existing Namespace.
3136   * @param nonceGroup Identifier for the source of the request, a client or process.
3137   * @param nonce A unique identifier for this operation from the client or process identified by
3138   * <code>nonceGroup</code> (the source must ensure each operation gets a unique id).
3139   * @return procedure id
3140   */
3141  long modifyNamespace(final NamespaceDescriptor newNsDescriptor, final long nonceGroup,
3142      final long nonce) throws IOException {
3143    checkInitialized();
3144
3145    TableName.isLegalNamespaceName(Bytes.toBytes(newNsDescriptor.getName()));
3146
3147    return MasterProcedureUtil.submitProcedure(new MasterProcedureUtil.NonceProcedureRunnable(this,
3148          nonceGroup, nonce) {
3149      @Override
3150      protected void run() throws IOException {
3151        NamespaceDescriptor oldNsDescriptor = getNamespace(newNsDescriptor.getName());
3152        getMaster().getMasterCoprocessorHost().preModifyNamespace(oldNsDescriptor, newNsDescriptor);
3153        // We need to wait for the procedure to potentially fail due to "prepare" sanity
3154        // checks. This will block only the beginning of the procedure. See HBASE-19953.
3155        ProcedurePrepareLatch latch = ProcedurePrepareLatch.createBlockingLatch();
3156        LOG.info(getClientIdAuditPrefix() + " modify " + newNsDescriptor);
3157        // Execute the operation synchronously - wait for the operation to complete before
3158        // continuing.
3159        setProcId(getClusterSchema().modifyNamespace(newNsDescriptor, getNonceKey(), latch));
3160        latch.await();
3161        getMaster().getMasterCoprocessorHost().postModifyNamespace(oldNsDescriptor,
3162          newNsDescriptor);
3163      }
3164
3165      @Override
3166      protected String getDescription() {
3167        return "ModifyNamespaceProcedure";
3168      }
3169    });
3170  }
3171
3172  /**
3173   * Delete an existing Namespace. Only empty Namespaces (no tables) can be removed.
3174   * @param nonceGroup Identifier for the source of the request, a client or process.
3175   * @param nonce A unique identifier for this operation from the client or process identified by
3176   * <code>nonceGroup</code> (the source must ensure each operation gets a unique id).
3177   * @return procedure id
3178   */
3179  long deleteNamespace(final String name, final long nonceGroup, final long nonce)
3180      throws IOException {
3181    checkInitialized();
3182
3183    return MasterProcedureUtil.submitProcedure(new MasterProcedureUtil.NonceProcedureRunnable(this,
3184          nonceGroup, nonce) {
3185      @Override
3186      protected void run() throws IOException {
3187        getMaster().getMasterCoprocessorHost().preDeleteNamespace(name);
3188        LOG.info(getClientIdAuditPrefix() + " delete " + name);
3189        // Execute the operation synchronously - wait for the operation to complete before
3190        // continuing.
3191        //
3192        // We need to wait for the procedure to potentially fail due to "prepare" sanity
3193        // checks. This will block only the beginning of the procedure. See HBASE-19953.
3194        ProcedurePrepareLatch latch = ProcedurePrepareLatch.createBlockingLatch();
3195        setProcId(submitProcedure(
3196              new DeleteNamespaceProcedure(procedureExecutor.getEnvironment(), name, latch)));
3197        latch.await();
3198        // Will not be invoked in the face of Exception thrown by the Procedure's execution
3199        getMaster().getMasterCoprocessorHost().postDeleteNamespace(name);
3200      }
3201
3202      @Override
3203      protected String getDescription() {
3204        return "DeleteNamespaceProcedure";
3205      }
3206    });
3207  }
3208
3209  /**
3210   * Get a Namespace
3211   * @param name Name of the Namespace
3212   * @return Namespace descriptor for <code>name</code>
3213   */
3214  NamespaceDescriptor getNamespace(String name) throws IOException {
3215    checkInitialized();
3216    if (this.cpHost != null) this.cpHost.preGetNamespaceDescriptor(name);
3217    NamespaceDescriptor nsd = this.clusterSchemaService.getNamespace(name);
3218    if (this.cpHost != null) this.cpHost.postGetNamespaceDescriptor(nsd);
3219    return nsd;
3220  }
3221
3222  /**
3223   * Get all Namespaces
3224   * @return All Namespace descriptors
3225   */
3226  List<NamespaceDescriptor> getNamespaces() throws IOException {
3227    checkInitialized();
3228    final List<NamespaceDescriptor> nsds = new ArrayList<>();
3229    if (cpHost != null) {
3230      cpHost.preListNamespaceDescriptors(nsds);
3231    }
3232    nsds.addAll(this.clusterSchemaService.getNamespaces());
3233    if (this.cpHost != null) {
3234      this.cpHost.postListNamespaceDescriptors(nsds);
3235    }
3236    return nsds;
3237  }
3238
3239  /**
3240   * List namespace names
3241   * @return All namespace names
3242   */
3243  public List<String> listNamespaces() throws IOException {
3244    checkInitialized();
3245    List<String> namespaces = new ArrayList<>();
3246    if (cpHost != null) {
3247      cpHost.preListNamespaces(namespaces);
3248    }
3249    for (NamespaceDescriptor namespace : clusterSchemaService.getNamespaces()) {
3250      namespaces.add(namespace.getName());
3251    }
3252    if (cpHost != null) {
3253      cpHost.postListNamespaces(namespaces);
3254    }
3255    return namespaces;
3256  }
3257
3258  @Override
3259  public List<TableName> listTableNamesByNamespace(String name) throws IOException {
3260    checkInitialized();
3261    return listTableNames(name, null, true);
3262  }
3263
3264  @Override
3265  public List<TableDescriptor> listTableDescriptorsByNamespace(String name) throws IOException {
3266    checkInitialized();
3267    return listTableDescriptors(name, null, null, true);
3268  }
3269
3270  @Override
3271  public boolean abortProcedure(final long procId, final boolean mayInterruptIfRunning)
3272      throws IOException {
3273    if (cpHost != null) {
3274      cpHost.preAbortProcedure(this.procedureExecutor, procId);
3275    }
3276
3277    final boolean result = this.procedureExecutor.abort(procId, mayInterruptIfRunning);
3278
3279    if (cpHost != null) {
3280      cpHost.postAbortProcedure();
3281    }
3282
3283    return result;
3284  }
3285
3286  @Override
3287  public List<Procedure<?>> getProcedures() throws IOException {
3288    if (cpHost != null) {
3289      cpHost.preGetProcedures();
3290    }
3291
3292    @SuppressWarnings({ "unchecked", "rawtypes" })
3293    List<Procedure<?>> procList = (List) this.procedureExecutor.getProcedures();
3294
3295    if (cpHost != null) {
3296      cpHost.postGetProcedures(procList);
3297    }
3298
3299    return procList;
3300  }
3301
3302  @Override
3303  public List<LockedResource> getLocks() throws IOException {
3304    if (cpHost != null) {
3305      cpHost.preGetLocks();
3306    }
3307
3308    MasterProcedureScheduler procedureScheduler =
3309      procedureExecutor.getEnvironment().getProcedureScheduler();
3310
3311    final List<LockedResource> lockedResources = procedureScheduler.getLocks();
3312
3313    if (cpHost != null) {
3314      cpHost.postGetLocks(lockedResources);
3315    }
3316
3317    return lockedResources;
3318  }
3319
3320  /**
3321   * Returns the list of table descriptors that match the specified request
3322   * @param namespace the namespace to query, or null if querying for all
3323   * @param regex The regular expression to match against, or null if querying for all
3324   * @param tableNameList the list of table names, or null if querying for all
3325   * @param includeSysTables False to match only against userspace tables
3326   * @return the list of table descriptors
3327   */
3328  public List<TableDescriptor> listTableDescriptors(final String namespace, final String regex,
3329      final List<TableName> tableNameList, final boolean includeSysTables)
3330  throws IOException {
3331    List<TableDescriptor> htds = new ArrayList<>();
3332    if (cpHost != null) {
3333      cpHost.preGetTableDescriptors(tableNameList, htds, regex);
3334    }
3335    htds = getTableDescriptors(htds, namespace, regex, tableNameList, includeSysTables);
3336    if (cpHost != null) {
3337      cpHost.postGetTableDescriptors(tableNameList, htds, regex);
3338    }
3339    return htds;
3340  }
3341
3342  /**
3343   * Returns the list of table names that match the specified request
3344   * @param regex The regular expression to match against, or null if querying for all
3345   * @param namespace the namespace to query, or null if querying for all
3346   * @param includeSysTables False to match only against userspace tables
3347   * @return the list of table names
3348   */
3349  public List<TableName> listTableNames(final String namespace, final String regex,
3350      final boolean includeSysTables) throws IOException {
3351    List<TableDescriptor> htds = new ArrayList<>();
3352    if (cpHost != null) {
3353      cpHost.preGetTableNames(htds, regex);
3354    }
3355    htds = getTableDescriptors(htds, namespace, regex, null, includeSysTables);
3356    if (cpHost != null) {
3357      cpHost.postGetTableNames(htds, regex);
3358    }
3359    List<TableName> result = new ArrayList<>(htds.size());
3360    for (TableDescriptor htd: htds) result.add(htd.getTableName());
3361    return result;
3362  }
3363
3364  /**
3365   * Return a list of table table descriptors after applying any provided filter parameters. Note
3366   * that the user-facing description of this filter logic is presented on the class-level javadoc
3367   * of {@link NormalizeTableFilterParams}.
3368   */
3369  private List<TableDescriptor> getTableDescriptors(final List<TableDescriptor> htds,
3370      final String namespace, final String regex, final List<TableName> tableNameList,
3371      final boolean includeSysTables)
3372  throws IOException {
3373    if (tableNameList == null || tableNameList.isEmpty()) {
3374      // request for all TableDescriptors
3375      Collection<TableDescriptor> allHtds;
3376      if (namespace != null && namespace.length() > 0) {
3377        // Do a check on the namespace existence. Will fail if does not exist.
3378        this.clusterSchemaService.getNamespace(namespace);
3379        allHtds = tableDescriptors.getByNamespace(namespace).values();
3380      } else {
3381        allHtds = tableDescriptors.getAll().values();
3382      }
3383      for (TableDescriptor desc: allHtds) {
3384        if (tableStateManager.isTablePresent(desc.getTableName())
3385            && (includeSysTables || !desc.getTableName().isSystemTable())) {
3386          htds.add(desc);
3387        }
3388      }
3389    } else {
3390      for (TableName s: tableNameList) {
3391        if (tableStateManager.isTablePresent(s)) {
3392          TableDescriptor desc = tableDescriptors.get(s);
3393          if (desc != null) {
3394            htds.add(desc);
3395          }
3396        }
3397      }
3398    }
3399
3400    // Retains only those matched by regular expression.
3401    if (regex != null) filterTablesByRegex(htds, Pattern.compile(regex));
3402    return htds;
3403  }
3404
3405  /**
3406   * Removes the table descriptors that don't match the pattern.
3407   * @param descriptors list of table descriptors to filter
3408   * @param pattern the regex to use
3409   */
3410  private static void filterTablesByRegex(final Collection<TableDescriptor> descriptors,
3411      final Pattern pattern) {
3412    final String defaultNS = NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR;
3413    Iterator<TableDescriptor> itr = descriptors.iterator();
3414    while (itr.hasNext()) {
3415      TableDescriptor htd = itr.next();
3416      String tableName = htd.getTableName().getNameAsString();
3417      boolean matched = pattern.matcher(tableName).matches();
3418      if (!matched && htd.getTableName().getNamespaceAsString().equals(defaultNS)) {
3419        matched = pattern.matcher(defaultNS + TableName.NAMESPACE_DELIM + tableName).matches();
3420      }
3421      if (!matched) {
3422        itr.remove();
3423      }
3424    }
3425  }
3426
3427  @Override
3428  public long getLastMajorCompactionTimestamp(TableName table) throws IOException {
3429    return getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS))
3430        .getLastMajorCompactionTimestamp(table);
3431  }
3432
3433  @Override
3434  public long getLastMajorCompactionTimestampForRegion(byte[] regionName) throws IOException {
3435    return getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS))
3436        .getLastMajorCompactionTimestamp(regionName);
3437  }
3438
3439  /**
3440   * Gets the mob file compaction state for a specific table.
3441   * Whether all the mob files are selected is known during the compaction execution, but
3442   * the statistic is done just before compaction starts, it is hard to know the compaction
3443   * type at that time, so the rough statistics are chosen for the mob file compaction. Only two
3444   * compaction states are available, CompactionState.MAJOR_AND_MINOR and CompactionState.NONE.
3445   * @param tableName The current table name.
3446   * @return If a given table is in mob file compaction now.
3447   */
3448  public CompactionState getMobCompactionState(TableName tableName) {
3449    AtomicInteger compactionsCount = mobCompactionStates.get(tableName);
3450    if (compactionsCount != null && compactionsCount.get() != 0) {
3451      return CompactionState.MAJOR_AND_MINOR;
3452    }
3453    return CompactionState.NONE;
3454  }
3455
3456  public void reportMobCompactionStart(TableName tableName) throws IOException {
3457    IdLock.Entry lockEntry = null;
3458    try {
3459      lockEntry = mobCompactionLock.getLockEntry(tableName.hashCode());
3460      AtomicInteger compactionsCount = mobCompactionStates.get(tableName);
3461      if (compactionsCount == null) {
3462        compactionsCount = new AtomicInteger(0);
3463        mobCompactionStates.put(tableName, compactionsCount);
3464      }
3465      compactionsCount.incrementAndGet();
3466    } finally {
3467      if (lockEntry != null) {
3468        mobCompactionLock.releaseLockEntry(lockEntry);
3469      }
3470    }
3471  }
3472
3473  public void reportMobCompactionEnd(TableName tableName) throws IOException {
3474    IdLock.Entry lockEntry = null;
3475    try {
3476      lockEntry = mobCompactionLock.getLockEntry(tableName.hashCode());
3477      AtomicInteger compactionsCount = mobCompactionStates.get(tableName);
3478      if (compactionsCount != null) {
3479        int count = compactionsCount.decrementAndGet();
3480        // remove the entry if the count is 0.
3481        if (count == 0) {
3482          mobCompactionStates.remove(tableName);
3483        }
3484      }
3485    } finally {
3486      if (lockEntry != null) {
3487        mobCompactionLock.releaseLockEntry(lockEntry);
3488      }
3489    }
3490  }
3491
3492
3493  /**
3494   * Queries the state of the {@link LoadBalancerTracker}. If the balancer is not initialized,
3495   * false is returned.
3496   *
3497   * @return The state of the load balancer, or false if the load balancer isn't defined.
3498   */
3499  public boolean isBalancerOn() {
3500    return !isInMaintenanceMode()
3501        && loadBalancerTracker != null
3502        && loadBalancerTracker.isBalancerOn();
3503  }
3504
3505  /**
3506   * Queries the state of the {@link RegionNormalizerTracker}. If it's not initialized,
3507   * false is returned.
3508   */
3509  public boolean isNormalizerOn() {
3510    return !isInMaintenanceMode()
3511        && regionNormalizerTracker != null
3512        && regionNormalizerTracker.isNormalizerOn();
3513  }
3514
3515  /**
3516   * Queries the state of the {@link SplitOrMergeTracker}. If it is not initialized,
3517   * false is returned. If switchType is illegal, false will return.
3518   * @param switchType see {@link org.apache.hadoop.hbase.client.MasterSwitchType}
3519   * @return The state of the switch
3520   */
3521  @Override
3522  public boolean isSplitOrMergeEnabled(MasterSwitchType switchType) {
3523    return !isInMaintenanceMode()
3524        && splitOrMergeTracker != null
3525        && splitOrMergeTracker.isSplitOrMergeEnabled(switchType);
3526  }
3527
3528  /**
3529   * Fetch the configured {@link LoadBalancer} class name. If none is set, a default is returned.
3530   * <p/>
3531   * Notice that, the base load balancer will always be {@link RSGroupBasedLoadBalancer} now, so
3532   * this method will return the balancer used inside each rs group.
3533   * @return The name of the {@link LoadBalancer} in use.
3534   */
3535  public String getLoadBalancerClassName() {
3536    return conf.get(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
3537      LoadBalancerFactory.getDefaultLoadBalancerClass().getName());
3538  }
3539
3540  /**
3541   * @return RegionNormalizerTracker instance
3542   */
3543  public RegionNormalizerTracker getRegionNormalizerTracker() {
3544    return regionNormalizerTracker;
3545  }
3546
3547  public SplitOrMergeTracker getSplitOrMergeTracker() {
3548    return splitOrMergeTracker;
3549  }
3550
3551  @Override
3552  public RSGroupBasedLoadBalancer getLoadBalancer() {
3553    return balancer;
3554  }
3555
3556  @Override
3557  public FavoredNodesManager getFavoredNodesManager() {
3558    return balancer.getFavoredNodesManager();
3559  }
3560
3561  private long executePeerProcedure(AbstractPeerProcedure<?> procedure) throws IOException {
3562    long procId = procedureExecutor.submitProcedure(procedure);
3563    procedure.getLatch().await();
3564    return procId;
3565  }
3566
3567  @Override
3568  public long addReplicationPeer(String peerId, ReplicationPeerConfig peerConfig, boolean enabled)
3569      throws ReplicationException, IOException {
3570    LOG.info(getClientIdAuditPrefix() + " creating replication peer, id=" + peerId + ", config=" +
3571      peerConfig + ", state=" + (enabled ? "ENABLED" : "DISABLED"));
3572    return executePeerProcedure(new AddPeerProcedure(peerId, peerConfig, enabled));
3573  }
3574
3575  @Override
3576  public long removeReplicationPeer(String peerId) throws ReplicationException, IOException {
3577    LOG.info(getClientIdAuditPrefix() + " removing replication peer, id=" + peerId);
3578    return executePeerProcedure(new RemovePeerProcedure(peerId));
3579  }
3580
3581  @Override
3582  public long enableReplicationPeer(String peerId) throws ReplicationException, IOException {
3583    LOG.info(getClientIdAuditPrefix() + " enable replication peer, id=" + peerId);
3584    return executePeerProcedure(new EnablePeerProcedure(peerId));
3585  }
3586
3587  @Override
3588  public long disableReplicationPeer(String peerId) throws ReplicationException, IOException {
3589    LOG.info(getClientIdAuditPrefix() + " disable replication peer, id=" + peerId);
3590    return executePeerProcedure(new DisablePeerProcedure(peerId));
3591  }
3592
3593  @Override
3594  public ReplicationPeerConfig getReplicationPeerConfig(String peerId)
3595      throws ReplicationException, IOException {
3596    if (cpHost != null) {
3597      cpHost.preGetReplicationPeerConfig(peerId);
3598    }
3599    LOG.info(getClientIdAuditPrefix() + " get replication peer config, id=" + peerId);
3600    ReplicationPeerConfig peerConfig = this.replicationPeerManager.getPeerConfig(peerId)
3601        .orElseThrow(() -> new ReplicationPeerNotFoundException(peerId));
3602    if (cpHost != null) {
3603      cpHost.postGetReplicationPeerConfig(peerId);
3604    }
3605    return peerConfig;
3606  }
3607
3608  @Override
3609  public long updateReplicationPeerConfig(String peerId, ReplicationPeerConfig peerConfig)
3610      throws ReplicationException, IOException {
3611    LOG.info(getClientIdAuditPrefix() + " update replication peer config, id=" + peerId +
3612      ", config=" + peerConfig);
3613    return executePeerProcedure(new UpdatePeerConfigProcedure(peerId, peerConfig));
3614  }
3615
3616  @Override
3617  public List<ReplicationPeerDescription> listReplicationPeers(String regex)
3618      throws ReplicationException, IOException {
3619    if (cpHost != null) {
3620      cpHost.preListReplicationPeers(regex);
3621    }
3622    LOG.debug("{} list replication peers, regex={}", getClientIdAuditPrefix(), regex);
3623    Pattern pattern = regex == null ? null : Pattern.compile(regex);
3624    List<ReplicationPeerDescription> peers =
3625      this.replicationPeerManager.listPeers(pattern);
3626    if (cpHost != null) {
3627      cpHost.postListReplicationPeers(regex);
3628    }
3629    return peers;
3630  }
3631
3632  @Override
3633  public long transitReplicationPeerSyncReplicationState(String peerId, SyncReplicationState state)
3634    throws ReplicationException, IOException {
3635    LOG.info(
3636      getClientIdAuditPrefix() +
3637        " transit current cluster state to {} in a synchronous replication peer id={}",
3638      state, peerId);
3639    return executePeerProcedure(new TransitPeerSyncReplicationStateProcedure(peerId, state));
3640  }
3641
3642  /**
3643   * Mark region server(s) as decommissioned (previously called 'draining') to prevent additional
3644   * regions from getting assigned to them. Also unload the regions on the servers asynchronously.0
3645   * @param servers Region servers to decommission.
3646   */
3647  public void decommissionRegionServers(final List<ServerName> servers, final boolean offload)
3648      throws IOException {
3649    List<ServerName> serversAdded = new ArrayList<>(servers.size());
3650    // Place the decommission marker first.
3651    String parentZnode = getZooKeeper().getZNodePaths().drainingZNode;
3652    for (ServerName server : servers) {
3653      try {
3654        String node = ZNodePaths.joinZNode(parentZnode, server.getServerName());
3655        ZKUtil.createAndFailSilent(getZooKeeper(), node);
3656      } catch (KeeperException ke) {
3657        throw new HBaseIOException(
3658          this.zooKeeper.prefix("Unable to decommission '" + server.getServerName() + "'."), ke);
3659      }
3660      if (this.serverManager.addServerToDrainList(server)) {
3661        serversAdded.add(server);
3662      }
3663    }
3664    // Move the regions off the decommissioned servers.
3665    if (offload) {
3666      final List<ServerName> destServers = this.serverManager.createDestinationServersList();
3667      for (ServerName server : serversAdded) {
3668        final List<RegionInfo> regionsOnServer = this.assignmentManager.getRegionsOnServer(server);
3669        for (RegionInfo hri : regionsOnServer) {
3670          ServerName dest = balancer.randomAssignment(hri, destServers);
3671          if (dest == null) {
3672            throw new HBaseIOException("Unable to determine a plan to move " + hri);
3673          }
3674          RegionPlan rp = new RegionPlan(hri, server, dest);
3675          this.assignmentManager.moveAsync(rp);
3676        }
3677      }
3678    }
3679  }
3680
3681  /**
3682   * List region servers marked as decommissioned (previously called 'draining') to not get regions
3683   * assigned to them.
3684   * @return List of decommissioned servers.
3685   */
3686  public List<ServerName> listDecommissionedRegionServers() {
3687    return this.serverManager.getDrainingServersList();
3688  }
3689
3690  /**
3691   * Remove decommission marker (previously called 'draining') from a region server to allow regions
3692   * assignments. Load regions onto the server asynchronously if a list of regions is given
3693   * @param server Region server to remove decommission marker from.
3694   */
3695  public void recommissionRegionServer(final ServerName server,
3696      final List<byte[]> encodedRegionNames) throws IOException {
3697    // Remove the server from decommissioned (draining) server list.
3698    String parentZnode = getZooKeeper().getZNodePaths().drainingZNode;
3699    String node = ZNodePaths.joinZNode(parentZnode, server.getServerName());
3700    try {
3701      ZKUtil.deleteNodeFailSilent(getZooKeeper(), node);
3702    } catch (KeeperException ke) {
3703      throw new HBaseIOException(
3704        this.zooKeeper.prefix("Unable to recommission '" + server.getServerName() + "'."), ke);
3705    }
3706    this.serverManager.removeServerFromDrainList(server);
3707
3708    // Load the regions onto the server if we are given a list of regions.
3709    if (encodedRegionNames == null || encodedRegionNames.isEmpty()) {
3710      return;
3711    }
3712    if (!this.serverManager.isServerOnline(server)) {
3713      return;
3714    }
3715    for (byte[] encodedRegionName : encodedRegionNames) {
3716      RegionState regionState =
3717        assignmentManager.getRegionStates().getRegionState(Bytes.toString(encodedRegionName));
3718      if (regionState == null) {
3719        LOG.warn("Unknown region " + Bytes.toStringBinary(encodedRegionName));
3720        continue;
3721      }
3722      RegionInfo hri = regionState.getRegion();
3723      if (server.equals(regionState.getServerName())) {
3724        LOG.info("Skipping move of region " + hri.getRegionNameAsString() +
3725          " because region already assigned to the same server " + server + ".");
3726        continue;
3727      }
3728      RegionPlan rp = new RegionPlan(hri, regionState.getServerName(), server);
3729      this.assignmentManager.moveAsync(rp);
3730    }
3731  }
3732
3733  @Override
3734  public LockManager getLockManager() {
3735    return lockManager;
3736  }
3737
3738  public QuotaObserverChore getQuotaObserverChore() {
3739    return this.quotaObserverChore;
3740  }
3741
3742  public SpaceQuotaSnapshotNotifier getSpaceQuotaSnapshotNotifier() {
3743    return this.spaceQuotaSnapshotNotifier;
3744  }
3745
3746  @SuppressWarnings("unchecked")
3747  private RemoteProcedure<MasterProcedureEnv, ?> getRemoteProcedure(long procId) {
3748    Procedure<?> procedure = procedureExecutor.getProcedure(procId);
3749    if (procedure == null) {
3750      return null;
3751    }
3752    assert procedure instanceof RemoteProcedure;
3753    return (RemoteProcedure<MasterProcedureEnv, ?>) procedure;
3754  }
3755
3756  public void remoteProcedureCompleted(long procId) {
3757    LOG.debug("Remote procedure done, pid={}", procId);
3758    RemoteProcedure<MasterProcedureEnv, ?> procedure = getRemoteProcedure(procId);
3759    if (procedure != null) {
3760      procedure.remoteOperationCompleted(procedureExecutor.getEnvironment());
3761    }
3762  }
3763
3764  public void remoteProcedureFailed(long procId, RemoteProcedureException error) {
3765    LOG.debug("Remote procedure failed, pid={}", procId, error);
3766    RemoteProcedure<MasterProcedureEnv, ?> procedure = getRemoteProcedure(procId);
3767    if (procedure != null) {
3768      procedure.remoteOperationFailed(procedureExecutor.getEnvironment(), error);
3769    }
3770  }
3771
3772  /**
3773   * Reopen regions provided in the argument
3774   *
3775   * @param tableName The current table name
3776   * @param regionNames The region names of the regions to reopen
3777   * @param nonceGroup Identifier for the source of the request, a client or process
3778   * @param nonce A unique identifier for this operation from the client or process identified by
3779   *   <code>nonceGroup</code> (the source must ensure each operation gets a unique id).
3780   * @return procedure Id
3781   * @throws IOException if reopening region fails while running procedure
3782   */
3783  long reopenRegions(final TableName tableName, final List<byte[]> regionNames,
3784      final long nonceGroup, final long nonce)
3785      throws IOException {
3786
3787    return MasterProcedureUtil
3788      .submitProcedure(new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {
3789
3790        @Override
3791        protected void run() throws IOException {
3792          submitProcedure(new ReopenTableRegionsProcedure(tableName, regionNames));
3793        }
3794
3795        @Override
3796        protected String getDescription() {
3797          return "ReopenTableRegionsProcedure";
3798        }
3799
3800      });
3801
3802  }
3803
3804  @Override
3805  public ReplicationPeerManager getReplicationPeerManager() {
3806    return replicationPeerManager;
3807  }
3808
3809  public HashMap<String, List<Pair<ServerName, ReplicationLoadSource>>>
3810      getReplicationLoad(ServerName[] serverNames) {
3811    List<ReplicationPeerDescription> peerList = this.getReplicationPeerManager().listPeers(null);
3812    if (peerList == null) {
3813      return null;
3814    }
3815    HashMap<String, List<Pair<ServerName, ReplicationLoadSource>>> replicationLoadSourceMap =
3816        new HashMap<>(peerList.size());
3817    peerList.stream()
3818        .forEach(peer -> replicationLoadSourceMap.put(peer.getPeerId(), new ArrayList<>()));
3819    for (ServerName serverName : serverNames) {
3820      List<ReplicationLoadSource> replicationLoadSources =
3821          getServerManager().getLoad(serverName).getReplicationLoadSourceList();
3822      for (ReplicationLoadSource replicationLoadSource : replicationLoadSources) {
3823        replicationLoadSourceMap.get(replicationLoadSource.getPeerID())
3824            .add(new Pair<>(serverName, replicationLoadSource));
3825      }
3826    }
3827    for (List<Pair<ServerName, ReplicationLoadSource>> loads : replicationLoadSourceMap.values()) {
3828      if (loads.size() > 0) {
3829        loads.sort(Comparator.comparingLong(load -> (-1) * load.getSecond().getReplicationLag()));
3830      }
3831    }
3832    return replicationLoadSourceMap;
3833  }
3834
3835  /**
3836   * This method modifies the master's configuration in order to inject replication-related features
3837   */
3838  @VisibleForTesting
3839  public static void decorateMasterConfiguration(Configuration conf) {
3840    String plugins = conf.get(HBASE_MASTER_LOGCLEANER_PLUGINS);
3841    String cleanerClass = ReplicationLogCleaner.class.getCanonicalName();
3842    if (plugins == null || !plugins.contains(cleanerClass)) {
3843      conf.set(HBASE_MASTER_LOGCLEANER_PLUGINS, plugins + "," + cleanerClass);
3844    }
3845    if (ReplicationUtils.isReplicationForBulkLoadDataEnabled(conf)) {
3846      plugins = conf.get(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS);
3847      cleanerClass = ReplicationHFileCleaner.class.getCanonicalName();
3848      if (!plugins.contains(cleanerClass)) {
3849        conf.set(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, plugins + "," + cleanerClass);
3850      }
3851    }
3852  }
3853
3854  public SnapshotQuotaObserverChore getSnapshotQuotaObserverChore() {
3855    return this.snapshotQuotaChore;
3856  }
3857
3858  @Override
3859  public SyncReplicationReplayWALManager getSyncReplicationReplayWALManager() {
3860    return this.syncReplicationReplayWALManager;
3861  }
3862
3863  @Override
3864  public Map<String, ReplicationStatus> getWalGroupsReplicationStatus() {
3865    if (!this.isOnline() || !LoadBalancer.isMasterCanHostUserRegions(conf)) {
3866      return new HashMap<>();
3867    }
3868    return super.getWalGroupsReplicationStatus();
3869  }
3870
3871  public HbckChore getHbckChore() {
3872    return this.hbckChore;
3873  }
3874
3875  @Override
3876  public String getClusterId() {
3877    if (activeMaster) {
3878      return super.getClusterId();
3879    }
3880    return cachedClusterId.getFromCacheOrFetch();
3881  }
3882
3883  public Optional<ServerName> getActiveMaster() {
3884    return activeMasterManager.getActiveMasterServerName();
3885  }
3886
3887  @Override
3888  public void runReplicationBarrierCleaner() {
3889    ReplicationBarrierCleaner rbc = this.replicationBarrierCleaner;
3890    if (rbc != null) {
3891      rbc.chore();
3892    }
3893  }
3894
3895  public MetaRegionLocationCache getMetaRegionLocationCache() {
3896    return this.metaRegionLocationCache;
3897  }
3898
3899  @Override
3900  public RSGroupInfoManager getRSGroupInfoManager() {
3901    return rsGroupInfoManager;
3902  }
3903}