001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master;
019
020import static org.apache.hadoop.hbase.HConstants.DEFAULT_HBASE_SPLIT_COORDINATED_BY_ZK;
021import static org.apache.hadoop.hbase.HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS;
022import static org.apache.hadoop.hbase.HConstants.HBASE_SPLIT_WAL_COORDINATED_BY_ZK;
023import static org.apache.hadoop.hbase.util.DNS.MASTER_HOSTNAME_KEY;
024
025import java.io.IOException;
026import java.io.InterruptedIOException;
027import java.lang.reflect.Constructor;
028import java.lang.reflect.InvocationTargetException;
029import java.net.InetAddress;
030import java.net.InetSocketAddress;
031import java.net.UnknownHostException;
032import java.util.ArrayList;
033import java.util.Arrays;
034import java.util.Collection;
035import java.util.Collections;
036import java.util.Comparator;
037import java.util.EnumSet;
038import java.util.HashMap;
039import java.util.Iterator;
040import java.util.LinkedList;
041import java.util.List;
042import java.util.Map;
043import java.util.Objects;
044import java.util.Optional;
045import java.util.Set;
046import java.util.concurrent.ExecutionException;
047import java.util.concurrent.Future;
048import java.util.concurrent.TimeUnit;
049import java.util.concurrent.TimeoutException;
050import java.util.concurrent.atomic.AtomicInteger;
051import java.util.regex.Pattern;
052import java.util.stream.Collectors;
053import javax.servlet.ServletException;
054import javax.servlet.http.HttpServlet;
055import javax.servlet.http.HttpServletRequest;
056import javax.servlet.http.HttpServletResponse;
057import org.apache.commons.io.IOUtils;
058import org.apache.commons.lang3.StringUtils;
059import org.apache.hadoop.conf.Configuration;
060import org.apache.hadoop.fs.FSDataOutputStream;
061import org.apache.hadoop.fs.Path;
062import org.apache.hadoop.hbase.ChoreService;
063import org.apache.hadoop.hbase.ClusterId;
064import org.apache.hadoop.hbase.ClusterMetrics;
065import org.apache.hadoop.hbase.ClusterMetrics.Option;
066import org.apache.hadoop.hbase.ClusterMetricsBuilder;
067import org.apache.hadoop.hbase.DoNotRetryIOException;
068import org.apache.hadoop.hbase.HBaseIOException;
069import org.apache.hadoop.hbase.HBaseInterfaceAudience;
070import org.apache.hadoop.hbase.HConstants;
071import org.apache.hadoop.hbase.InvalidFamilyOperationException;
072import org.apache.hadoop.hbase.MasterNotRunningException;
073import org.apache.hadoop.hbase.MetaTableAccessor;
074import org.apache.hadoop.hbase.NamespaceDescriptor;
075import org.apache.hadoop.hbase.PleaseHoldException;
076import org.apache.hadoop.hbase.RegionMetrics;
077import org.apache.hadoop.hbase.ReplicationPeerNotFoundException;
078import org.apache.hadoop.hbase.ServerMetrics;
079import org.apache.hadoop.hbase.ServerName;
080import org.apache.hadoop.hbase.TableName;
081import org.apache.hadoop.hbase.TableNotDisabledException;
082import org.apache.hadoop.hbase.TableNotFoundException;
083import org.apache.hadoop.hbase.UnknownRegionException;
084import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
085import org.apache.hadoop.hbase.client.CompactionState;
086import org.apache.hadoop.hbase.client.MasterSwitchType;
087import org.apache.hadoop.hbase.client.NormalizeTableFilterParams;
088import org.apache.hadoop.hbase.client.RegionInfo;
089import org.apache.hadoop.hbase.client.RegionInfoBuilder;
090import org.apache.hadoop.hbase.client.RegionStatesCount;
091import org.apache.hadoop.hbase.client.TableDescriptor;
092import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
093import org.apache.hadoop.hbase.client.TableState;
094import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
095import org.apache.hadoop.hbase.executor.ExecutorType;
096import org.apache.hadoop.hbase.favored.FavoredNodesManager;
097import org.apache.hadoop.hbase.http.InfoServer;
098import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils;
099import org.apache.hadoop.hbase.ipc.RpcServer;
100import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
101import org.apache.hadoop.hbase.log.HBaseMarkers;
102import org.apache.hadoop.hbase.master.MasterRpcServices.BalanceSwitchMode;
103import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
104import org.apache.hadoop.hbase.master.assignment.MergeTableRegionsProcedure;
105import org.apache.hadoop.hbase.master.assignment.RegionStateNode;
106import org.apache.hadoop.hbase.master.assignment.RegionStates;
107import org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure;
108import org.apache.hadoop.hbase.master.balancer.BalancerChore;
109import org.apache.hadoop.hbase.master.balancer.ClusterStatusChore;
110import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory;
111import org.apache.hadoop.hbase.master.cleaner.DirScanPool;
112import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
113import org.apache.hadoop.hbase.master.cleaner.LogCleaner;
114import org.apache.hadoop.hbase.master.cleaner.ReplicationBarrierCleaner;
115import org.apache.hadoop.hbase.master.cleaner.SnapshotCleanerChore;
116import org.apache.hadoop.hbase.master.janitor.CatalogJanitor;
117import org.apache.hadoop.hbase.master.locking.LockManager;
118import org.apache.hadoop.hbase.master.normalizer.RegionNormalizerFactory;
119import org.apache.hadoop.hbase.master.normalizer.RegionNormalizerManager;
120import org.apache.hadoop.hbase.master.procedure.CreateTableProcedure;
121import org.apache.hadoop.hbase.master.procedure.DeleteNamespaceProcedure;
122import org.apache.hadoop.hbase.master.procedure.DeleteTableProcedure;
123import org.apache.hadoop.hbase.master.procedure.DisableTableProcedure;
124import org.apache.hadoop.hbase.master.procedure.EnableTableProcedure;
125import org.apache.hadoop.hbase.master.procedure.InitMetaProcedure;
126import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants;
127import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
128import org.apache.hadoop.hbase.master.procedure.MasterProcedureScheduler;
129import org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil;
130import org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.NonceProcedureRunnable;
131import org.apache.hadoop.hbase.master.procedure.ModifyTableProcedure;
132import org.apache.hadoop.hbase.master.procedure.ProcedurePrepareLatch;
133import org.apache.hadoop.hbase.master.procedure.ProcedureSyncWait;
134import org.apache.hadoop.hbase.master.procedure.ReopenTableRegionsProcedure;
135import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure;
136import org.apache.hadoop.hbase.master.procedure.TruncateTableProcedure;
137import org.apache.hadoop.hbase.master.region.MasterRegion;
138import org.apache.hadoop.hbase.master.region.MasterRegionFactory;
139import org.apache.hadoop.hbase.master.replication.AbstractPeerProcedure;
140import org.apache.hadoop.hbase.master.replication.AddPeerProcedure;
141import org.apache.hadoop.hbase.master.replication.DisablePeerProcedure;
142import org.apache.hadoop.hbase.master.replication.EnablePeerProcedure;
143import org.apache.hadoop.hbase.master.replication.RemovePeerProcedure;
144import org.apache.hadoop.hbase.master.replication.ReplicationPeerManager;
145import org.apache.hadoop.hbase.master.replication.SyncReplicationReplayWALManager;
146import org.apache.hadoop.hbase.master.replication.TransitPeerSyncReplicationStateProcedure;
147import org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure;
148import org.apache.hadoop.hbase.master.slowlog.SlowLogMasterService;
149import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
150import org.apache.hadoop.hbase.master.zksyncer.MasterAddressSyncer;
151import org.apache.hadoop.hbase.master.zksyncer.MetaLocationSyncer;
152import org.apache.hadoop.hbase.mob.MobFileCleanerChore;
153import org.apache.hadoop.hbase.mob.MobFileCompactionChore;
154import org.apache.hadoop.hbase.monitoring.MemoryBoundedLogMessageBuffer;
155import org.apache.hadoop.hbase.monitoring.MonitoredTask;
156import org.apache.hadoop.hbase.monitoring.TaskMonitor;
157import org.apache.hadoop.hbase.procedure.MasterProcedureManagerHost;
158import org.apache.hadoop.hbase.procedure.flush.MasterFlushTableProcedureManager;
159import org.apache.hadoop.hbase.procedure2.LockedResource;
160import org.apache.hadoop.hbase.procedure2.Procedure;
161import org.apache.hadoop.hbase.procedure2.ProcedureEvent;
162import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
163import org.apache.hadoop.hbase.procedure2.RemoteProcedureDispatcher.RemoteProcedure;
164import org.apache.hadoop.hbase.procedure2.RemoteProcedureException;
165import org.apache.hadoop.hbase.procedure2.store.ProcedureStore;
166import org.apache.hadoop.hbase.procedure2.store.ProcedureStore.ProcedureStoreListener;
167import org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore;
168import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
169import org.apache.hadoop.hbase.quotas.MasterQuotasObserver;
170import org.apache.hadoop.hbase.quotas.QuotaObserverChore;
171import org.apache.hadoop.hbase.quotas.QuotaTableUtil;
172import org.apache.hadoop.hbase.quotas.QuotaUtil;
173import org.apache.hadoop.hbase.quotas.SnapshotQuotaObserverChore;
174import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot;
175import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus;
176import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshotNotifier;
177import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshotNotifierFactory;
178import org.apache.hadoop.hbase.quotas.SpaceViolationPolicy;
179import org.apache.hadoop.hbase.regionserver.HRegionServer;
180import org.apache.hadoop.hbase.regionserver.RSRpcServices;
181import org.apache.hadoop.hbase.replication.ReplicationException;
182import org.apache.hadoop.hbase.replication.ReplicationLoadSource;
183import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
184import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
185import org.apache.hadoop.hbase.replication.ReplicationUtils;
186import org.apache.hadoop.hbase.replication.SyncReplicationState;
187import org.apache.hadoop.hbase.replication.master.ReplicationHFileCleaner;
188import org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner;
189import org.apache.hadoop.hbase.replication.regionserver.ReplicationStatus;
190import org.apache.hadoop.hbase.rsgroup.RSGroupAdminEndpoint;
191import org.apache.hadoop.hbase.rsgroup.RSGroupBasedLoadBalancer;
192import org.apache.hadoop.hbase.rsgroup.RSGroupInfoManager;
193import org.apache.hadoop.hbase.rsgroup.RSGroupUtil;
194import org.apache.hadoop.hbase.security.AccessDeniedException;
195import org.apache.hadoop.hbase.security.SecurityConstants;
196import org.apache.hadoop.hbase.security.UserProvider;
197import org.apache.hadoop.hbase.trace.TraceUtil;
198import org.apache.hadoop.hbase.util.Addressing;
199import org.apache.hadoop.hbase.util.Bytes;
200import org.apache.hadoop.hbase.util.FutureUtils;
201import org.apache.hadoop.hbase.util.HBaseFsck;
202import org.apache.hadoop.hbase.util.HFileArchiveUtil;
203import org.apache.hadoop.hbase.util.IdLock;
204import org.apache.hadoop.hbase.util.ModifyRegionUtils;
205import org.apache.hadoop.hbase.util.Pair;
206import org.apache.hadoop.hbase.util.RetryCounter;
207import org.apache.hadoop.hbase.util.RetryCounterFactory;
208import org.apache.hadoop.hbase.util.TableDescriptorChecker;
209import org.apache.hadoop.hbase.util.Threads;
210import org.apache.hadoop.hbase.util.VersionInfo;
211import org.apache.hadoop.hbase.zookeeper.LoadBalancerTracker;
212import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker;
213import org.apache.hadoop.hbase.zookeeper.RegionNormalizerTracker;
214import org.apache.hadoop.hbase.zookeeper.SnapshotCleanupTracker;
215import org.apache.hadoop.hbase.zookeeper.ZKClusterId;
216import org.apache.hadoop.hbase.zookeeper.ZKUtil;
217import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
218import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
219import org.apache.yetus.audience.InterfaceAudience;
220import org.apache.zookeeper.KeeperException;
221import org.slf4j.Logger;
222import org.slf4j.LoggerFactory;
223
224import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
225import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
226import org.apache.hbase.thirdparty.com.google.common.collect.Maps;
227import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
228import org.apache.hbase.thirdparty.com.google.protobuf.Descriptors;
229import org.apache.hbase.thirdparty.com.google.protobuf.Service;
230import org.apache.hbase.thirdparty.org.eclipse.jetty.server.Server;
231import org.apache.hbase.thirdparty.org.eclipse.jetty.server.ServerConnector;
232import org.apache.hbase.thirdparty.org.eclipse.jetty.servlet.ServletHolder;
233import org.apache.hbase.thirdparty.org.eclipse.jetty.webapp.WebAppContext;
234
235import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter;
236import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoResponse;
237import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription;
238
239/**
240 * HMaster is the "master server" for HBase. An HBase cluster has one active
241 * master.  If many masters are started, all compete.  Whichever wins goes on to
242 * run the cluster.  All others park themselves in their constructor until
243 * master or cluster shutdown or until the active master loses its lease in
244 * zookeeper.  Thereafter, all running master jostle to take over master role.
245 *
246 * <p>The Master can be asked shutdown the cluster. See {@link #shutdown()}.  In
247 * this case it will tell all regionservers to go down and then wait on them
248 * all reporting in that they are down.  This master will then shut itself down.
249 *
250 * <p>You can also shutdown just this master.  Call {@link #stopMaster()}.
251 *
252 * @see org.apache.zookeeper.Watcher
253 */
254@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
255@SuppressWarnings("deprecation")
256public class HMaster extends HRegionServer implements MasterServices {
257  private static Logger LOG = LoggerFactory.getLogger(HMaster.class);
258
259  /**
260   * Protection against zombie master. Started once Master accepts active responsibility and
261   * starts taking over responsibilities. Allows a finite time window before giving up ownership.
262   */
263  private static class InitializationMonitor extends Thread {
264    /** The amount of time in milliseconds to sleep before checking initialization status. */
265    public static final String TIMEOUT_KEY = "hbase.master.initializationmonitor.timeout";
266    public static final long TIMEOUT_DEFAULT = TimeUnit.MILLISECONDS.convert(15, TimeUnit.MINUTES);
267
268    /**
269     * When timeout expired and initialization has not complete, call {@link System#exit(int)} when
270     * true, do nothing otherwise.
271     */
272    public static final String HALT_KEY = "hbase.master.initializationmonitor.haltontimeout";
273    public static final boolean HALT_DEFAULT = false;
274
275    private final HMaster master;
276    private final long timeout;
277    private final boolean haltOnTimeout;
278
279    /** Creates a Thread that monitors the {@link #isInitialized()} state. */
280    InitializationMonitor(HMaster master) {
281      super("MasterInitializationMonitor");
282      this.master = master;
283      this.timeout = master.getConfiguration().getLong(TIMEOUT_KEY, TIMEOUT_DEFAULT);
284      this.haltOnTimeout = master.getConfiguration().getBoolean(HALT_KEY, HALT_DEFAULT);
285      this.setDaemon(true);
286    }
287
288    @Override
289    public void run() {
290      try {
291        while (!master.isStopped() && master.isActiveMaster()) {
292          Thread.sleep(timeout);
293          if (master.isInitialized()) {
294            LOG.debug("Initialization completed within allotted tolerance. Monitor exiting.");
295          } else {
296            LOG.error("Master failed to complete initialization after " + timeout + "ms. Please"
297                + " consider submitting a bug report including a thread dump of this process.");
298            if (haltOnTimeout) {
299              LOG.error("Zombie Master exiting. Thread dump to stdout");
300              Threads.printThreadInfo(System.out, "Zombie HMaster");
301              System.exit(-1);
302            }
303          }
304        }
305      } catch (InterruptedException ie) {
306        LOG.trace("InitMonitor thread interrupted. Existing.");
307      }
308    }
309  }
310
311  // MASTER is name of the webapp and the attribute name used stuffing this
312  //instance into web context.
313  public static final String MASTER = "master";
314
315  // Manager and zk listener for master election
316  private final ActiveMasterManager activeMasterManager;
317  // Region server tracker
318  private RegionServerTracker regionServerTracker;
319  // Draining region server tracker
320  private DrainingServerTracker drainingServerTracker;
321  // Tracker for load balancer state
322  LoadBalancerTracker loadBalancerTracker;
323  // Tracker for meta location, if any client ZK quorum specified
324  MetaLocationSyncer metaLocationSyncer;
325  // Tracker for active master location, if any client ZK quorum specified
326  MasterAddressSyncer masterAddressSyncer;
327  // Tracker for auto snapshot cleanup state
328  SnapshotCleanupTracker snapshotCleanupTracker;
329
330  // Tracker for split and merge state
331  private SplitOrMergeTracker splitOrMergeTracker;
332
333  private ClusterSchemaService clusterSchemaService;
334
335  public static final String HBASE_MASTER_WAIT_ON_SERVICE_IN_SECONDS =
336    "hbase.master.wait.on.service.seconds";
337  public static final int DEFAULT_HBASE_MASTER_WAIT_ON_SERVICE_IN_SECONDS = 5 * 60;
338
339  public static final String HBASE_MASTER_CLEANER_INTERVAL = "hbase.master.cleaner.interval";
340
341  public static final int DEFAULT_HBASE_MASTER_CLEANER_INTERVAL = 600 * 1000;
342
343  // Metrics for the HMaster
344  final MetricsMaster metricsMaster;
345  // file system manager for the master FS operations
346  private MasterFileSystem fileSystemManager;
347  private MasterWalManager walManager;
348
349  // manager to manage procedure-based WAL splitting, can be null if current
350  // is zk-based WAL splitting. SplitWALManager will replace SplitLogManager
351  // and MasterWalManager, which means zk-based WAL splitting code will be
352  // useless after we switch to the procedure-based one. our eventual goal
353  // is to remove all the zk-based WAL splitting code.
354  private SplitWALManager splitWALManager;
355
356  // server manager to deal with region server info
357  private volatile ServerManager serverManager;
358
359  // manager of assignment nodes in zookeeper
360  private AssignmentManager assignmentManager;
361
362
363  /**
364   * Cache for the meta region replica's locations. Also tracks their changes to avoid stale
365   * cache entries.
366   */
367  private final MetaRegionLocationCache metaRegionLocationCache;
368
369  private RSGroupInfoManager rsGroupInfoManager;
370
371  // manager of replication
372  private ReplicationPeerManager replicationPeerManager;
373
374  private SyncReplicationReplayWALManager syncReplicationReplayWALManager;
375
376  // buffer for "fatal error" notices from region servers
377  // in the cluster. This is only used for assisting
378  // operations/debugging.
379  MemoryBoundedLogMessageBuffer rsFatals;
380
381  // flag set after we become the active master (used for testing)
382  private volatile boolean activeMaster = false;
383
384  // flag set after we complete initialization once active
385  private final ProcedureEvent<?> initialized = new ProcedureEvent<>("master initialized");
386
387  // flag set after master services are started,
388  // initialization may have not completed yet.
389  volatile boolean serviceStarted = false;
390
391  // Maximum time we should run balancer for
392  private final int maxBalancingTime;
393  // Maximum percent of regions in transition when balancing
394  private final double maxRitPercent;
395
396  private final LockManager lockManager = new LockManager(this);
397
398  private RSGroupBasedLoadBalancer balancer;
399  private BalancerChore balancerChore;
400  private RegionNormalizerManager regionNormalizerManager;
401  private ClusterStatusChore clusterStatusChore;
402  private ClusterStatusPublisher clusterStatusPublisherChore = null;
403  private SnapshotCleanerChore snapshotCleanerChore = null;
404
405  private HbckChore hbckChore;
406  CatalogJanitor catalogJanitorChore;
407  private DirScanPool cleanerPool;
408  private LogCleaner logCleaner;
409  private HFileCleaner hfileCleaner;
410  private ReplicationBarrierCleaner replicationBarrierCleaner;
411  private MobFileCleanerChore mobFileCleanerChore;
412  private MobFileCompactionChore mobFileCompactionChore;
413  // used to synchronize the mobCompactionStates
414  private final IdLock mobCompactionLock = new IdLock();
415  // save the information of mob compactions in tables.
416  // the key is table name, the value is the number of compactions in that table.
417  private Map<TableName, AtomicInteger> mobCompactionStates = Maps.newConcurrentMap();
418
419  MasterCoprocessorHost cpHost;
420
421  private final boolean preLoadTableDescriptors;
422
423  // Time stamps for when a hmaster became active
424  private long masterActiveTime;
425
426  // Time stamp for when HMaster finishes becoming Active Master
427  private long masterFinishedInitializationTime;
428
429  Map<String, Service> coprocessorServiceHandlers = Maps.newHashMap();
430
431  // monitor for snapshot of hbase tables
432  SnapshotManager snapshotManager;
433  // monitor for distributed procedures
434  private MasterProcedureManagerHost mpmHost;
435
436  private RegionsRecoveryChore regionsRecoveryChore = null;
437
438  private RegionsRecoveryConfigManager regionsRecoveryConfigManager = null;
439  // it is assigned after 'initialized' guard set to true, so should be volatile
440  private volatile MasterQuotaManager quotaManager;
441  private SpaceQuotaSnapshotNotifier spaceQuotaSnapshotNotifier;
442  private QuotaObserverChore quotaObserverChore;
443  private SnapshotQuotaObserverChore snapshotQuotaChore;
444
445  private ProcedureExecutor<MasterProcedureEnv> procedureExecutor;
446  private ProcedureStore procedureStore;
447
448  // the master local storage to store procedure data, etc.
449  private MasterRegion masterRegion;
450
451  // handle table states
452  private TableStateManager tableStateManager;
453
454  /** jetty server for master to redirect requests to regionserver infoServer */
455  private Server masterJettyServer;
456
457  // Determine if we should do normal startup or minimal "single-user" mode with no region
458  // servers and no user tables. Useful for repair and recovery of hbase:meta
459  private final boolean maintenanceMode;
460  static final String MAINTENANCE_MODE = "hbase.master.maintenance_mode";
461
462  // Cached clusterId on stand by masters to serve clusterID requests from clients.
463  private final CachedClusterId cachedClusterId;
464
465  public static class RedirectServlet extends HttpServlet {
466    private static final long serialVersionUID = 2894774810058302473L;
467    private final int regionServerInfoPort;
468    private final String regionServerHostname;
469
470    /**
471     * @param infoServer that we're trying to send all requests to
472     * @param hostname may be null. if given, will be used for redirects instead of host from client.
473     */
474    public RedirectServlet(InfoServer infoServer, String hostname) {
475       regionServerInfoPort = infoServer.getPort();
476       regionServerHostname = hostname;
477    }
478
479    @Override
480    public void doGet(HttpServletRequest request,
481        HttpServletResponse response) throws ServletException, IOException {
482      String redirectHost = regionServerHostname;
483      if(redirectHost == null) {
484        redirectHost = request.getServerName();
485        if(!Addressing.isLocalAddress(InetAddress.getByName(redirectHost))) {
486          LOG.warn("Couldn't resolve '" + redirectHost + "' as an address local to this node and '" +
487              MASTER_HOSTNAME_KEY + "' is not set; client will get an HTTP 400 response. If " +
488              "your HBase deployment relies on client accessible names that the region server process " +
489              "can't resolve locally, then you should set the previously mentioned configuration variable " +
490              "to an appropriate hostname.");
491          // no sending client provided input back to the client, so the goal host is just in the logs.
492          response.sendError(400, "Request was to a host that I can't resolve for any of the network interfaces on " +
493              "this node. If this is due to an intermediary such as an HTTP load balancer or other proxy, your HBase " +
494              "administrator can set '" + MASTER_HOSTNAME_KEY + "' to point to the correct hostname.");
495          return;
496        }
497      }
498      // TODO this scheme should come from looking at the scheme registered in the infoserver's http server for the
499      // host and port we're using, but it's buried way too deep to do that ATM.
500      String redirectUrl = request.getScheme() + "://"
501        + redirectHost + ":" + regionServerInfoPort
502        + request.getRequestURI();
503      response.sendRedirect(redirectUrl);
504    }
505  }
506
507  /**
508   * Initializes the HMaster. The steps are as follows:
509   * <p>
510   * <ol>
511   * <li>Initialize the local HRegionServer
512   * <li>Start the ActiveMasterManager.
513   * </ol>
514   * <p>
515   * Remaining steps of initialization occur in
516   * {@link #finishActiveMasterInitialization(MonitoredTask)} after the master becomes the
517   * active one.
518   */
519  public HMaster(final Configuration conf) throws IOException {
520    super(conf);
521    TraceUtil.initTracer(conf);
522    try {
523      if (conf.getBoolean(MAINTENANCE_MODE, false)) {
524        LOG.info("Detected {}=true via configuration.", MAINTENANCE_MODE);
525        maintenanceMode = true;
526      } else if (Boolean.getBoolean(MAINTENANCE_MODE)) {
527        LOG.info("Detected {}=true via environment variables.", MAINTENANCE_MODE);
528        maintenanceMode = true;
529      } else {
530        maintenanceMode = false;
531      }
532      this.rsFatals = new MemoryBoundedLogMessageBuffer(
533          conf.getLong("hbase.master.buffer.for.rs.fatals", 1 * 1024 * 1024));
534      LOG.info("hbase.rootdir={}, hbase.cluster.distributed={}", getDataRootDir(),
535          this.conf.getBoolean(HConstants.CLUSTER_DISTRIBUTED, false));
536
537      // Disable usage of meta replicas in the master
538      this.conf.setBoolean(HConstants.USE_META_REPLICAS, false);
539
540      decorateMasterConfiguration(this.conf);
541
542      // Hack! Maps DFSClient => Master for logs.  HDFS made this
543      // config param for task trackers, but we can piggyback off of it.
544      if (this.conf.get("mapreduce.task.attempt.id") == null) {
545        this.conf.set("mapreduce.task.attempt.id", "hb_m_" + this.serverName.toString());
546      }
547
548      this.metricsMaster = new MetricsMaster(new MetricsMasterWrapperImpl(this));
549
550      // preload table descriptor at startup
551      this.preLoadTableDescriptors = conf.getBoolean("hbase.master.preload.tabledescriptors", true);
552
553      this.maxBalancingTime = getMaxBalancingTime();
554      this.maxRitPercent = conf.getDouble(HConstants.HBASE_MASTER_BALANCER_MAX_RIT_PERCENT,
555          HConstants.DEFAULT_HBASE_MASTER_BALANCER_MAX_RIT_PERCENT);
556
557      // Do we publish the status?
558
559      boolean shouldPublish = conf.getBoolean(HConstants.STATUS_PUBLISHED,
560          HConstants.STATUS_PUBLISHED_DEFAULT);
561      Class<? extends ClusterStatusPublisher.Publisher> publisherClass =
562          conf.getClass(ClusterStatusPublisher.STATUS_PUBLISHER_CLASS,
563              ClusterStatusPublisher.DEFAULT_STATUS_PUBLISHER_CLASS,
564              ClusterStatusPublisher.Publisher.class);
565
566      if (shouldPublish) {
567        if (publisherClass == null) {
568          LOG.warn(HConstants.STATUS_PUBLISHED + " is true, but " +
569              ClusterStatusPublisher.DEFAULT_STATUS_PUBLISHER_CLASS +
570              " is not set - not publishing status");
571        } else {
572          clusterStatusPublisherChore = new ClusterStatusPublisher(this, conf, publisherClass);
573          LOG.debug("Created {}", this.clusterStatusPublisherChore);
574          getChoreService().scheduleChore(clusterStatusPublisherChore);
575        }
576      }
577
578      this.metaRegionLocationCache = new MetaRegionLocationCache(this.zooKeeper);
579      this.activeMasterManager = createActiveMasterManager(zooKeeper, serverName, this);
580
581      cachedClusterId = new CachedClusterId(this, conf);
582    } catch (Throwable t) {
583      // Make sure we log the exception. HMaster is often started via reflection and the
584      // cause of failed startup is lost.
585      LOG.error("Failed construction of Master", t);
586      throw t;
587    }
588  }
589
590  /**
591   * Protected to have custom implementations in tests override the default ActiveMaster
592   * implementation.
593   */
594  protected ActiveMasterManager createActiveMasterManager(ZKWatcher zk, ServerName sn,
595      org.apache.hadoop.hbase.Server server) throws InterruptedIOException {
596    return new ActiveMasterManager(zk, sn, server);
597  }
598
599  @Override
600  protected String getUseThisHostnameInstead(Configuration conf) {
601    return conf.get(MASTER_HOSTNAME_KEY);
602  }
603
604  // Main run loop. Calls through to the regionserver run loop AFTER becoming active Master; will
605  // block in here until then.
606  @Override
607  public void run() {
608    try {
609      Threads.setDaemonThreadRunning(new Thread(() -> {
610        try {
611          int infoPort = putUpJettyServer();
612          startActiveMasterManager(infoPort);
613        } catch (Throwable t) {
614          // Make sure we log the exception.
615          String error = "Failed to become Active Master";
616          LOG.error(error, t);
617          // Abort should have been called already.
618          if (!isAborted()) {
619            abort(error, t);
620          }
621        }
622      }), getName() + ":becomeActiveMaster");
623      // Fall in here even if we have been aborted. Need to run the shutdown services and
624      // the super run call will do this for us.
625      super.run();
626    } finally {
627      if (this.clusterSchemaService != null) {
628        // If on way out, then we are no longer active master.
629        this.clusterSchemaService.stopAsync();
630        try {
631          this.clusterSchemaService.awaitTerminated(
632              getConfiguration().getInt(HBASE_MASTER_WAIT_ON_SERVICE_IN_SECONDS,
633              DEFAULT_HBASE_MASTER_WAIT_ON_SERVICE_IN_SECONDS), TimeUnit.SECONDS);
634        } catch (TimeoutException te) {
635          LOG.warn("Failed shutdown of clusterSchemaService", te);
636        }
637      }
638      this.activeMaster = false;
639    }
640  }
641
642  // return the actual infoPort, -1 means disable info server.
643  private int putUpJettyServer() throws IOException {
644    if (!conf.getBoolean("hbase.master.infoserver.redirect", true)) {
645      return -1;
646    }
647    final int infoPort = conf.getInt("hbase.master.info.port.orig",
648      HConstants.DEFAULT_MASTER_INFOPORT);
649    // -1 is for disabling info server, so no redirecting
650    if (infoPort < 0 || infoServer == null) {
651      return -1;
652    }
653    if(infoPort == infoServer.getPort()) {
654      return infoPort;
655    }
656    final String addr = conf.get("hbase.master.info.bindAddress", "0.0.0.0");
657    if (!Addressing.isLocalAddress(InetAddress.getByName(addr))) {
658      String msg =
659          "Failed to start redirecting jetty server. Address " + addr
660              + " does not belong to this host. Correct configuration parameter: "
661              + "hbase.master.info.bindAddress";
662      LOG.error(msg);
663      throw new IOException(msg);
664    }
665
666    // TODO I'm pretty sure we could just add another binding to the InfoServer run by
667    // the RegionServer and have it run the RedirectServlet instead of standing up
668    // a second entire stack here.
669    masterJettyServer = new Server();
670    final ServerConnector connector = new ServerConnector(masterJettyServer);
671    connector.setHost(addr);
672    connector.setPort(infoPort);
673    masterJettyServer.addConnector(connector);
674    masterJettyServer.setStopAtShutdown(true);
675
676    final String redirectHostname =
677        StringUtils.isBlank(useThisHostnameInstead) ? null : useThisHostnameInstead;
678
679    final RedirectServlet redirect = new RedirectServlet(infoServer, redirectHostname);
680    final WebAppContext context = new WebAppContext(null, "/", null, null, null, null, WebAppContext.NO_SESSIONS);
681    context.addServlet(new ServletHolder(redirect), "/*");
682    context.setServer(masterJettyServer);
683
684    try {
685      masterJettyServer.start();
686    } catch (Exception e) {
687      throw new IOException("Failed to start redirecting jetty server", e);
688    }
689    return connector.getLocalPort();
690  }
691
692  /**
693   * For compatibility, if failed with regionserver credentials, try the master one
694   */
695  @Override
696  protected void login(UserProvider user, String host) throws IOException {
697    try {
698      super.login(user, host);
699    } catch (IOException ie) {
700      user.login(SecurityConstants.MASTER_KRB_KEYTAB_FILE,
701              SecurityConstants.MASTER_KRB_PRINCIPAL, host);
702    }
703  }
704
705  /**
706   * If configured to put regions on active master,
707   * wait till a backup master becomes active.
708   * Otherwise, loop till the server is stopped or aborted.
709   */
710  @Override
711  protected void waitForMasterActive(){
712    if (maintenanceMode) {
713      return;
714    }
715    boolean tablesOnMaster = LoadBalancer.isTablesOnMaster(conf);
716    while (!(tablesOnMaster && activeMaster) && !isStopped() && !isAborted()) {
717      sleeper.sleep();
718    }
719  }
720
721  @VisibleForTesting
722  public MasterRpcServices getMasterRpcServices() {
723    return (MasterRpcServices)rpcServices;
724  }
725
726  public boolean balanceSwitch(final boolean b) throws IOException {
727    return getMasterRpcServices().switchBalancer(b, BalanceSwitchMode.ASYNC);
728  }
729
730  @Override
731  protected String getProcessName() {
732    return MASTER;
733  }
734
735  @Override
736  protected boolean canCreateBaseZNode() {
737    return true;
738  }
739
740  @Override
741  protected boolean canUpdateTableDescriptor() {
742    return true;
743  }
744
745  @Override
746  protected boolean cacheTableDescriptor() {
747    return true;
748  }
749
750  @Override
751  protected RSRpcServices createRpcServices() throws IOException {
752    return new MasterRpcServices(this);
753  }
754
755  @Override
756  protected void configureInfoServer() {
757    infoServer.addUnprivilegedServlet("master-status", "/master-status", MasterStatusServlet.class);
758    infoServer.setAttribute(MASTER, this);
759    if (LoadBalancer.isTablesOnMaster(conf)) {
760      super.configureInfoServer();
761    }
762  }
763
764  @Override
765  protected Class<? extends HttpServlet> getDumpServlet() {
766    return MasterDumpServlet.class;
767  }
768
769  @Override
770  public MetricsMaster getMasterMetrics() {
771    return metricsMaster;
772  }
773
774  /**
775   * Initialize all ZK based system trackers. But do not include {@link RegionServerTracker}, it
776   * should have already been initialized along with {@link ServerManager}.
777   */
778  private void initializeZKBasedSystemTrackers()
779      throws IOException, KeeperException, ReplicationException {
780    this.balancer = new RSGroupBasedLoadBalancer();
781    this.balancer.setConf(conf);
782    this.loadBalancerTracker = new LoadBalancerTracker(zooKeeper, this);
783    this.loadBalancerTracker.start();
784
785    this.regionNormalizerManager =
786      RegionNormalizerFactory.createNormalizerManager(conf, zooKeeper, this);
787    this.regionNormalizerManager.start();
788
789    this.splitOrMergeTracker = new SplitOrMergeTracker(zooKeeper, conf, this);
790    this.splitOrMergeTracker.start();
791
792    // This is for backwards compatible. We do not need the CP for rs group now but if user want to
793    // load it, we need to enable rs group.
794    String[] cpClasses = conf.getStrings(MasterCoprocessorHost.MASTER_COPROCESSOR_CONF_KEY);
795    if (cpClasses != null) {
796      for (String cpClass : cpClasses) {
797        if (RSGroupAdminEndpoint.class.getName().equals(cpClass)) {
798          RSGroupUtil.enableRSGroup(conf);
799          break;
800        }
801      }
802    }
803    this.rsGroupInfoManager = RSGroupInfoManager.create(this);
804
805    this.replicationPeerManager = ReplicationPeerManager.create(zooKeeper, conf, clusterId);
806
807    this.drainingServerTracker = new DrainingServerTracker(zooKeeper, this, this.serverManager);
808    this.drainingServerTracker.start();
809
810    this.snapshotCleanupTracker = new SnapshotCleanupTracker(zooKeeper, this);
811    this.snapshotCleanupTracker.start();
812
813    String clientQuorumServers = conf.get(HConstants.CLIENT_ZOOKEEPER_QUORUM);
814    boolean clientZkObserverMode = conf.getBoolean(HConstants.CLIENT_ZOOKEEPER_OBSERVER_MODE,
815      HConstants.DEFAULT_CLIENT_ZOOKEEPER_OBSERVER_MODE);
816    if (clientQuorumServers != null && !clientZkObserverMode) {
817      // we need to take care of the ZK information synchronization
818      // if given client ZK are not observer nodes
819      ZKWatcher clientZkWatcher = new ZKWatcher(conf,
820          getProcessName() + ":" + rpcServices.getSocketAddress().getPort() + "-clientZK", this,
821          false, true);
822      this.metaLocationSyncer = new MetaLocationSyncer(zooKeeper, clientZkWatcher, this);
823      this.metaLocationSyncer.start();
824      this.masterAddressSyncer = new MasterAddressSyncer(zooKeeper, clientZkWatcher, this);
825      this.masterAddressSyncer.start();
826      // set cluster id is a one-go effort
827      ZKClusterId.setClusterId(clientZkWatcher, fileSystemManager.getClusterId());
828    }
829
830    // Set the cluster as up.  If new RSs, they'll be waiting on this before
831    // going ahead with their startup.
832    boolean wasUp = this.clusterStatusTracker.isClusterUp();
833    if (!wasUp) this.clusterStatusTracker.setClusterUp();
834
835    LOG.info("Active/primary master=" + this.serverName +
836        ", sessionid=0x" +
837        Long.toHexString(this.zooKeeper.getRecoverableZooKeeper().getSessionId()) +
838        ", setting cluster-up flag (Was=" + wasUp + ")");
839
840    // create/initialize the snapshot manager and other procedure managers
841    this.snapshotManager = new SnapshotManager();
842    this.mpmHost = new MasterProcedureManagerHost();
843    this.mpmHost.register(this.snapshotManager);
844    this.mpmHost.register(new MasterFlushTableProcedureManager());
845    this.mpmHost.loadProcedures(conf);
846    this.mpmHost.initialize(this, this.metricsMaster);
847  }
848
849  // Will be overriden in test to inject customized AssignmentManager
850  @VisibleForTesting
851  protected AssignmentManager createAssignmentManager(MasterServices master) {
852    return new AssignmentManager(master);
853  }
854
855  /**
856   * Finish initialization of HMaster after becoming the primary master.
857   * <p/>
858   * The startup order is a bit complicated but very important, do not change it unless you know
859   * what you are doing.
860   * <ol>
861   * <li>Initialize file system based components - file system manager, wal manager, table
862   * descriptors, etc</li>
863   * <li>Publish cluster id</li>
864   * <li>Here comes the most complicated part - initialize server manager, assignment manager and
865   * region server tracker
866   * <ol type='i'>
867   * <li>Create server manager</li>
868   * <li>Create procedure executor, load the procedures, but do not start workers. We will start it
869   * later after we finish scheduling SCPs to avoid scheduling duplicated SCPs for the same
870   * server</li>
871   * <li>Create assignment manager and start it, load the meta region state, but do not load data
872   * from meta region</li>
873   * <li>Start region server tracker, construct the online servers set and find out dead servers and
874   * schedule SCP for them. The online servers will be constructed by scanning zk, and we will also
875   * scan the wal directory to find out possible live region servers, and the differences between
876   * these two sets are the dead servers</li>
877   * </ol>
878   * </li>
879   * <li>If this is a new deploy, schedule a InitMetaProcedure to initialize meta</li>
880   * <li>Start necessary service threads - balancer, catalog janitor, executor services, and also
881   * the procedure executor, etc. Notice that the balancer must be created first as assignment
882   * manager may use it when assigning regions.</li>
883   * <li>Wait for meta to be initialized if necessary, start table state manager.</li>
884   * <li>Wait for enough region servers to check-in</li>
885   * <li>Let assignment manager load data from meta and construct region states</li>
886   * <li>Start all other things such as chore services, etc</li>
887   * </ol>
888   * <p/>
889   * Notice that now we will not schedule a special procedure to make meta online(unless the first
890   * time where meta has not been created yet), we will rely on SCP to bring meta online.
891   */
892  private void finishActiveMasterInitialization(MonitoredTask status) throws IOException,
893          InterruptedException, KeeperException, ReplicationException {
894    /*
895     * We are active master now... go initialize components we need to run.
896     */
897    status.setStatus("Initializing Master file system");
898
899    this.masterActiveTime = System.currentTimeMillis();
900    // TODO: Do this using Dependency Injection, using PicoContainer, Guice or Spring.
901
902    // always initialize the MemStoreLAB as we use a region to store data in master now, see
903    // localStore.
904    initializeMemStoreChunkCreator();
905    this.fileSystemManager = new MasterFileSystem(conf);
906    this.walManager = new MasterWalManager(this);
907
908    // warm-up HTDs cache on master initialization
909    if (preLoadTableDescriptors) {
910      status.setStatus("Pre-loading table descriptors");
911      this.tableDescriptors.getAll();
912    }
913
914    // Publish cluster ID; set it in Master too. The superclass RegionServer does this later but
915    // only after it has checked in with the Master. At least a few tests ask Master for clusterId
916    // before it has called its run method and before RegionServer has done the reportForDuty.
917    ClusterId clusterId = fileSystemManager.getClusterId();
918    status.setStatus("Publishing Cluster ID " + clusterId + " in ZooKeeper");
919    ZKClusterId.setClusterId(this.zooKeeper, fileSystemManager.getClusterId());
920    this.clusterId = clusterId.toString();
921
922    // Precaution. Put in place the old hbck1 lock file to fence out old hbase1s running their
923    // hbck1s against an hbase2 cluster; it could do damage. To skip this behavior, set
924    // hbase.write.hbck1.lock.file to false.
925    if (this.conf.getBoolean("hbase.write.hbck1.lock.file", true)) {
926      Pair<Path, FSDataOutputStream> result = null;
927      try {
928        result = HBaseFsck.checkAndMarkRunningHbck(this.conf,
929            HBaseFsck.createLockRetryCounterFactory(this.conf).create());
930      } finally {
931        if (result != null) {
932          IOUtils.closeQuietly(result.getSecond());
933        }
934      }
935    }
936
937    status.setStatus("Initialize ServerManager and schedule SCP for crash servers");
938    // The below two managers must be created before loading procedures, as they will be used during
939    // loading.
940    this.serverManager = createServerManager(this);
941    this.syncReplicationReplayWALManager = new SyncReplicationReplayWALManager(this);
942    if (!conf.getBoolean(HBASE_SPLIT_WAL_COORDINATED_BY_ZK,
943      DEFAULT_HBASE_SPLIT_COORDINATED_BY_ZK)) {
944      this.splitWALManager = new SplitWALManager(this);
945    }
946
947    // initialize master local region
948    masterRegion = MasterRegionFactory.create(this);
949    createProcedureExecutor();
950    Map<Class<?>, List<Procedure<MasterProcedureEnv>>> procsByType =
951      procedureExecutor.getActiveProceduresNoCopy().stream()
952        .collect(Collectors.groupingBy(p -> p.getClass()));
953
954    // Create Assignment Manager
955    this.assignmentManager = createAssignmentManager(this);
956    this.assignmentManager.start();
957    // TODO: TRSP can perform as the sub procedure for other procedures, so even if it is marked as
958    // completed, it could still be in the procedure list. This is a bit strange but is another
959    // story, need to verify the implementation for ProcedureExecutor and ProcedureStore.
960    List<TransitRegionStateProcedure> ritList =
961      procsByType.getOrDefault(TransitRegionStateProcedure.class, Collections.emptyList()).stream()
962        .filter(p -> !p.isFinished()).map(p -> (TransitRegionStateProcedure) p)
963        .collect(Collectors.toList());
964    this.assignmentManager.setupRIT(ritList);
965
966    // Start RegionServerTracker with listing of servers found with exiting SCPs -- these should
967    // be registered in the deadServers set -- and with the list of servernames out on the
968    // filesystem that COULD BE 'alive' (we'll schedule SCPs for each and let SCP figure it out).
969    // We also pass dirs that are already 'splitting'... so we can do some checks down in tracker.
970    // TODO: Generate the splitting and live Set in one pass instead of two as we currently do.
971    this.regionServerTracker = new RegionServerTracker(zooKeeper, this, this.serverManager);
972    this.regionServerTracker.start(
973      procsByType.getOrDefault(ServerCrashProcedure.class, Collections.emptyList()).stream()
974        .map(p -> (ServerCrashProcedure) p).map(p -> p.getServerName()).collect(Collectors.toSet()),
975      walManager.getLiveServersFromWALDir(), walManager.getSplittingServersFromWALDir());
976    // This manager will be started AFTER hbase:meta is confirmed on line.
977    // hbase.mirror.table.state.to.zookeeper is so hbase1 clients can connect. They read table
978    // state from zookeeper while hbase2 reads it from hbase:meta. Disable if no hbase1 clients.
979    this.tableStateManager =
980      this.conf.getBoolean(MirroringTableStateManager.MIRROR_TABLE_STATE_TO_ZK_KEY, true)
981        ?
982        new MirroringTableStateManager(this):
983        new TableStateManager(this);
984
985    status.setStatus("Initializing ZK system trackers");
986    initializeZKBasedSystemTrackers();
987    status.setStatus("Loading last flushed sequence id of regions");
988    try {
989      this.serverManager.loadLastFlushedSequenceIds();
990    } catch (IOException e) {
991      LOG.info("Failed to load last flushed sequence id of regions"
992          + " from file system", e);
993    }
994    // Set ourselves as active Master now our claim has succeeded up in zk.
995    this.activeMaster = true;
996
997    // Start the Zombie master detector after setting master as active, see HBASE-21535
998    Thread zombieDetector = new Thread(new InitializationMonitor(this),
999        "ActiveMasterInitializationMonitor-" + System.currentTimeMillis());
1000    zombieDetector.setDaemon(true);
1001    zombieDetector.start();
1002
1003    if (!maintenanceMode) {
1004      // Add the Observer to delete quotas on table deletion before starting all CPs by
1005      // default with quota support, avoiding if user specifically asks to not load this Observer.
1006      if (QuotaUtil.isQuotaEnabled(conf)) {
1007        updateConfigurationForQuotasObserver(conf);
1008      }
1009      // initialize master side coprocessors before we start handling requests
1010      status.setStatus("Initializing master coprocessors");
1011      this.cpHost = new MasterCoprocessorHost(this, this.conf);
1012    }
1013
1014    // Checking if meta needs initializing.
1015    status.setStatus("Initializing meta table if this is a new deploy");
1016    InitMetaProcedure initMetaProc = null;
1017    // Print out state of hbase:meta on startup; helps debugging.
1018    if (!this.assignmentManager.getRegionStates().hasTableRegionStates(TableName.META_TABLE_NAME)) {
1019      Optional<InitMetaProcedure> optProc = procedureExecutor.getProcedures().stream()
1020        .filter(p -> p instanceof InitMetaProcedure).map(o -> (InitMetaProcedure) o).findAny();
1021      initMetaProc = optProc.orElseGet(() -> {
1022        // schedule an init meta procedure if meta has not been deployed yet
1023        InitMetaProcedure temp = new InitMetaProcedure();
1024        procedureExecutor.submitProcedure(temp);
1025        return temp;
1026      });
1027    }
1028
1029    // initialize load balancer
1030    this.balancer.setMasterServices(this);
1031    this.balancer.setClusterMetrics(getClusterMetricsWithoutCoprocessor());
1032    this.balancer.initialize();
1033
1034    // start up all service threads.
1035    status.setStatus("Initializing master service threads");
1036    startServiceThreads();
1037    // wait meta to be initialized after we start procedure executor
1038    if (initMetaProc != null) {
1039      initMetaProc.await();
1040    }
1041    // Wake up this server to check in
1042    sleeper.skipSleepCycle();
1043
1044    // Wait for region servers to report in.
1045    // With this as part of master initialization, it precludes our being able to start a single
1046    // server that is both Master and RegionServer. Needs more thought. TODO.
1047    String statusStr = "Wait for region servers to report in";
1048    status.setStatus(statusStr);
1049    LOG.info(Objects.toString(status));
1050    waitForRegionServers(status);
1051
1052    // Check if master is shutting down because issue initializing regionservers or balancer.
1053    if (isStopped()) {
1054      return;
1055    }
1056
1057    status.setStatus("Starting assignment manager");
1058    // FIRST HBASE:META READ!!!!
1059    // The below cannot make progress w/o hbase:meta being online.
1060    // This is the FIRST attempt at going to hbase:meta. Meta on-lining is going on in background
1061    // as procedures run -- in particular SCPs for crashed servers... One should put up hbase:meta
1062    // if it is down. It may take a while to come online. So, wait here until meta if for sure
1063    // available. That's what waitForMetaOnline does.
1064    if (!waitForMetaOnline()) {
1065      return;
1066    }
1067    this.assignmentManager.joinCluster();
1068    // The below depends on hbase:meta being online.
1069    this.tableStateManager.start();
1070
1071    // for migrating from a version without HBASE-25099, and also for honoring the configuration
1072    // first.
1073    if (conf.get(HConstants.META_REPLICAS_NUM) != null) {
1074      int replicasNumInConf =
1075        conf.getInt(HConstants.META_REPLICAS_NUM, HConstants.DEFAULT_META_REPLICA_NUM);
1076      TableDescriptor metaDesc = tableDescriptors.get(TableName.META_TABLE_NAME);
1077      if (metaDesc.getRegionReplication() != replicasNumInConf) {
1078        // it is possible that we already have some replicas before upgrading, so we must set the
1079        // region replication number in meta TableDescriptor directly first, without creating a
1080        // ModifyTableProcedure, otherwise it may cause a double assign for the meta replicas.
1081        int existingReplicasCount =
1082          assignmentManager.getRegionStates().getRegionsOfTable(TableName.META_TABLE_NAME).size();
1083        if (existingReplicasCount > metaDesc.getRegionReplication()) {
1084          LOG.info("Update replica count of hbase:meta from {}(in TableDescriptor)" +
1085            " to {}(existing ZNodes)", metaDesc.getRegionReplication(), existingReplicasCount);
1086          metaDesc = TableDescriptorBuilder.newBuilder(metaDesc)
1087            .setRegionReplication(existingReplicasCount).build();
1088          tableDescriptors.update(metaDesc);
1089        }
1090        // check again, and issue a ModifyTableProcedure if needed
1091        if (metaDesc.getRegionReplication() != replicasNumInConf) {
1092          LOG.info(
1093            "The {} config is {} while the replica count in TableDescriptor is {}" +
1094              " for hbase:meta, altering...",
1095            HConstants.META_REPLICAS_NUM, replicasNumInConf, metaDesc.getRegionReplication());
1096          procedureExecutor.submitProcedure(new ModifyTableProcedure(
1097            procedureExecutor.getEnvironment(), TableDescriptorBuilder.newBuilder(metaDesc)
1098              .setRegionReplication(replicasNumInConf).build(),
1099            null, metaDesc, false));
1100        }
1101      }
1102    }
1103    // Below has to happen after tablestatemanager has started in the case where this hbase-2.x
1104    // is being started over an hbase-1.x dataset. tablestatemanager runs a migration as part
1105    // of its 'start' moving table state from zookeeper to hbase:meta. This migration needs to
1106    // complete before we do this next step processing offline regions else it fails reading
1107    // table states messing up master launch (namespace table, etc., are not assigned).
1108    this.assignmentManager.processOfflineRegions();
1109    // Initialize after meta is up as below scans meta
1110    if (getFavoredNodesManager() != null && !maintenanceMode) {
1111      SnapshotOfRegionAssignmentFromMeta snapshotOfRegionAssignment =
1112          new SnapshotOfRegionAssignmentFromMeta(getConnection());
1113      snapshotOfRegionAssignment.initialize();
1114      getFavoredNodesManager().initialize(snapshotOfRegionAssignment);
1115    }
1116
1117    // set cluster status again after user regions are assigned
1118    this.balancer.setClusterMetrics(getClusterMetricsWithoutCoprocessor());
1119
1120    // Start balancer and meta catalog janitor after meta and regions have been assigned.
1121    status.setStatus("Starting balancer and catalog janitor");
1122    this.clusterStatusChore = new ClusterStatusChore(this, balancer);
1123    getChoreService().scheduleChore(clusterStatusChore);
1124    this.balancerChore = new BalancerChore(this);
1125    getChoreService().scheduleChore(balancerChore);
1126    if (regionNormalizerManager != null) {
1127      getChoreService().scheduleChore(regionNormalizerManager.getRegionNormalizerChore());
1128    }
1129    this.catalogJanitorChore = new CatalogJanitor(this);
1130    getChoreService().scheduleChore(catalogJanitorChore);
1131    this.hbckChore = new HbckChore(this);
1132    getChoreService().scheduleChore(hbckChore);
1133    this.serverManager.startChore();
1134
1135    // Only for rolling upgrade, where we need to migrate the data in namespace table to meta table.
1136    if (!waitForNamespaceOnline()) {
1137      return;
1138    }
1139    status.setStatus("Starting cluster schema service");
1140    initClusterSchemaService();
1141
1142    if (this.cpHost != null) {
1143      try {
1144        this.cpHost.preMasterInitialization();
1145      } catch (IOException e) {
1146        LOG.error("Coprocessor preMasterInitialization() hook failed", e);
1147      }
1148    }
1149
1150    status.markComplete("Initialization successful");
1151    LOG.info(String.format("Master has completed initialization %.3fsec",
1152       (System.currentTimeMillis() - masterActiveTime) / 1000.0f));
1153    this.masterFinishedInitializationTime = System.currentTimeMillis();
1154    configurationManager.registerObserver(this.balancer);
1155    configurationManager.registerObserver(this.cleanerPool);
1156    configurationManager.registerObserver(this.hfileCleaner);
1157    configurationManager.registerObserver(this.logCleaner);
1158    configurationManager.registerObserver(this.regionsRecoveryConfigManager);
1159    // Set master as 'initialized'.
1160    setInitialized(true);
1161
1162    if (maintenanceMode) {
1163      LOG.info("Detected repair mode, skipping final initialization steps.");
1164      return;
1165    }
1166
1167    assignmentManager.checkIfShouldMoveSystemRegionAsync();
1168    status.setStatus("Starting quota manager");
1169    initQuotaManager();
1170    if (QuotaUtil.isQuotaEnabled(conf)) {
1171      // Create the quota snapshot notifier
1172      spaceQuotaSnapshotNotifier = createQuotaSnapshotNotifier();
1173      spaceQuotaSnapshotNotifier.initialize(getConnection());
1174      this.quotaObserverChore = new QuotaObserverChore(this, getMasterMetrics());
1175      // Start the chore to read the region FS space reports and act on them
1176      getChoreService().scheduleChore(quotaObserverChore);
1177
1178      this.snapshotQuotaChore = new SnapshotQuotaObserverChore(this, getMasterMetrics());
1179      // Start the chore to read snapshots and add their usage to table/NS quotas
1180      getChoreService().scheduleChore(snapshotQuotaChore);
1181    }
1182    final SlowLogMasterService slowLogMasterService = new SlowLogMasterService(conf, this);
1183    slowLogMasterService.init();
1184
1185    // clear the dead servers with same host name and port of online server because we are not
1186    // removing dead server with same hostname and port of rs which is trying to check in before
1187    // master initialization. See HBASE-5916.
1188    this.serverManager.clearDeadServersWithSameHostNameAndPortOfOnlineServer();
1189
1190    // Check and set the znode ACLs if needed in case we are overtaking a non-secure configuration
1191    status.setStatus("Checking ZNode ACLs");
1192    zooKeeper.checkAndSetZNodeAcls();
1193
1194    status.setStatus("Initializing MOB Cleaner");
1195    initMobCleaner();
1196
1197    status.setStatus("Calling postStartMaster coprocessors");
1198    if (this.cpHost != null) {
1199      // don't let cp initialization errors kill the master
1200      try {
1201        this.cpHost.postStartMaster();
1202      } catch (IOException ioe) {
1203        LOG.error("Coprocessor postStartMaster() hook failed", ioe);
1204      }
1205    }
1206
1207    zombieDetector.interrupt();
1208
1209    /*
1210     * After master has started up, lets do balancer post startup initialization. Since this runs
1211     * in activeMasterManager thread, it should be fine.
1212     */
1213    long start = System.currentTimeMillis();
1214    this.balancer.postMasterStartupInitialize();
1215    if (LOG.isDebugEnabled()) {
1216      LOG.debug("Balancer post startup initialization complete, took " + (
1217          (System.currentTimeMillis() - start) / 1000) + " seconds");
1218    }
1219  }
1220
1221  /**
1222   * Check hbase:meta is up and ready for reading. For use during Master startup only.
1223   * @return True if meta is UP and online and startup can progress. Otherwise, meta is not online
1224   *   and we will hold here until operator intervention.
1225   */
1226  @VisibleForTesting
1227  public boolean waitForMetaOnline() {
1228    return isRegionOnline(RegionInfoBuilder.FIRST_META_REGIONINFO);
1229  }
1230
1231  /**
1232   * @return True if region is online and scannable else false if an error or shutdown (Otherwise
1233   *   we just block in here holding up all forward-progess).
1234   */
1235  private boolean isRegionOnline(RegionInfo ri) {
1236    RetryCounter rc = null;
1237    while (!isStopped()) {
1238      RegionState rs = this.assignmentManager.getRegionStates().getRegionState(ri);
1239      if (rs.isOpened()) {
1240        if (this.getServerManager().isServerOnline(rs.getServerName())) {
1241          return true;
1242        }
1243      }
1244      // Region is not OPEN.
1245      Optional<Procedure<MasterProcedureEnv>> optProc = this.procedureExecutor.getProcedures().
1246          stream().filter(p -> p instanceof ServerCrashProcedure).findAny();
1247      // TODO: Add a page to refguide on how to do repair. Have this log message point to it.
1248      // Page will talk about loss of edits, how to schedule at least the meta WAL recovery, and
1249      // then how to assign including how to break region lock if one held.
1250      LOG.warn("{} is NOT online; state={}; ServerCrashProcedures={}. Master startup cannot " +
1251          "progress, in holding-pattern until region onlined.",
1252          ri.getRegionNameAsString(), rs, optProc.isPresent());
1253      // Check once-a-minute.
1254      if (rc == null) {
1255        rc = new RetryCounterFactory(1000).create();
1256      }
1257      Threads.sleep(rc.getBackoffTimeAndIncrementAttempts());
1258    }
1259    return false;
1260  }
1261
1262  /**
1263   * Check hbase:namespace table is assigned. If not, startup will hang looking for the ns table
1264   * <p/>
1265   * This is for rolling upgrading, later we will migrate the data in ns table to the ns family of
1266   * meta table. And if this is a new cluster, this method will return immediately as there will be
1267   * no namespace table/region.
1268   * @return True if namespace table is up/online.
1269   */
1270  private boolean waitForNamespaceOnline() throws IOException {
1271    TableState nsTableState =
1272      MetaTableAccessor.getTableState(getConnection(), TableName.NAMESPACE_TABLE_NAME);
1273    if (nsTableState == null || nsTableState.isDisabled()) {
1274      // this means we have already migrated the data and disabled or deleted the namespace table,
1275      // or this is a new deploy which does not have a namespace table from the beginning.
1276      return true;
1277    }
1278    List<RegionInfo> ris =
1279      this.assignmentManager.getRegionStates().getRegionsOfTable(TableName.NAMESPACE_TABLE_NAME);
1280    if (ris.isEmpty()) {
1281      // maybe this will not happen any more, but anyway, no harm to add a check here...
1282      return true;
1283    }
1284    // Else there are namespace regions up in meta. Ensure they are assigned before we go on.
1285    for (RegionInfo ri : ris) {
1286      if (!isRegionOnline(ri)) {
1287        return false;
1288      }
1289    }
1290    return true;
1291  }
1292
1293  /**
1294   * Adds the {@code MasterQuotasObserver} to the list of configured Master observers to
1295   * automatically remove quotas for a table when that table is deleted.
1296   */
1297  @VisibleForTesting
1298  public void updateConfigurationForQuotasObserver(Configuration conf) {
1299    // We're configured to not delete quotas on table deletion, so we don't need to add the obs.
1300    if (!conf.getBoolean(
1301          MasterQuotasObserver.REMOVE_QUOTA_ON_TABLE_DELETE,
1302          MasterQuotasObserver.REMOVE_QUOTA_ON_TABLE_DELETE_DEFAULT)) {
1303      return;
1304    }
1305    String[] masterCoprocs = conf.getStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY);
1306    final int length = null == masterCoprocs ? 0 : masterCoprocs.length;
1307    String[] updatedCoprocs = new String[length + 1];
1308    if (length > 0) {
1309      System.arraycopy(masterCoprocs, 0, updatedCoprocs, 0, masterCoprocs.length);
1310    }
1311    updatedCoprocs[length] = MasterQuotasObserver.class.getName();
1312    conf.setStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, updatedCoprocs);
1313  }
1314
1315  private void initMobCleaner() {
1316    this.mobFileCleanerChore = new MobFileCleanerChore(this);
1317    getChoreService().scheduleChore(mobFileCleanerChore);
1318    this.mobFileCompactionChore = new MobFileCompactionChore(this);
1319    getChoreService().scheduleChore(mobFileCompactionChore);
1320  }
1321
1322  /**
1323   * <p>
1324   * Create a {@link ServerManager} instance.
1325   * </p>
1326   * <p>
1327   * Will be overridden in tests.
1328   * </p>
1329   */
1330  @VisibleForTesting
1331  protected ServerManager createServerManager(final MasterServices master) throws IOException {
1332    // We put this out here in a method so can do a Mockito.spy and stub it out
1333    // w/ a mocked up ServerManager.
1334    setupClusterConnection();
1335    return new ServerManager(master);
1336  }
1337
1338  private void waitForRegionServers(final MonitoredTask status)
1339      throws IOException, InterruptedException {
1340    this.serverManager.waitForRegionServers(status);
1341  }
1342
1343  // Will be overridden in tests
1344  @VisibleForTesting
1345  protected void initClusterSchemaService() throws IOException, InterruptedException {
1346    this.clusterSchemaService = new ClusterSchemaServiceImpl(this);
1347    this.clusterSchemaService.startAsync();
1348    try {
1349      this.clusterSchemaService.awaitRunning(getConfiguration().getInt(
1350        HBASE_MASTER_WAIT_ON_SERVICE_IN_SECONDS,
1351        DEFAULT_HBASE_MASTER_WAIT_ON_SERVICE_IN_SECONDS), TimeUnit.SECONDS);
1352    } catch (TimeoutException toe) {
1353      throw new IOException("Timedout starting ClusterSchemaService", toe);
1354    }
1355  }
1356
1357  private void initQuotaManager() throws IOException {
1358    MasterQuotaManager quotaManager = new MasterQuotaManager(this);
1359    quotaManager.start();
1360    this.quotaManager = quotaManager;
1361  }
1362
1363  private SpaceQuotaSnapshotNotifier createQuotaSnapshotNotifier() {
1364    SpaceQuotaSnapshotNotifier notifier =
1365        SpaceQuotaSnapshotNotifierFactory.getInstance().create(getConfiguration());
1366    return notifier;
1367  }
1368
1369  boolean isCatalogJanitorEnabled() {
1370    return catalogJanitorChore != null ? catalogJanitorChore.getEnabled() : false;
1371  }
1372
1373  boolean isCleanerChoreEnabled() {
1374    boolean hfileCleanerFlag = true, logCleanerFlag = true;
1375
1376    if (hfileCleaner != null) {
1377      hfileCleanerFlag = hfileCleaner.getEnabled();
1378    }
1379
1380    if (logCleaner != null) {
1381      logCleanerFlag = logCleaner.getEnabled();
1382    }
1383
1384    return (hfileCleanerFlag && logCleanerFlag);
1385  }
1386
1387  @Override
1388  public ServerManager getServerManager() {
1389    return this.serverManager;
1390  }
1391
1392  @Override
1393  public MasterFileSystem getMasterFileSystem() {
1394    return this.fileSystemManager;
1395  }
1396
1397  @Override
1398  public MasterWalManager getMasterWalManager() {
1399    return this.walManager;
1400  }
1401
1402  @Override
1403  public SplitWALManager getSplitWALManager() {
1404    return splitWALManager;
1405  }
1406
1407  @Override
1408  public TableStateManager getTableStateManager() {
1409    return tableStateManager;
1410  }
1411
1412  /*
1413   * Start up all services. If any of these threads gets an unhandled exception
1414   * then they just die with a logged message.  This should be fine because
1415   * in general, we do not expect the master to get such unhandled exceptions
1416   *  as OOMEs; it should be lightly loaded. See what HRegionServer does if
1417   *  need to install an unexpected exception handler.
1418   */
1419  private void startServiceThreads() throws IOException {
1420    // Start the executor service pools
1421    this.executorService.startExecutorService(ExecutorType.MASTER_OPEN_REGION, conf.getInt(
1422      HConstants.MASTER_OPEN_REGION_THREADS, HConstants.MASTER_OPEN_REGION_THREADS_DEFAULT));
1423    this.executorService.startExecutorService(ExecutorType.MASTER_CLOSE_REGION, conf.getInt(
1424      HConstants.MASTER_CLOSE_REGION_THREADS, HConstants.MASTER_CLOSE_REGION_THREADS_DEFAULT));
1425    this.executorService.startExecutorService(ExecutorType.MASTER_SERVER_OPERATIONS,
1426      conf.getInt(HConstants.MASTER_SERVER_OPERATIONS_THREADS,
1427        HConstants.MASTER_SERVER_OPERATIONS_THREADS_DEFAULT));
1428    this.executorService.startExecutorService(ExecutorType.MASTER_META_SERVER_OPERATIONS,
1429      conf.getInt(HConstants.MASTER_META_SERVER_OPERATIONS_THREADS,
1430        HConstants.MASTER_META_SERVER_OPERATIONS_THREADS_DEFAULT));
1431    this.executorService.startExecutorService(ExecutorType.M_LOG_REPLAY_OPS, conf.getInt(
1432      HConstants.MASTER_LOG_REPLAY_OPS_THREADS, HConstants.MASTER_LOG_REPLAY_OPS_THREADS_DEFAULT));
1433    this.executorService.startExecutorService(ExecutorType.MASTER_SNAPSHOT_OPERATIONS, conf.getInt(
1434      SnapshotManager.SNAPSHOT_POOL_THREADS_KEY, SnapshotManager.SNAPSHOT_POOL_THREADS_DEFAULT));
1435
1436    // We depend on there being only one instance of this executor running
1437    // at a time. To do concurrency, would need fencing of enable/disable of
1438    // tables.
1439    // Any time changing this maxThreads to > 1, pls see the comment at
1440    // AccessController#postCompletedCreateTableAction
1441    this.executorService.startExecutorService(ExecutorType.MASTER_TABLE_OPERATIONS, 1);
1442    startProcedureExecutor();
1443
1444    // Create cleaner thread pool
1445    cleanerPool = new DirScanPool(conf);
1446    // Start log cleaner thread
1447    int cleanerInterval =
1448      conf.getInt(HBASE_MASTER_CLEANER_INTERVAL, DEFAULT_HBASE_MASTER_CLEANER_INTERVAL);
1449    this.logCleaner = new LogCleaner(cleanerInterval, this, conf,
1450      getMasterWalManager().getFileSystem(), getMasterWalManager().getOldLogDir(), cleanerPool);
1451    getChoreService().scheduleChore(logCleaner);
1452
1453    // start the hfile archive cleaner thread
1454    Path archiveDir = HFileArchiveUtil.getArchivePath(conf);
1455    Map<String, Object> params = new HashMap<>();
1456    params.put(MASTER, this);
1457    this.hfileCleaner = new HFileCleaner(cleanerInterval, this, conf,
1458      getMasterFileSystem().getFileSystem(), archiveDir, cleanerPool, params);
1459    getChoreService().scheduleChore(hfileCleaner);
1460
1461    // Regions Reopen based on very high storeFileRefCount is considered enabled
1462    // only if hbase.regions.recovery.store.file.ref.count has value > 0
1463    final int maxStoreFileRefCount = conf.getInt(
1464      HConstants.STORE_FILE_REF_COUNT_THRESHOLD,
1465      HConstants.DEFAULT_STORE_FILE_REF_COUNT_THRESHOLD);
1466    if (maxStoreFileRefCount > 0) {
1467      this.regionsRecoveryChore = new RegionsRecoveryChore(this, conf, this);
1468      getChoreService().scheduleChore(this.regionsRecoveryChore);
1469    } else {
1470      LOG.info("Reopening regions with very high storeFileRefCount is disabled. " +
1471          "Provide threshold value > 0 for {} to enable it.",
1472        HConstants.STORE_FILE_REF_COUNT_THRESHOLD);
1473    }
1474
1475    this.regionsRecoveryConfigManager = new RegionsRecoveryConfigManager(this);
1476
1477    replicationBarrierCleaner = new ReplicationBarrierCleaner(conf, this, getConnection(),
1478      replicationPeerManager);
1479    getChoreService().scheduleChore(replicationBarrierCleaner);
1480
1481    final boolean isSnapshotChoreEnabled = this.snapshotCleanupTracker
1482        .isSnapshotCleanupEnabled();
1483    this.snapshotCleanerChore = new SnapshotCleanerChore(this, conf, getSnapshotManager());
1484    if (isSnapshotChoreEnabled) {
1485      getChoreService().scheduleChore(this.snapshotCleanerChore);
1486    } else {
1487      if (LOG.isTraceEnabled()) {
1488        LOG.trace("Snapshot Cleaner Chore is disabled. Not starting up the chore..");
1489      }
1490    }
1491    serviceStarted = true;
1492    if (LOG.isTraceEnabled()) {
1493      LOG.trace("Started service threads");
1494    }
1495  }
1496
1497  @Override
1498  protected void stopServiceThreads() {
1499    if (masterJettyServer != null) {
1500      LOG.info("Stopping master jetty server");
1501      try {
1502        masterJettyServer.stop();
1503      } catch (Exception e) {
1504        LOG.error("Failed to stop master jetty server", e);
1505      }
1506    }
1507    stopChores();
1508
1509    super.stopServiceThreads();
1510    if (cleanerPool != null) {
1511      cleanerPool.shutdownNow();
1512      cleanerPool = null;
1513    }
1514
1515    LOG.debug("Stopping service threads");
1516
1517    // stop procedure executor prior to other services such as server manager and assignment
1518    // manager, as these services are important for some running procedures. See HBASE-24117 for
1519    // example.
1520    stopProcedureExecutor();
1521
1522    if (regionNormalizerManager != null) {
1523      regionNormalizerManager.stop();
1524    }
1525    if (this.quotaManager != null) {
1526      this.quotaManager.stop();
1527    }
1528
1529    if (this.activeMasterManager != null) {
1530      this.activeMasterManager.stop();
1531    }
1532    if (this.serverManager != null) {
1533      this.serverManager.stop();
1534    }
1535    if (this.assignmentManager != null) {
1536      this.assignmentManager.stop();
1537    }
1538
1539    if (masterRegion != null) {
1540      masterRegion.close(isAborted());
1541    }
1542    if (this.walManager != null) {
1543      this.walManager.stop();
1544    }
1545    if (this.fileSystemManager != null) {
1546      this.fileSystemManager.stop();
1547    }
1548    if (this.mpmHost != null) {
1549      this.mpmHost.stop("server shutting down.");
1550    }
1551    if (this.regionServerTracker != null) {
1552      this.regionServerTracker.stop();
1553    }
1554  }
1555
1556  private void createProcedureExecutor() throws IOException {
1557    MasterProcedureEnv procEnv = new MasterProcedureEnv(this);
1558    procedureStore =
1559      new RegionProcedureStore(this, masterRegion, new MasterProcedureEnv.FsUtilsLeaseRecovery(this));
1560    procedureStore.registerListener(new ProcedureStoreListener() {
1561
1562      @Override
1563      public void abortProcess() {
1564        abort("The Procedure Store lost the lease", null);
1565      }
1566    });
1567    MasterProcedureScheduler procedureScheduler = procEnv.getProcedureScheduler();
1568    procedureExecutor = new ProcedureExecutor<>(conf, procEnv, procedureStore, procedureScheduler);
1569    configurationManager.registerObserver(procEnv);
1570
1571    int cpus = Runtime.getRuntime().availableProcessors();
1572    final int numThreads = conf.getInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, Math.max(
1573      (cpus > 0 ? cpus / 4 : 0), MasterProcedureConstants.DEFAULT_MIN_MASTER_PROCEDURE_THREADS));
1574    final boolean abortOnCorruption =
1575      conf.getBoolean(MasterProcedureConstants.EXECUTOR_ABORT_ON_CORRUPTION,
1576        MasterProcedureConstants.DEFAULT_EXECUTOR_ABORT_ON_CORRUPTION);
1577    procedureStore.start(numThreads);
1578    // Just initialize it but do not start the workers, we will start the workers later by calling
1579    // startProcedureExecutor. See the javadoc for finishActiveMasterInitialization for more
1580    // details.
1581    procedureExecutor.init(numThreads, abortOnCorruption);
1582    if (!procEnv.getRemoteDispatcher().start()) {
1583      throw new HBaseIOException("Failed start of remote dispatcher");
1584    }
1585  }
1586
1587  private void startProcedureExecutor() throws IOException {
1588    procedureExecutor.startWorkers();
1589  }
1590
1591  /**
1592   * Turn on/off Snapshot Cleanup Chore
1593   *
1594   * @param on indicates whether Snapshot Cleanup Chore is to be run
1595   */
1596  void switchSnapshotCleanup(final boolean on, final boolean synchronous) {
1597    if (synchronous) {
1598      synchronized (this.snapshotCleanerChore) {
1599        switchSnapshotCleanup(on);
1600      }
1601    } else {
1602      switchSnapshotCleanup(on);
1603    }
1604  }
1605
1606  private void switchSnapshotCleanup(final boolean on) {
1607    try {
1608      snapshotCleanupTracker.setSnapshotCleanupEnabled(on);
1609      if (on) {
1610        if (!getChoreService().isChoreScheduled(this.snapshotCleanerChore)) {
1611          getChoreService().scheduleChore(this.snapshotCleanerChore);
1612        }
1613      } else {
1614        getChoreService().cancelChore(this.snapshotCleanerChore);
1615      }
1616    } catch (KeeperException e) {
1617      LOG.error("Error updating snapshot cleanup mode to {}", on, e);
1618    }
1619  }
1620
1621
1622  private void stopProcedureExecutor() {
1623    if (procedureExecutor != null) {
1624      configurationManager.deregisterObserver(procedureExecutor.getEnvironment());
1625      procedureExecutor.getEnvironment().getRemoteDispatcher().stop();
1626      procedureExecutor.stop();
1627      procedureExecutor.join();
1628      procedureExecutor = null;
1629    }
1630
1631    if (procedureStore != null) {
1632      procedureStore.stop(isAborted());
1633      procedureStore = null;
1634    }
1635  }
1636
1637  private void stopChores() {
1638    ChoreService choreService = getChoreService();
1639    if (choreService != null) {
1640      choreService.cancelChore(this.mobFileCleanerChore);
1641      choreService.cancelChore(this.mobFileCompactionChore);
1642      choreService.cancelChore(this.balancerChore);
1643      if (regionNormalizerManager != null) {
1644        choreService.cancelChore(regionNormalizerManager.getRegionNormalizerChore());
1645      }
1646      choreService.cancelChore(this.clusterStatusChore);
1647      choreService.cancelChore(this.catalogJanitorChore);
1648      choreService.cancelChore(this.clusterStatusPublisherChore);
1649      choreService.cancelChore(this.snapshotQuotaChore);
1650      choreService.cancelChore(this.logCleaner);
1651      choreService.cancelChore(this.hfileCleaner);
1652      choreService.cancelChore(this.replicationBarrierCleaner);
1653      choreService.cancelChore(this.snapshotCleanerChore);
1654      choreService.cancelChore(this.hbckChore);
1655      choreService.cancelChore(this.regionsRecoveryChore);
1656    }
1657  }
1658
1659  /**
1660   * @return Get remote side's InetAddress
1661   */
1662  InetAddress getRemoteInetAddress(final int port,
1663      final long serverStartCode) throws UnknownHostException {
1664    // Do it out here in its own little method so can fake an address when
1665    // mocking up in tests.
1666    InetAddress ia = RpcServer.getRemoteIp();
1667
1668    // The call could be from the local regionserver,
1669    // in which case, there is no remote address.
1670    if (ia == null && serverStartCode == startcode) {
1671      InetSocketAddress isa = rpcServices.getSocketAddress();
1672      if (isa != null && isa.getPort() == port) {
1673        ia = isa.getAddress();
1674      }
1675    }
1676    return ia;
1677  }
1678
1679  /**
1680   * @return Maximum time we should run balancer for
1681   */
1682  private int getMaxBalancingTime() {
1683    // if max balancing time isn't set, defaulting it to period time
1684    int maxBalancingTime = getConfiguration().getInt(HConstants.HBASE_BALANCER_MAX_BALANCING,
1685      getConfiguration()
1686        .getInt(HConstants.HBASE_BALANCER_PERIOD, HConstants.DEFAULT_HBASE_BALANCER_PERIOD));
1687    return maxBalancingTime;
1688  }
1689
1690  /**
1691   * @return Maximum number of regions in transition
1692   */
1693  private int getMaxRegionsInTransition() {
1694    int numRegions = this.assignmentManager.getRegionStates().getRegionAssignments().size();
1695    return Math.max((int) Math.floor(numRegions * this.maxRitPercent), 1);
1696  }
1697
1698  /**
1699   * It first sleep to the next balance plan start time. Meanwhile, throttling by the max
1700   * number regions in transition to protect availability.
1701   * @param nextBalanceStartTime The next balance plan start time
1702   * @param maxRegionsInTransition max number of regions in transition
1703   * @param cutoffTime when to exit balancer
1704   */
1705  private void balanceThrottling(long nextBalanceStartTime, int maxRegionsInTransition,
1706      long cutoffTime) {
1707    boolean interrupted = false;
1708
1709    // Sleep to next balance plan start time
1710    // But if there are zero regions in transition, it can skip sleep to speed up.
1711    while (!interrupted && System.currentTimeMillis() < nextBalanceStartTime
1712        && this.assignmentManager.getRegionStates().hasRegionsInTransition()) {
1713      try {
1714        Thread.sleep(100);
1715      } catch (InterruptedException ie) {
1716        interrupted = true;
1717      }
1718    }
1719
1720    // Throttling by max number regions in transition
1721    while (!interrupted
1722        && maxRegionsInTransition > 0
1723        && this.assignmentManager.getRegionStates().getRegionsInTransitionCount()
1724        >= maxRegionsInTransition && System.currentTimeMillis() <= cutoffTime) {
1725      try {
1726        // sleep if the number of regions in transition exceeds the limit
1727        Thread.sleep(100);
1728      } catch (InterruptedException ie) {
1729        interrupted = true;
1730      }
1731    }
1732
1733    if (interrupted) Thread.currentThread().interrupt();
1734  }
1735
1736  public boolean balance() throws IOException {
1737    return balance(false);
1738  }
1739
1740  /**
1741   * Checks master state before initiating action over region topology.
1742   * @param action the name of the action under consideration, for logging.
1743   * @return {@code true} when the caller should exit early, {@code false} otherwise.
1744   */
1745  @Override
1746  public boolean skipRegionManagementAction(final String action) {
1747    // Note: this method could be `default` on MasterServices if but for logging.
1748    if (!isInitialized()) {
1749      LOG.debug("Master has not been initialized, don't run {}.", action);
1750      return true;
1751    }
1752    if (this.getServerManager().isClusterShutdown()) {
1753      LOG.info("Cluster is shutting down, don't run {}.", action);
1754      return true;
1755    }
1756    if (isInMaintenanceMode()) {
1757      LOG.info("Master is in maintenance mode, don't run {}.", action);
1758      return true;
1759    }
1760    return false;
1761  }
1762
1763  public boolean balance(boolean force) throws IOException {
1764    if (loadBalancerTracker == null || !loadBalancerTracker.isBalancerOn()) {
1765      return false;
1766    }
1767    if (skipRegionManagementAction("balancer")) {
1768      return false;
1769    }
1770
1771    synchronized (this.balancer) {
1772        // Only allow one balance run at at time.
1773      if (this.assignmentManager.hasRegionsInTransition()) {
1774        List<RegionStateNode> regionsInTransition = assignmentManager.getRegionsInTransition();
1775        // if hbase:meta region is in transition, result of assignment cannot be recorded
1776        // ignore the force flag in that case
1777        boolean metaInTransition = assignmentManager.isMetaRegionInTransition();
1778        String prefix = force && !metaInTransition ? "R" : "Not r";
1779        List<RegionStateNode> toPrint = regionsInTransition;
1780        int max = 5;
1781        boolean truncated = false;
1782        if (regionsInTransition.size() > max) {
1783          toPrint = regionsInTransition.subList(0, max);
1784          truncated = true;
1785        }
1786        LOG.info(prefix + " not running balancer because " + regionsInTransition.size() +
1787          " region(s) in transition: " + toPrint + (truncated? "(truncated list)": ""));
1788        if (!force || metaInTransition) return false;
1789      }
1790      if (this.serverManager.areDeadServersInProgress()) {
1791        LOG.info("Not running balancer because processing dead regionserver(s): " +
1792          this.serverManager.getDeadServers());
1793        return false;
1794      }
1795
1796      if (this.cpHost != null) {
1797        try {
1798          if (this.cpHost.preBalance()) {
1799            LOG.debug("Coprocessor bypassing balancer request");
1800            return false;
1801          }
1802        } catch (IOException ioe) {
1803          LOG.error("Error invoking master coprocessor preBalance()", ioe);
1804          return false;
1805        }
1806      }
1807
1808      Map<TableName, Map<ServerName, List<RegionInfo>>> assignments =
1809        this.assignmentManager.getRegionStates()
1810          .getAssignmentsForBalancer(tableStateManager, this.serverManager.getOnlineServersList());
1811      for (Map<ServerName, List<RegionInfo>> serverMap : assignments.values()) {
1812        serverMap.keySet().removeAll(this.serverManager.getDrainingServersList());
1813      }
1814
1815      //Give the balancer the current cluster state.
1816      this.balancer.setClusterMetrics(getClusterMetricsWithoutCoprocessor());
1817
1818      List<RegionPlan> plans = this.balancer.balanceCluster(assignments);
1819
1820      if (skipRegionManagementAction("balancer")) {
1821        // make one last check that the cluster isn't shutting down before proceeding.
1822        return false;
1823      }
1824
1825      List<RegionPlan> sucRPs = executeRegionPlansWithThrottling(plans);
1826
1827      if (this.cpHost != null) {
1828        try {
1829          this.cpHost.postBalance(sucRPs);
1830        } catch (IOException ioe) {
1831          // balancing already succeeded so don't change the result
1832          LOG.error("Error invoking master coprocessor postBalance()", ioe);
1833        }
1834      }
1835    }
1836    // If LoadBalancer did not generate any plans, it means the cluster is already balanced.
1837    // Return true indicating a success.
1838    return true;
1839  }
1840
1841  /**
1842   * Execute region plans with throttling
1843   * @param plans to execute
1844   * @return succeeded plans
1845   */
1846  public List<RegionPlan> executeRegionPlansWithThrottling(List<RegionPlan> plans) {
1847    List<RegionPlan> successRegionPlans = new ArrayList<>();
1848    int maxRegionsInTransition = getMaxRegionsInTransition();
1849    long balanceStartTime = System.currentTimeMillis();
1850    long cutoffTime = balanceStartTime + this.maxBalancingTime;
1851    int rpCount = 0;  // number of RegionPlans balanced so far
1852    if (plans != null && !plans.isEmpty()) {
1853      int balanceInterval = this.maxBalancingTime / plans.size();
1854      LOG.info("Balancer plans size is " + plans.size() + ", the balance interval is "
1855          + balanceInterval + " ms, and the max number regions in transition is "
1856          + maxRegionsInTransition);
1857
1858      for (RegionPlan plan: plans) {
1859        LOG.info("balance " + plan);
1860        //TODO: bulk assign
1861        try {
1862          this.assignmentManager.moveAsync(plan);
1863        } catch (HBaseIOException hioe) {
1864          //should ignore failed plans here, avoiding the whole balance plans be aborted
1865          //later calls of balance() can fetch up the failed and skipped plans
1866          LOG.warn("Failed balance plan {}, skipping...", plan, hioe);
1867        }
1868        //rpCount records balance plans processed, does not care if a plan succeeds
1869        rpCount++;
1870        successRegionPlans.add(plan);
1871
1872        if (this.maxBalancingTime > 0) {
1873          balanceThrottling(balanceStartTime + rpCount * balanceInterval, maxRegionsInTransition,
1874            cutoffTime);
1875        }
1876
1877        // if performing next balance exceeds cutoff time, exit the loop
1878        if (this.maxBalancingTime > 0 && rpCount < plans.size()
1879          && System.currentTimeMillis() > cutoffTime) {
1880          // TODO: After balance, there should not be a cutoff time (keeping it as
1881          // a security net for now)
1882          LOG.debug("No more balancing till next balance run; maxBalanceTime="
1883              + this.maxBalancingTime);
1884          break;
1885        }
1886      }
1887    }
1888    return successRegionPlans;
1889  }
1890
1891  @Override
1892  public RegionNormalizerManager getRegionNormalizerManager() {
1893    return regionNormalizerManager;
1894  }
1895
1896  @Override
1897  public boolean normalizeRegions(
1898    final NormalizeTableFilterParams ntfp,
1899    final boolean isHighPriority
1900  ) throws IOException {
1901    if (regionNormalizerManager == null || !regionNormalizerManager.isNormalizerOn()) {
1902      LOG.debug("Region normalization is disabled, don't run region normalizer.");
1903      return false;
1904    }
1905    if (skipRegionManagementAction("region normalizer")) {
1906      return false;
1907    }
1908    if (assignmentManager.hasRegionsInTransition()) {
1909      return false;
1910    }
1911
1912    final Set<TableName> matchingTables = getTableDescriptors(new LinkedList<>(),
1913      ntfp.getNamespace(), ntfp.getRegex(), ntfp.getTableNames(), false)
1914      .stream()
1915      .map(TableDescriptor::getTableName)
1916      .collect(Collectors.toSet());
1917    final Set<TableName> allEnabledTables =
1918      tableStateManager.getTablesInStates(TableState.State.ENABLED);
1919    final List<TableName> targetTables =
1920      new ArrayList<>(Sets.intersection(matchingTables, allEnabledTables));
1921    Collections.shuffle(targetTables);
1922    return regionNormalizerManager.normalizeRegions(targetTables, isHighPriority);
1923  }
1924
1925  /**
1926   * @return Client info for use as prefix on an audit log string; who did an action
1927   */
1928  @Override
1929  public String getClientIdAuditPrefix() {
1930    return "Client=" + RpcServer.getRequestUserName().orElse(null)
1931        + "/" + RpcServer.getRemoteAddress().orElse(null);
1932  }
1933
1934  /**
1935   * Switch for the background CatalogJanitor thread.
1936   * Used for testing.  The thread will continue to run.  It will just be a noop
1937   * if disabled.
1938   * @param b If false, the catalog janitor won't do anything.
1939   */
1940  public void setCatalogJanitorEnabled(final boolean b) {
1941    this.catalogJanitorChore.setEnabled(b);
1942  }
1943
1944  @Override
1945  public long mergeRegions(
1946      final RegionInfo[] regionsToMerge,
1947      final boolean forcible,
1948      final long ng,
1949      final long nonce) throws IOException {
1950    checkInitialized();
1951
1952    if (!isSplitOrMergeEnabled(MasterSwitchType.MERGE)) {
1953      String regionsStr = Arrays.deepToString(regionsToMerge);
1954      LOG.warn("Merge switch is off! skip merge of " + regionsStr);
1955      throw new DoNotRetryIOException("Merge of " + regionsStr +
1956          " failed because merge switch is off");
1957    }
1958
1959    final String mergeRegionsStr = Arrays.stream(regionsToMerge).map(RegionInfo::getEncodedName)
1960      .collect(Collectors.joining(", "));
1961    return MasterProcedureUtil.submitProcedure(new NonceProcedureRunnable(this, ng, nonce) {
1962      @Override
1963      protected void run() throws IOException {
1964        getMaster().getMasterCoprocessorHost().preMergeRegions(regionsToMerge);
1965        String aid = getClientIdAuditPrefix();
1966        LOG.info("{} merge regions {}", aid, mergeRegionsStr);
1967        submitProcedure(new MergeTableRegionsProcedure(procedureExecutor.getEnvironment(),
1968            regionsToMerge, forcible));
1969        getMaster().getMasterCoprocessorHost().postMergeRegions(regionsToMerge);
1970      }
1971
1972      @Override
1973      protected String getDescription() {
1974        return "MergeTableProcedure";
1975      }
1976    });
1977  }
1978
1979  @Override
1980  public long splitRegion(final RegionInfo regionInfo, final byte[] splitRow,
1981      final long nonceGroup, final long nonce)
1982  throws IOException {
1983    checkInitialized();
1984
1985    if (!isSplitOrMergeEnabled(MasterSwitchType.SPLIT)) {
1986      LOG.warn("Split switch is off! skip split of " + regionInfo);
1987      throw new DoNotRetryIOException("Split region " + regionInfo.getRegionNameAsString() +
1988          " failed due to split switch off");
1989    }
1990
1991    return MasterProcedureUtil.submitProcedure(
1992        new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {
1993      @Override
1994      protected void run() throws IOException {
1995        getMaster().getMasterCoprocessorHost().preSplitRegion(regionInfo.getTable(), splitRow);
1996        LOG.info(getClientIdAuditPrefix() + " split " + regionInfo.getRegionNameAsString());
1997
1998        // Execute the operation asynchronously
1999        submitProcedure(getAssignmentManager().createSplitProcedure(regionInfo, splitRow));
2000      }
2001
2002      @Override
2003      protected String getDescription() {
2004        return "SplitTableProcedure";
2005      }
2006    });
2007  }
2008
2009  private void warmUpRegion(ServerName server, RegionInfo region) {
2010    FutureUtils.addListener(asyncClusterConnection.getRegionServerAdmin(server)
2011      .warmupRegion(RequestConverter.buildWarmupRegionRequest(region)), (r, e) -> {
2012        if (e != null) {
2013          LOG.warn("Failed to warm up region {} on server {}", region, server, e);
2014        }
2015      });
2016  }
2017
2018  // Public so can be accessed by tests. Blocks until move is done.
2019  // Replace with an async implementation from which you can get
2020  // a success/failure result.
2021  @VisibleForTesting
2022  public void move(final byte[] encodedRegionName, byte[] destServerName) throws IOException {
2023    RegionState regionState = assignmentManager.getRegionStates().
2024      getRegionState(Bytes.toString(encodedRegionName));
2025
2026    RegionInfo hri;
2027    if (regionState != null) {
2028      hri = regionState.getRegion();
2029    } else {
2030      throw new UnknownRegionException(Bytes.toStringBinary(encodedRegionName));
2031    }
2032
2033    ServerName dest;
2034    List<ServerName> exclude = hri.getTable().isSystemTable() ? assignmentManager.getExcludedServersForSystemTable()
2035        : new ArrayList<>(1);
2036    if (destServerName != null && exclude.contains(ServerName.valueOf(Bytes.toString(destServerName)))) {
2037      LOG.info(
2038          Bytes.toString(encodedRegionName) + " can not move to " + Bytes.toString(destServerName)
2039              + " because the server is in exclude list");
2040      destServerName = null;
2041    }
2042    if (destServerName == null || destServerName.length == 0) {
2043      LOG.info("Passed destination servername is null/empty so " +
2044        "choosing a server at random");
2045      exclude.add(regionState.getServerName());
2046      final List<ServerName> destServers = this.serverManager.createDestinationServersList(exclude);
2047      dest = balancer.randomAssignment(hri, destServers);
2048      if (dest == null) {
2049        LOG.debug("Unable to determine a plan to assign " + hri);
2050        return;
2051      }
2052    } else {
2053      ServerName candidate = ServerName.valueOf(Bytes.toString(destServerName));
2054      dest = balancer.randomAssignment(hri, Lists.newArrayList(candidate));
2055      if (dest == null) {
2056        LOG.debug("Unable to determine a plan to assign " + hri);
2057        return;
2058      }
2059      // TODO: deal with table on master for rs group.
2060      if (dest.equals(serverName)) {
2061        // To avoid unnecessary region moving later by balancer. Don't put user
2062        // regions on master.
2063        LOG.debug("Skipping move of region " + hri.getRegionNameAsString() +
2064          " to avoid unnecessary region moving later by load balancer," +
2065          " because it should not be on master");
2066        return;
2067      }
2068    }
2069
2070    if (dest.equals(regionState.getServerName())) {
2071      LOG.debug("Skipping move of region " + hri.getRegionNameAsString()
2072        + " because region already assigned to the same server " + dest + ".");
2073      return;
2074    }
2075
2076    // Now we can do the move
2077    RegionPlan rp = new RegionPlan(hri, regionState.getServerName(), dest);
2078    assert rp.getDestination() != null: rp.toString() + " " + dest;
2079
2080    try {
2081      checkInitialized();
2082      if (this.cpHost != null) {
2083        this.cpHost.preMove(hri, rp.getSource(), rp.getDestination());
2084      }
2085
2086      TransitRegionStateProcedure proc =
2087        this.assignmentManager.createMoveRegionProcedure(rp.getRegionInfo(), rp.getDestination());
2088      // Warmup the region on the destination before initiating the move.
2089      // A region server could reject the close request because it either does not
2090      // have the specified region or the region is being split.
2091      warmUpRegion(rp.getDestination(), hri);
2092
2093      LOG.info(getClientIdAuditPrefix() + " move " + rp + ", running balancer");
2094      Future<byte[]> future = ProcedureSyncWait.submitProcedure(this.procedureExecutor, proc);
2095      try {
2096        // Is this going to work? Will we throw exception on error?
2097        // TODO: CompletableFuture rather than this stunted Future.
2098        future.get();
2099      } catch (InterruptedException | ExecutionException e) {
2100        throw new HBaseIOException(e);
2101      }
2102      if (this.cpHost != null) {
2103        this.cpHost.postMove(hri, rp.getSource(), rp.getDestination());
2104      }
2105    } catch (IOException ioe) {
2106      if (ioe instanceof HBaseIOException) {
2107        throw (HBaseIOException)ioe;
2108      }
2109      throw new HBaseIOException(ioe);
2110    }
2111  }
2112
2113  @Override
2114  public long createTable(final TableDescriptor tableDescriptor, final byte[][] splitKeys,
2115      final long nonceGroup, final long nonce) throws IOException {
2116    checkInitialized();
2117    TableDescriptor desc = getMasterCoprocessorHost().preCreateTableRegionsInfos(tableDescriptor);
2118    if (desc == null) {
2119      throw new IOException("Creation for " + tableDescriptor + " is canceled by CP");
2120    }
2121    String namespace = desc.getTableName().getNamespaceAsString();
2122    this.clusterSchemaService.getNamespace(namespace);
2123
2124    RegionInfo[] newRegions = ModifyRegionUtils.createRegionInfos(desc, splitKeys);
2125    TableDescriptorChecker.sanityCheck(conf, desc);
2126
2127    return MasterProcedureUtil
2128      .submitProcedure(new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {
2129        @Override
2130        protected void run() throws IOException {
2131          getMaster().getMasterCoprocessorHost().preCreateTable(desc, newRegions);
2132
2133          LOG.info(getClientIdAuditPrefix() + " create " + desc);
2134
2135          // TODO: We can handle/merge duplicate requests, and differentiate the case of
2136          // TableExistsException by saying if the schema is the same or not.
2137          //
2138          // We need to wait for the procedure to potentially fail due to "prepare" sanity
2139          // checks. This will block only the beginning of the procedure. See HBASE-19953.
2140          ProcedurePrepareLatch latch = ProcedurePrepareLatch.createBlockingLatch();
2141          submitProcedure(
2142            new CreateTableProcedure(procedureExecutor.getEnvironment(), desc, newRegions, latch));
2143          latch.await();
2144
2145          getMaster().getMasterCoprocessorHost().postCreateTable(desc, newRegions);
2146        }
2147
2148        @Override
2149        protected String getDescription() {
2150          return "CreateTableProcedure";
2151        }
2152      });
2153  }
2154
2155  @Override
2156  public long createSystemTable(final TableDescriptor tableDescriptor) throws IOException {
2157    if (isStopped()) {
2158      throw new MasterNotRunningException();
2159    }
2160
2161    TableName tableName = tableDescriptor.getTableName();
2162    if (!(tableName.isSystemTable())) {
2163      throw new IllegalArgumentException(
2164        "Only system table creation can use this createSystemTable API");
2165    }
2166
2167    RegionInfo[] newRegions = ModifyRegionUtils.createRegionInfos(tableDescriptor, null);
2168
2169    LOG.info(getClientIdAuditPrefix() + " create " + tableDescriptor);
2170
2171    // This special create table is called locally to master.  Therefore, no RPC means no need
2172    // to use nonce to detect duplicated RPC call.
2173    long procId = this.procedureExecutor.submitProcedure(
2174      new CreateTableProcedure(procedureExecutor.getEnvironment(), tableDescriptor, newRegions));
2175
2176    return procId;
2177  }
2178
2179  private void startActiveMasterManager(int infoPort) throws KeeperException {
2180    String backupZNode = ZNodePaths.joinZNode(
2181      zooKeeper.getZNodePaths().backupMasterAddressesZNode, serverName.toString());
2182    /*
2183    * Add a ZNode for ourselves in the backup master directory since we
2184    * may not become the active master. If so, we want the actual active
2185    * master to know we are backup masters, so that it won't assign
2186    * regions to us if so configured.
2187    *
2188    * If we become the active master later, ActiveMasterManager will delete
2189    * this node explicitly.  If we crash before then, ZooKeeper will delete
2190    * this node for us since it is ephemeral.
2191    */
2192    LOG.info("Adding backup master ZNode " + backupZNode);
2193    if (!MasterAddressTracker.setMasterAddress(zooKeeper, backupZNode, serverName, infoPort)) {
2194      LOG.warn("Failed create of " + backupZNode + " by " + serverName);
2195    }
2196    this.activeMasterManager.setInfoPort(infoPort);
2197    int timeout = conf.getInt(HConstants.ZK_SESSION_TIMEOUT, HConstants.DEFAULT_ZK_SESSION_TIMEOUT);
2198    // If we're a backup master, stall until a primary to write this address
2199    if (conf.getBoolean(HConstants.MASTER_TYPE_BACKUP, HConstants.DEFAULT_MASTER_TYPE_BACKUP)) {
2200      LOG.debug("HMaster started in backup mode. Stalling until master znode is written.");
2201      // This will only be a minute or so while the cluster starts up,
2202      // so don't worry about setting watches on the parent znode
2203      while (!activeMasterManager.hasActiveMaster()) {
2204        LOG.debug("Waiting for master address and cluster state znode to be written.");
2205        Threads.sleep(timeout);
2206      }
2207    }
2208    MonitoredTask status = TaskMonitor.get().createStatus("Master startup");
2209    status.setDescription("Master startup");
2210    try {
2211      if (activeMasterManager.blockUntilBecomingActiveMaster(timeout, status)) {
2212        finishActiveMasterInitialization(status);
2213      }
2214    } catch (Throwable t) {
2215      status.setStatus("Failed to become active: " + t.getMessage());
2216      LOG.error(HBaseMarkers.FATAL, "Failed to become active master", t);
2217      // HBASE-5680: Likely hadoop23 vs hadoop 20.x/1.x incompatibility
2218      if (t instanceof NoClassDefFoundError && t.getMessage().
2219          contains("org/apache/hadoop/hdfs/protocol/HdfsConstants$SafeModeAction")) {
2220        // improved error message for this special case
2221        abort("HBase is having a problem with its Hadoop jars.  You may need to recompile " +
2222          "HBase against Hadoop version " + org.apache.hadoop.util.VersionInfo.getVersion() +
2223          " or change your hadoop jars to start properly", t);
2224      } else {
2225        abort("Unhandled exception. Starting shutdown.", t);
2226      }
2227    } finally {
2228      status.cleanup();
2229    }
2230  }
2231
2232  private static boolean isCatalogTable(final TableName tableName) {
2233    return tableName.equals(TableName.META_TABLE_NAME);
2234  }
2235
2236  @Override
2237  public long deleteTable(
2238      final TableName tableName,
2239      final long nonceGroup,
2240      final long nonce) throws IOException {
2241    checkInitialized();
2242
2243    return MasterProcedureUtil.submitProcedure(
2244        new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {
2245      @Override
2246      protected void run() throws IOException {
2247        getMaster().getMasterCoprocessorHost().preDeleteTable(tableName);
2248
2249        LOG.info(getClientIdAuditPrefix() + " delete " + tableName);
2250
2251        // TODO: We can handle/merge duplicate request
2252        //
2253        // We need to wait for the procedure to potentially fail due to "prepare" sanity
2254        // checks. This will block only the beginning of the procedure. See HBASE-19953.
2255        ProcedurePrepareLatch latch = ProcedurePrepareLatch.createBlockingLatch();
2256        submitProcedure(new DeleteTableProcedure(procedureExecutor.getEnvironment(),
2257            tableName, latch));
2258        latch.await();
2259
2260        getMaster().getMasterCoprocessorHost().postDeleteTable(tableName);
2261      }
2262
2263      @Override
2264      protected String getDescription() {
2265        return "DeleteTableProcedure";
2266      }
2267    });
2268  }
2269
2270  @Override
2271  public long truncateTable(
2272      final TableName tableName,
2273      final boolean preserveSplits,
2274      final long nonceGroup,
2275      final long nonce) throws IOException {
2276    checkInitialized();
2277
2278    return MasterProcedureUtil.submitProcedure(
2279        new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {
2280      @Override
2281      protected void run() throws IOException {
2282        getMaster().getMasterCoprocessorHost().preTruncateTable(tableName);
2283
2284        LOG.info(getClientIdAuditPrefix() + " truncate " + tableName);
2285        ProcedurePrepareLatch latch = ProcedurePrepareLatch.createLatch(2, 0);
2286        submitProcedure(new TruncateTableProcedure(procedureExecutor.getEnvironment(),
2287            tableName, preserveSplits, latch));
2288        latch.await();
2289
2290        getMaster().getMasterCoprocessorHost().postTruncateTable(tableName);
2291      }
2292
2293      @Override
2294      protected String getDescription() {
2295        return "TruncateTableProcedure";
2296      }
2297    });
2298  }
2299
2300  @Override
2301  public long addColumn(final TableName tableName, final ColumnFamilyDescriptor column,
2302      final long nonceGroup, final long nonce) throws IOException {
2303    checkInitialized();
2304    checkTableExists(tableName);
2305
2306    return modifyTable(tableName, new TableDescriptorGetter() {
2307
2308      @Override
2309      public TableDescriptor get() throws IOException {
2310        TableDescriptor old = getTableDescriptors().get(tableName);
2311        if (old.hasColumnFamily(column.getName())) {
2312          throw new InvalidFamilyOperationException("Column family '" + column.getNameAsString()
2313              + "' in table '" + tableName + "' already exists so cannot be added");
2314        }
2315
2316        return TableDescriptorBuilder.newBuilder(old).setColumnFamily(column).build();
2317      }
2318    }, nonceGroup, nonce, true);
2319  }
2320
2321  /**
2322   * Implement to return TableDescriptor after pre-checks
2323   */
2324  protected interface TableDescriptorGetter {
2325    TableDescriptor get() throws IOException;
2326  }
2327
2328  @Override
2329  public long modifyColumn(final TableName tableName, final ColumnFamilyDescriptor descriptor,
2330      final long nonceGroup, final long nonce) throws IOException {
2331    checkInitialized();
2332    checkTableExists(tableName);
2333    return modifyTable(tableName, new TableDescriptorGetter() {
2334
2335      @Override
2336      public TableDescriptor get() throws IOException {
2337        TableDescriptor old = getTableDescriptors().get(tableName);
2338        if (!old.hasColumnFamily(descriptor.getName())) {
2339          throw new InvalidFamilyOperationException("Family '" + descriptor.getNameAsString()
2340              + "' does not exist, so it cannot be modified");
2341        }
2342
2343        return TableDescriptorBuilder.newBuilder(old).modifyColumnFamily(descriptor).build();
2344      }
2345    }, nonceGroup, nonce, true);
2346  }
2347
2348  @Override
2349  public long deleteColumn(final TableName tableName, final byte[] columnName,
2350      final long nonceGroup, final long nonce) throws IOException {
2351    checkInitialized();
2352    checkTableExists(tableName);
2353
2354    return modifyTable(tableName, new TableDescriptorGetter() {
2355
2356      @Override
2357      public TableDescriptor get() throws IOException {
2358        TableDescriptor old = getTableDescriptors().get(tableName);
2359
2360        if (!old.hasColumnFamily(columnName)) {
2361          throw new InvalidFamilyOperationException("Family '" + Bytes.toString(columnName)
2362              + "' does not exist, so it cannot be deleted");
2363        }
2364        if (old.getColumnFamilyCount() == 1) {
2365          throw new InvalidFamilyOperationException("Family '" + Bytes.toString(columnName)
2366              + "' is the only column family in the table, so it cannot be deleted");
2367        }
2368        return TableDescriptorBuilder.newBuilder(old).removeColumnFamily(columnName).build();
2369      }
2370    }, nonceGroup, nonce, true);
2371  }
2372
2373  @Override
2374  public long enableTable(final TableName tableName, final long nonceGroup, final long nonce)
2375      throws IOException {
2376    checkInitialized();
2377
2378    return MasterProcedureUtil.submitProcedure(
2379        new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {
2380      @Override
2381      protected void run() throws IOException {
2382        getMaster().getMasterCoprocessorHost().preEnableTable(tableName);
2383
2384        // Normally, it would make sense for this authorization check to exist inside
2385        // AccessController, but because the authorization check is done based on internal state
2386        // (rather than explicit permissions) we'll do the check here instead of in the
2387        // coprocessor.
2388        MasterQuotaManager quotaManager = getMasterQuotaManager();
2389        if (quotaManager != null) {
2390          if (quotaManager.isQuotaInitialized()) {
2391              SpaceQuotaSnapshot currSnapshotOfTable =
2392                  QuotaTableUtil.getCurrentSnapshotFromQuotaTable(getConnection(), tableName);
2393              if (currSnapshotOfTable != null) {
2394                SpaceQuotaStatus quotaStatus = currSnapshotOfTable.getQuotaStatus();
2395                if (quotaStatus.isInViolation()
2396                    && SpaceViolationPolicy.DISABLE == quotaStatus.getPolicy().orElse(null)) {
2397                throw new AccessDeniedException("Enabling the table '" + tableName
2398                    + "' is disallowed due to a violated space quota.");
2399              }
2400            }
2401          } else if (LOG.isTraceEnabled()) {
2402            LOG.trace("Unable to check for space quotas as the MasterQuotaManager is not enabled");
2403          }
2404        }
2405
2406        LOG.info(getClientIdAuditPrefix() + " enable " + tableName);
2407
2408        // Execute the operation asynchronously - client will check the progress of the operation
2409        // In case the request is from a <1.1 client before returning,
2410        // we want to make sure that the table is prepared to be
2411        // enabled (the table is locked and the table state is set).
2412        // Note: if the procedure throws exception, we will catch it and rethrow.
2413        final ProcedurePrepareLatch prepareLatch = ProcedurePrepareLatch.createLatch();
2414        submitProcedure(new EnableTableProcedure(procedureExecutor.getEnvironment(),
2415            tableName, prepareLatch));
2416        prepareLatch.await();
2417
2418        getMaster().getMasterCoprocessorHost().postEnableTable(tableName);
2419      }
2420
2421      @Override
2422      protected String getDescription() {
2423        return "EnableTableProcedure";
2424      }
2425    });
2426  }
2427
2428  @Override
2429  public long disableTable(final TableName tableName, final long nonceGroup, final long nonce)
2430      throws IOException {
2431    checkInitialized();
2432
2433    return MasterProcedureUtil.submitProcedure(
2434        new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {
2435      @Override
2436      protected void run() throws IOException {
2437        getMaster().getMasterCoprocessorHost().preDisableTable(tableName);
2438
2439        LOG.info(getClientIdAuditPrefix() + " disable " + tableName);
2440
2441        // Execute the operation asynchronously - client will check the progress of the operation
2442        // In case the request is from a <1.1 client before returning,
2443        // we want to make sure that the table is prepared to be
2444        // enabled (the table is locked and the table state is set).
2445        // Note: if the procedure throws exception, we will catch it and rethrow.
2446        //
2447        // We need to wait for the procedure to potentially fail due to "prepare" sanity
2448        // checks. This will block only the beginning of the procedure. See HBASE-19953.
2449        final ProcedurePrepareLatch prepareLatch = ProcedurePrepareLatch.createBlockingLatch();
2450        submitProcedure(new DisableTableProcedure(procedureExecutor.getEnvironment(),
2451            tableName, false, prepareLatch));
2452        prepareLatch.await();
2453
2454        getMaster().getMasterCoprocessorHost().postDisableTable(tableName);
2455      }
2456
2457      @Override
2458      protected String getDescription() {
2459        return "DisableTableProcedure";
2460      }
2461    });
2462  }
2463
2464  private long modifyTable(final TableName tableName,
2465      final TableDescriptorGetter newDescriptorGetter, final long nonceGroup, final long nonce,
2466      final boolean shouldCheckDescriptor) throws IOException {
2467    return MasterProcedureUtil
2468        .submitProcedure(new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {
2469          @Override
2470          protected void run() throws IOException {
2471            TableDescriptor oldDescriptor = getMaster().getTableDescriptors().get(tableName);
2472            TableDescriptor newDescriptor = getMaster().getMasterCoprocessorHost()
2473                .preModifyTable(tableName, oldDescriptor, newDescriptorGetter.get());
2474            TableDescriptorChecker.sanityCheck(conf, newDescriptor);
2475            LOG.info("{} modify table {} from {} to {}", getClientIdAuditPrefix(), tableName,
2476                oldDescriptor, newDescriptor);
2477
2478            // Execute the operation synchronously - wait for the operation completes before
2479            // continuing.
2480            //
2481            // We need to wait for the procedure to potentially fail due to "prepare" sanity
2482            // checks. This will block only the beginning of the procedure. See HBASE-19953.
2483            ProcedurePrepareLatch latch = ProcedurePrepareLatch.createBlockingLatch();
2484            submitProcedure(new ModifyTableProcedure(procedureExecutor.getEnvironment(),
2485                newDescriptor, latch, oldDescriptor, shouldCheckDescriptor));
2486            latch.await();
2487
2488            getMaster().getMasterCoprocessorHost().postModifyTable(tableName, oldDescriptor,
2489              newDescriptor);
2490          }
2491
2492          @Override
2493          protected String getDescription() {
2494            return "ModifyTableProcedure";
2495          }
2496        });
2497
2498  }
2499
2500  @Override
2501  public long modifyTable(final TableName tableName, final TableDescriptor newDescriptor,
2502      final long nonceGroup, final long nonce) throws IOException {
2503    checkInitialized();
2504    return modifyTable(tableName, new TableDescriptorGetter() {
2505      @Override
2506      public TableDescriptor get() throws IOException {
2507        return newDescriptor;
2508      }
2509    }, nonceGroup, nonce, false);
2510
2511  }
2512
2513  public long restoreSnapshot(final SnapshotDescription snapshotDesc,
2514      final long nonceGroup, final long nonce, final boolean restoreAcl) throws IOException {
2515    checkInitialized();
2516    getSnapshotManager().checkSnapshotSupport();
2517
2518    // Ensure namespace exists. Will throw exception if non-known NS.
2519    final TableName dstTable = TableName.valueOf(snapshotDesc.getTable());
2520    getClusterSchema().getNamespace(dstTable.getNamespaceAsString());
2521
2522    return MasterProcedureUtil.submitProcedure(
2523        new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {
2524      @Override
2525      protected void run() throws IOException {
2526          setProcId(
2527            getSnapshotManager().restoreOrCloneSnapshot(snapshotDesc, getNonceKey(), restoreAcl));
2528      }
2529
2530      @Override
2531      protected String getDescription() {
2532        return "RestoreSnapshotProcedure";
2533      }
2534    });
2535  }
2536
2537  private void checkTableExists(final TableName tableName)
2538    throws IOException, TableNotFoundException {
2539    if (!tableDescriptors.exists(tableName)) {
2540      throw new TableNotFoundException(tableName);
2541    }
2542  }
2543
2544  @Override
2545  public void checkTableModifiable(final TableName tableName)
2546      throws IOException, TableNotFoundException, TableNotDisabledException {
2547    if (isCatalogTable(tableName)) {
2548      throw new IOException("Can't modify catalog tables");
2549    }
2550    checkTableExists(tableName);
2551    TableState ts = getTableStateManager().getTableState(tableName);
2552    if (!ts.isDisabled()) {
2553      throw new TableNotDisabledException("Not DISABLED; " + ts);
2554    }
2555  }
2556
2557  public ClusterMetrics getClusterMetricsWithoutCoprocessor() throws InterruptedIOException {
2558    return getClusterMetricsWithoutCoprocessor(EnumSet.allOf(Option.class));
2559  }
2560
2561  public ClusterMetrics getClusterMetricsWithoutCoprocessor(EnumSet<Option> options)
2562      throws InterruptedIOException {
2563    ClusterMetricsBuilder builder = ClusterMetricsBuilder.newBuilder();
2564    // given that hbase1 can't submit the request with Option,
2565    // we return all information to client if the list of Option is empty.
2566    if (options.isEmpty()) {
2567      options = EnumSet.allOf(Option.class);
2568    }
2569
2570    for (Option opt : options) {
2571      switch (opt) {
2572        case HBASE_VERSION: builder.setHBaseVersion(VersionInfo.getVersion()); break;
2573        case CLUSTER_ID: builder.setClusterId(getClusterId()); break;
2574        case MASTER: builder.setMasterName(getServerName()); break;
2575        case BACKUP_MASTERS: builder.setBackerMasterNames(getBackupMasters()); break;
2576        case LIVE_SERVERS: {
2577          if (serverManager != null) {
2578            builder.setLiveServerMetrics(serverManager.getOnlineServers().entrySet().stream()
2579              .collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue())));
2580          }
2581          break;
2582        }
2583        case DEAD_SERVERS: {
2584          if (serverManager != null) {
2585            builder.setDeadServerNames(new ArrayList<>(
2586              serverManager.getDeadServers().copyServerNames()));
2587          }
2588          break;
2589        }
2590        case MASTER_COPROCESSORS: {
2591          if (cpHost != null) {
2592            builder.setMasterCoprocessorNames(Arrays.asList(getMasterCoprocessors()));
2593          }
2594          break;
2595        }
2596        case REGIONS_IN_TRANSITION: {
2597          if (assignmentManager != null) {
2598            builder.setRegionsInTransition(assignmentManager.getRegionStates()
2599                .getRegionsStateInTransition());
2600          }
2601          break;
2602        }
2603        case BALANCER_ON: {
2604          if (loadBalancerTracker != null) {
2605            builder.setBalancerOn(loadBalancerTracker.isBalancerOn());
2606          }
2607          break;
2608        }
2609        case MASTER_INFO_PORT: {
2610          if (infoServer != null) {
2611            builder.setMasterInfoPort(infoServer.getPort());
2612          }
2613          break;
2614        }
2615        case SERVERS_NAME: {
2616          if (serverManager != null) {
2617            builder.setServerNames(serverManager.getOnlineServersList());
2618          }
2619          break;
2620        }
2621        case TABLE_TO_REGIONS_COUNT: {
2622          if (isActiveMaster() && isInitialized() && assignmentManager != null) {
2623            try {
2624              Map<TableName, RegionStatesCount> tableRegionStatesCountMap = new HashMap<>();
2625              Map<String, TableDescriptor> tableDescriptorMap = getTableDescriptors().getAll();
2626              for (TableDescriptor tableDescriptor : tableDescriptorMap.values()) {
2627                TableName tableName = tableDescriptor.getTableName();
2628                RegionStatesCount regionStatesCount = assignmentManager
2629                  .getRegionStatesCount(tableName);
2630                tableRegionStatesCountMap.put(tableName, regionStatesCount);
2631              }
2632              builder.setTableRegionStatesCount(tableRegionStatesCountMap);
2633            } catch (IOException e) {
2634              LOG.error("Error while populating TABLE_TO_REGIONS_COUNT for Cluster Metrics..", e);
2635            }
2636          }
2637          break;
2638        }
2639      }
2640    }
2641    return builder.build();
2642  }
2643
2644  /**
2645   * @return cluster status
2646   */
2647  public ClusterMetrics getClusterMetrics() throws IOException {
2648    return getClusterMetrics(EnumSet.allOf(Option.class));
2649  }
2650
2651  public ClusterMetrics getClusterMetrics(EnumSet<Option> options) throws IOException {
2652    if (cpHost != null) {
2653      cpHost.preGetClusterMetrics();
2654    }
2655    ClusterMetrics status = getClusterMetricsWithoutCoprocessor(options);
2656    if (cpHost != null) {
2657      cpHost.postGetClusterMetrics(status);
2658    }
2659    return status;
2660  }
2661
2662  List<ServerName> getBackupMasters() {
2663    return activeMasterManager.getBackupMasters();
2664  }
2665
2666  /**
2667   * The set of loaded coprocessors is stored in a static set. Since it's
2668   * statically allocated, it does not require that HMaster's cpHost be
2669   * initialized prior to accessing it.
2670   * @return a String representation of the set of names of the loaded coprocessors.
2671   */
2672  public static String getLoadedCoprocessors() {
2673    return CoprocessorHost.getLoadedCoprocessors().toString();
2674  }
2675
2676  /**
2677   * @return timestamp in millis when HMaster was started.
2678   */
2679  public long getMasterStartTime() {
2680    return startcode;
2681  }
2682
2683  /**
2684   * @return timestamp in millis when HMaster became the active master.
2685   */
2686  public long getMasterActiveTime() {
2687    return masterActiveTime;
2688  }
2689
2690  /**
2691   * @return timestamp in millis when HMaster finished becoming the active master
2692   */
2693  public long getMasterFinishedInitializationTime() {
2694    return masterFinishedInitializationTime;
2695  }
2696
2697  public int getNumWALFiles() {
2698    return 0;
2699  }
2700
2701  public ProcedureStore getProcedureStore() {
2702    return procedureStore;
2703  }
2704
2705  public int getRegionServerInfoPort(final ServerName sn) {
2706    int port = this.serverManager.getInfoPort(sn);
2707    return port == 0 ? conf.getInt(HConstants.REGIONSERVER_INFO_PORT,
2708      HConstants.DEFAULT_REGIONSERVER_INFOPORT) : port;
2709  }
2710
2711  @Override
2712  public String getRegionServerVersion(ServerName sn) {
2713    // Will return "0.0.0" if the server is not online to prevent move system region to unknown
2714    // version RS.
2715    return this.serverManager.getVersion(sn);
2716  }
2717
2718  @Override
2719  public void checkIfShouldMoveSystemRegionAsync() {
2720    assignmentManager.checkIfShouldMoveSystemRegionAsync();
2721  }
2722
2723  /**
2724   * @return array of coprocessor SimpleNames.
2725   */
2726  public String[] getMasterCoprocessors() {
2727    Set<String> masterCoprocessors = getMasterCoprocessorHost().getCoprocessors();
2728    return masterCoprocessors.toArray(new String[masterCoprocessors.size()]);
2729  }
2730
2731  @Override
2732  public void abort(String reason, Throwable cause) {
2733    if (!setAbortRequested() || isStopped()) {
2734      LOG.debug("Abort called but aborted={}, stopped={}", isAborted(), isStopped());
2735      return;
2736    }
2737    if (cpHost != null) {
2738      // HBASE-4014: dump a list of loaded coprocessors.
2739      LOG.error(HBaseMarkers.FATAL, "Master server abort: loaded coprocessors are: " +
2740          getLoadedCoprocessors());
2741    }
2742    String msg = "***** ABORTING master " + this + ": " + reason + " *****";
2743    if (cause != null) {
2744      LOG.error(HBaseMarkers.FATAL, msg, cause);
2745    } else {
2746      LOG.error(HBaseMarkers.FATAL, msg);
2747    }
2748
2749    try {
2750      stopMaster();
2751    } catch (IOException e) {
2752      LOG.error("Exception occurred while stopping master", e);
2753    }
2754  }
2755
2756  @Override
2757  public ZKWatcher getZooKeeper() {
2758    return zooKeeper;
2759  }
2760
2761  @Override
2762  public MasterCoprocessorHost getMasterCoprocessorHost() {
2763    return cpHost;
2764  }
2765
2766  @Override
2767  public MasterQuotaManager getMasterQuotaManager() {
2768    return quotaManager;
2769  }
2770
2771  @Override
2772  public ProcedureExecutor<MasterProcedureEnv> getMasterProcedureExecutor() {
2773    return procedureExecutor;
2774  }
2775
2776  @Override
2777  public ServerName getServerName() {
2778    return this.serverName;
2779  }
2780
2781  @Override
2782  public AssignmentManager getAssignmentManager() {
2783    return this.assignmentManager;
2784  }
2785
2786  @Override
2787  public CatalogJanitor getCatalogJanitor() {
2788    return this.catalogJanitorChore;
2789  }
2790
2791  public MemoryBoundedLogMessageBuffer getRegionServerFatalLogBuffer() {
2792    return rsFatals;
2793  }
2794
2795  /**
2796   * Shutdown the cluster.
2797   * Master runs a coordinated stop of all RegionServers and then itself.
2798   */
2799  public void shutdown() throws IOException {
2800    if (cpHost != null) {
2801      cpHost.preShutdown();
2802    }
2803
2804    // Tell the servermanager cluster shutdown has been called. This makes it so when Master is
2805    // last running server, it'll stop itself. Next, we broadcast the cluster shutdown by setting
2806    // the cluster status as down. RegionServers will notice this change in state and will start
2807    // shutting themselves down. When last has exited, Master can go down.
2808    if (this.serverManager != null) {
2809      this.serverManager.shutdownCluster();
2810    }
2811    if (this.clusterStatusTracker != null) {
2812      try {
2813        this.clusterStatusTracker.setClusterDown();
2814      } catch (KeeperException e) {
2815        LOG.error("ZooKeeper exception trying to set cluster as down in ZK", e);
2816      }
2817    }
2818    // Stop the procedure executor. Will stop any ongoing assign, unassign, server crash etc.,
2819    // processing so we can go down.
2820    if (this.procedureExecutor != null) {
2821      this.procedureExecutor.stop();
2822    }
2823    // Shutdown our cluster connection. This will kill any hosted RPCs that might be going on;
2824    // this is what we want especially if the Master is in startup phase doing call outs to
2825    // hbase:meta, etc. when cluster is down. Without ths connection close, we'd have to wait on
2826    // the rpc to timeout.
2827    if (this.asyncClusterConnection != null) {
2828      this.asyncClusterConnection.close();
2829    }
2830  }
2831
2832  public void stopMaster() throws IOException {
2833    if (cpHost != null) {
2834      cpHost.preStopMaster();
2835    }
2836    stop("Stopped by " + Thread.currentThread().getName());
2837  }
2838
2839  @Override
2840  public void stop(String msg) {
2841    if (!isStopped()) {
2842      super.stop(msg);
2843      if (this.activeMasterManager != null) {
2844        this.activeMasterManager.stop();
2845      }
2846    }
2847  }
2848
2849  @VisibleForTesting
2850  protected void checkServiceStarted() throws ServerNotRunningYetException {
2851    if (!serviceStarted) {
2852      throw new ServerNotRunningYetException("Server is not running yet");
2853    }
2854  }
2855
2856  public static class MasterStoppedException extends DoNotRetryIOException {
2857    MasterStoppedException() {
2858      super();
2859    }
2860  }
2861
2862  void checkInitialized() throws PleaseHoldException, ServerNotRunningYetException,
2863      MasterNotRunningException, MasterStoppedException {
2864    checkServiceStarted();
2865    if (!isInitialized()) {
2866      throw new PleaseHoldException("Master is initializing");
2867    }
2868    if (isStopped()) {
2869      throw new MasterStoppedException();
2870    }
2871  }
2872
2873  /**
2874   * Report whether this master is currently the active master or not.
2875   * If not active master, we are parked on ZK waiting to become active.
2876   *
2877   * This method is used for testing.
2878   *
2879   * @return true if active master, false if not.
2880   */
2881  @Override
2882  public boolean isActiveMaster() {
2883    return activeMaster;
2884  }
2885
2886  /**
2887   * Report whether this master has completed with its initialization and is
2888   * ready.  If ready, the master is also the active master.  A standby master
2889   * is never ready.
2890   *
2891   * This method is used for testing.
2892   *
2893   * @return true if master is ready to go, false if not.
2894   */
2895  @Override
2896  public boolean isInitialized() {
2897    return initialized.isReady();
2898  }
2899
2900  /**
2901   * Report whether this master is in maintenance mode.
2902   *
2903   * @return true if master is in maintenanceMode
2904   */
2905  @Override
2906  public boolean isInMaintenanceMode() {
2907    return maintenanceMode;
2908  }
2909
2910  @VisibleForTesting
2911  public void setInitialized(boolean isInitialized) {
2912    procedureExecutor.getEnvironment().setEventReady(initialized, isInitialized);
2913  }
2914
2915  @Override
2916  public ProcedureEvent<?> getInitializedEvent() {
2917    return initialized;
2918  }
2919
2920  /**
2921   * Compute the average load across all region servers.
2922   * Currently, this uses a very naive computation - just uses the number of
2923   * regions being served, ignoring stats about number of requests.
2924   * @return the average load
2925   */
2926  public double getAverageLoad() {
2927    if (this.assignmentManager == null) {
2928      return 0;
2929    }
2930
2931    RegionStates regionStates = this.assignmentManager.getRegionStates();
2932    if (regionStates == null) {
2933      return 0;
2934    }
2935    return regionStates.getAverageLoad();
2936  }
2937
2938  @Override
2939  public boolean registerService(Service instance) {
2940    /*
2941     * No stacking of instances is allowed for a single service name
2942     */
2943    Descriptors.ServiceDescriptor serviceDesc = instance.getDescriptorForType();
2944    String serviceName = CoprocessorRpcUtils.getServiceName(serviceDesc);
2945    if (coprocessorServiceHandlers.containsKey(serviceName)) {
2946      LOG.error("Coprocessor service "+serviceName+
2947          " already registered, rejecting request from "+instance
2948      );
2949      return false;
2950    }
2951
2952    coprocessorServiceHandlers.put(serviceName, instance);
2953    if (LOG.isDebugEnabled()) {
2954      LOG.debug("Registered master coprocessor service: service="+serviceName);
2955    }
2956    return true;
2957  }
2958
2959  /**
2960   * Utility for constructing an instance of the passed HMaster class.
2961   * @param masterClass
2962   * @return HMaster instance.
2963   */
2964  public static HMaster constructMaster(Class<? extends HMaster> masterClass,
2965      final Configuration conf)  {
2966    try {
2967      Constructor<? extends HMaster> c = masterClass.getConstructor(Configuration.class);
2968      return c.newInstance(conf);
2969    } catch(Exception e) {
2970      Throwable error = e;
2971      if (e instanceof InvocationTargetException &&
2972          ((InvocationTargetException)e).getTargetException() != null) {
2973        error = ((InvocationTargetException)e).getTargetException();
2974      }
2975      throw new RuntimeException("Failed construction of Master: " + masterClass.toString() + ". "
2976        , error);
2977    }
2978  }
2979
2980  /**
2981   * @see org.apache.hadoop.hbase.master.HMasterCommandLine
2982   */
2983  public static void main(String [] args) {
2984    LOG.info("STARTING service " + HMaster.class.getSimpleName());
2985    VersionInfo.logVersion();
2986    new HMasterCommandLine(HMaster.class).doMain(args);
2987  }
2988
2989  public HFileCleaner getHFileCleaner() {
2990    return this.hfileCleaner;
2991  }
2992
2993  public LogCleaner getLogCleaner() {
2994    return this.logCleaner;
2995  }
2996
2997  /**
2998   * @return the underlying snapshot manager
2999   */
3000  @Override
3001  public SnapshotManager getSnapshotManager() {
3002    return this.snapshotManager;
3003  }
3004
3005  /**
3006   * @return the underlying MasterProcedureManagerHost
3007   */
3008  @Override
3009  public MasterProcedureManagerHost getMasterProcedureManagerHost() {
3010    return mpmHost;
3011  }
3012
3013  @Override
3014  public ClusterSchema getClusterSchema() {
3015    return this.clusterSchemaService;
3016  }
3017
3018  /**
3019   * Create a new Namespace.
3020   * @param namespaceDescriptor descriptor for new Namespace
3021   * @param nonceGroup Identifier for the source of the request, a client or process.
3022   * @param nonce A unique identifier for this operation from the client or process identified by
3023   * <code>nonceGroup</code> (the source must ensure each operation gets a unique id).
3024   * @return procedure id
3025   */
3026  long createNamespace(final NamespaceDescriptor namespaceDescriptor, final long nonceGroup,
3027      final long nonce) throws IOException {
3028    checkInitialized();
3029
3030    TableName.isLegalNamespaceName(Bytes.toBytes(namespaceDescriptor.getName()));
3031
3032    return MasterProcedureUtil.submitProcedure(new MasterProcedureUtil.NonceProcedureRunnable(this,
3033          nonceGroup, nonce) {
3034      @Override
3035      protected void run() throws IOException {
3036        getMaster().getMasterCoprocessorHost().preCreateNamespace(namespaceDescriptor);
3037        // We need to wait for the procedure to potentially fail due to "prepare" sanity
3038        // checks. This will block only the beginning of the procedure. See HBASE-19953.
3039        ProcedurePrepareLatch latch = ProcedurePrepareLatch.createBlockingLatch();
3040        LOG.info(getClientIdAuditPrefix() + " creating " + namespaceDescriptor);
3041        // Execute the operation synchronously - wait for the operation to complete before
3042        // continuing.
3043        setProcId(getClusterSchema().createNamespace(namespaceDescriptor, getNonceKey(), latch));
3044        latch.await();
3045        getMaster().getMasterCoprocessorHost().postCreateNamespace(namespaceDescriptor);
3046      }
3047
3048      @Override
3049      protected String getDescription() {
3050        return "CreateNamespaceProcedure";
3051      }
3052    });
3053  }
3054
3055  /**
3056   * Modify an existing Namespace.
3057   * @param nonceGroup Identifier for the source of the request, a client or process.
3058   * @param nonce A unique identifier for this operation from the client or process identified by
3059   * <code>nonceGroup</code> (the source must ensure each operation gets a unique id).
3060   * @return procedure id
3061   */
3062  long modifyNamespace(final NamespaceDescriptor newNsDescriptor, final long nonceGroup,
3063      final long nonce) throws IOException {
3064    checkInitialized();
3065
3066    TableName.isLegalNamespaceName(Bytes.toBytes(newNsDescriptor.getName()));
3067
3068    return MasterProcedureUtil.submitProcedure(new MasterProcedureUtil.NonceProcedureRunnable(this,
3069          nonceGroup, nonce) {
3070      @Override
3071      protected void run() throws IOException {
3072        NamespaceDescriptor oldNsDescriptor = getNamespace(newNsDescriptor.getName());
3073        getMaster().getMasterCoprocessorHost().preModifyNamespace(oldNsDescriptor, newNsDescriptor);
3074        // We need to wait for the procedure to potentially fail due to "prepare" sanity
3075        // checks. This will block only the beginning of the procedure. See HBASE-19953.
3076        ProcedurePrepareLatch latch = ProcedurePrepareLatch.createBlockingLatch();
3077        LOG.info(getClientIdAuditPrefix() + " modify " + newNsDescriptor);
3078        // Execute the operation synchronously - wait for the operation to complete before
3079        // continuing.
3080        setProcId(getClusterSchema().modifyNamespace(newNsDescriptor, getNonceKey(), latch));
3081        latch.await();
3082        getMaster().getMasterCoprocessorHost().postModifyNamespace(oldNsDescriptor,
3083          newNsDescriptor);
3084      }
3085
3086      @Override
3087      protected String getDescription() {
3088        return "ModifyNamespaceProcedure";
3089      }
3090    });
3091  }
3092
3093  /**
3094   * Delete an existing Namespace. Only empty Namespaces (no tables) can be removed.
3095   * @param nonceGroup Identifier for the source of the request, a client or process.
3096   * @param nonce A unique identifier for this operation from the client or process identified by
3097   * <code>nonceGroup</code> (the source must ensure each operation gets a unique id).
3098   * @return procedure id
3099   */
3100  long deleteNamespace(final String name, final long nonceGroup, final long nonce)
3101      throws IOException {
3102    checkInitialized();
3103
3104    return MasterProcedureUtil.submitProcedure(new MasterProcedureUtil.NonceProcedureRunnable(this,
3105          nonceGroup, nonce) {
3106      @Override
3107      protected void run() throws IOException {
3108        getMaster().getMasterCoprocessorHost().preDeleteNamespace(name);
3109        LOG.info(getClientIdAuditPrefix() + " delete " + name);
3110        // Execute the operation synchronously - wait for the operation to complete before
3111        // continuing.
3112        //
3113        // We need to wait for the procedure to potentially fail due to "prepare" sanity
3114        // checks. This will block only the beginning of the procedure. See HBASE-19953.
3115        ProcedurePrepareLatch latch = ProcedurePrepareLatch.createBlockingLatch();
3116        setProcId(submitProcedure(
3117              new DeleteNamespaceProcedure(procedureExecutor.getEnvironment(), name, latch)));
3118        latch.await();
3119        // Will not be invoked in the face of Exception thrown by the Procedure's execution
3120        getMaster().getMasterCoprocessorHost().postDeleteNamespace(name);
3121      }
3122
3123      @Override
3124      protected String getDescription() {
3125        return "DeleteNamespaceProcedure";
3126      }
3127    });
3128  }
3129
3130  /**
3131   * Get a Namespace
3132   * @param name Name of the Namespace
3133   * @return Namespace descriptor for <code>name</code>
3134   */
3135  NamespaceDescriptor getNamespace(String name) throws IOException {
3136    checkInitialized();
3137    if (this.cpHost != null) this.cpHost.preGetNamespaceDescriptor(name);
3138    NamespaceDescriptor nsd = this.clusterSchemaService.getNamespace(name);
3139    if (this.cpHost != null) this.cpHost.postGetNamespaceDescriptor(nsd);
3140    return nsd;
3141  }
3142
3143  /**
3144   * Get all Namespaces
3145   * @return All Namespace descriptors
3146   */
3147  List<NamespaceDescriptor> getNamespaces() throws IOException {
3148    checkInitialized();
3149    final List<NamespaceDescriptor> nsds = new ArrayList<>();
3150    if (cpHost != null) {
3151      cpHost.preListNamespaceDescriptors(nsds);
3152    }
3153    nsds.addAll(this.clusterSchemaService.getNamespaces());
3154    if (this.cpHost != null) {
3155      this.cpHost.postListNamespaceDescriptors(nsds);
3156    }
3157    return nsds;
3158  }
3159
3160  /**
3161   * List namespace names
3162   * @return All namespace names
3163   */
3164  public List<String> listNamespaces() throws IOException {
3165    checkInitialized();
3166    List<String> namespaces = new ArrayList<>();
3167    if (cpHost != null) {
3168      cpHost.preListNamespaces(namespaces);
3169    }
3170    for (NamespaceDescriptor namespace : clusterSchemaService.getNamespaces()) {
3171      namespaces.add(namespace.getName());
3172    }
3173    if (cpHost != null) {
3174      cpHost.postListNamespaces(namespaces);
3175    }
3176    return namespaces;
3177  }
3178
3179  @Override
3180  public List<TableName> listTableNamesByNamespace(String name) throws IOException {
3181    checkInitialized();
3182    return listTableNames(name, null, true);
3183  }
3184
3185  @Override
3186  public List<TableDescriptor> listTableDescriptorsByNamespace(String name) throws IOException {
3187    checkInitialized();
3188    return listTableDescriptors(name, null, null, true);
3189  }
3190
3191  @Override
3192  public boolean abortProcedure(final long procId, final boolean mayInterruptIfRunning)
3193      throws IOException {
3194    if (cpHost != null) {
3195      cpHost.preAbortProcedure(this.procedureExecutor, procId);
3196    }
3197
3198    final boolean result = this.procedureExecutor.abort(procId, mayInterruptIfRunning);
3199
3200    if (cpHost != null) {
3201      cpHost.postAbortProcedure();
3202    }
3203
3204    return result;
3205  }
3206
3207  @Override
3208  public List<Procedure<?>> getProcedures() throws IOException {
3209    if (cpHost != null) {
3210      cpHost.preGetProcedures();
3211    }
3212
3213    @SuppressWarnings({ "unchecked", "rawtypes" })
3214    List<Procedure<?>> procList = (List) this.procedureExecutor.getProcedures();
3215
3216    if (cpHost != null) {
3217      cpHost.postGetProcedures(procList);
3218    }
3219
3220    return procList;
3221  }
3222
3223  @Override
3224  public List<LockedResource> getLocks() throws IOException {
3225    if (cpHost != null) {
3226      cpHost.preGetLocks();
3227    }
3228
3229    MasterProcedureScheduler procedureScheduler =
3230      procedureExecutor.getEnvironment().getProcedureScheduler();
3231
3232    final List<LockedResource> lockedResources = procedureScheduler.getLocks();
3233
3234    if (cpHost != null) {
3235      cpHost.postGetLocks(lockedResources);
3236    }
3237
3238    return lockedResources;
3239  }
3240
3241  /**
3242   * Returns the list of table descriptors that match the specified request
3243   * @param namespace the namespace to query, or null if querying for all
3244   * @param regex The regular expression to match against, or null if querying for all
3245   * @param tableNameList the list of table names, or null if querying for all
3246   * @param includeSysTables False to match only against userspace tables
3247   * @return the list of table descriptors
3248   */
3249  public List<TableDescriptor> listTableDescriptors(final String namespace, final String regex,
3250      final List<TableName> tableNameList, final boolean includeSysTables)
3251  throws IOException {
3252    List<TableDescriptor> htds = new ArrayList<>();
3253    if (cpHost != null) {
3254      cpHost.preGetTableDescriptors(tableNameList, htds, regex);
3255    }
3256    htds = getTableDescriptors(htds, namespace, regex, tableNameList, includeSysTables);
3257    if (cpHost != null) {
3258      cpHost.postGetTableDescriptors(tableNameList, htds, regex);
3259    }
3260    return htds;
3261  }
3262
3263  /**
3264   * Returns the list of table names that match the specified request
3265   * @param regex The regular expression to match against, or null if querying for all
3266   * @param namespace the namespace to query, or null if querying for all
3267   * @param includeSysTables False to match only against userspace tables
3268   * @return the list of table names
3269   */
3270  public List<TableName> listTableNames(final String namespace, final String regex,
3271      final boolean includeSysTables) throws IOException {
3272    List<TableDescriptor> htds = new ArrayList<>();
3273    if (cpHost != null) {
3274      cpHost.preGetTableNames(htds, regex);
3275    }
3276    htds = getTableDescriptors(htds, namespace, regex, null, includeSysTables);
3277    if (cpHost != null) {
3278      cpHost.postGetTableNames(htds, regex);
3279    }
3280    List<TableName> result = new ArrayList<>(htds.size());
3281    for (TableDescriptor htd: htds) result.add(htd.getTableName());
3282    return result;
3283  }
3284
3285  /**
3286   * Return a list of table table descriptors after applying any provided filter parameters. Note
3287   * that the user-facing description of this filter logic is presented on the class-level javadoc
3288   * of {@link NormalizeTableFilterParams}.
3289   */
3290  private List<TableDescriptor> getTableDescriptors(final List<TableDescriptor> htds,
3291      final String namespace, final String regex, final List<TableName> tableNameList,
3292      final boolean includeSysTables)
3293  throws IOException {
3294    if (tableNameList == null || tableNameList.isEmpty()) {
3295      // request for all TableDescriptors
3296      Collection<TableDescriptor> allHtds;
3297      if (namespace != null && namespace.length() > 0) {
3298        // Do a check on the namespace existence. Will fail if does not exist.
3299        this.clusterSchemaService.getNamespace(namespace);
3300        allHtds = tableDescriptors.getByNamespace(namespace).values();
3301      } else {
3302        allHtds = tableDescriptors.getAll().values();
3303      }
3304      for (TableDescriptor desc: allHtds) {
3305        if (tableStateManager.isTablePresent(desc.getTableName())
3306            && (includeSysTables || !desc.getTableName().isSystemTable())) {
3307          htds.add(desc);
3308        }
3309      }
3310    } else {
3311      for (TableName s: tableNameList) {
3312        if (tableStateManager.isTablePresent(s)) {
3313          TableDescriptor desc = tableDescriptors.get(s);
3314          if (desc != null) {
3315            htds.add(desc);
3316          }
3317        }
3318      }
3319    }
3320
3321    // Retains only those matched by regular expression.
3322    if (regex != null) filterTablesByRegex(htds, Pattern.compile(regex));
3323    return htds;
3324  }
3325
3326  /**
3327   * Removes the table descriptors that don't match the pattern.
3328   * @param descriptors list of table descriptors to filter
3329   * @param pattern the regex to use
3330   */
3331  private static void filterTablesByRegex(final Collection<TableDescriptor> descriptors,
3332      final Pattern pattern) {
3333    final String defaultNS = NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR;
3334    Iterator<TableDescriptor> itr = descriptors.iterator();
3335    while (itr.hasNext()) {
3336      TableDescriptor htd = itr.next();
3337      String tableName = htd.getTableName().getNameAsString();
3338      boolean matched = pattern.matcher(tableName).matches();
3339      if (!matched && htd.getTableName().getNamespaceAsString().equals(defaultNS)) {
3340        matched = pattern.matcher(defaultNS + TableName.NAMESPACE_DELIM + tableName).matches();
3341      }
3342      if (!matched) {
3343        itr.remove();
3344      }
3345    }
3346  }
3347
3348  @Override
3349  public long getLastMajorCompactionTimestamp(TableName table) throws IOException {
3350    return getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS))
3351        .getLastMajorCompactionTimestamp(table);
3352  }
3353
3354  @Override
3355  public long getLastMajorCompactionTimestampForRegion(byte[] regionName) throws IOException {
3356    return getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS))
3357        .getLastMajorCompactionTimestamp(regionName);
3358  }
3359
3360  /**
3361   * Gets the mob file compaction state for a specific table.
3362   * Whether all the mob files are selected is known during the compaction execution, but
3363   * the statistic is done just before compaction starts, it is hard to know the compaction
3364   * type at that time, so the rough statistics are chosen for the mob file compaction. Only two
3365   * compaction states are available, CompactionState.MAJOR_AND_MINOR and CompactionState.NONE.
3366   * @param tableName The current table name.
3367   * @return If a given table is in mob file compaction now.
3368   */
3369  public GetRegionInfoResponse.CompactionState getMobCompactionState(TableName tableName) {
3370    AtomicInteger compactionsCount = mobCompactionStates.get(tableName);
3371    if (compactionsCount != null && compactionsCount.get() != 0) {
3372      return GetRegionInfoResponse.CompactionState.MAJOR_AND_MINOR;
3373    }
3374    return GetRegionInfoResponse.CompactionState.NONE;
3375  }
3376
3377  public void reportMobCompactionStart(TableName tableName) throws IOException {
3378    IdLock.Entry lockEntry = null;
3379    try {
3380      lockEntry = mobCompactionLock.getLockEntry(tableName.hashCode());
3381      AtomicInteger compactionsCount = mobCompactionStates.get(tableName);
3382      if (compactionsCount == null) {
3383        compactionsCount = new AtomicInteger(0);
3384        mobCompactionStates.put(tableName, compactionsCount);
3385      }
3386      compactionsCount.incrementAndGet();
3387    } finally {
3388      if (lockEntry != null) {
3389        mobCompactionLock.releaseLockEntry(lockEntry);
3390      }
3391    }
3392  }
3393
3394  public void reportMobCompactionEnd(TableName tableName) throws IOException {
3395    IdLock.Entry lockEntry = null;
3396    try {
3397      lockEntry = mobCompactionLock.getLockEntry(tableName.hashCode());
3398      AtomicInteger compactionsCount = mobCompactionStates.get(tableName);
3399      if (compactionsCount != null) {
3400        int count = compactionsCount.decrementAndGet();
3401        // remove the entry if the count is 0.
3402        if (count == 0) {
3403          mobCompactionStates.remove(tableName);
3404        }
3405      }
3406    } finally {
3407      if (lockEntry != null) {
3408        mobCompactionLock.releaseLockEntry(lockEntry);
3409      }
3410    }
3411  }
3412
3413
3414  /**
3415   * Queries the state of the {@link LoadBalancerTracker}. If the balancer is not initialized,
3416   * false is returned.
3417   *
3418   * @return The state of the load balancer, or false if the load balancer isn't defined.
3419   */
3420  public boolean isBalancerOn() {
3421    return !isInMaintenanceMode()
3422        && loadBalancerTracker != null
3423        && loadBalancerTracker.isBalancerOn();
3424  }
3425
3426  /**
3427   * Queries the state of the {@link RegionNormalizerTracker}. If it's not initialized,
3428   * false is returned.
3429   */
3430  public boolean isNormalizerOn() {
3431    return !isInMaintenanceMode()
3432      && getRegionNormalizerManager().isNormalizerOn();
3433  }
3434
3435  /**
3436   * Queries the state of the {@link SplitOrMergeTracker}. If it is not initialized,
3437   * false is returned. If switchType is illegal, false will return.
3438   * @param switchType see {@link org.apache.hadoop.hbase.client.MasterSwitchType}
3439   * @return The state of the switch
3440   */
3441  @Override
3442  public boolean isSplitOrMergeEnabled(MasterSwitchType switchType) {
3443    return !isInMaintenanceMode()
3444        && splitOrMergeTracker != null
3445        && splitOrMergeTracker.isSplitOrMergeEnabled(switchType);
3446  }
3447
3448  /**
3449   * Fetch the configured {@link LoadBalancer} class name. If none is set, a default is returned.
3450   * <p/>
3451   * Notice that, the base load balancer will always be {@link RSGroupBasedLoadBalancer} now, so
3452   * this method will return the balancer used inside each rs group.
3453   * @return The name of the {@link LoadBalancer} in use.
3454   */
3455  public String getLoadBalancerClassName() {
3456    return conf.get(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
3457      LoadBalancerFactory.getDefaultLoadBalancerClass().getName());
3458  }
3459
3460  public SplitOrMergeTracker getSplitOrMergeTracker() {
3461    return splitOrMergeTracker;
3462  }
3463
3464  @Override
3465  public RSGroupBasedLoadBalancer getLoadBalancer() {
3466    return balancer;
3467  }
3468
3469  @Override
3470  public FavoredNodesManager getFavoredNodesManager() {
3471    return balancer.getFavoredNodesManager();
3472  }
3473
3474  private long executePeerProcedure(AbstractPeerProcedure<?> procedure) throws IOException {
3475    long procId = procedureExecutor.submitProcedure(procedure);
3476    procedure.getLatch().await();
3477    return procId;
3478  }
3479
3480  @Override
3481  public long addReplicationPeer(String peerId, ReplicationPeerConfig peerConfig, boolean enabled)
3482      throws ReplicationException, IOException {
3483    LOG.info(getClientIdAuditPrefix() + " creating replication peer, id=" + peerId + ", config=" +
3484      peerConfig + ", state=" + (enabled ? "ENABLED" : "DISABLED"));
3485    return executePeerProcedure(new AddPeerProcedure(peerId, peerConfig, enabled));
3486  }
3487
3488  @Override
3489  public long removeReplicationPeer(String peerId) throws ReplicationException, IOException {
3490    LOG.info(getClientIdAuditPrefix() + " removing replication peer, id=" + peerId);
3491    return executePeerProcedure(new RemovePeerProcedure(peerId));
3492  }
3493
3494  @Override
3495  public long enableReplicationPeer(String peerId) throws ReplicationException, IOException {
3496    LOG.info(getClientIdAuditPrefix() + " enable replication peer, id=" + peerId);
3497    return executePeerProcedure(new EnablePeerProcedure(peerId));
3498  }
3499
3500  @Override
3501  public long disableReplicationPeer(String peerId) throws ReplicationException, IOException {
3502    LOG.info(getClientIdAuditPrefix() + " disable replication peer, id=" + peerId);
3503    return executePeerProcedure(new DisablePeerProcedure(peerId));
3504  }
3505
3506  @Override
3507  public ReplicationPeerConfig getReplicationPeerConfig(String peerId)
3508      throws ReplicationException, IOException {
3509    if (cpHost != null) {
3510      cpHost.preGetReplicationPeerConfig(peerId);
3511    }
3512    LOG.info(getClientIdAuditPrefix() + " get replication peer config, id=" + peerId);
3513    ReplicationPeerConfig peerConfig = this.replicationPeerManager.getPeerConfig(peerId)
3514        .orElseThrow(() -> new ReplicationPeerNotFoundException(peerId));
3515    if (cpHost != null) {
3516      cpHost.postGetReplicationPeerConfig(peerId);
3517    }
3518    return peerConfig;
3519  }
3520
3521  @Override
3522  public long updateReplicationPeerConfig(String peerId, ReplicationPeerConfig peerConfig)
3523      throws ReplicationException, IOException {
3524    LOG.info(getClientIdAuditPrefix() + " update replication peer config, id=" + peerId +
3525      ", config=" + peerConfig);
3526    return executePeerProcedure(new UpdatePeerConfigProcedure(peerId, peerConfig));
3527  }
3528
3529  @Override
3530  public List<ReplicationPeerDescription> listReplicationPeers(String regex)
3531      throws ReplicationException, IOException {
3532    if (cpHost != null) {
3533      cpHost.preListReplicationPeers(regex);
3534    }
3535    LOG.debug("{} list replication peers, regex={}", getClientIdAuditPrefix(), regex);
3536    Pattern pattern = regex == null ? null : Pattern.compile(regex);
3537    List<ReplicationPeerDescription> peers =
3538      this.replicationPeerManager.listPeers(pattern);
3539    if (cpHost != null) {
3540      cpHost.postListReplicationPeers(regex);
3541    }
3542    return peers;
3543  }
3544
3545  @Override
3546  public long transitReplicationPeerSyncReplicationState(String peerId, SyncReplicationState state)
3547    throws ReplicationException, IOException {
3548    LOG.info(
3549      getClientIdAuditPrefix() +
3550        " transit current cluster state to {} in a synchronous replication peer id={}",
3551      state, peerId);
3552    return executePeerProcedure(new TransitPeerSyncReplicationStateProcedure(peerId, state));
3553  }
3554
3555  /**
3556   * Mark region server(s) as decommissioned (previously called 'draining') to prevent additional
3557   * regions from getting assigned to them. Also unload the regions on the servers asynchronously.0
3558   * @param servers Region servers to decommission.
3559   */
3560  public void decommissionRegionServers(final List<ServerName> servers, final boolean offload)
3561      throws IOException {
3562    List<ServerName> serversAdded = new ArrayList<>(servers.size());
3563    // Place the decommission marker first.
3564    String parentZnode = getZooKeeper().getZNodePaths().drainingZNode;
3565    for (ServerName server : servers) {
3566      try {
3567        String node = ZNodePaths.joinZNode(parentZnode, server.getServerName());
3568        ZKUtil.createAndFailSilent(getZooKeeper(), node);
3569      } catch (KeeperException ke) {
3570        throw new HBaseIOException(
3571          this.zooKeeper.prefix("Unable to decommission '" + server.getServerName() + "'."), ke);
3572      }
3573      if (this.serverManager.addServerToDrainList(server)) {
3574        serversAdded.add(server);
3575      }
3576    }
3577    // Move the regions off the decommissioned servers.
3578    if (offload) {
3579      final List<ServerName> destServers = this.serverManager.createDestinationServersList();
3580      for (ServerName server : serversAdded) {
3581        final List<RegionInfo> regionsOnServer = this.assignmentManager.getRegionsOnServer(server);
3582        for (RegionInfo hri : regionsOnServer) {
3583          ServerName dest = balancer.randomAssignment(hri, destServers);
3584          if (dest == null) {
3585            throw new HBaseIOException("Unable to determine a plan to move " + hri);
3586          }
3587          RegionPlan rp = new RegionPlan(hri, server, dest);
3588          this.assignmentManager.moveAsync(rp);
3589        }
3590      }
3591    }
3592  }
3593
3594  /**
3595   * List region servers marked as decommissioned (previously called 'draining') to not get regions
3596   * assigned to them.
3597   * @return List of decommissioned servers.
3598   */
3599  public List<ServerName> listDecommissionedRegionServers() {
3600    return this.serverManager.getDrainingServersList();
3601  }
3602
3603  /**
3604   * Remove decommission marker (previously called 'draining') from a region server to allow regions
3605   * assignments. Load regions onto the server asynchronously if a list of regions is given
3606   * @param server Region server to remove decommission marker from.
3607   */
3608  public void recommissionRegionServer(final ServerName server,
3609      final List<byte[]> encodedRegionNames) throws IOException {
3610    // Remove the server from decommissioned (draining) server list.
3611    String parentZnode = getZooKeeper().getZNodePaths().drainingZNode;
3612    String node = ZNodePaths.joinZNode(parentZnode, server.getServerName());
3613    try {
3614      ZKUtil.deleteNodeFailSilent(getZooKeeper(), node);
3615    } catch (KeeperException ke) {
3616      throw new HBaseIOException(
3617        this.zooKeeper.prefix("Unable to recommission '" + server.getServerName() + "'."), ke);
3618    }
3619    this.serverManager.removeServerFromDrainList(server);
3620
3621    // Load the regions onto the server if we are given a list of regions.
3622    if (encodedRegionNames == null || encodedRegionNames.isEmpty()) {
3623      return;
3624    }
3625    if (!this.serverManager.isServerOnline(server)) {
3626      return;
3627    }
3628    for (byte[] encodedRegionName : encodedRegionNames) {
3629      RegionState regionState =
3630        assignmentManager.getRegionStates().getRegionState(Bytes.toString(encodedRegionName));
3631      if (regionState == null) {
3632        LOG.warn("Unknown region " + Bytes.toStringBinary(encodedRegionName));
3633        continue;
3634      }
3635      RegionInfo hri = regionState.getRegion();
3636      if (server.equals(regionState.getServerName())) {
3637        LOG.info("Skipping move of region " + hri.getRegionNameAsString() +
3638          " because region already assigned to the same server " + server + ".");
3639        continue;
3640      }
3641      RegionPlan rp = new RegionPlan(hri, regionState.getServerName(), server);
3642      this.assignmentManager.moveAsync(rp);
3643    }
3644  }
3645
3646  @Override
3647  public LockManager getLockManager() {
3648    return lockManager;
3649  }
3650
3651  public QuotaObserverChore getQuotaObserverChore() {
3652    return this.quotaObserverChore;
3653  }
3654
3655  public SpaceQuotaSnapshotNotifier getSpaceQuotaSnapshotNotifier() {
3656    return this.spaceQuotaSnapshotNotifier;
3657  }
3658
3659  @SuppressWarnings("unchecked")
3660  private RemoteProcedure<MasterProcedureEnv, ?> getRemoteProcedure(long procId) {
3661    Procedure<?> procedure = procedureExecutor.getProcedure(procId);
3662    if (procedure == null) {
3663      return null;
3664    }
3665    assert procedure instanceof RemoteProcedure;
3666    return (RemoteProcedure<MasterProcedureEnv, ?>) procedure;
3667  }
3668
3669  public void remoteProcedureCompleted(long procId) {
3670    LOG.debug("Remote procedure done, pid={}", procId);
3671    RemoteProcedure<MasterProcedureEnv, ?> procedure = getRemoteProcedure(procId);
3672    if (procedure != null) {
3673      procedure.remoteOperationCompleted(procedureExecutor.getEnvironment());
3674    }
3675  }
3676
3677  public void remoteProcedureFailed(long procId, RemoteProcedureException error) {
3678    LOG.debug("Remote procedure failed, pid={}", procId, error);
3679    RemoteProcedure<MasterProcedureEnv, ?> procedure = getRemoteProcedure(procId);
3680    if (procedure != null) {
3681      procedure.remoteOperationFailed(procedureExecutor.getEnvironment(), error);
3682    }
3683  }
3684
3685  /**
3686   * Reopen regions provided in the argument
3687   *
3688   * @param tableName The current table name
3689   * @param regionNames The region names of the regions to reopen
3690   * @param nonceGroup Identifier for the source of the request, a client or process
3691   * @param nonce A unique identifier for this operation from the client or process identified by
3692   *   <code>nonceGroup</code> (the source must ensure each operation gets a unique id).
3693   * @return procedure Id
3694   * @throws IOException if reopening region fails while running procedure
3695   */
3696  long reopenRegions(final TableName tableName, final List<byte[]> regionNames,
3697      final long nonceGroup, final long nonce)
3698      throws IOException {
3699
3700    return MasterProcedureUtil
3701      .submitProcedure(new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {
3702
3703        @Override
3704        protected void run() throws IOException {
3705          submitProcedure(new ReopenTableRegionsProcedure(tableName, regionNames));
3706        }
3707
3708        @Override
3709        protected String getDescription() {
3710          return "ReopenTableRegionsProcedure";
3711        }
3712
3713      });
3714
3715  }
3716
3717  @Override
3718  public ReplicationPeerManager getReplicationPeerManager() {
3719    return replicationPeerManager;
3720  }
3721
3722  public HashMap<String, List<Pair<ServerName, ReplicationLoadSource>>>
3723      getReplicationLoad(ServerName[] serverNames) {
3724    List<ReplicationPeerDescription> peerList = this.getReplicationPeerManager().listPeers(null);
3725    if (peerList == null) {
3726      return null;
3727    }
3728    HashMap<String, List<Pair<ServerName, ReplicationLoadSource>>> replicationLoadSourceMap =
3729        new HashMap<>(peerList.size());
3730    peerList.stream()
3731        .forEach(peer -> replicationLoadSourceMap.put(peer.getPeerId(), new ArrayList<>()));
3732    for (ServerName serverName : serverNames) {
3733      List<ReplicationLoadSource> replicationLoadSources =
3734          getServerManager().getLoad(serverName).getReplicationLoadSourceList();
3735      for (ReplicationLoadSource replicationLoadSource : replicationLoadSources) {
3736        replicationLoadSourceMap.get(replicationLoadSource.getPeerID())
3737            .add(new Pair<>(serverName, replicationLoadSource));
3738      }
3739    }
3740    for (List<Pair<ServerName, ReplicationLoadSource>> loads : replicationLoadSourceMap.values()) {
3741      if (loads.size() > 0) {
3742        loads.sort(Comparator.comparingLong(load -> (-1) * load.getSecond().getReplicationLag()));
3743      }
3744    }
3745    return replicationLoadSourceMap;
3746  }
3747
3748  /**
3749   * This method modifies the master's configuration in order to inject replication-related features
3750   */
3751  @VisibleForTesting
3752  public static void decorateMasterConfiguration(Configuration conf) {
3753    String plugins = conf.get(HBASE_MASTER_LOGCLEANER_PLUGINS);
3754    String cleanerClass = ReplicationLogCleaner.class.getCanonicalName();
3755    if (plugins == null || !plugins.contains(cleanerClass)) {
3756      conf.set(HBASE_MASTER_LOGCLEANER_PLUGINS, plugins + "," + cleanerClass);
3757    }
3758    if (ReplicationUtils.isReplicationForBulkLoadDataEnabled(conf)) {
3759      plugins = conf.get(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS);
3760      cleanerClass = ReplicationHFileCleaner.class.getCanonicalName();
3761      if (!plugins.contains(cleanerClass)) {
3762        conf.set(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, plugins + "," + cleanerClass);
3763      }
3764    }
3765  }
3766
3767  public SnapshotQuotaObserverChore getSnapshotQuotaObserverChore() {
3768    return this.snapshotQuotaChore;
3769  }
3770
3771  @Override
3772  public SyncReplicationReplayWALManager getSyncReplicationReplayWALManager() {
3773    return this.syncReplicationReplayWALManager;
3774  }
3775
3776  @Override
3777  public Map<String, ReplicationStatus> getWalGroupsReplicationStatus() {
3778    if (!this.isOnline() || !LoadBalancer.isMasterCanHostUserRegions(conf)) {
3779      return new HashMap<>();
3780    }
3781    return super.getWalGroupsReplicationStatus();
3782  }
3783
3784  public HbckChore getHbckChore() {
3785    return this.hbckChore;
3786  }
3787
3788  @Override
3789  public String getClusterId() {
3790    if (activeMaster) {
3791      return super.getClusterId();
3792    }
3793    return cachedClusterId.getFromCacheOrFetch();
3794  }
3795
3796  public Optional<ServerName> getActiveMaster() {
3797    return activeMasterManager.getActiveMasterServerName();
3798  }
3799
3800  @Override
3801  public void runReplicationBarrierCleaner() {
3802    ReplicationBarrierCleaner rbc = this.replicationBarrierCleaner;
3803    if (rbc != null) {
3804      rbc.chore();
3805    }
3806  }
3807
3808  public MetaRegionLocationCache getMetaRegionLocationCache() {
3809    return this.metaRegionLocationCache;
3810  }
3811
3812  @Override
3813  public RSGroupInfoManager getRSGroupInfoManager() {
3814    return rsGroupInfoManager;
3815  }
3816
3817  /**
3818   * Get the compaction state of the table
3819   *
3820   * @param tableName The table name
3821   * @return CompactionState Compaction state of the table
3822   */
3823  public CompactionState getCompactionState(final TableName tableName) {
3824    CompactionState compactionState = CompactionState.NONE;
3825    try {
3826      List<RegionInfo> regions =
3827        assignmentManager.getRegionStates().getRegionsOfTable(tableName, false);
3828      for (RegionInfo regionInfo : regions) {
3829        ServerName serverName =
3830          assignmentManager.getRegionStates().getRegionServerOfRegion(regionInfo);
3831        if (serverName == null) {
3832          continue;
3833        }
3834        ServerMetrics sl = serverManager.getLoad(serverName);
3835        if (sl == null) {
3836          continue;
3837        }
3838        RegionMetrics regionMetrics = sl.getRegionMetrics().get(regionInfo.getRegionName());
3839        if (regionMetrics.getCompactionState() == CompactionState.MAJOR) {
3840          if (compactionState == CompactionState.MINOR) {
3841            compactionState = CompactionState.MAJOR_AND_MINOR;
3842          } else {
3843            compactionState = CompactionState.MAJOR;
3844          }
3845        } else if (regionMetrics.getCompactionState() == CompactionState.MINOR) {
3846          if (compactionState == CompactionState.MAJOR) {
3847            compactionState = CompactionState.MAJOR_AND_MINOR;
3848          } else {
3849            compactionState = CompactionState.MINOR;
3850          }
3851        }
3852      }
3853    } catch (Exception e) {
3854      compactionState = null;
3855      LOG.error("Exception when get compaction state for " + tableName.getNameAsString(), e);
3856    }
3857    return compactionState;
3858  }
3859}