View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.master;
20  
21  import java.io.IOException;
22  import java.io.InterruptedIOException;
23  import java.lang.reflect.Constructor;
24  import java.lang.reflect.InvocationTargetException;
25  import java.net.InetAddress;
26  import java.net.InetSocketAddress;
27  import java.net.UnknownHostException;
28  import java.util.ArrayList;
29  import java.util.Arrays;
30  import java.util.Collection;
31  import java.util.Collections;
32  import java.util.Comparator;
33  import java.util.HashSet;
34  import java.util.Iterator;
35  import java.util.List;
36  import java.util.Map;
37  import java.util.Set;
38  import java.util.concurrent.TimeUnit;
39  import java.util.concurrent.atomic.AtomicInteger;
40  import java.util.concurrent.atomic.AtomicReference;
41  import java.util.regex.Pattern;
42  
43  import javax.servlet.ServletException;
44  import javax.servlet.http.HttpServlet;
45  import javax.servlet.http.HttpServletRequest;
46  import javax.servlet.http.HttpServletResponse;
47  
48  import org.apache.commons.logging.Log;
49  import org.apache.commons.logging.LogFactory;
50  import org.apache.hadoop.conf.Configuration;
51  import org.apache.hadoop.fs.Path;
52  import org.apache.hadoop.hbase.ClusterStatus;
53  import org.apache.hadoop.hbase.CoordinatedStateException;
54  import org.apache.hadoop.hbase.CoordinatedStateManager;
55  import org.apache.hadoop.hbase.DoNotRetryIOException;
56  import org.apache.hadoop.hbase.HBaseIOException;
57  import org.apache.hadoop.hbase.HBaseInterfaceAudience;
58  import org.apache.hadoop.hbase.HColumnDescriptor;
59  import org.apache.hadoop.hbase.HConstants;
60  import org.apache.hadoop.hbase.HRegionInfo;
61  import org.apache.hadoop.hbase.HTableDescriptor;
62  import org.apache.hadoop.hbase.MasterNotRunningException;
63  import org.apache.hadoop.hbase.MetaTableAccessor;
64  import org.apache.hadoop.hbase.NamespaceDescriptor;
65  import org.apache.hadoop.hbase.NamespaceNotFoundException;
66  import org.apache.hadoop.hbase.PleaseHoldException;
67  import org.apache.hadoop.hbase.Server;
68  import org.apache.hadoop.hbase.ServerLoad;
69  import org.apache.hadoop.hbase.ServerName;
70  import org.apache.hadoop.hbase.TableDescriptors;
71  import org.apache.hadoop.hbase.TableName;
72  import org.apache.hadoop.hbase.TableNotDisabledException;
73  import org.apache.hadoop.hbase.TableNotFoundException;
74  import org.apache.hadoop.hbase.UnknownRegionException;
75  import org.apache.hadoop.hbase.classification.InterfaceAudience;
76  import org.apache.hadoop.hbase.client.RegionReplicaUtil;
77  import org.apache.hadoop.hbase.client.Result;
78  import org.apache.hadoop.hbase.client.TableState;
79  import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
80  import org.apache.hadoop.hbase.exceptions.DeserializationException;
81  import org.apache.hadoop.hbase.executor.ExecutorType;
82  import org.apache.hadoop.hbase.ipc.RpcServer;
83  import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
84  import org.apache.hadoop.hbase.master.MasterRpcServices.BalanceSwitchMode;
85  import org.apache.hadoop.hbase.master.balancer.BalancerChore;
86  import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer;
87  import org.apache.hadoop.hbase.master.balancer.ClusterStatusChore;
88  import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory;
89  import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
90  import org.apache.hadoop.hbase.master.cleaner.LogCleaner;
91  import org.apache.hadoop.hbase.master.handler.DispatchMergingRegionHandler;
92  import org.apache.hadoop.hbase.master.procedure.AddColumnFamilyProcedure;
93  import org.apache.hadoop.hbase.master.procedure.CreateTableProcedure;
94  import org.apache.hadoop.hbase.master.procedure.DeleteColumnFamilyProcedure;
95  import org.apache.hadoop.hbase.master.procedure.DeleteTableProcedure;
96  import org.apache.hadoop.hbase.master.procedure.DisableTableProcedure;
97  import org.apache.hadoop.hbase.master.procedure.EnableTableProcedure;
98  import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants;
99  import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
100 import org.apache.hadoop.hbase.master.procedure.ModifyColumnFamilyProcedure;
101 import org.apache.hadoop.hbase.master.procedure.ModifyTableProcedure;
102 import org.apache.hadoop.hbase.master.procedure.ProcedurePrepareLatch;
103 import org.apache.hadoop.hbase.master.procedure.ProcedureSyncWait;
104 import org.apache.hadoop.hbase.master.procedure.TruncateTableProcedure;
105 import org.apache.hadoop.hbase.master.normalizer.RegionNormalizer;
106 import org.apache.hadoop.hbase.master.normalizer.RegionNormalizerChore;
107 import org.apache.hadoop.hbase.master.normalizer.RegionNormalizerFactory;
108 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
109 import org.apache.hadoop.hbase.mob.MobConstants;
110 import org.apache.hadoop.hbase.monitoring.MemoryBoundedLogMessageBuffer;
111 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
112 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
113 import org.apache.hadoop.hbase.procedure.MasterProcedureManagerHost;
114 import org.apache.hadoop.hbase.procedure.flush.MasterFlushTableProcedureManager;
115 import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
116 import org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore;
117 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState;
118 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionServerInfo;
119 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode;
120 import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
121 import org.apache.hadoop.hbase.quotas.RegionStateListener;
122 import org.apache.hadoop.hbase.regionserver.HRegionServer;
123 import org.apache.hadoop.hbase.regionserver.RSRpcServices;
124 import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
125 import org.apache.hadoop.hbase.regionserver.RegionSplitPolicy;
126 import org.apache.hadoop.hbase.replication.regionserver.Replication;
127 import org.apache.hadoop.hbase.security.UserProvider;
128 import org.apache.hadoop.hbase.util.Addressing;
129 import org.apache.hadoop.hbase.util.Bytes;
130 import org.apache.hadoop.hbase.util.CompressionTest;
131 import org.apache.hadoop.hbase.util.EncryptionTest;
132 import org.apache.hadoop.hbase.util.FSUtils;
133 import org.apache.hadoop.hbase.util.HFileArchiveUtil;
134 import org.apache.hadoop.hbase.util.HasThread;
135 import org.apache.hadoop.hbase.util.IdLock;
136 import org.apache.hadoop.hbase.util.ModifyRegionUtils;
137 import org.apache.hadoop.hbase.util.Pair;
138 import org.apache.hadoop.hbase.util.Threads;
139 import org.apache.hadoop.hbase.util.VersionInfo;
140 import org.apache.hadoop.hbase.util.ZKDataMigrator;
141 import org.apache.hadoop.hbase.zookeeper.DrainingServerTracker;
142 import org.apache.hadoop.hbase.zookeeper.LoadBalancerTracker;
143 import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker;
144 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
145 import org.apache.hadoop.hbase.zookeeper.RegionServerTracker;
146 import org.apache.hadoop.hbase.zookeeper.ZKClusterId;
147 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
148 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
149 import org.apache.zookeeper.KeeperException;
150 import org.mortbay.jetty.Connector;
151 import org.mortbay.jetty.nio.SelectChannelConnector;
152 import org.mortbay.jetty.servlet.Context;
153 
154 import com.google.common.annotations.VisibleForTesting;
155 import com.google.common.collect.Maps;
156 import com.google.protobuf.Descriptors;
157 import com.google.protobuf.Service;
158 
159 /**
160  * HMaster is the "master server" for HBase. An HBase cluster has one active
161  * master.  If many masters are started, all compete.  Whichever wins goes on to
162  * run the cluster.  All others park themselves in their constructor until
163  * master or cluster shutdown or until the active master loses its lease in
164  * zookeeper.  Thereafter, all running master jostle to take over master role.
165  *
166  * <p>The Master can be asked shutdown the cluster. See {@link #shutdown()}.  In
167  * this case it will tell all regionservers to go down and then wait on them
168  * all reporting in that they are down.  This master will then shut itself down.
169  *
170  * <p>You can also shutdown just this master.  Call {@link #stopMaster()}.
171  *
172  * @see org.apache.zookeeper.Watcher
173  */
174 @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
175 @SuppressWarnings("deprecation")
176 public class HMaster extends HRegionServer implements MasterServices, Server {
177   private static final Log LOG = LogFactory.getLog(HMaster.class.getName());
178 
179   /**
180    * Protection against zombie master. Started once Master accepts active responsibility and
181    * starts taking over responsibilities. Allows a finite time window before giving up ownership.
182    */
183   private static class InitializationMonitor extends HasThread {
184     /** The amount of time in milliseconds to sleep before checking initialization status. */
185     public static final String TIMEOUT_KEY = "hbase.master.initializationmonitor.timeout";
186     public static final long TIMEOUT_DEFAULT = TimeUnit.MILLISECONDS.convert(15, TimeUnit.MINUTES);
187 
188     /**
189      * When timeout expired and initialization has not complete, call {@link System#exit(int)} when
190      * true, do nothing otherwise.
191      */
192     public static final String HALT_KEY = "hbase.master.initializationmonitor.haltontimeout";
193     public static final boolean HALT_DEFAULT = false;
194 
195     private final HMaster master;
196     private final long timeout;
197     private final boolean haltOnTimeout;
198 
199     /** Creates a Thread that monitors the {@link #isInitialized()} state. */
200     InitializationMonitor(HMaster master) {
201       super("MasterInitializationMonitor");
202       this.master = master;
203       this.timeout = master.getConfiguration().getLong(TIMEOUT_KEY, TIMEOUT_DEFAULT);
204       this.haltOnTimeout = master.getConfiguration().getBoolean(HALT_KEY, HALT_DEFAULT);
205       this.setDaemon(true);
206     }
207 
208     @Override
209     public void run() {
210       try {
211         while (!master.isStopped() && master.isActiveMaster()) {
212           Thread.sleep(timeout);
213           if (master.isInitialized()) {
214             LOG.debug("Initialization completed within allotted tolerance. Monitor exiting.");
215           } else {
216             LOG.error("Master failed to complete initialization after " + timeout + "ms. Please"
217                 + " consider submitting a bug report including a thread dump of this process.");
218             if (haltOnTimeout) {
219               LOG.error("Zombie Master exiting. Thread dump to stdout");
220               Threads.printThreadInfo(System.out, "Zombie HMaster");
221               System.exit(-1);
222             }
223           }
224         }
225       } catch (InterruptedException ie) {
226         LOG.trace("InitMonitor thread interrupted. Existing.");
227       }
228     }
229   }
230 
231   // MASTER is name of the webapp and the attribute name used stuffing this
232   //instance into web context.
233   public static final String MASTER = "master";
234 
235   // Manager and zk listener for master election
236   private final ActiveMasterManager activeMasterManager;
237   // Region server tracker
238   RegionServerTracker regionServerTracker;
239   // Draining region server tracker
240   private DrainingServerTracker drainingServerTracker;
241   // Tracker for load balancer state
242   LoadBalancerTracker loadBalancerTracker;
243 
244   /** Namespace stuff */
245   private TableNamespaceManager tableNamespaceManager;
246 
247   // Metrics for the HMaster
248   final MetricsMaster metricsMaster;
249   // file system manager for the master FS operations
250   private MasterFileSystem fileSystemManager;
251 
252   // server manager to deal with region server info
253   volatile ServerManager serverManager;
254 
255   // manager of assignment nodes in zookeeper
256   AssignmentManager assignmentManager;
257 
258   // buffer for "fatal error" notices from region servers
259   // in the cluster. This is only used for assisting
260   // operations/debugging.
261   MemoryBoundedLogMessageBuffer rsFatals;
262 
263   // flag set after we become the active master (used for testing)
264   private volatile boolean isActiveMaster = false;
265 
266   // flag set after we complete initialization once active,
267   // it is not private since it's used in unit tests
268   volatile boolean initialized = false;
269 
270   // flag set after master services are started,
271   // initialization may have not completed yet.
272   volatile boolean serviceStarted = false;
273 
274   // flag set after we complete assignMeta.
275   private volatile boolean serverCrashProcessingEnabled = false;
276 
277   LoadBalancer balancer;
278   RegionNormalizer normalizer;
279   private boolean normalizerEnabled = false;
280   private BalancerChore balancerChore;
281   private RegionNormalizerChore normalizerChore;
282   private ClusterStatusChore clusterStatusChore;
283   private ClusterStatusPublisher clusterStatusPublisherChore = null;
284 
285   CatalogJanitor catalogJanitorChore;
286   private LogCleaner logCleaner;
287   private HFileCleaner hfileCleaner;
288   private ExpiredMobFileCleanerChore expiredMobFileCleanerChore;
289   private MobCompactionChore mobCompactChore;
290   private MasterMobCompactionThread mobCompactThread;
291   // used to synchronize the mobCompactionStates
292   private final IdLock mobCompactionLock = new IdLock();
293   // save the information of mob compactions in tables.
294   // the key is table name, the value is the number of compactions in that table.
295   private Map<TableName, AtomicInteger> mobCompactionStates = Maps.newConcurrentMap();
296 
297   MasterCoprocessorHost cpHost;
298 
299   private final boolean preLoadTableDescriptors;
300 
301   // Time stamps for when a hmaster became active
302   private long masterActiveTime;
303 
304   //should we check the compression codec type at master side, default true, HBASE-6370
305   private final boolean masterCheckCompression;
306 
307   //should we check encryption settings at master side, default true
308   private final boolean masterCheckEncryption;
309 
310   Map<String, Service> coprocessorServiceHandlers = Maps.newHashMap();
311 
312   // monitor for snapshot of hbase tables
313   SnapshotManager snapshotManager;
314   // monitor for distributed procedures
315   MasterProcedureManagerHost mpmHost;
316 
317   // it is assigned after 'initialized' guard set to true, so should be volatile
318   private volatile MasterQuotaManager quotaManager;
319 
320   private ProcedureExecutor<MasterProcedureEnv> procedureExecutor;
321   private WALProcedureStore procedureStore;
322 
323   // handle table states
324   private TableStateManager tableStateManager;
325 
326   /** flag used in test cases in order to simulate RS failures during master initialization */
327   private volatile boolean initializationBeforeMetaAssignment = false;
328 
329   /** jetty server for master to redirect requests to regionserver infoServer */
330   private org.mortbay.jetty.Server masterJettyServer;
331 
332   public static class RedirectServlet extends HttpServlet {
333     private static final long serialVersionUID = 2894774810058302472L;
334     private static int regionServerInfoPort;
335 
336     @Override
337     public void doGet(HttpServletRequest request,
338         HttpServletResponse response) throws ServletException, IOException {
339       String redirectUrl = request.getScheme() + "://"
340         + request.getServerName() + ":" + regionServerInfoPort
341         + request.getRequestURI();
342       response.sendRedirect(redirectUrl);
343     }
344   }
345 
346   /**
347    * Initializes the HMaster. The steps are as follows:
348    * <p>
349    * <ol>
350    * <li>Initialize the local HRegionServer
351    * <li>Start the ActiveMasterManager.
352    * </ol>
353    * <p>
354    * Remaining steps of initialization occur in
355    * #finishActiveMasterInitialization(MonitoredTask) after
356    * the master becomes the active one.
357    *
358    * @throws KeeperException
359    * @throws IOException
360    */
361   public HMaster(final Configuration conf, CoordinatedStateManager csm)
362       throws IOException, KeeperException {
363     super(conf, csm);
364     this.rsFatals = new MemoryBoundedLogMessageBuffer(
365       conf.getLong("hbase.master.buffer.for.rs.fatals", 1*1024*1024));
366 
367     LOG.info("hbase.rootdir=" + FSUtils.getRootDir(this.conf) +
368       ", hbase.cluster.distributed=" + this.conf.getBoolean(HConstants.CLUSTER_DISTRIBUTED, false));
369 
370     // Disable usage of meta replicas in the master
371     this.conf.setBoolean(HConstants.USE_META_REPLICAS, false);
372 
373     Replication.decorateMasterConfiguration(this.conf);
374 
375     // Hack! Maps DFSClient => Master for logs.  HDFS made this
376     // config param for task trackers, but we can piggyback off of it.
377     if (this.conf.get("mapreduce.task.attempt.id") == null) {
378       this.conf.set("mapreduce.task.attempt.id", "hb_m_" + this.serverName.toString());
379     }
380 
381     // should we check the compression codec type at master side, default true, HBASE-6370
382     this.masterCheckCompression = conf.getBoolean("hbase.master.check.compression", true);
383 
384     // should we check encryption settings at master side, default true
385     this.masterCheckEncryption = conf.getBoolean("hbase.master.check.encryption", true);
386 
387     this.metricsMaster = new MetricsMaster( new MetricsMasterWrapperImpl(this));
388 
389     // preload table descriptor at startup
390     this.preLoadTableDescriptors = conf.getBoolean("hbase.master.preload.tabledescriptors", true);
391 
392     // Do we publish the status?
393 
394     boolean shouldPublish = conf.getBoolean(HConstants.STATUS_PUBLISHED,
395         HConstants.STATUS_PUBLISHED_DEFAULT);
396     Class<? extends ClusterStatusPublisher.Publisher> publisherClass =
397         conf.getClass(ClusterStatusPublisher.STATUS_PUBLISHER_CLASS,
398             ClusterStatusPublisher.DEFAULT_STATUS_PUBLISHER_CLASS,
399             ClusterStatusPublisher.Publisher.class);
400 
401     if (shouldPublish) {
402       if (publisherClass == null) {
403         LOG.warn(HConstants.STATUS_PUBLISHED + " is true, but " +
404             ClusterStatusPublisher.DEFAULT_STATUS_PUBLISHER_CLASS +
405             " is not set - not publishing status");
406       } else {
407         clusterStatusPublisherChore = new ClusterStatusPublisher(this, conf, publisherClass);
408         getChoreService().scheduleChore(clusterStatusPublisherChore);
409       }
410     }
411 
412     // Some unit tests don't need a cluster, so no zookeeper at all
413     if (!conf.getBoolean("hbase.testing.nocluster", false)) {
414       activeMasterManager = new ActiveMasterManager(zooKeeper, this.serverName, this);
415       int infoPort = putUpJettyServer();
416       startActiveMasterManager(infoPort);
417     } else {
418       activeMasterManager = null;
419     }
420   }
421 
422   // return the actual infoPort, -1 means disable info server.
423   private int putUpJettyServer() throws IOException {
424     if (!conf.getBoolean("hbase.master.infoserver.redirect", true)) {
425       return -1;
426     }
427     int infoPort = conf.getInt("hbase.master.info.port.orig",
428       HConstants.DEFAULT_MASTER_INFOPORT);
429     // -1 is for disabling info server, so no redirecting
430     if (infoPort < 0 || infoServer == null) {
431       return -1;
432     }
433     String addr = conf.get("hbase.master.info.bindAddress", "0.0.0.0");
434     if (!Addressing.isLocalAddress(InetAddress.getByName(addr))) {
435       String msg =
436           "Failed to start redirecting jetty server. Address " + addr
437               + " does not belong to this host. Correct configuration parameter: "
438               + "hbase.master.info.bindAddress";
439       LOG.error(msg);
440       throw new IOException(msg);
441     }
442 
443     RedirectServlet.regionServerInfoPort = infoServer.getPort();
444     if(RedirectServlet.regionServerInfoPort == infoPort) {
445       return infoPort;
446     }
447     masterJettyServer = new org.mortbay.jetty.Server();
448     Connector connector = new SelectChannelConnector();
449     connector.setHost(addr);
450     connector.setPort(infoPort);
451     masterJettyServer.addConnector(connector);
452     masterJettyServer.setStopAtShutdown(true);
453     Context context = new Context(masterJettyServer, "/", Context.NO_SESSIONS);
454     context.addServlet(RedirectServlet.class, "/*");
455     try {
456       masterJettyServer.start();
457     } catch (Exception e) {
458       throw new IOException("Failed to start redirecting jetty server", e);
459     }
460     return connector.getLocalPort();
461   }
462 
463   @Override
464   protected TableDescriptors getFsTableDescriptors() throws IOException {
465     return super.getFsTableDescriptors();
466   }
467 
468   /**
469    * For compatibility, if failed with regionserver credentials, try the master one
470    */
471   @Override
472   protected void login(UserProvider user, String host) throws IOException {
473     try {
474       super.login(user, host);
475     } catch (IOException ie) {
476       user.login("hbase.master.keytab.file",
477         "hbase.master.kerberos.principal", host);
478     }
479   }
480 
481   /**
482    * If configured to put regions on active master,
483    * wait till a backup master becomes active.
484    * Otherwise, loop till the server is stopped or aborted.
485    */
486   @Override
487   protected void waitForMasterActive(){
488     boolean tablesOnMaster = BaseLoadBalancer.tablesOnMaster(conf);
489     while (!(tablesOnMaster && isActiveMaster)
490         && !isStopped() && !isAborted()) {
491       sleeper.sleep();
492     }
493   }
494 
495   @VisibleForTesting
496   public MasterRpcServices getMasterRpcServices() {
497     return (MasterRpcServices)rpcServices;
498   }
499 
500   public boolean balanceSwitch(final boolean b) throws IOException {
501     return getMasterRpcServices().switchBalancer(b, BalanceSwitchMode.ASYNC);
502   }
503 
504   @Override
505   protected String getProcessName() {
506     return MASTER;
507   }
508 
509   @Override
510   protected boolean canCreateBaseZNode() {
511     return true;
512   }
513 
514   @Override
515   protected boolean canUpdateTableDescriptor() {
516     return true;
517   }
518 
519   @Override
520   protected RSRpcServices createRpcServices() throws IOException {
521     return new MasterRpcServices(this);
522   }
523 
524   @Override
525   protected void configureInfoServer() {
526     infoServer.addServlet("master-status", "/master-status", MasterStatusServlet.class);
527     infoServer.setAttribute(MASTER, this);
528     if (BaseLoadBalancer.tablesOnMaster(conf)) {
529       super.configureInfoServer();
530     }
531   }
532 
533   @Override
534   protected Class<? extends HttpServlet> getDumpServlet() {
535     return MasterDumpServlet.class;
536   }
537 
538   /**
539    * Emit the HMaster metrics, such as region in transition metrics.
540    * Surrounding in a try block just to be sure metrics doesn't abort HMaster.
541    */
542   @Override
543   protected void doMetrics() {
544     try {
545       if (assignmentManager != null) {
546         assignmentManager.updateRegionsInTransitionMetrics();
547       }
548     } catch (Throwable e) {
549       LOG.error("Couldn't update metrics: " + e.getMessage());
550     }
551   }
552 
553   MetricsMaster getMasterMetrics() {
554     return metricsMaster;
555   }
556 
557   /**
558    * Initialize all ZK based system trackers.
559    * @throws IOException
560    * @throws InterruptedException
561    * @throws KeeperException
562    * @throws CoordinatedStateException
563    */
564   void initializeZKBasedSystemTrackers() throws IOException,
565       InterruptedException, KeeperException, CoordinatedStateException {
566     this.balancer = LoadBalancerFactory.getLoadBalancer(conf);
567     this.normalizer = RegionNormalizerFactory.getRegionNormalizer(conf);
568     this.normalizer.setMasterServices(this);
569     this.normalizerEnabled = conf.getBoolean(HConstants.HBASE_NORMALIZER_ENABLED, false);
570     this.loadBalancerTracker = new LoadBalancerTracker(zooKeeper, this);
571     this.loadBalancerTracker.start();
572     this.assignmentManager = new AssignmentManager(this, serverManager,
573       this.balancer, this.service, this.metricsMaster,
574       this.tableLockManager, tableStateManager);
575 
576     this.regionServerTracker = new RegionServerTracker(zooKeeper, this,
577         this.serverManager);
578     this.regionServerTracker.start();
579 
580     this.drainingServerTracker = new DrainingServerTracker(zooKeeper, this,
581       this.serverManager);
582     this.drainingServerTracker.start();
583 
584     // Set the cluster as up.  If new RSs, they'll be waiting on this before
585     // going ahead with their startup.
586     boolean wasUp = this.clusterStatusTracker.isClusterUp();
587     if (!wasUp) this.clusterStatusTracker.setClusterUp();
588 
589     LOG.info("Server active/primary master=" + this.serverName +
590         ", sessionid=0x" +
591         Long.toHexString(this.zooKeeper.getRecoverableZooKeeper().getSessionId()) +
592         ", setting cluster-up flag (Was=" + wasUp + ")");
593 
594     // create/initialize the snapshot manager and other procedure managers
595     this.snapshotManager = new SnapshotManager();
596     this.mpmHost = new MasterProcedureManagerHost();
597     this.mpmHost.register(this.snapshotManager);
598     this.mpmHost.register(new MasterFlushTableProcedureManager());
599     this.mpmHost.loadProcedures(conf);
600     this.mpmHost.initialize(this, this.metricsMaster);
601 
602   }
603 
604   /**
605    * Finish initialization of HMaster after becoming the primary master.
606    *
607    * <ol>
608    * <li>Initialize master components - file system manager, server manager,
609    *     assignment manager, region server tracker, etc</li>
610    * <li>Start necessary service threads - balancer, catalog janior,
611    *     executor services, etc</li>
612    * <li>Set cluster as UP in ZooKeeper</li>
613    * <li>Wait for RegionServers to check-in</li>
614    * <li>Split logs and perform data recovery, if necessary</li>
615    * <li>Ensure assignment of meta/namespace regions<li>
616    * <li>Handle either fresh cluster start or master failover</li>
617    * </ol>
618    *
619    * @throws IOException
620    * @throws InterruptedException
621    * @throws KeeperException
622    * @throws CoordinatedStateException
623    */
624   private void finishActiveMasterInitialization(MonitoredTask status)
625       throws IOException, InterruptedException, KeeperException, CoordinatedStateException {
626 
627     isActiveMaster = true;
628     Thread zombieDetector = new Thread(new InitializationMonitor(this));
629     zombieDetector.start();
630 
631     /*
632      * We are active master now... go initialize components we need to run.
633      * Note, there may be dross in zk from previous runs; it'll get addressed
634      * below after we determine if cluster startup or failover.
635      */
636 
637     status.setStatus("Initializing Master file system");
638 
639     this.masterActiveTime = System.currentTimeMillis();
640     // TODO: Do this using Dependency Injection, using PicoContainer, Guice or Spring.
641     this.fileSystemManager = new MasterFileSystem(this, this);
642 
643     // enable table descriptors cache
644     this.tableDescriptors.setCacheOn();
645     // set the META's descriptor to the correct replication
646     this.tableDescriptors.get(TableName.META_TABLE_NAME).setRegionReplication(
647         conf.getInt(HConstants.META_REPLICAS_NUM, HConstants.DEFAULT_META_REPLICA_NUM));
648     // warm-up HTDs cache on master initialization
649     if (preLoadTableDescriptors) {
650       status.setStatus("Pre-loading table descriptors");
651       this.tableDescriptors.getAll();
652     }
653 
654     // publish cluster ID
655     status.setStatus("Publishing Cluster ID in ZooKeeper");
656     ZKClusterId.setClusterId(this.zooKeeper, fileSystemManager.getClusterId());
657     this.serverManager = createServerManager(this, this);
658 
659     setupClusterConnection();
660 
661     // Invalidate all write locks held previously
662     this.tableLockManager.reapWriteLocks();
663     this.tableStateManager = new TableStateManager(this);
664 
665     status.setStatus("Initializing ZK system trackers");
666     initializeZKBasedSystemTrackers();
667 
668     // initialize master side coprocessors before we start handling requests
669     status.setStatus("Initializing master coprocessors");
670     this.cpHost = new MasterCoprocessorHost(this, this.conf);
671 
672     // start up all service threads.
673     status.setStatus("Initializing master service threads");
674     startServiceThreads();
675 
676     // Wake up this server to check in
677     sleeper.skipSleepCycle();
678 
679     // Wait for region servers to report in
680     this.serverManager.waitForRegionServers(status);
681     // Check zk for region servers that are up but didn't register
682     for (ServerName sn: this.regionServerTracker.getOnlineServers()) {
683       // The isServerOnline check is opportunistic, correctness is handled inside
684       if (!this.serverManager.isServerOnline(sn)
685           && serverManager.checkAndRecordNewServer(sn, ServerLoad.EMPTY_SERVERLOAD)) {
686         LOG.info("Registered server found up in zk but who has not yet reported in: " + sn);
687       }
688     }
689 
690     // get a list for previously failed RS which need log splitting work
691     // we recover hbase:meta region servers inside master initialization and
692     // handle other failed servers in SSH in order to start up master node ASAP
693     Set<ServerName> previouslyFailedServers =
694       this.fileSystemManager.getFailedServersFromLogFolders();
695 
696     // log splitting for hbase:meta server
697     ServerName oldMetaServerLocation = metaTableLocator.getMetaRegionLocation(this.getZooKeeper());
698     if (oldMetaServerLocation != null && previouslyFailedServers.contains(oldMetaServerLocation)) {
699       splitMetaLogBeforeAssignment(oldMetaServerLocation);
700       // Note: we can't remove oldMetaServerLocation from previousFailedServers list because it
701       // may also host user regions
702     }
703     Set<ServerName> previouslyFailedMetaRSs = getPreviouselyFailedMetaServersFromZK();
704     // need to use union of previouslyFailedMetaRSs recorded in ZK and previouslyFailedServers
705     // instead of previouslyFailedMetaRSs alone to address the following two situations:
706     // 1) the chained failure situation(recovery failed multiple times in a row).
707     // 2) master get killed right before it could delete the recovering hbase:meta from ZK while the
708     // same server still has non-meta wals to be replayed so that
709     // removeStaleRecoveringRegionsFromZK can't delete the stale hbase:meta region
710     // Passing more servers into splitMetaLog is all right. If a server doesn't have hbase:meta wal,
711     // there is no op for the server.
712     previouslyFailedMetaRSs.addAll(previouslyFailedServers);
713 
714     this.initializationBeforeMetaAssignment = true;
715 
716     // Wait for regionserver to finish initialization.
717     if (BaseLoadBalancer.tablesOnMaster(conf)) {
718       waitForServerOnline();
719     }
720 
721     //initialize load balancer
722     this.balancer.setClusterStatus(getClusterStatus());
723     this.balancer.setMasterServices(this);
724     this.balancer.initialize();
725 
726     // Check if master is shutting down because of some issue
727     // in initializing the regionserver or the balancer.
728     if (isStopped()) return;
729 
730     // Make sure meta assigned before proceeding.
731     status.setStatus("Assigning Meta Region");
732     assignMeta(status, previouslyFailedMetaRSs, HRegionInfo.DEFAULT_REPLICA_ID);
733     // check if master is shutting down because above assignMeta could return even hbase:meta isn't
734     // assigned when master is shutting down
735     if (isStopped()) return;
736 
737     // migrating existent table state from zk, so splitters
738     // and recovery process treat states properly.
739     for (Map.Entry<TableName, TableState.State> entry : ZKDataMigrator
740         .queryForTableStates(getZooKeeper()).entrySet()) {
741       LOG.info("Converting state from zk to new states:" + entry);
742       tableStateManager.setTableState(entry.getKey(), entry.getValue());
743     }
744     ZKUtil.deleteChildrenRecursively(getZooKeeper(), getZooKeeper().tableZNode);
745 
746     status.setStatus("Submitting log splitting work for previously failed region servers");
747     // Master has recovered hbase:meta region server and we put
748     // other failed region servers in a queue to be handled later by SSH
749     for (ServerName tmpServer : previouslyFailedServers) {
750       this.serverManager.processDeadServer(tmpServer, true);
751     }
752 
753     // Fix up assignment manager status
754     status.setStatus("Starting assignment manager");
755     this.assignmentManager.joinCluster();
756 
757     // set cluster status again after user regions are assigned
758     this.balancer.setClusterStatus(getClusterStatus());
759 
760     // Start balancer and meta catalog janitor after meta and regions have been assigned.
761     status.setStatus("Starting balancer and catalog janitor");
762     this.clusterStatusChore = new ClusterStatusChore(this, balancer);
763     getChoreService().scheduleChore(clusterStatusChore);
764     this.balancerChore = new BalancerChore(this);
765     getChoreService().scheduleChore(balancerChore);
766     this.normalizerChore = new RegionNormalizerChore(this);
767     getChoreService().scheduleChore(normalizerChore);
768     this.catalogJanitorChore = new CatalogJanitor(this, this);
769     getChoreService().scheduleChore(catalogJanitorChore);
770 
771     status.setStatus("Starting namespace manager");
772     initNamespace();
773 
774     if (this.cpHost != null) {
775       try {
776         this.cpHost.preMasterInitialization();
777       } catch (IOException e) {
778         LOG.error("Coprocessor preMasterInitialization() hook failed", e);
779       }
780     }
781 
782     status.markComplete("Initialization successful");
783     LOG.info("Master has completed initialization");
784     configurationManager.registerObserver(this.balancer);
785     // Set master as 'initialized'.
786     initialized = true;
787     // assign the meta replicas
788     Set<ServerName> EMPTY_SET = new HashSet<ServerName>();
789     int numReplicas = conf.getInt(HConstants.META_REPLICAS_NUM,
790            HConstants.DEFAULT_META_REPLICA_NUM);
791     for (int i = 1; i < numReplicas; i++) {
792       assignMeta(status, EMPTY_SET, i);
793     }
794     unassignExcessMetaReplica(zooKeeper, numReplicas);
795 
796     status.setStatus("Starting quota manager");
797     initQuotaManager();
798 
799     // clear the dead servers with same host name and port of online server because we are not
800     // removing dead server with same hostname and port of rs which is trying to check in before
801     // master initialization. See HBASE-5916.
802     this.serverManager.clearDeadServersWithSameHostNameAndPortOfOnlineServer();
803 
804     // Check and set the znode ACLs if needed in case we are overtaking a non-secure configuration
805     status.setStatus("Checking ZNode ACLs");
806     zooKeeper.checkAndSetZNodeAcls();
807 
808     status.setStatus("Calling postStartMaster coprocessors");
809 
810     this.expiredMobFileCleanerChore = new ExpiredMobFileCleanerChore(this);
811     getChoreService().scheduleChore(expiredMobFileCleanerChore);
812 
813     int mobCompactionPeriod = conf.getInt(MobConstants.MOB_COMPACTION_CHORE_PERIOD,
814       MobConstants.DEFAULT_MOB_COMPACTION_CHORE_PERIOD);
815     if (mobCompactionPeriod > 0) {
816       this.mobCompactChore = new MobCompactionChore(this, mobCompactionPeriod);
817       getChoreService().scheduleChore(mobCompactChore);
818     } else {
819       LOG
820         .info("The period is " + mobCompactionPeriod + " seconds, MobCompactionChore is disabled");
821     }
822     this.mobCompactThread = new MasterMobCompactionThread(this);
823 
824     if (this.cpHost != null) {
825       // don't let cp initialization errors kill the master
826       try {
827         this.cpHost.postStartMaster();
828       } catch (IOException ioe) {
829         LOG.error("Coprocessor postStartMaster() hook failed", ioe);
830       }
831     }
832 
833     zombieDetector.interrupt();
834   }
835 
836   /**
837    * Create a {@link ServerManager} instance.
838    * @param master
839    * @param services
840    * @return An instance of {@link ServerManager}
841    * @throws org.apache.hadoop.hbase.ZooKeeperConnectionException
842    * @throws IOException
843    */
844   ServerManager createServerManager(final Server master,
845       final MasterServices services)
846   throws IOException {
847     // We put this out here in a method so can do a Mockito.spy and stub it out
848     // w/ a mocked up ServerManager.
849     return new ServerManager(master, services);
850   }
851 
852   private void unassignExcessMetaReplica(ZooKeeperWatcher zkw, int numMetaReplicasConfigured) {
853     // unassign the unneeded replicas (for e.g., if the previous master was configured
854     // with a replication of 3 and now it is 2, we need to unassign the 1 unneeded replica)
855     try {
856       List<String> metaReplicaZnodes = zooKeeper.getMetaReplicaNodes();
857       for (String metaReplicaZnode : metaReplicaZnodes) {
858         int replicaId = zooKeeper.getMetaReplicaIdFromZnode(metaReplicaZnode);
859         if (replicaId >= numMetaReplicasConfigured) {
860           RegionState r = MetaTableLocator.getMetaRegionState(zkw, replicaId);
861           LOG.info("Closing excess replica of meta region " + r.getRegion());
862           // send a close and wait for a max of 30 seconds
863           ServerManager.closeRegionSilentlyAndWait(getConnection(), r.getServerName(),
864               r.getRegion(), 30000);
865           ZKUtil.deleteNode(zkw, zkw.getZNodeForReplica(replicaId));
866         }
867       }
868     } catch (Exception ex) {
869       // ignore the exception since we don't want the master to be wedged due to potential
870       // issues in the cleanup of the extra regions. We can do that cleanup via hbck or manually
871       LOG.warn("Ignoring exception " + ex);
872     }
873   }
874 
875   /**
876    * Check <code>hbase:meta</code> is assigned. If not, assign it.
877    * @param status MonitoredTask
878    * @param previouslyFailedMetaRSs
879    * @param replicaId
880    * @throws InterruptedException
881    * @throws IOException
882    * @throws KeeperException
883    */
884   void assignMeta(MonitoredTask status, Set<ServerName> previouslyFailedMetaRSs, int replicaId)
885       throws InterruptedException, IOException, KeeperException {
886     // Work on meta region
887     int assigned = 0;
888     long timeout = this.conf.getLong("hbase.catalog.verification.timeout", 1000);
889     if (replicaId == HRegionInfo.DEFAULT_REPLICA_ID) {
890       status.setStatus("Assigning hbase:meta region");
891     } else {
892       status.setStatus("Assigning hbase:meta region, replicaId " + replicaId);
893     }
894 
895     // Get current meta state from zk.
896     RegionState metaState = MetaTableLocator.getMetaRegionState(getZooKeeper(), replicaId);
897     HRegionInfo hri = RegionReplicaUtil.getRegionInfoForReplica(HRegionInfo.FIRST_META_REGIONINFO,
898         replicaId);
899     RegionStates regionStates = assignmentManager.getRegionStates();
900     regionStates.createRegionState(hri, metaState.getState(),
901         metaState.getServerName(), null);
902 
903     if (!metaState.isOpened() || !metaTableLocator.verifyMetaRegionLocation(
904         this.getConnection(), this.getZooKeeper(), timeout, replicaId)) {
905       ServerName currentMetaServer = metaState.getServerName();
906       if (serverManager.isServerOnline(currentMetaServer)) {
907         if (replicaId == HRegionInfo.DEFAULT_REPLICA_ID) {
908           LOG.info("Meta was in transition on " + currentMetaServer);
909         } else {
910           LOG.info("Meta with replicaId " + replicaId + " was in transition on " +
911                     currentMetaServer);
912         }
913         assignmentManager.processRegionsInTransition(Arrays.asList(metaState));
914       } else {
915         if (currentMetaServer != null) {
916           if (replicaId == HRegionInfo.DEFAULT_REPLICA_ID) {
917             splitMetaLogBeforeAssignment(currentMetaServer);
918             regionStates.logSplit(HRegionInfo.FIRST_META_REGIONINFO);
919             previouslyFailedMetaRSs.add(currentMetaServer);
920           }
921         }
922         LOG.info("Re-assigning hbase:meta with replicaId, " + replicaId +
923             " it was on " + currentMetaServer);
924         assignmentManager.assignMeta(hri);
925       }
926       assigned++;
927     }
928 
929     if (replicaId == HRegionInfo.DEFAULT_REPLICA_ID)
930       getTableStateManager().setTableState(TableName.META_TABLE_NAME, TableState.State.ENABLED);
931     // TODO: should we prevent from using state manager before meta was initialized?
932     // tableStateManager.start();
933 
934     if ((RecoveryMode.LOG_REPLAY == this.getMasterFileSystem().getLogRecoveryMode())
935         && (!previouslyFailedMetaRSs.isEmpty())) {
936       // replay WAL edits mode need new hbase:meta RS is assigned firstly
937       status.setStatus("replaying log for Meta Region");
938       this.fileSystemManager.splitMetaLog(previouslyFailedMetaRSs);
939     }
940 
941     this.assignmentManager.setEnabledTable(TableName.META_TABLE_NAME);
942     tableStateManager.start();
943 
944     // Make sure a hbase:meta location is set. We need to enable SSH here since
945     // if the meta region server is died at this time, we need it to be re-assigned
946     // by SSH so that system tables can be assigned.
947     // No need to wait for meta is assigned = 0 when meta is just verified.
948     if (replicaId == HRegionInfo.DEFAULT_REPLICA_ID) enableCrashedServerProcessing(assigned != 0);
949     LOG.info("hbase:meta with replicaId " + replicaId + " assigned=" + assigned + ", location="
950       + metaTableLocator.getMetaRegionLocation(this.getZooKeeper(), replicaId));
951     status.setStatus("META assigned.");
952   }
953 
954   void initNamespace() throws IOException {
955     //create namespace manager
956     tableNamespaceManager = new TableNamespaceManager(this);
957     tableNamespaceManager.start();
958   }
959 
960   void initQuotaManager() throws IOException {
961     MasterQuotaManager quotaManager = new MasterQuotaManager(this);
962     this.assignmentManager.setRegionStateListener((RegionStateListener)quotaManager);
963     quotaManager.start();
964     this.quotaManager = quotaManager;
965   }
966 
967   boolean isCatalogJanitorEnabled() {
968     return catalogJanitorChore != null ?
969       catalogJanitorChore.getEnabled() : false;
970   }
971 
972   private void splitMetaLogBeforeAssignment(ServerName currentMetaServer) throws IOException {
973     if (RecoveryMode.LOG_REPLAY == this.getMasterFileSystem().getLogRecoveryMode()) {
974       // In log replay mode, we mark hbase:meta region as recovering in ZK
975       Set<HRegionInfo> regions = new HashSet<HRegionInfo>();
976       regions.add(HRegionInfo.FIRST_META_REGIONINFO);
977       this.fileSystemManager.prepareLogReplay(currentMetaServer, regions);
978     } else {
979       // In recovered.edits mode: create recovered edits file for hbase:meta server
980       this.fileSystemManager.splitMetaLog(currentMetaServer);
981     }
982   }
983 
984   private void enableCrashedServerProcessing(final boolean waitForMeta)
985   throws IOException, InterruptedException {
986     // If crashed server processing is disabled, we enable it and expire those dead but not expired
987     // servers. This is required so that if meta is assigning to a server which dies after
988     // assignMeta starts assignment, ServerCrashProcedure can re-assign it. Otherwise, we will be
989     // stuck here waiting forever if waitForMeta is specified.
990     if (!serverCrashProcessingEnabled) {
991       serverCrashProcessingEnabled = true;
992       this.serverManager.processQueuedDeadServers();
993     }
994 
995     if (waitForMeta) {
996       metaTableLocator.waitMetaRegionLocation(this.getZooKeeper());
997     }
998   }
999 
1000   /**
1001    * This function returns a set of region server names under hbase:meta recovering region ZK node
1002    * @return Set of meta server names which were recorded in ZK
1003    * @throws KeeperException
1004    */
1005   private Set<ServerName> getPreviouselyFailedMetaServersFromZK() throws KeeperException {
1006     Set<ServerName> result = new HashSet<ServerName>();
1007     String metaRecoveringZNode = ZKUtil.joinZNode(zooKeeper.recoveringRegionsZNode,
1008       HRegionInfo.FIRST_META_REGIONINFO.getEncodedName());
1009     List<String> regionFailedServers = ZKUtil.listChildrenNoWatch(zooKeeper, metaRecoveringZNode);
1010     if (regionFailedServers == null) return result;
1011 
1012     for(String failedServer : regionFailedServers) {
1013       ServerName server = ServerName.parseServerName(failedServer);
1014       result.add(server);
1015     }
1016     return result;
1017   }
1018 
1019   @Override
1020   public TableDescriptors getTableDescriptors() {
1021     return this.tableDescriptors;
1022   }
1023 
1024   @Override
1025   public ServerManager getServerManager() {
1026     return this.serverManager;
1027   }
1028 
1029   @Override
1030   public MasterFileSystem getMasterFileSystem() {
1031     return this.fileSystemManager;
1032   }
1033 
1034   @Override
1035   public TableStateManager getTableStateManager() {
1036     return tableStateManager;
1037   }
1038 
1039   /*
1040    * Start up all services. If any of these threads gets an unhandled exception
1041    * then they just die with a logged message.  This should be fine because
1042    * in general, we do not expect the master to get such unhandled exceptions
1043    *  as OOMEs; it should be lightly loaded. See what HRegionServer does if
1044    *  need to install an unexpected exception handler.
1045    */
1046   private void startServiceThreads() throws IOException{
1047    // Start the executor service pools
1048    this.service.startExecutorService(ExecutorType.MASTER_OPEN_REGION,
1049       conf.getInt("hbase.master.executor.openregion.threads", 5));
1050    this.service.startExecutorService(ExecutorType.MASTER_CLOSE_REGION,
1051       conf.getInt("hbase.master.executor.closeregion.threads", 5));
1052    this.service.startExecutorService(ExecutorType.MASTER_SERVER_OPERATIONS,
1053       conf.getInt("hbase.master.executor.serverops.threads", 5));
1054    this.service.startExecutorService(ExecutorType.MASTER_META_SERVER_OPERATIONS,
1055       conf.getInt("hbase.master.executor.meta.serverops.threads", 5));
1056    this.service.startExecutorService(ExecutorType.M_LOG_REPLAY_OPS,
1057       conf.getInt("hbase.master.executor.logreplayops.threads", 10));
1058 
1059    // We depend on there being only one instance of this executor running
1060    // at a time.  To do concurrency, would need fencing of enable/disable of
1061    // tables.
1062    // Any time changing this maxThreads to > 1, pls see the comment at
1063    // AccessController#postCreateTableHandler
1064    this.service.startExecutorService(ExecutorType.MASTER_TABLE_OPERATIONS, 1);
1065    startProcedureExecutor();
1066 
1067    // Start log cleaner thread
1068    int cleanerInterval = conf.getInt("hbase.master.cleaner.interval", 60 * 1000);
1069    this.logCleaner =
1070       new LogCleaner(cleanerInterval,
1071          this, conf, getMasterFileSystem().getFileSystem(),
1072          getMasterFileSystem().getOldLogDir());
1073     getChoreService().scheduleChore(logCleaner);
1074 
1075    //start the hfile archive cleaner thread
1076     Path archiveDir = HFileArchiveUtil.getArchivePath(conf);
1077     this.hfileCleaner = new HFileCleaner(cleanerInterval, this, conf, getMasterFileSystem()
1078         .getFileSystem(), archiveDir);
1079     getChoreService().scheduleChore(hfileCleaner);
1080     serviceStarted = true;
1081     if (LOG.isTraceEnabled()) {
1082       LOG.trace("Started service threads");
1083     }
1084   }
1085 
1086   @Override
1087   protected void sendShutdownInterrupt() {
1088     super.sendShutdownInterrupt();
1089     stopProcedureExecutor();
1090   }
1091 
1092   @Override
1093   protected void stopServiceThreads() {
1094     if (masterJettyServer != null) {
1095       LOG.info("Stopping master jetty server");
1096       try {
1097         masterJettyServer.stop();
1098       } catch (Exception e) {
1099         LOG.error("Failed to stop master jetty server", e);
1100       }
1101     }
1102     super.stopServiceThreads();
1103     stopChores();
1104 
1105     // Wait for all the remaining region servers to report in IFF we were
1106     // running a cluster shutdown AND we were NOT aborting.
1107     if (!isAborted() && this.serverManager != null &&
1108         this.serverManager.isClusterShutdown()) {
1109       this.serverManager.letRegionServersShutdown();
1110     }
1111     if (LOG.isDebugEnabled()) {
1112       LOG.debug("Stopping service threads");
1113     }
1114     // Clean up and close up shop
1115     if (this.logCleaner != null) this.logCleaner.cancel(true);
1116     if (this.hfileCleaner != null) this.hfileCleaner.cancel(true);
1117     if (this.quotaManager != null) this.quotaManager.stop();
1118     if (this.activeMasterManager != null) this.activeMasterManager.stop();
1119     if (this.serverManager != null) this.serverManager.stop();
1120     if (this.assignmentManager != null) this.assignmentManager.stop();
1121     if (this.fileSystemManager != null) this.fileSystemManager.stop();
1122     if (this.mpmHost != null) this.mpmHost.stop("server shutting down.");
1123   }
1124 
1125   private void startProcedureExecutor() throws IOException {
1126     final MasterProcedureEnv procEnv = new MasterProcedureEnv(this);
1127     final Path logDir = new Path(fileSystemManager.getRootDir(),
1128         MasterProcedureConstants.MASTER_PROCEDURE_LOGDIR);
1129 
1130     procedureStore = new WALProcedureStore(conf, fileSystemManager.getFileSystem(), logDir,
1131         new MasterProcedureEnv.WALStoreLeaseRecovery(this));
1132     procedureStore.registerListener(new MasterProcedureEnv.MasterProcedureStoreListener(this));
1133     procedureExecutor = new ProcedureExecutor(conf, procEnv, procedureStore,
1134         procEnv.getProcedureQueue());
1135 
1136     final int numThreads = conf.getInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS,
1137         Math.max(Runtime.getRuntime().availableProcessors(),
1138           MasterProcedureConstants.DEFAULT_MIN_MASTER_PROCEDURE_THREADS));
1139     final boolean abortOnCorruption = conf.getBoolean(
1140         MasterProcedureConstants.EXECUTOR_ABORT_ON_CORRUPTION,
1141         MasterProcedureConstants.DEFAULT_EXECUTOR_ABORT_ON_CORRUPTION);
1142     procedureStore.start(numThreads);
1143     procedureExecutor.start(numThreads, abortOnCorruption);
1144   }
1145 
1146   private void stopProcedureExecutor() {
1147     if (procedureExecutor != null) {
1148       procedureExecutor.stop();
1149     }
1150 
1151     if (procedureStore != null) {
1152       procedureStore.stop(isAborted());
1153     }
1154   }
1155 
1156   private void stopChores() {
1157     if (this.expiredMobFileCleanerChore != null) {
1158       this.expiredMobFileCleanerChore.cancel(true);
1159     }
1160     if (this.mobCompactChore != null) {
1161       this.mobCompactChore.cancel(true);
1162     }
1163     if (this.balancerChore != null) {
1164       this.balancerChore.cancel(true);
1165     }
1166     if (this.normalizerChore != null) {
1167       this.normalizerChore.cancel(true);
1168     }
1169     if (this.clusterStatusChore != null) {
1170       this.clusterStatusChore.cancel(true);
1171     }
1172     if (this.catalogJanitorChore != null) {
1173       this.catalogJanitorChore.cancel(true);
1174     }
1175     if (this.clusterStatusPublisherChore != null){
1176       clusterStatusPublisherChore.cancel(true);
1177     }
1178     if (this.mobCompactThread != null) {
1179       this.mobCompactThread.close();
1180     }
1181   }
1182 
1183   /**
1184    * @return Get remote side's InetAddress
1185    * @throws UnknownHostException
1186    */
1187   InetAddress getRemoteInetAddress(final int port,
1188       final long serverStartCode) throws UnknownHostException {
1189     // Do it out here in its own little method so can fake an address when
1190     // mocking up in tests.
1191     InetAddress ia = RpcServer.getRemoteIp();
1192 
1193     // The call could be from the local regionserver,
1194     // in which case, there is no remote address.
1195     if (ia == null && serverStartCode == startcode) {
1196       InetSocketAddress isa = rpcServices.getSocketAddress();
1197       if (isa != null && isa.getPort() == port) {
1198         ia = isa.getAddress();
1199       }
1200     }
1201     return ia;
1202   }
1203 
1204   /**
1205    * @return Maximum time we should run balancer for
1206    */
1207   private int getBalancerCutoffTime() {
1208     int balancerCutoffTime = getConfiguration().getInt("hbase.balancer.max.balancing", -1);
1209     if (balancerCutoffTime == -1) {
1210       // if cutoff time isn't set, defaulting it to period time
1211       int balancerPeriod = getConfiguration().getInt("hbase.balancer.period", 300000);
1212       balancerCutoffTime = balancerPeriod;
1213     }
1214     return balancerCutoffTime;
1215   }
1216 
1217   public boolean balance() throws IOException {
1218     // if master not initialized, don't run balancer.
1219     if (!this.initialized) {
1220       LOG.debug("Master has not been initialized, don't run balancer.");
1221       return false;
1222     }
1223     // Do this call outside of synchronized block.
1224     int maximumBalanceTime = getBalancerCutoffTime();
1225     synchronized (this.balancer) {
1226       // If balance not true, don't run balancer.
1227       if (!this.loadBalancerTracker.isBalancerOn()) return false;
1228       // Only allow one balance run at at time.
1229       if (this.assignmentManager.getRegionStates().isRegionsInTransition()) {
1230         Map<String, RegionState> regionsInTransition =
1231           this.assignmentManager.getRegionStates().getRegionsInTransition();
1232         LOG.debug("Not running balancer because " + regionsInTransition.size() +
1233           " region(s) in transition: " + org.apache.commons.lang.StringUtils.
1234             abbreviate(regionsInTransition.toString(), 256));
1235         return false;
1236       }
1237       if (this.serverManager.areDeadServersInProgress()) {
1238         LOG.debug("Not running balancer because processing dead regionserver(s): " +
1239           this.serverManager.getDeadServers());
1240         return false;
1241       }
1242 
1243       if (this.cpHost != null) {
1244         try {
1245           if (this.cpHost.preBalance()) {
1246             LOG.debug("Coprocessor bypassing balancer request");
1247             return false;
1248           }
1249         } catch (IOException ioe) {
1250           LOG.error("Error invoking master coprocessor preBalance()", ioe);
1251           return false;
1252         }
1253       }
1254 
1255       Map<TableName, Map<ServerName, List<HRegionInfo>>> assignmentsByTable =
1256         this.assignmentManager.getRegionStates().getAssignmentsByTable();
1257 
1258       List<RegionPlan> plans = new ArrayList<RegionPlan>();
1259       //Give the balancer the current cluster state.
1260       this.balancer.setClusterStatus(getClusterStatus());
1261       for (Map<ServerName, List<HRegionInfo>> assignments : assignmentsByTable.values()) {
1262         List<RegionPlan> partialPlans = this.balancer.balanceCluster(assignments);
1263         if (partialPlans != null) plans.addAll(partialPlans);
1264       }
1265       long cutoffTime = System.currentTimeMillis() + maximumBalanceTime;
1266       int rpCount = 0;  // number of RegionPlans balanced so far
1267       long totalRegPlanExecTime = 0;
1268       if (plans != null && !plans.isEmpty()) {
1269         for (RegionPlan plan: plans) {
1270           LOG.info("balance " + plan);
1271           long balStartTime = System.currentTimeMillis();
1272           //TODO: bulk assign
1273           this.assignmentManager.balance(plan);
1274           totalRegPlanExecTime += System.currentTimeMillis()-balStartTime;
1275           rpCount++;
1276           if (rpCount < plans.size() &&
1277               // if performing next balance exceeds cutoff time, exit the loop
1278               (System.currentTimeMillis() + (totalRegPlanExecTime / rpCount)) > cutoffTime) {
1279             //TODO: After balance, there should not be a cutoff time (keeping it as
1280             // a security net for now)
1281             LOG.debug("No more balancing till next balance run; maximumBalanceTime=" +
1282               maximumBalanceTime);
1283             break;
1284           }
1285         }
1286       }
1287       if (this.cpHost != null) {
1288         try {
1289           this.cpHost.postBalance(rpCount < plans.size() ? plans.subList(0, rpCount) : plans);
1290         } catch (IOException ioe) {
1291           // balancing already succeeded so don't change the result
1292           LOG.error("Error invoking master coprocessor postBalance()", ioe);
1293         }
1294       }
1295     }
1296     // If LoadBalancer did not generate any plans, it means the cluster is already balanced.
1297     // Return true indicating a success.
1298     return true;
1299   }
1300 
1301   /**
1302    * Perform normalization of cluster (invoked by {@link RegionNormalizerChore}).
1303    *
1304    * @return true if normalization step was performed successfully, false otherwise
1305    *   (specifically, if HMaster hasn't been initialized properly or normalization
1306    *   is globally disabled)
1307    * @throws IOException
1308    */
1309   public boolean normalizeRegions() throws IOException {
1310     if (!this.initialized) {
1311       LOG.debug("Master has not been initialized, don't run region normalizer.");
1312       return false;
1313     }
1314 
1315     if (!this.normalizerEnabled) {
1316       LOG.debug("Region normalization is disabled, don't run region normalizer.");
1317       return false;
1318     }
1319 
1320     synchronized (this.normalizer) {
1321       // Don't run the normalizer concurrently
1322       List<TableName> allEnabledTables = new ArrayList<>(
1323         this.tableStateManager.getTablesInStates(TableState.State.ENABLED));
1324 
1325       Collections.shuffle(allEnabledTables);
1326 
1327       for (TableName table : allEnabledTables) {
1328         if (quotaManager.getNamespaceQuotaManager() != null &&
1329             quotaManager.getNamespaceQuotaManager().getState(table.getNamespaceAsString()) != null){
1330           LOG.debug("Skipping normalizing " + table + " since its namespace has quota");
1331           continue;
1332         }
1333         if (table.isSystemTable() || !getTableDescriptors().getDescriptor(table).
1334             getHTableDescriptor().isNormalizationEnabled()) {
1335           LOG.debug("Skipping normalization for table: " + table + ", as it's either system"
1336             + " table or doesn't have auto normalization turned on");
1337           continue;
1338         }
1339         this.normalizer.computePlanForTable(table).execute(clusterConnection.getAdmin());
1340       }
1341     }
1342     // If Region did not generate any plans, it means the cluster is already balanced.
1343     // Return true indicating a success.
1344     return true;
1345   }
1346 
1347   /**
1348    * @return Client info for use as prefix on an audit log string; who did an action
1349    */
1350   String getClientIdAuditPrefix() {
1351     return "Client=" + RpcServer.getRequestUserName() + "/" + RpcServer.getRemoteAddress();
1352   }
1353 
1354   /**
1355    * Switch for the background CatalogJanitor thread.
1356    * Used for testing.  The thread will continue to run.  It will just be a noop
1357    * if disabled.
1358    * @param b If false, the catalog janitor won't do anything.
1359    */
1360   public void setCatalogJanitorEnabled(final boolean b) {
1361     this.catalogJanitorChore.setEnabled(b);
1362   }
1363 
1364   @Override
1365   public void dispatchMergingRegions(final HRegionInfo region_a,
1366       final HRegionInfo region_b, final boolean forcible) throws IOException {
1367     checkInitialized();
1368     this.service.submit(new DispatchMergingRegionHandler(this,
1369       this.catalogJanitorChore, region_a, region_b, forcible));
1370   }
1371 
1372   void move(final byte[] encodedRegionName,
1373       final byte[] destServerName) throws HBaseIOException {
1374     RegionState regionState = assignmentManager.getRegionStates().
1375       getRegionState(Bytes.toString(encodedRegionName));
1376     if (regionState == null) {
1377       throw new UnknownRegionException(Bytes.toStringBinary(encodedRegionName));
1378     }
1379 
1380     HRegionInfo hri = regionState.getRegion();
1381     ServerName dest;
1382     if (destServerName == null || destServerName.length == 0) {
1383       LOG.info("Passed destination servername is null/empty so " +
1384         "choosing a server at random");
1385       final List<ServerName> destServers = this.serverManager.createDestinationServersList(
1386         regionState.getServerName());
1387       dest = balancer.randomAssignment(hri, destServers);
1388       if (dest == null) {
1389         LOG.debug("Unable to determine a plan to assign " + hri);
1390         return;
1391       }
1392     } else {
1393       dest = ServerName.valueOf(Bytes.toString(destServerName));
1394       if (dest.equals(serverName) && balancer instanceof BaseLoadBalancer
1395           && !((BaseLoadBalancer)balancer).shouldBeOnMaster(hri)) {
1396         // To avoid unnecessary region moving later by balancer. Don't put user
1397         // regions on master. Regions on master could be put on other region
1398         // server intentionally by test however.
1399         LOG.debug("Skipping move of region " + hri.getRegionNameAsString()
1400           + " to avoid unnecessary region moving later by load balancer,"
1401           + " because it should not be on master");
1402         return;
1403       }
1404     }
1405 
1406     if (dest.equals(regionState.getServerName())) {
1407       LOG.debug("Skipping move of region " + hri.getRegionNameAsString()
1408         + " because region already assigned to the same server " + dest + ".");
1409       return;
1410     }
1411 
1412     // Now we can do the move
1413     RegionPlan rp = new RegionPlan(hri, regionState.getServerName(), dest);
1414 
1415     try {
1416       checkInitialized();
1417       if (this.cpHost != null) {
1418         if (this.cpHost.preMove(hri, rp.getSource(), rp.getDestination())) {
1419           return;
1420         }
1421       }
1422       // warmup the region on the destination before initiating the move. this call
1423       // is synchronous and takes some time. doing it before the source region gets
1424       // closed
1425       serverManager.sendRegionWarmup(rp.getDestination(), hri);
1426 
1427       LOG.info(getClientIdAuditPrefix() + " move " + rp + ", running balancer");
1428       this.assignmentManager.balance(rp);
1429       if (this.cpHost != null) {
1430         this.cpHost.postMove(hri, rp.getSource(), rp.getDestination());
1431       }
1432     } catch (IOException ioe) {
1433       if (ioe instanceof HBaseIOException) {
1434         throw (HBaseIOException)ioe;
1435       }
1436       throw new HBaseIOException(ioe);
1437     }
1438   }
1439 
1440   @Override
1441   public long createTable(
1442       final HTableDescriptor hTableDescriptor,
1443       final byte [][] splitKeys,
1444       final long nonceGroup,
1445       final long nonce) throws IOException {
1446     if (isStopped()) {
1447       throw new MasterNotRunningException();
1448     }
1449 
1450     String namespace = hTableDescriptor.getTableName().getNamespaceAsString();
1451     ensureNamespaceExists(namespace);
1452 
1453     HRegionInfo[] newRegions = ModifyRegionUtils.createHRegionInfos(hTableDescriptor, splitKeys);
1454     checkInitialized();
1455     sanityCheckTableDescriptor(hTableDescriptor);
1456 
1457     if (cpHost != null) {
1458       cpHost.preCreateTable(hTableDescriptor, newRegions);
1459     }
1460     LOG.info(getClientIdAuditPrefix() + " create " + hTableDescriptor);
1461 
1462     // TODO: We can handle/merge duplicate requests, and differentiate the case of
1463     //       TableExistsException by saying if the schema is the same or not.
1464     ProcedurePrepareLatch latch = ProcedurePrepareLatch.createLatch();
1465     long procId = this.procedureExecutor.submitProcedure(
1466       new CreateTableProcedure(
1467         procedureExecutor.getEnvironment(), hTableDescriptor, newRegions, latch),
1468       nonceGroup,
1469       nonce);
1470     latch.await();
1471 
1472     if (cpHost != null) {
1473       cpHost.postCreateTable(hTableDescriptor, newRegions);
1474     }
1475 
1476     return procId;
1477   }
1478 
1479   /**
1480    * Checks whether the table conforms to some sane limits, and configured
1481    * values (compression, etc) work. Throws an exception if something is wrong.
1482    * @throws IOException
1483    */
1484   private void sanityCheckTableDescriptor(final HTableDescriptor htd) throws IOException {
1485     final String CONF_KEY = "hbase.table.sanity.checks";
1486     boolean logWarn = false;
1487     if (!conf.getBoolean(CONF_KEY, true)) {
1488       logWarn = true;
1489     }
1490     String tableVal = htd.getConfigurationValue(CONF_KEY);
1491     if (tableVal != null && !Boolean.valueOf(tableVal)) {
1492       logWarn = true;
1493     }
1494 
1495     // check max file size
1496     long maxFileSizeLowerLimit = 2 * 1024 * 1024L; // 2M is the default lower limit
1497     long maxFileSize = htd.getMaxFileSize();
1498     if (maxFileSize < 0) {
1499       maxFileSize = conf.getLong(HConstants.HREGION_MAX_FILESIZE, maxFileSizeLowerLimit);
1500     }
1501     if (maxFileSize < conf.getLong("hbase.hregion.max.filesize.limit", maxFileSizeLowerLimit)) {
1502       String message = "MAX_FILESIZE for table descriptor or "
1503           + "\"hbase.hregion.max.filesize\" (" + maxFileSize
1504           + ") is too small, which might cause over splitting into unmanageable "
1505           + "number of regions.";
1506       warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);
1507     }
1508 
1509     // check flush size
1510     long flushSizeLowerLimit = 1024 * 1024L; // 1M is the default lower limit
1511     long flushSize = htd.getMemStoreFlushSize();
1512     if (flushSize < 0) {
1513       flushSize = conf.getLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, flushSizeLowerLimit);
1514     }
1515     if (flushSize < conf.getLong("hbase.hregion.memstore.flush.size.limit", flushSizeLowerLimit)) {
1516       String message = "MEMSTORE_FLUSHSIZE for table descriptor or "
1517           + "\"hbase.hregion.memstore.flush.size\" ("+flushSize+") is too small, which might cause"
1518           + " very frequent flushing.";
1519       warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);
1520     }
1521 
1522     // check that coprocessors and other specified plugin classes can be loaded
1523     try {
1524       checkClassLoading(conf, htd);
1525     } catch (Exception ex) {
1526       warnOrThrowExceptionForFailure(logWarn, CONF_KEY, ex.getMessage(), null);
1527     }
1528 
1529     // check compression can be loaded
1530     try {
1531       checkCompression(htd);
1532     } catch (IOException e) {
1533       warnOrThrowExceptionForFailure(logWarn, CONF_KEY, e.getMessage(), e);
1534     }
1535 
1536     // check encryption can be loaded
1537     try {
1538       checkEncryption(conf, htd);
1539     } catch (IOException e) {
1540       warnOrThrowExceptionForFailure(logWarn, CONF_KEY, e.getMessage(), e);
1541     }
1542 
1543     // check that we have at least 1 CF
1544     if (htd.getColumnFamilies().length == 0) {
1545       String message = "Table should have at least one column family.";
1546       warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);
1547     }
1548 
1549     for (HColumnDescriptor hcd : htd.getColumnFamilies()) {
1550       if (hcd.getTimeToLive() <= 0) {
1551         String message = "TTL for column family " + hcd.getNameAsString() + " must be positive.";
1552         warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);
1553       }
1554 
1555       // check blockSize
1556       if (hcd.getBlocksize() < 1024 || hcd.getBlocksize() > 16 * 1024 * 1024) {
1557         String message = "Block size for column family " + hcd.getNameAsString()
1558             + "  must be between 1K and 16MB.";
1559         warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);
1560       }
1561 
1562       // check versions
1563       if (hcd.getMinVersions() < 0) {
1564         String message = "Min versions for column family " + hcd.getNameAsString()
1565           + "  must be positive.";
1566         warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);
1567       }
1568       // max versions already being checked
1569 
1570       // HBASE-13776 Setting illegal versions for HColumnDescriptor
1571       //  does not throw IllegalArgumentException
1572       // check minVersions <= maxVerions
1573       if (hcd.getMinVersions() > hcd.getMaxVersions()) {
1574         String message = "Min versions for column family " + hcd.getNameAsString()
1575             + " must be less than the Max versions.";
1576         warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);
1577       }
1578 
1579       // check replication scope
1580       if (hcd.getScope() < 0) {
1581         String message = "Replication scope for column family "
1582           + hcd.getNameAsString() + "  must be positive.";
1583         warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);
1584       }
1585 
1586       // TODO: should we check coprocessors and encryption ?
1587     }
1588   }
1589 
1590   // HBASE-13350 - Helper method to log warning on sanity check failures if checks disabled.
1591   private static void warnOrThrowExceptionForFailure(boolean logWarn, String confKey,
1592       String message, Exception cause) throws IOException {
1593     if (!logWarn) {
1594       throw new DoNotRetryIOException(message + " Set " + confKey +
1595           " to false at conf or table descriptor if you want to bypass sanity checks", cause);
1596     }
1597     LOG.warn(message);
1598   }
1599 
1600   private void startActiveMasterManager(int infoPort) throws KeeperException {
1601     String backupZNode = ZKUtil.joinZNode(
1602       zooKeeper.backupMasterAddressesZNode, serverName.toString());
1603     /*
1604     * Add a ZNode for ourselves in the backup master directory since we
1605     * may not become the active master. If so, we want the actual active
1606     * master to know we are backup masters, so that it won't assign
1607     * regions to us if so configured.
1608     *
1609     * If we become the active master later, ActiveMasterManager will delete
1610     * this node explicitly.  If we crash before then, ZooKeeper will delete
1611     * this node for us since it is ephemeral.
1612     */
1613     LOG.info("Adding backup master ZNode " + backupZNode);
1614     if (!MasterAddressTracker.setMasterAddress(zooKeeper, backupZNode,
1615         serverName, infoPort)) {
1616       LOG.warn("Failed create of " + backupZNode + " by " + serverName);
1617     }
1618 
1619     activeMasterManager.setInfoPort(infoPort);
1620     // Start a thread to try to become the active master, so we won't block here
1621     Threads.setDaemonThreadRunning(new Thread(new Runnable() {
1622       @Override
1623       public void run() {
1624         int timeout = conf.getInt(HConstants.ZK_SESSION_TIMEOUT,
1625           HConstants.DEFAULT_ZK_SESSION_TIMEOUT);
1626         // If we're a backup master, stall until a primary to writes his address
1627         if (conf.getBoolean(HConstants.MASTER_TYPE_BACKUP,
1628           HConstants.DEFAULT_MASTER_TYPE_BACKUP)) {
1629           LOG.debug("HMaster started in backup mode. "
1630             + "Stalling until master znode is written.");
1631           // This will only be a minute or so while the cluster starts up,
1632           // so don't worry about setting watches on the parent znode
1633           while (!activeMasterManager.hasActiveMaster()) {
1634             LOG.debug("Waiting for master address ZNode to be written "
1635               + "(Also watching cluster state node)");
1636             Threads.sleep(timeout);
1637           }
1638         }
1639         MonitoredTask status = TaskMonitor.get().createStatus("Master startup");
1640         status.setDescription("Master startup");
1641         try {
1642           if (activeMasterManager.blockUntilBecomingActiveMaster(timeout, status)) {
1643             finishActiveMasterInitialization(status);
1644           }
1645         } catch (Throwable t) {
1646           status.setStatus("Failed to become active: " + t.getMessage());
1647           LOG.fatal("Failed to become active master", t);
1648           // HBASE-5680: Likely hadoop23 vs hadoop 20.x/1.x incompatibility
1649           if (t instanceof NoClassDefFoundError &&
1650             t.getMessage()
1651               .contains("org/apache/hadoop/hdfs/protocol/HdfsConstants$SafeModeAction")) {
1652             // improved error message for this special case
1653             abort("HBase is having a problem with its Hadoop jars.  You may need to "
1654               + "recompile HBase against Hadoop version "
1655               + org.apache.hadoop.util.VersionInfo.getVersion()
1656               + " or change your hadoop jars to start properly", t);
1657           } else {
1658             abort("Unhandled exception. Starting shutdown.", t);
1659           }
1660         } finally {
1661           status.cleanup();
1662         }
1663       }
1664     }, getServerName().toShortString() + ".activeMasterManager"));
1665   }
1666 
1667   private void checkCompression(final HTableDescriptor htd)
1668   throws IOException {
1669     if (!this.masterCheckCompression) return;
1670     for (HColumnDescriptor hcd : htd.getColumnFamilies()) {
1671       checkCompression(hcd);
1672     }
1673   }
1674 
1675   private void checkCompression(final HColumnDescriptor hcd)
1676   throws IOException {
1677     if (!this.masterCheckCompression) return;
1678     CompressionTest.testCompression(hcd.getCompressionType());
1679     CompressionTest.testCompression(hcd.getCompactionCompressionType());
1680   }
1681 
1682   private void checkEncryption(final Configuration conf, final HTableDescriptor htd)
1683   throws IOException {
1684     if (!this.masterCheckEncryption) return;
1685     for (HColumnDescriptor hcd : htd.getColumnFamilies()) {
1686       checkEncryption(conf, hcd);
1687     }
1688   }
1689 
1690   private void checkEncryption(final Configuration conf, final HColumnDescriptor hcd)
1691   throws IOException {
1692     if (!this.masterCheckEncryption) return;
1693     EncryptionTest.testEncryption(conf, hcd.getEncryptionType(), hcd.getEncryptionKey());
1694   }
1695 
1696   private void checkClassLoading(final Configuration conf, final HTableDescriptor htd)
1697   throws IOException {
1698     RegionSplitPolicy.getSplitPolicyClass(htd, conf);
1699     RegionCoprocessorHost.testTableCoprocessorAttrs(conf, htd);
1700   }
1701 
1702   private static boolean isCatalogTable(final TableName tableName) {
1703     return tableName.equals(TableName.META_TABLE_NAME);
1704   }
1705 
1706   @Override
1707   public long deleteTable(
1708       final TableName tableName,
1709       final long nonceGroup,
1710       final long nonce) throws IOException {
1711     checkInitialized();
1712     if (cpHost != null) {
1713       cpHost.preDeleteTable(tableName);
1714     }
1715     LOG.info(getClientIdAuditPrefix() + " delete " + tableName);
1716 
1717     // TODO: We can handle/merge duplicate request
1718     ProcedurePrepareLatch latch = ProcedurePrepareLatch.createLatch();
1719     long procId = this.procedureExecutor.submitProcedure(
1720       new DeleteTableProcedure(procedureExecutor.getEnvironment(), tableName, latch),
1721       nonceGroup,
1722       nonce);
1723     latch.await();
1724 
1725     if (cpHost != null) {
1726       cpHost.postDeleteTable(tableName);
1727     }
1728 
1729     return procId;
1730   }
1731 
1732   @Override
1733   public long truncateTable(
1734       final TableName tableName,
1735       final boolean preserveSplits,
1736       final long nonceGroup,
1737       final long nonce) throws IOException {
1738     checkInitialized();
1739     if (cpHost != null) {
1740       cpHost.preTruncateTable(tableName);
1741     }
1742     LOG.info(getClientIdAuditPrefix() + " truncate " + tableName);
1743 
1744     long procId = this.procedureExecutor.submitProcedure(
1745       new TruncateTableProcedure(procedureExecutor.getEnvironment(), tableName, preserveSplits),
1746       nonceGroup,
1747       nonce);
1748     ProcedureSyncWait.waitForProcedureToComplete(procedureExecutor, procId);
1749 
1750     if (cpHost != null) {
1751       cpHost.postTruncateTable(tableName);
1752     }
1753     return procId;
1754   }
1755 
1756   @Override
1757   public void addColumn(
1758       final TableName tableName,
1759       final HColumnDescriptor columnDescriptor,
1760       final long nonceGroup,
1761       final long nonce)
1762       throws IOException {
1763     checkInitialized();
1764     checkCompression(columnDescriptor);
1765     checkEncryption(conf, columnDescriptor);
1766     if (cpHost != null) {
1767       if (cpHost.preAddColumn(tableName, columnDescriptor)) {
1768         return;
1769       }
1770     }
1771     // Execute the operation synchronously - wait for the operation to complete before continuing.
1772     long procId = this.procedureExecutor.submitProcedure(
1773       new AddColumnFamilyProcedure(procedureExecutor.getEnvironment(), tableName, columnDescriptor),
1774       nonceGroup,
1775       nonce);
1776     ProcedureSyncWait.waitForProcedureToComplete(procedureExecutor, procId);
1777     if (cpHost != null) {
1778       cpHost.postAddColumn(tableName, columnDescriptor);
1779     }
1780   }
1781 
1782   @Override
1783   public void modifyColumn(
1784       final TableName tableName,
1785       final HColumnDescriptor descriptor,
1786       final long nonceGroup,
1787       final long nonce)
1788       throws IOException {
1789     checkInitialized();
1790     checkCompression(descriptor);
1791     checkEncryption(conf, descriptor);
1792     if (cpHost != null) {
1793       if (cpHost.preModifyColumn(tableName, descriptor)) {
1794         return;
1795       }
1796     }
1797     LOG.info(getClientIdAuditPrefix() + " modify " + descriptor);
1798 
1799     // Execute the operation synchronously - wait for the operation to complete before continuing.
1800     long procId = this.procedureExecutor.submitProcedure(
1801       new ModifyColumnFamilyProcedure(procedureExecutor.getEnvironment(), tableName, descriptor),
1802       nonceGroup,
1803       nonce);
1804     ProcedureSyncWait.waitForProcedureToComplete(procedureExecutor, procId);
1805 
1806     if (cpHost != null) {
1807       cpHost.postModifyColumn(tableName, descriptor);
1808     }
1809   }
1810 
1811   @Override
1812   public void deleteColumn(
1813       final TableName tableName,
1814       final byte[] columnName,
1815       final long nonceGroup,
1816       final long nonce)
1817       throws IOException {
1818     checkInitialized();
1819     if (cpHost != null) {
1820       if (cpHost.preDeleteColumn(tableName, columnName)) {
1821         return;
1822       }
1823     }
1824     LOG.info(getClientIdAuditPrefix() + " delete " + Bytes.toString(columnName));
1825 
1826     // Execute the operation synchronously - wait for the operation to complete before continuing.
1827     long procId = this.procedureExecutor.submitProcedure(
1828       new DeleteColumnFamilyProcedure(procedureExecutor.getEnvironment(), tableName, columnName),
1829       nonceGroup,
1830       nonce);
1831     ProcedureSyncWait.waitForProcedureToComplete(procedureExecutor, procId);
1832 
1833     if (cpHost != null) {
1834       cpHost.postDeleteColumn(tableName, columnName);
1835     }
1836   }
1837 
1838   @Override
1839   public long enableTable(
1840       final TableName tableName,
1841       final long nonceGroup,
1842       final long nonce) throws IOException {
1843     checkInitialized();
1844     if (cpHost != null) {
1845       cpHost.preEnableTable(tableName);
1846     }
1847     LOG.info(getClientIdAuditPrefix() + " enable " + tableName);
1848 
1849     // Execute the operation asynchronously - client will check the progress of the operation
1850     final ProcedurePrepareLatch prepareLatch = ProcedurePrepareLatch.createLatch();
1851     long procId = this.procedureExecutor.submitProcedure(
1852       new EnableTableProcedure(procedureExecutor.getEnvironment(), tableName, false, prepareLatch),
1853       nonceGroup,
1854       nonce);
1855     // Before returning to client, we want to make sure that the table is prepared to be
1856     // enabled (the table is locked and the table state is set).
1857     //
1858     // Note: if the procedure throws exception, we will catch it and rethrow.
1859     prepareLatch.await();
1860 
1861     if (cpHost != null) {
1862       cpHost.postEnableTable(tableName);
1863     }
1864 
1865     return procId;
1866   }
1867 
1868   @Override
1869   public long disableTable(
1870       final TableName tableName,
1871       final long nonceGroup,
1872       final long nonce) throws IOException {
1873     checkInitialized();
1874     if (cpHost != null) {
1875       cpHost.preDisableTable(tableName);
1876     }
1877     LOG.info(getClientIdAuditPrefix() + " disable " + tableName);
1878 
1879     // Execute the operation asynchronously - client will check the progress of the operation
1880     final ProcedurePrepareLatch prepareLatch = ProcedurePrepareLatch.createLatch();
1881     // Execute the operation asynchronously - client will check the progress of the operation
1882     long procId = this.procedureExecutor.submitProcedure(
1883       new DisableTableProcedure(procedureExecutor.getEnvironment(), tableName, false, prepareLatch),
1884       nonceGroup,
1885       nonce);
1886     // Before returning to client, we want to make sure that the table is prepared to be
1887     // enabled (the table is locked and the table state is set).
1888     //
1889     // Note: if the procedure throws exception, we will catch it and rethrow.
1890     prepareLatch.await();
1891 
1892     if (cpHost != null) {
1893       cpHost.postDisableTable(tableName);
1894     }
1895 
1896     return procId;
1897   }
1898 
1899   /**
1900    * Return the region and current deployment for the region containing
1901    * the given row. If the region cannot be found, returns null. If it
1902    * is found, but not currently deployed, the second element of the pair
1903    * may be null.
1904    */
1905   @VisibleForTesting // Used by TestMaster.
1906   Pair<HRegionInfo, ServerName> getTableRegionForRow(
1907       final TableName tableName, final byte [] rowKey)
1908   throws IOException {
1909     final AtomicReference<Pair<HRegionInfo, ServerName>> result =
1910       new AtomicReference<Pair<HRegionInfo, ServerName>>(null);
1911 
1912     MetaTableAccessor.Visitor visitor = new MetaTableAccessor.Visitor() {
1913         @Override
1914         public boolean visit(Result data) throws IOException {
1915           if (data == null || data.size() <= 0) {
1916             return true;
1917           }
1918           Pair<HRegionInfo, ServerName> pair = HRegionInfo.getHRegionInfoAndServerName(data);
1919           if (pair == null) {
1920             return false;
1921           }
1922           if (!pair.getFirst().getTable().equals(tableName)) {
1923             return false;
1924           }
1925           result.set(pair);
1926           return true;
1927         }
1928     };
1929 
1930     MetaTableAccessor.scanMeta(clusterConnection, visitor, tableName, rowKey, 1);
1931     return result.get();
1932   }
1933 
1934   @Override
1935   public long modifyTable(
1936       final TableName tableName,
1937       final HTableDescriptor descriptor,
1938       final long nonceGroup,
1939       final long nonce)
1940       throws IOException {
1941     checkInitialized();
1942     sanityCheckTableDescriptor(descriptor);
1943     if (cpHost != null) {
1944       cpHost.preModifyTable(tableName, descriptor);
1945     }
1946 
1947     LOG.info(getClientIdAuditPrefix() + " modify " + tableName);
1948 
1949     // Execute the operation synchronously - wait for the operation completes before continuing.
1950     long procId = this.procedureExecutor.submitProcedure(
1951       new ModifyTableProcedure(procedureExecutor.getEnvironment(), descriptor),
1952       nonceGroup,
1953       nonce);
1954 
1955     ProcedureSyncWait.waitForProcedureToComplete(procedureExecutor, procId);
1956 
1957     if (cpHost != null) {
1958       cpHost.postModifyTable(tableName, descriptor);
1959     }
1960 
1961     return procId;
1962   }
1963 
1964   @Override
1965   public void checkTableModifiable(final TableName tableName)
1966       throws IOException, TableNotFoundException, TableNotDisabledException {
1967     if (isCatalogTable(tableName)) {
1968       throw new IOException("Can't modify catalog tables");
1969     }
1970     if (!MetaTableAccessor.tableExists(getConnection(), tableName)) {
1971       throw new TableNotFoundException(tableName);
1972     }
1973     if (!getAssignmentManager().getTableStateManager().
1974         isTableState(tableName, TableState.State.DISABLED)) {
1975       throw new TableNotDisabledException(tableName);
1976     }
1977   }
1978 
1979   /**
1980    * @return cluster status
1981    */
1982   public ClusterStatus getClusterStatus() throws InterruptedIOException {
1983     // Build Set of backup masters from ZK nodes
1984     List<String> backupMasterStrings;
1985     try {
1986       backupMasterStrings = ZKUtil.listChildrenNoWatch(this.zooKeeper,
1987         this.zooKeeper.backupMasterAddressesZNode);
1988     } catch (KeeperException e) {
1989       LOG.warn(this.zooKeeper.prefix("Unable to list backup servers"), e);
1990       backupMasterStrings = null;
1991     }
1992 
1993     List<ServerName> backupMasters = null;
1994     if (backupMasterStrings != null && !backupMasterStrings.isEmpty()) {
1995       backupMasters = new ArrayList<ServerName>(backupMasterStrings.size());
1996       for (String s: backupMasterStrings) {
1997         try {
1998           byte [] bytes;
1999           try {
2000             bytes = ZKUtil.getData(this.zooKeeper, ZKUtil.joinZNode(
2001                 this.zooKeeper.backupMasterAddressesZNode, s));
2002           } catch (InterruptedException e) {
2003             throw new InterruptedIOException();
2004           }
2005           if (bytes != null) {
2006             ServerName sn;
2007             try {
2008               sn = ServerName.parseFrom(bytes);
2009             } catch (DeserializationException e) {
2010               LOG.warn("Failed parse, skipping registering backup server", e);
2011               continue;
2012             }
2013             backupMasters.add(sn);
2014           }
2015         } catch (KeeperException e) {
2016           LOG.warn(this.zooKeeper.prefix("Unable to get information about " +
2017                    "backup servers"), e);
2018         }
2019       }
2020       Collections.sort(backupMasters, new Comparator<ServerName>() {
2021         @Override
2022         public int compare(ServerName s1, ServerName s2) {
2023           return s1.getServerName().compareTo(s2.getServerName());
2024         }});
2025     }
2026 
2027     String clusterId = fileSystemManager != null ?
2028       fileSystemManager.getClusterId().toString() : null;
2029     Map<String, RegionState> regionsInTransition = assignmentManager != null ?
2030       assignmentManager.getRegionStates().getRegionsInTransition() : null;
2031     String[] coprocessors = cpHost != null ? getMasterCoprocessors() : null;
2032     boolean balancerOn = loadBalancerTracker != null ?
2033       loadBalancerTracker.isBalancerOn() : false;
2034     Map<ServerName, ServerLoad> onlineServers = null;
2035     Set<ServerName> deadServers = null;
2036     if (serverManager != null) {
2037       deadServers = serverManager.getDeadServers().copyServerNames();
2038       onlineServers = serverManager.getOnlineServers();
2039     }
2040     return new ClusterStatus(VersionInfo.getVersion(), clusterId,
2041       onlineServers, deadServers, serverName, backupMasters,
2042       regionsInTransition, coprocessors, balancerOn);
2043   }
2044 
2045   /**
2046    * The set of loaded coprocessors is stored in a static set. Since it's
2047    * statically allocated, it does not require that HMaster's cpHost be
2048    * initialized prior to accessing it.
2049    * @return a String representation of the set of names of the loaded
2050    * coprocessors.
2051    */
2052   public static String getLoadedCoprocessors() {
2053     return CoprocessorHost.getLoadedCoprocessors().toString();
2054   }
2055 
2056   /**
2057    * @return timestamp in millis when HMaster was started.
2058    */
2059   public long getMasterStartTime() {
2060     return startcode;
2061   }
2062 
2063   /**
2064    * @return timestamp in millis when HMaster became the active master.
2065    */
2066   public long getMasterActiveTime() {
2067     return masterActiveTime;
2068   }
2069 
2070   public int getRegionServerInfoPort(final ServerName sn) {
2071     RegionServerInfo info = this.regionServerTracker.getRegionServerInfo(sn);
2072     if (info == null || info.getInfoPort() == 0) {
2073       return conf.getInt(HConstants.REGIONSERVER_INFO_PORT,
2074         HConstants.DEFAULT_REGIONSERVER_INFOPORT);
2075     }
2076     return info.getInfoPort();
2077   }
2078 
2079   /**
2080    * @return array of coprocessor SimpleNames.
2081    */
2082   public String[] getMasterCoprocessors() {
2083     Set<String> masterCoprocessors = getMasterCoprocessorHost().getCoprocessors();
2084     return masterCoprocessors.toArray(new String[masterCoprocessors.size()]);
2085   }
2086 
2087   @Override
2088   public void abort(final String msg, final Throwable t) {
2089     if (isAborted() || isStopped()) {
2090       return;
2091     }
2092     if (cpHost != null) {
2093       // HBASE-4014: dump a list of loaded coprocessors.
2094       LOG.fatal("Master server abort: loaded coprocessors are: " +
2095           getLoadedCoprocessors());
2096     }
2097     if (t != null) LOG.fatal(msg, t);
2098     stop(msg);
2099   }
2100 
2101   @Override
2102   public ZooKeeperWatcher getZooKeeper() {
2103     return zooKeeper;
2104   }
2105 
2106   @Override
2107   public MasterCoprocessorHost getMasterCoprocessorHost() {
2108     return cpHost;
2109   }
2110 
2111   @Override
2112   public MasterQuotaManager getMasterQuotaManager() {
2113     return quotaManager;
2114   }
2115 
2116   @Override
2117   public ProcedureExecutor<MasterProcedureEnv> getMasterProcedureExecutor() {
2118     return procedureExecutor;
2119   }
2120 
2121   @Override
2122   public ServerName getServerName() {
2123     return this.serverName;
2124   }
2125 
2126   @Override
2127   public AssignmentManager getAssignmentManager() {
2128     return this.assignmentManager;
2129   }
2130 
2131   public MemoryBoundedLogMessageBuffer getRegionServerFatalLogBuffer() {
2132     return rsFatals;
2133   }
2134 
2135   public void shutdown() {
2136     if (cpHost != null) {
2137       try {
2138         cpHost.preShutdown();
2139       } catch (IOException ioe) {
2140         LOG.error("Error call master coprocessor preShutdown()", ioe);
2141       }
2142     }
2143 
2144     if (this.serverManager != null) {
2145       this.serverManager.shutdownCluster();
2146     }
2147     if (this.clusterStatusTracker != null){
2148       try {
2149         this.clusterStatusTracker.setClusterDown();
2150       } catch (KeeperException e) {
2151         LOG.error("ZooKeeper exception trying to set cluster as down in ZK", e);
2152       }
2153     }
2154   }
2155 
2156   public void stopMaster() {
2157     if (cpHost != null) {
2158       try {
2159         cpHost.preStopMaster();
2160       } catch (IOException ioe) {
2161         LOG.error("Error call master coprocessor preStopMaster()", ioe);
2162       }
2163     }
2164     stop("Stopped by " + Thread.currentThread().getName());
2165   }
2166 
2167   void checkServiceStarted() throws ServerNotRunningYetException {
2168     if (!serviceStarted) {
2169       throw new ServerNotRunningYetException("Server is not running yet");
2170     }
2171   }
2172 
2173   void checkInitialized() throws PleaseHoldException, ServerNotRunningYetException {
2174     checkServiceStarted();
2175     if (!this.initialized) {
2176       throw new PleaseHoldException("Master is initializing");
2177     }
2178   }
2179 
2180   void checkNamespaceManagerReady() throws IOException {
2181     checkInitialized();
2182     if (tableNamespaceManager == null ||
2183         !tableNamespaceManager.isTableAvailableAndInitialized()) {
2184       throw new IOException("Table Namespace Manager not ready yet, try again later");
2185     }
2186   }
2187   /**
2188    * Report whether this master is currently the active master or not.
2189    * If not active master, we are parked on ZK waiting to become active.
2190    *
2191    * This method is used for testing.
2192    *
2193    * @return true if active master, false if not.
2194    */
2195   public boolean isActiveMaster() {
2196     return isActiveMaster;
2197   }
2198 
2199   /**
2200    * Report whether this master has completed with its initialization and is
2201    * ready.  If ready, the master is also the active master.  A standby master
2202    * is never ready.
2203    *
2204    * This method is used for testing.
2205    *
2206    * @return true if master is ready to go, false if not.
2207    */
2208   @Override
2209   public boolean isInitialized() {
2210     return initialized;
2211   }
2212 
2213   /**
2214    * ServerCrashProcessingEnabled is set false before completing assignMeta to prevent processing
2215    * of crashed servers.
2216    * @return true if assignMeta has completed;
2217    */
2218   @Override
2219   public boolean isServerCrashProcessingEnabled() {
2220     return this.serverCrashProcessingEnabled;
2221   }
2222 
2223   @VisibleForTesting
2224   public void setServerCrashProcessingEnabled(final boolean b) {
2225     this.serverCrashProcessingEnabled = b;
2226   }
2227 
2228   /**
2229    * Report whether this master has started initialization and is about to do meta region assignment
2230    * @return true if master is in initialization &amp; about to assign hbase:meta regions
2231    */
2232   public boolean isInitializationStartsMetaRegionAssignment() {
2233     return this.initializationBeforeMetaAssignment;
2234   }
2235 
2236   public void assignRegion(HRegionInfo hri) {
2237     assignmentManager.assign(hri);
2238   }
2239 
2240   /**
2241    * Compute the average load across all region servers.
2242    * Currently, this uses a very naive computation - just uses the number of
2243    * regions being served, ignoring stats about number of requests.
2244    * @return the average load
2245    */
2246   public double getAverageLoad() {
2247     if (this.assignmentManager == null) {
2248       return 0;
2249     }
2250 
2251     RegionStates regionStates = this.assignmentManager.getRegionStates();
2252     if (regionStates == null) {
2253       return 0;
2254     }
2255     return regionStates.getAverageLoad();
2256   }
2257 
2258   @Override
2259   public boolean registerService(Service instance) {
2260     /*
2261      * No stacking of instances is allowed for a single service name
2262      */
2263     Descriptors.ServiceDescriptor serviceDesc = instance.getDescriptorForType();
2264     if (coprocessorServiceHandlers.containsKey(serviceDesc.getFullName())) {
2265       LOG.error("Coprocessor service "+serviceDesc.getFullName()+
2266           " already registered, rejecting request from "+instance
2267       );
2268       return false;
2269     }
2270 
2271     coprocessorServiceHandlers.put(serviceDesc.getFullName(), instance);
2272     if (LOG.isDebugEnabled()) {
2273       LOG.debug("Registered master coprocessor service: service="+serviceDesc.getFullName());
2274     }
2275     return true;
2276   }
2277 
2278   /**
2279    * Utility for constructing an instance of the passed HMaster class.
2280    * @param masterClass
2281    * @param conf
2282    * @return HMaster instance.
2283    */
2284   public static HMaster constructMaster(Class<? extends HMaster> masterClass,
2285       final Configuration conf, final CoordinatedStateManager cp)  {
2286     try {
2287       Constructor<? extends HMaster> c =
2288         masterClass.getConstructor(Configuration.class, CoordinatedStateManager.class);
2289       return c.newInstance(conf, cp);
2290     } catch (InvocationTargetException ite) {
2291       Throwable target = ite.getTargetException() != null?
2292         ite.getTargetException(): ite;
2293       if (target.getCause() != null) target = target.getCause();
2294       throw new RuntimeException("Failed construction of Master: " +
2295         masterClass.toString(), target);
2296     } catch (Exception e) {
2297       throw new RuntimeException("Failed construction of Master: " +
2298         masterClass.toString() + ((e.getCause() != null)?
2299           e.getCause().getMessage(): ""), e);
2300     }
2301   }
2302 
2303   /**
2304    * @see org.apache.hadoop.hbase.master.HMasterCommandLine
2305    */
2306   public static void main(String [] args) {
2307     VersionInfo.logVersion();
2308     new HMasterCommandLine(HMaster.class).doMain(args);
2309   }
2310 
2311   public HFileCleaner getHFileCleaner() {
2312     return this.hfileCleaner;
2313   }
2314 
2315   /**
2316    * Exposed for TESTING!
2317    * @return the underlying snapshot manager
2318    */
2319   public SnapshotManager getSnapshotManagerForTesting() {
2320     return this.snapshotManager;
2321   }
2322 
2323   @Override
2324   public void createNamespace(NamespaceDescriptor descriptor) throws IOException {
2325     TableName.isLegalNamespaceName(Bytes.toBytes(descriptor.getName()));
2326     checkNamespaceManagerReady();
2327     if (cpHost != null) {
2328       if (cpHost.preCreateNamespace(descriptor)) {
2329         return;
2330       }
2331     }
2332     LOG.info(getClientIdAuditPrefix() + " creating " + descriptor);
2333     tableNamespaceManager.create(descriptor);
2334     if (cpHost != null) {
2335       cpHost.postCreateNamespace(descriptor);
2336     }
2337   }
2338 
2339   @Override
2340   public void modifyNamespace(NamespaceDescriptor descriptor) throws IOException {
2341     TableName.isLegalNamespaceName(Bytes.toBytes(descriptor.getName()));
2342     checkNamespaceManagerReady();
2343     if (cpHost != null) {
2344       if (cpHost.preModifyNamespace(descriptor)) {
2345         return;
2346       }
2347     }
2348     LOG.info(getClientIdAuditPrefix() + " modify " + descriptor);
2349     tableNamespaceManager.update(descriptor);
2350     if (cpHost != null) {
2351       cpHost.postModifyNamespace(descriptor);
2352     }
2353   }
2354 
2355   @Override
2356   public void deleteNamespace(String name) throws IOException {
2357     checkNamespaceManagerReady();
2358     if (cpHost != null) {
2359       if (cpHost.preDeleteNamespace(name)) {
2360         return;
2361       }
2362     }
2363     LOG.info(getClientIdAuditPrefix() + " delete " + name);
2364     tableNamespaceManager.remove(name);
2365     if (cpHost != null) {
2366       cpHost.postDeleteNamespace(name);
2367     }
2368   }
2369 
2370   /**
2371    * Ensure that the specified namespace exists, otherwise throws a NamespaceNotFoundException
2372    *
2373    * @param name the namespace to check
2374    * @throws IOException if the namespace manager is not ready yet.
2375    * @throws NamespaceNotFoundException if the namespace does not exists
2376    */
2377   private void ensureNamespaceExists(final String name)
2378       throws IOException, NamespaceNotFoundException {
2379     checkNamespaceManagerReady();
2380     NamespaceDescriptor nsd = tableNamespaceManager.get(name);
2381     if (nsd == null) {
2382       throw new NamespaceNotFoundException(name);
2383     }
2384   }
2385 
2386   @Override
2387   public NamespaceDescriptor getNamespaceDescriptor(String name) throws IOException {
2388     checkNamespaceManagerReady();
2389 
2390     if (cpHost != null) {
2391       cpHost.preGetNamespaceDescriptor(name);
2392     }
2393 
2394     NamespaceDescriptor nsd = tableNamespaceManager.get(name);
2395     if (nsd == null) {
2396       throw new NamespaceNotFoundException(name);
2397     }
2398 
2399     if (cpHost != null) {
2400       cpHost.postGetNamespaceDescriptor(nsd);
2401     }
2402 
2403     return nsd;
2404   }
2405 
2406   @Override
2407   public List<NamespaceDescriptor> listNamespaceDescriptors() throws IOException {
2408     checkNamespaceManagerReady();
2409 
2410     final List<NamespaceDescriptor> descriptors = new ArrayList<NamespaceDescriptor>();
2411     boolean bypass = false;
2412     if (cpHost != null) {
2413       bypass = cpHost.preListNamespaceDescriptors(descriptors);
2414     }
2415 
2416     if (!bypass) {
2417       descriptors.addAll(tableNamespaceManager.list());
2418 
2419       if (cpHost != null) {
2420         cpHost.postListNamespaceDescriptors(descriptors);
2421       }
2422     }
2423     return descriptors;
2424   }
2425 
2426   @Override
2427   public List<HTableDescriptor> listTableDescriptorsByNamespace(String name) throws IOException {
2428     ensureNamespaceExists(name);
2429     return listTableDescriptors(name, null, null, true);
2430   }
2431 
2432   @Override
2433   public List<TableName> listTableNamesByNamespace(String name) throws IOException {
2434     ensureNamespaceExists(name);
2435     return listTableNames(name, null, true);
2436   }
2437 
2438   /**
2439    * Returns the list of table descriptors that match the specified request
2440    *
2441    * @param namespace the namespace to query, or null if querying for all
2442    * @param regex The regular expression to match against, or null if querying for all
2443    * @param tableNameList the list of table names, or null if querying for all
2444    * @param includeSysTables False to match only against userspace tables
2445    * @return the list of table descriptors
2446    */
2447   public List<HTableDescriptor> listTableDescriptors(final String namespace, final String regex,
2448       final List<TableName> tableNameList, final boolean includeSysTables)
2449       throws IOException {
2450     final List<HTableDescriptor> descriptors = new ArrayList<HTableDescriptor>();
2451 
2452     boolean bypass = false;
2453     if (cpHost != null) {
2454       bypass = cpHost.preGetTableDescriptors(tableNameList, descriptors, regex);
2455     }
2456 
2457     if (!bypass) {
2458       if (tableNameList == null || tableNameList.size() == 0) {
2459         // request for all TableDescriptors
2460         Collection<HTableDescriptor> htds;
2461         if (namespace != null && namespace.length() > 0) {
2462           htds = tableDescriptors.getByNamespace(namespace).values();
2463         } else {
2464           htds = tableDescriptors.getAll().values();
2465         }
2466 
2467         for (HTableDescriptor desc: htds) {
2468           if (tableStateManager.isTablePresent(desc.getTableName())
2469               && (includeSysTables || !desc.getTableName().isSystemTable())) {
2470             descriptors.add(desc);
2471           }
2472         }
2473       } else {
2474         for (TableName s: tableNameList) {
2475           if (tableStateManager.isTablePresent(s)) {
2476             HTableDescriptor desc = tableDescriptors.get(s);
2477             if (desc != null) {
2478               descriptors.add(desc);
2479             }
2480           }
2481         }
2482       }
2483 
2484       // Retains only those matched by regular expression.
2485       if (regex != null) {
2486         filterTablesByRegex(descriptors, Pattern.compile(regex));
2487       }
2488 
2489       if (cpHost != null) {
2490         cpHost.postGetTableDescriptors(tableNameList, descriptors, regex);
2491       }
2492     }
2493     return descriptors;
2494   }
2495 
2496   /**
2497    * Returns the list of table names that match the specified request
2498    * @param regex The regular expression to match against, or null if querying for all
2499    * @param namespace the namespace to query, or null if querying for all
2500    * @param includeSysTables False to match only against userspace tables
2501    * @return the list of table names
2502    */
2503   public List<TableName> listTableNames(final String namespace, final String regex,
2504       final boolean includeSysTables) throws IOException {
2505     final List<HTableDescriptor> descriptors = new ArrayList<HTableDescriptor>();
2506 
2507     boolean bypass = false;
2508     if (cpHost != null) {
2509       bypass = cpHost.preGetTableNames(descriptors, regex);
2510     }
2511 
2512     if (!bypass) {
2513       // get all descriptors
2514       Collection<HTableDescriptor> htds;
2515       if (namespace != null && namespace.length() > 0) {
2516         htds = tableDescriptors.getByNamespace(namespace).values();
2517       } else {
2518         htds = tableDescriptors.getAll().values();
2519       }
2520 
2521       for (HTableDescriptor htd: htds) {
2522         if (includeSysTables || !htd.getTableName().isSystemTable()) {
2523           descriptors.add(htd);
2524         }
2525       }
2526 
2527       // Retains only those matched by regular expression.
2528       if (regex != null) {
2529         filterTablesByRegex(descriptors, Pattern.compile(regex));
2530       }
2531 
2532       if (cpHost != null) {
2533         cpHost.postGetTableNames(descriptors, regex);
2534       }
2535     }
2536 
2537     List<TableName> result = new ArrayList<TableName>(descriptors.size());
2538     for (HTableDescriptor htd: descriptors) {
2539       result.add(htd.getTableName());
2540     }
2541     return result;
2542   }
2543 
2544 
2545   /**
2546    * Removes the table descriptors that don't match the pattern.
2547    * @param descriptors list of table descriptors to filter
2548    * @param pattern the regex to use
2549    */
2550   private static void filterTablesByRegex(final Collection<HTableDescriptor> descriptors,
2551       final Pattern pattern) {
2552     final String defaultNS = NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR;
2553     Iterator<HTableDescriptor> itr = descriptors.iterator();
2554     while (itr.hasNext()) {
2555       HTableDescriptor htd = itr.next();
2556       String tableName = htd.getTableName().getNameAsString();
2557       boolean matched = pattern.matcher(tableName).matches();
2558       if (!matched && htd.getTableName().getNamespaceAsString().equals(defaultNS)) {
2559         matched = pattern.matcher(defaultNS + TableName.NAMESPACE_DELIM + tableName).matches();
2560       }
2561       if (!matched) {
2562         itr.remove();
2563       }
2564     }
2565   }
2566 
2567   @Override
2568   public long getLastMajorCompactionTimestamp(TableName table) throws IOException {
2569     return getClusterStatus().getLastMajorCompactionTsForTable(table);
2570   }
2571 
2572   @Override
2573   public long getLastMajorCompactionTimestampForRegion(byte[] regionName) throws IOException {
2574     return getClusterStatus().getLastMajorCompactionTsForRegion(regionName);
2575   }
2576 
2577   /**
2578    * Gets the mob file compaction state for a specific table.
2579    * Whether all the mob files are selected is known during the compaction execution, but
2580    * the statistic is done just before compaction starts, it is hard to know the compaction
2581    * type at that time, so the rough statistics are chosen for the mob file compaction. Only two
2582    * compaction states are available, CompactionState.MAJOR_AND_MINOR and CompactionState.NONE.
2583    * @param tableName The current table name.
2584    * @return If a given table is in mob file compaction now.
2585    */
2586   public CompactionState getMobCompactionState(TableName tableName) {
2587     AtomicInteger compactionsCount = mobCompactionStates.get(tableName);
2588     if (compactionsCount != null && compactionsCount.get() != 0) {
2589       return CompactionState.MAJOR_AND_MINOR;
2590     }
2591     return CompactionState.NONE;
2592   }
2593 
2594   public void reportMobCompactionStart(TableName tableName) throws IOException {
2595     IdLock.Entry lockEntry = null;
2596     try {
2597       lockEntry = mobCompactionLock.getLockEntry(tableName.hashCode());
2598       AtomicInteger compactionsCount = mobCompactionStates.get(tableName);
2599       if (compactionsCount == null) {
2600         compactionsCount = new AtomicInteger(0);
2601         mobCompactionStates.put(tableName, compactionsCount);
2602       }
2603       compactionsCount.incrementAndGet();
2604     } finally {
2605       if (lockEntry != null) {
2606         mobCompactionLock.releaseLockEntry(lockEntry);
2607       }
2608     }
2609   }
2610 
2611   public void reportMobCompactionEnd(TableName tableName) throws IOException {
2612     IdLock.Entry lockEntry = null;
2613     try {
2614       lockEntry = mobCompactionLock.getLockEntry(tableName.hashCode());
2615       AtomicInteger compactionsCount = mobCompactionStates.get(tableName);
2616       if (compactionsCount != null) {
2617         int count = compactionsCount.decrementAndGet();
2618         // remove the entry if the count is 0.
2619         if (count == 0) {
2620           mobCompactionStates.remove(tableName);
2621         }
2622       }
2623     } finally {
2624       if (lockEntry != null) {
2625         mobCompactionLock.releaseLockEntry(lockEntry);
2626       }
2627     }
2628   }
2629 
2630   /**
2631    * Requests mob compaction.
2632    * @param tableName The table the compact.
2633    * @param columns The compacted columns.
2634    * @param allFiles Whether add all mob files into the compaction.
2635    */
2636   public void requestMobCompaction(TableName tableName,
2637     List<HColumnDescriptor> columns, boolean allFiles) throws IOException {
2638     mobCompactThread.requestMobCompaction(conf, fs, tableName, columns,
2639       tableLockManager, allFiles);
2640   }
2641 
2642   /**
2643    * Queries the state of the {@link LoadBalancerTracker}. If the balancer is not initialized,
2644    * false is returned.
2645    *
2646    * @return The state of the load balancer, or false if the load balancer isn't defined.
2647    */
2648   public boolean isBalancerOn() {
2649     if (null == loadBalancerTracker) return false;
2650     return loadBalancerTracker.isBalancerOn();
2651   }
2652 
2653   /**
2654    * Fetch the configured {@link LoadBalancer} class name. If none is set, a default is returned.
2655    *
2656    * @return The name of the {@link LoadBalancer} in use.
2657    */
2658   public String getLoadBalancerClassName() {
2659     return conf.get(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, LoadBalancerFactory
2660         .getDefaultLoadBalancerClass().getName());
2661   }
2662 }