View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.master;
20  
21  import java.io.IOException;
22  import java.io.InterruptedIOException;
23  import java.lang.reflect.Constructor;
24  import java.lang.reflect.InvocationTargetException;
25  import java.net.InetAddress;
26  import java.net.InetSocketAddress;
27  import java.net.UnknownHostException;
28  import java.util.ArrayList;
29  import java.util.Arrays;
30  import java.util.Collection;
31  import java.util.Collections;
32  import java.util.Comparator;
33  import java.util.HashSet;
34  import java.util.Iterator;
35  import java.util.List;
36  import java.util.Map;
37  import java.util.Set;
38  import java.util.concurrent.TimeUnit;
39  import java.util.concurrent.atomic.AtomicReference;
40  import java.util.regex.Pattern;
41  
42  import javax.servlet.ServletException;
43  import javax.servlet.http.HttpServlet;
44  import javax.servlet.http.HttpServletRequest;
45  import javax.servlet.http.HttpServletResponse;
46  
47  import org.apache.commons.logging.Log;
48  import org.apache.commons.logging.LogFactory;
49  import org.apache.hadoop.conf.Configuration;
50  import org.apache.hadoop.fs.Path;
51  import org.apache.hadoop.hbase.ClusterStatus;
52  import org.apache.hadoop.hbase.CoordinatedStateException;
53  import org.apache.hadoop.hbase.CoordinatedStateManager;
54  import org.apache.hadoop.hbase.DoNotRetryIOException;
55  import org.apache.hadoop.hbase.HBaseIOException;
56  import org.apache.hadoop.hbase.HBaseInterfaceAudience;
57  import org.apache.hadoop.hbase.HColumnDescriptor;
58  import org.apache.hadoop.hbase.HConstants;
59  import org.apache.hadoop.hbase.HRegionInfo;
60  import org.apache.hadoop.hbase.HTableDescriptor;
61  import org.apache.hadoop.hbase.MasterNotRunningException;
62  import org.apache.hadoop.hbase.MetaTableAccessor;
63  import org.apache.hadoop.hbase.NamespaceDescriptor;
64  import org.apache.hadoop.hbase.NamespaceNotFoundException;
65  import org.apache.hadoop.hbase.PleaseHoldException;
66  import org.apache.hadoop.hbase.Server;
67  import org.apache.hadoop.hbase.ServerLoad;
68  import org.apache.hadoop.hbase.ServerName;
69  import org.apache.hadoop.hbase.TableDescriptors;
70  import org.apache.hadoop.hbase.TableName;
71  import org.apache.hadoop.hbase.TableNotDisabledException;
72  import org.apache.hadoop.hbase.TableNotFoundException;
73  import org.apache.hadoop.hbase.UnknownRegionException;
74  import org.apache.hadoop.hbase.classification.InterfaceAudience;
75  import org.apache.hadoop.hbase.client.RegionReplicaUtil;
76  import org.apache.hadoop.hbase.client.Result;
77  import org.apache.hadoop.hbase.client.TableState;
78  import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
79  import org.apache.hadoop.hbase.exceptions.DeserializationException;
80  import org.apache.hadoop.hbase.executor.ExecutorType;
81  import org.apache.hadoop.hbase.ipc.RpcServer;
82  import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
83  import org.apache.hadoop.hbase.master.MasterRpcServices.BalanceSwitchMode;
84  import org.apache.hadoop.hbase.master.balancer.BalancerChore;
85  import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer;
86  import org.apache.hadoop.hbase.master.balancer.ClusterStatusChore;
87  import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory;
88  import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
89  import org.apache.hadoop.hbase.master.cleaner.LogCleaner;
90  import org.apache.hadoop.hbase.master.handler.DispatchMergingRegionHandler;
91  import org.apache.hadoop.hbase.master.procedure.AddColumnFamilyProcedure;
92  import org.apache.hadoop.hbase.master.procedure.CreateTableProcedure;
93  import org.apache.hadoop.hbase.master.procedure.DeleteColumnFamilyProcedure;
94  import org.apache.hadoop.hbase.master.procedure.DeleteTableProcedure;
95  import org.apache.hadoop.hbase.master.procedure.DisableTableProcedure;
96  import org.apache.hadoop.hbase.master.procedure.EnableTableProcedure;
97  import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants;
98  import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
99  import org.apache.hadoop.hbase.master.procedure.ModifyColumnFamilyProcedure;
100 import org.apache.hadoop.hbase.master.procedure.ModifyTableProcedure;
101 import org.apache.hadoop.hbase.master.procedure.ProcedurePrepareLatch;
102 import org.apache.hadoop.hbase.master.procedure.ProcedureSyncWait;
103 import org.apache.hadoop.hbase.master.procedure.TruncateTableProcedure;
104 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
105 import org.apache.hadoop.hbase.monitoring.MemoryBoundedLogMessageBuffer;
106 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
107 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
108 import org.apache.hadoop.hbase.procedure.MasterProcedureManagerHost;
109 import org.apache.hadoop.hbase.procedure.flush.MasterFlushTableProcedureManager;
110 import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
111 import org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore;
112 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionServerInfo;
113 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode;
114 import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
115 import org.apache.hadoop.hbase.quotas.RegionStateListener;
116 import org.apache.hadoop.hbase.regionserver.HRegionServer;
117 import org.apache.hadoop.hbase.regionserver.RSRpcServices;
118 import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
119 import org.apache.hadoop.hbase.regionserver.RegionSplitPolicy;
120 import org.apache.hadoop.hbase.replication.regionserver.Replication;
121 import org.apache.hadoop.hbase.security.UserProvider;
122 import org.apache.hadoop.hbase.util.Addressing;
123 import org.apache.hadoop.hbase.util.Bytes;
124 import org.apache.hadoop.hbase.util.CompressionTest;
125 import org.apache.hadoop.hbase.util.EncryptionTest;
126 import org.apache.hadoop.hbase.util.FSUtils;
127 import org.apache.hadoop.hbase.util.HFileArchiveUtil;
128 import org.apache.hadoop.hbase.util.HasThread;
129 import org.apache.hadoop.hbase.util.ModifyRegionUtils;
130 import org.apache.hadoop.hbase.util.Pair;
131 import org.apache.hadoop.hbase.util.Threads;
132 import org.apache.hadoop.hbase.util.VersionInfo;
133 import org.apache.hadoop.hbase.util.ZKDataMigrator;
134 import org.apache.hadoop.hbase.zookeeper.DrainingServerTracker;
135 import org.apache.hadoop.hbase.zookeeper.LoadBalancerTracker;
136 import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker;
137 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
138 import org.apache.hadoop.hbase.zookeeper.RegionServerTracker;
139 import org.apache.hadoop.hbase.zookeeper.ZKClusterId;
140 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
141 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
142 import org.apache.zookeeper.KeeperException;
143 import org.mortbay.jetty.Connector;
144 import org.mortbay.jetty.nio.SelectChannelConnector;
145 import org.mortbay.jetty.servlet.Context;
146 
147 import com.google.common.annotations.VisibleForTesting;
148 import com.google.common.collect.Maps;
149 import com.google.protobuf.Descriptors;
150 import com.google.protobuf.Service;
151 
152 /**
153  * HMaster is the "master server" for HBase. An HBase cluster has one active
154  * master.  If many masters are started, all compete.  Whichever wins goes on to
155  * run the cluster.  All others park themselves in their constructor until
156  * master or cluster shutdown or until the active master loses its lease in
157  * zookeeper.  Thereafter, all running master jostle to take over master role.
158  *
159  * <p>The Master can be asked shutdown the cluster. See {@link #shutdown()}.  In
160  * this case it will tell all regionservers to go down and then wait on them
161  * all reporting in that they are down.  This master will then shut itself down.
162  *
163  * <p>You can also shutdown just this master.  Call {@link #stopMaster()}.
164  *
165  * @see org.apache.zookeeper.Watcher
166  */
167 @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
168 @SuppressWarnings("deprecation")
169 public class HMaster extends HRegionServer implements MasterServices, Server {
170   private static final Log LOG = LogFactory.getLog(HMaster.class.getName());
171 
172   /**
173    * Protection against zombie master. Started once Master accepts active responsibility and
174    * starts taking over responsibilities. Allows a finite time window before giving up ownership.
175    */
176   private static class InitializationMonitor extends HasThread {
177     /** The amount of time in milliseconds to sleep before checking initialization status. */
178     public static final String TIMEOUT_KEY = "hbase.master.initializationmonitor.timeout";
179     public static final long TIMEOUT_DEFAULT = TimeUnit.MILLISECONDS.convert(15, TimeUnit.MINUTES);
180 
181     /**
182      * When timeout expired and initialization has not complete, call {@link System#exit(int)} when
183      * true, do nothing otherwise.
184      */
185     public static final String HALT_KEY = "hbase.master.initializationmonitor.haltontimeout";
186     public static final boolean HALT_DEFAULT = false;
187 
188     private final HMaster master;
189     private final long timeout;
190     private final boolean haltOnTimeout;
191 
192     /** Creates a Thread that monitors the {@link #isInitialized()} state. */
193     InitializationMonitor(HMaster master) {
194       super("MasterInitializationMonitor");
195       this.master = master;
196       this.timeout = master.getConfiguration().getLong(TIMEOUT_KEY, TIMEOUT_DEFAULT);
197       this.haltOnTimeout = master.getConfiguration().getBoolean(HALT_KEY, HALT_DEFAULT);
198       this.setDaemon(true);
199     }
200 
201     @Override
202     public void run() {
203       try {
204         while (!master.isStopped() && master.isActiveMaster()) {
205           Thread.sleep(timeout);
206           if (master.isInitialized()) {
207             LOG.debug("Initialization completed within allotted tolerance. Monitor exiting.");
208           } else {
209             LOG.error("Master failed to complete initialization after " + timeout + "ms. Please"
210                 + " consider submitting a bug report including a thread dump of this process.");
211             if (haltOnTimeout) {
212               LOG.error("Zombie Master exiting. Thread dump to stdout");
213               Threads.printThreadInfo(System.out, "Zombie HMaster");
214               System.exit(-1);
215             }
216           }
217         }
218       } catch (InterruptedException ie) {
219         LOG.trace("InitMonitor thread interrupted. Existing.");
220       }
221     }
222   }
223 
224   // MASTER is name of the webapp and the attribute name used stuffing this
225   //instance into web context.
226   public static final String MASTER = "master";
227 
228   // Manager and zk listener for master election
229   private final ActiveMasterManager activeMasterManager;
230   // Region server tracker
231   RegionServerTracker regionServerTracker;
232   // Draining region server tracker
233   private DrainingServerTracker drainingServerTracker;
234   // Tracker for load balancer state
235   LoadBalancerTracker loadBalancerTracker;
236 
237   /** Namespace stuff */
238   private TableNamespaceManager tableNamespaceManager;
239 
240   // Metrics for the HMaster
241   final MetricsMaster metricsMaster;
242   // file system manager for the master FS operations
243   private MasterFileSystem fileSystemManager;
244 
245   // server manager to deal with region server info
246   volatile ServerManager serverManager;
247 
248   // manager of assignment nodes in zookeeper
249   AssignmentManager assignmentManager;
250 
251   // buffer for "fatal error" notices from region servers
252   // in the cluster. This is only used for assisting
253   // operations/debugging.
254   MemoryBoundedLogMessageBuffer rsFatals;
255 
256   // flag set after we become the active master (used for testing)
257   private volatile boolean isActiveMaster = false;
258 
259   // flag set after we complete initialization once active,
260   // it is not private since it's used in unit tests
261   volatile boolean initialized = false;
262 
263   // flag set after master services are started,
264   // initialization may have not completed yet.
265   volatile boolean serviceStarted = false;
266 
267   // flag set after we complete assignMeta.
268   private volatile boolean serverShutdownHandlerEnabled = false;
269 
270   LoadBalancer balancer;
271   private BalancerChore balancerChore;
272   private ClusterStatusChore clusterStatusChore;
273   private ClusterStatusPublisher clusterStatusPublisherChore = null;
274 
275   CatalogJanitor catalogJanitorChore;
276   private LogCleaner logCleaner;
277   private HFileCleaner hfileCleaner;
278 
279   MasterCoprocessorHost cpHost;
280 
281   private final boolean preLoadTableDescriptors;
282 
283   // Time stamps for when a hmaster became active
284   private long masterActiveTime;
285 
286   //should we check the compression codec type at master side, default true, HBASE-6370
287   private final boolean masterCheckCompression;
288 
289   //should we check encryption settings at master side, default true
290   private final boolean masterCheckEncryption;
291 
292   Map<String, Service> coprocessorServiceHandlers = Maps.newHashMap();
293 
294   // monitor for snapshot of hbase tables
295   SnapshotManager snapshotManager;
296   // monitor for distributed procedures
297   MasterProcedureManagerHost mpmHost;
298 
299   // it is assigned after 'initialized' guard set to true, so should be volatile
300   private volatile MasterQuotaManager quotaManager;
301 
302   private ProcedureExecutor<MasterProcedureEnv> procedureExecutor;
303   private WALProcedureStore procedureStore;
304 
305   // handle table states
306   private TableStateManager tableStateManager;
307 
308   /** flag used in test cases in order to simulate RS failures during master initialization */
309   private volatile boolean initializationBeforeMetaAssignment = false;
310 
311   /** jetty server for master to redirect requests to regionserver infoServer */
312   private org.mortbay.jetty.Server masterJettyServer;
313 
314   public static class RedirectServlet extends HttpServlet {
315     private static final long serialVersionUID = 2894774810058302472L;
316     private static int regionServerInfoPort;
317 
318     @Override
319     public void doGet(HttpServletRequest request,
320         HttpServletResponse response) throws ServletException, IOException {
321       String redirectUrl = request.getScheme() + "://"
322         + request.getServerName() + ":" + regionServerInfoPort
323         + request.getRequestURI();
324       response.sendRedirect(redirectUrl);
325     }
326   }
327 
328   /**
329    * Initializes the HMaster. The steps are as follows:
330    * <p>
331    * <ol>
332    * <li>Initialize the local HRegionServer
333    * <li>Start the ActiveMasterManager.
334    * </ol>
335    * <p>
336    * Remaining steps of initialization occur in
337    * #finishActiveMasterInitialization(MonitoredTask) after
338    * the master becomes the active one.
339    *
340    * @throws KeeperException
341    * @throws IOException
342    */
343   public HMaster(final Configuration conf, CoordinatedStateManager csm)
344       throws IOException, KeeperException {
345     super(conf, csm);
346     this.rsFatals = new MemoryBoundedLogMessageBuffer(
347       conf.getLong("hbase.master.buffer.for.rs.fatals", 1*1024*1024));
348 
349     LOG.info("hbase.rootdir=" + FSUtils.getRootDir(this.conf) +
350         ", hbase.cluster.distributed=" + this.conf.getBoolean(HConstants.CLUSTER_DISTRIBUTED, false));
351 
352     // Disable usage of meta replicas in the master
353     this.conf.setBoolean(HConstants.USE_META_REPLICAS, false);
354 
355     Replication.decorateMasterConfiguration(this.conf);
356 
357     // Hack! Maps DFSClient => Master for logs.  HDFS made this
358     // config param for task trackers, but we can piggyback off of it.
359     if (this.conf.get("mapreduce.task.attempt.id") == null) {
360       this.conf.set("mapreduce.task.attempt.id", "hb_m_" + this.serverName.toString());
361     }
362 
363     // should we check the compression codec type at master side, default true, HBASE-6370
364     this.masterCheckCompression = conf.getBoolean("hbase.master.check.compression", true);
365 
366     // should we check encryption settings at master side, default true
367     this.masterCheckEncryption = conf.getBoolean("hbase.master.check.encryption", true);
368 
369     this.metricsMaster = new MetricsMaster( new MetricsMasterWrapperImpl(this));
370 
371     // preload table descriptor at startup
372     this.preLoadTableDescriptors = conf.getBoolean("hbase.master.preload.tabledescriptors", true);
373 
374     // Do we publish the status?
375 
376     boolean shouldPublish = conf.getBoolean(HConstants.STATUS_PUBLISHED,
377         HConstants.STATUS_PUBLISHED_DEFAULT);
378     Class<? extends ClusterStatusPublisher.Publisher> publisherClass =
379         conf.getClass(ClusterStatusPublisher.STATUS_PUBLISHER_CLASS,
380             ClusterStatusPublisher.DEFAULT_STATUS_PUBLISHER_CLASS,
381             ClusterStatusPublisher.Publisher.class);
382 
383     if (shouldPublish) {
384       if (publisherClass == null) {
385         LOG.warn(HConstants.STATUS_PUBLISHED + " is true, but " +
386             ClusterStatusPublisher.DEFAULT_STATUS_PUBLISHER_CLASS +
387             " is not set - not publishing status");
388       } else {
389         clusterStatusPublisherChore = new ClusterStatusPublisher(this, conf, publisherClass);
390         getChoreService().scheduleChore(clusterStatusPublisherChore);
391       }
392     }
393     activeMasterManager = new ActiveMasterManager(zooKeeper, this.serverName, this);
394     int infoPort = putUpJettyServer();
395     startActiveMasterManager(infoPort);
396   }
397 
398   // return the actual infoPort, -1 means disable info server.
399   private int putUpJettyServer() throws IOException {
400     if (!conf.getBoolean("hbase.master.infoserver.redirect", true)) {
401       return -1;
402     }
403     int infoPort = conf.getInt("hbase.master.info.port.orig",
404       HConstants.DEFAULT_MASTER_INFOPORT);
405     // -1 is for disabling info server, so no redirecting
406     if (infoPort < 0 || infoServer == null) {
407       return -1;
408     }
409     String addr = conf.get("hbase.master.info.bindAddress", "0.0.0.0");
410     if (!Addressing.isLocalAddress(InetAddress.getByName(addr))) {
411       String msg =
412           "Failed to start redirecting jetty server. Address " + addr
413               + " does not belong to this host. Correct configuration parameter: "
414               + "hbase.master.info.bindAddress";
415       LOG.error(msg);
416       throw new IOException(msg);
417     }
418 
419     RedirectServlet.regionServerInfoPort = infoServer.getPort();
420     if(RedirectServlet.regionServerInfoPort == infoPort) {
421       return infoPort;
422     }
423     masterJettyServer = new org.mortbay.jetty.Server();
424     Connector connector = new SelectChannelConnector();
425     connector.setHost(addr);
426     connector.setPort(infoPort);
427     masterJettyServer.addConnector(connector);
428     masterJettyServer.setStopAtShutdown(true);
429     Context context = new Context(masterJettyServer, "/", Context.NO_SESSIONS);
430     context.addServlet(RedirectServlet.class, "/*");
431     try {
432       masterJettyServer.start();
433     } catch (Exception e) {
434       throw new IOException("Failed to start redirecting jetty server", e);
435     }
436     return connector.getLocalPort();
437   }
438 
439   @Override
440   protected TableDescriptors getFsTableDescriptors() throws IOException {
441     return super.getFsTableDescriptors();
442   }
443 
444   /**
445    * For compatibility, if failed with regionserver credentials, try the master one
446    */
447   @Override
448   protected void login(UserProvider user, String host) throws IOException {
449     try {
450       super.login(user, host);
451     } catch (IOException ie) {
452       user.login("hbase.master.keytab.file",
453         "hbase.master.kerberos.principal", host);
454     }
455   }
456 
457   /**
458    * If configured to put regions on active master,
459    * wait till a backup master becomes active.
460    * Otherwise, loop till the server is stopped or aborted.
461    */
462   @Override
463   protected void waitForMasterActive(){
464     boolean tablesOnMaster = BaseLoadBalancer.tablesOnMaster(conf);
465     while (!(tablesOnMaster && isActiveMaster)
466         && !isStopped() && !isAborted()) {
467       sleeper.sleep();
468     }
469   }
470 
471   @VisibleForTesting
472   public MasterRpcServices getMasterRpcServices() {
473     return (MasterRpcServices)rpcServices;
474   }
475 
476   public boolean balanceSwitch(final boolean b) throws IOException {
477     return getMasterRpcServices().switchBalancer(b, BalanceSwitchMode.ASYNC);
478   }
479 
480   @Override
481   protected String getProcessName() {
482     return MASTER;
483   }
484 
485   @Override
486   protected boolean canCreateBaseZNode() {
487     return true;
488   }
489 
490   @Override
491   protected boolean canUpdateTableDescriptor() {
492     return true;
493   }
494 
495   @Override
496   protected RSRpcServices createRpcServices() throws IOException {
497     return new MasterRpcServices(this);
498   }
499 
500   @Override
501   protected void configureInfoServer() {
502     infoServer.addServlet("master-status", "/master-status", MasterStatusServlet.class);
503     infoServer.setAttribute(MASTER, this);
504     if (BaseLoadBalancer.tablesOnMaster(conf)) {
505       super.configureInfoServer();
506     }
507   }
508 
509   @Override
510   protected Class<? extends HttpServlet> getDumpServlet() {
511     return MasterDumpServlet.class;
512   }
513 
514   /**
515    * Emit the HMaster metrics, such as region in transition metrics.
516    * Surrounding in a try block just to be sure metrics doesn't abort HMaster.
517    */
518   @Override
519   protected void doMetrics() {
520     try {
521       if (assignmentManager != null) {
522         assignmentManager.updateRegionsInTransitionMetrics();
523       }
524     } catch (Throwable e) {
525       LOG.error("Couldn't update metrics: " + e.getMessage());
526     }
527   }
528 
529   MetricsMaster getMasterMetrics() {
530     return metricsMaster;
531   }
532 
533   /**
534    * Initialize all ZK based system trackers.
535    * @throws IOException
536    * @throws InterruptedException
537    * @throws KeeperException
538    * @throws CoordinatedStateException
539    */
540   void initializeZKBasedSystemTrackers() throws IOException,
541       InterruptedException, KeeperException, CoordinatedStateException {
542     this.balancer = LoadBalancerFactory.getLoadBalancer(conf);
543     this.loadBalancerTracker = new LoadBalancerTracker(zooKeeper, this);
544     this.loadBalancerTracker.start();
545     this.assignmentManager = new AssignmentManager(this, serverManager,
546       this.balancer, this.service, this.metricsMaster,
547       this.tableLockManager, tableStateManager);
548 
549     this.regionServerTracker = new RegionServerTracker(zooKeeper, this,
550         this.serverManager);
551     this.regionServerTracker.start();
552 
553     this.drainingServerTracker = new DrainingServerTracker(zooKeeper, this,
554       this.serverManager);
555     this.drainingServerTracker.start();
556 
557     // Set the cluster as up.  If new RSs, they'll be waiting on this before
558     // going ahead with their startup.
559     boolean wasUp = this.clusterStatusTracker.isClusterUp();
560     if (!wasUp) this.clusterStatusTracker.setClusterUp();
561 
562     LOG.info("Server active/primary master=" + this.serverName +
563         ", sessionid=0x" +
564         Long.toHexString(this.zooKeeper.getRecoverableZooKeeper().getSessionId()) +
565         ", setting cluster-up flag (Was=" + wasUp + ")");
566 
567     // create/initialize the snapshot manager and other procedure managers
568     this.snapshotManager = new SnapshotManager();
569     this.mpmHost = new MasterProcedureManagerHost();
570     this.mpmHost.register(this.snapshotManager);
571     this.mpmHost.register(new MasterFlushTableProcedureManager());
572     this.mpmHost.loadProcedures(conf);
573     this.mpmHost.initialize(this, this.metricsMaster);
574 
575   }
576 
577   /**
578    * Finish initialization of HMaster after becoming the primary master.
579    *
580    * <ol>
581    * <li>Initialize master components - file system manager, server manager,
582    *     assignment manager, region server tracker, etc</li>
583    * <li>Start necessary service threads - balancer, catalog janior,
584    *     executor services, etc</li>
585    * <li>Set cluster as UP in ZooKeeper</li>
586    * <li>Wait for RegionServers to check-in</li>
587    * <li>Split logs and perform data recovery, if necessary</li>
588    * <li>Ensure assignment of meta/namespace regions<li>
589    * <li>Handle either fresh cluster start or master failover</li>
590    * </ol>
591    *
592    * @throws IOException
593    * @throws InterruptedException
594    * @throws KeeperException
595    * @throws CoordinatedStateException
596    */
597   private void finishActiveMasterInitialization(MonitoredTask status)
598       throws IOException, InterruptedException, KeeperException, CoordinatedStateException {
599 
600     isActiveMaster = true;
601     Thread zombieDetector = new Thread(new InitializationMonitor(this));
602     zombieDetector.start();
603 
604     /*
605      * We are active master now... go initialize components we need to run.
606      * Note, there may be dross in zk from previous runs; it'll get addressed
607      * below after we determine if cluster startup or failover.
608      */
609 
610     status.setStatus("Initializing Master file system");
611 
612     this.masterActiveTime = System.currentTimeMillis();
613     // TODO: Do this using Dependency Injection, using PicoContainer, Guice or Spring.
614     this.fileSystemManager = new MasterFileSystem(this, this);
615 
616     // enable table descriptors cache
617     this.tableDescriptors.setCacheOn();
618     // set the META's descriptor to the correct replication
619     this.tableDescriptors.get(TableName.META_TABLE_NAME).setRegionReplication(
620         conf.getInt(HConstants.META_REPLICAS_NUM, HConstants.DEFAULT_META_REPLICA_NUM));
621     // warm-up HTDs cache on master initialization
622     if (preLoadTableDescriptors) {
623       status.setStatus("Pre-loading table descriptors");
624       this.tableDescriptors.getAll();
625     }
626 
627     // publish cluster ID
628     status.setStatus("Publishing Cluster ID in ZooKeeper");
629     ZKClusterId.setClusterId(this.zooKeeper, fileSystemManager.getClusterId());
630     this.serverManager = createServerManager(this, this);
631 
632     setupClusterConnection();
633 
634     // Invalidate all write locks held previously
635     this.tableLockManager.reapWriteLocks();
636     this.tableStateManager = new TableStateManager(this);
637 
638     status.setStatus("Initializing ZK system trackers");
639     initializeZKBasedSystemTrackers();
640 
641     // initialize master side coprocessors before we start handling requests
642     status.setStatus("Initializing master coprocessors");
643     this.cpHost = new MasterCoprocessorHost(this, this.conf);
644 
645     // start up all service threads.
646     status.setStatus("Initializing master service threads");
647     startServiceThreads();
648 
649     // Wake up this server to check in
650     sleeper.skipSleepCycle();
651 
652     // Wait for region servers to report in
653     this.serverManager.waitForRegionServers(status);
654     // Check zk for region servers that are up but didn't register
655     for (ServerName sn: this.regionServerTracker.getOnlineServers()) {
656       // The isServerOnline check is opportunistic, correctness is handled inside
657       if (!this.serverManager.isServerOnline(sn)
658           && serverManager.checkAndRecordNewServer(sn, ServerLoad.EMPTY_SERVERLOAD)) {
659         LOG.info("Registered server found up in zk but who has not yet reported in: " + sn);
660       }
661     }
662 
663     // get a list for previously failed RS which need log splitting work
664     // we recover hbase:meta region servers inside master initialization and
665     // handle other failed servers in SSH in order to start up master node ASAP
666     Set<ServerName> previouslyFailedServers = this.fileSystemManager
667         .getFailedServersFromLogFolders();
668 
669     // remove stale recovering regions from previous run
670     this.fileSystemManager.removeStaleRecoveringRegionsFromZK(previouslyFailedServers);
671 
672     // log splitting for hbase:meta server
673     ServerName oldMetaServerLocation = metaTableLocator.getMetaRegionLocation(this.getZooKeeper());
674     if (oldMetaServerLocation != null && previouslyFailedServers.contains(oldMetaServerLocation)) {
675       splitMetaLogBeforeAssignment(oldMetaServerLocation);
676       // Note: we can't remove oldMetaServerLocation from previousFailedServers list because it
677       // may also host user regions
678     }
679     Set<ServerName> previouslyFailedMetaRSs = getPreviouselyFailedMetaServersFromZK();
680     // need to use union of previouslyFailedMetaRSs recorded in ZK and previouslyFailedServers
681     // instead of previouslyFailedMetaRSs alone to address the following two situations:
682     // 1) the chained failure situation(recovery failed multiple times in a row).
683     // 2) master get killed right before it could delete the recovering hbase:meta from ZK while the
684     // same server still has non-meta wals to be replayed so that
685     // removeStaleRecoveringRegionsFromZK can't delete the stale hbase:meta region
686     // Passing more servers into splitMetaLog is all right. If a server doesn't have hbase:meta wal,
687     // there is no op for the server.
688     previouslyFailedMetaRSs.addAll(previouslyFailedServers);
689 
690     this.initializationBeforeMetaAssignment = true;
691 
692     // Wait for regionserver to finish initialization.
693     if (BaseLoadBalancer.tablesOnMaster(conf)) {
694       waitForServerOnline();
695     }
696 
697     //initialize load balancer
698     this.balancer.setClusterStatus(getClusterStatus());
699     this.balancer.setMasterServices(this);
700     this.balancer.initialize();
701 
702     // Check if master is shutting down because of some issue
703     // in initializing the regionserver or the balancer.
704     if(isStopped()) return;
705 
706     // Make sure meta assigned before proceeding.
707     status.setStatus("Assigning Meta Region");
708     assignMeta(status, previouslyFailedMetaRSs, HRegionInfo.DEFAULT_REPLICA_ID);
709     // check if master is shutting down because above assignMeta could return even hbase:meta isn't
710     // assigned when master is shutting down
711     if(isStopped()) return;
712 
713     // migrating existent table state from zk, so splitters
714     // and recovery process treat states properly.
715     for (Map.Entry<TableName, TableState.State> entry : ZKDataMigrator
716         .queryForTableStates(getZooKeeper()).entrySet()) {
717       LOG.info("Converting state from zk to new states:" + entry);
718       tableStateManager.setTableState(entry.getKey(), entry.getValue());
719     }
720     ZKUtil.deleteChildrenRecursively(getZooKeeper(), getZooKeeper().tableZNode);
721 
722     status.setStatus("Submitting log splitting work for previously failed region servers");
723     // Master has recovered hbase:meta region server and we put
724     // other failed region servers in a queue to be handled later by SSH
725     for (ServerName tmpServer : previouslyFailedServers) {
726       this.serverManager.processDeadServer(tmpServer, true);
727     }
728 
729     // Fix up assignment manager status
730     status.setStatus("Starting assignment manager");
731     this.assignmentManager.joinCluster();
732 
733     //set cluster status again after user regions are assigned
734     this.balancer.setClusterStatus(getClusterStatus());
735 
736     // Start balancer and meta catalog janitor after meta and regions have
737     // been assigned.
738     status.setStatus("Starting balancer and catalog janitor");
739     this.clusterStatusChore = new ClusterStatusChore(this, balancer);
740     getChoreService().scheduleChore(clusterStatusChore);
741     this.balancerChore = new BalancerChore(this);
742     getChoreService().scheduleChore(balancerChore);
743     this.catalogJanitorChore = new CatalogJanitor(this, this);
744     getChoreService().scheduleChore(catalogJanitorChore);
745 
746     status.setStatus("Starting namespace manager");
747     initNamespace();
748 
749     if (this.cpHost != null) {
750       try {
751         this.cpHost.preMasterInitialization();
752       } catch (IOException e) {
753         LOG.error("Coprocessor preMasterInitialization() hook failed", e);
754       }
755     }
756 
757     status.markComplete("Initialization successful");
758     LOG.info("Master has completed initialization");
759     configurationManager.registerObserver(this.balancer);
760     initialized = true;
761     // assign the meta replicas
762     Set<ServerName> EMPTY_SET = new HashSet<ServerName>();
763     int numReplicas = conf.getInt(HConstants.META_REPLICAS_NUM,
764            HConstants.DEFAULT_META_REPLICA_NUM);
765     for (int i = 1; i < numReplicas; i++) {
766       assignMeta(status, EMPTY_SET, i);
767     }
768     unassignExcessMetaReplica(zooKeeper, numReplicas);
769 
770     status.setStatus("Starting quota manager");
771     initQuotaManager();
772 
773     // clear the dead servers with same host name and port of online server because we are not
774     // removing dead server with same hostname and port of rs which is trying to check in before
775     // master initialization. See HBASE-5916.
776     this.serverManager.clearDeadServersWithSameHostNameAndPortOfOnlineServer();
777 
778     if (this.cpHost != null) {
779       // don't let cp initialization errors kill the master
780       try {
781         this.cpHost.postStartMaster();
782       } catch (IOException ioe) {
783         LOG.error("Coprocessor postStartMaster() hook failed", ioe);
784       }
785     }
786 
787     zombieDetector.interrupt();
788   }
789 
790   /**
791    * Create a {@link ServerManager} instance.
792    * @param master
793    * @param services
794    * @return An instance of {@link ServerManager}
795    * @throws org.apache.hadoop.hbase.ZooKeeperConnectionException
796    * @throws IOException
797    */
798   ServerManager createServerManager(final Server master,
799       final MasterServices services)
800   throws IOException {
801     // We put this out here in a method so can do a Mockito.spy and stub it out
802     // w/ a mocked up ServerManager.
803     return new ServerManager(master, services);
804   }
805 
806   private void unassignExcessMetaReplica(ZooKeeperWatcher zkw, int numMetaReplicasConfigured) {
807     // unassign the unneeded replicas (for e.g., if the previous master was configured
808     // with a replication of 3 and now it is 2, we need to unassign the 1 unneeded replica)
809     try {
810       List<String> metaReplicaZnodes = zooKeeper.getMetaReplicaNodes();
811       for (String metaReplicaZnode : metaReplicaZnodes) {
812         int replicaId = zooKeeper.getMetaReplicaIdFromZnode(metaReplicaZnode);
813         if (replicaId >= numMetaReplicasConfigured) {
814           RegionState r = MetaTableLocator.getMetaRegionState(zkw, replicaId);
815           LOG.info("Closing excess replica of meta region " + r.getRegion());
816           // send a close and wait for a max of 30 seconds
817           ServerManager.closeRegionSilentlyAndWait(getConnection(), r.getServerName(),
818               r.getRegion(), 30000);
819           ZKUtil.deleteNode(zkw, zkw.getZNodeForReplica(replicaId));
820         }
821       }
822     } catch (Exception ex) {
823       // ignore the exception since we don't want the master to be wedged due to potential
824       // issues in the cleanup of the extra regions. We can do that cleanup via hbck or manually
825       LOG.warn("Ignoring exception " + ex);
826     }
827   }
828 
829   /**
830    * Check <code>hbase:meta</code> is assigned. If not, assign it.
831    * @param status MonitoredTask
832    * @param previouslyFailedMetaRSs
833    * @param replicaId
834    * @throws InterruptedException
835    * @throws IOException
836    * @throws KeeperException
837    */
838   void assignMeta(MonitoredTask status, Set<ServerName> previouslyFailedMetaRSs, int replicaId)
839       throws InterruptedException, IOException, KeeperException {
840     // Work on meta region
841     int assigned = 0;
842     long timeout = this.conf.getLong("hbase.catalog.verification.timeout", 1000);
843     if (replicaId == HRegionInfo.DEFAULT_REPLICA_ID) {
844       status.setStatus("Assigning hbase:meta region");
845     } else {
846       status.setStatus("Assigning hbase:meta region, replicaId " + replicaId);
847     }
848 
849     // Get current meta state from zk.
850     RegionState metaState = MetaTableLocator.getMetaRegionState(getZooKeeper(), replicaId);
851     HRegionInfo hri = RegionReplicaUtil.getRegionInfoForReplica(HRegionInfo.FIRST_META_REGIONINFO,
852         replicaId);
853     RegionStates regionStates = assignmentManager.getRegionStates();
854     regionStates.createRegionState(hri, metaState.getState(),
855         metaState.getServerName(), null);
856 
857     if (!metaState.isOpened() || !metaTableLocator.verifyMetaRegionLocation(
858         this.getConnection(), this.getZooKeeper(), timeout, replicaId)) {
859       ServerName currentMetaServer = metaState.getServerName();
860       if (serverManager.isServerOnline(currentMetaServer)) {
861         if (replicaId == HRegionInfo.DEFAULT_REPLICA_ID) {
862           LOG.info("Meta was in transition on " + currentMetaServer);
863         } else {
864           LOG.info("Meta with replicaId " + replicaId + " was in transition on " +
865                     currentMetaServer);
866         }
867         assignmentManager.processRegionsInTransition(Arrays.asList(metaState));
868       } else {
869         if (currentMetaServer != null) {
870           if (replicaId == HRegionInfo.DEFAULT_REPLICA_ID) {
871             splitMetaLogBeforeAssignment(currentMetaServer);
872             regionStates.logSplit(HRegionInfo.FIRST_META_REGIONINFO);
873             previouslyFailedMetaRSs.add(currentMetaServer);
874           }
875         }
876         LOG.info("Re-assigning hbase:meta with replicaId, " + replicaId +
877             " it was on " + currentMetaServer);
878         assignmentManager.assignMeta(hri);
879       }
880       assigned++;
881     }
882 
883     if (replicaId == HRegionInfo.DEFAULT_REPLICA_ID)
884       getTableStateManager().setTableState(TableName.META_TABLE_NAME, TableState.State.ENABLED);
885     // TODO: should we prevent from using state manager before meta was initialized?
886     // tableStateManager.start();
887 
888     if ((RecoveryMode.LOG_REPLAY == this.getMasterFileSystem().getLogRecoveryMode())
889         && (!previouslyFailedMetaRSs.isEmpty())) {
890       // replay WAL edits mode need new hbase:meta RS is assigned firstly
891       status.setStatus("replaying log for Meta Region");
892       this.fileSystemManager.splitMetaLog(previouslyFailedMetaRSs);
893     }
894 
895     this.assignmentManager.setEnabledTable(TableName.META_TABLE_NAME);
896     tableStateManager.start();
897 
898     // Make sure a hbase:meta location is set. We need to enable SSH here since
899     // if the meta region server is died at this time, we need it to be re-assigned
900     // by SSH so that system tables can be assigned.
901     // No need to wait for meta is assigned = 0 when meta is just verified.
902     if (replicaId == HRegionInfo.DEFAULT_REPLICA_ID) enableServerShutdownHandler(assigned != 0);
903     LOG.info("hbase:meta with replicaId " + replicaId + " assigned=" + assigned + ", location="
904       + metaTableLocator.getMetaRegionLocation(this.getZooKeeper(), replicaId));
905     status.setStatus("META assigned.");
906   }
907 
908   void initNamespace() throws IOException {
909     //create namespace manager
910     tableNamespaceManager = new TableNamespaceManager(this);
911     tableNamespaceManager.start();
912   }
913 
914   void initQuotaManager() throws IOException {
915     MasterQuotaManager quotaManager = new MasterQuotaManager(this);
916     this.assignmentManager.setRegionStateListener((RegionStateListener)quotaManager);
917     quotaManager.start();
918     this.quotaManager = quotaManager;
919   }
920 
921   boolean isCatalogJanitorEnabled() {
922     return catalogJanitorChore != null ?
923       catalogJanitorChore.getEnabled() : false;
924   }
925 
926   private void splitMetaLogBeforeAssignment(ServerName currentMetaServer) throws IOException {
927     if (RecoveryMode.LOG_REPLAY == this.getMasterFileSystem().getLogRecoveryMode()) {
928       // In log replay mode, we mark hbase:meta region as recovering in ZK
929       Set<HRegionInfo> regions = new HashSet<HRegionInfo>();
930       regions.add(HRegionInfo.FIRST_META_REGIONINFO);
931       this.fileSystemManager.prepareLogReplay(currentMetaServer, regions);
932     } else {
933       // In recovered.edits mode: create recovered edits file for hbase:meta server
934       this.fileSystemManager.splitMetaLog(currentMetaServer);
935     }
936   }
937 
938   private void enableServerShutdownHandler(
939       final boolean waitForMeta) throws IOException, InterruptedException {
940     // If ServerShutdownHandler is disabled, we enable it and expire those dead
941     // but not expired servers. This is required so that if meta is assigning to
942     // a server which dies after assignMeta starts assignment,
943     // SSH can re-assign it. Otherwise, we will be
944     // stuck here waiting forever if waitForMeta is specified.
945     if (!serverShutdownHandlerEnabled) {
946       serverShutdownHandlerEnabled = true;
947       this.serverManager.processQueuedDeadServers();
948     }
949 
950     if (waitForMeta) {
951       metaTableLocator.waitMetaRegionLocation(this.getZooKeeper());
952     }
953   }
954 
955   /**
956    * This function returns a set of region server names under hbase:meta recovering region ZK node
957    * @return Set of meta server names which were recorded in ZK
958    * @throws KeeperException
959    */
960   private Set<ServerName> getPreviouselyFailedMetaServersFromZK() throws KeeperException {
961     Set<ServerName> result = new HashSet<ServerName>();
962     String metaRecoveringZNode = ZKUtil.joinZNode(zooKeeper.recoveringRegionsZNode,
963       HRegionInfo.FIRST_META_REGIONINFO.getEncodedName());
964     List<String> regionFailedServers = ZKUtil.listChildrenNoWatch(zooKeeper, metaRecoveringZNode);
965     if (regionFailedServers == null) return result;
966 
967     for(String failedServer : regionFailedServers) {
968       ServerName server = ServerName.parseServerName(failedServer);
969       result.add(server);
970     }
971     return result;
972   }
973 
974   @Override
975   public TableDescriptors getTableDescriptors() {
976     return this.tableDescriptors;
977   }
978 
979   @Override
980   public ServerManager getServerManager() {
981     return this.serverManager;
982   }
983 
984   @Override
985   public MasterFileSystem getMasterFileSystem() {
986     return this.fileSystemManager;
987   }
988 
989   @Override
990   public TableStateManager getTableStateManager() {
991     return tableStateManager;
992   }
993 
994   /*
995    * Start up all services. If any of these threads gets an unhandled exception
996    * then they just die with a logged message.  This should be fine because
997    * in general, we do not expect the master to get such unhandled exceptions
998    *  as OOMEs; it should be lightly loaded. See what HRegionServer does if
999    *  need to install an unexpected exception handler.
1000    */
1001   private void startServiceThreads() throws IOException{
1002    // Start the executor service pools
1003    this.service.startExecutorService(ExecutorType.MASTER_OPEN_REGION,
1004       conf.getInt("hbase.master.executor.openregion.threads", 5));
1005    this.service.startExecutorService(ExecutorType.MASTER_CLOSE_REGION,
1006       conf.getInt("hbase.master.executor.closeregion.threads", 5));
1007    this.service.startExecutorService(ExecutorType.MASTER_SERVER_OPERATIONS,
1008       conf.getInt("hbase.master.executor.serverops.threads", 5));
1009    this.service.startExecutorService(ExecutorType.MASTER_META_SERVER_OPERATIONS,
1010       conf.getInt("hbase.master.executor.meta.serverops.threads", 5));
1011    this.service.startExecutorService(ExecutorType.M_LOG_REPLAY_OPS,
1012       conf.getInt("hbase.master.executor.logreplayops.threads", 10));
1013 
1014    // We depend on there being only one instance of this executor running
1015    // at a time.  To do concurrency, would need fencing of enable/disable of
1016    // tables.
1017    // Any time changing this maxThreads to > 1, pls see the comment at
1018    // AccessController#postCreateTableHandler
1019    this.service.startExecutorService(ExecutorType.MASTER_TABLE_OPERATIONS, 1);
1020    startProcedureExecutor();
1021 
1022    // Start log cleaner thread
1023    int cleanerInterval = conf.getInt("hbase.master.cleaner.interval", 60 * 1000);
1024    this.logCleaner =
1025       new LogCleaner(cleanerInterval,
1026          this, conf, getMasterFileSystem().getFileSystem(),
1027          getMasterFileSystem().getOldLogDir());
1028     getChoreService().scheduleChore(logCleaner);
1029 
1030    //start the hfile archive cleaner thread
1031     Path archiveDir = HFileArchiveUtil.getArchivePath(conf);
1032     this.hfileCleaner = new HFileCleaner(cleanerInterval, this, conf, getMasterFileSystem()
1033         .getFileSystem(), archiveDir);
1034     getChoreService().scheduleChore(hfileCleaner);
1035     serviceStarted = true;
1036     if (LOG.isTraceEnabled()) {
1037       LOG.trace("Started service threads");
1038     }
1039   }
1040 
1041   @Override
1042   protected void sendShutdownInterrupt() {
1043     super.sendShutdownInterrupt();
1044     stopProcedureExecutor();
1045   }
1046 
1047   @Override
1048   protected void stopServiceThreads() {
1049     if (masterJettyServer != null) {
1050       LOG.info("Stopping master jetty server");
1051       try {
1052         masterJettyServer.stop();
1053       } catch (Exception e) {
1054         LOG.error("Failed to stop master jetty server", e);
1055       }
1056     }
1057     super.stopServiceThreads();
1058     stopChores();
1059 
1060     // Wait for all the remaining region servers to report in IFF we were
1061     // running a cluster shutdown AND we were NOT aborting.
1062     if (!isAborted() && this.serverManager != null &&
1063         this.serverManager.isClusterShutdown()) {
1064       this.serverManager.letRegionServersShutdown();
1065     }
1066     if (LOG.isDebugEnabled()) {
1067       LOG.debug("Stopping service threads");
1068     }
1069     // Clean up and close up shop
1070     if (this.logCleaner != null) this.logCleaner.cancel(true);
1071     if (this.hfileCleaner != null) this.hfileCleaner.cancel(true);
1072     if (this.quotaManager != null) this.quotaManager.stop();
1073     if (this.activeMasterManager != null) this.activeMasterManager.stop();
1074     if (this.serverManager != null) this.serverManager.stop();
1075     if (this.assignmentManager != null) this.assignmentManager.stop();
1076     if (this.fileSystemManager != null) this.fileSystemManager.stop();
1077     if (this.mpmHost != null) this.mpmHost.stop("server shutting down.");
1078   }
1079 
1080   private void startProcedureExecutor() throws IOException {
1081     final MasterProcedureEnv procEnv = new MasterProcedureEnv(this);
1082     final Path logDir = new Path(fileSystemManager.getRootDir(),
1083         MasterProcedureConstants.MASTER_PROCEDURE_LOGDIR);
1084 
1085     procedureStore = new WALProcedureStore(conf, fileSystemManager.getFileSystem(), logDir,
1086         new MasterProcedureEnv.WALStoreLeaseRecovery(this));
1087     procedureStore.registerListener(new MasterProcedureEnv.MasterProcedureStoreListener(this));
1088     procedureExecutor = new ProcedureExecutor(conf, procEnv, procedureStore,
1089         procEnv.getProcedureQueue());
1090 
1091     final int numThreads = conf.getInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS,
1092         Math.max(Runtime.getRuntime().availableProcessors(),
1093           MasterProcedureConstants.DEFAULT_MIN_MASTER_PROCEDURE_THREADS));
1094     procedureStore.start(numThreads);
1095     procedureExecutor.start(numThreads);
1096   }
1097 
1098   private void stopProcedureExecutor() {
1099     if (procedureExecutor != null) {
1100       procedureExecutor.stop();
1101     }
1102 
1103     if (procedureStore != null) {
1104       procedureStore.stop(isAborted());
1105     }
1106   }
1107 
1108   private void stopChores() {
1109     if (this.balancerChore != null) {
1110       this.balancerChore.cancel(true);
1111     }
1112     if (this.clusterStatusChore != null) {
1113       this.clusterStatusChore.cancel(true);
1114     }
1115     if (this.catalogJanitorChore != null) {
1116       this.catalogJanitorChore.cancel(true);
1117     }
1118     if (this.clusterStatusPublisherChore != null){
1119       clusterStatusPublisherChore.cancel(true);
1120     }
1121   }
1122 
1123   /**
1124    * @return Get remote side's InetAddress
1125    * @throws UnknownHostException
1126    */
1127   InetAddress getRemoteInetAddress(final int port,
1128       final long serverStartCode) throws UnknownHostException {
1129     // Do it out here in its own little method so can fake an address when
1130     // mocking up in tests.
1131     InetAddress ia = RpcServer.getRemoteIp();
1132 
1133     // The call could be from the local regionserver,
1134     // in which case, there is no remote address.
1135     if (ia == null && serverStartCode == startcode) {
1136       InetSocketAddress isa = rpcServices.getSocketAddress();
1137       if (isa != null && isa.getPort() == port) {
1138         ia = isa.getAddress();
1139       }
1140     }
1141     return ia;
1142   }
1143 
1144   /**
1145    * @return Maximum time we should run balancer for
1146    */
1147   private int getBalancerCutoffTime() {
1148     int balancerCutoffTime = getConfiguration().getInt("hbase.balancer.max.balancing", -1);
1149     if (balancerCutoffTime == -1) {
1150       // if cutoff time isn't set, defaulting it to period time
1151       int balancerPeriod = getConfiguration().getInt("hbase.balancer.period", 300000);
1152       balancerCutoffTime = balancerPeriod;
1153     }
1154     return balancerCutoffTime;
1155   }
1156 
1157   public boolean balance() throws IOException {
1158     // if master not initialized, don't run balancer.
1159     if (!this.initialized) {
1160       LOG.debug("Master has not been initialized, don't run balancer.");
1161       return false;
1162     }
1163     // Do this call outside of synchronized block.
1164     int maximumBalanceTime = getBalancerCutoffTime();
1165     synchronized (this.balancer) {
1166       // If balance not true, don't run balancer.
1167       if (!this.loadBalancerTracker.isBalancerOn()) return false;
1168       // Only allow one balance run at at time.
1169       if (this.assignmentManager.getRegionStates().isRegionsInTransition()) {
1170         Map<String, RegionState> regionsInTransition =
1171           this.assignmentManager.getRegionStates().getRegionsInTransition();
1172         LOG.debug("Not running balancer because " + regionsInTransition.size() +
1173           " region(s) in transition: " + org.apache.commons.lang.StringUtils.
1174             abbreviate(regionsInTransition.toString(), 256));
1175         return false;
1176       }
1177       if (this.serverManager.areDeadServersInProgress()) {
1178         LOG.debug("Not running balancer because processing dead regionserver(s): " +
1179           this.serverManager.getDeadServers());
1180         return false;
1181       }
1182 
1183       if (this.cpHost != null) {
1184         try {
1185           if (this.cpHost.preBalance()) {
1186             LOG.debug("Coprocessor bypassing balancer request");
1187             return false;
1188           }
1189         } catch (IOException ioe) {
1190           LOG.error("Error invoking master coprocessor preBalance()", ioe);
1191           return false;
1192         }
1193       }
1194 
1195       Map<TableName, Map<ServerName, List<HRegionInfo>>> assignmentsByTable =
1196         this.assignmentManager.getRegionStates().getAssignmentsByTable();
1197 
1198       List<RegionPlan> plans = new ArrayList<RegionPlan>();
1199       //Give the balancer the current cluster state.
1200       this.balancer.setClusterStatus(getClusterStatus());
1201       for (Map<ServerName, List<HRegionInfo>> assignments : assignmentsByTable.values()) {
1202         List<RegionPlan> partialPlans = this.balancer.balanceCluster(assignments);
1203         if (partialPlans != null) plans.addAll(partialPlans);
1204       }
1205       long cutoffTime = System.currentTimeMillis() + maximumBalanceTime;
1206       int rpCount = 0;  // number of RegionPlans balanced so far
1207       long totalRegPlanExecTime = 0;
1208       if (plans != null && !plans.isEmpty()) {
1209         for (RegionPlan plan: plans) {
1210           LOG.info("balance " + plan);
1211           long balStartTime = System.currentTimeMillis();
1212           //TODO: bulk assign
1213           this.assignmentManager.balance(plan);
1214           totalRegPlanExecTime += System.currentTimeMillis()-balStartTime;
1215           rpCount++;
1216           if (rpCount < plans.size() &&
1217               // if performing next balance exceeds cutoff time, exit the loop
1218               (System.currentTimeMillis() + (totalRegPlanExecTime / rpCount)) > cutoffTime) {
1219             //TODO: After balance, there should not be a cutoff time (keeping it as
1220             // a security net for now)
1221             LOG.debug("No more balancing till next balance run; maximumBalanceTime=" +
1222               maximumBalanceTime);
1223             break;
1224           }
1225         }
1226       }
1227       if (this.cpHost != null) {
1228         try {
1229           this.cpHost.postBalance(rpCount < plans.size() ? plans.subList(0, rpCount) : plans);
1230         } catch (IOException ioe) {
1231           // balancing already succeeded so don't change the result
1232           LOG.error("Error invoking master coprocessor postBalance()", ioe);
1233         }
1234       }
1235     }
1236     // If LoadBalancer did not generate any plans, it means the cluster is already balanced.
1237     // Return true indicating a success.
1238     return true;
1239   }
1240 
1241   /**
1242    * @return Client info for use as prefix on an audit log string; who did an action
1243    */
1244   String getClientIdAuditPrefix() {
1245     return "Client=" + RpcServer.getRequestUserName() + "/" + RpcServer.getRemoteAddress();
1246   }
1247 
1248   /**
1249    * Switch for the background CatalogJanitor thread.
1250    * Used for testing.  The thread will continue to run.  It will just be a noop
1251    * if disabled.
1252    * @param b If false, the catalog janitor won't do anything.
1253    */
1254   public void setCatalogJanitorEnabled(final boolean b) {
1255     this.catalogJanitorChore.setEnabled(b);
1256   }
1257 
1258   @Override
1259   public void dispatchMergingRegions(final HRegionInfo region_a,
1260       final HRegionInfo region_b, final boolean forcible) throws IOException {
1261     checkInitialized();
1262     this.service.submit(new DispatchMergingRegionHandler(this,
1263         this.catalogJanitorChore, region_a, region_b, forcible));
1264   }
1265 
1266   void move(final byte[] encodedRegionName,
1267       final byte[] destServerName) throws HBaseIOException {
1268     RegionState regionState = assignmentManager.getRegionStates().
1269       getRegionState(Bytes.toString(encodedRegionName));
1270     if (regionState == null) {
1271       throw new UnknownRegionException(Bytes.toStringBinary(encodedRegionName));
1272     }
1273 
1274     HRegionInfo hri = regionState.getRegion();
1275     ServerName dest;
1276     if (destServerName == null || destServerName.length == 0) {
1277       LOG.info("Passed destination servername is null/empty so " +
1278         "choosing a server at random");
1279       final List<ServerName> destServers = this.serverManager.createDestinationServersList(
1280         regionState.getServerName());
1281       dest = balancer.randomAssignment(hri, destServers);
1282       if (dest == null) {
1283         LOG.debug("Unable to determine a plan to assign " + hri);
1284         return;
1285       }
1286     } else {
1287       dest = ServerName.valueOf(Bytes.toString(destServerName));
1288       if (dest.equals(serverName) && balancer instanceof BaseLoadBalancer
1289           && !((BaseLoadBalancer)balancer).shouldBeOnMaster(hri)) {
1290         // To avoid unnecessary region moving later by balancer. Don't put user
1291         // regions on master. Regions on master could be put on other region
1292         // server intentionally by test however.
1293         LOG.debug("Skipping move of region " + hri.getRegionNameAsString()
1294           + " to avoid unnecessary region moving later by load balancer,"
1295           + " because it should not be on master");
1296         return;
1297       }
1298     }
1299 
1300     if (dest.equals(regionState.getServerName())) {
1301       LOG.debug("Skipping move of region " + hri.getRegionNameAsString()
1302         + " because region already assigned to the same server " + dest + ".");
1303       return;
1304     }
1305 
1306     // Now we can do the move
1307     RegionPlan rp = new RegionPlan(hri, regionState.getServerName(), dest);
1308 
1309     try {
1310       checkInitialized();
1311       if (this.cpHost != null) {
1312         if (this.cpHost.preMove(hri, rp.getSource(), rp.getDestination())) {
1313           return;
1314         }
1315       }
1316       // warmup the region on the destination before initiating the move. this call
1317       // is synchronous and takes some time. doing it before the source region gets
1318       // closed
1319       serverManager.sendRegionWarmup(rp.getDestination(), hri);
1320 
1321       LOG.info(getClientIdAuditPrefix() + " move " + rp + ", running balancer");
1322       this.assignmentManager.balance(rp);
1323       if (this.cpHost != null) {
1324         this.cpHost.postMove(hri, rp.getSource(), rp.getDestination());
1325       }
1326     } catch (IOException ioe) {
1327       if (ioe instanceof HBaseIOException) {
1328         throw (HBaseIOException)ioe;
1329       }
1330       throw new HBaseIOException(ioe);
1331     }
1332   }
1333 
1334   @Override
1335   public long createTable(HTableDescriptor hTableDescriptor,
1336       byte [][] splitKeys) throws IOException {
1337     if (isStopped()) {
1338       throw new MasterNotRunningException();
1339     }
1340 
1341     String namespace = hTableDescriptor.getTableName().getNamespaceAsString();
1342     ensureNamespaceExists(namespace);
1343 
1344     HRegionInfo[] newRegions = ModifyRegionUtils.createHRegionInfos(hTableDescriptor, splitKeys);
1345     checkInitialized();
1346     sanityCheckTableDescriptor(hTableDescriptor);
1347     this.quotaManager.checkNamespaceTableAndRegionQuota(hTableDescriptor.getTableName(),
1348       newRegions.length);
1349     if (cpHost != null) {
1350       cpHost.preCreateTable(hTableDescriptor, newRegions);
1351     }
1352     LOG.info(getClientIdAuditPrefix() + " create " + hTableDescriptor);
1353 
1354     // TODO: We can handle/merge duplicate requests, and differentiate the case of
1355     //       TableExistsException by saying if the schema is the same or not.
1356     ProcedurePrepareLatch latch = ProcedurePrepareLatch.createLatch();
1357     long procId = this.procedureExecutor.submitProcedure(
1358       new CreateTableProcedure(procedureExecutor.getEnvironment(),
1359         hTableDescriptor, newRegions, latch));
1360     latch.await();
1361 
1362     if (cpHost != null) {
1363       cpHost.postCreateTable(hTableDescriptor, newRegions);
1364     }
1365 
1366     return procId;
1367   }
1368 
1369   /**
1370    * Checks whether the table conforms to some sane limits, and configured
1371    * values (compression, etc) work. Throws an exception if something is wrong.
1372    * @throws IOException
1373    */
1374   private void sanityCheckTableDescriptor(final HTableDescriptor htd) throws IOException {
1375     final String CONF_KEY = "hbase.table.sanity.checks";
1376     boolean logWarn = false;
1377     if (!conf.getBoolean(CONF_KEY, true)) {
1378       logWarn = true;
1379     }
1380     String tableVal = htd.getConfigurationValue(CONF_KEY);
1381     if (tableVal != null && !Boolean.valueOf(tableVal)) {
1382       logWarn = true;
1383     }
1384 
1385     // check max file size
1386     long maxFileSizeLowerLimit = 2 * 1024 * 1024L; // 2M is the default lower limit
1387     long maxFileSize = htd.getMaxFileSize();
1388     if (maxFileSize < 0) {
1389       maxFileSize = conf.getLong(HConstants.HREGION_MAX_FILESIZE, maxFileSizeLowerLimit);
1390     }
1391     if (maxFileSize < conf.getLong("hbase.hregion.max.filesize.limit", maxFileSizeLowerLimit)) {
1392       String message = "MAX_FILESIZE for table descriptor or "
1393           + "\"hbase.hregion.max.filesize\" (" + maxFileSize
1394           + ") is too small, which might cause over splitting into unmanageable "
1395           + "number of regions.";
1396       warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);
1397     }
1398 
1399     // check flush size
1400     long flushSizeLowerLimit = 1024 * 1024L; // 1M is the default lower limit
1401     long flushSize = htd.getMemStoreFlushSize();
1402     if (flushSize < 0) {
1403       flushSize = conf.getLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, flushSizeLowerLimit);
1404     }
1405     if (flushSize < conf.getLong("hbase.hregion.memstore.flush.size.limit", flushSizeLowerLimit)) {
1406       String message = "MEMSTORE_FLUSHSIZE for table descriptor or "
1407           + "\"hbase.hregion.memstore.flush.size\" ("+flushSize+") is too small, which might cause"
1408           + " very frequent flushing.";
1409       warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);
1410     }
1411 
1412     // check that coprocessors and other specified plugin classes can be loaded
1413     try {
1414       checkClassLoading(conf, htd);
1415     } catch (Exception ex) {
1416       warnOrThrowExceptionForFailure(logWarn, CONF_KEY, ex.getMessage(), null);
1417     }
1418 
1419     // check compression can be loaded
1420     try {
1421       checkCompression(htd);
1422     } catch (IOException e) {
1423       warnOrThrowExceptionForFailure(logWarn, CONF_KEY, e.getMessage(), e);
1424     }
1425 
1426     // check encryption can be loaded
1427     try {
1428       checkEncryption(conf, htd);
1429     } catch (IOException e) {
1430       warnOrThrowExceptionForFailure(logWarn, CONF_KEY, e.getMessage(), e);
1431     }
1432 
1433     // check that we have at least 1 CF
1434     if (htd.getColumnFamilies().length == 0) {
1435       String message = "Table should have at least one column family.";
1436       warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);
1437     }
1438 
1439     for (HColumnDescriptor hcd : htd.getColumnFamilies()) {
1440       if (hcd.getTimeToLive() <= 0) {
1441         String message = "TTL for column family " + hcd.getNameAsString() + " must be positive.";
1442         warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);
1443       }
1444 
1445       // check blockSize
1446       if (hcd.getBlocksize() < 1024 || hcd.getBlocksize() > 16 * 1024 * 1024) {
1447         String message = "Block size for column family " + hcd.getNameAsString()
1448             + "  must be between 1K and 16MB.";
1449         warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);
1450       }
1451 
1452       // check versions
1453       if (hcd.getMinVersions() < 0) {
1454         String message = "Min versions for column family " + hcd.getNameAsString()
1455           + "  must be positive.";
1456         warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);
1457       }
1458       // max versions already being checked
1459 
1460       // check replication scope
1461       if (hcd.getScope() < 0) {
1462         String message = "Replication scope for column family "
1463           + hcd.getNameAsString() + "  must be positive.";
1464         warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);
1465       }
1466 
1467       // TODO: should we check coprocessors and encryption ?
1468     }
1469   }
1470 
1471   // HBASE-13350 - Helper method to log warning on sanity check failures if checks disabled.
1472   private static void warnOrThrowExceptionForFailure(boolean logWarn, String confKey,
1473       String message, Exception cause) throws IOException {
1474     if (!logWarn) {
1475       throw new DoNotRetryIOException(message + " Set " + confKey +
1476           " to false at conf or table descriptor if you want to bypass sanity checks", cause);
1477     }
1478     LOG.warn(message);
1479   }
1480 
1481   private void startActiveMasterManager(int infoPort) throws KeeperException {
1482     String backupZNode = ZKUtil.joinZNode(
1483       zooKeeper.backupMasterAddressesZNode, serverName.toString());
1484     /*
1485     * Add a ZNode for ourselves in the backup master directory since we
1486     * may not become the active master. If so, we want the actual active
1487     * master to know we are backup masters, so that it won't assign
1488     * regions to us if so configured.
1489     *
1490     * If we become the active master later, ActiveMasterManager will delete
1491     * this node explicitly.  If we crash before then, ZooKeeper will delete
1492     * this node for us since it is ephemeral.
1493     */
1494     LOG.info("Adding backup master ZNode " + backupZNode);
1495     if (!MasterAddressTracker.setMasterAddress(zooKeeper, backupZNode,
1496         serverName, infoPort)) {
1497       LOG.warn("Failed create of " + backupZNode + " by " + serverName);
1498     }
1499 
1500     activeMasterManager.setInfoPort(infoPort);
1501     // Start a thread to try to become the active master, so we won't block here
1502     Threads.setDaemonThreadRunning(new Thread(new Runnable() {
1503       @Override
1504       public void run() {
1505         int timeout = conf.getInt(HConstants.ZK_SESSION_TIMEOUT,
1506           HConstants.DEFAULT_ZK_SESSION_TIMEOUT);
1507         // If we're a backup master, stall until a primary to writes his address
1508         if (conf.getBoolean(HConstants.MASTER_TYPE_BACKUP,
1509             HConstants.DEFAULT_MASTER_TYPE_BACKUP)) {
1510           LOG.debug("HMaster started in backup mode. "
1511             + "Stalling until master znode is written.");
1512           // This will only be a minute or so while the cluster starts up,
1513           // so don't worry about setting watches on the parent znode
1514           while (!activeMasterManager.hasActiveMaster()) {
1515             LOG.debug("Waiting for master address ZNode to be written "
1516               + "(Also watching cluster state node)");
1517             Threads.sleep(timeout);
1518           }
1519         }
1520         MonitoredTask status = TaskMonitor.get().createStatus("Master startup");
1521         status.setDescription("Master startup");
1522         try {
1523           if (activeMasterManager.blockUntilBecomingActiveMaster(timeout, status)) {
1524             finishActiveMasterInitialization(status);
1525           }
1526         } catch (Throwable t) {
1527           status.setStatus("Failed to become active: " + t.getMessage());
1528           LOG.fatal("Failed to become active master", t);
1529           // HBASE-5680: Likely hadoop23 vs hadoop 20.x/1.x incompatibility
1530           if (t instanceof NoClassDefFoundError &&
1531               t.getMessage()
1532                   .contains("org/apache/hadoop/hdfs/protocol/FSConstants$SafeModeAction")) {
1533             // improved error message for this special case
1534             abort("HBase is having a problem with its Hadoop jars.  You may need to "
1535               + "recompile HBase against Hadoop version "
1536               +  org.apache.hadoop.util.VersionInfo.getVersion()
1537               + " or change your hadoop jars to start properly", t);
1538           } else {
1539             abort("Unhandled exception. Starting shutdown.", t);
1540           }
1541         } finally {
1542           status.cleanup();
1543         }
1544       }
1545     }, getServerName().toShortString() + ".activeMasterManager"));
1546   }
1547 
1548   private void checkCompression(final HTableDescriptor htd)
1549   throws IOException {
1550     if (!this.masterCheckCompression) return;
1551     for (HColumnDescriptor hcd : htd.getColumnFamilies()) {
1552       checkCompression(hcd);
1553     }
1554   }
1555 
1556   private void checkCompression(final HColumnDescriptor hcd)
1557   throws IOException {
1558     if (!this.masterCheckCompression) return;
1559     CompressionTest.testCompression(hcd.getCompression());
1560     CompressionTest.testCompression(hcd.getCompactionCompression());
1561   }
1562 
1563   private void checkEncryption(final Configuration conf, final HTableDescriptor htd)
1564   throws IOException {
1565     if (!this.masterCheckEncryption) return;
1566     for (HColumnDescriptor hcd : htd.getColumnFamilies()) {
1567       checkEncryption(conf, hcd);
1568     }
1569   }
1570 
1571   private void checkEncryption(final Configuration conf, final HColumnDescriptor hcd)
1572   throws IOException {
1573     if (!this.masterCheckEncryption) return;
1574     EncryptionTest.testEncryption(conf, hcd.getEncryptionType(), hcd.getEncryptionKey());
1575   }
1576 
1577   private void checkClassLoading(final Configuration conf, final HTableDescriptor htd)
1578   throws IOException {
1579     RegionSplitPolicy.getSplitPolicyClass(htd, conf);
1580     RegionCoprocessorHost.testTableCoprocessorAttrs(conf, htd);
1581   }
1582 
1583   private static boolean isCatalogTable(final TableName tableName) {
1584     return tableName.equals(TableName.META_TABLE_NAME);
1585   }
1586 
1587   @Override
1588   public long deleteTable(final TableName tableName) throws IOException {
1589     checkInitialized();
1590     if (cpHost != null) {
1591       cpHost.preDeleteTable(tableName);
1592     }
1593     LOG.info(getClientIdAuditPrefix() + " delete " + tableName);
1594 
1595     // TODO: We can handle/merge duplicate request
1596     ProcedurePrepareLatch latch = ProcedurePrepareLatch.createLatch();
1597     long procId = this.procedureExecutor.submitProcedure(
1598         new DeleteTableProcedure(procedureExecutor.getEnvironment(), tableName, latch));
1599     latch.await();
1600 
1601     if (cpHost != null) {
1602       cpHost.postDeleteTable(tableName);
1603     }
1604 
1605     return procId;
1606   }
1607 
1608   @Override
1609   public void truncateTable(TableName tableName, boolean preserveSplits) throws IOException {
1610     checkInitialized();
1611     if (cpHost != null) {
1612       cpHost.preTruncateTable(tableName);
1613     }
1614     LOG.info(getClientIdAuditPrefix() + " truncate " + tableName);
1615 
1616     long procId = this.procedureExecutor.submitProcedure(
1617         new TruncateTableProcedure(procedureExecutor.getEnvironment(), tableName, preserveSplits));
1618     ProcedureSyncWait.waitForProcedureToComplete(procedureExecutor, procId);
1619 
1620     if (cpHost != null) {
1621       cpHost.postTruncateTable(tableName);
1622     }
1623   }
1624 
1625   @Override
1626   public void addColumn(final TableName tableName, final HColumnDescriptor columnDescriptor)
1627       throws IOException {
1628     checkInitialized();
1629     checkCompression(columnDescriptor);
1630     checkEncryption(conf, columnDescriptor);
1631     if (cpHost != null) {
1632       if (cpHost.preAddColumn(tableName, columnDescriptor)) {
1633         return;
1634       }
1635     }
1636     // Execute the operation synchronously - wait for the operation to complete before continuing.
1637     long procId =
1638         this.procedureExecutor.submitProcedure(new AddColumnFamilyProcedure(procedureExecutor
1639             .getEnvironment(), tableName, columnDescriptor));
1640     ProcedureSyncWait.waitForProcedureToComplete(procedureExecutor, procId);
1641     if (cpHost != null) {
1642       cpHost.postAddColumn(tableName, columnDescriptor);
1643     }
1644   }
1645 
1646   @Override
1647   public void modifyColumn(TableName tableName, HColumnDescriptor descriptor)
1648       throws IOException {
1649     checkInitialized();
1650     checkCompression(descriptor);
1651     checkEncryption(conf, descriptor);
1652     if (cpHost != null) {
1653       if (cpHost.preModifyColumn(tableName, descriptor)) {
1654         return;
1655       }
1656     }
1657     LOG.info(getClientIdAuditPrefix() + " modify " + descriptor);
1658 
1659     // Execute the operation synchronously - wait for the operation to complete before continuing.
1660     long procId =
1661         this.procedureExecutor.submitProcedure(new ModifyColumnFamilyProcedure(procedureExecutor
1662             .getEnvironment(), tableName, descriptor));
1663     ProcedureSyncWait.waitForProcedureToComplete(procedureExecutor, procId);
1664 
1665     if (cpHost != null) {
1666       cpHost.postModifyColumn(tableName, descriptor);
1667     }
1668   }
1669 
1670   @Override
1671   public void deleteColumn(final TableName tableName, final byte[] columnName)
1672       throws IOException {
1673     checkInitialized();
1674     if (cpHost != null) {
1675       if (cpHost.preDeleteColumn(tableName, columnName)) {
1676         return;
1677       }
1678     }
1679     LOG.info(getClientIdAuditPrefix() + " delete " + Bytes.toString(columnName));
1680 
1681     // Execute the operation synchronously - wait for the operation to complete before continuing.
1682     long procId =
1683         this.procedureExecutor.submitProcedure(new DeleteColumnFamilyProcedure(procedureExecutor
1684             .getEnvironment(), tableName, columnName));
1685     ProcedureSyncWait.waitForProcedureToComplete(procedureExecutor, procId);
1686 
1687     if (cpHost != null) {
1688       cpHost.postDeleteColumn(tableName, columnName);
1689     }
1690   }
1691 
1692   @Override
1693   public long enableTable(final TableName tableName) throws IOException {
1694     checkInitialized();
1695     if (cpHost != null) {
1696       cpHost.preEnableTable(tableName);
1697     }
1698     LOG.info(getClientIdAuditPrefix() + " enable " + tableName);
1699 
1700     // Execute the operation asynchronously - client will check the progress of the operation
1701     final ProcedurePrepareLatch prepareLatch = ProcedurePrepareLatch.createLatch();
1702     long procId =
1703         this.procedureExecutor.submitProcedure(new EnableTableProcedure(procedureExecutor
1704             .getEnvironment(), tableName, false, prepareLatch));
1705     // Before returning to client, we want to make sure that the table is prepared to be
1706     // enabled (the table is locked and the table state is set).
1707     //
1708     // Note: if the procedure throws exception, we will catch it and rethrow.
1709     prepareLatch.await();
1710 
1711     if (cpHost != null) {
1712       cpHost.postEnableTable(tableName);
1713     }
1714 
1715     return procId;
1716   }
1717 
1718   @Override
1719   public long disableTable(final TableName tableName) throws IOException {
1720     checkInitialized();
1721     if (cpHost != null) {
1722       cpHost.preDisableTable(tableName);
1723     }
1724     LOG.info(getClientIdAuditPrefix() + " disable " + tableName);
1725 
1726     // Execute the operation asynchronously - client will check the progress of the operation
1727     final ProcedurePrepareLatch prepareLatch = ProcedurePrepareLatch.createLatch();
1728     // Execute the operation asynchronously - client will check the progress of the operation
1729     long procId =
1730         this.procedureExecutor.submitProcedure(new DisableTableProcedure(procedureExecutor
1731             .getEnvironment(), tableName, false, prepareLatch));
1732     // Before returning to client, we want to make sure that the table is prepared to be
1733     // enabled (the table is locked and the table state is set).
1734     //
1735     // Note: if the procedure throws exception, we will catch it and rethrow.
1736     prepareLatch.await();
1737 
1738     if (cpHost != null) {
1739       cpHost.postDisableTable(tableName);
1740     }
1741 
1742     return procId;
1743   }
1744 
1745   /**
1746    * Return the region and current deployment for the region containing
1747    * the given row. If the region cannot be found, returns null. If it
1748    * is found, but not currently deployed, the second element of the pair
1749    * may be null.
1750    */
1751   @VisibleForTesting // Used by TestMaster.
1752   Pair<HRegionInfo, ServerName> getTableRegionForRow(
1753       final TableName tableName, final byte [] rowKey)
1754   throws IOException {
1755     final AtomicReference<Pair<HRegionInfo, ServerName>> result =
1756       new AtomicReference<Pair<HRegionInfo, ServerName>>(null);
1757 
1758     MetaTableAccessor.Visitor visitor = new MetaTableAccessor.Visitor() {
1759         @Override
1760         public boolean visit(Result data) throws IOException {
1761           if (data == null || data.size() <= 0) {
1762             return true;
1763           }
1764           Pair<HRegionInfo, ServerName> pair = HRegionInfo.getHRegionInfoAndServerName(data);
1765           if (pair == null) {
1766             return false;
1767           }
1768           if (!pair.getFirst().getTable().equals(tableName)) {
1769             return false;
1770           }
1771           result.set(pair);
1772           return true;
1773         }
1774     };
1775 
1776     MetaTableAccessor.scanMeta(clusterConnection, visitor, tableName, rowKey, 1);
1777     return result.get();
1778   }
1779 
1780   @Override
1781   public void modifyTable(final TableName tableName, final HTableDescriptor descriptor)
1782       throws IOException {
1783     checkInitialized();
1784     sanityCheckTableDescriptor(descriptor);
1785     if (cpHost != null) {
1786       cpHost.preModifyTable(tableName, descriptor);
1787     }
1788 
1789     LOG.info(getClientIdAuditPrefix() + " modify " + tableName);
1790 
1791     // Execute the operation synchronously - wait for the operation completes before continuing.
1792     long procId = this.procedureExecutor.submitProcedure(
1793         new ModifyTableProcedure(procedureExecutor.getEnvironment(), descriptor));
1794 
1795     ProcedureSyncWait.waitForProcedureToComplete(procedureExecutor, procId);
1796 
1797     if (cpHost != null) {
1798       cpHost.postModifyTable(tableName, descriptor);
1799     }
1800   }
1801 
1802   @Override
1803   public void checkTableModifiable(final TableName tableName)
1804       throws IOException, TableNotFoundException, TableNotDisabledException {
1805     if (isCatalogTable(tableName)) {
1806       throw new IOException("Can't modify catalog tables");
1807     }
1808     if (!MetaTableAccessor.tableExists(getConnection(), tableName)) {
1809       throw new TableNotFoundException(tableName);
1810     }
1811     if (!getAssignmentManager().getTableStateManager().
1812         isTableState(tableName, TableState.State.DISABLED)) {
1813       throw new TableNotDisabledException(tableName);
1814     }
1815   }
1816 
1817   /**
1818    * @return cluster status
1819    */
1820   public ClusterStatus getClusterStatus() throws InterruptedIOException {
1821     // Build Set of backup masters from ZK nodes
1822     List<String> backupMasterStrings;
1823     try {
1824       backupMasterStrings = ZKUtil.listChildrenNoWatch(this.zooKeeper,
1825         this.zooKeeper.backupMasterAddressesZNode);
1826     } catch (KeeperException e) {
1827       LOG.warn(this.zooKeeper.prefix("Unable to list backup servers"), e);
1828       backupMasterStrings = null;
1829     }
1830 
1831     List<ServerName> backupMasters = null;
1832     if (backupMasterStrings != null && !backupMasterStrings.isEmpty()) {
1833       backupMasters = new ArrayList<ServerName>(backupMasterStrings.size());
1834       for (String s: backupMasterStrings) {
1835         try {
1836           byte [] bytes;
1837           try {
1838             bytes = ZKUtil.getData(this.zooKeeper, ZKUtil.joinZNode(
1839                 this.zooKeeper.backupMasterAddressesZNode, s));
1840           } catch (InterruptedException e) {
1841             throw new InterruptedIOException();
1842           }
1843           if (bytes != null) {
1844             ServerName sn;
1845             try {
1846               sn = ServerName.parseFrom(bytes);
1847             } catch (DeserializationException e) {
1848               LOG.warn("Failed parse, skipping registering backup server", e);
1849               continue;
1850             }
1851             backupMasters.add(sn);
1852           }
1853         } catch (KeeperException e) {
1854           LOG.warn(this.zooKeeper.prefix("Unable to get information about " +
1855                    "backup servers"), e);
1856         }
1857       }
1858       Collections.sort(backupMasters, new Comparator<ServerName>() {
1859         @Override
1860         public int compare(ServerName s1, ServerName s2) {
1861           return s1.getServerName().compareTo(s2.getServerName());
1862         }});
1863     }
1864 
1865     String clusterId = fileSystemManager != null ?
1866       fileSystemManager.getClusterId().toString() : null;
1867     Map<String, RegionState> regionsInTransition = assignmentManager != null ?
1868       assignmentManager.getRegionStates().getRegionsInTransition() : null;
1869     String[] coprocessors = cpHost != null ? getMasterCoprocessors() : null;
1870     boolean balancerOn = loadBalancerTracker != null ?
1871       loadBalancerTracker.isBalancerOn() : false;
1872     Map<ServerName, ServerLoad> onlineServers = null;
1873     Set<ServerName> deadServers = null;
1874     if (serverManager != null) {
1875       deadServers = serverManager.getDeadServers().copyServerNames();
1876       onlineServers = serverManager.getOnlineServers();
1877     }
1878     return new ClusterStatus(VersionInfo.getVersion(), clusterId,
1879       onlineServers, deadServers, serverName, backupMasters,
1880       regionsInTransition, coprocessors, balancerOn);
1881   }
1882 
1883   /**
1884    * The set of loaded coprocessors is stored in a static set. Since it's
1885    * statically allocated, it does not require that HMaster's cpHost be
1886    * initialized prior to accessing it.
1887    * @return a String representation of the set of names of the loaded
1888    * coprocessors.
1889    */
1890   public static String getLoadedCoprocessors() {
1891     return CoprocessorHost.getLoadedCoprocessors().toString();
1892   }
1893 
1894   /**
1895    * @return timestamp in millis when HMaster was started.
1896    */
1897   public long getMasterStartTime() {
1898     return startcode;
1899   }
1900 
1901   /**
1902    * @return timestamp in millis when HMaster became the active master.
1903    */
1904   public long getMasterActiveTime() {
1905     return masterActiveTime;
1906   }
1907 
1908   public int getRegionServerInfoPort(final ServerName sn) {
1909     RegionServerInfo info = this.regionServerTracker.getRegionServerInfo(sn);
1910     if (info == null || info.getInfoPort() == 0) {
1911       return conf.getInt(HConstants.REGIONSERVER_INFO_PORT,
1912         HConstants.DEFAULT_REGIONSERVER_INFOPORT);
1913     }
1914     return info.getInfoPort();
1915   }
1916 
1917   /**
1918    * @return array of coprocessor SimpleNames.
1919    */
1920   public String[] getMasterCoprocessors() {
1921     Set<String> masterCoprocessors = getMasterCoprocessorHost().getCoprocessors();
1922     return masterCoprocessors.toArray(new String[masterCoprocessors.size()]);
1923   }
1924 
1925   @Override
1926   public void abort(final String msg, final Throwable t) {
1927     if (isAborted() || isStopped()) {
1928       return;
1929     }
1930     if (cpHost != null) {
1931       // HBASE-4014: dump a list of loaded coprocessors.
1932       LOG.fatal("Master server abort: loaded coprocessors are: " +
1933           getLoadedCoprocessors());
1934     }
1935     if (t != null) LOG.fatal(msg, t);
1936     stop(msg);
1937   }
1938 
1939   @Override
1940   public ZooKeeperWatcher getZooKeeper() {
1941     return zooKeeper;
1942   }
1943 
1944   @Override
1945   public MasterCoprocessorHost getMasterCoprocessorHost() {
1946     return cpHost;
1947   }
1948 
1949   @Override
1950   public MasterQuotaManager getMasterQuotaManager() {
1951     return quotaManager;
1952   }
1953 
1954   @Override
1955   public ProcedureExecutor<MasterProcedureEnv> getMasterProcedureExecutor() {
1956     return procedureExecutor;
1957   }
1958 
1959   @Override
1960   public ServerName getServerName() {
1961     return this.serverName;
1962   }
1963 
1964   @Override
1965   public AssignmentManager getAssignmentManager() {
1966     return this.assignmentManager;
1967   }
1968 
1969   public MemoryBoundedLogMessageBuffer getRegionServerFatalLogBuffer() {
1970     return rsFatals;
1971   }
1972 
1973   public void shutdown() {
1974     if (cpHost != null) {
1975       try {
1976         cpHost.preShutdown();
1977       } catch (IOException ioe) {
1978         LOG.error("Error call master coprocessor preShutdown()", ioe);
1979       }
1980     }
1981 
1982     if (this.serverManager != null) {
1983       this.serverManager.shutdownCluster();
1984     }
1985     if (this.clusterStatusTracker != null){
1986       try {
1987         this.clusterStatusTracker.setClusterDown();
1988       } catch (KeeperException e) {
1989         LOG.error("ZooKeeper exception trying to set cluster as down in ZK", e);
1990       }
1991     }
1992   }
1993 
1994   public void stopMaster() {
1995     if (cpHost != null) {
1996       try {
1997         cpHost.preStopMaster();
1998       } catch (IOException ioe) {
1999         LOG.error("Error call master coprocessor preStopMaster()", ioe);
2000       }
2001     }
2002     stop("Stopped by " + Thread.currentThread().getName());
2003   }
2004 
2005   void checkServiceStarted() throws ServerNotRunningYetException {
2006     if (!serviceStarted) {
2007       throw new ServerNotRunningYetException("Server is not running yet");
2008     }
2009   }
2010 
2011   void checkInitialized() throws PleaseHoldException, ServerNotRunningYetException {
2012     checkServiceStarted();
2013     if (!this.initialized) {
2014       throw new PleaseHoldException("Master is initializing");
2015     }
2016   }
2017 
2018   void checkNamespaceManagerReady() throws IOException {
2019     checkInitialized();
2020     if (tableNamespaceManager == null ||
2021         !tableNamespaceManager.isTableAvailableAndInitialized()) {
2022       throw new IOException("Table Namespace Manager not ready yet, try again later");
2023     }
2024   }
2025   /**
2026    * Report whether this master is currently the active master or not.
2027    * If not active master, we are parked on ZK waiting to become active.
2028    *
2029    * This method is used for testing.
2030    *
2031    * @return true if active master, false if not.
2032    */
2033   public boolean isActiveMaster() {
2034     return isActiveMaster;
2035   }
2036 
2037   /**
2038    * Report whether this master has completed with its initialization and is
2039    * ready.  If ready, the master is also the active master.  A standby master
2040    * is never ready.
2041    *
2042    * This method is used for testing.
2043    *
2044    * @return true if master is ready to go, false if not.
2045    */
2046   @Override
2047   public boolean isInitialized() {
2048     return initialized;
2049   }
2050 
2051   /**
2052    * ServerShutdownHandlerEnabled is set false before completing
2053    * assignMeta to prevent processing of ServerShutdownHandler.
2054    * @return true if assignMeta has completed;
2055    */
2056   @Override
2057   public boolean isServerShutdownHandlerEnabled() {
2058     return this.serverShutdownHandlerEnabled;
2059   }
2060 
2061   /**
2062    * Report whether this master has started initialization and is about to do meta region assignment
2063    * @return true if master is in initialization & about to assign hbase:meta regions
2064    */
2065   public boolean isInitializationStartsMetaRegionAssignment() {
2066     return this.initializationBeforeMetaAssignment;
2067   }
2068 
2069   public void assignRegion(HRegionInfo hri) {
2070     assignmentManager.assign(hri);
2071   }
2072 
2073   /**
2074    * Compute the average load across all region servers.
2075    * Currently, this uses a very naive computation - just uses the number of
2076    * regions being served, ignoring stats about number of requests.
2077    * @return the average load
2078    */
2079   public double getAverageLoad() {
2080     if (this.assignmentManager == null) {
2081       return 0;
2082     }
2083 
2084     RegionStates regionStates = this.assignmentManager.getRegionStates();
2085     if (regionStates == null) {
2086       return 0;
2087     }
2088     return regionStates.getAverageLoad();
2089   }
2090 
2091   @Override
2092   public boolean registerService(Service instance) {
2093     /*
2094      * No stacking of instances is allowed for a single service name
2095      */
2096     Descriptors.ServiceDescriptor serviceDesc = instance.getDescriptorForType();
2097     if (coprocessorServiceHandlers.containsKey(serviceDesc.getFullName())) {
2098       LOG.error("Coprocessor service "+serviceDesc.getFullName()+
2099           " already registered, rejecting request from "+instance
2100       );
2101       return false;
2102     }
2103 
2104     coprocessorServiceHandlers.put(serviceDesc.getFullName(), instance);
2105     if (LOG.isDebugEnabled()) {
2106       LOG.debug("Registered master coprocessor service: service="+serviceDesc.getFullName());
2107     }
2108     return true;
2109   }
2110 
2111   /**
2112    * Utility for constructing an instance of the passed HMaster class.
2113    * @param masterClass
2114    * @param conf
2115    * @return HMaster instance.
2116    */
2117   public static HMaster constructMaster(Class<? extends HMaster> masterClass,
2118       final Configuration conf, final CoordinatedStateManager cp)  {
2119     try {
2120       Constructor<? extends HMaster> c =
2121         masterClass.getConstructor(Configuration.class, CoordinatedStateManager.class);
2122       return c.newInstance(conf, cp);
2123     } catch (InvocationTargetException ite) {
2124       Throwable target = ite.getTargetException() != null?
2125         ite.getTargetException(): ite;
2126       if (target.getCause() != null) target = target.getCause();
2127       throw new RuntimeException("Failed construction of Master: " +
2128         masterClass.toString(), target);
2129     } catch (Exception e) {
2130       throw new RuntimeException("Failed construction of Master: " +
2131         masterClass.toString() + ((e.getCause() != null)?
2132           e.getCause().getMessage(): ""), e);
2133     }
2134   }
2135 
2136   /**
2137    * @see org.apache.hadoop.hbase.master.HMasterCommandLine
2138    */
2139   public static void main(String [] args) {
2140     VersionInfo.logVersion();
2141     new HMasterCommandLine(HMaster.class).doMain(args);
2142   }
2143 
2144   public HFileCleaner getHFileCleaner() {
2145     return this.hfileCleaner;
2146   }
2147 
2148   /**
2149    * Exposed for TESTING!
2150    * @return the underlying snapshot manager
2151    */
2152   public SnapshotManager getSnapshotManagerForTesting() {
2153     return this.snapshotManager;
2154   }
2155 
2156   @Override
2157   public void createNamespace(NamespaceDescriptor descriptor) throws IOException {
2158     TableName.isLegalNamespaceName(Bytes.toBytes(descriptor.getName()));
2159     checkNamespaceManagerReady();
2160     if (cpHost != null) {
2161       if (cpHost.preCreateNamespace(descriptor)) {
2162         return;
2163       }
2164     }
2165     LOG.info(getClientIdAuditPrefix() + " creating " + descriptor);
2166     tableNamespaceManager.create(descriptor);
2167     if (cpHost != null) {
2168       cpHost.postCreateNamespace(descriptor);
2169     }
2170   }
2171 
2172   @Override
2173   public void modifyNamespace(NamespaceDescriptor descriptor) throws IOException {
2174     TableName.isLegalNamespaceName(Bytes.toBytes(descriptor.getName()));
2175     checkNamespaceManagerReady();
2176     if (cpHost != null) {
2177       if (cpHost.preModifyNamespace(descriptor)) {
2178         return;
2179       }
2180     }
2181     LOG.info(getClientIdAuditPrefix() + " modify " + descriptor);
2182     tableNamespaceManager.update(descriptor);
2183     if (cpHost != null) {
2184       cpHost.postModifyNamespace(descriptor);
2185     }
2186   }
2187 
2188   @Override
2189   public void deleteNamespace(String name) throws IOException {
2190     checkNamespaceManagerReady();
2191     if (cpHost != null) {
2192       if (cpHost.preDeleteNamespace(name)) {
2193         return;
2194       }
2195     }
2196     LOG.info(getClientIdAuditPrefix() + " delete " + name);
2197     tableNamespaceManager.remove(name);
2198     if (cpHost != null) {
2199       cpHost.postDeleteNamespace(name);
2200     }
2201   }
2202 
2203   /**
2204    * Ensure that the specified namespace exists, otherwise throws a NamespaceNotFoundException
2205    *
2206    * @param name the namespace to check
2207    * @throws IOException if the namespace manager is not ready yet.
2208    * @throws NamespaceNotFoundException if the namespace does not exists
2209    */
2210   private void ensureNamespaceExists(final String name)
2211       throws IOException, NamespaceNotFoundException {
2212     checkNamespaceManagerReady();
2213     NamespaceDescriptor nsd = tableNamespaceManager.get(name);
2214     if (nsd == null) {
2215       throw new NamespaceNotFoundException(name);
2216     }
2217   }
2218 
2219   @Override
2220   public NamespaceDescriptor getNamespaceDescriptor(String name) throws IOException {
2221     checkNamespaceManagerReady();
2222 
2223     if (cpHost != null) {
2224       cpHost.preGetNamespaceDescriptor(name);
2225     }
2226 
2227     NamespaceDescriptor nsd = tableNamespaceManager.get(name);
2228     if (nsd == null) {
2229       throw new NamespaceNotFoundException(name);
2230     }
2231 
2232     if (cpHost != null) {
2233       cpHost.postGetNamespaceDescriptor(nsd);
2234     }
2235 
2236     return nsd;
2237   }
2238 
2239   @Override
2240   public List<NamespaceDescriptor> listNamespaceDescriptors() throws IOException {
2241     checkNamespaceManagerReady();
2242 
2243     final List<NamespaceDescriptor> descriptors = new ArrayList<NamespaceDescriptor>();
2244     boolean bypass = false;
2245     if (cpHost != null) {
2246       bypass = cpHost.preListNamespaceDescriptors(descriptors);
2247     }
2248 
2249     if (!bypass) {
2250       descriptors.addAll(tableNamespaceManager.list());
2251 
2252       if (cpHost != null) {
2253         cpHost.postListNamespaceDescriptors(descriptors);
2254       }
2255     }
2256     return descriptors;
2257   }
2258 
2259   @Override
2260   public List<HTableDescriptor> listTableDescriptorsByNamespace(String name) throws IOException {
2261     ensureNamespaceExists(name);
2262     return listTableDescriptors(name, null, null, true);
2263   }
2264 
2265   @Override
2266   public List<TableName> listTableNamesByNamespace(String name) throws IOException {
2267     ensureNamespaceExists(name);
2268     return listTableNames(name, null, true);
2269   }
2270 
2271   /**
2272    * Returns the list of table descriptors that match the specified request
2273    *
2274    * @param namespace the namespace to query, or null if querying for all
2275    * @param regex The regular expression to match against, or null if querying for all
2276    * @param tableNameList the list of table names, or null if querying for all
2277    * @param includeSysTables False to match only against userspace tables
2278    * @return the list of table descriptors
2279    */
2280   public List<HTableDescriptor> listTableDescriptors(final String namespace, final String regex,
2281       final List<TableName> tableNameList, final boolean includeSysTables)
2282       throws IOException {
2283     final List<HTableDescriptor> descriptors = new ArrayList<HTableDescriptor>();
2284 
2285     boolean bypass = false;
2286     if (cpHost != null) {
2287       bypass = cpHost.preGetTableDescriptors(tableNameList, descriptors, regex);
2288     }
2289 
2290     if (!bypass) {
2291       if (tableNameList == null || tableNameList.size() == 0) {
2292         // request for all TableDescriptors
2293         Collection<HTableDescriptor> htds;
2294         if (namespace != null && namespace.length() > 0) {
2295           htds = tableDescriptors.getByNamespace(namespace).values();
2296         } else {
2297           htds = tableDescriptors.getAll().values();
2298         }
2299 
2300         for (HTableDescriptor desc: htds) {
2301           if (tableStateManager.isTablePresent(desc.getTableName())
2302               && (includeSysTables || !desc.getTableName().isSystemTable())) {
2303             descriptors.add(desc);
2304           }
2305         }
2306       } else {
2307         for (TableName s: tableNameList) {
2308           if (tableStateManager.isTablePresent(s)) {
2309             HTableDescriptor desc = tableDescriptors.get(s);
2310             if (desc != null) {
2311               descriptors.add(desc);
2312             }
2313           }
2314         }
2315       }
2316 
2317       // Retains only those matched by regular expression.
2318       if (regex != null) {
2319         filterTablesByRegex(descriptors, Pattern.compile(regex));
2320       }
2321 
2322       if (cpHost != null) {
2323         cpHost.postGetTableDescriptors(tableNameList, descriptors, regex);
2324       }
2325     }
2326     return descriptors;
2327   }
2328 
2329   /**
2330    * Returns the list of table names that match the specified request
2331    * @param regex The regular expression to match against, or null if querying for all
2332    * @param namespace the namespace to query, or null if querying for all
2333    * @param includeSysTables False to match only against userspace tables
2334    * @return the list of table names
2335    */
2336   public List<TableName> listTableNames(final String namespace, final String regex,
2337       final boolean includeSysTables) throws IOException {
2338     final List<HTableDescriptor> descriptors = new ArrayList<HTableDescriptor>();
2339 
2340     boolean bypass = false;
2341     if (cpHost != null) {
2342       bypass = cpHost.preGetTableNames(descriptors, regex);
2343     }
2344 
2345     if (!bypass) {
2346       // get all descriptors
2347       Collection<HTableDescriptor> htds;
2348       if (namespace != null && namespace.length() > 0) {
2349         htds = tableDescriptors.getByNamespace(namespace).values();
2350       } else {
2351         htds = tableDescriptors.getAll().values();
2352       }
2353 
2354       for (HTableDescriptor htd: htds) {
2355         if (includeSysTables || !htd.getTableName().isSystemTable()) {
2356           descriptors.add(htd);
2357         }
2358       }
2359 
2360       // Retains only those matched by regular expression.
2361       if (regex != null) {
2362         filterTablesByRegex(descriptors, Pattern.compile(regex));
2363       }
2364 
2365       if (cpHost != null) {
2366         cpHost.postGetTableNames(descriptors, regex);
2367       }
2368     }
2369 
2370     List<TableName> result = new ArrayList<TableName>(descriptors.size());
2371     for (HTableDescriptor htd: descriptors) {
2372       result.add(htd.getTableName());
2373     }
2374     return result;
2375   }
2376 
2377 
2378   /**
2379    * Removes the table descriptors that don't match the pattern.
2380    * @param descriptors list of table descriptors to filter
2381    * @param pattern the regex to use
2382    */
2383   private static void filterTablesByRegex(final Collection<HTableDescriptor> descriptors,
2384       final Pattern pattern) {
2385     final String defaultNS = NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR;
2386     Iterator<HTableDescriptor> itr = descriptors.iterator();
2387     while (itr.hasNext()) {
2388       HTableDescriptor htd = itr.next();
2389       String tableName = htd.getTableName().getNameAsString();
2390       boolean matched = pattern.matcher(tableName).matches();
2391       if (!matched && htd.getTableName().getNamespaceAsString().equals(defaultNS)) {
2392         matched = pattern.matcher(defaultNS + TableName.NAMESPACE_DELIM + tableName).matches();
2393       }
2394       if (!matched) {
2395         itr.remove();
2396       }
2397     }
2398   }
2399 
2400   @Override
2401   public long getLastMajorCompactionTimestamp(TableName table) throws IOException {
2402     return getClusterStatus().getLastMajorCompactionTsForTable(table);
2403   }
2404 
2405   @Override
2406   public long getLastMajorCompactionTimestampForRegion(byte[] regionName) throws IOException {
2407     return getClusterStatus().getLastMajorCompactionTsForRegion(regionName);
2408   }
2409 
2410   /**
2411    * Queries the state of the {@link LoadBalancerTracker}. If the balancer is not initialized,
2412    * false is returned.
2413    *
2414    * @return The state of the load balancer, or false if the load balancer isn't defined.
2415    */
2416   public boolean isBalancerOn() {
2417     if (null == loadBalancerTracker) return false;
2418     return loadBalancerTracker.isBalancerOn();
2419   }
2420 
2421   /**
2422    * Fetch the configured {@link LoadBalancer} class name. If none is set, a default is returned.
2423    *
2424    * @return The name of the {@link LoadBalancer} in use.
2425    */
2426   public String getLoadBalancerClassName() {
2427     return conf.get(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, LoadBalancerFactory
2428         .getDefaultLoadBalancerClass().getName());
2429   }
2430 }