View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.master;
20  
21  import com.google.common.annotations.VisibleForTesting;
22  import com.google.common.collect.Lists;
23  import com.google.common.collect.Maps;
24  import com.google.protobuf.Descriptors;
25  import com.google.protobuf.Service;
26  
27  import java.io.IOException;
28  import java.io.InterruptedIOException;
29  import java.lang.reflect.Constructor;
30  import java.lang.reflect.InvocationTargetException;
31  import java.net.InetAddress;
32  import java.net.InetSocketAddress;
33  import java.net.UnknownHostException;
34  import java.util.ArrayList;
35  import java.util.Collection;
36  import java.util.Collections;
37  import java.util.Comparator;
38  import java.util.Iterator;
39  import java.util.List;
40  import java.util.Map;
41  import java.util.Map.Entry;
42  import java.util.Set;
43  import java.util.concurrent.TimeUnit;
44  import java.util.concurrent.atomic.AtomicInteger;
45  import java.util.concurrent.atomic.AtomicReference;
46  import java.util.regex.Pattern;
47
48  import javax.servlet.ServletException;
49  import javax.servlet.http.HttpServlet;
50  import javax.servlet.http.HttpServletRequest;
51  import javax.servlet.http.HttpServletResponse;
52
53  import org.apache.commons.logging.Log;
54  import org.apache.commons.logging.LogFactory;
55  import org.apache.hadoop.conf.Configuration;
56  import org.apache.hadoop.fs.Path;
57  import org.apache.hadoop.hbase.ClusterStatus;
58  import org.apache.hadoop.hbase.CoordinatedStateException;
59  import org.apache.hadoop.hbase.CoordinatedStateManager;
60  import org.apache.hadoop.hbase.DoNotRetryIOException;
61  import org.apache.hadoop.hbase.HBaseIOException;
62  import org.apache.hadoop.hbase.HBaseInterfaceAudience;
63  import org.apache.hadoop.hbase.HColumnDescriptor;
64  import org.apache.hadoop.hbase.HConstants;
65  import org.apache.hadoop.hbase.HRegionInfo;
66  import org.apache.hadoop.hbase.HTableDescriptor;
67  import org.apache.hadoop.hbase.MasterNotRunningException;
68  import org.apache.hadoop.hbase.MetaTableAccessor;
69  import org.apache.hadoop.hbase.NamespaceDescriptor;
70  import org.apache.hadoop.hbase.PleaseHoldException;
71  import org.apache.hadoop.hbase.ProcedureInfo;
72  import org.apache.hadoop.hbase.RegionStateListener;
73  import org.apache.hadoop.hbase.ScheduledChore;
74  import org.apache.hadoop.hbase.ServerLoad;
75  import org.apache.hadoop.hbase.ServerName;
76  import org.apache.hadoop.hbase.TableDescriptors;
77  import org.apache.hadoop.hbase.TableName;
78  import org.apache.hadoop.hbase.TableNotDisabledException;
79  import org.apache.hadoop.hbase.TableNotFoundException;
80  import org.apache.hadoop.hbase.UnknownRegionException;
81  import org.apache.hadoop.hbase.classification.InterfaceAudience;
82  import org.apache.hadoop.hbase.client.MasterSwitchType;
83  import org.apache.hadoop.hbase.client.Result;
84  import org.apache.hadoop.hbase.client.TableState;
85  import org.apache.hadoop.hbase.coprocessor.BypassCoprocessorException;
86  import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
87  import org.apache.hadoop.hbase.exceptions.DeserializationException;
88  import org.apache.hadoop.hbase.exceptions.MergeRegionException;
89  import org.apache.hadoop.hbase.executor.ExecutorType;
90  import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils;
91  import org.apache.hadoop.hbase.ipc.RpcServer;
92  import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
93  import org.apache.hadoop.hbase.master.MasterRpcServices.BalanceSwitchMode;
94  import org.apache.hadoop.hbase.master.balancer.BalancerChore;
95  import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer;
96  import org.apache.hadoop.hbase.master.balancer.ClusterStatusChore;
97  import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory;
98  import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
99  import org.apache.hadoop.hbase.master.cleaner.LogCleaner;
100 import org.apache.hadoop.hbase.master.cleaner.ReplicationZKLockCleanerChore;
101 import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan;
102 import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan.PlanType;
103 import org.apache.hadoop.hbase.master.normalizer.RegionNormalizer;
104 import org.apache.hadoop.hbase.master.normalizer.RegionNormalizerChore;
105 import org.apache.hadoop.hbase.master.normalizer.RegionNormalizerFactory;
106 import org.apache.hadoop.hbase.master.procedure.AddColumnFamilyProcedure;
107 import org.apache.hadoop.hbase.master.procedure.CreateTableProcedure;
108 import org.apache.hadoop.hbase.master.procedure.DeleteColumnFamilyProcedure;
109 import org.apache.hadoop.hbase.master.procedure.DeleteTableProcedure;
110 import org.apache.hadoop.hbase.master.procedure.DisableTableProcedure;
111 import org.apache.hadoop.hbase.master.procedure.DispatchMergingRegionsProcedure;
112 import org.apache.hadoop.hbase.master.procedure.EnableTableProcedure;
113 import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants;
114 import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
115 import org.apache.hadoop.hbase.master.procedure.MasterProcedureScheduler.ProcedureEvent;
116 import org.apache.hadoop.hbase.master.procedure.ModifyColumnFamilyProcedure;
117 import org.apache.hadoop.hbase.master.procedure.ModifyTableProcedure;
118 import org.apache.hadoop.hbase.master.procedure.ProcedurePrepareLatch;
119 import org.apache.hadoop.hbase.master.procedure.ProcedureSyncWait;
120 import org.apache.hadoop.hbase.master.procedure.TruncateTableProcedure;
121 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
122 import org.apache.hadoop.hbase.mob.MobConstants;
123 import org.apache.hadoop.hbase.monitoring.MemoryBoundedLogMessageBuffer;
124 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
125 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
126 import org.apache.hadoop.hbase.procedure.MasterProcedureManagerHost;
127 import org.apache.hadoop.hbase.procedure.flush.MasterFlushTableProcedureManager;
128 import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
129 import org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore;
130 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState;
131 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionServerInfo;
132 import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
133 import org.apache.hadoop.hbase.regionserver.DefaultStoreEngine;
134 import org.apache.hadoop.hbase.regionserver.HRegionServer;
135 import org.apache.hadoop.hbase.regionserver.HStore;
136 import org.apache.hadoop.hbase.regionserver.RSRpcServices;
137 import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
138 import org.apache.hadoop.hbase.regionserver.RegionSplitPolicy;
139 import org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy;
140 import org.apache.hadoop.hbase.regionserver.compactions.FIFOCompactionPolicy;
141 import org.apache.hadoop.hbase.replication.ReplicationFactory;
142 import org.apache.hadoop.hbase.replication.ReplicationQueuesZKImpl;
143 import org.apache.hadoop.hbase.replication.master.TableCFsUpdater;
144 import org.apache.hadoop.hbase.replication.regionserver.Replication;
145 import org.apache.hadoop.hbase.security.UserProvider;
146 import org.apache.hadoop.hbase.util.Addressing;
147 import org.apache.hadoop.hbase.util.Bytes;
148 import org.apache.hadoop.hbase.util.CompressionTest;
149 import org.apache.hadoop.hbase.util.EncryptionTest;
150 import org.apache.hadoop.hbase.util.FSUtils;
151 import org.apache.hadoop.hbase.util.HFileArchiveUtil;
152 import org.apache.hadoop.hbase.util.HasThread;
153 import org.apache.hadoop.hbase.util.IdLock;
154 import org.apache.hadoop.hbase.util.ModifyRegionUtils;
155 import org.apache.hadoop.hbase.util.Pair;
156 import org.apache.hadoop.hbase.util.Threads;
157 import org.apache.hadoop.hbase.util.VersionInfo;
158 import org.apache.hadoop.hbase.util.ZKDataMigrator;
159 import org.apache.hadoop.hbase.zookeeper.DrainingServerTracker;
160 import org.apache.hadoop.hbase.zookeeper.LoadBalancerTracker;
161 import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker;
162 import org.apache.hadoop.hbase.zookeeper.MasterMaintenanceModeTracker;
163 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
164 import org.apache.hadoop.hbase.zookeeper.RegionNormalizerTracker;
165 import org.apache.hadoop.hbase.zookeeper.RegionServerTracker;
166 import org.apache.hadoop.hbase.zookeeper.SplitOrMergeTracker;
167 import org.apache.hadoop.hbase.zookeeper.ZKClusterId;
168 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
169 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
170 import org.apache.zookeeper.KeeperException;
171 import org.mortbay.jetty.Connector;
172 import org.mortbay.jetty.nio.SelectChannelConnector;
173 import org.mortbay.jetty.servlet.Context;
174
175 /**
176  * HMaster is the "master server" for HBase. An HBase cluster has one active
177  * master.  If many masters are started, all compete.  Whichever wins goes on to
178  * run the cluster.  All others park themselves in their constructor until
179  * master or cluster shutdown or until the active master loses its lease in
180  * zookeeper.  Thereafter, all running master jostle to take over master role.
181  *
182  * <p>The Master can be asked shutdown the cluster. See {@link #shutdown()}.  In
183  * this case it will tell all regionservers to go down and then wait on them
184  * all reporting in that they are down.  This master will then shut itself down.
185  *
186  * <p>You can also shutdown just this master.  Call {@link #stopMaster()}.
187  *
188  * @see org.apache.zookeeper.Watcher
189  */
190 @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
191 @SuppressWarnings("deprecation")
192 public class HMaster extends HRegionServer implements MasterServices {
193   private static final Log LOG = LogFactory.getLog(HMaster.class.getName());
194
195   /**
196    * Protection against zombie master. Started once Master accepts active responsibility and
197    * starts taking over responsibilities. Allows a finite time window before giving up ownership.
198    */
199   private static class InitializationMonitor extends HasThread {
200     /** The amount of time in milliseconds to sleep before checking initialization status. */
201     public static final String TIMEOUT_KEY = "hbase.master.initializationmonitor.timeout";
202     public static final long TIMEOUT_DEFAULT = TimeUnit.MILLISECONDS.convert(15, TimeUnit.MINUTES);
203
204     /**
205      * When timeout expired and initialization has not complete, call {@link System#exit(int)} when
206      * true, do nothing otherwise.
207      */
208     public static final String HALT_KEY = "hbase.master.initializationmonitor.haltontimeout";
209     public static final boolean HALT_DEFAULT = false;
210
211     private final HMaster master;
212     private final long timeout;
213     private final boolean haltOnTimeout;
214
215     /** Creates a Thread that monitors the {@link #isInitialized()} state. */
216     InitializationMonitor(HMaster master) {
217       super("MasterInitializationMonitor");
218       this.master = master;
219       this.timeout = master.getConfiguration().getLong(TIMEOUT_KEY, TIMEOUT_DEFAULT);
220       this.haltOnTimeout = master.getConfiguration().getBoolean(HALT_KEY, HALT_DEFAULT);
221       this.setDaemon(true);
222     }
223
224     @Override
225     public void run() {
226       try {
227         while (!master.isStopped() && master.isActiveMaster()) {
228           Thread.sleep(timeout);
229           if (master.isInitialized()) {
230             LOG.debug("Initialization completed within allotted tolerance. Monitor exiting.");
231           } else {
232             LOG.error("Master failed to complete initialization after " + timeout + "ms. Please"
233                 + " consider submitting a bug report including a thread dump of this process.");
234             if (haltOnTimeout) {
235               LOG.error("Zombie Master exiting. Thread dump to stdout");
236               Threads.printThreadInfo(System.out, "Zombie HMaster");
237               System.exit(-1);
238             }
239           }
240         }
241       } catch (InterruptedException ie) {
242         LOG.trace("InitMonitor thread interrupted. Existing.");
243       }
244     }
245   }
246
247   // MASTER is name of the webapp and the attribute name used stuffing this
248   //instance into web context.
249   public static final String MASTER = "master";
250
251   // Manager and zk listener for master election
252   private final ActiveMasterManager activeMasterManager;
253   // Region server tracker
254   RegionServerTracker regionServerTracker;
255   // Draining region server tracker
256   private DrainingServerTracker drainingServerTracker;
257   // Tracker for load balancer state
258   LoadBalancerTracker loadBalancerTracker;
259
260   // Tracker for split and merge state
261   private SplitOrMergeTracker splitOrMergeTracker;
262
263   // Tracker for region normalizer state
264   private RegionNormalizerTracker regionNormalizerTracker;
265
266   //Tracker for master maintenance mode setting
267   private MasterMaintenanceModeTracker maintenanceModeTracker;
268
269   private ClusterSchemaService clusterSchemaService;
270
271   // Metrics for the HMaster
272   final MetricsMaster metricsMaster;
273   // file system manager for the master FS operations
274   private MasterFileSystem fileSystemManager;
275   private MasterWalManager walManager;
276
277   // server manager to deal with region server info
278   private volatile ServerManager serverManager;
279
280   // manager of assignment nodes in zookeeper
281   private AssignmentManager assignmentManager;
282
283   // buffer for "fatal error" notices from region servers
284   // in the cluster. This is only used for assisting
285   // operations/debugging.
286   MemoryBoundedLogMessageBuffer rsFatals;
287
288   // flag set after we become the active master (used for testing)
289   private volatile boolean isActiveMaster = false;
290
291   // flag set after we complete initialization once active,
292   // it is not private since it's used in unit tests
293   private final ProcedureEvent initialized = new ProcedureEvent("master initialized");
294
295   // flag set after master services are started,
296   // initialization may have not completed yet.
297   volatile boolean serviceStarted = false;
298
299   // flag set after we complete assignMeta.
300   private final ProcedureEvent serverCrashProcessingEnabled =
301     new ProcedureEvent("server crash processing");
302
303   private LoadBalancer balancer;
304   private RegionNormalizer normalizer;
305   private BalancerChore balancerChore;
306   private RegionNormalizerChore normalizerChore;
307   private ClusterStatusChore clusterStatusChore;
308   private ClusterStatusPublisher clusterStatusPublisherChore = null;
309   private PeriodicDoMetrics periodicDoMetricsChore = null;
310
311   CatalogJanitor catalogJanitorChore;
312   private ReplicationZKLockCleanerChore replicationZKLockCleanerChore;
313   private LogCleaner logCleaner;
314   private HFileCleaner hfileCleaner;
315   private ExpiredMobFileCleanerChore expiredMobFileCleanerChore;
316   private MobCompactionChore mobCompactChore;
317   private MasterMobCompactionThread mobCompactThread;
318   // used to synchronize the mobCompactionStates
319   private final IdLock mobCompactionLock = new IdLock();
320   // save the information of mob compactions in tables.
321   // the key is table name, the value is the number of compactions in that table.
322   private Map<TableName, AtomicInteger> mobCompactionStates = Maps.newConcurrentMap();
323
324   MasterCoprocessorHost cpHost;
325
326   private final boolean preLoadTableDescriptors;
327
328   // Time stamps for when a hmaster became active
329   private long masterActiveTime;
330
331   //should we check the compression codec type at master side, default true, HBASE-6370
332   private final boolean masterCheckCompression;
333
334   //should we check encryption settings at master side, default true
335   private final boolean masterCheckEncryption;
336
337   Map<String, Service> coprocessorServiceHandlers = Maps.newHashMap();
338
339   // monitor for snapshot of hbase tables
340   SnapshotManager snapshotManager;
341   // monitor for distributed procedures
342   private MasterProcedureManagerHost mpmHost;
343
344   // it is assigned after 'initialized' guard set to true, so should be volatile
345   private volatile MasterQuotaManager quotaManager;
346
347   private ProcedureExecutor<MasterProcedureEnv> procedureExecutor;
348   private WALProcedureStore procedureStore;
349
350   // handle table states
351   private TableStateManager tableStateManager;
352
353   private long splitPlanCount;
354   private long mergePlanCount;
355
356   /** flag used in test cases in order to simulate RS failures during master initialization */
357   private volatile boolean initializationBeforeMetaAssignment = false;
358
359   /** jetty server for master to redirect requests to regionserver infoServer */
360   private org.mortbay.jetty.Server masterJettyServer;
361
362   public static class RedirectServlet extends HttpServlet {
363     private static final long serialVersionUID = 2894774810058302472L;
364     private static int regionServerInfoPort;
365
366     @Override
367     public void doGet(HttpServletRequest request,
368         HttpServletResponse response) throws ServletException, IOException {
369       String redirectUrl = request.getScheme() + "://"
370         + request.getServerName() + ":" + regionServerInfoPort
371         + request.getRequestURI();
372       response.sendRedirect(redirectUrl);
373     }
374   }
375
376   private static class PeriodicDoMetrics extends ScheduledChore {
377     private final HMaster server;
378     public PeriodicDoMetrics(int doMetricsInterval, final HMaster server) {
379       super(server.getServerName() + "-DoMetricsChore", server, doMetricsInterval);
380       this.server = server;
381     }
382
383     @Override
384     protected void chore() {
385       server.doMetrics();
386     }
387   }
388
389   /**
390    * Initializes the HMaster. The steps are as follows:
391    * <p>
392    * <ol>
393    * <li>Initialize the local HRegionServer
394    * <li>Start the ActiveMasterManager.
395    * </ol>
396    * <p>
397    * Remaining steps of initialization occur in
398    * #finishActiveMasterInitialization(MonitoredTask) after
399    * the master becomes the active one.
400    */
401   public HMaster(final Configuration conf, CoordinatedStateManager csm)
402       throws IOException, KeeperException {
403     super(conf, csm);
404     this.rsFatals = new MemoryBoundedLogMessageBuffer(
405       conf.getLong("hbase.master.buffer.for.rs.fatals", 1*1024*1024));
406
407     LOG.info("hbase.rootdir=" + FSUtils.getRootDir(this.conf) +
408       ", hbase.cluster.distributed=" + this.conf.getBoolean(HConstants.CLUSTER_DISTRIBUTED, false));
409
410     // Disable usage of meta replicas in the master
411     this.conf.setBoolean(HConstants.USE_META_REPLICAS, false);
412
413     Replication.decorateMasterConfiguration(this.conf);
414
415     // Hack! Maps DFSClient => Master for logs.  HDFS made this
416     // config param for task trackers, but we can piggyback off of it.
417     if (this.conf.get("mapreduce.task.attempt.id") == null) {
418       this.conf.set("mapreduce.task.attempt.id", "hb_m_" + this.serverName.toString());
419     }
420
421     // should we check the compression codec type at master side, default true, HBASE-6370
422     this.masterCheckCompression = conf.getBoolean("hbase.master.check.compression", true);
423
424     // should we check encryption settings at master side, default true
425     this.masterCheckEncryption = conf.getBoolean("hbase.master.check.encryption", true);
426
427     this.metricsMaster = new MetricsMaster(new MetricsMasterWrapperImpl(this));
428
429     // preload table descriptor at startup
430     this.preLoadTableDescriptors = conf.getBoolean("hbase.master.preload.tabledescriptors", true);
431
432     // Do we publish the status?
433
434     boolean shouldPublish = conf.getBoolean(HConstants.STATUS_PUBLISHED,
435         HConstants.STATUS_PUBLISHED_DEFAULT);
436     Class<? extends ClusterStatusPublisher.Publisher> publisherClass =
437         conf.getClass(ClusterStatusPublisher.STATUS_PUBLISHER_CLASS,
438             ClusterStatusPublisher.DEFAULT_STATUS_PUBLISHER_CLASS,
439             ClusterStatusPublisher.Publisher.class);
440
441     if (shouldPublish) {
442       if (publisherClass == null) {
443         LOG.warn(HConstants.STATUS_PUBLISHED + " is true, but " +
444             ClusterStatusPublisher.DEFAULT_STATUS_PUBLISHER_CLASS +
445             " is not set - not publishing status");
446       } else {
447         clusterStatusPublisherChore = new ClusterStatusPublisher(this, conf, publisherClass);
448         getChoreService().scheduleChore(clusterStatusPublisherChore);
449       }
450     }
451
452     // Some unit tests don't need a cluster, so no zookeeper at all
453     if (!conf.getBoolean("hbase.testing.nocluster", false)) {
454       activeMasterManager = new ActiveMasterManager(zooKeeper, this.serverName, this);
455       int infoPort = putUpJettyServer();
456       startActiveMasterManager(infoPort);
457     } else {
458       activeMasterManager = null;
459     }
460   }
461
462   // return the actual infoPort, -1 means disable info server.
463   private int putUpJettyServer() throws IOException {
464     if (!conf.getBoolean("hbase.master.infoserver.redirect", true)) {
465       return -1;
466     }
467     int infoPort = conf.getInt("hbase.master.info.port.orig",
468       HConstants.DEFAULT_MASTER_INFOPORT);
469     // -1 is for disabling info server, so no redirecting
470     if (infoPort < 0 || infoServer == null) {
471       return -1;
472     }
473     String addr = conf.get("hbase.master.info.bindAddress", "0.0.0.0");
474     if (!Addressing.isLocalAddress(InetAddress.getByName(addr))) {
475       String msg =
476           "Failed to start redirecting jetty server. Address " + addr
477               + " does not belong to this host. Correct configuration parameter: "
478               + "hbase.master.info.bindAddress";
479       LOG.error(msg);
480       throw new IOException(msg);
481     }
482
483     RedirectServlet.regionServerInfoPort = infoServer.getPort();
484     if(RedirectServlet.regionServerInfoPort == infoPort) {
485       return infoPort;
486     }
487     masterJettyServer = new org.mortbay.jetty.Server();
488     Connector connector = new SelectChannelConnector();
489     connector.setHost(addr);
490     connector.setPort(infoPort);
491     masterJettyServer.addConnector(connector);
492     masterJettyServer.setStopAtShutdown(true);
493     Context context = new Context(masterJettyServer, "/", Context.NO_SESSIONS);
494     context.addServlet(RedirectServlet.class, "/*");
495     try {
496       masterJettyServer.start();
497     } catch (Exception e) {
498       throw new IOException("Failed to start redirecting jetty server", e);
499     }
500     return connector.getLocalPort();
501   }
502
503   @Override
504   protected TableDescriptors getFsTableDescriptors() throws IOException {
505     return super.getFsTableDescriptors();
506   }
507
508   /**
509    * For compatibility, if failed with regionserver credentials, try the master one
510    */
511   @Override
512   protected void login(UserProvider user, String host) throws IOException {
513     try {
514       super.login(user, host);
515     } catch (IOException ie) {
516       user.login("hbase.master.keytab.file",
517         "hbase.master.kerberos.principal", host);
518     }
519   }
520
521   /**
522    * If configured to put regions on active master,
523    * wait till a backup master becomes active.
524    * Otherwise, loop till the server is stopped or aborted.
525    */
526   @Override
527   protected void waitForMasterActive(){
528     boolean tablesOnMaster = BaseLoadBalancer.tablesOnMaster(conf);
529     while (!(tablesOnMaster && isActiveMaster)
530         && !isStopped() && !isAborted()) {
531       sleeper.sleep();
532     }
533   }
534
535   @VisibleForTesting
536   public MasterRpcServices getMasterRpcServices() {
537     return (MasterRpcServices)rpcServices;
538   }
539
540   public boolean balanceSwitch(final boolean b) throws IOException {
541     return getMasterRpcServices().switchBalancer(b, BalanceSwitchMode.ASYNC);
542   }
543
544   @Override
545   protected String getProcessName() {
546     return MASTER;
547   }
548
549   @Override
550   protected boolean canCreateBaseZNode() {
551     return true;
552   }
553
554   @Override
555   protected boolean canUpdateTableDescriptor() {
556     return true;
557   }
558
559   @Override
560   protected RSRpcServices createRpcServices() throws IOException {
561     return new MasterRpcServices(this);
562   }
563
564   @Override
565   protected void configureInfoServer() {
566     infoServer.addServlet("master-status", "/master-status", MasterStatusServlet.class);
567     infoServer.setAttribute(MASTER, this);
568     if (BaseLoadBalancer.tablesOnMaster(conf)) {
569       super.configureInfoServer();
570     }
571   }
572
573   @Override
574   protected Class<? extends HttpServlet> getDumpServlet() {
575     return MasterDumpServlet.class;
576   }
577
578   /**
579    * Emit the HMaster metrics, such as region in transition metrics.
580    * Surrounding in a try block just to be sure metrics doesn't abort HMaster.
581    */
582   private void doMetrics() {
583     try {
584       if (assignmentManager != null) {
585         assignmentManager.updateRegionsInTransitionMetrics();
586       }
587     } catch (Throwable e) {
588       LOG.error("Couldn't update metrics: " + e.getMessage());
589     }
590   }
591
592   MetricsMaster getMasterMetrics() {
593     return metricsMaster;
594   }
595
596   /**
597    * Initialize all ZK based system trackers.
598    */
599   void initializeZKBasedSystemTrackers() throws IOException,
600       InterruptedException, KeeperException, CoordinatedStateException {
601     this.balancer = LoadBalancerFactory.getLoadBalancer(conf);
602     this.normalizer = RegionNormalizerFactory.getRegionNormalizer(conf);
603     this.normalizer.setMasterServices(this);
604     this.normalizer.setMasterRpcServices((MasterRpcServices)rpcServices);
605     this.loadBalancerTracker = new LoadBalancerTracker(zooKeeper, this);
606     this.loadBalancerTracker.start();
607
608     this.regionNormalizerTracker = new RegionNormalizerTracker(zooKeeper, this);
609     this.regionNormalizerTracker.start();
610
611     this.splitOrMergeTracker = new SplitOrMergeTracker(zooKeeper, conf, this);
612     this.splitOrMergeTracker.start();
613
614     this.assignmentManager = new AssignmentManager(this, serverManager,
615       this.balancer, this.service, this.metricsMaster,
616       this.tableLockManager, tableStateManager);
617
618     this.regionServerTracker = new RegionServerTracker(zooKeeper, this, this.serverManager);
619     this.regionServerTracker.start();
620
621     this.drainingServerTracker = new DrainingServerTracker(zooKeeper, this, this.serverManager);
622     this.drainingServerTracker.start();
623
624     this.maintenanceModeTracker = new MasterMaintenanceModeTracker(zooKeeper);
625     this.maintenanceModeTracker.start();
626
627     // Set the cluster as up.  If new RSs, they'll be waiting on this before
628     // going ahead with their startup.
629     boolean wasUp = this.clusterStatusTracker.isClusterUp();
630     if (!wasUp) this.clusterStatusTracker.setClusterUp();
631
632     LOG.info("Server active/primary master=" + this.serverName +
633         ", sessionid=0x" +
634         Long.toHexString(this.zooKeeper.getRecoverableZooKeeper().getSessionId()) +
635         ", setting cluster-up flag (Was=" + wasUp + ")");
636
637     // create/initialize the snapshot manager and other procedure managers
638     this.snapshotManager = new SnapshotManager();
639     this.mpmHost = new MasterProcedureManagerHost();
640     this.mpmHost.register(this.snapshotManager);
641     this.mpmHost.register(new MasterFlushTableProcedureManager());
642     this.mpmHost.loadProcedures(conf);
643     this.mpmHost.initialize(this, this.metricsMaster);
644
645   }
646
647   /**
648    * Finish initialization of HMaster after becoming the primary master.
649    *
650    * <ol>
651    * <li>Initialize master components - file system manager, server manager,
652    *     assignment manager, region server tracker, etc</li>
653    * <li>Start necessary service threads - balancer, catalog janior,
654    *     executor services, etc</li>
655    * <li>Set cluster as UP in ZooKeeper</li>
656    * <li>Wait for RegionServers to check-in</li>
657    * <li>Split logs and perform data recovery, if necessary</li>
658    * <li>Ensure assignment of meta/namespace regions<li>
659    * <li>Handle either fresh cluster start or master failover</li>
660    * </ol>
661    */
662   private void finishActiveMasterInitialization(MonitoredTask status)
663       throws IOException, InterruptedException, KeeperException, CoordinatedStateException {
664
665     isActiveMaster = true;
666     Thread zombieDetector = new Thread(new InitializationMonitor(this));
667     zombieDetector.start();
668
669     /*
670      * We are active master now... go initialize components we need to run.
671      * Note, there may be dross in zk from previous runs; it'll get addressed
672      * below after we determine if cluster startup or failover.
673      */
674
675     status.setStatus("Initializing Master file system");
676
677     this.masterActiveTime = System.currentTimeMillis();
678     // TODO: Do this using Dependency Injection, using PicoContainer, Guice or Spring.
679     this.fileSystemManager = new MasterFileSystem(this);
680     this.walManager = new MasterWalManager(this);
681
682     // enable table descriptors cache
683     this.tableDescriptors.setCacheOn();
684     // set the META's descriptor to the correct replication
685     this.tableDescriptors.get(TableName.META_TABLE_NAME).setRegionReplication(
686         conf.getInt(HConstants.META_REPLICAS_NUM, HConstants.DEFAULT_META_REPLICA_NUM));
687     // warm-up HTDs cache on master initialization
688     if (preLoadTableDescriptors) {
689       status.setStatus("Pre-loading table descriptors");
690       this.tableDescriptors.getAll();
691     }
692
693     // publish cluster ID
694     status.setStatus("Publishing Cluster ID in ZooKeeper");
695     ZKClusterId.setClusterId(this.zooKeeper, fileSystemManager.getClusterId());
696
697     this.serverManager = createServerManager(this);
698
699     // Invalidate all write locks held previously
700     this.tableLockManager.reapWriteLocks();
701     this.tableStateManager = new TableStateManager(this);
702
703     status.setStatus("Initializing ZK system trackers");
704     initializeZKBasedSystemTrackers();
705
706     // This is for backwards compatibility
707     // See HBASE-11393
708     status.setStatus("Update TableCFs node in ZNode");
709     TableCFsUpdater tableCFsUpdater = new TableCFsUpdater(zooKeeper,
710             conf, this.clusterConnection);
711     tableCFsUpdater.update();
712
713     // initialize master side coprocessors before we start handling requests
714     status.setStatus("Initializing master coprocessors");
715     this.cpHost = new MasterCoprocessorHost(this, this.conf);
716
717     // start up all service threads.
718     status.setStatus("Initializing master service threads");
719     startServiceThreads();
720
721     // Wake up this server to check in
722     sleeper.skipSleepCycle();
723
724     // Wait for region servers to report in
725     status.setStatus("Wait for region servers to report in");
726     waitForRegionServers(status);
727
728     // get a list for previously failed RS which need log splitting work
729     // we recover hbase:meta region servers inside master initialization and
730     // handle other failed servers in SSH in order to start up master node ASAP
731     MasterMetaBootstrap metaBootstrap = createMetaBootstrap(this, status);
732     metaBootstrap.splitMetaLogsBeforeAssignment();
733
734     this.initializationBeforeMetaAssignment = true;
735
736     // Wait for regionserver to finish initialization.
737     if (BaseLoadBalancer.tablesOnMaster(conf)) {
738       waitForServerOnline();
739     }
740
741     //initialize load balancer
742     this.balancer.setClusterStatus(getClusterStatus());
743     this.balancer.setMasterServices(this);
744     this.balancer.initialize();
745
746     // Check if master is shutting down because of some issue
747     // in initializing the regionserver or the balancer.
748     if (isStopped()) return;
749
750     // Make sure meta assigned before proceeding.
751     status.setStatus("Assigning Meta Region");
752     metaBootstrap.assignMeta();
753
754     // check if master is shutting down because above assignMeta could return even hbase:meta isn't
755     // assigned when master is shutting down
756     if (isStopped()) return;
757
758     // migrating existent table state from zk, so splitters
759     // and recovery process treat states properly.
760     for (Map.Entry<TableName, TableState.State> entry : ZKDataMigrator
761         .queryForTableStates(getZooKeeper()).entrySet()) {
762       LOG.info("Converting state from zk to new states:" + entry);
763       tableStateManager.setTableState(entry.getKey(), entry.getValue());
764     }
765     ZKUtil.deleteChildrenRecursively(getZooKeeper(), getZooKeeper().tableZNode);
766
767     status.setStatus("Submitting log splitting work for previously failed region servers");
768     metaBootstrap.processDeadServers();
769
770     // Fix up assignment manager status
771     status.setStatus("Starting assignment manager");
772     this.assignmentManager.joinCluster();
773
774     // set cluster status again after user regions are assigned
775     this.balancer.setClusterStatus(getClusterStatus());
776
777     // Start balancer and meta catalog janitor after meta and regions have been assigned.
778     status.setStatus("Starting balancer and catalog janitor");
779     this.clusterStatusChore = new ClusterStatusChore(this, balancer);
780     getChoreService().scheduleChore(clusterStatusChore);
781     this.balancerChore = new BalancerChore(this);
782     getChoreService().scheduleChore(balancerChore);
783     this.normalizerChore = new RegionNormalizerChore(this);
784     getChoreService().scheduleChore(normalizerChore);
785     this.catalogJanitorChore = new CatalogJanitor(this, this);
786     getChoreService().scheduleChore(catalogJanitorChore);
787
788     // Do Metrics periodically
789     periodicDoMetricsChore = new PeriodicDoMetrics(msgInterval, this);
790     getChoreService().scheduleChore(periodicDoMetricsChore);
791
792     status.setStatus("Starting cluster schema service");
793     initClusterSchemaService();
794
795     if (this.cpHost != null) {
796       try {
797         this.cpHost.preMasterInitialization();
798       } catch (IOException e) {
799         LOG.error("Coprocessor preMasterInitialization() hook failed", e);
800       }
801     }
802
803     status.markComplete("Initialization successful");
804     LOG.info("Master has completed initialization");
805     configurationManager.registerObserver(this.balancer);
806
807     // Set master as 'initialized'.
808     setInitialized(true);
809
810     status.setStatus("Assign meta replicas");
811     metaBootstrap.assignMetaReplicas();
812
813     status.setStatus("Starting quota manager");
814     initQuotaManager();
815
816     // clear the dead servers with same host name and port of online server because we are not
817     // removing dead server with same hostname and port of rs which is trying to check in before
818     // master initialization. See HBASE-5916.
819     this.serverManager.clearDeadServersWithSameHostNameAndPortOfOnlineServer();
820
821     // Check and set the znode ACLs if needed in case we are overtaking a non-secure configuration
822     status.setStatus("Checking ZNode ACLs");
823     zooKeeper.checkAndSetZNodeAcls();
824
825     status.setStatus("Initializing MOB Cleaner");
826     initMobCleaner();
827
828     status.setStatus("Calling postStartMaster coprocessors");
829     if (this.cpHost != null) {
830       // don't let cp initialization errors kill the master
831       try {
832         this.cpHost.postStartMaster();
833       } catch (IOException ioe) {
834         LOG.error("Coprocessor postStartMaster() hook failed", ioe);
835       }
836     }
837
838     zombieDetector.interrupt();
839   }
840
841   private void initMobCleaner() {
842     this.expiredMobFileCleanerChore = new ExpiredMobFileCleanerChore(this);
843     getChoreService().scheduleChore(expiredMobFileCleanerChore);
844
845     int mobCompactionPeriod = conf.getInt(MobConstants.MOB_COMPACTION_CHORE_PERIOD,
846         MobConstants.DEFAULT_MOB_COMPACTION_CHORE_PERIOD);
847     if (mobCompactionPeriod > 0) {
848       this.mobCompactChore = new MobCompactionChore(this, mobCompactionPeriod);
849       getChoreService().scheduleChore(mobCompactChore);
850     } else {
851       LOG
852         .info("The period is " + mobCompactionPeriod + " seconds, MobCompactionChore is disabled");
853     }
854     this.mobCompactThread = new MasterMobCompactionThread(this);
855   }
856
857   /**
858    * Create a {@link MasterMetaBootstrap} instance.
859    */
860   MasterMetaBootstrap createMetaBootstrap(final HMaster master, final MonitoredTask status) {
861     // We put this out here in a method so can do a Mockito.spy and stub it out
862     // w/ a mocked up MasterMetaBootstrap.
863     return new MasterMetaBootstrap(master, status);
864   }
865
866   /**
867    * Create a {@link ServerManager} instance.
868    */
869   ServerManager createServerManager(final MasterServices master) throws IOException {
870     // We put this out here in a method so can do a Mockito.spy and stub it out
871     // w/ a mocked up ServerManager.
872     setupClusterConnection();
873     return new ServerManager(master);
874   }
875
876   private void waitForRegionServers(final MonitoredTask status)
877       throws IOException, InterruptedException {
878     this.serverManager.waitForRegionServers(status);
879     // Check zk for region servers that are up but didn't register
880     for (ServerName sn: this.regionServerTracker.getOnlineServers()) {
881       // The isServerOnline check is opportunistic, correctness is handled inside
882       if (!this.serverManager.isServerOnline(sn)
883           && serverManager.checkAndRecordNewServer(sn, ServerLoad.EMPTY_SERVERLOAD)) {
884         LOG.info("Registered server found up in zk but who has not yet reported in: " + sn);
885       }
886     }
887   }
888
889   void initClusterSchemaService() throws IOException, InterruptedException {
890     this.clusterSchemaService = new ClusterSchemaServiceImpl(this);
891     this.clusterSchemaService.startAndWait();
892     if (!this.clusterSchemaService.isRunning()) throw new HBaseIOException("Failed start");
893   }
894
895   void initQuotaManager() throws IOException {
896     MasterQuotaManager quotaManager = new MasterQuotaManager(this);
897     this.assignmentManager.setRegionStateListener((RegionStateListener)quotaManager);
898     quotaManager.start();
899     this.quotaManager = quotaManager;
900   }
901
902   boolean isCatalogJanitorEnabled() {
903     return catalogJanitorChore != null ?
904       catalogJanitorChore.getEnabled() : false;
905   }
906
907   @Override
908   public TableDescriptors getTableDescriptors() {
909     return this.tableDescriptors;
910   }
911
912   @Override
913   public ServerManager getServerManager() {
914     return this.serverManager;
915   }
916
917   @Override
918   public MasterFileSystem getMasterFileSystem() {
919     return this.fileSystemManager;
920   }
921
922   @Override
923   public MasterWalManager getMasterWalManager() {
924     return this.walManager;
925   }
926
927   @Override
928   public TableStateManager getTableStateManager() {
929     return tableStateManager;
930   }
931
932   /*
933    * Start up all services. If any of these threads gets an unhandled exception
934    * then they just die with a logged message.  This should be fine because
935    * in general, we do not expect the master to get such unhandled exceptions
936    *  as OOMEs; it should be lightly loaded. See what HRegionServer does if
937    *  need to install an unexpected exception handler.
938    */
939   private void startServiceThreads() throws IOException{
940    // Start the executor service pools
941    this.service.startExecutorService(ExecutorType.MASTER_OPEN_REGION,
942       conf.getInt("hbase.master.executor.openregion.threads", 5));
943    this.service.startExecutorService(ExecutorType.MASTER_CLOSE_REGION,
944       conf.getInt("hbase.master.executor.closeregion.threads", 5));
945    this.service.startExecutorService(ExecutorType.MASTER_SERVER_OPERATIONS,
946       conf.getInt("hbase.master.executor.serverops.threads", 5));
947    this.service.startExecutorService(ExecutorType.MASTER_META_SERVER_OPERATIONS,
948       conf.getInt("hbase.master.executor.meta.serverops.threads", 5));
949    this.service.startExecutorService(ExecutorType.M_LOG_REPLAY_OPS,
950       conf.getInt("hbase.master.executor.logreplayops.threads", 10));
951
952    // We depend on there being only one instance of this executor running
953    // at a time.  To do concurrency, would need fencing of enable/disable of
954    // tables.
955    // Any time changing this maxThreads to > 1, pls see the comment at
956    // AccessController#postCompletedCreateTableAction
957    this.service.startExecutorService(ExecutorType.MASTER_TABLE_OPERATIONS, 1);
958    startProcedureExecutor();
959
960    // Start log cleaner thread
961    int cleanerInterval = conf.getInt("hbase.master.cleaner.interval", 60 * 1000);
962    this.logCleaner =
963       new LogCleaner(cleanerInterval,
964          this, conf, getMasterWalManager().getFileSystem(),
965          getMasterWalManager().getOldLogDir());
966     getChoreService().scheduleChore(logCleaner);
967
968    //start the hfile archive cleaner thread
969     Path archiveDir = HFileArchiveUtil.getArchivePath(conf);
970     this.hfileCleaner = new HFileCleaner(cleanerInterval, this, conf, getMasterFileSystem()
971         .getFileSystem(), archiveDir);
972     getChoreService().scheduleChore(hfileCleaner);
973     serviceStarted = true;
974     if (LOG.isTraceEnabled()) {
975       LOG.trace("Started service threads");
976     }
977     if (conf.getClass("hbase.region.replica.replication.replicationQueues.class",
978         ReplicationFactory.defaultReplicationQueueClass) == ReplicationQueuesZKImpl.class && !conf
979         .getBoolean(HConstants.ZOOKEEPER_USEMULTI, true)) {
980       try {
981         replicationZKLockCleanerChore = new ReplicationZKLockCleanerChore(this, this,
982             cleanerInterval, this.getZooKeeper(), this.conf);
983         getChoreService().scheduleChore(replicationZKLockCleanerChore);
984       } catch (Exception e) {
985         LOG.error("start replicationZKLockCleanerChore failed", e);
986       }
987     }
988   }
989
990   @Override
991   protected void sendShutdownInterrupt() {
992     super.sendShutdownInterrupt();
993     stopProcedureExecutor();
994   }
995
996   @Override
997   protected void stopServiceThreads() {
998     if (masterJettyServer != null) {
999       LOG.info("Stopping master jetty server");
1000       try {
1001         masterJettyServer.stop();
1002       } catch (Exception e) {
1003         LOG.error("Failed to stop master jetty server", e);
1004       }
1005     }
1006     super.stopServiceThreads();
1007     stopChores();
1008
1009     // Wait for all the remaining region servers to report in IFF we were
1010     // running a cluster shutdown AND we were NOT aborting.
1011     if (!isAborted() && this.serverManager != null &&
1012         this.serverManager.isClusterShutdown()) {
1013       this.serverManager.letRegionServersShutdown();
1014     }
1015     if (LOG.isDebugEnabled()) {
1016       LOG.debug("Stopping service threads");
1017     }
1018     // Clean up and close up shop
1019     if (this.logCleaner != null) this.logCleaner.cancel(true);
1020     if (this.hfileCleaner != null) this.hfileCleaner.cancel(true);
1021     if (this.replicationZKLockCleanerChore != null) this.replicationZKLockCleanerChore.cancel(true);
1022     if (this.quotaManager != null) this.quotaManager.stop();
1023     if (this.activeMasterManager != null) this.activeMasterManager.stop();
1024     if (this.serverManager != null) this.serverManager.stop();
1025     if (this.assignmentManager != null) this.assignmentManager.stop();
1026     if (this.walManager != null) this.walManager.stop();
1027     if (this.fileSystemManager != null) this.fileSystemManager.stop();
1028     if (this.mpmHost != null) this.mpmHost.stop("server shutting down.");
1029   }
1030
1031   private void startProcedureExecutor() throws IOException {
1032     final MasterProcedureEnv procEnv = new MasterProcedureEnv(this);
1033     final Path logDir = new Path(fileSystemManager.getRootDir(),
1034         MasterProcedureConstants.MASTER_PROCEDURE_LOGDIR);
1035
1036     procedureStore = new WALProcedureStore(conf, fileSystemManager.getFileSystem(), logDir,
1037         new MasterProcedureEnv.WALStoreLeaseRecovery(this));
1038     procedureStore.registerListener(new MasterProcedureEnv.MasterProcedureStoreListener(this));
1039     procedureExecutor = new ProcedureExecutor(conf, procEnv, procedureStore,
1040         procEnv.getProcedureQueue());
1041
1042     final int numThreads = conf.getInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS,
1043         Math.max(Runtime.getRuntime().availableProcessors(),
1044           MasterProcedureConstants.DEFAULT_MIN_MASTER_PROCEDURE_THREADS));
1045     final boolean abortOnCorruption = conf.getBoolean(
1046         MasterProcedureConstants.EXECUTOR_ABORT_ON_CORRUPTION,
1047         MasterProcedureConstants.DEFAULT_EXECUTOR_ABORT_ON_CORRUPTION);
1048     procedureStore.start(numThreads);
1049     procedureExecutor.start(numThreads, abortOnCorruption);
1050   }
1051
1052   private void stopProcedureExecutor() {
1053     if (procedureExecutor != null) {
1054       procedureExecutor.stop();
1055     }
1056
1057     if (procedureStore != null) {
1058       procedureStore.stop(isAborted());
1059     }
1060   }
1061
1062   private void stopChores() {
1063     if (this.expiredMobFileCleanerChore != null) {
1064       this.expiredMobFileCleanerChore.cancel(true);
1065     }
1066     if (this.mobCompactChore != null) {
1067       this.mobCompactChore.cancel(true);
1068     }
1069     if (this.balancerChore != null) {
1070       this.balancerChore.cancel(true);
1071     }
1072     if (this.normalizerChore != null) {
1073       this.normalizerChore.cancel(true);
1074     }
1075     if (this.clusterStatusChore != null) {
1076       this.clusterStatusChore.cancel(true);
1077     }
1078     if (this.catalogJanitorChore != null) {
1079       this.catalogJanitorChore.cancel(true);
1080     }
1081     if (this.clusterStatusPublisherChore != null){
1082       clusterStatusPublisherChore.cancel(true);
1083     }
1084     if (this.mobCompactThread != null) {
1085       this.mobCompactThread.close();
1086     }
1087
1088     if (this.periodicDoMetricsChore != null) {
1089       periodicDoMetricsChore.cancel();
1090     }
1091   }
1092
1093   /**
1094    * @return Get remote side's InetAddress
1095    */
1096   InetAddress getRemoteInetAddress(final int port,
1097       final long serverStartCode) throws UnknownHostException {
1098     // Do it out here in its own little method so can fake an address when
1099     // mocking up in tests.
1100     InetAddress ia = RpcServer.getRemoteIp();
1101
1102     // The call could be from the local regionserver,
1103     // in which case, there is no remote address.
1104     if (ia == null && serverStartCode == startcode) {
1105       InetSocketAddress isa = rpcServices.getSocketAddress();
1106       if (isa != null && isa.getPort() == port) {
1107         ia = isa.getAddress();
1108       }
1109     }
1110     return ia;
1111   }
1112
1113   /**
1114    * @return Maximum time we should run balancer for
1115    */
1116   private int getBalancerCutoffTime() {
1117     int balancerCutoffTime = getConfiguration().getInt("hbase.balancer.max.balancing", -1);
1118     if (balancerCutoffTime == -1) {
1119       // if cutoff time isn't set, defaulting it to period time
1120       int balancerPeriod = getConfiguration().getInt("hbase.balancer.period", 300000);
1121       balancerCutoffTime = balancerPeriod;
1122     }
1123     return balancerCutoffTime;
1124   }
1125
1126   public boolean balance() throws IOException {
1127     return balance(false);
1128   }
1129
1130   public boolean balance(boolean force) throws IOException {
1131     // if master not initialized, don't run balancer.
1132     if (!isInitialized()) {
1133       LOG.debug("Master has not been initialized, don't run balancer.");
1134       return false;
1135     }
1136
1137     if (isInMaintenanceMode()) {
1138       LOG.info("Master is in maintenanceMode mode, don't run balancer.");
1139       return false;
1140     }
1141
1142     // Do this call outside of synchronized block.
1143     int maximumBalanceTime = getBalancerCutoffTime();
1144     synchronized (this.balancer) {
1145       // If balance not true, don't run balancer.
1146       if (!this.loadBalancerTracker.isBalancerOn()) return false;
1147       // Only allow one balance run at at time.
1148       if (this.assignmentManager.getRegionStates().isRegionsInTransition()) {
1149         Set<RegionState> regionsInTransition =
1150           this.assignmentManager.getRegionStates().getRegionsInTransition();
1151         // if hbase:meta region is in transition, result of assignment cannot be recorded
1152         // ignore the force flag in that case
1153         boolean metaInTransition = assignmentManager.getRegionStates().isMetaRegionInTransition();
1154         String prefix = force && !metaInTransition ? "R" : "Not r";
1155         LOG.debug(prefix + "unning balancer because " + regionsInTransition.size() +
1156           " region(s) in transition: " + org.apache.commons.lang.StringUtils.
1157             abbreviate(regionsInTransition.toString(), 256));
1158         if (!force || metaInTransition) return false;
1159       }
1160       if (this.serverManager.areDeadServersInProgress()) {
1161         LOG.debug("Not running balancer because processing dead regionserver(s): " +
1162           this.serverManager.getDeadServers());
1163         return false;
1164       }
1165
1166       if (this.cpHost != null) {
1167         try {
1168           if (this.cpHost.preBalance()) {
1169             LOG.debug("Coprocessor bypassing balancer request");
1170             return false;
1171           }
1172         } catch (IOException ioe) {
1173           LOG.error("Error invoking master coprocessor preBalance()", ioe);
1174           return false;
1175         }
1176       }
1177
1178       Map<TableName, Map<ServerName, List<HRegionInfo>>> assignmentsByTable =
1179         this.assignmentManager.getRegionStates().getAssignmentsByTable();
1180
1181       List<RegionPlan> plans = new ArrayList<RegionPlan>();
1182
1183       //Give the balancer the current cluster state.
1184       this.balancer.setClusterStatus(getClusterStatus());
1185       for (Entry<TableName, Map<ServerName, List<HRegionInfo>>> e : assignmentsByTable.entrySet()) {
1186         List<RegionPlan> partialPlans = this.balancer.balanceCluster(e.getKey(), e.getValue());
1187         if (partialPlans != null) plans.addAll(partialPlans);
1188       }
1189
1190       long cutoffTime = System.currentTimeMillis() + maximumBalanceTime;
1191       int rpCount = 0;  // number of RegionPlans balanced so far
1192       long totalRegPlanExecTime = 0;
1193       if (plans != null && !plans.isEmpty()) {
1194         for (RegionPlan plan: plans) {
1195           LOG.info("balance " + plan);
1196           long balStartTime = System.currentTimeMillis();
1197           //TODO: bulk assign
1198           this.assignmentManager.balance(plan);
1199           totalRegPlanExecTime += System.currentTimeMillis()-balStartTime;
1200           rpCount++;
1201           if (rpCount < plans.size() &&
1202               // if performing next balance exceeds cutoff time, exit the loop
1203               (System.currentTimeMillis() + (totalRegPlanExecTime / rpCount)) > cutoffTime) {
1204             //TODO: After balance, there should not be a cutoff time (keeping it as
1205             // a security net for now)
1206             LOG.debug("No more balancing till next balance run; maximumBalanceTime=" +
1207               maximumBalanceTime);
1208             break;
1209           }
1210         }
1211       }
1212       if (this.cpHost != null) {
1213         try {
1214           this.cpHost.postBalance(rpCount < plans.size() ? plans.subList(0, rpCount) : plans);
1215         } catch (IOException ioe) {
1216           // balancing already succeeded so don't change the result
1217           LOG.error("Error invoking master coprocessor postBalance()", ioe);
1218         }
1219       }
1220     }
1221     // If LoadBalancer did not generate any plans, it means the cluster is already balanced.
1222     // Return true indicating a success.
1223     return true;
1224   }
1225
1226   @Override
1227   @VisibleForTesting
1228   public RegionNormalizer getRegionNormalizer() {
1229     return this.normalizer;
1230   }
1231
1232   /**
1233    * Perform normalization of cluster (invoked by {@link RegionNormalizerChore}).
1234    *
1235    * @return true if normalization step was performed successfully, false otherwise
1236    *    (specifically, if HMaster hasn't been initialized properly or normalization
1237    *    is globally disabled)
1238    */
1239   public boolean normalizeRegions() throws IOException {
1240     if (!isInitialized()) {
1241       LOG.debug("Master has not been initialized, don't run region normalizer.");
1242       return false;
1243     }
1244
1245     if (isInMaintenanceMode()) {
1246       LOG.info("Master is in maintenance mode, don't run region normalizer.");
1247       return false;
1248     }
1249
1250     if (!this.regionNormalizerTracker.isNormalizerOn()) {
1251       LOG.debug("Region normalization is disabled, don't run region normalizer.");
1252       return false;
1253     }
1254
1255     synchronized (this.normalizer) {
1256       // Don't run the normalizer concurrently
1257       List<TableName> allEnabledTables = new ArrayList<>(
1258         this.tableStateManager.getTablesInStates(TableState.State.ENABLED));
1259
1260       Collections.shuffle(allEnabledTables);
1261
1262       for (TableName table : allEnabledTables) {
1263         if (isInMaintenanceMode()) {
1264           LOG.debug("Master is in maintenance mode, stop running region normalizer.");
1265           return false;
1266         }
1267
1268         HTableDescriptor tblDesc = getTableDescriptors().get(table);
1269         if (table.isSystemTable() || (tblDesc != null &&
1270             !tblDesc.isNormalizationEnabled())) {
1271           LOG.debug("Skipping normalization for table: " + table + ", as it's either system"
1272               + " table or doesn't have auto normalization turned on");
1273           continue;
1274         }
1275         List<NormalizationPlan> plans = this.normalizer.computePlanForTable(table);
1276         if (plans != null) {
1277           for (NormalizationPlan plan : plans) {
1278             plan.execute(clusterConnection.getAdmin());
1279             if (plan.getType() == PlanType.SPLIT) {
1280               splitPlanCount++;
1281             } else if (plan.getType() == PlanType.MERGE) {
1282               mergePlanCount++;
1283             }
1284           }
1285         }
1286       }
1287     }
1288     // If Region did not generate any plans, it means the cluster is already balanced.
1289     // Return true indicating a success.
1290     return true;
1291   }
1292
1293   /**
1294    * @return Client info for use as prefix on an audit log string; who did an action
1295    */
1296   String getClientIdAuditPrefix() {
1297     return "Client=" + RpcServer.getRequestUserName() + "/" + RpcServer.getRemoteAddress();
1298   }
1299
1300   /**
1301    * Switch for the background CatalogJanitor thread.
1302    * Used for testing.  The thread will continue to run.  It will just be a noop
1303    * if disabled.
1304    * @param b If false, the catalog janitor won't do anything.
1305    */
1306   public void setCatalogJanitorEnabled(final boolean b) {
1307     this.catalogJanitorChore.setEnabled(b);
1308   }
1309
1310   @Override
1311   public long dispatchMergingRegions(
1312       final HRegionInfo regionInfoA,
1313       final HRegionInfo regionInfoB,
1314       final boolean forcible,
1315       final long nonceGroup,
1316       final long nonce) throws IOException {
1317     checkInitialized();
1318
1319     TableName tableName = regionInfoA.getTable();
1320     if (tableName == null || regionInfoB.getTable() == null) {
1321       throw new UnknownRegionException ("Can't merge regions without table associated");
1322     }
1323
1324     if (!tableName.equals(regionInfoB.getTable())) {
1325       throw new IOException ("Cannot merge regions from two different tables");
1326     }
1327
1328     if (regionInfoA.compareTo(regionInfoB) == 0) {
1329       throw new MergeRegionException(
1330         "Cannot merge a region to itself " + regionInfoA + ", " + regionInfoB);
1331     }
1332
1333     HRegionInfo [] regionsToMerge = new HRegionInfo[2];
1334     regionsToMerge [0] = regionInfoA;
1335     regionsToMerge [1] = regionInfoB;
1336
1337     if (cpHost != null) {
1338       cpHost.preDispatchMerge(regionInfoA, regionInfoB);
1339     }
1340
1341     LOG.info(getClientIdAuditPrefix() + " Merge regions "
1342         + regionInfoA.getEncodedName() + " and " + regionInfoB.getEncodedName());
1343
1344     long procId = this.procedureExecutor.submitProcedure(
1345       new DispatchMergingRegionsProcedure(
1346         procedureExecutor.getEnvironment(), tableName, regionsToMerge, forcible),
1347       nonceGroup,
1348       nonce);
1349
1350     if (cpHost != null) {
1351       cpHost.postDispatchMerge(regionInfoA, regionInfoB);
1352     }
1353     return procId;
1354   }
1355
1356   void move(final byte[] encodedRegionName,
1357       final byte[] destServerName) throws HBaseIOException {
1358     RegionState regionState = assignmentManager.getRegionStates().
1359       getRegionState(Bytes.toString(encodedRegionName));
1360
1361     HRegionInfo hri;
1362     if (regionState != null) {
1363       hri = regionState.getRegion();
1364     } else {
1365       throw new UnknownRegionException(Bytes.toStringBinary(encodedRegionName));
1366     }
1367
1368     ServerName dest;
1369     if (destServerName == null || destServerName.length == 0) {
1370       LOG.info("Passed destination servername is null/empty so " +
1371         "choosing a server at random");
1372       final List<ServerName> destServers = this.serverManager.createDestinationServersList(
1373         regionState.getServerName());
1374       dest = balancer.randomAssignment(hri, destServers);
1375       if (dest == null) {
1376         LOG.debug("Unable to determine a plan to assign " + hri);
1377         return;
1378       }
1379     } else {
1380       ServerName candidate = ServerName.valueOf(Bytes.toString(destServerName));
1381       dest = balancer.randomAssignment(hri, Lists.newArrayList(candidate));
1382       if (dest == null) {
1383         LOG.debug("Unable to determine a plan to assign " + hri);
1384         return;
1385       }
1386       if (dest.equals(serverName) && balancer instanceof BaseLoadBalancer
1387           && !((BaseLoadBalancer)balancer).shouldBeOnMaster(hri)) {
1388         // To avoid unnecessary region moving later by balancer. Don't put user
1389         // regions on master. Regions on master could be put on other region
1390         // server intentionally by test however.
1391         LOG.debug("Skipping move of region " + hri.getRegionNameAsString()
1392           + " to avoid unnecessary region moving later by load balancer,"
1393           + " because it should not be on master");
1394         return;
1395       }
1396     }
1397
1398     if (dest.equals(regionState.getServerName())) {
1399       LOG.debug("Skipping move of region " + hri.getRegionNameAsString()
1400         + " because region already assigned to the same server " + dest + ".");
1401       return;
1402     }
1403
1404     // Now we can do the move
1405     RegionPlan rp = new RegionPlan(hri, regionState.getServerName(), dest);
1406
1407     try {
1408       checkInitialized();
1409       if (this.cpHost != null) {
1410         if (this.cpHost.preMove(hri, rp.getSource(), rp.getDestination())) {
1411           return;
1412         }
1413       }
1414       // warmup the region on the destination before initiating the move. this call
1415       // is synchronous and takes some time. doing it before the source region gets
1416       // closed
1417       serverManager.sendRegionWarmup(rp.getDestination(), hri);
1418
1419       LOG.info(getClientIdAuditPrefix() + " move " + rp + ", running balancer");
1420       this.assignmentManager.balance(rp);
1421       if (this.cpHost != null) {
1422         this.cpHost.postMove(hri, rp.getSource(), rp.getDestination());
1423       }
1424     } catch (IOException ioe) {
1425       if (ioe instanceof HBaseIOException) {
1426         throw (HBaseIOException)ioe;
1427       }
1428       throw new HBaseIOException(ioe);
1429     }
1430   }
1431
1432   @Override
1433   public long createTable(
1434       final HTableDescriptor hTableDescriptor,
1435       final byte [][] splitKeys,
1436       final long nonceGroup,
1437       final long nonce) throws IOException {
1438     if (isStopped()) {
1439       throw new MasterNotRunningException();
1440     }
1441     checkInitialized();
1442     String namespace = hTableDescriptor.getTableName().getNamespaceAsString();
1443     this.clusterSchemaService.getNamespace(namespace);
1444
1445     HRegionInfo[] newRegions = ModifyRegionUtils.createHRegionInfos(hTableDescriptor, splitKeys);
1446     checkInitialized();
1447     sanityCheckTableDescriptor(hTableDescriptor);
1448     if (cpHost != null) {
1449       cpHost.preCreateTable(hTableDescriptor, newRegions);
1450     }
1451     LOG.info(getClientIdAuditPrefix() + " create " + hTableDescriptor);
1452
1453     // TODO: We can handle/merge duplicate requests, and differentiate the case of
1454     //       TableExistsException by saying if the schema is the same or not.
1455     ProcedurePrepareLatch latch = ProcedurePrepareLatch.createLatch();
1456     long procId = this.procedureExecutor.submitProcedure(
1457       new CreateTableProcedure(
1458         procedureExecutor.getEnvironment(), hTableDescriptor, newRegions, latch),
1459       nonceGroup,
1460       nonce);
1461     latch.await();
1462
1463     if (cpHost != null) {
1464       cpHost.postCreateTable(hTableDescriptor, newRegions);
1465     }
1466
1467     return procId;
1468   }
1469
1470   /**
1471    * Checks whether the table conforms to some sane limits, and configured
1472    * values (compression, etc) work. Throws an exception if something is wrong.
1473    * @throws IOException
1474    */
1475   private void sanityCheckTableDescriptor(final HTableDescriptor htd) throws IOException {
1476     final String CONF_KEY = "hbase.table.sanity.checks";
1477     boolean logWarn = false;
1478     if (!conf.getBoolean(CONF_KEY, true)) {
1479       logWarn = true;
1480     }
1481     String tableVal = htd.getConfigurationValue(CONF_KEY);
1482     if (tableVal != null && !Boolean.valueOf(tableVal)) {
1483       logWarn = true;
1484     }
1485
1486     // check max file size
1487     long maxFileSizeLowerLimit = 2 * 1024 * 1024L; // 2M is the default lower limit
1488     long maxFileSize = htd.getMaxFileSize();
1489     if (maxFileSize < 0) {
1490       maxFileSize = conf.getLong(HConstants.HREGION_MAX_FILESIZE, maxFileSizeLowerLimit);
1491     }
1492     if (maxFileSize < conf.getLong("hbase.hregion.max.filesize.limit", maxFileSizeLowerLimit)) {
1493       String message = "MAX_FILESIZE for table descriptor or "
1494           + "\"hbase.hregion.max.filesize\" (" + maxFileSize
1495           + ") is too small, which might cause over splitting into unmanageable "
1496           + "number of regions.";
1497       warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);
1498     }
1499
1500     // check flush size
1501     long flushSizeLowerLimit = 1024 * 1024L; // 1M is the default lower limit
1502     long flushSize = htd.getMemStoreFlushSize();
1503     if (flushSize < 0) {
1504       flushSize = conf.getLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, flushSizeLowerLimit);
1505     }
1506     if (flushSize < conf.getLong("hbase.hregion.memstore.flush.size.limit", flushSizeLowerLimit)) {
1507       String message = "MEMSTORE_FLUSHSIZE for table descriptor or "
1508           + "\"hbase.hregion.memstore.flush.size\" ("+flushSize+") is too small, which might cause"
1509           + " very frequent flushing.";
1510       warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);
1511     }
1512
1513     // check that coprocessors and other specified plugin classes can be loaded
1514     try {
1515       checkClassLoading(conf, htd);
1516     } catch (Exception ex) {
1517       warnOrThrowExceptionForFailure(logWarn, CONF_KEY, ex.getMessage(), null);
1518     }
1519
1520     // check compression can be loaded
1521     try {
1522       checkCompression(htd);
1523     } catch (IOException e) {
1524       warnOrThrowExceptionForFailure(logWarn, CONF_KEY, e.getMessage(), e);
1525     }
1526
1527     // check encryption can be loaded
1528     try {
1529       checkEncryption(conf, htd);
1530     } catch (IOException e) {
1531       warnOrThrowExceptionForFailure(logWarn, CONF_KEY, e.getMessage(), e);
1532     }
1533     // Verify compaction policy
1534     try{
1535       checkCompactionPolicy(conf, htd);
1536     } catch(IOException e){
1537       warnOrThrowExceptionForFailure(false, CONF_KEY, e.getMessage(), e);
1538     }
1539     // check that we have at least 1 CF
1540     if (htd.getColumnFamilies().length == 0) {
1541       String message = "Table should have at least one column family.";
1542       warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);
1543     }
1544
1545     for (HColumnDescriptor hcd : htd.getColumnFamilies()) {
1546       if (hcd.getTimeToLive() <= 0) {
1547         String message = "TTL for column family " + hcd.getNameAsString() + " must be positive.";
1548         warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);
1549       }
1550
1551       // check blockSize
1552       if (hcd.getBlocksize() < 1024 || hcd.getBlocksize() > 16 * 1024 * 1024) {
1553         String message = "Block size for column family " + hcd.getNameAsString()
1554             + "  must be between 1K and 16MB.";
1555         warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);
1556       }
1557
1558       // check versions
1559       if (hcd.getMinVersions() < 0) {
1560         String message = "Min versions for column family " + hcd.getNameAsString()
1561           + "  must be positive.";
1562         warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);
1563       }
1564       // max versions already being checked
1565
1566       // HBASE-13776 Setting illegal versions for HColumnDescriptor
1567       //  does not throw IllegalArgumentException
1568       // check minVersions <= maxVerions
1569       if (hcd.getMinVersions() > hcd.getMaxVersions()) {
1570         String message = "Min versions for column family " + hcd.getNameAsString()
1571             + " must be less than the Max versions.";
1572         warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);
1573       }
1574
1575       // check replication scope
1576       if (hcd.getScope() < 0) {
1577         String message = "Replication scope for column family "
1578           + hcd.getNameAsString() + "  must be positive.";
1579         warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);
1580       }
1581
1582       // check data replication factor, it can be 0(default value) when user has not explicitly
1583       // set the value, in this case we use default replication factor set in the file system.
1584       if (hcd.getDFSReplication() < 0) {
1585         String message = "HFile Replication for column family " + hcd.getNameAsString()
1586             + "  must be greater than zero.";
1587         warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);
1588       }
1589
1590       // TODO: should we check coprocessors and encryption ?
1591     }
1592   }
1593
1594   private void checkCompactionPolicy(Configuration conf, HTableDescriptor htd)
1595       throws IOException {
1596     // FIFO compaction has some requirements
1597     // Actually FCP ignores periodic major compactions
1598     String className =
1599         htd.getConfigurationValue(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY);
1600     if (className == null) {
1601       className =
1602           conf.get(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY,
1603             ExploringCompactionPolicy.class.getName());
1604     }
1605
1606     int blockingFileCount = HStore.DEFAULT_BLOCKING_STOREFILE_COUNT;
1607     String sv = htd.getConfigurationValue(HStore.BLOCKING_STOREFILES_KEY);
1608     if (sv != null) {
1609       blockingFileCount = Integer.parseInt(sv);
1610     } else {
1611       blockingFileCount = conf.getInt(HStore.BLOCKING_STOREFILES_KEY, blockingFileCount);
1612     }
1613
1614     for (HColumnDescriptor hcd : htd.getColumnFamilies()) {
1615       String compactionPolicy =
1616           hcd.getConfigurationValue(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY);
1617       if (compactionPolicy == null) {
1618         compactionPolicy = className;
1619       }
1620       if (!compactionPolicy.equals(FIFOCompactionPolicy.class.getName())) {
1621         continue;
1622       }
1623       // FIFOCompaction
1624       String message = null;
1625
1626       // 1. Check TTL
1627       if (hcd.getTimeToLive() == HColumnDescriptor.DEFAULT_TTL) {
1628         message = "Default TTL is not supported for FIFO compaction";
1629         throw new IOException(message);
1630       }
1631
1632       // 2. Check min versions
1633       if (hcd.getMinVersions() > 0) {
1634         message = "MIN_VERSION > 0 is not supported for FIFO compaction";
1635         throw new IOException(message);
1636       }
1637
1638       // 3. blocking file count
1639       String sbfc = htd.getConfigurationValue(HStore.BLOCKING_STOREFILES_KEY);
1640       if (sbfc != null) {
1641         blockingFileCount = Integer.parseInt(sbfc);
1642       }
1643       if (blockingFileCount < 1000) {
1644         message =
1645             "blocking file count '" + HStore.BLOCKING_STOREFILES_KEY + "' " + blockingFileCount
1646                 + " is below recommended minimum of 1000";
1647         throw new IOException(message);
1648       }
1649     }
1650   }
1651
1652   // HBASE-13350 - Helper method to log warning on sanity check failures if checks disabled.
1653   private static void warnOrThrowExceptionForFailure(boolean logWarn, String confKey,
1654       String message, Exception cause) throws IOException {
1655     if (!logWarn) {
1656       throw new DoNotRetryIOException(message + " Set " + confKey +
1657           " to false at conf or table descriptor if you want to bypass sanity checks", cause);
1658     }
1659     LOG.warn(message);
1660   }
1661
1662   private void startActiveMasterManager(int infoPort) throws KeeperException {
1663     String backupZNode = ZKUtil.joinZNode(
1664       zooKeeper.backupMasterAddressesZNode, serverName.toString());
1665     /*
1666     * Add a ZNode for ourselves in the backup master directory since we
1667     * may not become the active master. If so, we want the actual active
1668     * master to know we are backup masters, so that it won't assign
1669     * regions to us if so configured.
1670     *
1671     * If we become the active master later, ActiveMasterManager will delete
1672     * this node explicitly.  If we crash before then, ZooKeeper will delete
1673     * this node for us since it is ephemeral.
1674     */
1675     LOG.info("Adding backup master ZNode " + backupZNode);
1676     if (!MasterAddressTracker.setMasterAddress(zooKeeper, backupZNode,
1677         serverName, infoPort)) {
1678       LOG.warn("Failed create of " + backupZNode + " by " + serverName);
1679     }
1680
1681     activeMasterManager.setInfoPort(infoPort);
1682     // Start a thread to try to become the active master, so we won't block here
1683     Threads.setDaemonThreadRunning(new Thread(new Runnable() {
1684       @Override
1685       public void run() {
1686         int timeout = conf.getInt(HConstants.ZK_SESSION_TIMEOUT,
1687           HConstants.DEFAULT_ZK_SESSION_TIMEOUT);
1688         // If we're a backup master, stall until a primary to writes his address
1689         if (conf.getBoolean(HConstants.MASTER_TYPE_BACKUP,
1690           HConstants.DEFAULT_MASTER_TYPE_BACKUP)) {
1691           LOG.debug("HMaster started in backup mode. "
1692             + "Stalling until master znode is written.");
1693           // This will only be a minute or so while the cluster starts up,
1694           // so don't worry about setting watches on the parent znode
1695           while (!activeMasterManager.hasActiveMaster()) {
1696             LOG.debug("Waiting for master address ZNode to be written "
1697               + "(Also watching cluster state node)");
1698             Threads.sleep(timeout);
1699           }
1700         }
1701         MonitoredTask status = TaskMonitor.get().createStatus("Master startup");
1702         status.setDescription("Master startup");
1703         try {
1704           if (activeMasterManager.blockUntilBecomingActiveMaster(timeout, status)) {
1705             finishActiveMasterInitialization(status);
1706           }
1707         } catch (Throwable t) {
1708           status.setStatus("Failed to become active: " + t.getMessage());
1709           LOG.fatal("Failed to become active master", t);
1710           // HBASE-5680: Likely hadoop23 vs hadoop 20.x/1.x incompatibility
1711           if (t instanceof NoClassDefFoundError &&
1712             t.getMessage()
1713               .contains("org/apache/hadoop/hdfs/protocol/HdfsConstants$SafeModeAction")) {
1714             // improved error message for this special case
1715             abort("HBase is having a problem with its Hadoop jars.  You may need to "
1716               + "recompile HBase against Hadoop version "
1717               + org.apache.hadoop.util.VersionInfo.getVersion()
1718               + " or change your hadoop jars to start properly", t);
1719           } else {
1720             abort("Unhandled exception. Starting shutdown.", t);
1721           }
1722         } finally {
1723           status.cleanup();
1724         }
1725       }
1726     }, getServerName().toShortString() + ".activeMasterManager"));
1727   }
1728
1729   private void checkCompression(final HTableDescriptor htd)
1730   throws IOException {
1731     if (!this.masterCheckCompression) return;
1732     for (HColumnDescriptor hcd : htd.getColumnFamilies()) {
1733       checkCompression(hcd);
1734     }
1735   }
1736
1737   private void checkCompression(final HColumnDescriptor hcd)
1738   throws IOException {
1739     if (!this.masterCheckCompression) return;
1740     CompressionTest.testCompression(hcd.getCompressionType());
1741     CompressionTest.testCompression(hcd.getCompactionCompressionType());
1742   }
1743
1744   private void checkEncryption(final Configuration conf, final HTableDescriptor htd)
1745   throws IOException {
1746     if (!this.masterCheckEncryption) return;
1747     for (HColumnDescriptor hcd : htd.getColumnFamilies()) {
1748       checkEncryption(conf, hcd);
1749     }
1750   }
1751
1752   private void checkEncryption(final Configuration conf, final HColumnDescriptor hcd)
1753   throws IOException {
1754     if (!this.masterCheckEncryption) return;
1755     EncryptionTest.testEncryption(conf, hcd.getEncryptionType(), hcd.getEncryptionKey());
1756   }
1757
1758   private void checkClassLoading(final Configuration conf, final HTableDescriptor htd)
1759   throws IOException {
1760     RegionSplitPolicy.getSplitPolicyClass(htd, conf);
1761     RegionCoprocessorHost.testTableCoprocessorAttrs(conf, htd);
1762   }
1763
1764   private static boolean isCatalogTable(final TableName tableName) {
1765     return tableName.equals(TableName.META_TABLE_NAME);
1766   }
1767
1768   @Override
1769   public long deleteTable(
1770       final TableName tableName,
1771       final long nonceGroup,
1772       final long nonce) throws IOException {
1773     checkInitialized();
1774     if (cpHost != null) {
1775       cpHost.preDeleteTable(tableName);
1776     }
1777     LOG.info(getClientIdAuditPrefix() + " delete " + tableName);
1778
1779     // TODO: We can handle/merge duplicate request
1780     ProcedurePrepareLatch latch = ProcedurePrepareLatch.createLatch();
1781     long procId = this.procedureExecutor.submitProcedure(
1782       new DeleteTableProcedure(procedureExecutor.getEnvironment(), tableName, latch),
1783       nonceGroup,
1784       nonce);
1785     latch.await();
1786
1787     if (cpHost != null) {
1788       cpHost.postDeleteTable(tableName);
1789     }
1790
1791     return procId;
1792   }
1793
1794   @Override
1795   public long truncateTable(
1796       final TableName tableName,
1797       final boolean preserveSplits,
1798       final long nonceGroup,
1799       final long nonce) throws IOException {
1800     checkInitialized();
1801     if (cpHost != null) {
1802       cpHost.preTruncateTable(tableName);
1803     }
1804     LOG.info(getClientIdAuditPrefix() + " truncate " + tableName);
1805
1806     long procId = this.procedureExecutor.submitProcedure(
1807       new TruncateTableProcedure(procedureExecutor.getEnvironment(), tableName, preserveSplits),
1808       nonceGroup,
1809       nonce);
1810     ProcedureSyncWait.waitForProcedureToComplete(procedureExecutor, procId);
1811
1812     if (cpHost != null) {
1813       cpHost.postTruncateTable(tableName);
1814     }
1815     return procId;
1816   }
1817
1818   @Override
1819   public long addColumn(
1820       final TableName tableName,
1821       final HColumnDescriptor columnDescriptor,
1822       final long nonceGroup,
1823       final long nonce)
1824       throws IOException {
1825     checkInitialized();
1826     checkCompression(columnDescriptor);
1827     checkEncryption(conf, columnDescriptor);
1828     if (cpHost != null) {
1829       if (cpHost.preAddColumn(tableName, columnDescriptor)) {
1830         return -1;
1831       }
1832     }
1833     // Execute the operation synchronously - wait for the operation to complete before continuing.
1834     long procId = this.procedureExecutor.submitProcedure(
1835       new AddColumnFamilyProcedure(procedureExecutor.getEnvironment(), tableName, columnDescriptor),
1836       nonceGroup,
1837       nonce);
1838     ProcedureSyncWait.waitForProcedureToComplete(procedureExecutor, procId);
1839     if (cpHost != null) {
1840       cpHost.postAddColumn(tableName, columnDescriptor);
1841     }
1842     return procId;
1843   }
1844
1845   @Override
1846   public long modifyColumn(
1847       final TableName tableName,
1848       final HColumnDescriptor descriptor,
1849       final long nonceGroup,
1850       final long nonce)
1851       throws IOException {
1852     checkInitialized();
1853     checkCompression(descriptor);
1854     checkEncryption(conf, descriptor);
1855     if (cpHost != null) {
1856       if (cpHost.preModifyColumn(tableName, descriptor)) {
1857         return -1;
1858       }
1859     }
1860     LOG.info(getClientIdAuditPrefix() + " modify " + descriptor);
1861
1862     // Execute the operation synchronously - wait for the operation to complete before continuing.
1863     long procId = this.procedureExecutor.submitProcedure(
1864       new ModifyColumnFamilyProcedure(procedureExecutor.getEnvironment(), tableName, descriptor),
1865       nonceGroup,
1866       nonce);
1867     ProcedureSyncWait.waitForProcedureToComplete(procedureExecutor, procId);
1868
1869     if (cpHost != null) {
1870       cpHost.postModifyColumn(tableName, descriptor);
1871     }
1872     return procId;
1873   }
1874
1875   @Override
1876   public long deleteColumn(
1877       final TableName tableName,
1878       final byte[] columnName,
1879       final long nonceGroup,
1880       final long nonce)
1881       throws IOException {
1882     checkInitialized();
1883     if (cpHost != null) {
1884       if (cpHost.preDeleteColumn(tableName, columnName)) {
1885         return -1;
1886       }
1887     }
1888     LOG.info(getClientIdAuditPrefix() + " delete " + Bytes.toString(columnName));
1889
1890     // Execute the operation synchronously - wait for the operation to complete before continuing.
1891     long procId = this.procedureExecutor.submitProcedure(
1892       new DeleteColumnFamilyProcedure(procedureExecutor.getEnvironment(), tableName, columnName),
1893       nonceGroup,
1894       nonce);
1895     ProcedureSyncWait.waitForProcedureToComplete(procedureExecutor, procId);
1896
1897     if (cpHost != null) {
1898       cpHost.postDeleteColumn(tableName, columnName);
1899     }
1900     return procId;
1901   }
1902
1903   @Override
1904   public long enableTable(
1905       final TableName tableName,
1906       final long nonceGroup,
1907       final long nonce) throws IOException {
1908     checkInitialized();
1909     if (cpHost != null) {
1910       cpHost.preEnableTable(tableName);
1911     }
1912     LOG.info(getClientIdAuditPrefix() + " enable " + tableName);
1913
1914     // Execute the operation asynchronously - client will check the progress of the operation
1915     final ProcedurePrepareLatch prepareLatch = ProcedurePrepareLatch.createLatch();
1916     long procId = this.procedureExecutor.submitProcedure(
1917       new EnableTableProcedure(procedureExecutor.getEnvironment(), tableName, false, prepareLatch),
1918       nonceGroup,
1919       nonce);
1920     // Before returning to client, we want to make sure that the table is prepared to be
1921     // enabled (the table is locked and the table state is set).
1922     //
1923     // Note: if the procedure throws exception, we will catch it and rethrow.
1924     prepareLatch.await();
1925
1926     if (cpHost != null) {
1927       cpHost.postEnableTable(tableName);
1928     }
1929
1930     return procId;
1931   }
1932
1933   @Override
1934   public long disableTable(
1935       final TableName tableName,
1936       final long nonceGroup,
1937       final long nonce) throws IOException {
1938     checkInitialized();
1939     if (cpHost != null) {
1940       cpHost.preDisableTable(tableName);
1941     }
1942     LOG.info(getClientIdAuditPrefix() + " disable " + tableName);
1943
1944     // Execute the operation asynchronously - client will check the progress of the operation
1945     final ProcedurePrepareLatch prepareLatch = ProcedurePrepareLatch.createLatch();
1946     // Execute the operation asynchronously - client will check the progress of the operation
1947     long procId = this.procedureExecutor.submitProcedure(
1948       new DisableTableProcedure(procedureExecutor.getEnvironment(), tableName, false, prepareLatch),
1949       nonceGroup,
1950       nonce);
1951     // Before returning to client, we want to make sure that the table is prepared to be
1952     // enabled (the table is locked and the table state is set).
1953     //
1954     // Note: if the procedure throws exception, we will catch it and rethrow.
1955     prepareLatch.await();
1956
1957     if (cpHost != null) {
1958       cpHost.postDisableTable(tableName);
1959     }
1960
1961     return procId;
1962   }
1963
1964   /**
1965    * Return the region and current deployment for the region containing
1966    * the given row. If the region cannot be found, returns null. If it
1967    * is found, but not currently deployed, the second element of the pair
1968    * may be null.
1969    */
1970   @VisibleForTesting // Used by TestMaster.
1971   Pair<HRegionInfo, ServerName> getTableRegionForRow(
1972       final TableName tableName, final byte [] rowKey)
1973   throws IOException {
1974     final AtomicReference<Pair<HRegionInfo, ServerName>> result =
1975       new AtomicReference<Pair<HRegionInfo, ServerName>>(null);
1976
1977     MetaTableAccessor.Visitor visitor = new MetaTableAccessor.Visitor() {
1978         @Override
1979         public boolean visit(Result data) throws IOException {
1980           if (data == null || data.size() <= 0) {
1981             return true;
1982           }
1983           Pair<HRegionInfo, ServerName> pair =
1984               new Pair(MetaTableAccessor.getHRegionInfo(data),
1985                   MetaTableAccessor.getServerName(data,0));
1986           if (pair == null) {
1987             return false;
1988           }
1989           if (!pair.getFirst().getTable().equals(tableName)) {
1990             return false;
1991           }
1992           result.set(pair);
1993           return true;
1994         }
1995     };
1996
1997     MetaTableAccessor.scanMeta(clusterConnection, visitor, tableName, rowKey, 1);
1998     return result.get();
1999   }
2000
2001   @Override
2002   public long modifyTable(
2003       final TableName tableName,
2004       final HTableDescriptor descriptor,
2005       final long nonceGroup,
2006       final long nonce)
2007       throws IOException {
2008     checkInitialized();
2009     sanityCheckTableDescriptor(descriptor);
2010     if (cpHost != null) {
2011       cpHost.preModifyTable(tableName, descriptor);
2012     }
2013
2014     LOG.info(getClientIdAuditPrefix() + " modify " + tableName);
2015
2016     // Execute the operation synchronously - wait for the operation completes before continuing.
2017     long procId = this.procedureExecutor.submitProcedure(
2018       new ModifyTableProcedure(procedureExecutor.getEnvironment(), descriptor),
2019       nonceGroup,
2020       nonce);
2021
2022     ProcedureSyncWait.waitForProcedureToComplete(procedureExecutor, procId);
2023
2024     if (cpHost != null) {
2025       cpHost.postModifyTable(tableName, descriptor);
2026     }
2027
2028     return procId;
2029   }
2030
2031   @Override
2032   public void checkTableModifiable(final TableName tableName)
2033       throws IOException, TableNotFoundException, TableNotDisabledException {
2034     if (isCatalogTable(tableName)) {
2035       throw new IOException("Can't modify catalog tables");
2036     }
2037     if (!MetaTableAccessor.tableExists(getConnection(), tableName)) {
2038       throw new TableNotFoundException(tableName);
2039     }
2040     if (!getTableStateManager().isTableState(tableName, TableState.State.DISABLED)) {
2041       throw new TableNotDisabledException(tableName);
2042     }
2043   }
2044
2045   /**
2046    * @return cluster status
2047    */
2048   public ClusterStatus getClusterStatus() throws InterruptedIOException {
2049     // Build Set of backup masters from ZK nodes
2050     List<String> backupMasterStrings;
2051     try {
2052       backupMasterStrings = ZKUtil.listChildrenNoWatch(this.zooKeeper,
2053         this.zooKeeper.backupMasterAddressesZNode);
2054     } catch (KeeperException e) {
2055       LOG.warn(this.zooKeeper.prefix("Unable to list backup servers"), e);
2056       backupMasterStrings = null;
2057     }
2058
2059     List<ServerName> backupMasters = null;
2060     if (backupMasterStrings != null && !backupMasterStrings.isEmpty()) {
2061       backupMasters = new ArrayList<ServerName>(backupMasterStrings.size());
2062       for (String s: backupMasterStrings) {
2063         try {
2064           byte [] bytes;
2065           try {
2066             bytes = ZKUtil.getData(this.zooKeeper, ZKUtil.joinZNode(
2067                 this.zooKeeper.backupMasterAddressesZNode, s));
2068           } catch (InterruptedException e) {
2069             throw new InterruptedIOException();
2070           }
2071           if (bytes != null) {
2072             ServerName sn;
2073             try {
2074               sn = ServerName.parseFrom(bytes);
2075             } catch (DeserializationException e) {
2076               LOG.warn("Failed parse, skipping registering backup server", e);
2077               continue;
2078             }
2079             backupMasters.add(sn);
2080           }
2081         } catch (KeeperException e) {
2082           LOG.warn(this.zooKeeper.prefix("Unable to get information about " +
2083                    "backup servers"), e);
2084         }
2085       }
2086       Collections.sort(backupMasters, new Comparator<ServerName>() {
2087         @Override
2088         public int compare(ServerName s1, ServerName s2) {
2089           return s1.getServerName().compareTo(s2.getServerName());
2090         }});
2091     }
2092
2093     String clusterId = fileSystemManager != null ?
2094       fileSystemManager.getClusterId().toString() : null;
2095     Set<RegionState> regionsInTransition = assignmentManager != null ?
2096       assignmentManager.getRegionStates().getRegionsInTransition() : null;
2097     String[] coprocessors = cpHost != null ? getMasterCoprocessors() : null;
2098     boolean balancerOn = loadBalancerTracker != null ?
2099       loadBalancerTracker.isBalancerOn() : false;
2100     Map<ServerName, ServerLoad> onlineServers = null;
2101     Set<ServerName> deadServers = null;
2102     if (serverManager != null) {
2103       deadServers = serverManager.getDeadServers().copyServerNames();
2104       onlineServers = serverManager.getOnlineServers();
2105     }
2106     return new ClusterStatus(VersionInfo.getVersion(), clusterId,
2107       onlineServers, deadServers, serverName, backupMasters,
2108       regionsInTransition, coprocessors, balancerOn);
2109   }
2110
2111   /**
2112    * The set of loaded coprocessors is stored in a static set. Since it's
2113    * statically allocated, it does not require that HMaster's cpHost be
2114    * initialized prior to accessing it.
2115    * @return a String representation of the set of names of the loaded coprocessors.
2116    */
2117   public static String getLoadedCoprocessors() {
2118     return CoprocessorHost.getLoadedCoprocessors().toString();
2119   }
2120
2121   /**
2122    * @return timestamp in millis when HMaster was started.
2123    */
2124   public long getMasterStartTime() {
2125     return startcode;
2126   }
2127
2128   /**
2129    * @return timestamp in millis when HMaster became the active master.
2130    */
2131   public long getMasterActiveTime() {
2132     return masterActiveTime;
2133   }
2134
2135   public int getNumWALFiles() {
2136     return procedureStore != null ? procedureStore.getActiveLogs().size() : 0;
2137   }
2138
2139   public WALProcedureStore getWalProcedureStore() {
2140     return procedureStore;
2141   }
2142
2143   public int getRegionServerInfoPort(final ServerName sn) {
2144     RegionServerInfo info = this.regionServerTracker.getRegionServerInfo(sn);
2145     if (info == null || info.getInfoPort() == 0) {
2146       return conf.getInt(HConstants.REGIONSERVER_INFO_PORT,
2147         HConstants.DEFAULT_REGIONSERVER_INFOPORT);
2148     }
2149     return info.getInfoPort();
2150   }
2151
2152   public String getRegionServerVersion(final ServerName sn) {
2153     RegionServerInfo info = this.regionServerTracker.getRegionServerInfo(sn);
2154     if (info != null && info.hasVersionInfo()) {
2155       return info.getVersionInfo().getVersion();
2156     }
2157     return "Unknown";
2158   }
2159
2160   /**
2161    * @return array of coprocessor SimpleNames.
2162    */
2163   public String[] getMasterCoprocessors() {
2164     Set<String> masterCoprocessors = getMasterCoprocessorHost().getCoprocessors();
2165     return masterCoprocessors.toArray(new String[masterCoprocessors.size()]);
2166   }
2167
2168   @Override
2169   public void abort(final String msg, final Throwable t) {
2170     if (isAborted() || isStopped()) {
2171       return;
2172     }
2173     if (cpHost != null) {
2174       // HBASE-4014: dump a list of loaded coprocessors.
2175       LOG.fatal("Master server abort: loaded coprocessors are: " +
2176           getLoadedCoprocessors());
2177     }
2178     if (t != null) LOG.fatal(msg, t);
2179     stopMaster();
2180   }
2181
2182   @Override
2183   public ZooKeeperWatcher getZooKeeper() {
2184     return zooKeeper;
2185   }
2186
2187   @Override
2188   public MasterCoprocessorHost getMasterCoprocessorHost() {
2189     return cpHost;
2190   }
2191
2192   @Override
2193   public MasterQuotaManager getMasterQuotaManager() {
2194     return quotaManager;
2195   }
2196
2197   @Override
2198   public ProcedureExecutor<MasterProcedureEnv> getMasterProcedureExecutor() {
2199     return procedureExecutor;
2200   }
2201
2202   @Override
2203   public ServerName getServerName() {
2204     return this.serverName;
2205   }
2206
2207   @Override
2208   public AssignmentManager getAssignmentManager() {
2209     return this.assignmentManager;
2210   }
2211
2212   @Override
2213   public CatalogJanitor getCatalogJanitor() {
2214     return this.catalogJanitorChore;
2215   }
2216
2217   public MemoryBoundedLogMessageBuffer getRegionServerFatalLogBuffer() {
2218     return rsFatals;
2219   }
2220
2221   public void shutdown() {
2222     if (cpHost != null) {
2223       try {
2224         cpHost.preShutdown();
2225       } catch (IOException ioe) {
2226         LOG.error("Error call master coprocessor preShutdown()", ioe);
2227       }
2228     }
2229
2230     if (this.serverManager != null) {
2231       this.serverManager.shutdownCluster();
2232     }
2233     if (this.clusterStatusTracker != null){
2234       try {
2235         this.clusterStatusTracker.setClusterDown();
2236       } catch (KeeperException e) {
2237         LOG.error("ZooKeeper exception trying to set cluster as down in ZK", e);
2238       }
2239     }
2240   }
2241
2242   public void stopMaster() {
2243     if (cpHost != null) {
2244       try {
2245         cpHost.preStopMaster();
2246       } catch (IOException ioe) {
2247         LOG.error("Error call master coprocessor preStopMaster()", ioe);
2248       }
2249     }
2250     stop("Stopped by " + Thread.currentThread().getName());
2251   }
2252
2253   void checkServiceStarted() throws ServerNotRunningYetException {
2254     if (!serviceStarted) {
2255       throw new ServerNotRunningYetException("Server is not running yet");
2256     }
2257   }
2258
2259   void checkInitialized() throws PleaseHoldException, ServerNotRunningYetException {
2260     checkServiceStarted();
2261     if (!isInitialized()) throw new PleaseHoldException("Master is initializing");
2262   }
2263
2264   /**
2265    * Report whether this master is currently the active master or not.
2266    * If not active master, we are parked on ZK waiting to become active.
2267    *
2268    * This method is used for testing.
2269    *
2270    * @return true if active master, false if not.
2271    */
2272   @Override
2273   public boolean isActiveMaster() {
2274     return isActiveMaster;
2275   }
2276
2277   /**
2278    * Report whether this master has completed with its initialization and is
2279    * ready.  If ready, the master is also the active master.  A standby master
2280    * is never ready.
2281    *
2282    * This method is used for testing.
2283    *
2284    * @return true if master is ready to go, false if not.
2285    */
2286   @Override
2287   public boolean isInitialized() {
2288     return initialized.isReady();
2289   }
2290
2291   /**
2292    * Report whether this master is in maintenance mode.
2293    *
2294    * @return true if master is in maintenanceMode
2295    */
2296   @Override
2297   public boolean isInMaintenanceMode() {
2298     return maintenanceModeTracker.isInMaintenanceMode();
2299   }
2300
2301   @VisibleForTesting
2302   public void setInitialized(boolean isInitialized) {
2303     procedureExecutor.getEnvironment().setEventReady(initialized, isInitialized);
2304   }
2305
2306   public ProcedureEvent getInitializedEvent() {
2307     return initialized;
2308   }
2309
2310   /**
2311    * ServerCrashProcessingEnabled is set false before completing assignMeta to prevent processing
2312    * of crashed servers.
2313    * @return true if assignMeta has completed;
2314    */
2315   @Override
2316   public boolean isServerCrashProcessingEnabled() {
2317     return serverCrashProcessingEnabled.isReady();
2318   }
2319
2320   @VisibleForTesting
2321   public void setServerCrashProcessingEnabled(final boolean b) {
2322     procedureExecutor.getEnvironment().setEventReady(serverCrashProcessingEnabled, b);
2323   }
2324
2325   public ProcedureEvent getServerCrashProcessingEnabledEvent() {
2326     return serverCrashProcessingEnabled;
2327   }
2328
2329   /**
2330    * Report whether this master has started initialization and is about to do meta region assignment
2331    * @return true if master is in initialization &amp; about to assign hbase:meta regions
2332    */
2333   public boolean isInitializationStartsMetaRegionAssignment() {
2334     return this.initializationBeforeMetaAssignment;
2335   }
2336
2337   /**
2338    * Compute the average load across all region servers.
2339    * Currently, this uses a very naive computation - just uses the number of
2340    * regions being served, ignoring stats about number of requests.
2341    * @return the average load
2342    */
2343   public double getAverageLoad() {
2344     if (this.assignmentManager == null) {
2345       return 0;
2346     }
2347
2348     RegionStates regionStates = this.assignmentManager.getRegionStates();
2349     if (regionStates == null) {
2350       return 0;
2351     }
2352     return regionStates.getAverageLoad();
2353   }
2354
2355   /*
2356    * @return the count of region split plans executed
2357    */
2358   public long getSplitPlanCount() {
2359     return splitPlanCount;
2360   }
2361
2362   /*
2363    * @return the count of region merge plans executed
2364    */
2365   public long getMergePlanCount() {
2366     return mergePlanCount;
2367   }
2368
2369   @Override
2370   public boolean registerService(Service instance) {
2371     /*
2372      * No stacking of instances is allowed for a single service name
2373      */
2374     Descriptors.ServiceDescriptor serviceDesc = instance.getDescriptorForType();
2375     String serviceName = CoprocessorRpcUtils.getServiceName(serviceDesc);
2376     if (coprocessorServiceHandlers.containsKey(serviceName)) {
2377       LOG.error("Coprocessor service "+serviceName+
2378           " already registered, rejecting request from "+instance
2379       );
2380       return false;
2381     }
2382
2383     coprocessorServiceHandlers.put(serviceName, instance);
2384     if (LOG.isDebugEnabled()) {
2385       LOG.debug("Registered master coprocessor service: service="+serviceName);
2386     }
2387     return true;
2388   }
2389
2390   /**
2391    * Utility for constructing an instance of the passed HMaster class.
2392    * @param masterClass
2393    * @return HMaster instance.
2394    */
2395   public static HMaster constructMaster(Class<? extends HMaster> masterClass,
2396       final Configuration conf, final CoordinatedStateManager cp)  {
2397     try {
2398       Constructor<? extends HMaster> c =
2399         masterClass.getConstructor(Configuration.class, CoordinatedStateManager.class);
2400       return c.newInstance(conf, cp);
2401     } catch(Exception e) {
2402       Throwable error = e;
2403       if (e instanceof InvocationTargetException &&
2404           ((InvocationTargetException)e).getTargetException() != null) {
2405         error = ((InvocationTargetException)e).getTargetException();
2406       }
2407       throw new RuntimeException("Failed construction of Master: " + masterClass.toString() + ". "
2408         , error);
2409     }
2410   }
2411
2412   /**
2413    * @see org.apache.hadoop.hbase.master.HMasterCommandLine
2414    */
2415   public static void main(String [] args) {
2416     VersionInfo.logVersion();
2417     new HMasterCommandLine(HMaster.class).doMain(args);
2418   }
2419
2420   public HFileCleaner getHFileCleaner() {
2421     return this.hfileCleaner;
2422   }
2423
2424   /**
2425    * @return the underlying snapshot manager
2426    */
2427   @Override
2428   public SnapshotManager getSnapshotManager() {
2429     return this.snapshotManager;
2430   }
2431
2432   /**
2433    * @return the underlying MasterProcedureManagerHost
2434    */
2435   @Override
2436   public MasterProcedureManagerHost getMasterProcedureManagerHost() {
2437     return mpmHost;
2438   }
2439
2440   @Override
2441   public ClusterSchema getClusterSchema() {
2442     return this.clusterSchemaService;
2443   }
2444
2445   /**
2446    * Create a new Namespace.
2447    * @param namespaceDescriptor descriptor for new Namespace
2448    * @param nonceGroup Identifier for the source of the request, a client or process.
2449    * @param nonce A unique identifier for this operation from the client or process identified by
2450    * <code>nonceGroup</code> (the source must ensure each operation gets a unique id).
2451    * @return procedure id
2452    */
2453   long createNamespace(final NamespaceDescriptor namespaceDescriptor, final long nonceGroup,
2454       final long nonce)
2455   throws IOException {
2456     checkInitialized();
2457     TableName.isLegalNamespaceName(Bytes.toBytes(namespaceDescriptor.getName()));
2458     if (this.cpHost != null && this.cpHost.preCreateNamespace(namespaceDescriptor)) {
2459       throw new BypassCoprocessorException();
2460     }
2461     LOG.info(getClientIdAuditPrefix() + " creating " + namespaceDescriptor);
2462     // Execute the operation synchronously - wait for the operation to complete before continuing.
2463     long procId = getClusterSchema().createNamespace(namespaceDescriptor, nonceGroup, nonce);
2464     if (this.cpHost != null) this.cpHost.postCreateNamespace(namespaceDescriptor);
2465     return procId;
2466   }
2467
2468   /**
2469    * Modify an existing Namespace.
2470    * @param nonceGroup Identifier for the source of the request, a client or process.
2471    * @param nonce A unique identifier for this operation from the client or process identified by
2472    * <code>nonceGroup</code> (the source must ensure each operation gets a unique id).
2473    * @return procedure id
2474    */
2475   long modifyNamespace(final NamespaceDescriptor namespaceDescriptor, final long nonceGroup,
2476       final long nonce)
2477   throws IOException {
2478     checkInitialized();
2479     TableName.isLegalNamespaceName(Bytes.toBytes(namespaceDescriptor.getName()));
2480     if (this.cpHost != null && this.cpHost.preModifyNamespace(namespaceDescriptor)) {
2481       throw new BypassCoprocessorException();
2482     }
2483     LOG.info(getClientIdAuditPrefix() + " modify " + namespaceDescriptor);
2484     // Execute the operation synchronously - wait for the operation to complete before continuing.
2485     long procId = getClusterSchema().modifyNamespace(namespaceDescriptor, nonceGroup, nonce);
2486     if (this.cpHost != null) this.cpHost.postModifyNamespace(namespaceDescriptor);
2487     return procId;
2488   }
2489
2490   /**
2491    * Delete an existing Namespace. Only empty Namespaces (no tables) can be removed.
2492    * @param nonceGroup Identifier for the source of the request, a client or process.
2493    * @param nonce A unique identifier for this operation from the client or process identified by
2494    * <code>nonceGroup</code> (the source must ensure each operation gets a unique id).
2495    * @return procedure id
2496    */
2497   long deleteNamespace(final String name, final long nonceGroup, final long nonce)
2498   throws IOException {
2499     checkInitialized();
2500     if (this.cpHost != null && this.cpHost.preDeleteNamespace(name)) {
2501       throw new BypassCoprocessorException();
2502     }
2503     LOG.info(getClientIdAuditPrefix() + " delete " + name);
2504     // Execute the operation synchronously - wait for the operation to complete before continuing.
2505     long procId = getClusterSchema().deleteNamespace(name, nonceGroup, nonce);
2506     if (this.cpHost != null) this.cpHost.postDeleteNamespace(name);
2507     return procId;
2508   }
2509
2510   /**
2511    * Get a Namespace
2512    * @param name Name of the Namespace
2513    * @return Namespace descriptor for <code>name</code>
2514    */
2515   NamespaceDescriptor getNamespace(String name) throws IOException {
2516     checkInitialized();
2517     if (this.cpHost != null) this.cpHost.preGetNamespaceDescriptor(name);
2518     NamespaceDescriptor nsd = this.clusterSchemaService.getNamespace(name);
2519     if (this.cpHost != null) this.cpHost.postGetNamespaceDescriptor(nsd);
2520     return nsd;
2521   }
2522
2523   /**
2524    * Get all Namespaces
2525    * @return All Namespace descriptors
2526    */
2527   List<NamespaceDescriptor> getNamespaces() throws IOException {
2528     checkInitialized();
2529     final List<NamespaceDescriptor> nsds = new ArrayList<NamespaceDescriptor>();
2530     boolean bypass = false;
2531     if (cpHost != null) {
2532       bypass = cpHost.preListNamespaceDescriptors(nsds);
2533     }
2534     if (!bypass) {
2535       nsds.addAll(this.clusterSchemaService.getNamespaces());
2536       if (this.cpHost != null) this.cpHost.postListNamespaceDescriptors(nsds);
2537     }
2538     return nsds;
2539   }
2540
2541   @Override
2542   public List<TableName> listTableNamesByNamespace(String name) throws IOException {
2543     checkInitialized();
2544     return listTableNames(name, null, true);
2545   }
2546
2547   @Override
2548   public List<HTableDescriptor> listTableDescriptorsByNamespace(String name) throws IOException {
2549     checkInitialized();
2550     return listTableDescriptors(name, null, null, true);
2551   }
2552
2553   @Override
2554   public boolean abortProcedure(final long procId, final boolean mayInterruptIfRunning)
2555       throws IOException {
2556     if (cpHost != null) {
2557       cpHost.preAbortProcedure(this.procedureExecutor, procId);
2558     }
2559
2560     final boolean result = this.procedureExecutor.abort(procId, mayInterruptIfRunning);
2561
2562     if (cpHost != null) {
2563       cpHost.postAbortProcedure();
2564     }
2565
2566     return result;
2567   }
2568
2569   @Override
2570   public List<ProcedureInfo> listProcedures() throws IOException {
2571     if (cpHost != null) {
2572       cpHost.preListProcedures();
2573     }
2574
2575     final List<ProcedureInfo> procInfoList = this.procedureExecutor.listProcedures();
2576
2577     if (cpHost != null) {
2578       cpHost.postListProcedures(procInfoList);
2579     }
2580
2581     return procInfoList;
2582   }
2583
2584   /**
2585    * Returns the list of table descriptors that match the specified request
2586    * @param namespace the namespace to query, or null if querying for all
2587    * @param regex The regular expression to match against, or null if querying for all
2588    * @param tableNameList the list of table names, or null if querying for all
2589    * @param includeSysTables False to match only against userspace tables
2590    * @return the list of table descriptors
2591    */
2592   public List<HTableDescriptor> listTableDescriptors(final String namespace, final String regex,
2593       final List<TableName> tableNameList, final boolean includeSysTables)
2594   throws IOException {
2595     List<HTableDescriptor> htds = new ArrayList<HTableDescriptor>();
2596     boolean bypass = cpHost != null?
2597         cpHost.preGetTableDescriptors(tableNameList, htds, regex): false;
2598     if (!bypass) {
2599       htds = getTableDescriptors(htds, namespace, regex, tableNameList, includeSysTables);
2600       if (cpHost != null) {
2601         cpHost.postGetTableDescriptors(tableNameList, htds, regex);
2602       }
2603     }
2604     return htds;
2605   }
2606
2607   /**
2608    * Returns the list of table names that match the specified request
2609    * @param regex The regular expression to match against, or null if querying for all
2610    * @param namespace the namespace to query, or null if querying for all
2611    * @param includeSysTables False to match only against userspace tables
2612    * @return the list of table names
2613    */
2614   public List<TableName> listTableNames(final String namespace, final String regex,
2615       final boolean includeSysTables) throws IOException {
2616     List<HTableDescriptor> htds = new ArrayList<HTableDescriptor>();
2617     boolean bypass = cpHost != null? cpHost.preGetTableNames(htds, regex): false;
2618     if (!bypass) {
2619       htds = getTableDescriptors(htds, namespace, regex, null, includeSysTables);
2620       if (cpHost != null) cpHost.postGetTableNames(htds, regex);
2621     }
2622     List<TableName> result = new ArrayList<TableName>(htds.size());
2623     for (HTableDescriptor htd: htds) result.add(htd.getTableName());
2624     return result;
2625   }
2626
2627   /**
2628    * @return list of table table descriptors after filtering by regex and whether to include system
2629    *    tables, etc.
2630    * @throws IOException
2631    */
2632   private List<HTableDescriptor> getTableDescriptors(final List<HTableDescriptor> htds,
2633       final String namespace, final String regex, final List<TableName> tableNameList,
2634       final boolean includeSysTables)
2635   throws IOException {
2636     if (tableNameList == null || tableNameList.size() == 0) {
2637       // request for all TableDescriptors
2638       Collection<HTableDescriptor> allHtds;
2639       if (namespace != null && namespace.length() > 0) {
2640         // Do a check on the namespace existence. Will fail if does not exist.
2641         this.clusterSchemaService.getNamespace(namespace);
2642         allHtds = tableDescriptors.getByNamespace(namespace).values();
2643       } else {
2644         allHtds = tableDescriptors.getAll().values();
2645       }
2646       for (HTableDescriptor desc: allHtds) {
2647         if (tableStateManager.isTablePresent(desc.getTableName())
2648             && (includeSysTables || !desc.getTableName().isSystemTable())) {
2649           htds.add(desc);
2650         }
2651       }
2652     } else {
2653       for (TableName s: tableNameList) {
2654         if (tableStateManager.isTablePresent(s)) {
2655           HTableDescriptor desc = tableDescriptors.get(s);
2656           if (desc != null) {
2657             htds.add(desc);
2658           }
2659         }
2660       }
2661     }
2662
2663     // Retains only those matched by regular expression.
2664     if (regex != null) filterTablesByRegex(htds, Pattern.compile(regex));
2665     return htds;
2666   }
2667
2668   /**
2669    * Removes the table descriptors that don't match the pattern.
2670    * @param descriptors list of table descriptors to filter
2671    * @param pattern the regex to use
2672    */
2673   private static void filterTablesByRegex(final Collection<HTableDescriptor> descriptors,
2674       final Pattern pattern) {
2675     final String defaultNS = NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR;
2676     Iterator<HTableDescriptor> itr = descriptors.iterator();
2677     while (itr.hasNext()) {
2678       HTableDescriptor htd = itr.next();
2679       String tableName = htd.getTableName().getNameAsString();
2680       boolean matched = pattern.matcher(tableName).matches();
2681       if (!matched && htd.getTableName().getNamespaceAsString().equals(defaultNS)) {
2682         matched = pattern.matcher(defaultNS + TableName.NAMESPACE_DELIM + tableName).matches();
2683       }
2684       if (!matched) {
2685         itr.remove();
2686       }
2687     }
2688   }
2689
2690   @Override
2691   public long getLastMajorCompactionTimestamp(TableName table) throws IOException {
2692     return getClusterStatus().getLastMajorCompactionTsForTable(table);
2693   }
2694
2695   @Override
2696   public long getLastMajorCompactionTimestampForRegion(byte[] regionName) throws IOException {
2697     return getClusterStatus().getLastMajorCompactionTsForRegion(regionName);
2698   }
2699
2700   /**
2701    * Gets the mob file compaction state for a specific table.
2702    * Whether all the mob files are selected is known during the compaction execution, but
2703    * the statistic is done just before compaction starts, it is hard to know the compaction
2704    * type at that time, so the rough statistics are chosen for the mob file compaction. Only two
2705    * compaction states are available, CompactionState.MAJOR_AND_MINOR and CompactionState.NONE.
2706    * @param tableName The current table name.
2707    * @return If a given table is in mob file compaction now.
2708    */
2709   public CompactionState getMobCompactionState(TableName tableName) {
2710     AtomicInteger compactionsCount = mobCompactionStates.get(tableName);
2711     if (compactionsCount != null && compactionsCount.get() != 0) {
2712       return CompactionState.MAJOR_AND_MINOR;
2713     }
2714     return CompactionState.NONE;
2715   }
2716
2717   public void reportMobCompactionStart(TableName tableName) throws IOException {
2718     IdLock.Entry lockEntry = null;
2719     try {
2720       lockEntry = mobCompactionLock.getLockEntry(tableName.hashCode());
2721       AtomicInteger compactionsCount = mobCompactionStates.get(tableName);
2722       if (compactionsCount == null) {
2723         compactionsCount = new AtomicInteger(0);
2724         mobCompactionStates.put(tableName, compactionsCount);
2725       }
2726       compactionsCount.incrementAndGet();
2727     } finally {
2728       if (lockEntry != null) {
2729         mobCompactionLock.releaseLockEntry(lockEntry);
2730       }
2731     }
2732   }
2733
2734   public void reportMobCompactionEnd(TableName tableName) throws IOException {
2735     IdLock.Entry lockEntry = null;
2736     try {
2737       lockEntry = mobCompactionLock.getLockEntry(tableName.hashCode());
2738       AtomicInteger compactionsCount = mobCompactionStates.get(tableName);
2739       if (compactionsCount != null) {
2740         int count = compactionsCount.decrementAndGet();
2741         // remove the entry if the count is 0.
2742         if (count == 0) {
2743           mobCompactionStates.remove(tableName);
2744         }
2745       }
2746     } finally {
2747       if (lockEntry != null) {
2748         mobCompactionLock.releaseLockEntry(lockEntry);
2749       }
2750     }
2751   }
2752
2753   /**
2754    * Requests mob compaction.
2755    * @param tableName The table the compact.
2756    * @param columns The compacted columns.
2757    * @param allFiles Whether add all mob files into the compaction.
2758    */
2759   public void requestMobCompaction(TableName tableName,
2760     List<HColumnDescriptor> columns, boolean allFiles) throws IOException {
2761     mobCompactThread.requestMobCompaction(conf, fs, tableName, columns,
2762       tableLockManager, allFiles);
2763   }
2764
2765   /**
2766    * Queries the state of the {@link LoadBalancerTracker}. If the balancer is not initialized,
2767    * false is returned.
2768    *
2769    * @return The state of the load balancer, or false if the load balancer isn't defined.
2770    */
2771   public boolean isBalancerOn() {
2772     if (null == loadBalancerTracker || isInMaintenanceMode()) {
2773       return false;
2774     }
2775     return loadBalancerTracker.isBalancerOn();
2776   }
2777
2778   /**
2779    * Queries the state of the {@link RegionNormalizerTracker}. If it's not initialized,
2780    * false is returned.
2781    */
2782   public boolean isNormalizerOn() {
2783     return (null == regionNormalizerTracker || isInMaintenanceMode()) ?
2784         false: regionNormalizerTracker.isNormalizerOn();
2785   }
2786
2787   /**
2788    * Queries the state of the {@link SplitOrMergeTracker}. If it is not initialized,
2789    * false is returned. If switchType is illegal, false will return.
2790    * @param switchType see {@link org.apache.hadoop.hbase.client.MasterSwitchType}
2791    * @return The state of the switch
2792    */
2793   public boolean isSplitOrMergeEnabled(MasterSwitchType switchType) {
2794     if (null == splitOrMergeTracker || isInMaintenanceMode()) {
2795       return false;
2796     }
2797     return splitOrMergeTracker.isSplitOrMergeEnabled(switchType);
2798   }
2799
2800   /**
2801    * Fetch the configured {@link LoadBalancer} class name. If none is set, a default is returned.
2802    *
2803    * @return The name of the {@link LoadBalancer} in use.
2804    */
2805   public String getLoadBalancerClassName() {
2806     return conf.get(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, LoadBalancerFactory
2807         .getDefaultLoadBalancerClass().getName());
2808   }
2809
2810   /**
2811    * @return RegionNormalizerTracker instance
2812    */
2813   public RegionNormalizerTracker getRegionNormalizerTracker() {
2814     return regionNormalizerTracker;
2815   }
2816
2817   public SplitOrMergeTracker getSplitOrMergeTracker() {
2818     return splitOrMergeTracker;
2819   }
2820
2821   @Override
2822   public LoadBalancer getLoadBalancer() {
2823     return balancer;
2824   }
2825 }