001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master;
019
020import java.io.File;
021import java.io.IOException;
022import java.util.List;
023import org.apache.hadoop.conf.Configuration;
024import org.apache.hadoop.hbase.HConstants;
025import org.apache.hadoop.hbase.LocalHBaseCluster;
026import org.apache.hadoop.hbase.MasterNotRunningException;
027import org.apache.hadoop.hbase.ZNodeClearer;
028import org.apache.hadoop.hbase.ZooKeeperConnectionException;
029import org.apache.hadoop.hbase.client.Admin;
030import org.apache.hadoop.hbase.client.Connection;
031import org.apache.hadoop.hbase.client.ConnectionFactory;
032import org.apache.hadoop.hbase.regionserver.HRegionServer;
033import org.apache.hadoop.hbase.util.JVMClusterUtil;
034import org.apache.hadoop.hbase.util.ServerCommandLine;
035import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
036import org.apache.hadoop.hbase.zookeeper.ZKAuthentication;
037import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
038import org.apache.yetus.audience.InterfaceAudience;
039import org.apache.zookeeper.KeeperException;
040import org.slf4j.Logger;
041import org.slf4j.LoggerFactory;
042
043import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine;
044import org.apache.hbase.thirdparty.org.apache.commons.cli.GnuParser;
045import org.apache.hbase.thirdparty.org.apache.commons.cli.Options;
046import org.apache.hbase.thirdparty.org.apache.commons.cli.ParseException;
047
048@InterfaceAudience.Private
049public class HMasterCommandLine extends ServerCommandLine {
050  private static final Logger LOG = LoggerFactory.getLogger(HMasterCommandLine.class);
051
052  private static final String USAGE = "Usage: Master [opts] start|stop|clear\n"
053    + " start  Start Master. If local mode, start Master and RegionServer in same JVM\n"
054    + " stop   Start cluster shutdown; Master signals RegionServer shutdown\n"
055    + " clear  Delete the master znode in ZooKeeper after a master crashes\n "
056    + " where [opts] are:\n"
057    + "   --minRegionServers=<servers>   Minimum RegionServers needed to host user tables.\n"
058    + "   --localRegionServers=<servers> "
059    + "RegionServers to start in master process when in standalone mode.\n"
060    + "   --masters=<servers>            Masters to start in this process.\n"
061    + "   --backup                       Master should start in backup mode\n"
062    + "   --shutDownCluster                    "
063    + "Start Cluster shutdown; Master signals RegionServer shutdown";
064
065  private final Class<? extends HMaster> masterClass;
066
067  public HMasterCommandLine(Class<? extends HMaster> masterClass) {
068    this.masterClass = masterClass;
069  }
070
071  @Override
072  protected String getUsage() {
073    return USAGE;
074  }
075
076  @Override
077  public int run(String args[]) throws Exception {
078    boolean shutDownCluster = false;
079    Options opt = new Options();
080    opt.addOption("localRegionServers", true,
081      "RegionServers to start in master process when running standalone");
082    opt.addOption("masters", true, "Masters to start in this process");
083    opt.addOption("minRegionServers", true, "Minimum RegionServers needed to host user tables");
084    opt.addOption("backup", false, "Do not try to become HMaster until the primary fails");
085    opt.addOption("shutDownCluster", false,
086      "`hbase master stop --shutDownCluster` shuts down cluster");
087
088    CommandLine cmd;
089    try {
090      cmd = new GnuParser().parse(opt, args);
091    } catch (ParseException e) {
092      LOG.error("Could not parse: ", e);
093      usage(null);
094      return 1;
095    }
096
097    if (cmd.hasOption("minRegionServers")) {
098      String val = cmd.getOptionValue("minRegionServers");
099      getConf().setInt("hbase.regions.server.count.min", Integer.parseInt(val));
100      LOG.debug("minRegionServers set to " + val);
101    }
102
103    // minRegionServers used to be minServers. Support it too.
104    if (cmd.hasOption("minServers")) {
105      String val = cmd.getOptionValue("minServers");
106      getConf().setInt("hbase.regions.server.count.min", Integer.parseInt(val));
107      LOG.debug("minServers set to " + val);
108    }
109
110    // check if we are the backup master - override the conf if so
111    if (cmd.hasOption("backup")) {
112      getConf().setBoolean(HConstants.MASTER_TYPE_BACKUP, true);
113    }
114
115    // How many regionservers to startup in this process (we run regionservers in same process as
116    // master when we are in local/standalone mode. Useful testing)
117    if (cmd.hasOption("localRegionServers")) {
118      String val = cmd.getOptionValue("localRegionServers");
119      getConf().setInt("hbase.regionservers", Integer.parseInt(val));
120      LOG.debug("localRegionServers set to " + val);
121    }
122    // How many masters to startup inside this process; useful testing
123    if (cmd.hasOption("masters")) {
124      String val = cmd.getOptionValue("masters");
125      getConf().setInt("hbase.masters", Integer.parseInt(val));
126      LOG.debug("masters set to " + val);
127    }
128
129    // Checking whether to shut down cluster or not
130    if (cmd.hasOption("shutDownCluster")) {
131      shutDownCluster = true;
132    }
133
134    @SuppressWarnings("unchecked")
135    List<String> remainingArgs = cmd.getArgList();
136    if (remainingArgs.size() != 1) {
137      usage(null);
138      return 1;
139    }
140
141    String command = remainingArgs.get(0);
142
143    if ("start".equals(command)) {
144      return startMaster();
145    } else if ("stop".equals(command)) {
146      if (shutDownCluster) {
147        return stopMaster();
148      }
149      System.err.println("To shutdown the master run "
150        + "hbase-daemon.sh stop master or send a kill signal to the HMaster pid, "
151        + "and to stop HBase Cluster run \"stop-hbase.sh\" or \"hbase master "
152        + "stop --shutDownCluster\"");
153      return 1;
154    } else if ("clear".equals(command)) {
155      return (ZNodeClearer.clear(getConf()) ? 0 : 1);
156    } else {
157      usage("Invalid command: " + command);
158      return 1;
159    }
160  }
161
162  private int startMaster() {
163    Configuration conf = getConf();
164    try {
165      // If 'local', defer to LocalHBaseCluster instance. Starts master
166      // and regionserver both in the one JVM.
167      if (LocalHBaseCluster.isLocal(conf)) {
168        DefaultMetricsSystem.setMiniClusterMode(true);
169        final MiniZooKeeperCluster zooKeeperCluster = new MiniZooKeeperCluster(conf);
170        File zkDataPath = new File(conf.get(HConstants.ZOOKEEPER_DATA_DIR));
171
172        // find out the default client port
173        int zkClientPort = 0;
174
175        // If the zookeeper client port is specified in server quorum, use it.
176        String zkserver = conf.get(HConstants.ZOOKEEPER_QUORUM);
177        if (zkserver != null) {
178          String[] zkservers = zkserver.split(",");
179
180          if (zkservers.length > 1) {
181            // In local mode deployment, we have the master + a region server and zookeeper server
182            // started in the same process. Therefore, we only support one zookeeper server.
183            String errorMsg = "Could not start ZK with " + zkservers.length
184              + " ZK servers in local mode deployment. Aborting as clients (e.g. shell) will not "
185              + "be able to find this ZK quorum.";
186            System.err.println(errorMsg);
187            throw new IOException(errorMsg);
188          }
189
190          String[] parts = zkservers[0].split(":");
191
192          if (parts.length == 2) {
193            // the second part is the client port
194            zkClientPort = Integer.parseInt(parts[1]);
195          }
196        }
197        // If the client port could not be find in server quorum conf, try another conf
198        if (zkClientPort == 0) {
199          zkClientPort = conf.getInt(HConstants.ZOOKEEPER_CLIENT_PORT, 0);
200          // The client port has to be set by now; if not, throw exception.
201          if (zkClientPort == 0) {
202            throw new IOException("No config value for " + HConstants.ZOOKEEPER_CLIENT_PORT);
203          }
204        }
205        zooKeeperCluster.setDefaultClientPort(zkClientPort);
206        // set the ZK tick time if specified
207        int zkTickTime = conf.getInt(HConstants.ZOOKEEPER_TICK_TIME, 0);
208        if (zkTickTime > 0) {
209          zooKeeperCluster.setTickTime(zkTickTime);
210        }
211
212        // login the zookeeper server principal (if using security)
213        ZKAuthentication.loginServer(conf, HConstants.ZK_SERVER_KEYTAB_FILE,
214          HConstants.ZK_SERVER_KERBEROS_PRINCIPAL, null);
215        int localZKClusterSessionTimeout =
216          conf.getInt(HConstants.ZK_SESSION_TIMEOUT + ".localHBaseCluster", 10 * 1000);
217        conf.setInt(HConstants.ZK_SESSION_TIMEOUT, localZKClusterSessionTimeout);
218        LOG.info("Starting a zookeeper cluster");
219        int clientPort = zooKeeperCluster.startup(zkDataPath);
220        if (clientPort != zkClientPort) {
221          String errorMsg = "Could not start ZK at requested port of " + zkClientPort
222            + ".  ZK was started at port: " + clientPort
223            + ".  Aborting as clients (e.g. shell) will not be able to find " + "this ZK quorum.";
224          System.err.println(errorMsg);
225          throw new IOException(errorMsg);
226        }
227        conf.set(HConstants.ZOOKEEPER_CLIENT_PORT, Integer.toString(clientPort));
228
229        // Need to have the zk cluster shutdown when master is shutdown.
230        // Run a subclass that does the zk cluster shutdown on its way out.
231        int mastersCount = conf.getInt("hbase.masters", 1);
232        int regionServersCount = conf.getInt("hbase.regionservers", 1);
233        // Set start timeout to 5 minutes for cmd line start operations
234        conf.setIfUnset("hbase.master.start.timeout.localHBaseCluster", "300000");
235        LOG.info("Starting up instance of localHBaseCluster; master=" + mastersCount
236          + ", regionserversCount=" + regionServersCount);
237        LocalHBaseCluster cluster = new LocalHBaseCluster(conf, mastersCount, regionServersCount,
238          LocalHMaster.class, HRegionServer.class);
239        ((LocalHMaster) cluster.getMaster(0)).setZKCluster(zooKeeperCluster);
240        cluster.startup();
241        waitOnMasterThreads(cluster);
242      } else {
243        logProcessInfo(getConf());
244        HMaster master = HMaster.constructMaster(masterClass, conf);
245        if (master.isStopped()) {
246          LOG.info("Won't bring the Master up as a shutdown is requested");
247          return 1;
248        }
249        master.start();
250        master.join();
251        if (master.isAborted()) throw new RuntimeException("HMaster Aborted");
252      }
253    } catch (Throwable t) {
254      LOG.error("Master exiting", t);
255      return 1;
256    }
257    return 0;
258  }
259
260  @SuppressWarnings("resource")
261  private int stopMaster() {
262    Configuration conf = getConf();
263    // Don't try more than once
264    conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 0);
265    try (Connection connection = ConnectionFactory.createConnection(conf)) {
266      try (Admin admin = connection.getAdmin()) {
267        admin.shutdown();
268      } catch (Throwable t) {
269        LOG.error("Failed to stop master", t);
270        return 1;
271      }
272    } catch (MasterNotRunningException e) {
273      LOG.error("Master not running");
274      return 1;
275    } catch (ZooKeeperConnectionException e) {
276      LOG.error("ZooKeeper not available");
277      return 1;
278    } catch (IOException e) {
279      LOG.error("Got IOException: " + e.getMessage(), e);
280      return 1;
281    }
282    return 0;
283  }
284
285  private void waitOnMasterThreads(LocalHBaseCluster cluster) throws InterruptedException {
286    List<JVMClusterUtil.MasterThread> masters = cluster.getMasters();
287    List<JVMClusterUtil.RegionServerThread> regionservers = cluster.getRegionServers();
288
289    if (masters != null) {
290      for (JVMClusterUtil.MasterThread t : masters) {
291        t.join();
292        if (t.getMaster().isAborted()) {
293          closeAllRegionServerThreads(regionservers);
294          throw new RuntimeException("HMaster Aborted");
295        }
296      }
297    }
298  }
299
300  private static void
301    closeAllRegionServerThreads(List<JVMClusterUtil.RegionServerThread> regionservers) {
302    for (JVMClusterUtil.RegionServerThread t : regionservers) {
303      t.getRegionServer().stop("HMaster Aborted; Bringing down regions servers");
304    }
305  }
306
307  /*
308   * Version of master that will shutdown the passed zk cluster on its way out.
309   */
310  public static class LocalHMaster extends HMaster {
311    private MiniZooKeeperCluster zkcluster = null;
312
313    public LocalHMaster(Configuration conf)
314      throws IOException, KeeperException, InterruptedException {
315      super(conf);
316    }
317
318    @Override
319    public void run() {
320      super.run();
321      if (this.zkcluster != null) {
322        try {
323          this.zkcluster.shutdown();
324        } catch (IOException e) {
325          LOG.error("Failed to shutdown MiniZooKeeperCluster", e);
326        }
327      }
328    }
329
330    void setZKCluster(final MiniZooKeeperCluster zkcluster) {
331      this.zkcluster = zkcluster;
332    }
333  }
334}