001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.master; 019 020import java.io.File; 021import java.io.IOException; 022import java.util.List; 023import org.apache.hadoop.conf.Configuration; 024import org.apache.hadoop.hbase.HConstants; 025import org.apache.hadoop.hbase.LocalHBaseCluster; 026import org.apache.hadoop.hbase.MasterNotRunningException; 027import org.apache.hadoop.hbase.ZNodeClearer; 028import org.apache.hadoop.hbase.ZooKeeperConnectionException; 029import org.apache.hadoop.hbase.client.Admin; 030import org.apache.hadoop.hbase.client.Connection; 031import org.apache.hadoop.hbase.client.ConnectionFactory; 032import org.apache.hadoop.hbase.regionserver.HRegionServer; 033import org.apache.hadoop.hbase.util.JVMClusterUtil; 034import org.apache.hadoop.hbase.util.ServerCommandLine; 035import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster; 036import org.apache.hadoop.hbase.zookeeper.ZKAuthentication; 037import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; 038import org.apache.yetus.audience.InterfaceAudience; 039import org.apache.zookeeper.KeeperException; 040import org.slf4j.Logger; 041import org.slf4j.LoggerFactory; 042 043import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine; 044import org.apache.hbase.thirdparty.org.apache.commons.cli.GnuParser; 045import org.apache.hbase.thirdparty.org.apache.commons.cli.Options; 046import org.apache.hbase.thirdparty.org.apache.commons.cli.ParseException; 047 048@InterfaceAudience.Private 049public class HMasterCommandLine extends ServerCommandLine { 050 private static final Logger LOG = LoggerFactory.getLogger(HMasterCommandLine.class); 051 052 private static final String USAGE = "Usage: Master [opts] start|stop|clear\n" 053 + " start Start Master. If local mode, start Master and RegionServer in same JVM\n" 054 + " stop Start cluster shutdown; Master signals RegionServer shutdown\n" 055 + " clear Delete the master znode in ZooKeeper after a master crashes\n " 056 + " where [opts] are:\n" 057 + " --minRegionServers=<servers> Minimum RegionServers needed to host user tables.\n" 058 + " --localRegionServers=<servers> " 059 + "RegionServers to start in master process when in standalone mode.\n" 060 + " --masters=<servers> Masters to start in this process.\n" 061 + " --backup Master should start in backup mode\n" 062 + " --shutDownCluster " 063 + "Start Cluster shutdown; Master signals RegionServer shutdown"; 064 065 private final Class<? extends HMaster> masterClass; 066 067 public HMasterCommandLine(Class<? extends HMaster> masterClass) { 068 this.masterClass = masterClass; 069 } 070 071 @Override 072 protected String getUsage() { 073 return USAGE; 074 } 075 076 @Override 077 public int run(String args[]) throws Exception { 078 boolean shutDownCluster = false; 079 Options opt = new Options(); 080 opt.addOption("localRegionServers", true, 081 "RegionServers to start in master process when running standalone"); 082 opt.addOption("masters", true, "Masters to start in this process"); 083 opt.addOption("minRegionServers", true, "Minimum RegionServers needed to host user tables"); 084 opt.addOption("backup", false, "Do not try to become HMaster until the primary fails"); 085 opt.addOption("shutDownCluster", false, 086 "`hbase master stop --shutDownCluster` shuts down cluster"); 087 088 CommandLine cmd; 089 try { 090 cmd = new GnuParser().parse(opt, args); 091 } catch (ParseException e) { 092 LOG.error("Could not parse: ", e); 093 usage(null); 094 return 1; 095 } 096 097 if (cmd.hasOption("minRegionServers")) { 098 String val = cmd.getOptionValue("minRegionServers"); 099 getConf().setInt("hbase.regions.server.count.min", Integer.parseInt(val)); 100 LOG.debug("minRegionServers set to " + val); 101 } 102 103 // minRegionServers used to be minServers. Support it too. 104 if (cmd.hasOption("minServers")) { 105 String val = cmd.getOptionValue("minServers"); 106 getConf().setInt("hbase.regions.server.count.min", Integer.parseInt(val)); 107 LOG.debug("minServers set to " + val); 108 } 109 110 // check if we are the backup master - override the conf if so 111 if (cmd.hasOption("backup")) { 112 getConf().setBoolean(HConstants.MASTER_TYPE_BACKUP, true); 113 } 114 115 // How many regionservers to startup in this process (we run regionservers in same process as 116 // master when we are in local/standalone mode. Useful testing) 117 if (cmd.hasOption("localRegionServers")) { 118 String val = cmd.getOptionValue("localRegionServers"); 119 getConf().setInt("hbase.regionservers", Integer.parseInt(val)); 120 LOG.debug("localRegionServers set to " + val); 121 } 122 // How many masters to startup inside this process; useful testing 123 if (cmd.hasOption("masters")) { 124 String val = cmd.getOptionValue("masters"); 125 getConf().setInt("hbase.masters", Integer.parseInt(val)); 126 LOG.debug("masters set to " + val); 127 } 128 129 // Checking whether to shut down cluster or not 130 if (cmd.hasOption("shutDownCluster")) { 131 shutDownCluster = true; 132 } 133 134 @SuppressWarnings("unchecked") 135 List<String> remainingArgs = cmd.getArgList(); 136 if (remainingArgs.size() != 1) { 137 usage(null); 138 return 1; 139 } 140 141 String command = remainingArgs.get(0); 142 143 if ("start".equals(command)) { 144 return startMaster(); 145 } else if ("stop".equals(command)) { 146 if (shutDownCluster) { 147 return stopMaster(); 148 } 149 System.err.println("To shutdown the master run " 150 + "hbase-daemon.sh stop master or send a kill signal to the HMaster pid, " 151 + "and to stop HBase Cluster run \"stop-hbase.sh\" or \"hbase master " 152 + "stop --shutDownCluster\""); 153 return 1; 154 } else if ("clear".equals(command)) { 155 return (ZNodeClearer.clear(getConf()) ? 0 : 1); 156 } else { 157 usage("Invalid command: " + command); 158 return 1; 159 } 160 } 161 162 private int startMaster() { 163 Configuration conf = getConf(); 164 try { 165 // If 'local', defer to LocalHBaseCluster instance. Starts master 166 // and regionserver both in the one JVM. 167 if (LocalHBaseCluster.isLocal(conf)) { 168 DefaultMetricsSystem.setMiniClusterMode(true); 169 final MiniZooKeeperCluster zooKeeperCluster = new MiniZooKeeperCluster(conf); 170 File zkDataPath = new File(conf.get(HConstants.ZOOKEEPER_DATA_DIR)); 171 172 // find out the default client port 173 int zkClientPort = 0; 174 175 // If the zookeeper client port is specified in server quorum, use it. 176 String zkserver = conf.get(HConstants.ZOOKEEPER_QUORUM); 177 if (zkserver != null) { 178 String[] zkservers = zkserver.split(","); 179 180 if (zkservers.length > 1) { 181 // In local mode deployment, we have the master + a region server and zookeeper server 182 // started in the same process. Therefore, we only support one zookeeper server. 183 String errorMsg = "Could not start ZK with " + zkservers.length 184 + " ZK servers in local mode deployment. Aborting as clients (e.g. shell) will not " 185 + "be able to find this ZK quorum."; 186 System.err.println(errorMsg); 187 throw new IOException(errorMsg); 188 } 189 190 String[] parts = zkservers[0].split(":"); 191 192 if (parts.length == 2) { 193 // the second part is the client port 194 zkClientPort = Integer.parseInt(parts[1]); 195 } 196 } 197 // If the client port could not be find in server quorum conf, try another conf 198 if (zkClientPort == 0) { 199 zkClientPort = conf.getInt(HConstants.ZOOKEEPER_CLIENT_PORT, 0); 200 // The client port has to be set by now; if not, throw exception. 201 if (zkClientPort == 0) { 202 throw new IOException("No config value for " + HConstants.ZOOKEEPER_CLIENT_PORT); 203 } 204 } 205 zooKeeperCluster.setDefaultClientPort(zkClientPort); 206 // set the ZK tick time if specified 207 int zkTickTime = conf.getInt(HConstants.ZOOKEEPER_TICK_TIME, 0); 208 if (zkTickTime > 0) { 209 zooKeeperCluster.setTickTime(zkTickTime); 210 } 211 212 // login the zookeeper server principal (if using security) 213 ZKAuthentication.loginServer(conf, HConstants.ZK_SERVER_KEYTAB_FILE, 214 HConstants.ZK_SERVER_KERBEROS_PRINCIPAL, null); 215 int localZKClusterSessionTimeout = 216 conf.getInt(HConstants.ZK_SESSION_TIMEOUT + ".localHBaseCluster", 10 * 1000); 217 conf.setInt(HConstants.ZK_SESSION_TIMEOUT, localZKClusterSessionTimeout); 218 LOG.info("Starting a zookeeper cluster"); 219 int clientPort = zooKeeperCluster.startup(zkDataPath); 220 if (clientPort != zkClientPort) { 221 String errorMsg = "Could not start ZK at requested port of " + zkClientPort 222 + ". ZK was started at port: " + clientPort 223 + ". Aborting as clients (e.g. shell) will not be able to find " + "this ZK quorum."; 224 System.err.println(errorMsg); 225 throw new IOException(errorMsg); 226 } 227 conf.set(HConstants.ZOOKEEPER_CLIENT_PORT, Integer.toString(clientPort)); 228 229 // Need to have the zk cluster shutdown when master is shutdown. 230 // Run a subclass that does the zk cluster shutdown on its way out. 231 int mastersCount = conf.getInt("hbase.masters", 1); 232 int regionServersCount = conf.getInt("hbase.regionservers", 1); 233 // Set start timeout to 5 minutes for cmd line start operations 234 conf.setIfUnset("hbase.master.start.timeout.localHBaseCluster", "300000"); 235 LOG.info("Starting up instance of localHBaseCluster; master=" + mastersCount 236 + ", regionserversCount=" + regionServersCount); 237 LocalHBaseCluster cluster = new LocalHBaseCluster(conf, mastersCount, regionServersCount, 238 LocalHMaster.class, HRegionServer.class); 239 ((LocalHMaster) cluster.getMaster(0)).setZKCluster(zooKeeperCluster); 240 cluster.startup(); 241 waitOnMasterThreads(cluster); 242 } else { 243 logProcessInfo(getConf()); 244 HMaster master = HMaster.constructMaster(masterClass, conf); 245 if (master.isStopped()) { 246 LOG.info("Won't bring the Master up as a shutdown is requested"); 247 return 1; 248 } 249 master.start(); 250 master.join(); 251 if (master.isAborted()) throw new RuntimeException("HMaster Aborted"); 252 } 253 } catch (Throwable t) { 254 LOG.error("Master exiting", t); 255 return 1; 256 } 257 return 0; 258 } 259 260 @SuppressWarnings("resource") 261 private int stopMaster() { 262 Configuration conf = getConf(); 263 // Don't try more than once 264 conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 0); 265 try (Connection connection = ConnectionFactory.createConnection(conf)) { 266 try (Admin admin = connection.getAdmin()) { 267 admin.shutdown(); 268 } catch (Throwable t) { 269 LOG.error("Failed to stop master", t); 270 return 1; 271 } 272 } catch (MasterNotRunningException e) { 273 LOG.error("Master not running"); 274 return 1; 275 } catch (ZooKeeperConnectionException e) { 276 LOG.error("ZooKeeper not available"); 277 return 1; 278 } catch (IOException e) { 279 LOG.error("Got IOException: " + e.getMessage(), e); 280 return 1; 281 } 282 return 0; 283 } 284 285 private void waitOnMasterThreads(LocalHBaseCluster cluster) throws InterruptedException { 286 List<JVMClusterUtil.MasterThread> masters = cluster.getMasters(); 287 List<JVMClusterUtil.RegionServerThread> regionservers = cluster.getRegionServers(); 288 289 if (masters != null) { 290 for (JVMClusterUtil.MasterThread t : masters) { 291 t.join(); 292 if (t.getMaster().isAborted()) { 293 closeAllRegionServerThreads(regionservers); 294 throw new RuntimeException("HMaster Aborted"); 295 } 296 } 297 } 298 } 299 300 private static void 301 closeAllRegionServerThreads(List<JVMClusterUtil.RegionServerThread> regionservers) { 302 for (JVMClusterUtil.RegionServerThread t : regionservers) { 303 t.getRegionServer().stop("HMaster Aborted; Bringing down regions servers"); 304 } 305 } 306 307 /* 308 * Version of master that will shutdown the passed zk cluster on its way out. 309 */ 310 public static class LocalHMaster extends HMaster { 311 private MiniZooKeeperCluster zkcluster = null; 312 313 public LocalHMaster(Configuration conf) 314 throws IOException, KeeperException, InterruptedException { 315 super(conf); 316 } 317 318 @Override 319 public void run() { 320 super.run(); 321 if (this.zkcluster != null) { 322 try { 323 this.zkcluster.shutdown(); 324 } catch (IOException e) { 325 LOG.error("Failed to shutdown MiniZooKeeperCluster", e); 326 } 327 } 328 } 329 330 void setZKCluster(final MiniZooKeeperCluster zkcluster) { 331 this.zkcluster = zkcluster; 332 } 333 } 334}