001/** 002 * 003 * Licensed to the Apache Software Foundation (ASF) under one 004 * or more contributor license agreements. See the NOTICE file 005 * distributed with this work for additional information 006 * regarding copyright ownership. The ASF licenses this file 007 * to you under the Apache License, Version 2.0 (the 008 * "License"); you may not use this file except in compliance 009 * with the License. You may obtain a copy of the License at 010 * 011 * http://www.apache.org/licenses/LICENSE-2.0 012 * 013 * Unless required by applicable law or agreed to in writing, software 014 * distributed under the License is distributed on an "AS IS" BASIS, 015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 016 * See the License for the specific language governing permissions and 017 * limitations under the License. 018 */ 019package org.apache.hadoop.hbase.master; 020 021import java.io.File; 022import java.io.IOException; 023import java.util.List; 024 025import org.apache.hadoop.conf.Configuration; 026import org.apache.hadoop.hbase.HConstants; 027import org.apache.hadoop.hbase.LocalHBaseCluster; 028import org.apache.hadoop.hbase.MasterNotRunningException; 029import org.apache.hadoop.hbase.ZNodeClearer; 030import org.apache.hadoop.hbase.ZooKeeperConnectionException; 031import org.apache.hadoop.hbase.trace.TraceUtil; 032import org.apache.yetus.audience.InterfaceAudience; 033import org.apache.hadoop.hbase.client.Admin; 034import org.apache.hadoop.hbase.client.Connection; 035import org.apache.hadoop.hbase.client.ConnectionFactory; 036import org.apache.hadoop.hbase.regionserver.HRegionServer; 037import org.apache.hadoop.hbase.util.JVMClusterUtil; 038import org.apache.hadoop.hbase.util.ServerCommandLine; 039import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster; 040import org.apache.hadoop.hbase.zookeeper.ZKUtil; 041import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; 042import org.apache.zookeeper.KeeperException; 043import org.slf4j.Logger; 044import org.slf4j.LoggerFactory; 045 046import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine; 047import org.apache.hbase.thirdparty.org.apache.commons.cli.GnuParser; 048import org.apache.hbase.thirdparty.org.apache.commons.cli.Options; 049import org.apache.hbase.thirdparty.org.apache.commons.cli.ParseException; 050 051@InterfaceAudience.Private 052public class HMasterCommandLine extends ServerCommandLine { 053 private static final Logger LOG = LoggerFactory.getLogger(HMasterCommandLine.class); 054 055 private static final String USAGE = 056 "Usage: Master [opts] start|stop|clear\n" + 057 " start Start Master. If local mode, start Master and RegionServer in same JVM\n" + 058 " stop Start cluster shutdown; Master signals RegionServer shutdown\n" + 059 " clear Delete the master znode in ZooKeeper after a master crashes\n "+ 060 " where [opts] are:\n" + 061 " --minRegionServers=<servers> Minimum RegionServers needed to host user tables.\n" + 062 " --localRegionServers=<servers> " + 063 "RegionServers to start in master process when in standalone mode.\n" + 064 " --masters=<servers> Masters to start in this process.\n" + 065 " --backup Master should start in backup mode\n" + 066 " --shutDownCluster " + 067 "Start Cluster shutdown; Master signals RegionServer shutdown"; 068 069 private final Class<? extends HMaster> masterClass; 070 071 public HMasterCommandLine(Class<? extends HMaster> masterClass) { 072 this.masterClass = masterClass; 073 } 074 075 @Override 076 protected String getUsage() { 077 return USAGE; 078 } 079 080 @Override 081 public int run(String args[]) throws Exception { 082 boolean shutDownCluster = false; 083 Options opt = new Options(); 084 opt.addOption("localRegionServers", true, 085 "RegionServers to start in master process when running standalone"); 086 opt.addOption("masters", true, "Masters to start in this process"); 087 opt.addOption("minRegionServers", true, "Minimum RegionServers needed to host user tables"); 088 opt.addOption("backup", false, "Do not try to become HMaster until the primary fails"); 089 opt.addOption("shutDownCluster", false, "`hbase master stop --shutDownCluster` shuts down cluster"); 090 091 CommandLine cmd; 092 try { 093 cmd = new GnuParser().parse(opt, args); 094 } catch (ParseException e) { 095 LOG.error("Could not parse: ", e); 096 usage(null); 097 return 1; 098 } 099 100 101 if (cmd.hasOption("minRegionServers")) { 102 String val = cmd.getOptionValue("minRegionServers"); 103 getConf().setInt("hbase.regions.server.count.min", 104 Integer.parseInt(val)); 105 LOG.debug("minRegionServers set to " + val); 106 } 107 108 // minRegionServers used to be minServers. Support it too. 109 if (cmd.hasOption("minServers")) { 110 String val = cmd.getOptionValue("minServers"); 111 getConf().setInt("hbase.regions.server.count.min", Integer.parseInt(val)); 112 LOG.debug("minServers set to " + val); 113 } 114 115 // check if we are the backup master - override the conf if so 116 if (cmd.hasOption("backup")) { 117 getConf().setBoolean(HConstants.MASTER_TYPE_BACKUP, true); 118 } 119 120 // How many regionservers to startup in this process (we run regionservers in same process as 121 // master when we are in local/standalone mode. Useful testing) 122 if (cmd.hasOption("localRegionServers")) { 123 String val = cmd.getOptionValue("localRegionServers"); 124 getConf().setInt("hbase.regionservers", Integer.parseInt(val)); 125 LOG.debug("localRegionServers set to " + val); 126 } 127 // How many masters to startup inside this process; useful testing 128 if (cmd.hasOption("masters")) { 129 String val = cmd.getOptionValue("masters"); 130 getConf().setInt("hbase.masters", Integer.parseInt(val)); 131 LOG.debug("masters set to " + val); 132 } 133 134 // Checking whether to shut down cluster or not 135 if (cmd.hasOption("shutDownCluster")) { 136 shutDownCluster = true; 137 } 138 139 @SuppressWarnings("unchecked") 140 List<String> remainingArgs = cmd.getArgList(); 141 if (remainingArgs.size() != 1) { 142 usage(null); 143 return 1; 144 } 145 146 String command = remainingArgs.get(0); 147 148 if ("start".equals(command)) { 149 return startMaster(); 150 } else if ("stop".equals(command)) { 151 if (shutDownCluster) { 152 return stopMaster(); 153 } 154 System.err.println( 155 "To shutdown the master run " + 156 "hbase-daemon.sh stop master or send a kill signal to " + 157 "the HMaster pid, " + 158 "and to stop HBase Cluster run \"stop-hbase.sh\" or \"hbase master stop --shutDownCluster\""); 159 return 1; 160 } else if ("clear".equals(command)) { 161 return (ZNodeClearer.clear(getConf()) ? 0 : 1); 162 } else { 163 usage("Invalid command: " + command); 164 return 1; 165 } 166 } 167 168 private int startMaster() { 169 Configuration conf = getConf(); 170 TraceUtil.initTracer(conf); 171 172 try { 173 // If 'local', defer to LocalHBaseCluster instance. Starts master 174 // and regionserver both in the one JVM. 175 if (LocalHBaseCluster.isLocal(conf)) { 176 DefaultMetricsSystem.setMiniClusterMode(true); 177 final MiniZooKeeperCluster zooKeeperCluster = new MiniZooKeeperCluster(conf); 178 File zkDataPath = new File(conf.get(HConstants.ZOOKEEPER_DATA_DIR)); 179 180 // find out the default client port 181 int zkClientPort = 0; 182 183 // If the zookeeper client port is specified in server quorum, use it. 184 String zkserver = conf.get(HConstants.ZOOKEEPER_QUORUM); 185 if (zkserver != null) { 186 String[] zkservers = zkserver.split(","); 187 188 if (zkservers.length > 1) { 189 // In local mode deployment, we have the master + a region server and zookeeper server 190 // started in the same process. Therefore, we only support one zookeeper server. 191 String errorMsg = "Could not start ZK with " + zkservers.length + 192 " ZK servers in local mode deployment. Aborting as clients (e.g. shell) will not " 193 + "be able to find this ZK quorum."; 194 System.err.println(errorMsg); 195 throw new IOException(errorMsg); 196 } 197 198 String[] parts = zkservers[0].split(":"); 199 200 if (parts.length == 2) { 201 // the second part is the client port 202 zkClientPort = Integer.parseInt(parts [1]); 203 } 204 } 205 // If the client port could not be find in server quorum conf, try another conf 206 if (zkClientPort == 0) { 207 zkClientPort = conf.getInt(HConstants.ZOOKEEPER_CLIENT_PORT, 0); 208 // The client port has to be set by now; if not, throw exception. 209 if (zkClientPort == 0) { 210 throw new IOException("No config value for " + HConstants.ZOOKEEPER_CLIENT_PORT); 211 } 212 } 213 zooKeeperCluster.setDefaultClientPort(zkClientPort); 214 // set the ZK tick time if specified 215 int zkTickTime = conf.getInt(HConstants.ZOOKEEPER_TICK_TIME, 0); 216 if (zkTickTime > 0) { 217 zooKeeperCluster.setTickTime(zkTickTime); 218 } 219 220 // login the zookeeper server principal (if using security) 221 ZKUtil.loginServer(conf, HConstants.ZK_SERVER_KEYTAB_FILE, 222 HConstants.ZK_SERVER_KERBEROS_PRINCIPAL, null); 223 int localZKClusterSessionTimeout = 224 conf.getInt(HConstants.ZK_SESSION_TIMEOUT + ".localHBaseCluster", 10*1000); 225 conf.setInt(HConstants.ZK_SESSION_TIMEOUT, localZKClusterSessionTimeout); 226 LOG.info("Starting a zookeeper cluster"); 227 int clientPort = zooKeeperCluster.startup(zkDataPath); 228 if (clientPort != zkClientPort) { 229 String errorMsg = "Could not start ZK at requested port of " + 230 zkClientPort + ". ZK was started at port: " + clientPort + 231 ". Aborting as clients (e.g. shell) will not be able to find " + 232 "this ZK quorum."; 233 System.err.println(errorMsg); 234 throw new IOException(errorMsg); 235 } 236 conf.set(HConstants.ZOOKEEPER_CLIENT_PORT, Integer.toString(clientPort)); 237 238 // Need to have the zk cluster shutdown when master is shutdown. 239 // Run a subclass that does the zk cluster shutdown on its way out. 240 int mastersCount = conf.getInt("hbase.masters", 1); 241 int regionServersCount = conf.getInt("hbase.regionservers", 1); 242 // Set start timeout to 5 minutes for cmd line start operations 243 conf.setIfUnset("hbase.master.start.timeout.localHBaseCluster", "300000"); 244 LOG.info("Starting up instance of localHBaseCluster; master=" + mastersCount + 245 ", regionserversCount=" + regionServersCount); 246 LocalHBaseCluster cluster = new LocalHBaseCluster(conf, mastersCount, regionServersCount, 247 LocalHMaster.class, HRegionServer.class); 248 ((LocalHMaster)cluster.getMaster(0)).setZKCluster(zooKeeperCluster); 249 cluster.startup(); 250 waitOnMasterThreads(cluster); 251 } else { 252 logProcessInfo(getConf()); 253 HMaster master = HMaster.constructMaster(masterClass, conf); 254 if (master.isStopped()) { 255 LOG.info("Won't bring the Master up as a shutdown is requested"); 256 return 1; 257 } 258 master.start(); 259 master.join(); 260 if(master.isAborted()) 261 throw new RuntimeException("HMaster Aborted"); 262 } 263 } catch (Throwable t) { 264 LOG.error("Master exiting", t); 265 return 1; 266 } 267 return 0; 268 } 269 270 @SuppressWarnings("resource") 271 private int stopMaster() { 272 Configuration conf = getConf(); 273 // Don't try more than once 274 conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 0); 275 try (Connection connection = ConnectionFactory.createConnection(conf)) { 276 try (Admin admin = connection.getAdmin()) { 277 admin.shutdown(); 278 } catch (Throwable t) { 279 LOG.error("Failed to stop master", t); 280 return 1; 281 } 282 } catch (MasterNotRunningException e) { 283 LOG.error("Master not running"); 284 return 1; 285 } catch (ZooKeeperConnectionException e) { 286 LOG.error("ZooKeeper not available"); 287 return 1; 288 } catch (IOException e) { 289 LOG.error("Got IOException: " +e.getMessage(), e); 290 return 1; 291 } 292 return 0; 293 } 294 295 private void waitOnMasterThreads(LocalHBaseCluster cluster) throws InterruptedException{ 296 List<JVMClusterUtil.MasterThread> masters = cluster.getMasters(); 297 List<JVMClusterUtil.RegionServerThread> regionservers = cluster.getRegionServers(); 298 299 if (masters != null) { 300 for (JVMClusterUtil.MasterThread t : masters) { 301 t.join(); 302 if(t.getMaster().isAborted()) { 303 closeAllRegionServerThreads(regionservers); 304 throw new RuntimeException("HMaster Aborted"); 305 } 306 } 307 } 308 } 309 310 private static void closeAllRegionServerThreads( 311 List<JVMClusterUtil.RegionServerThread> regionservers) { 312 for(JVMClusterUtil.RegionServerThread t : regionservers){ 313 t.getRegionServer().stop("HMaster Aborted; Bringing down regions servers"); 314 } 315 } 316 317 /* 318 * Version of master that will shutdown the passed zk cluster on its way out. 319 */ 320 public static class LocalHMaster extends HMaster { 321 private MiniZooKeeperCluster zkcluster = null; 322 323 public LocalHMaster(Configuration conf) 324 throws IOException, KeeperException, InterruptedException { 325 super(conf); 326 } 327 328 @Override 329 public void run() { 330 super.run(); 331 if (this.zkcluster != null) { 332 try { 333 this.zkcluster.shutdown(); 334 } catch (IOException e) { 335 e.printStackTrace(); 336 } 337 } 338 } 339 340 void setZKCluster(final MiniZooKeeperCluster zkcluster) { 341 this.zkcluster = zkcluster; 342 } 343 } 344}