View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.master;
20  
21  import java.io.File;
22  import java.io.IOException;
23  import java.util.List;
24  
25  import org.apache.commons.cli.CommandLine;
26  import org.apache.commons.cli.GnuParser;
27  import org.apache.commons.cli.Options;
28  import org.apache.commons.cli.ParseException;
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  import org.apache.hadoop.conf.Configuration;
32  import org.apache.hadoop.hbase.CoordinatedStateManager;
33  import org.apache.hadoop.hbase.CoordinatedStateManagerFactory;
34  import org.apache.hadoop.hbase.HConstants;
35  import org.apache.hadoop.hbase.LocalHBaseCluster;
36  import org.apache.hadoop.hbase.MasterNotRunningException;
37  import org.apache.hadoop.hbase.ZNodeClearer;
38  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
39  import org.apache.hadoop.hbase.classification.InterfaceAudience;
40  import org.apache.hadoop.hbase.client.Admin;
41  import org.apache.hadoop.hbase.client.Connection;
42  import org.apache.hadoop.hbase.client.ConnectionFactory;
43  import org.apache.hadoop.hbase.regionserver.HRegionServer;
44  import org.apache.hadoop.hbase.util.JVMClusterUtil;
45  import org.apache.hadoop.hbase.util.ServerCommandLine;
46  import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
47  import org.apache.hadoop.hbase.zookeeper.ZKUtil;
48  import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
49  import org.apache.zookeeper.KeeperException;
50  
51  @InterfaceAudience.Private
52  public class HMasterCommandLine extends ServerCommandLine {
53    private static final Log LOG = LogFactory.getLog(HMasterCommandLine.class);
54  
55    private static final String USAGE =
56      "Usage: Master [opts] start|stop|clear\n" +
57      " start  Start Master. If local mode, start Master and RegionServer in same JVM\n" +
58      " stop   Start cluster shutdown; Master signals RegionServer shutdown\n" +
59      " clear  Delete the master znode in ZooKeeper after a master crashes\n "+
60      " where [opts] are:\n" +
61      "   --minRegionServers=<servers>   Minimum RegionServers needed to host user tables.\n" +
62      "   --localRegionServers=<servers> " +
63        "RegionServers to start in master process when in standalone mode.\n" +
64      "   --masters=<servers>            Masters to start in this process.\n" +
65      "   --backup                       Master should start in backup mode";
66  
67    private final Class<? extends HMaster> masterClass;
68  
69    public HMasterCommandLine(Class<? extends HMaster> masterClass) {
70      this.masterClass = masterClass;
71    }
72  
73    protected String getUsage() {
74      return USAGE;
75    }
76  
77  
78    public int run(String args[]) throws Exception {
79      Options opt = new Options();
80      opt.addOption("localRegionServers", true,
81        "RegionServers to start in master process when running standalone");
82      opt.addOption("masters", true, "Masters to start in this process");
83      opt.addOption("minRegionServers", true, "Minimum RegionServers needed to host user tables");
84      opt.addOption("backup", false, "Do not try to become HMaster until the primary fails");
85  
86      CommandLine cmd;
87      try {
88        cmd = new GnuParser().parse(opt, args);
89      } catch (ParseException e) {
90        LOG.error("Could not parse: ", e);
91        usage(null);
92        return 1;
93      }
94  
95  
96      if (cmd.hasOption("minRegionServers")) {
97        String val = cmd.getOptionValue("minRegionServers");
98        getConf().setInt("hbase.regions.server.count.min",
99                    Integer.parseInt(val));
100       LOG.debug("minRegionServers set to " + val);
101     }
102 
103     // minRegionServers used to be minServers.  Support it too.
104     if (cmd.hasOption("minServers")) {
105       String val = cmd.getOptionValue("minServers");
106       getConf().setInt("hbase.regions.server.count.min",
107                   Integer.parseInt(val));
108       LOG.debug("minServers set to " + val);
109     }
110 
111     // check if we are the backup master - override the conf if so
112     if (cmd.hasOption("backup")) {
113       getConf().setBoolean(HConstants.MASTER_TYPE_BACKUP, true);
114     }
115 
116     // How many regionservers to startup in this process (we run regionservers in same process as
117     // master when we are in local/standalone mode. Useful testing)
118     if (cmd.hasOption("localRegionServers")) {
119       String val = cmd.getOptionValue("localRegionServers");
120       getConf().setInt("hbase.regionservers", Integer.parseInt(val));
121       LOG.debug("localRegionServers set to " + val);
122     }
123     // How many masters to startup inside this process; useful testing
124     if (cmd.hasOption("masters")) {
125       String val = cmd.getOptionValue("masters");
126       getConf().setInt("hbase.masters", Integer.parseInt(val));
127       LOG.debug("masters set to " + val);
128     }
129 
130     @SuppressWarnings("unchecked")
131     List<String> remainingArgs = cmd.getArgList();
132     if (remainingArgs.size() != 1) {
133       usage(null);
134       return 1;
135     }
136 
137     String command = remainingArgs.get(0);
138 
139     if ("start".equals(command)) {
140       return startMaster();
141     } else if ("stop".equals(command)) {
142       return stopMaster();
143     } else if ("clear".equals(command)) {
144       return (ZNodeClearer.clear(getConf()) ? 0 : 1);
145     } else {
146       usage("Invalid command: " + command);
147       return 1;
148     }
149   }
150 
151   private int startMaster() {
152     Configuration conf = getConf();
153     try {
154       // If 'local', defer to LocalHBaseCluster instance.  Starts master
155       // and regionserver both in the one JVM.
156       if (LocalHBaseCluster.isLocal(conf)) {
157         DefaultMetricsSystem.setMiniClusterMode(true);
158         final MiniZooKeeperCluster zooKeeperCluster = new MiniZooKeeperCluster(conf);
159         File zkDataPath = new File(conf.get(HConstants.ZOOKEEPER_DATA_DIR));
160 
161         // find out the default client port
162         int zkClientPort = 0;
163 
164         // If the zookeeper client port is specified in server quorum, use it.
165         String zkserver = conf.get(HConstants.ZOOKEEPER_QUORUM);
166         if (zkserver != null) {
167           String[] zkservers = zkserver.split(",");
168 
169           if (zkservers.length > 1) {
170             // In local mode deployment, we have the master + a region server and zookeeper server
171             // started in the same process. Therefore, we only support one zookeeper server.
172             String errorMsg = "Could not start ZK with " + zkservers.length +
173                 " ZK servers in local mode deployment. Aborting as clients (e.g. shell) will not "
174                 + "be able to find this ZK quorum.";
175               System.err.println(errorMsg);
176               throw new IOException(errorMsg);
177           }
178 
179           String[] parts = zkservers[0].split(":");
180 
181           if (parts.length == 2) {
182             // the second part is the client port
183             zkClientPort = Integer.parseInt(parts [1]);
184           }
185         }
186         // If the client port could not be find in server quorum conf, try another conf
187         if (zkClientPort == 0) {
188           zkClientPort = conf.getInt(HConstants.ZOOKEEPER_CLIENT_PORT, 0);
189           // The client port has to be set by now; if not, throw exception.
190           if (zkClientPort == 0) {
191             throw new IOException("No config value for " + HConstants.ZOOKEEPER_CLIENT_PORT);
192           }
193         }
194         zooKeeperCluster.setDefaultClientPort(zkClientPort);
195         // set the ZK tick time if specified
196         int zkTickTime = conf.getInt(HConstants.ZOOKEEPER_TICK_TIME, 0);
197         if (zkTickTime > 0) {
198           zooKeeperCluster.setTickTime(zkTickTime);
199         }
200 
201         // login the zookeeper server principal (if using security)
202         ZKUtil.loginServer(conf, "hbase.zookeeper.server.keytab.file",
203           "hbase.zookeeper.server.kerberos.principal", null);
204         int localZKClusterSessionTimeout =
205           conf.getInt(HConstants.ZK_SESSION_TIMEOUT + ".localHBaseCluster", 10*1000);
206         conf.setInt(HConstants.ZK_SESSION_TIMEOUT, localZKClusterSessionTimeout);
207         LOG.info("Starting a zookeeper cluster");
208         int clientPort = zooKeeperCluster.startup(zkDataPath);
209         if (clientPort != zkClientPort) {
210           String errorMsg = "Could not start ZK at requested port of " +
211             zkClientPort + ".  ZK was started at port: " + clientPort +
212             ".  Aborting as clients (e.g. shell) will not be able to find " +
213             "this ZK quorum.";
214           System.err.println(errorMsg);
215           throw new IOException(errorMsg);
216         }
217         conf.set(HConstants.ZOOKEEPER_CLIENT_PORT, Integer.toString(clientPort));
218 
219         // Need to have the zk cluster shutdown when master is shutdown.
220         // Run a subclass that does the zk cluster shutdown on its way out.
221         int mastersCount = conf.getInt("hbase.masters", 1);
222         int regionServersCount = conf.getInt("hbase.regionservers", 1);
223         LOG.info("Starting up instance of localHBaseCluster; master=" + mastersCount +
224           ", regionserversCount=" + regionServersCount);
225         LocalHBaseCluster cluster = new LocalHBaseCluster(conf, mastersCount, regionServersCount,
226           LocalHMaster.class, HRegionServer.class);
227         ((LocalHMaster)cluster.getMaster(0)).setZKCluster(zooKeeperCluster);
228         cluster.startup();
229         waitOnMasterThreads(cluster);
230       } else {
231         logProcessInfo(getConf());
232         CoordinatedStateManager csm =
233           CoordinatedStateManagerFactory.getCoordinatedStateManager(conf);
234         HMaster master = HMaster.constructMaster(masterClass, conf, csm);
235         if (master.isStopped()) {
236           LOG.info("Won't bring the Master up as a shutdown is requested");
237           return 1;
238         }
239         master.start();
240         master.join();
241         if(master.isAborted())
242           throw new RuntimeException("HMaster Aborted");
243       }
244     } catch (Throwable t) {
245       LOG.error("Master exiting", t);
246       return 1;
247     }
248     return 0;
249   }
250 
251   @SuppressWarnings("resource")
252   private int stopMaster() {
253     Configuration conf = getConf();
254     // Don't try more than once
255     conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
256     try (Connection connection = ConnectionFactory.createConnection(conf)) {
257       try (Admin admin = connection.getAdmin()) {
258         connection.getAdmin().shutdown();
259       } catch (Throwable t) {
260         LOG.error("Failed to stop master", t);
261         return 1;
262       }
263     } catch (MasterNotRunningException e) {
264       LOG.error("Master not running");
265       return 1;
266     } catch (ZooKeeperConnectionException e) {
267       LOG.error("ZooKeeper not available");
268       return 1;
269     } catch (IOException e) {
270       LOG.error("Got IOException: " +e.getMessage(), e);
271       return 1;
272     }
273     return 0;
274   }
275 
276   private void waitOnMasterThreads(LocalHBaseCluster cluster) throws InterruptedException{
277     List<JVMClusterUtil.MasterThread> masters = cluster.getMasters();
278     List<JVMClusterUtil.RegionServerThread> regionservers = cluster.getRegionServers();
279 
280     if (masters != null) {
281       for (JVMClusterUtil.MasterThread t : masters) {
282         t.join();
283         if(t.getMaster().isAborted()) {
284           closeAllRegionServerThreads(regionservers);
285           throw new RuntimeException("HMaster Aborted");
286         }
287       }
288     }
289   }
290 
291   private static void closeAllRegionServerThreads(
292       List<JVMClusterUtil.RegionServerThread> regionservers) {
293     for(JVMClusterUtil.RegionServerThread t : regionservers){
294       t.getRegionServer().stop("HMaster Aborted; Bringing down regions servers");
295     }
296   }
297 
298   /*
299    * Version of master that will shutdown the passed zk cluster on its way out.
300    */
301   public static class LocalHMaster extends HMaster {
302     private MiniZooKeeperCluster zkcluster = null;
303 
304     public LocalHMaster(Configuration conf, CoordinatedStateManager csm)
305     throws IOException, KeeperException, InterruptedException {
306       super(conf, csm);
307     }
308 
309     @Override
310     public void run() {
311       super.run();
312       if (this.zkcluster != null) {
313         try {
314           this.zkcluster.shutdown();
315         } catch (IOException e) {
316           e.printStackTrace();
317         }
318       }
319     }
320 
321     void setZKCluster(final MiniZooKeeperCluster zkcluster) {
322       this.zkcluster = zkcluster;
323     }
324   }
325 }