View Javadoc

1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.zookeeper;
21  
22  import java.io.BufferedReader;
23  import java.io.IOException;
24  import java.io.InputStreamReader;
25  import java.io.PrintWriter;
26  import java.net.InetSocketAddress;
27  import java.net.InetAddress;
28  import java.net.Socket;
29  import java.util.ArrayList;
30  import java.util.Arrays;
31  import java.util.LinkedList;
32  import java.util.List;
33  import java.util.Properties;
34  import java.util.HashMap;
35  import java.util.Map;
36  
37  import javax.security.auth.login.LoginException;
38  import javax.security.auth.login.AppConfigurationEntry;
39  import javax.security.auth.login.AppConfigurationEntry.LoginModuleControlFlag;
40  import org.apache.hadoop.security.SecurityUtil;
41  import org.apache.hadoop.security.authentication.util.KerberosUtil;
42  
43  import org.apache.commons.lang.StringUtils;
44  import org.apache.commons.logging.Log;
45  import org.apache.commons.logging.LogFactory;
46  import org.apache.hadoop.conf.Configuration;
47  import org.apache.hadoop.hbase.EmptyWatcher;
48  import org.apache.hadoop.hbase.HConstants;
49  import org.apache.hadoop.hbase.ServerName;
50  import org.apache.hadoop.hbase.executor.RegionTransitionData;
51  import org.apache.hadoop.hbase.util.Bytes;
52  import org.apache.hadoop.hbase.util.Threads;
53  import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp.CreateAndFailSilent;
54  import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp.DeleteNodeFailSilent;
55  import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp.SetData;
56  import org.apache.zookeeper.AsyncCallback;
57  import org.apache.zookeeper.CreateMode;
58  import org.apache.zookeeper.KeeperException;
59  import org.apache.zookeeper.KeeperException.NoNodeException;
60  import org.apache.zookeeper.Op;
61  import org.apache.zookeeper.Watcher;
62  import org.apache.zookeeper.ZooDefs.Ids;
63  import org.apache.zookeeper.ZooKeeper;
64  import org.apache.zookeeper.client.ZooKeeperSaslClient;
65  import org.apache.zookeeper.data.ACL;
66  import org.apache.zookeeper.data.Stat;
67  import org.apache.zookeeper.server.ZooKeeperSaslServer;
68  import org.apache.zookeeper.proto.CreateRequest;
69  import org.apache.zookeeper.proto.DeleteRequest;
70  import org.apache.zookeeper.proto.SetDataRequest;
71  
72  /**
73   * Internal HBase utility class for ZooKeeper.
74   *
75   * <p>Contains only static methods and constants.
76   *
77   * <p>Methods all throw {@link KeeperException} if there is an unexpected
78   * zookeeper exception, so callers of these methods must handle appropriately.
79   * If ZK is required for the operation, the server will need to be aborted.
80   */
81  public class ZKUtil {
82    private static final Log LOG = LogFactory.getLog(ZKUtil.class);
83  
84    // TODO: Replace this with ZooKeeper constant when ZOOKEEPER-277 is resolved.
85    public static final char ZNODE_PATH_SEPARATOR = '/';
86    private static int zkDumpConnectionTimeOut;
87  
88    /**
89     * Creates a new connection to ZooKeeper, pulling settings and ensemble config
90     * from the specified configuration object using methods from {@link ZKConfig}.
91     *
92     * Sets the connection status monitoring watcher to the specified watcher.
93     *
94     * @param conf configuration to pull ensemble and other settings from
95     * @param watcher watcher to monitor connection changes
96     * @return connection to zookeeper
97     * @throws IOException if unable to connect to zk or config problem
98     */
99    public static RecoverableZooKeeper connect(Configuration conf, Watcher watcher)
100   throws IOException {
101     Properties properties = ZKConfig.makeZKProps(conf);
102     String ensemble = ZKConfig.getZKQuorumServersString(properties);
103     return connect(conf, ensemble, watcher);
104   }
105 
106   public static RecoverableZooKeeper connect(Configuration conf, String ensemble,
107       Watcher watcher)
108   throws IOException {
109     return connect(conf, ensemble, watcher, "");
110   }
111 
112   public static RecoverableZooKeeper connect(Configuration conf, String ensemble,
113       Watcher watcher, final String descriptor)
114   throws IOException {
115     if(ensemble == null) {
116       throw new IOException("Unable to determine ZooKeeper ensemble");
117     }
118     int timeout = conf.getInt(HConstants.ZK_SESSION_TIMEOUT,
119         HConstants.DEFAULT_ZK_SESSION_TIMEOUT);
120     LOG.debug(descriptor + " opening connection to ZooKeeper with ensemble (" +
121         ensemble + ")");
122     int retry = conf.getInt("zookeeper.recovery.retry", 3);
123     int retryIntervalMillis =
124       conf.getInt("zookeeper.recovery.retry.intervalmill", 1000);
125     zkDumpConnectionTimeOut = conf.getInt("zookeeper.dump.connection.timeout",
126         1000);
127     return new RecoverableZooKeeper(ensemble, timeout, watcher,
128         retry, retryIntervalMillis);
129   }
130 
131   /**
132    * Log in the current zookeeper server process using the given configuration
133    * keys for the credential file and login principal.
134    *
135    * <p><strong>This is only applicable when running on secure hbase</strong>
136    * On regular HBase (without security features), this will safely be ignored.
137    * </p>
138    *
139    * @param conf The configuration data to use
140    * @param keytabFileKey Property key used to configure the path to the credential file
141    * @param userNameKey Property key used to configure the login principal
142    * @param hostname Current hostname to use in any credentials
143    * @throws IOException underlying exception from SecurityUtil.login() call
144    */
145   public static void loginServer(Configuration conf, String keytabFileKey,
146       String userNameKey, String hostname) throws IOException {
147     login(conf, keytabFileKey, userNameKey, hostname,
148           ZooKeeperSaslServer.LOGIN_CONTEXT_NAME_KEY,
149           JaasConfiguration.SERVER_KEYTAB_KERBEROS_CONFIG_NAME);
150   }
151 
152   /**
153    * Log in the current zookeeper client using the given configuration
154    * keys for the credential file and login principal.
155    *
156    * <p><strong>This is only applicable when running on secure hbase</strong>
157    * On regular HBase (without security features), this will safely be ignored.
158    * </p>
159    *
160    * @param conf The configuration data to use
161    * @param keytabFileKey Property key used to configure the path to the credential file
162    * @param userNameKey Property key used to configure the login principal
163    * @param hostname Current hostname to use in any credentials
164    * @throws IOException underlying exception from SecurityUtil.login() call
165    */
166   public static void loginClient(Configuration conf, String keytabFileKey,
167       String userNameKey, String hostname) throws IOException {
168     login(conf, keytabFileKey, userNameKey, hostname,
169           ZooKeeperSaslClient.LOGIN_CONTEXT_NAME_KEY,
170           JaasConfiguration.CLIENT_KEYTAB_KERBEROS_CONFIG_NAME);
171   }
172 
173   /**
174    * Log in the current process using the given configuration keys for the
175    * credential file and login principal.
176    *
177    * <p><strong>This is only applicable when running on secure hbase</strong>
178    * On regular HBase (without security features), this will safely be ignored.
179    * </p>
180    *
181    * @param conf The configuration data to use
182    * @param keytabFileKey Property key used to configure the path to the credential file
183    * @param userNameKey Property key used to configure the login principal
184    * @param hostname Current hostname to use in any credentials
185    * @param loginContextProperty property name to expose the entry name
186    * @param loginContextName jaas entry name
187    * @throws IOException underlying exception from SecurityUtil.login() call
188    */
189   private static void login(Configuration conf, String keytabFileKey,
190       String userNameKey, String hostname,
191       String loginContextProperty, String loginContextName)
192       throws IOException {
193     if (!isSecureZooKeeper(conf))
194       return;
195 
196     // User has specified a jaas.conf, keep this one as the good one.
197     // HBASE_OPTS="-Djava.security.auth.login.config=jaas.conf"
198     if (System.getProperty("java.security.auth.login.config") != null)
199       return;
200 
201     // No keytab specified, no auth
202     String keytabFilename = conf.get(keytabFileKey);
203     if (keytabFilename == null) {
204       LOG.warn("no keytab specified for: " + keytabFileKey);
205       return;
206     }
207 
208     String principalConfig = conf.get(userNameKey, System.getProperty("user.name"));
209     String principalName = SecurityUtil.getServerPrincipal(principalConfig, hostname);
210 
211     // Initialize the "jaas.conf" for keyTab/principal,
212     // If keyTab is not specified use the Ticket Cache.
213     // and set the zookeeper login context name.
214     JaasConfiguration jaasConf = new JaasConfiguration(loginContextName,
215         principalName, keytabFilename);
216     javax.security.auth.login.Configuration.setConfiguration(jaasConf);
217     System.setProperty(loginContextProperty, loginContextName);
218   }
219 
220   /**
221    * A JAAS configuration that defines the login modules that we want to use for login.
222    */
223   private static class JaasConfiguration extends javax.security.auth.login.Configuration {
224     private static final String SERVER_KEYTAB_KERBEROS_CONFIG_NAME =
225       "zookeeper-server-keytab-kerberos";
226     private static final String CLIENT_KEYTAB_KERBEROS_CONFIG_NAME =
227       "zookeeper-client-keytab-kerberos";
228 
229     private static final Map<String, String> BASIC_JAAS_OPTIONS =
230       new HashMap<String,String>();
231     static {
232       String jaasEnvVar = System.getenv("HBASE_JAAS_DEBUG");
233       if (jaasEnvVar != null && "true".equalsIgnoreCase(jaasEnvVar)) {
234         BASIC_JAAS_OPTIONS.put("debug", "true");
235       }
236     }
237 
238     private static final Map<String,String> KEYTAB_KERBEROS_OPTIONS =
239       new HashMap<String,String>();
240     static {
241       KEYTAB_KERBEROS_OPTIONS.put("doNotPrompt", "true");
242       KEYTAB_KERBEROS_OPTIONS.put("storeKey", "true");
243       KEYTAB_KERBEROS_OPTIONS.put("refreshKrb5Config", "true");
244       KEYTAB_KERBEROS_OPTIONS.putAll(BASIC_JAAS_OPTIONS);
245     }
246 
247     private static final AppConfigurationEntry KEYTAB_KERBEROS_LOGIN =
248       new AppConfigurationEntry(KerberosUtil.getKrb5LoginModuleName(),
249                                 LoginModuleControlFlag.REQUIRED,
250                                 KEYTAB_KERBEROS_OPTIONS);
251 
252     private static final AppConfigurationEntry[] KEYTAB_KERBEROS_CONF =
253       new AppConfigurationEntry[]{KEYTAB_KERBEROS_LOGIN};
254 
255     private javax.security.auth.login.Configuration baseConfig;
256     private final String loginContextName;
257     private final boolean useTicketCache;
258     private final String keytabFile;
259     private final String principal;
260 
261     public JaasConfiguration(String loginContextName, String principal) {
262       this(loginContextName, principal, null, true);
263     }
264 
265     public JaasConfiguration(String loginContextName, String principal, String keytabFile) {
266       this(loginContextName, principal, keytabFile, keytabFile == null || keytabFile.length() == 0);
267     }
268 
269     private JaasConfiguration(String loginContextName, String principal,
270                              String keytabFile, boolean useTicketCache) {
271       try {
272         this.baseConfig = javax.security.auth.login.Configuration.getConfiguration();
273       } catch (SecurityException e) {
274         this.baseConfig = null;
275       }
276       this.loginContextName = loginContextName;
277       this.useTicketCache = useTicketCache;
278       this.keytabFile = keytabFile;
279       this.principal = principal;
280       LOG.info("JaasConfiguration loginContextName=" + loginContextName +
281                " principal=" + principal + " useTicketCache=" + useTicketCache +
282                " keytabFile=" + keytabFile);
283     }
284 
285     @Override
286     public AppConfigurationEntry[] getAppConfigurationEntry(String appName) {
287       if (loginContextName.equals(appName)) {
288         if (!useTicketCache) {
289           KEYTAB_KERBEROS_OPTIONS.put("keyTab", keytabFile);
290           KEYTAB_KERBEROS_OPTIONS.put("useKeyTab", "true");
291         }
292         KEYTAB_KERBEROS_OPTIONS.put("principal", principal);
293         KEYTAB_KERBEROS_OPTIONS.put("useTicketCache", useTicketCache ? "true" : "false");
294         return KEYTAB_KERBEROS_CONF;
295       }
296       if (baseConfig != null) return baseConfig.getAppConfigurationEntry(appName);
297       return(null);
298     }
299   }
300 
301   //
302   // Helper methods
303   //
304 
305   /**
306    * Join the prefix znode name with the suffix znode name to generate a proper
307    * full znode name.
308    *
309    * Assumes prefix does not end with slash and suffix does not begin with it.
310    *
311    * @param prefix beginning of znode name
312    * @param suffix ending of znode name
313    * @return result of properly joining prefix with suffix
314    */
315   public static String joinZNode(String prefix, String suffix) {
316     return prefix + ZNODE_PATH_SEPARATOR + suffix;
317   }
318 
319   /**
320    * Returns the full path of the immediate parent of the specified node.
321    * @param node path to get parent of
322    * @return parent of path, null if passed the root node or an invalid node
323    */
324   public static String getParent(String node) {
325     int idx = node.lastIndexOf(ZNODE_PATH_SEPARATOR);
326     return idx <= 0 ? null : node.substring(0, idx);
327   }
328 
329   /**
330    * Get the name of the current node from the specified fully-qualified path.
331    * @param path fully-qualified path
332    * @return name of the current node
333    */
334   public static String getNodeName(String path) {
335     return path.substring(path.lastIndexOf("/")+1);
336   }
337 
338   /**
339    * Get the key to the ZK ensemble for this configuration without
340    * adding a name at the end
341    * @param conf Configuration to use to build the key
342    * @return ensemble key without a name
343    */
344   public static String getZooKeeperClusterKey(Configuration conf) {
345     return getZooKeeperClusterKey(conf, null);
346   }
347 
348   /**
349    * Get the key to the ZK ensemble for this configuration and append
350    * a name at the end
351    * @param conf Configuration to use to build the key
352    * @param name Name that should be appended at the end if not empty or null
353    * @return ensemble key with a name (if any)
354    */
355   public static String getZooKeeperClusterKey(Configuration conf, String name) {
356     String ensemble = conf.get(HConstants.ZOOKEEPER_QUORUM.replaceAll(
357         "[\\t\\n\\x0B\\f\\r]", ""));
358     StringBuilder builder = new StringBuilder(ensemble);
359     builder.append(":");
360     builder.append(conf.get(HConstants.ZOOKEEPER_CLIENT_PORT));
361     builder.append(":");
362     builder.append(conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT));
363     if (name != null && !name.isEmpty()) {
364       builder.append(",");
365       builder.append(name);
366     }
367     return builder.toString();
368   }
369 
370   /**
371    * Apply the settings in the given key to the given configuration, this is
372    * used to communicate with distant clusters
373    * @param conf configuration object to configure
374    * @param key string that contains the 3 required configuratins
375    * @throws IOException
376    */
377   public static void applyClusterKeyToConf(Configuration conf, String key)
378       throws IOException{
379     String[] parts = transformClusterKey(key);
380     conf.set(HConstants.ZOOKEEPER_QUORUM, parts[0]);
381     conf.set(HConstants.ZOOKEEPER_CLIENT_PORT, parts[1]);
382     conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, parts[2]);
383   }
384 
385   /**
386    * Separate the given key into the three configurations it should contain:
387    * hbase.zookeeper.quorum, hbase.zookeeper.client.port
388    * and zookeeper.znode.parent
389    * @param key
390    * @return the three configuration in the described order
391    * @throws IOException
392    */
393   public static String[] transformClusterKey(String key) throws IOException {
394     String[] parts = key.split(":");
395     if (parts.length != 3) {
396       throw new IOException("Cluster key passed " + key + " is invalid, the format should be:" +
397           HConstants.ZOOKEEPER_QUORUM + ":hbase.zookeeper.client.port:"
398           + HConstants.ZOOKEEPER_ZNODE_PARENT);
399     }
400     return parts;
401   }
402 
403   //
404   // Existence checks and watches
405   //
406 
407   /**
408    * Watch the specified znode for delete/create/change events.  The watcher is
409    * set whether or not the node exists.  If the node already exists, the method
410    * returns true.  If the node does not exist, the method returns false.
411    *
412    * @param zkw zk reference
413    * @param znode path of node to watch
414    * @return true if znode exists, false if does not exist or error
415    * @throws KeeperException if unexpected zookeeper exception
416    */
417   public static boolean watchAndCheckExists(ZooKeeperWatcher zkw, String znode)
418   throws KeeperException {
419     try {
420       Stat s = zkw.getRecoverableZooKeeper().exists(znode, zkw);
421       boolean exists = s != null ? true : false;
422       if (exists) {
423         LOG.debug(zkw.prefix("Set watcher on existing znode " + znode));
424       } else {
425         LOG.debug(zkw.prefix(znode+" does not exist. Watcher is set."));
426       }
427       return exists;
428     } catch (KeeperException e) {
429       LOG.warn(zkw.prefix("Unable to set watcher on znode " + znode), e);
430       zkw.keeperException(e);
431       return false;
432     } catch (InterruptedException e) {
433       LOG.warn(zkw.prefix("Unable to set watcher on znode " + znode), e);
434       zkw.interruptedException(e);
435       return false;
436     }
437   }
438 
439   /**
440    * Check if the specified node exists.  Sets no watches.
441    *
442    * @param zkw zk reference
443    * @param znode path of node to watch
444    * @return version of the node if it exists, -1 if does not exist
445    * @throws KeeperException if unexpected zookeeper exception
446    */
447   public static int checkExists(ZooKeeperWatcher zkw, String znode)
448   throws KeeperException {
449     try {
450       Stat s = zkw.getRecoverableZooKeeper().exists(znode, null);
451       return s != null ? s.getVersion() : -1;
452     } catch (KeeperException e) {
453       LOG.warn(zkw.prefix("Unable to set watcher on znode (" + znode + ")"), e);
454       zkw.keeperException(e);
455       return -1;
456     } catch (InterruptedException e) {
457       LOG.warn(zkw.prefix("Unable to set watcher on znode (" + znode + ")"), e);
458       zkw.interruptedException(e);
459       return -1;
460     }
461   }
462 
463   //
464   // Znode listings
465   //
466 
467   /**
468    * Lists the children znodes of the specified znode.  Also sets a watch on
469    * the specified znode which will capture a NodeDeleted event on the specified
470    * znode as well as NodeChildrenChanged if any children of the specified znode
471    * are created or deleted.
472    *
473    * Returns null if the specified node does not exist.  Otherwise returns a
474    * list of children of the specified node.  If the node exists but it has no
475    * children, an empty list will be returned.
476    *
477    * @param zkw zk reference
478    * @param znode path of node to list and watch children of
479    * @return list of children of the specified node, an empty list if the node
480    *          exists but has no children, and null if the node does not exist
481    * @throws KeeperException if unexpected zookeeper exception
482    */
483   public static List<String> listChildrenAndWatchForNewChildren(
484       ZooKeeperWatcher zkw, String znode)
485   throws KeeperException {
486     try {
487       List<String> children = zkw.getRecoverableZooKeeper().getChildren(znode, zkw);
488       return children;
489     } catch(KeeperException.NoNodeException ke) {
490       LOG.debug(zkw.prefix("Unable to list children of znode " + znode + " " +
491           "because node does not exist (not an error)"));
492       return null;
493     } catch (KeeperException e) {
494       LOG.warn(zkw.prefix("Unable to list children of znode " + znode + " "), e);
495       zkw.keeperException(e);
496       return null;
497     } catch (InterruptedException e) {
498       LOG.warn(zkw.prefix("Unable to list children of znode " + znode + " "), e);
499       zkw.interruptedException(e);
500       return null;
501     }
502   }
503 
504   /**
505    * List all the children of the specified znode, setting a watch for children
506    * changes and also setting a watch on every individual child in order to get
507    * the NodeCreated and NodeDeleted events.
508    * @param zkw zookeeper reference
509    * @param znode node to get children of and watch
510    * @return list of znode names, null if the node doesn't exist
511    * @throws KeeperException
512    */
513   public static List<String> listChildrenAndWatchThem(ZooKeeperWatcher zkw,
514       String znode) throws KeeperException {
515     List<String> children = listChildrenAndWatchForNewChildren(zkw, znode);
516     if (children == null) {
517       return null;
518     }
519     for (String child : children) {
520       watchAndCheckExists(zkw, joinZNode(znode, child));
521     }
522     return children;
523   }
524 
525   /**
526    * Lists the children of the specified znode without setting any watches.
527    *
528    * Used to list the currently online regionservers and their addresses.
529    *
530    * Sets no watches at all, this method is best effort.
531    *
532    * Returns an empty list if the node has no children.  Returns null if the
533    * parent node itself does not exist.
534    *
535    * @param zkw zookeeper reference
536    * @param znode node to get children of as addresses
537    * @return list of data of children of specified znode, empty if no children,
538    *         null if parent does not exist
539    * @throws KeeperException if unexpected zookeeper exception
540    */
541   public static List<String> listChildrenNoWatch(
542       ZooKeeperWatcher zkw, String znode)
543   throws KeeperException {
544     List<String> children = null;
545     try {
546       // List the children without watching
547       children = zkw.getRecoverableZooKeeper().getChildren(znode, null);
548     } catch(KeeperException.NoNodeException nne) {
549       return null;
550     } catch(InterruptedException ie) {
551       zkw.interruptedException(ie);
552     }
553     return children;
554   }
555 
556   /**
557    * Simple class to hold a node path and node data.
558    */
559   public static class NodeAndData {
560     private String node;
561     private byte [] data;
562     public NodeAndData(String node, byte [] data) {
563       this.node = node;
564       this.data = data;
565     }
566     public String getNode() {
567       return node;
568     }
569     public byte [] getData() {
570       return data;
571     }
572     @Override
573     public String toString() {
574       return node + " (" + RegionTransitionData.fromBytes(data) + ")";
575     }
576     public boolean isEmpty() {
577       return (data.length == 0);
578     }
579   }
580 
581   /**
582    * Checks if the specified znode has any children.  Sets no watches.
583    *
584    * Returns true if the node exists and has children.  Returns false if the
585    * node does not exist or if the node does not have any children.
586    *
587    * Used during master initialization to determine if the master is a
588    * failed-over-to master or the first master during initial cluster startup.
589    * If the directory for regionserver ephemeral nodes is empty then this is
590    * a cluster startup, if not then it is not cluster startup.
591    *
592    * @param zkw zk reference
593    * @param znode path of node to check for children of
594    * @return true if node has children, false if not or node does not exist
595    * @throws KeeperException if unexpected zookeeper exception
596    */
597   public static boolean nodeHasChildren(ZooKeeperWatcher zkw, String znode)
598   throws KeeperException {
599     try {
600       return !zkw.getRecoverableZooKeeper().getChildren(znode, null).isEmpty();
601     } catch(KeeperException.NoNodeException ke) {
602       LOG.debug(zkw.prefix("Unable to list children of znode " + znode + " " +
603       "because node does not exist (not an error)"));
604       return false;
605     } catch (KeeperException e) {
606       LOG.warn(zkw.prefix("Unable to list children of znode " + znode), e);
607       zkw.keeperException(e);
608       return false;
609     } catch (InterruptedException e) {
610       LOG.warn(zkw.prefix("Unable to list children of znode " + znode), e);
611       zkw.interruptedException(e);
612       return false;
613     }
614   }
615 
616   /**
617    * Get the number of children of the specified node.
618    *
619    * If the node does not exist or has no children, returns 0.
620    *
621    * Sets no watches at all.
622    *
623    * @param zkw zk reference
624    * @param znode path of node to count children of
625    * @return number of children of specified node, 0 if none or parent does not
626    *         exist
627    * @throws KeeperException if unexpected zookeeper exception
628    */
629   public static int getNumberOfChildren(ZooKeeperWatcher zkw, String znode)
630   throws KeeperException {
631     try {
632       Stat stat = zkw.getRecoverableZooKeeper().exists(znode, null);
633       return stat == null ? 0 : stat.getNumChildren();
634     } catch(KeeperException e) {
635       LOG.warn(zkw.prefix("Unable to get children of node " + znode));
636       zkw.keeperException(e);
637     } catch(InterruptedException e) {
638       zkw.interruptedException(e);
639     }
640     return 0;
641   }
642 
643   //
644   // Data retrieval
645   //
646 
647   /**
648    * Get znode data. Does not set a watcher.
649    * @return ZNode data
650    */
651   public static byte [] getData(ZooKeeperWatcher zkw, String znode)
652   throws KeeperException {
653     try {
654       byte [] data = zkw.getRecoverableZooKeeper().getData(znode, null, null);
655       logRetrievedMsg(zkw, znode, data, false);
656       return data;
657     } catch (KeeperException.NoNodeException e) {
658       LOG.debug(zkw.prefix("Unable to get data of znode " + znode + " " +
659         "because node does not exist (not an error)"));
660       return null;
661     } catch (KeeperException e) {
662       LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
663       zkw.keeperException(e);
664       return null;
665     } catch (InterruptedException e) {
666       LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
667       zkw.interruptedException(e);
668       return null;
669     }
670   }
671 
672   /**
673    * Get the data at the specified znode and set a watch.
674    *
675    * Returns the data and sets a watch if the node exists.  Returns null and no
676    * watch is set if the node does not exist or there is an exception.
677    *
678    * @param zkw zk reference
679    * @param znode path of node
680    * @return data of the specified znode, or null
681    * @throws KeeperException if unexpected zookeeper exception
682    */
683   public static byte [] getDataAndWatch(ZooKeeperWatcher zkw, String znode)
684   throws KeeperException {
685     return getDataInternal(zkw, znode, null, true);
686   }
687 
688   /**
689    * Get the data at the specified znode and set a watch.
690    *
691    * Returns the data and sets a watch if the node exists.  Returns null and no
692    * watch is set if the node does not exist or there is an exception.
693    *
694    * @param zkw zk reference
695    * @param znode path of node
696    * @param stat object to populate the version of the znode
697    * @return data of the specified znode, or null
698    * @throws KeeperException if unexpected zookeeper exception
699    */
700   public static byte[] getDataAndWatch(ZooKeeperWatcher zkw, String znode,
701       Stat stat) throws KeeperException {
702     return getDataInternal(zkw, znode, stat, true);
703   }
704 
705   private static byte[] getDataInternal(ZooKeeperWatcher zkw, String znode, Stat stat,
706       boolean watcherSet)
707       throws KeeperException {
708     try {
709       byte [] data = zkw.getRecoverableZooKeeper().getData(znode, zkw, stat);
710       logRetrievedMsg(zkw, znode, data, watcherSet);
711       return data;
712     } catch (KeeperException.NoNodeException e) {
713       LOG.debug(zkw.prefix("Unable to get data of znode " + znode + " " +
714         "because node does not exist (not an error)"));
715       return null;
716     } catch (KeeperException e) {
717       LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
718       zkw.keeperException(e);
719       return null;
720     } catch (InterruptedException e) {
721       LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
722       zkw.interruptedException(e);
723       return null;
724     }
725   }
726 
727   /**
728    * Get the data at the specified znode without setting a watch.
729    *
730    * Returns the data if the node exists.  Returns null if the node does not
731    * exist.
732    *
733    * Sets the stats of the node in the passed Stat object.  Pass a null stat if
734    * not interested.
735    *
736    * @param zkw zk reference
737    * @param znode path of node
738    * @param stat node status to get if node exists
739    * @return data of the specified znode, or null if node does not exist
740    * @throws KeeperException if unexpected zookeeper exception
741    */
742   public static byte [] getDataNoWatch(ZooKeeperWatcher zkw, String znode,
743       Stat stat)
744   throws KeeperException {
745     try {
746       byte [] data = zkw.getRecoverableZooKeeper().getData(znode, null, stat);
747       logRetrievedMsg(zkw, znode, data, false);
748       return data;
749     } catch (KeeperException.NoNodeException e) {
750       LOG.debug(zkw.prefix("Unable to get data of znode " + znode + " " +
751           "because node does not exist (not necessarily an error)"));
752       return null;
753     } catch (KeeperException e) {
754       LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
755       zkw.keeperException(e);
756       return null;
757     } catch (InterruptedException e) {
758       LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
759       zkw.interruptedException(e);
760       return null;
761     }
762   }
763 
764   /**
765    * Returns the date of child znodes of the specified znode.  Also sets a watch on
766    * the specified znode which will capture a NodeDeleted event on the specified
767    * znode as well as NodeChildrenChanged if any children of the specified znode
768    * are created or deleted.
769    *
770    * Returns null if the specified node does not exist.  Otherwise returns a
771    * list of children of the specified node.  If the node exists but it has no
772    * children, an empty list will be returned.
773    *
774    * @param zkw zk reference
775    * @param baseNode path of node to list and watch children of
776    * @return list of data of children of the specified node, an empty list if the node
777    *          exists but has no children, and null if the node does not exist
778    * @throws KeeperException if unexpected zookeeper exception
779    */
780   public static List<NodeAndData> getChildDataAndWatchForNewChildren(
781       ZooKeeperWatcher zkw, String baseNode) throws KeeperException {
782     List<String> nodes =
783       ZKUtil.listChildrenAndWatchForNewChildren(zkw, baseNode);
784     List<NodeAndData> newNodes = new ArrayList<NodeAndData>();
785     if (nodes != null) {
786       for (String node : nodes) {
787         String nodePath = ZKUtil.joinZNode(baseNode, node);
788         byte[] data = ZKUtil.getDataAndWatch(zkw, nodePath);
789         newNodes.add(new NodeAndData(nodePath, data));
790       }
791     }
792     return newNodes;
793   }
794 
795   /**
796    * Update the data of an existing node with the expected version to have the
797    * specified data.
798    *
799    * Throws an exception if there is a version mismatch or some other problem.
800    *
801    * Sets no watches under any conditions.
802    *
803    * @param zkw zk reference
804    * @param znode
805    * @param data
806    * @param expectedVersion
807    * @throws KeeperException if unexpected zookeeper exception
808    * @throws KeeperException.BadVersionException if version mismatch
809    */
810   public static void updateExistingNodeData(ZooKeeperWatcher zkw, String znode,
811       byte [] data, int expectedVersion)
812   throws KeeperException {
813     try {
814       zkw.getRecoverableZooKeeper().setData(znode, data, expectedVersion);
815     } catch(InterruptedException ie) {
816       zkw.interruptedException(ie);
817     }
818   }
819 
820   //
821   // Data setting
822   //
823 
824   /**
825    * Sets the data of the existing znode to be the specified data.  Ensures that
826    * the current data has the specified expected version.
827    *
828    * <p>If the node does not exist, a {@link NoNodeException} will be thrown.
829    *
830    * <p>If their is a version mismatch, method returns null.
831    *
832    * <p>No watches are set but setting data will trigger other watchers of this
833    * node.
834    *
835    * <p>If there is another problem, a KeeperException will be thrown.
836    *
837    * @param zkw zk reference
838    * @param znode path of node
839    * @param data data to set for node
840    * @param expectedVersion version expected when setting data
841    * @return true if data set, false if version mismatch
842    * @throws KeeperException if unexpected zookeeper exception
843    */
844   public static boolean setData(ZooKeeperWatcher zkw, String znode,
845       byte [] data, int expectedVersion)
846   throws KeeperException, KeeperException.NoNodeException {
847     try {
848       return zkw.getRecoverableZooKeeper().setData(znode, data, expectedVersion) != null;
849     } catch (InterruptedException e) {
850       zkw.interruptedException(e);
851       return false;
852     }
853   }
854 
855   /**
856    * Set data into node creating node if it doesn't yet exist.
857    * Does not set watch.
858    *
859    * @param zkw zk reference
860    * @param znode path of node
861    * @param data data to set for node
862    * @throws KeeperException
863    */
864   public static void createSetData(final ZooKeeperWatcher zkw, final String znode,
865       final byte [] data)
866   throws KeeperException {
867     if (checkExists(zkw, znode) == -1) {
868       ZKUtil.createWithParents(zkw, znode, data);
869     } else {
870       ZKUtil.setData(zkw, znode, data);
871     }
872   }
873 
874   /**
875    * Sets the data of the existing znode to be the specified data.  The node
876    * must exist but no checks are done on the existing data or version.
877    *
878    * <p>If the node does not exist, a {@link NoNodeException} will be thrown.
879    *
880    * <p>No watches are set but setting data will trigger other watchers of this
881    * node.
882    *
883    * <p>If there is another problem, a KeeperException will be thrown.
884    *
885    * @param zkw zk reference
886    * @param znode path of node
887    * @param data data to set for node
888    * @throws KeeperException if unexpected zookeeper exception
889    */
890   public static void setData(ZooKeeperWatcher zkw, String znode, byte [] data)
891   throws KeeperException, KeeperException.NoNodeException {
892     setData(zkw, (SetData)ZKUtilOp.setData(znode, data));
893   }
894 
895   private static void setData(ZooKeeperWatcher zkw, SetData setData)
896   throws KeeperException, KeeperException.NoNodeException {
897     SetDataRequest sd = (SetDataRequest)toZooKeeperOp(zkw, setData).toRequestRecord();
898     setData(zkw, sd.getPath(), sd.getData(), sd.getVersion());
899   }
900 
901   /**
902    * Returns whether or not secure authentication is enabled
903    * (whether <code>hbase.security.authentication</code> is set to
904    * <code>kerberos</code>.
905    */
906   public static boolean isSecureZooKeeper(Configuration conf) {
907     // hbase shell need to use:
908     //    -Djava.security.auth.login.config=user-jaas.conf
909     // since each user has a different jaas.conf
910     if (System.getProperty("java.security.auth.login.config") != null)
911       return true;
912 
913     // Master & RSs uses hbase.zookeeper.client.*
914     return("kerberos".equalsIgnoreCase(conf.get("hbase.security.authentication")) &&
915          conf.get("hbase.zookeeper.client.keytab.file") != null);
916   }
917 
918   private static ArrayList<ACL> createACL(ZooKeeperWatcher zkw, String node) {
919     if (isSecureZooKeeper(zkw.getConfiguration())) {
920       // Certain znodes are accessed directly by the client,
921       // so they must be readable by non-authenticated clients
922       if ((node.equals(zkw.baseZNode) == true) ||
923           (node.equals(zkw.rootServerZNode) == true) ||
924           (node.equals(zkw.masterAddressZNode) == true) ||
925           (node.equals(zkw.clusterIdZNode) == true) ||
926           (node.equals(zkw.rsZNode) == true) ||
927           (node.equals(zkw.backupMasterAddressesZNode) == true) ||
928           (node.startsWith(zkw.masterTableZNode) == true) ||
929           (node.startsWith(zkw.masterTableZNode92) == true)) {
930         return ZooKeeperWatcher.CREATOR_ALL_AND_WORLD_READABLE;
931       }
932       return Ids.CREATOR_ALL_ACL;
933     } else {
934       return Ids.OPEN_ACL_UNSAFE;
935     }
936   }
937 
938   //
939   // Node creation
940   //
941 
942   /**
943    *
944    * Set the specified znode to be an ephemeral node carrying the specified
945    * data.
946    *
947    * If the node is created successfully, a watcher is also set on the node.
948    *
949    * If the node is not created successfully because it already exists, this
950    * method will also set a watcher on the node.
951    *
952    * If there is another problem, a KeeperException will be thrown.
953    *
954    * @param zkw zk reference
955    * @param znode path of node
956    * @param data data of node
957    * @return true if node created, false if not, watch set in both cases
958    * @throws KeeperException if unexpected zookeeper exception
959    */
960   public static boolean createEphemeralNodeAndWatch(ZooKeeperWatcher zkw,
961       String znode, byte [] data)
962   throws KeeperException {
963     boolean ret = true;
964     try {
965       zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode),
966           CreateMode.EPHEMERAL);
967     } catch (KeeperException.NodeExistsException nee) {
968       ret = false;
969     } catch (InterruptedException e) {
970       LOG.info("Interrupted", e);
971       Thread.currentThread().interrupt();
972     }
973     if(!watchAndCheckExists(zkw, znode)) {
974       // It did exist but now it doesn't, try again
975       return createEphemeralNodeAndWatch(zkw, znode, data);
976     }
977     return ret;
978   }
979 
980   /**
981    * Creates the specified znode to be a persistent node carrying the specified
982    * data.
983    *
984    * Returns true if the node was successfully created, false if the node
985    * already existed.
986    *
987    * If the node is created successfully, a watcher is also set on the node.
988    *
989    * If the node is not created successfully because it already exists, this
990    * method will also set a watcher on the node but return false.
991    *
992    * If there is another problem, a KeeperException will be thrown.
993    *
994    * @param zkw zk reference
995    * @param znode path of node
996    * @param data data of node
997    * @return true if node created, false if not, watch set in both cases
998    * @throws KeeperException if unexpected zookeeper exception
999    */
1000   public static boolean createNodeIfNotExistsAndWatch(
1001       ZooKeeperWatcher zkw, String znode, byte [] data)
1002   throws KeeperException {
1003     boolean ret = true;
1004     try {
1005       zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode),
1006           CreateMode.PERSISTENT);
1007     } catch (KeeperException.NodeExistsException nee) {
1008       ret = false;
1009     } catch (InterruptedException e) {
1010       zkw.interruptedException(e);
1011       return false;
1012     }
1013     try {
1014       zkw.getRecoverableZooKeeper().exists(znode, zkw);
1015     } catch (InterruptedException e) {
1016       zkw.interruptedException(e);
1017       return false;
1018     }
1019     return ret;
1020   }
1021 
1022   /**
1023    * Creates the specified node with the specified data and watches it.
1024    *
1025    * <p>Throws an exception if the node already exists.
1026    *
1027    * <p>The node created is persistent and open access.
1028    *
1029    * <p>Returns the version number of the created node if successful.
1030    *
1031    * @param zkw zk reference
1032    * @param znode path of node to create
1033    * @param data data of node to create
1034    * @return version of node created
1035    * @throws KeeperException if unexpected zookeeper exception
1036    * @throws KeeperException.NodeExistsException if node already exists
1037    */
1038   public static int createAndWatch(ZooKeeperWatcher zkw,
1039       String znode, byte [] data)
1040   throws KeeperException, KeeperException.NodeExistsException {
1041     try {
1042       zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode),
1043           CreateMode.PERSISTENT);
1044       return zkw.getRecoverableZooKeeper().exists(znode, zkw).getVersion();
1045     } catch (InterruptedException e) {
1046       zkw.interruptedException(e);
1047       return -1;
1048     }
1049   }
1050 
1051   /**
1052    * Async creates the specified node with the specified data.
1053    *
1054    * <p>Throws an exception if the node already exists.
1055    *
1056    * <p>The node created is persistent and open access.
1057    *
1058    * @param zkw zk reference
1059    * @param znode path of node to create
1060    * @param data data of node to create
1061    * @param cb
1062    * @param ctx
1063    * @throws KeeperException if unexpected zookeeper exception
1064    * @throws KeeperException.NodeExistsException if node already exists
1065    */
1066   public static void asyncCreate(ZooKeeperWatcher zkw,
1067       String znode, byte [] data, final AsyncCallback.StringCallback cb,
1068       final Object ctx) {
1069     zkw.getRecoverableZooKeeper().getZooKeeper().create(znode, data,
1070         createACL(zkw, znode), CreateMode.PERSISTENT, cb, ctx);
1071   }
1072 
1073   /**
1074    * Creates the specified node, iff the node does not exist.  Does not set a
1075    * watch and fails silently if the node already exists.
1076    *
1077    * The node created is persistent and open access.
1078    *
1079    * @param zkw zk reference
1080    * @param znode path of node
1081    * @throws KeeperException if unexpected zookeeper exception
1082    */
1083   public static void createAndFailSilent(ZooKeeperWatcher zkw,
1084       String znode) throws KeeperException {
1085     createAndFailSilent(zkw, znode, new byte[0]);
1086   }
1087 
1088   /**
1089    * Creates the specified node containing specified data, iff the node does not exist.  Does
1090    * not set a watch and fails silently if the node already exists.
1091    *
1092    * The node created is persistent and open access.
1093    *
1094    * @param zkw zk reference
1095    * @param znode path of node
1096    * @param data a byte array data to store in the znode
1097    * @throws KeeperException if unexpected zookeeper exception
1098    */
1099   public static void createAndFailSilent(ZooKeeperWatcher zkw,
1100       String znode, byte[] data) throws KeeperException {
1101     createAndFailSilent(zkw,
1102       (CreateAndFailSilent)ZKUtilOp.createAndFailSilent(znode, data));
1103   }
1104 
1105   private static void createAndFailSilent(ZooKeeperWatcher zkw, CreateAndFailSilent cafs)
1106   throws KeeperException {
1107     CreateRequest create = (CreateRequest)toZooKeeperOp(zkw, cafs).toRequestRecord();
1108     String znode = create.getPath();
1109     try {
1110       RecoverableZooKeeper zk = zkw.getRecoverableZooKeeper();
1111       if (zk.exists(znode, false) == null) {
1112         zk.create(znode, create.getData(), create.getAcl(), CreateMode.fromFlag(create.getFlags()));
1113       }
1114     } catch(KeeperException.NodeExistsException nee) {
1115     } catch(KeeperException.NoAuthException nee){
1116       try {
1117         if (null == zkw.getRecoverableZooKeeper().exists(znode, false)) {
1118           // If we failed to create the file and it does not already exist.
1119           throw(nee);
1120         }
1121       } catch (InterruptedException ie) {
1122         zkw.interruptedException(ie);
1123       }
1124 
1125     } catch(InterruptedException ie) {
1126       zkw.interruptedException(ie);
1127     }
1128   }
1129 
1130   /**
1131    * Creates the specified node and all parent nodes required for it to exist.
1132    *
1133    * No watches are set and no errors are thrown if the node already exists.
1134    *
1135    * The nodes created are persistent and open access.
1136    *
1137    * @param zkw zk reference
1138    * @param znode path of node
1139    * @throws KeeperException if unexpected zookeeper exception
1140    */
1141   public static void createWithParents(ZooKeeperWatcher zkw, String znode)
1142   throws KeeperException {
1143     createWithParents(zkw, znode, new byte[0]);
1144   }
1145 
1146   /**
1147    * Creates the specified node and all parent nodes required for it to exist.  The creation of
1148    * parent znodes is not atomic with the leafe znode creation but the data is written atomically
1149    * when the leaf node is created.
1150    *
1151    * No watches are set and no errors are thrown if the node already exists.
1152    *
1153    * The nodes created are persistent and open access.
1154    *
1155    * @param zkw zk reference
1156    * @param znode path of node
1157    * @throws KeeperException if unexpected zookeeper exception
1158    */
1159   public static void createWithParents(ZooKeeperWatcher zkw, String znode, byte[] data)
1160   throws KeeperException {
1161     try {
1162       if(znode == null) {
1163         return;
1164       }
1165       zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode),
1166           CreateMode.PERSISTENT);
1167     } catch(KeeperException.NodeExistsException nee) {
1168       return;
1169     } catch(KeeperException.NoNodeException nne) {
1170       createWithParents(zkw, getParent(znode));
1171       createWithParents(zkw, znode, data);
1172     } catch(InterruptedException ie) {
1173       zkw.interruptedException(ie);
1174     }
1175   }
1176 
1177   //
1178   // Deletes
1179   //
1180 
1181   /**
1182    * Delete the specified node.  Sets no watches.  Throws all exceptions.
1183    */
1184   public static void deleteNode(ZooKeeperWatcher zkw, String node)
1185   throws KeeperException {
1186     deleteNode(zkw, node, -1);
1187   }
1188 
1189   /**
1190    * Delete the specified node with the specified version.  Sets no watches.
1191    * Throws all exceptions.
1192    */
1193   public static boolean deleteNode(ZooKeeperWatcher zkw, String node,
1194       int version)
1195   throws KeeperException {
1196     try {
1197       zkw.getRecoverableZooKeeper().delete(node, version);
1198       return true;
1199     } catch(KeeperException.BadVersionException bve) {
1200       return false;
1201     } catch(InterruptedException ie) {
1202       zkw.interruptedException(ie);
1203       return false;
1204     }
1205   }
1206 
1207   /**
1208    * Deletes the specified node.  Fails silent if the node does not exist.
1209    * @param zkw
1210    * @param node
1211    * @throws KeeperException
1212    */
1213   public static void deleteNodeFailSilent(ZooKeeperWatcher zkw, String node)
1214   throws KeeperException {
1215     deleteNodeFailSilent(zkw,
1216       (DeleteNodeFailSilent)ZKUtilOp.deleteNodeFailSilent(node));
1217   }
1218 
1219   private static void deleteNodeFailSilent(ZooKeeperWatcher zkw,
1220       DeleteNodeFailSilent dnfs) throws KeeperException {
1221     DeleteRequest delete = (DeleteRequest)toZooKeeperOp(zkw, dnfs).toRequestRecord();
1222     try {
1223       zkw.getRecoverableZooKeeper().delete(delete.getPath(), delete.getVersion());
1224     } catch(KeeperException.NoNodeException nne) {
1225     } catch(InterruptedException ie) {
1226       zkw.interruptedException(ie);
1227     }
1228   }
1229 
1230   /**
1231    * Delete the specified node and all of it's children.
1232    * <p>
1233    * If the node does not exist, just returns.
1234    * <p>
1235    * Sets no watches. Throws all exceptions besides dealing with deletion of
1236    * children.
1237    */
1238   public static void deleteNodeRecursively(ZooKeeperWatcher zkw, String node)
1239   throws KeeperException {
1240     try {
1241       List<String> children = ZKUtil.listChildrenNoWatch(zkw, node);
1242       // the node is already deleted, so we just finish
1243       if (children == null) return;
1244 
1245       if(!children.isEmpty()) {
1246         for(String child : children) {
1247           deleteNodeRecursively(zkw, joinZNode(node, child));
1248         }
1249       }
1250       zkw.getRecoverableZooKeeper().delete(node, -1);
1251     } catch(InterruptedException ie) {
1252       zkw.interruptedException(ie);
1253     }
1254   }
1255 
1256   /**
1257    * Delete all the children of the specified node but not the node itself.
1258    *
1259    * Sets no watches.  Throws all exceptions besides dealing with deletion of
1260    * children.
1261    */
1262   public static void deleteChildrenRecursively(ZooKeeperWatcher zkw, String node)
1263   throws KeeperException {
1264     List<String> children = ZKUtil.listChildrenNoWatch(zkw, node);
1265     if (children == null || children.isEmpty()) return;
1266     for(String child : children) {
1267       deleteNodeRecursively(zkw, joinZNode(node, child));
1268     }
1269   }
1270 
1271   /**
1272    * Represents an action taken by ZKUtil, e.g. createAndFailSilent.
1273    * These actions are higher-level than {@link ZKOp} actions, which represent
1274    * individual actions in the ZooKeeper API, like create.
1275    */
1276   public abstract static class ZKUtilOp {
1277     private String path;
1278 
1279     private ZKUtilOp(String path) {
1280       this.path = path;
1281     }
1282 
1283     /**
1284      * @return a createAndFailSilent ZKUtilOp
1285      */
1286     public static ZKUtilOp createAndFailSilent(String path, byte[] data) {
1287       return new CreateAndFailSilent(path, data);
1288     }
1289 
1290     /**
1291      * @return a deleteNodeFailSilent ZKUtilOP
1292      */
1293     public static ZKUtilOp deleteNodeFailSilent(String path) {
1294       return new DeleteNodeFailSilent(path);
1295     }
1296 
1297     /**
1298      * @return a setData ZKUtilOp
1299      */
1300     public static ZKUtilOp setData(String path, byte [] data) {
1301       return new SetData(path, data);
1302     }
1303 
1304     /**
1305      * @return path to znode where the ZKOp will occur
1306      */
1307     public String getPath() {
1308       return path;
1309     }
1310 
1311     /**
1312      * ZKUtilOp representing createAndFailSilent in ZooKeeper
1313      * (attempt to create node, ignore error if already exists)
1314      */
1315     public static class CreateAndFailSilent extends ZKUtilOp {
1316       private byte [] data;
1317 
1318       private CreateAndFailSilent(String path, byte [] data) {
1319         super(path);
1320         this.data = data;
1321       }
1322 
1323       public byte[] getData() {
1324         return data;
1325       }
1326 
1327       @Override
1328       public boolean equals(Object o) {
1329         if (this == o) return true;
1330         if (!(o instanceof CreateAndFailSilent)) return false;
1331 
1332         CreateAndFailSilent op = (CreateAndFailSilent) o;
1333         return getPath().equals(op.getPath()) && Arrays.equals(data, op.data);
1334       }
1335     }
1336 
1337     /**
1338      * ZKUtilOp representing deleteNodeFailSilent in ZooKeeper
1339      * (attempt to delete node, ignore error if node doesn't exist)
1340      */
1341     public static class DeleteNodeFailSilent extends ZKUtilOp {
1342       private DeleteNodeFailSilent(String path) {
1343         super(path);
1344       }
1345 
1346       @Override
1347       public boolean equals(Object o) {
1348         if (this == o) return true;
1349         if (!(o instanceof DeleteNodeFailSilent)) return false;
1350 
1351         return super.equals(o);
1352       }
1353     }
1354 
1355     /**
1356      * @return ZKUtilOp representing setData in ZooKeeper
1357      */
1358     public static class SetData extends ZKUtilOp {
1359       private byte [] data;
1360 
1361       private SetData(String path, byte [] data) {
1362         super(path);
1363         this.data = data;
1364       }
1365 
1366       public byte[] getData() {
1367         return data;
1368       }
1369 
1370       @Override
1371       public boolean equals(Object o) {
1372         if (this == o) return true;
1373         if (!(o instanceof SetData)) return false;
1374 
1375         SetData op = (SetData) o;
1376         return getPath().equals(op.getPath()) && Arrays.equals(data, op.data);
1377       }
1378     }
1379   }
1380 
1381   /**
1382    * Convert from ZKUtilOp to ZKOp
1383    */
1384   private static Op toZooKeeperOp(ZooKeeperWatcher zkw, ZKUtilOp op)
1385   throws UnsupportedOperationException {
1386     if(op == null) return null;
1387 
1388     if (op instanceof CreateAndFailSilent) {
1389       CreateAndFailSilent cafs = (CreateAndFailSilent)op;
1390       return Op.create(cafs.getPath(), cafs.getData(), createACL(zkw, cafs.getPath()),
1391         CreateMode.PERSISTENT);
1392     } else if (op instanceof DeleteNodeFailSilent) {
1393       DeleteNodeFailSilent dnfs = (DeleteNodeFailSilent)op;
1394       return Op.delete(dnfs.getPath(), -1);
1395     } else if (op instanceof SetData) {
1396       SetData sd = (SetData)op;
1397       return Op.setData(sd.getPath(), sd.getData(), -1);
1398     } else {
1399       throw new UnsupportedOperationException("Unexpected ZKUtilOp type: "
1400         + op.getClass().getName());
1401     }
1402   }
1403 
1404   /**
1405    * If hbase.zookeeper.useMulti is true, use ZooKeeper's multi-update functionality.
1406    * Otherwise, run the list of operations sequentially.
1407    *
1408    * If all of the following are true:
1409    * - runSequentialOnMultiFailure is true
1410    * - hbase.zookeeper.useMulti is true
1411    * - on calling multi, we get a ZooKeeper exception that can be handled by a sequential call(*)
1412    * Then:
1413    * - we retry the operations one-by-one (sequentially)
1414    *
1415    * Note *: an example is receiving a NodeExistsException from a "create" call.  Without multi,
1416    * a user could call "createAndFailSilent" to ensure that a node exists if they don't care who
1417    * actually created the node (i.e. the NodeExistsException from ZooKeeper is caught).
1418    * This will cause all operations in the multi to fail, however, because
1419    * the NodeExistsException that zk.create throws will fail the multi transaction.
1420    * In this case, if the previous conditions hold, the commands are run sequentially, which should
1421    * result in the correct final state, but means that the operations will not run atomically.
1422    *
1423    * @throws KeeperException
1424    */
1425   public static void multiOrSequential(ZooKeeperWatcher zkw, List<ZKUtilOp> ops,
1426       boolean runSequentialOnMultiFailure) throws KeeperException {
1427     if (ops == null) return;
1428     boolean useMulti = zkw.getConfiguration().getBoolean(HConstants.ZOOKEEPER_USEMULTI, false);
1429 
1430     if (useMulti) {
1431       List<Op> zkOps = new LinkedList<Op>();
1432       for (ZKUtilOp op : ops) {
1433         zkOps.add(toZooKeeperOp(zkw, op));
1434       }
1435       try {
1436         zkw.getRecoverableZooKeeper().multi(zkOps);
1437       } catch (KeeperException ke) {
1438        switch (ke.code()) {
1439          case NODEEXISTS:
1440          case NONODE:
1441          case BADVERSION:
1442          case NOAUTH:
1443            // if we get an exception that could be solved by running sequentially
1444            // (and the client asked us to), then break out and run sequentially
1445            if (runSequentialOnMultiFailure) {
1446              LOG.info("On call to ZK.multi, received exception: " + ke.toString() + "."
1447                + "  Attempting to run operations sequentially because"
1448                + " runSequentialOnMultiFailure is: " + runSequentialOnMultiFailure + ".");
1449              processSequentially(zkw, ops);
1450              break;
1451            }
1452           default:
1453             throw ke;
1454         }
1455       } catch (InterruptedException ie) {
1456         zkw.interruptedException(ie);
1457       }
1458     } else {
1459       // run sequentially
1460       processSequentially(zkw, ops);
1461     }
1462   }
1463 
1464   private static void processSequentially(ZooKeeperWatcher zkw, List<ZKUtilOp> ops)
1465       throws KeeperException, NoNodeException {
1466     for (ZKUtilOp op : ops) {
1467       if (op instanceof CreateAndFailSilent) {
1468         createAndFailSilent(zkw, (CreateAndFailSilent) op);
1469       } else if (op instanceof DeleteNodeFailSilent) {
1470         deleteNodeFailSilent(zkw, (DeleteNodeFailSilent) op);
1471       } else if (op instanceof SetData) {
1472         setData(zkw, (SetData) op);
1473       } else {
1474         throw new UnsupportedOperationException("Unexpected ZKUtilOp type: "
1475             + op.getClass().getName());
1476       }
1477     }
1478   }
1479 
1480   //
1481   // ZooKeeper cluster information
1482   //
1483 
1484   /** @return String dump of everything in ZooKeeper. */
1485   public static String dump(ZooKeeperWatcher zkw) {
1486     StringBuilder sb = new StringBuilder();
1487     try {
1488       sb.append("HBase is rooted at ").append(zkw.baseZNode);
1489       sb.append("\nActive master address: ").append(
1490           ServerName.parseVersionedServerName(getData(zkw, zkw.masterAddressZNode)));
1491       sb.append("\nBackup master addresses:");
1492       for (String child : listChildrenNoWatch(zkw,
1493                                               zkw.backupMasterAddressesZNode)) {
1494         sb.append("\n ").append(child);
1495       }
1496       sb.append("\nRegion server holding ROOT: ").append(
1497           Bytes.toStringBinary(getData(zkw, zkw.rootServerZNode)));
1498       sb.append("\nRegion servers:");
1499       for (String child : listChildrenNoWatch(zkw, zkw.rsZNode)) {
1500         sb.append("\n ").append(child);
1501       }
1502       try {
1503         getReplicationZnodesDump(zkw, sb);
1504       } catch (KeeperException ke) {
1505         LOG.warn("Couldn't get the replication znode dump." + ke.getStackTrace());
1506       }
1507       sb.append("\nQuorum Server Statistics:");
1508       String[] servers = zkw.getQuorum().split(",");
1509       for (String server : servers) {
1510         sb.append("\n ").append(server);
1511         try {
1512           String[] stat = getServerStats(server, ZKUtil.zkDumpConnectionTimeOut);
1513 
1514           if (stat == null) {
1515             sb.append("[Error] invalid quorum server: " + server);
1516             break;
1517           }
1518 
1519           for (String s : stat) {
1520             sb.append("\n  ").append(s);
1521           }
1522         } catch (Exception e) {
1523           sb.append("\n  ERROR: ").append(e.getMessage());
1524         }
1525       }
1526     } catch (KeeperException ke) {
1527       sb.append("\nFATAL ZooKeeper Exception!\n");
1528       sb.append("\n" + ke.getMessage());
1529     }
1530     return sb.toString();
1531   }
1532 
1533   private static void getReplicationZnodesDump(ZooKeeperWatcher zkw, StringBuilder sb)
1534       throws KeeperException {
1535     String replicationZNodeName = zkw.getConfiguration().get("zookeeper.znode.replication",
1536       "replication");
1537     String replicationZnode = joinZNode(zkw.baseZNode, replicationZNodeName);
1538     if (ZKUtil.checkExists(zkw, replicationZnode) == -1)
1539       return;
1540     // do a ls -r on this znode
1541     List<String> stack = new LinkedList<String>();
1542     stack.add(replicationZnode);
1543     do {
1544       String znodeToProcess = stack.remove(stack.size() - 1);
1545       sb.append("\n").append(znodeToProcess).append(": ")
1546           .append(Bytes.toString(ZKUtil.getData(zkw, znodeToProcess)));
1547       for (String zNodeChild : ZKUtil.listChildrenNoWatch(zkw, znodeToProcess)) {
1548         stack.add(ZKUtil.joinZNode(znodeToProcess, zNodeChild));
1549       }
1550     } while (stack.size() > 0);
1551   }
1552   /**
1553    * Gets the statistics from the given server.
1554    *
1555    * @param server  The server to get the statistics from.
1556    * @param timeout  The socket timeout to use.
1557    * @return The array of response strings.
1558    * @throws IOException When the socket communication fails.
1559    */
1560   public static String[] getServerStats(String server, int timeout)
1561   throws IOException {
1562     String[] sp = server.split(":");
1563     if (sp == null || sp.length == 0) {
1564       return null;
1565     }
1566 
1567     String host = sp[0];
1568     int port = sp.length > 1 ? Integer.parseInt(sp[1])
1569         : HConstants.DEFAULT_ZOOKEPER_CLIENT_PORT;
1570 
1571     Socket socket = new Socket();
1572     InetSocketAddress sockAddr = new InetSocketAddress(host, port);
1573     socket.connect(sockAddr, timeout);
1574 
1575     socket.setSoTimeout(timeout);
1576     PrintWriter out = new PrintWriter(socket.getOutputStream(), true);
1577     BufferedReader in = new BufferedReader(new InputStreamReader(
1578       socket.getInputStream()));
1579     out.println("stat");
1580     out.flush();
1581     ArrayList<String> res = new ArrayList<String>();
1582     while (true) {
1583       String line = in.readLine();
1584       if (line != null) {
1585         res.add(line);
1586       } else {
1587         break;
1588       }
1589     }
1590     socket.close();
1591     return res.toArray(new String[res.size()]);
1592   }
1593 
1594   private static void logRetrievedMsg(final ZooKeeperWatcher zkw,
1595       final String znode, final byte [] data, final boolean watcherSet) {
1596     if (!LOG.isDebugEnabled()) return;
1597     LOG.debug(zkw.prefix("Retrieved " + ((data == null)? 0: data.length) +
1598       " byte(s) of data from znode " + znode +
1599       (watcherSet? " and set watcher; ": "; data=") +
1600       (data == null? "null": data.length == 0? "empty": (
1601           znode.startsWith(zkw.assignmentZNode) ?
1602               RegionTransitionData.fromBytes(data).toString()
1603               : StringUtils.abbreviate(Bytes.toStringBinary(data), 32)))));
1604   }
1605 
1606   /**
1607    * Waits for HBase installation's base (parent) znode to become available.
1608    * @throws IOException on ZK errors
1609    */
1610   public static void waitForBaseZNode(Configuration conf) throws IOException {
1611     LOG.info("Waiting until the base znode is available");
1612     String parentZNode = conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT,
1613         HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT);
1614     ZooKeeper zk = new ZooKeeper(ZKConfig.getZKQuorumServersString(conf),
1615         conf.getInt(HConstants.ZK_SESSION_TIMEOUT,
1616         HConstants.DEFAULT_ZK_SESSION_TIMEOUT), EmptyWatcher.instance);
1617 
1618     final int maxTimeMs = 10000;
1619     final int maxNumAttempts = maxTimeMs / HConstants.SOCKET_RETRY_WAIT_MS;
1620 
1621     KeeperException keeperEx = null;
1622     try {
1623       try {
1624         for (int attempt = 0; attempt < maxNumAttempts; ++attempt) {
1625           try {
1626             if (zk.exists(parentZNode, false) != null) {
1627               LOG.info("Parent znode exists: " + parentZNode);
1628               keeperEx = null;
1629               break;
1630             }
1631           } catch (KeeperException e) {
1632             keeperEx = e;
1633           }
1634           Threads.sleepWithoutInterrupt(HConstants.SOCKET_RETRY_WAIT_MS);
1635         }
1636       } finally {
1637         zk.close();
1638       }
1639     } catch (InterruptedException ex) {
1640       Thread.currentThread().interrupt();
1641     }
1642 
1643     if (keeperEx != null) {
1644       throw new IOException(keeperEx);
1645     }
1646   }
1647 
1648   /**
1649    * Recursively print the current state of ZK (non-transactional)
1650    * @param root name of the root directory in zk to print
1651    * @throws KeeperException
1652    */
1653   public static void logZKTree(ZooKeeperWatcher zkw, String root) {
1654     if (!LOG.isDebugEnabled()) return;
1655     LOG.debug("Current zk system:");
1656     String prefix = "|-";
1657     LOG.debug(prefix + root);
1658     try {
1659       logZKTree(zkw, root, prefix);
1660     } catch (KeeperException e) {
1661       throw new RuntimeException(e);
1662     }
1663   }
1664 
1665   /**
1666    * Helper method to print the current state of the ZK tree.
1667    * @see #logZKTree(ZooKeeperWatcher, String)
1668    * @throws KeeperException if an unexpected exception occurs
1669    */
1670   protected static void logZKTree(ZooKeeperWatcher zkw, String root, String prefix) throws KeeperException {
1671     List<String> children = ZKUtil.listChildrenNoWatch(zkw, root);
1672     if (children == null) return;
1673     for (String child : children) {
1674       LOG.debug(prefix + child);
1675       String node = ZKUtil.joinZNode(root.equals("/") ? "" : root, child);
1676       logZKTree(zkw, node, prefix + "---");
1677     }
1678   }
1679 
1680 }