View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.zookeeper;
20  
21  import java.io.BufferedReader;
22  import java.io.IOException;
23  import java.io.InputStreamReader;
24  import java.io.PrintWriter;
25  import java.net.InetSocketAddress;
26  import java.net.Socket;
27  import java.util.ArrayList;
28  import java.util.Arrays;
29  import java.util.Deque;
30  import java.util.HashMap;
31  import java.util.LinkedList;
32  import java.util.List;
33  import java.util.Map;
34  
35  import javax.security.auth.login.AppConfigurationEntry;
36  import javax.security.auth.login.AppConfigurationEntry.LoginModuleControlFlag;
37  
38  import org.apache.commons.lang.StringUtils;
39  import org.apache.commons.logging.Log;
40  import org.apache.commons.logging.LogFactory;
41  import org.apache.hadoop.conf.Configuration;
42  import org.apache.hadoop.hbase.HBaseConfiguration;
43  import org.apache.hadoop.hbase.HConstants;
44  import org.apache.hadoop.hbase.ServerName;
45  import org.apache.hadoop.hbase.classification.InterfaceAudience;
46  import org.apache.hadoop.hbase.exceptions.DeserializationException;
47  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
48  import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos;
49  import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionStoreSequenceIds;
50  import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
51  import org.apache.hadoop.hbase.util.ByteStringer;
52  import org.apache.hadoop.hbase.util.Bytes;
53  import org.apache.hadoop.hbase.util.Threads;
54  import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp.CreateAndFailSilent;
55  import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp.DeleteNodeFailSilent;
56  import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp.SetData;
57  import org.apache.hadoop.security.SecurityUtil;
58  import org.apache.hadoop.security.authentication.util.KerberosUtil;
59  import org.apache.zookeeper.AsyncCallback;
60  import org.apache.zookeeper.CreateMode;
61  import org.apache.zookeeper.KeeperException;
62  import org.apache.zookeeper.KeeperException.NoNodeException;
63  import org.apache.zookeeper.Op;
64  import org.apache.zookeeper.Watcher;
65  import org.apache.zookeeper.ZooDefs.Ids;
66  import org.apache.zookeeper.ZooDefs.Perms;
67  import org.apache.zookeeper.ZooKeeper;
68  import org.apache.zookeeper.client.ZooKeeperSaslClient;
69  import org.apache.zookeeper.data.ACL;
70  import org.apache.zookeeper.data.Id;
71  import org.apache.zookeeper.data.Stat;
72  import org.apache.zookeeper.proto.CreateRequest;
73  import org.apache.zookeeper.proto.DeleteRequest;
74  import org.apache.zookeeper.proto.SetDataRequest;
75  import org.apache.zookeeper.server.ZooKeeperSaslServer;
76  
77  import com.google.common.annotations.VisibleForTesting;
78  import com.google.protobuf.InvalidProtocolBufferException;
79  
80  /**
81   * Internal HBase utility class for ZooKeeper.
82   *
83   * <p>Contains only static methods and constants.
84   *
85   * <p>Methods all throw {@link KeeperException} if there is an unexpected
86   * zookeeper exception, so callers of these methods must handle appropriately.
87   * If ZK is required for the operation, the server will need to be aborted.
88   */
89  @InterfaceAudience.Private
90  public class ZKUtil {
91    private static final Log LOG = LogFactory.getLog(ZKUtil.class);
92  
93    // TODO: Replace this with ZooKeeper constant when ZOOKEEPER-277 is resolved.
94    public static final char ZNODE_PATH_SEPARATOR = '/';
95    private static int zkDumpConnectionTimeOut;
96  
97    // The Quorum for the ZK cluster can have one the following format (see examples below):
98    // (1). s1,s2,s3 (no client port in the list, the client port could be obtained from clientPort)
99    // (2). s1:p1,s2:p2,s3:p3 (with client port, which could be same or different for each server,
100   //      in this case, the clientPort would be ignored)
101   // (3). s1:p1,s2,s3:p3 (mix of (1) and (2) - if port is not specified in a server, it would use
102   //      the clientPort; otherwise, it would use the specified port)
103   @VisibleForTesting
104   public static class ZKClusterKey {
105     public String quorumString;
106     public int clientPort;
107     public String znodeParent;
108 
109     ZKClusterKey(String quorumString, int clientPort, String znodeParent) {
110       this.quorumString = quorumString;
111       this.clientPort = clientPort;
112       this.znodeParent = znodeParent;
113     }
114   }
115 
116   /**
117    * Creates a new connection to ZooKeeper, pulling settings and ensemble config
118    * from the specified configuration object using methods from {@link ZKConfig}.
119    *
120    * Sets the connection status monitoring watcher to the specified watcher.
121    *
122    * @param conf configuration to pull ensemble and other settings from
123    * @param watcher watcher to monitor connection changes
124    * @return connection to zookeeper
125    * @throws IOException if unable to connect to zk or config problem
126    */
127   public static RecoverableZooKeeper connect(Configuration conf, Watcher watcher)
128   throws IOException {
129     String ensemble = ZKConfig.getZKQuorumServersString(conf);
130     return connect(conf, ensemble, watcher);
131   }
132 
133   public static RecoverableZooKeeper connect(Configuration conf, String ensemble,
134       Watcher watcher)
135   throws IOException {
136     return connect(conf, ensemble, watcher, null);
137   }
138 
139   public static RecoverableZooKeeper connect(Configuration conf, String ensemble,
140       Watcher watcher, final String identifier)
141   throws IOException {
142     if(ensemble == null) {
143       throw new IOException("Unable to determine ZooKeeper ensemble");
144     }
145     int timeout = conf.getInt(HConstants.ZK_SESSION_TIMEOUT,
146         HConstants.DEFAULT_ZK_SESSION_TIMEOUT);
147     if (LOG.isTraceEnabled()) {
148       LOG.trace(identifier + " opening connection to ZooKeeper ensemble=" + ensemble);
149     }
150     int retry = conf.getInt("zookeeper.recovery.retry", 3);
151     int retryIntervalMillis =
152       conf.getInt("zookeeper.recovery.retry.intervalmill", 1000);
153     zkDumpConnectionTimeOut = conf.getInt("zookeeper.dump.connection.timeout",
154         1000);
155     return new RecoverableZooKeeper(ensemble, timeout, watcher,
156         retry, retryIntervalMillis, identifier);
157   }
158 
159   /**
160    * Log in the current zookeeper server process using the given configuration
161    * keys for the credential file and login principal.
162    *
163    * <p><strong>This is only applicable when running on secure hbase</strong>
164    * On regular HBase (without security features), this will safely be ignored.
165    * </p>
166    *
167    * @param conf The configuration data to use
168    * @param keytabFileKey Property key used to configure the path to the credential file
169    * @param userNameKey Property key used to configure the login principal
170    * @param hostname Current hostname to use in any credentials
171    * @throws IOException underlying exception from SecurityUtil.login() call
172    */
173   public static void loginServer(Configuration conf, String keytabFileKey,
174       String userNameKey, String hostname) throws IOException {
175     login(conf, keytabFileKey, userNameKey, hostname,
176           ZooKeeperSaslServer.LOGIN_CONTEXT_NAME_KEY,
177           JaasConfiguration.SERVER_KEYTAB_KERBEROS_CONFIG_NAME);
178   }
179 
180   /**
181    * Log in the current zookeeper client using the given configuration
182    * keys for the credential file and login principal.
183    *
184    * <p><strong>This is only applicable when running on secure hbase</strong>
185    * On regular HBase (without security features), this will safely be ignored.
186    * </p>
187    *
188    * @param conf The configuration data to use
189    * @param keytabFileKey Property key used to configure the path to the credential file
190    * @param userNameKey Property key used to configure the login principal
191    * @param hostname Current hostname to use in any credentials
192    * @throws IOException underlying exception from SecurityUtil.login() call
193    */
194   public static void loginClient(Configuration conf, String keytabFileKey,
195       String userNameKey, String hostname) throws IOException {
196     login(conf, keytabFileKey, userNameKey, hostname,
197           ZooKeeperSaslClient.LOGIN_CONTEXT_NAME_KEY,
198           JaasConfiguration.CLIENT_KEYTAB_KERBEROS_CONFIG_NAME);
199   }
200 
201   /**
202    * Log in the current process using the given configuration keys for the
203    * credential file and login principal.
204    *
205    * <p><strong>This is only applicable when running on secure hbase</strong>
206    * On regular HBase (without security features), this will safely be ignored.
207    * </p>
208    *
209    * @param conf The configuration data to use
210    * @param keytabFileKey Property key used to configure the path to the credential file
211    * @param userNameKey Property key used to configure the login principal
212    * @param hostname Current hostname to use in any credentials
213    * @param loginContextProperty property name to expose the entry name
214    * @param loginContextName jaas entry name
215    * @throws IOException underlying exception from SecurityUtil.login() call
216    */
217   private static void login(Configuration conf, String keytabFileKey,
218       String userNameKey, String hostname,
219       String loginContextProperty, String loginContextName)
220       throws IOException {
221     if (!isSecureZooKeeper(conf))
222       return;
223 
224     // User has specified a jaas.conf, keep this one as the good one.
225     // HBASE_OPTS="-Djava.security.auth.login.config=jaas.conf"
226     if (System.getProperty("java.security.auth.login.config") != null)
227       return;
228 
229     // No keytab specified, no auth
230     String keytabFilename = conf.get(keytabFileKey);
231     if (keytabFilename == null) {
232       LOG.warn("no keytab specified for: " + keytabFileKey);
233       return;
234     }
235 
236     String principalConfig = conf.get(userNameKey, System.getProperty("user.name"));
237     String principalName = SecurityUtil.getServerPrincipal(principalConfig, hostname);
238 
239     // Initialize the "jaas.conf" for keyTab/principal,
240     // If keyTab is not specified use the Ticket Cache.
241     // and set the zookeeper login context name.
242     JaasConfiguration jaasConf = new JaasConfiguration(loginContextName,
243         principalName, keytabFilename);
244     javax.security.auth.login.Configuration.setConfiguration(jaasConf);
245     System.setProperty(loginContextProperty, loginContextName);
246   }
247 
248   /**
249    * A JAAS configuration that defines the login modules that we want to use for login.
250    */
251   private static class JaasConfiguration extends javax.security.auth.login.Configuration {
252     private static final String SERVER_KEYTAB_KERBEROS_CONFIG_NAME =
253       "zookeeper-server-keytab-kerberos";
254     private static final String CLIENT_KEYTAB_KERBEROS_CONFIG_NAME =
255       "zookeeper-client-keytab-kerberos";
256 
257     private static final Map<String, String> BASIC_JAAS_OPTIONS =
258       new HashMap<String,String>();
259     static {
260       String jaasEnvVar = System.getenv("HBASE_JAAS_DEBUG");
261       if (jaasEnvVar != null && "true".equalsIgnoreCase(jaasEnvVar)) {
262         BASIC_JAAS_OPTIONS.put("debug", "true");
263       }
264     }
265 
266     private static final Map<String,String> KEYTAB_KERBEROS_OPTIONS =
267       new HashMap<String,String>();
268     static {
269       KEYTAB_KERBEROS_OPTIONS.put("doNotPrompt", "true");
270       KEYTAB_KERBEROS_OPTIONS.put("storeKey", "true");
271       KEYTAB_KERBEROS_OPTIONS.put("refreshKrb5Config", "true");
272       KEYTAB_KERBEROS_OPTIONS.putAll(BASIC_JAAS_OPTIONS);
273     }
274 
275     private static final AppConfigurationEntry KEYTAB_KERBEROS_LOGIN =
276       new AppConfigurationEntry(KerberosUtil.getKrb5LoginModuleName(),
277                                 LoginModuleControlFlag.REQUIRED,
278                                 KEYTAB_KERBEROS_OPTIONS);
279 
280     private static final AppConfigurationEntry[] KEYTAB_KERBEROS_CONF =
281       new AppConfigurationEntry[]{KEYTAB_KERBEROS_LOGIN};
282 
283     private javax.security.auth.login.Configuration baseConfig;
284     private final String loginContextName;
285     private final boolean useTicketCache;
286     private final String keytabFile;
287     private final String principal;
288 
289     public JaasConfiguration(String loginContextName, String principal, String keytabFile) {
290       this(loginContextName, principal, keytabFile, keytabFile == null || keytabFile.length() == 0);
291     }
292 
293     private JaasConfiguration(String loginContextName, String principal,
294                              String keytabFile, boolean useTicketCache) {
295       try {
296         this.baseConfig = javax.security.auth.login.Configuration.getConfiguration();
297       } catch (SecurityException e) {
298         this.baseConfig = null;
299       }
300       this.loginContextName = loginContextName;
301       this.useTicketCache = useTicketCache;
302       this.keytabFile = keytabFile;
303       this.principal = principal;
304       LOG.info("JaasConfiguration loginContextName=" + loginContextName +
305                " principal=" + principal + " useTicketCache=" + useTicketCache +
306                " keytabFile=" + keytabFile);
307     }
308 
309     @Override
310     public AppConfigurationEntry[] getAppConfigurationEntry(String appName) {
311       if (loginContextName.equals(appName)) {
312         if (!useTicketCache) {
313           KEYTAB_KERBEROS_OPTIONS.put("keyTab", keytabFile);
314           KEYTAB_KERBEROS_OPTIONS.put("useKeyTab", "true");
315         }
316         KEYTAB_KERBEROS_OPTIONS.put("principal", principal);
317         KEYTAB_KERBEROS_OPTIONS.put("useTicketCache", useTicketCache ? "true" : "false");
318         return KEYTAB_KERBEROS_CONF;
319       }
320       if (baseConfig != null) return baseConfig.getAppConfigurationEntry(appName);
321       return(null);
322     }
323   }
324 
325   //
326   // Helper methods
327   //
328 
329   /**
330    * Join the prefix znode name with the suffix znode name to generate a proper
331    * full znode name.
332    *
333    * Assumes prefix does not end with slash and suffix does not begin with it.
334    *
335    * @param prefix beginning of znode name
336    * @param suffix ending of znode name
337    * @return result of properly joining prefix with suffix
338    */
339   public static String joinZNode(String prefix, String suffix) {
340     return prefix + ZNODE_PATH_SEPARATOR + suffix;
341   }
342 
343   /**
344    * Returns the full path of the immediate parent of the specified node.
345    * @param node path to get parent of
346    * @return parent of path, null if passed the root node or an invalid node
347    */
348   public static String getParent(String node) {
349     int idx = node.lastIndexOf(ZNODE_PATH_SEPARATOR);
350     return idx <= 0 ? null : node.substring(0, idx);
351   }
352 
353   /**
354    * Get the name of the current node from the specified fully-qualified path.
355    * @param path fully-qualified path
356    * @return name of the current node
357    */
358   public static String getNodeName(String path) {
359     return path.substring(path.lastIndexOf("/")+1);
360   }
361 
362   /**
363    * Get the key to the ZK ensemble for this configuration without
364    * adding a name at the end
365    * @param conf Configuration to use to build the key
366    * @return ensemble key without a name
367    */
368   public static String getZooKeeperClusterKey(Configuration conf) {
369     return getZooKeeperClusterKey(conf, null);
370   }
371 
372   /**
373    * Get the key to the ZK ensemble for this configuration and append
374    * a name at the end
375    * @param conf Configuration to use to build the key
376    * @param name Name that should be appended at the end if not empty or null
377    * @return ensemble key with a name (if any)
378    */
379   public static String getZooKeeperClusterKey(Configuration conf, String name) {
380     String ensemble = conf.get(HConstants.ZOOKEEPER_QUORUM).replaceAll(
381         "[\\t\\n\\x0B\\f\\r]", "");
382     StringBuilder builder = new StringBuilder(ensemble);
383     builder.append(":");
384     builder.append(conf.get(HConstants.ZOOKEEPER_CLIENT_PORT));
385     builder.append(":");
386     builder.append(conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT));
387     if (name != null && !name.isEmpty()) {
388       builder.append(",");
389       builder.append(name);
390     }
391     return builder.toString();
392   }
393 
394   /**
395    * Apply the settings in the given key to the given configuration, this is
396    * used to communicate with distant clusters
397    * @param conf configuration object to configure
398    * @param key string that contains the 3 required configuratins
399    * @throws IOException
400    */
401   public static void applyClusterKeyToConf(Configuration conf, String key)
402       throws IOException{
403     ZKClusterKey zkClusterKey = transformClusterKey(key);
404     conf.set(HConstants.ZOOKEEPER_QUORUM, zkClusterKey.quorumString);
405     conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, zkClusterKey.clientPort);
406     conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, zkClusterKey.znodeParent);
407   }
408 
409   /**
410    * Separate the given key into the three configurations it should contain:
411    * hbase.zookeeper.quorum, hbase.zookeeper.client.port
412    * and zookeeper.znode.parent
413    * @param key
414    * @return the three configuration in the described order
415    * @throws IOException
416    */
417   public static ZKClusterKey transformClusterKey(String key) throws IOException {
418     String[] parts = key.split(":");
419 
420     if (parts.length == 3) {
421       return new ZKClusterKey(parts [0], Integer.parseInt(parts [1]), parts [2]);
422     }
423 
424     if (parts.length > 3) {
425       // The quorum could contain client port in server:clientport format, try to transform more.
426       String zNodeParent = parts [parts.length - 1];
427       String clientPort = parts [parts.length - 2];
428 
429       // The first part length is the total length minus the lengths of other parts and minus 2 ":"
430       int endQuorumIndex = key.length() - zNodeParent.length() - clientPort.length() - 2;
431       String quorumStringInput = key.substring(0, endQuorumIndex);
432       String[] serverHosts = quorumStringInput.split(",");
433 
434       // The common case is that every server has its own client port specified - this means
435       // that (total parts - the ZNodeParent part - the ClientPort part) is equal to
436       // (the number of "," + 1) - "+ 1" because the last server has no ",".
437       if ((parts.length - 2) == (serverHosts.length + 1)) {
438         return new ZKClusterKey(quorumStringInput, Integer.parseInt(clientPort), zNodeParent);
439       }
440 
441       // For the uncommon case that some servers has no port specified, we need to build the
442       // server:clientport list using default client port for servers without specified port.
443       return new ZKClusterKey(
444         ZKConfig.buildQuorumServerString(serverHosts, clientPort),
445         Integer.parseInt(clientPort),
446         zNodeParent);
447     }
448 
449     throw new IOException("Cluster key passed " + key + " is invalid, the format should be:" +
450           HConstants.ZOOKEEPER_QUORUM + ":" + HConstants.ZOOKEEPER_CLIENT_PORT + ":"
451           + HConstants.ZOOKEEPER_ZNODE_PARENT);
452   }
453 
454   /**
455    * Standardize the ZK quorum string: make it a "server:clientport" list, separated by ','
456    * @param quorumStringInput a string contains a list of servers for ZK quorum
457    * @param clientPort the default client port
458    * @return the string for a list of "server:port" separated by ","
459    */
460   @VisibleForTesting
461   public static String standardizeQuorumServerString(String quorumStringInput, String clientPort) {
462     String[] serverHosts = quorumStringInput.split(",");
463     return ZKConfig.buildQuorumServerString(serverHosts, clientPort);
464   }
465 
466   //
467   // Existence checks and watches
468   //
469 
470   /**
471    * Watch the specified znode for delete/create/change events.  The watcher is
472    * set whether or not the node exists.  If the node already exists, the method
473    * returns true.  If the node does not exist, the method returns false.
474    *
475    * @param zkw zk reference
476    * @param znode path of node to watch
477    * @return true if znode exists, false if does not exist or error
478    * @throws KeeperException if unexpected zookeeper exception
479    */
480   public static boolean watchAndCheckExists(ZooKeeperWatcher zkw, String znode)
481   throws KeeperException {
482     try {
483       Stat s = zkw.getRecoverableZooKeeper().exists(znode, zkw);
484       boolean exists = s != null ? true : false;
485       if (exists) {
486         LOG.debug(zkw.prefix("Set watcher on existing znode=" + znode));
487       } else {
488         LOG.debug(zkw.prefix("Set watcher on znode that does not yet exist, " + znode));
489       }
490       return exists;
491     } catch (KeeperException e) {
492       LOG.warn(zkw.prefix("Unable to set watcher on znode " + znode), e);
493       zkw.keeperException(e);
494       return false;
495     } catch (InterruptedException e) {
496       LOG.warn(zkw.prefix("Unable to set watcher on znode " + znode), e);
497       zkw.interruptedException(e);
498       return false;
499     }
500   }
501 
502   /**
503    * Watch the specified znode, but only if exists. Useful when watching
504    * for deletions. Uses .getData() (and handles NoNodeException) instead
505    * of .exists() to accomplish this, as .getData() will only set a watch if
506    * the znode exists.
507    * @param zkw zk reference
508    * @param znode path of node to watch
509    * @return true if the watch is set, false if node does not exists
510    * @throws KeeperException if unexpected zookeeper exception
511    */
512   public static boolean setWatchIfNodeExists(ZooKeeperWatcher zkw, String znode)
513       throws KeeperException {
514     try {
515       zkw.getRecoverableZooKeeper().getData(znode, true, null);
516       return true;
517     } catch (NoNodeException e) {
518       return false;
519     } catch (InterruptedException e) {
520       LOG.warn(zkw.prefix("Unable to set watcher on znode " + znode), e);
521       zkw.interruptedException(e);
522       return false;
523     }
524   }
525 
526   /**
527    * Check if the specified node exists.  Sets no watches.
528    *
529    * @param zkw zk reference
530    * @param znode path of node to watch
531    * @return version of the node if it exists, -1 if does not exist
532    * @throws KeeperException if unexpected zookeeper exception
533    */
534   public static int checkExists(ZooKeeperWatcher zkw, String znode)
535   throws KeeperException {
536     try {
537       Stat s = zkw.getRecoverableZooKeeper().exists(znode, null);
538       return s != null ? s.getVersion() : -1;
539     } catch (KeeperException e) {
540       LOG.warn(zkw.prefix("Unable to set watcher on znode (" + znode + ")"), e);
541       zkw.keeperException(e);
542       return -1;
543     } catch (InterruptedException e) {
544       LOG.warn(zkw.prefix("Unable to set watcher on znode (" + znode + ")"), e);
545       zkw.interruptedException(e);
546       return -1;
547     }
548   }
549 
550   //
551   // Znode listings
552   //
553 
554   /**
555    * Lists the children znodes of the specified znode.  Also sets a watch on
556    * the specified znode which will capture a NodeDeleted event on the specified
557    * znode as well as NodeChildrenChanged if any children of the specified znode
558    * are created or deleted.
559    *
560    * Returns null if the specified node does not exist.  Otherwise returns a
561    * list of children of the specified node.  If the node exists but it has no
562    * children, an empty list will be returned.
563    *
564    * @param zkw zk reference
565    * @param znode path of node to list and watch children of
566    * @return list of children of the specified node, an empty list if the node
567    *          exists but has no children, and null if the node does not exist
568    * @throws KeeperException if unexpected zookeeper exception
569    */
570   public static List<String> listChildrenAndWatchForNewChildren(
571       ZooKeeperWatcher zkw, String znode)
572   throws KeeperException {
573     try {
574       List<String> children = zkw.getRecoverableZooKeeper().getChildren(znode, zkw);
575       return children;
576     } catch(KeeperException.NoNodeException ke) {
577       LOG.debug(zkw.prefix("Unable to list children of znode " + znode + " " +
578           "because node does not exist (not an error)"));
579       return null;
580     } catch (KeeperException e) {
581       LOG.warn(zkw.prefix("Unable to list children of znode " + znode + " "), e);
582       zkw.keeperException(e);
583       return null;
584     } catch (InterruptedException e) {
585       LOG.warn(zkw.prefix("Unable to list children of znode " + znode + " "), e);
586       zkw.interruptedException(e);
587       return null;
588     }
589   }
590 
591   /**
592    * List all the children of the specified znode, setting a watch for children
593    * changes and also setting a watch on every individual child in order to get
594    * the NodeCreated and NodeDeleted events.
595    * @param zkw zookeeper reference
596    * @param znode node to get children of and watch
597    * @return list of znode names, null if the node doesn't exist
598    * @throws KeeperException
599    */
600   public static List<String> listChildrenAndWatchThem(ZooKeeperWatcher zkw,
601       String znode) throws KeeperException {
602     List<String> children = listChildrenAndWatchForNewChildren(zkw, znode);
603     if (children == null) {
604       return null;
605     }
606     for (String child : children) {
607       watchAndCheckExists(zkw, joinZNode(znode, child));
608     }
609     return children;
610   }
611 
612   /**
613    * Lists the children of the specified znode without setting any watches.
614    *
615    * Sets no watches at all, this method is best effort.
616    *
617    * Returns an empty list if the node has no children.  Returns null if the
618    * parent node itself does not exist.
619    *
620    * @param zkw zookeeper reference
621    * @param znode node to get children
622    * @return list of data of children of specified znode, empty if no children,
623    *         null if parent does not exist
624    * @throws KeeperException if unexpected zookeeper exception
625    */
626   public static List<String> listChildrenNoWatch(ZooKeeperWatcher zkw, String znode)
627   throws KeeperException {
628     List<String> children = null;
629     try {
630       // List the children without watching
631       children = zkw.getRecoverableZooKeeper().getChildren(znode, null);
632     } catch(KeeperException.NoNodeException nne) {
633       return null;
634     } catch(InterruptedException ie) {
635       zkw.interruptedException(ie);
636     }
637     return children;
638   }
639 
640   /**
641    * Simple class to hold a node path and node data.
642    * @deprecated Unused
643    */
644   @Deprecated
645   public static class NodeAndData {
646     private String node;
647     private byte [] data;
648     public NodeAndData(String node, byte [] data) {
649       this.node = node;
650       this.data = data;
651     }
652     public String getNode() {
653       return node;
654     }
655     public byte [] getData() {
656       return data;
657     }
658     @Override
659     public String toString() {
660       return node;
661     }
662     public boolean isEmpty() {
663       return (data == null || data.length == 0);
664     }
665   }
666 
667   /**
668    * Checks if the specified znode has any children.  Sets no watches.
669    *
670    * Returns true if the node exists and has children.  Returns false if the
671    * node does not exist or if the node does not have any children.
672    *
673    * Used during master initialization to determine if the master is a
674    * failed-over-to master or the first master during initial cluster startup.
675    * If the directory for regionserver ephemeral nodes is empty then this is
676    * a cluster startup, if not then it is not cluster startup.
677    *
678    * @param zkw zk reference
679    * @param znode path of node to check for children of
680    * @return true if node has children, false if not or node does not exist
681    * @throws KeeperException if unexpected zookeeper exception
682    */
683   public static boolean nodeHasChildren(ZooKeeperWatcher zkw, String znode)
684   throws KeeperException {
685     try {
686       return !zkw.getRecoverableZooKeeper().getChildren(znode, null).isEmpty();
687     } catch(KeeperException.NoNodeException ke) {
688       LOG.debug(zkw.prefix("Unable to list children of znode " + znode + " " +
689       "because node does not exist (not an error)"));
690       return false;
691     } catch (KeeperException e) {
692       LOG.warn(zkw.prefix("Unable to list children of znode " + znode), e);
693       zkw.keeperException(e);
694       return false;
695     } catch (InterruptedException e) {
696       LOG.warn(zkw.prefix("Unable to list children of znode " + znode), e);
697       zkw.interruptedException(e);
698       return false;
699     }
700   }
701 
702   /**
703    * Get the number of children of the specified node.
704    *
705    * If the node does not exist or has no children, returns 0.
706    *
707    * Sets no watches at all.
708    *
709    * @param zkw zk reference
710    * @param znode path of node to count children of
711    * @return number of children of specified node, 0 if none or parent does not
712    *         exist
713    * @throws KeeperException if unexpected zookeeper exception
714    */
715   public static int getNumberOfChildren(ZooKeeperWatcher zkw, String znode)
716   throws KeeperException {
717     try {
718       Stat stat = zkw.getRecoverableZooKeeper().exists(znode, null);
719       return stat == null ? 0 : stat.getNumChildren();
720     } catch(KeeperException e) {
721       LOG.warn(zkw.prefix("Unable to get children of node " + znode));
722       zkw.keeperException(e);
723     } catch(InterruptedException e) {
724       zkw.interruptedException(e);
725     }
726     return 0;
727   }
728 
729   //
730   // Data retrieval
731   //
732 
733   /**
734    * Get znode data. Does not set a watcher.
735    * @return ZNode data, null if the node does not exist or if there is an
736    *  error.
737    */
738   public static byte [] getData(ZooKeeperWatcher zkw, String znode)
739       throws KeeperException, InterruptedException {
740     try {
741       byte [] data = zkw.getRecoverableZooKeeper().getData(znode, null, null);
742       logRetrievedMsg(zkw, znode, data, false);
743       return data;
744     } catch (KeeperException.NoNodeException e) {
745       LOG.debug(zkw.prefix("Unable to get data of znode " + znode + " " +
746           "because node does not exist (not an error)"));
747       return null;
748     } catch (KeeperException e) {
749       LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
750       zkw.keeperException(e);
751       return null;
752     }
753   }
754 
755   /**
756    * Get the data at the specified znode and set a watch.
757    *
758    * Returns the data and sets a watch if the node exists.  Returns null and no
759    * watch is set if the node does not exist or there is an exception.
760    *
761    * @param zkw zk reference
762    * @param znode path of node
763    * @return data of the specified znode, or null
764    * @throws KeeperException if unexpected zookeeper exception
765    */
766   public static byte [] getDataAndWatch(ZooKeeperWatcher zkw, String znode)
767   throws KeeperException {
768     return getDataInternal(zkw, znode, null, true);
769   }
770 
771   /**
772    * Get the data at the specified znode and set a watch.
773    *
774    * Returns the data and sets a watch if the node exists.  Returns null and no
775    * watch is set if the node does not exist or there is an exception.
776    *
777    * @param zkw zk reference
778    * @param znode path of node
779    * @param stat object to populate the version of the znode
780    * @return data of the specified znode, or null
781    * @throws KeeperException if unexpected zookeeper exception
782    */
783   public static byte[] getDataAndWatch(ZooKeeperWatcher zkw, String znode,
784       Stat stat) throws KeeperException {
785     return getDataInternal(zkw, znode, stat, true);
786   }
787 
788   private static byte[] getDataInternal(ZooKeeperWatcher zkw, String znode, Stat stat,
789       boolean watcherSet)
790       throws KeeperException {
791     try {
792       byte [] data = zkw.getRecoverableZooKeeper().getData(znode, zkw, stat);
793       logRetrievedMsg(zkw, znode, data, watcherSet);
794       return data;
795     } catch (KeeperException.NoNodeException e) {
796       // This log can get pretty annoying when we cycle on 100ms waits.
797       // Enable trace if you really want to see it.
798       LOG.trace(zkw.prefix("Unable to get data of znode " + znode + " " +
799         "because node does not exist (not an error)"));
800       return null;
801     } catch (KeeperException e) {
802       LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
803       zkw.keeperException(e);
804       return null;
805     } catch (InterruptedException e) {
806       LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
807       zkw.interruptedException(e);
808       return null;
809     }
810   }
811 
812   /**
813    * Get the data at the specified znode without setting a watch.
814    *
815    * Returns the data if the node exists.  Returns null if the node does not
816    * exist.
817    *
818    * Sets the stats of the node in the passed Stat object.  Pass a null stat if
819    * not interested.
820    *
821    * @param zkw zk reference
822    * @param znode path of node
823    * @param stat node status to get if node exists
824    * @return data of the specified znode, or null if node does not exist
825    * @throws KeeperException if unexpected zookeeper exception
826    */
827   public static byte [] getDataNoWatch(ZooKeeperWatcher zkw, String znode,
828       Stat stat)
829   throws KeeperException {
830     try {
831       byte [] data = zkw.getRecoverableZooKeeper().getData(znode, null, stat);
832       logRetrievedMsg(zkw, znode, data, false);
833       return data;
834     } catch (KeeperException.NoNodeException e) {
835       LOG.debug(zkw.prefix("Unable to get data of znode " + znode + " " +
836           "because node does not exist (not necessarily an error)"));
837       return null;
838     } catch (KeeperException e) {
839       LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
840       zkw.keeperException(e);
841       return null;
842     } catch (InterruptedException e) {
843       LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
844       zkw.interruptedException(e);
845       return null;
846     }
847   }
848 
849   /**
850    * Returns the date of child znodes of the specified znode.  Also sets a watch on
851    * the specified znode which will capture a NodeDeleted event on the specified
852    * znode as well as NodeChildrenChanged if any children of the specified znode
853    * are created or deleted.
854    *
855    * Returns null if the specified node does not exist.  Otherwise returns a
856    * list of children of the specified node.  If the node exists but it has no
857    * children, an empty list will be returned.
858    *
859    * @param zkw zk reference
860    * @param baseNode path of node to list and watch children of
861    * @return list of data of children of the specified node, an empty list if the node
862    *          exists but has no children, and null if the node does not exist
863    * @throws KeeperException if unexpected zookeeper exception
864    * @deprecated Unused
865    */
866   @Deprecated
867   public static List<NodeAndData> getChildDataAndWatchForNewChildren(
868       ZooKeeperWatcher zkw, String baseNode) throws KeeperException {
869     List<String> nodes =
870       ZKUtil.listChildrenAndWatchForNewChildren(zkw, baseNode);
871     if (nodes != null) {
872       List<NodeAndData> newNodes = new ArrayList<NodeAndData>();
873       for (String node : nodes) {
874         String nodePath = ZKUtil.joinZNode(baseNode, node);
875         byte[] data = ZKUtil.getDataAndWatch(zkw, nodePath);
876         newNodes.add(new NodeAndData(nodePath, data));
877       }
878       return newNodes;
879     }
880     return null;
881   }
882 
883   /**
884    * Update the data of an existing node with the expected version to have the
885    * specified data.
886    *
887    * Throws an exception if there is a version mismatch or some other problem.
888    *
889    * Sets no watches under any conditions.
890    *
891    * @param zkw zk reference
892    * @param znode
893    * @param data
894    * @param expectedVersion
895    * @throws KeeperException if unexpected zookeeper exception
896    * @throws KeeperException.BadVersionException if version mismatch
897    * @deprecated Unused
898    */
899   @Deprecated
900   public static void updateExistingNodeData(ZooKeeperWatcher zkw, String znode,
901       byte [] data, int expectedVersion)
902   throws KeeperException {
903     try {
904       zkw.getRecoverableZooKeeper().setData(znode, data, expectedVersion);
905     } catch(InterruptedException ie) {
906       zkw.interruptedException(ie);
907     }
908   }
909 
910   //
911   // Data setting
912   //
913 
914   /**
915    * Sets the data of the existing znode to be the specified data.  Ensures that
916    * the current data has the specified expected version.
917    *
918    * <p>If the node does not exist, a {@link NoNodeException} will be thrown.
919    *
920    * <p>If their is a version mismatch, method returns null.
921    *
922    * <p>No watches are set but setting data will trigger other watchers of this
923    * node.
924    *
925    * <p>If there is another problem, a KeeperException will be thrown.
926    *
927    * @param zkw zk reference
928    * @param znode path of node
929    * @param data data to set for node
930    * @param expectedVersion version expected when setting data
931    * @return true if data set, false if version mismatch
932    * @throws KeeperException if unexpected zookeeper exception
933    */
934   public static boolean setData(ZooKeeperWatcher zkw, String znode,
935       byte [] data, int expectedVersion)
936   throws KeeperException, KeeperException.NoNodeException {
937     try {
938       return zkw.getRecoverableZooKeeper().setData(znode, data, expectedVersion) != null;
939     } catch (InterruptedException e) {
940       zkw.interruptedException(e);
941       return false;
942     }
943   }
944 
945   /**
946    * Set data into node creating node if it doesn't yet exist.
947    * Does not set watch.
948    *
949    * @param zkw zk reference
950    * @param znode path of node
951    * @param data data to set for node
952    * @throws KeeperException
953    */
954   public static void createSetData(final ZooKeeperWatcher zkw, final String znode,
955       final byte [] data)
956   throws KeeperException {
957     if (checkExists(zkw, znode) == -1) {
958       ZKUtil.createWithParents(zkw, znode, data);
959     } else {
960       ZKUtil.setData(zkw, znode, data);
961     }
962   }
963 
964   /**
965    * Sets the data of the existing znode to be the specified data.  The node
966    * must exist but no checks are done on the existing data or version.
967    *
968    * <p>If the node does not exist, a {@link NoNodeException} will be thrown.
969    *
970    * <p>No watches are set but setting data will trigger other watchers of this
971    * node.
972    *
973    * <p>If there is another problem, a KeeperException will be thrown.
974    *
975    * @param zkw zk reference
976    * @param znode path of node
977    * @param data data to set for node
978    * @throws KeeperException if unexpected zookeeper exception
979    */
980   public static void setData(ZooKeeperWatcher zkw, String znode, byte [] data)
981   throws KeeperException, KeeperException.NoNodeException {
982     setData(zkw, (SetData)ZKUtilOp.setData(znode, data));
983   }
984 
985   private static void setData(ZooKeeperWatcher zkw, SetData setData)
986   throws KeeperException, KeeperException.NoNodeException {
987     SetDataRequest sd = (SetDataRequest)toZooKeeperOp(zkw, setData).toRequestRecord();
988     setData(zkw, sd.getPath(), sd.getData(), sd.getVersion());
989   }
990 
991   /**
992    * Returns whether or not secure authentication is enabled
993    * (whether <code>hbase.security.authentication</code> is set to
994    * <code>kerberos</code>.
995    */
996   public static boolean isSecureZooKeeper(Configuration conf) {
997     // Detection for embedded HBase client with jaas configuration
998     // defined for third party programs.
999     try {
1000       javax.security.auth.login.Configuration testConfig =
1001           javax.security.auth.login.Configuration.getConfiguration();
1002       if (testConfig.getAppConfigurationEntry("Client") == null
1003           && testConfig.getAppConfigurationEntry(
1004             JaasConfiguration.CLIENT_KEYTAB_KERBEROS_CONFIG_NAME) == null
1005           && testConfig.getAppConfigurationEntry(
1006               JaasConfiguration.SERVER_KEYTAB_KERBEROS_CONFIG_NAME) == null) {
1007         return false;
1008       }
1009     } catch(Exception e) {
1010       // No Jaas configuration defined.
1011       return false;
1012     }
1013 
1014     // Master & RSs uses hbase.zookeeper.client.*
1015     return "kerberos".equalsIgnoreCase(conf.get("hbase.security.authentication"));
1016   }
1017 
1018   private static ArrayList<ACL> createACL(ZooKeeperWatcher zkw, String node) {
1019     return createACL(zkw, node, isSecureZooKeeper(zkw.getConfiguration()));
1020   }
1021 
1022   public static ArrayList<ACL> createACL(ZooKeeperWatcher zkw, String node,
1023     boolean isSecureZooKeeper) {
1024     if (!node.startsWith(zkw.baseZNode)) {
1025       return Ids.OPEN_ACL_UNSAFE;
1026     }
1027     if (isSecureZooKeeper) {
1028       String superUser = zkw.getConfiguration().get("hbase.superuser");
1029       ArrayList<ACL> acls = new ArrayList<ACL>();
1030       // add permission to hbase supper user
1031       if (superUser != null) {
1032         acls.add(new ACL(Perms.ALL, new Id("auth", superUser)));
1033       }
1034       // Certain znodes are accessed directly by the client,
1035       // so they must be readable by non-authenticated clients
1036       if (zkw.isClientReadable(node)) {
1037         acls.addAll(Ids.CREATOR_ALL_ACL);
1038         acls.addAll(Ids.READ_ACL_UNSAFE);
1039       } else {
1040         acls.addAll(Ids.CREATOR_ALL_ACL);
1041       }
1042       return acls;
1043     } else {
1044       return Ids.OPEN_ACL_UNSAFE;
1045     }
1046   }
1047 
1048   //
1049   // Node creation
1050   //
1051 
1052   /**
1053    *
1054    * Set the specified znode to be an ephemeral node carrying the specified
1055    * data.
1056    *
1057    * If the node is created successfully, a watcher is also set on the node.
1058    *
1059    * If the node is not created successfully because it already exists, this
1060    * method will also set a watcher on the node.
1061    *
1062    * If there is another problem, a KeeperException will be thrown.
1063    *
1064    * @param zkw zk reference
1065    * @param znode path of node
1066    * @param data data of node
1067    * @return true if node created, false if not, watch set in both cases
1068    * @throws KeeperException if unexpected zookeeper exception
1069    */
1070   public static boolean createEphemeralNodeAndWatch(ZooKeeperWatcher zkw,
1071       String znode, byte [] data)
1072   throws KeeperException {
1073     boolean ret = true;
1074     try {
1075       zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode),
1076           CreateMode.EPHEMERAL);
1077     } catch (KeeperException.NodeExistsException nee) {
1078       ret = false;
1079     } catch (InterruptedException e) {
1080       LOG.info("Interrupted", e);
1081       Thread.currentThread().interrupt();
1082     }
1083     if(!watchAndCheckExists(zkw, znode)) {
1084       // It did exist but now it doesn't, try again
1085       return createEphemeralNodeAndWatch(zkw, znode, data);
1086     }
1087     return ret;
1088   }
1089 
1090   /**
1091    * Creates the specified znode to be a persistent node carrying the specified
1092    * data.
1093    *
1094    * Returns true if the node was successfully created, false if the node
1095    * already existed.
1096    *
1097    * If the node is created successfully, a watcher is also set on the node.
1098    *
1099    * If the node is not created successfully because it already exists, this
1100    * method will also set a watcher on the node but return false.
1101    *
1102    * If there is another problem, a KeeperException will be thrown.
1103    *
1104    * @param zkw zk reference
1105    * @param znode path of node
1106    * @param data data of node
1107    * @return true if node created, false if not, watch set in both cases
1108    * @throws KeeperException if unexpected zookeeper exception
1109    */
1110   public static boolean createNodeIfNotExistsAndWatch(
1111       ZooKeeperWatcher zkw, String znode, byte [] data)
1112   throws KeeperException {
1113     boolean ret = true;
1114     try {
1115       zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode),
1116           CreateMode.PERSISTENT);
1117     } catch (KeeperException.NodeExistsException nee) {
1118       ret = false;
1119     } catch (InterruptedException e) {
1120       zkw.interruptedException(e);
1121       return false;
1122     }
1123     try {
1124       zkw.getRecoverableZooKeeper().exists(znode, zkw);
1125     } catch (InterruptedException e) {
1126       zkw.interruptedException(e);
1127       return false;
1128     }
1129     return ret;
1130   }
1131 
1132   /**
1133    * Creates the specified znode with the specified data but does not watch it.
1134    *
1135    * Returns the znode of the newly created node
1136    *
1137    * If there is another problem, a KeeperException will be thrown.
1138    *
1139    * @param zkw zk reference
1140    * @param znode path of node
1141    * @param data data of node
1142    * @param createMode specifying whether the node to be created is ephemeral and/or sequential
1143    * @return true name of the newly created znode or null
1144    * @throws KeeperException if unexpected zookeeper exception
1145    */
1146   public static String createNodeIfNotExistsNoWatch(ZooKeeperWatcher zkw, String znode,
1147       byte[] data, CreateMode createMode) throws KeeperException {
1148 
1149     String createdZNode = null;
1150     try {
1151       createdZNode = zkw.getRecoverableZooKeeper().create(znode, data,
1152           createACL(zkw, znode), createMode);
1153     } catch (KeeperException.NodeExistsException nee) {
1154       return znode;
1155     } catch (InterruptedException e) {
1156       zkw.interruptedException(e);
1157       return null;
1158     }
1159     return createdZNode;
1160   }
1161 
1162   /**
1163    * Creates the specified node with the specified data and watches it.
1164    *
1165    * <p>Throws an exception if the node already exists.
1166    *
1167    * <p>The node created is persistent and open access.
1168    *
1169    * <p>Returns the version number of the created node if successful.
1170    *
1171    * @param zkw zk reference
1172    * @param znode path of node to create
1173    * @param data data of node to create
1174    * @return version of node created
1175    * @throws KeeperException if unexpected zookeeper exception
1176    * @throws KeeperException.NodeExistsException if node already exists
1177    */
1178   public static int createAndWatch(ZooKeeperWatcher zkw,
1179       String znode, byte [] data)
1180   throws KeeperException, KeeperException.NodeExistsException {
1181     try {
1182       zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode),
1183           CreateMode.PERSISTENT);
1184       Stat stat = zkw.getRecoverableZooKeeper().exists(znode, zkw);
1185       if (stat == null){
1186         // Likely a race condition. Someone deleted the znode.
1187         throw KeeperException.create(KeeperException.Code.SYSTEMERROR,
1188             "ZK.exists returned null (i.e.: znode does not exist) for znode=" + znode);
1189       }
1190      return stat.getVersion();
1191     } catch (InterruptedException e) {
1192       zkw.interruptedException(e);
1193       return -1;
1194     }
1195   }
1196 
1197   /**
1198    * Async creates the specified node with the specified data.
1199    *
1200    * <p>Throws an exception if the node already exists.
1201    *
1202    * <p>The node created is persistent and open access.
1203    *
1204    * @param zkw zk reference
1205    * @param znode path of node to create
1206    * @param data data of node to create
1207    * @param cb
1208    * @param ctx
1209    */
1210   public static void asyncCreate(ZooKeeperWatcher zkw,
1211       String znode, byte [] data, final AsyncCallback.StringCallback cb,
1212       final Object ctx) {
1213     zkw.getRecoverableZooKeeper().getZooKeeper().create(znode, data,
1214         createACL(zkw, znode), CreateMode.PERSISTENT, cb, ctx);
1215   }
1216 
1217   /**
1218    * Creates the specified node, iff the node does not exist.  Does not set a
1219    * watch and fails silently if the node already exists.
1220    *
1221    * The node created is persistent and open access.
1222    *
1223    * @param zkw zk reference
1224    * @param znode path of node
1225    * @throws KeeperException if unexpected zookeeper exception
1226    */
1227   public static void createAndFailSilent(ZooKeeperWatcher zkw,
1228       String znode) throws KeeperException {
1229     createAndFailSilent(zkw, znode, new byte[0]);
1230   }
1231 
1232   /**
1233    * Creates the specified node containing specified data, iff the node does not exist.  Does
1234    * not set a watch and fails silently if the node already exists.
1235    *
1236    * The node created is persistent and open access.
1237    *
1238    * @param zkw zk reference
1239    * @param znode path of node
1240    * @param data a byte array data to store in the znode
1241    * @throws KeeperException if unexpected zookeeper exception
1242    */
1243   public static void createAndFailSilent(ZooKeeperWatcher zkw,
1244       String znode, byte[] data)
1245   throws KeeperException {
1246     createAndFailSilent(zkw,
1247         (CreateAndFailSilent)ZKUtilOp.createAndFailSilent(znode, data));
1248   }
1249 
1250   private static void createAndFailSilent(ZooKeeperWatcher zkw, CreateAndFailSilent cafs)
1251   throws KeeperException {
1252     CreateRequest create = (CreateRequest)toZooKeeperOp(zkw, cafs).toRequestRecord();
1253     String znode = create.getPath();
1254     try {
1255       RecoverableZooKeeper zk = zkw.getRecoverableZooKeeper();
1256       if (zk.exists(znode, false) == null) {
1257         zk.create(znode, create.getData(), create.getAcl(), CreateMode.fromFlag(create.getFlags()));
1258       }
1259     } catch(KeeperException.NodeExistsException nee) {
1260     } catch(KeeperException.NoAuthException nee){
1261       try {
1262         if (null == zkw.getRecoverableZooKeeper().exists(znode, false)) {
1263           // If we failed to create the file and it does not already exist.
1264           throw(nee);
1265         }
1266       } catch (InterruptedException ie) {
1267         zkw.interruptedException(ie);
1268       }
1269     } catch(InterruptedException ie) {
1270       zkw.interruptedException(ie);
1271     }
1272   }
1273 
1274   /**
1275    * Creates the specified node and all parent nodes required for it to exist.
1276    *
1277    * No watches are set and no errors are thrown if the node already exists.
1278    *
1279    * The nodes created are persistent and open access.
1280    *
1281    * @param zkw zk reference
1282    * @param znode path of node
1283    * @throws KeeperException if unexpected zookeeper exception
1284    */
1285   public static void createWithParents(ZooKeeperWatcher zkw, String znode)
1286   throws KeeperException {
1287     createWithParents(zkw, znode, new byte[0]);
1288   }
1289 
1290   /**
1291    * Creates the specified node and all parent nodes required for it to exist.  The creation of
1292    * parent znodes is not atomic with the leafe znode creation but the data is written atomically
1293    * when the leaf node is created.
1294    *
1295    * No watches are set and no errors are thrown if the node already exists.
1296    *
1297    * The nodes created are persistent and open access.
1298    *
1299    * @param zkw zk reference
1300    * @param znode path of node
1301    * @throws KeeperException if unexpected zookeeper exception
1302    */
1303   public static void createWithParents(ZooKeeperWatcher zkw, String znode, byte[] data)
1304   throws KeeperException {
1305     try {
1306       if(znode == null) {
1307         return;
1308       }
1309       zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode),
1310           CreateMode.PERSISTENT);
1311     } catch(KeeperException.NodeExistsException nee) {
1312       return;
1313     } catch(KeeperException.NoNodeException nne) {
1314       createWithParents(zkw, getParent(znode));
1315       createWithParents(zkw, znode, data);
1316     } catch(InterruptedException ie) {
1317       zkw.interruptedException(ie);
1318     }
1319   }
1320 
1321   //
1322   // Deletes
1323   //
1324 
1325   /**
1326    * Delete the specified node.  Sets no watches.  Throws all exceptions.
1327    */
1328   public static void deleteNode(ZooKeeperWatcher zkw, String node)
1329   throws KeeperException {
1330     deleteNode(zkw, node, -1);
1331   }
1332 
1333   /**
1334    * Delete the specified node with the specified version.  Sets no watches.
1335    * Throws all exceptions.
1336    */
1337   public static boolean deleteNode(ZooKeeperWatcher zkw, String node,
1338       int version)
1339   throws KeeperException {
1340     try {
1341       zkw.getRecoverableZooKeeper().delete(node, version);
1342       return true;
1343     } catch(KeeperException.BadVersionException bve) {
1344       return false;
1345     } catch(InterruptedException ie) {
1346       zkw.interruptedException(ie);
1347       return false;
1348     }
1349   }
1350 
1351   /**
1352    * Deletes the specified node.  Fails silent if the node does not exist.
1353    * @param zkw
1354    * @param node
1355    * @throws KeeperException
1356    */
1357   public static void deleteNodeFailSilent(ZooKeeperWatcher zkw, String node)
1358   throws KeeperException {
1359     deleteNodeFailSilent(zkw,
1360       (DeleteNodeFailSilent)ZKUtilOp.deleteNodeFailSilent(node));
1361   }
1362 
1363   private static void deleteNodeFailSilent(ZooKeeperWatcher zkw,
1364       DeleteNodeFailSilent dnfs) throws KeeperException {
1365     DeleteRequest delete = (DeleteRequest)toZooKeeperOp(zkw, dnfs).toRequestRecord();
1366     try {
1367       zkw.getRecoverableZooKeeper().delete(delete.getPath(), delete.getVersion());
1368     } catch(KeeperException.NoNodeException nne) {
1369     } catch(InterruptedException ie) {
1370       zkw.interruptedException(ie);
1371     }
1372   }
1373 
1374 
1375   /**
1376    * Delete the specified node and all of it's children.
1377    * <p>
1378    * If the node does not exist, just returns.
1379    * <p>
1380    * Sets no watches. Throws all exceptions besides dealing with deletion of
1381    * children.
1382    */
1383   public static void deleteNodeRecursively(ZooKeeperWatcher zkw, String node)
1384   throws KeeperException {
1385     deleteNodeRecursivelyMultiOrSequential(zkw, true, node);
1386   }
1387 
1388   /**
1389    * Delete all the children of the specified node but not the node itself.
1390    *
1391    * Sets no watches.  Throws all exceptions besides dealing with deletion of
1392    * children.
1393    *
1394    * If hbase.zookeeper.useMulti is true, use ZooKeeper's multi-update functionality.
1395    * Otherwise, run the list of operations sequentially.
1396    *
1397    * @throws KeeperException
1398    */
1399   public static void deleteChildrenRecursively(ZooKeeperWatcher zkw, String node)
1400       throws KeeperException {
1401     deleteChildrenRecursivelyMultiOrSequential(zkw, true, node);
1402   }
1403 
1404   /**
1405    * Delete all the children of the specified node but not the node itself. This
1406    * will first traverse the znode tree for listing the children and then delete
1407    * these znodes using multi-update api or sequential based on the specified
1408    * configurations.
1409    * <p>
1410    * Sets no watches. Throws all exceptions besides dealing with deletion of
1411    * children.
1412    * <p>
1413    * If hbase.zookeeper.useMulti is true, use ZooKeeper's multi-update
1414    * functionality. Otherwise, run the list of operations sequentially.
1415    * <p>
1416    * If all of the following are true:
1417    * <ul>
1418    * <li>runSequentialOnMultiFailure is true
1419    * <li>hbase.zookeeper.useMulti is true
1420    * </ul>
1421    * on calling multi, we get a ZooKeeper exception that can be handled by a
1422    * sequential call(*), we retry the operations one-by-one (sequentially).
1423    *
1424    * @param zkw
1425    *          - zk reference
1426    * @param runSequentialOnMultiFailure
1427    *          - if true when we get a ZooKeeper exception that could retry the
1428    *          operations one-by-one (sequentially)
1429    * @param pathRoots
1430    *          - path of the parent node(s)
1431    * @throws KeeperException.NotEmptyException
1432    *           if node has children while deleting
1433    * @throws KeeperException
1434    *           if unexpected ZooKeeper exception
1435    * @throws IllegalArgumentException
1436    *           if an invalid path is specified
1437    */
1438   public static void deleteChildrenRecursivelyMultiOrSequential(
1439       ZooKeeperWatcher zkw, boolean runSequentialOnMultiFailure,
1440       String... pathRoots) throws KeeperException {
1441     if (pathRoots == null || pathRoots.length <= 0) {
1442       LOG.warn("Given path is not valid!");
1443       return;
1444     }
1445     List<ZKUtilOp> ops = new ArrayList<ZKUtil.ZKUtilOp>();
1446     for (String eachRoot : pathRoots) {
1447       List<String> children = listChildrenBFSNoWatch(zkw, eachRoot);
1448       // Delete the leaves first and eventually get rid of the root
1449       for (int i = children.size() - 1; i >= 0; --i) {
1450         ops.add(ZKUtilOp.deleteNodeFailSilent(children.get(i)));
1451       }
1452     }
1453     // atleast one element should exist
1454     if (ops.size() > 0) {
1455       multiOrSequential(zkw, ops, runSequentialOnMultiFailure);
1456     }
1457   }
1458 
1459   /**
1460    * Delete the specified node and its children. This traverse the
1461    * znode tree for listing the children and then delete
1462    * these znodes including the parent using multi-update api or
1463    * sequential based on the specified configurations.
1464    * <p>
1465    * Sets no watches. Throws all exceptions besides dealing with deletion of
1466    * children.
1467    * <p>
1468    * If hbase.zookeeper.useMulti is true, use ZooKeeper's multi-update
1469    * functionality. Otherwise, run the list of operations sequentially.
1470    * <p>
1471    * If all of the following are true:
1472    * <ul>
1473    * <li>runSequentialOnMultiFailure is true
1474    * <li>hbase.zookeeper.useMulti is true
1475    * </ul>
1476    * on calling multi, we get a ZooKeeper exception that can be handled by a
1477    * sequential call(*), we retry the operations one-by-one (sequentially).
1478    *
1479    * @param zkw
1480    *          - zk reference
1481    * @param runSequentialOnMultiFailure
1482    *          - if true when we get a ZooKeeper exception that could retry the
1483    *          operations one-by-one (sequentially)
1484    * @param pathRoots
1485    *          - path of the parent node(s)
1486    * @throws KeeperException.NotEmptyException
1487    *           if node has children while deleting
1488    * @throws KeeperException
1489    *           if unexpected ZooKeeper exception
1490    * @throws IllegalArgumentException
1491    *           if an invalid path is specified
1492    */
1493   public static void deleteNodeRecursivelyMultiOrSequential(ZooKeeperWatcher zkw,
1494       boolean runSequentialOnMultiFailure, String... pathRoots) throws KeeperException {
1495     if (pathRoots == null || pathRoots.length <= 0) {
1496       LOG.warn("Given path is not valid!");
1497       return;
1498     }
1499     List<ZKUtilOp> ops = new ArrayList<ZKUtil.ZKUtilOp>();
1500     for (String eachRoot : pathRoots) {
1501       // Zookeeper Watches are one time triggers; When children of parent nodes are deleted
1502       // recursively, must set another watch, get notified of delete node
1503       List<String> children = listChildrenBFSAndWatchThem(zkw, eachRoot);
1504       // Delete the leaves first and eventually get rid of the root
1505       for (int i = children.size() - 1; i >= 0; --i) {
1506         ops.add(ZKUtilOp.deleteNodeFailSilent(children.get(i)));
1507       }
1508       try {
1509         if (zkw.getRecoverableZooKeeper().exists(eachRoot, zkw) != null) {
1510           ops.add(ZKUtilOp.deleteNodeFailSilent(eachRoot));
1511         }
1512       } catch (InterruptedException e) {
1513         zkw.interruptedException(e);
1514       }
1515     }
1516     // atleast one element should exist
1517     if (ops.size() > 0) {
1518       multiOrSequential(zkw, ops, runSequentialOnMultiFailure);
1519     }
1520   }
1521 
1522   /**
1523    * BFS Traversal of all the children under path, with the entries in the list,
1524    * in the same order as that of the traversal. Lists all the children without
1525    * setting any watches.
1526    *
1527    * @param zkw
1528    *          - zk reference
1529    * @param znode
1530    *          - path of node
1531    * @return list of children znodes under the path
1532    * @throws KeeperException
1533    *           if unexpected ZooKeeper exception
1534    */
1535   private static List<String> listChildrenBFSNoWatch(ZooKeeperWatcher zkw,
1536       final String znode) throws KeeperException {
1537     Deque<String> queue = new LinkedList<String>();
1538     List<String> tree = new ArrayList<String>();
1539     queue.add(znode);
1540     while (true) {
1541       String node = queue.pollFirst();
1542       if (node == null) {
1543         break;
1544       }
1545       List<String> children = listChildrenNoWatch(zkw, node);
1546       if (children == null) {
1547         continue;
1548       }
1549       for (final String child : children) {
1550         final String childPath = node + "/" + child;
1551         queue.add(childPath);
1552         tree.add(childPath);
1553       }
1554     }
1555     return tree;
1556   }
1557 
1558   /**
1559    * BFS Traversal of all the children under path, with the entries in the list,
1560    * in the same order as that of the traversal.
1561    * Lists all the children and set watches on to them.
1562    *
1563    * @param zkw
1564    *          - zk reference
1565    * @param znode
1566    *          - path of node
1567    * @return list of children znodes under the path
1568    * @throws KeeperException
1569    *           if unexpected ZooKeeper exception
1570    */
1571   private static List<String> listChildrenBFSAndWatchThem(ZooKeeperWatcher zkw, final String znode)
1572       throws KeeperException {
1573     Deque<String> queue = new LinkedList<String>();
1574     List<String> tree = new ArrayList<String>();
1575     queue.add(znode);
1576     while (true) {
1577       String node = queue.pollFirst();
1578       if (node == null) {
1579         break;
1580       }
1581       List<String> children = listChildrenAndWatchThem(zkw, node);
1582       if (children == null) {
1583         continue;
1584       }
1585       for (final String child : children) {
1586         final String childPath = node + "/" + child;
1587         queue.add(childPath);
1588         tree.add(childPath);
1589       }
1590     }
1591     return tree;
1592   }
1593 
1594   /**
1595    * Represents an action taken by ZKUtil, e.g. createAndFailSilent.
1596    * These actions are higher-level than ZKOp actions, which represent
1597    * individual actions in the ZooKeeper API, like create.
1598    */
1599   public abstract static class ZKUtilOp {
1600     private String path;
1601 
1602     private ZKUtilOp(String path) {
1603       this.path = path;
1604     }
1605 
1606     /**
1607      * @return a createAndFailSilent ZKUtilOp
1608      */
1609     public static ZKUtilOp createAndFailSilent(String path, byte[] data) {
1610       return new CreateAndFailSilent(path, data);
1611     }
1612 
1613     /**
1614      * @return a deleteNodeFailSilent ZKUtilOP
1615      */
1616     public static ZKUtilOp deleteNodeFailSilent(String path) {
1617       return new DeleteNodeFailSilent(path);
1618     }
1619 
1620     /**
1621      * @return a setData ZKUtilOp
1622      */
1623     public static ZKUtilOp setData(String path, byte [] data) {
1624       return new SetData(path, data);
1625     }
1626 
1627     /**
1628      * @return path to znode where the ZKOp will occur
1629      */
1630     public String getPath() {
1631       return path;
1632     }
1633 
1634     /**
1635      * ZKUtilOp representing createAndFailSilent in ZooKeeper
1636      * (attempt to create node, ignore error if already exists)
1637      */
1638     public static class CreateAndFailSilent extends ZKUtilOp {
1639       private byte [] data;
1640 
1641       private CreateAndFailSilent(String path, byte [] data) {
1642         super(path);
1643         this.data = data;
1644       }
1645 
1646       public byte[] getData() {
1647         return data;
1648       }
1649 
1650       @Override
1651       public boolean equals(Object o) {
1652         if (this == o) return true;
1653         if (!(o instanceof CreateAndFailSilent)) return false;
1654 
1655         CreateAndFailSilent op = (CreateAndFailSilent) o;
1656         return getPath().equals(op.getPath()) && Arrays.equals(data, op.data);
1657       }
1658 
1659       @Override
1660       public int hashCode() {
1661         int ret = 17 + getPath().hashCode() * 31;
1662         return ret * 31 + Bytes.hashCode(data);
1663       }
1664     }
1665 
1666     /**
1667      * ZKUtilOp representing deleteNodeFailSilent in ZooKeeper
1668      * (attempt to delete node, ignore error if node doesn't exist)
1669      */
1670     public static class DeleteNodeFailSilent extends ZKUtilOp {
1671       private DeleteNodeFailSilent(String path) {
1672         super(path);
1673       }
1674 
1675       @Override
1676       public boolean equals(Object o) {
1677         if (this == o) return true;
1678         if (!(o instanceof DeleteNodeFailSilent)) return false;
1679 
1680         return super.equals(o);
1681       }
1682 
1683       @Override
1684       public int hashCode() {
1685         return getPath().hashCode();
1686       }
1687     }
1688 
1689     /**
1690      * ZKUtilOp representing setData in ZooKeeper
1691      */
1692     public static class SetData extends ZKUtilOp {
1693       private byte [] data;
1694 
1695       private SetData(String path, byte [] data) {
1696         super(path);
1697         this.data = data;
1698       }
1699 
1700       public byte[] getData() {
1701         return data;
1702       }
1703 
1704       @Override
1705       public boolean equals(Object o) {
1706         if (this == o) return true;
1707         if (!(o instanceof SetData)) return false;
1708 
1709         SetData op = (SetData) o;
1710         return getPath().equals(op.getPath()) && Arrays.equals(data, op.data);
1711       }
1712 
1713       @Override
1714       public int hashCode() {
1715         int ret = getPath().hashCode();
1716         return ret * 31 + Bytes.hashCode(data);
1717       }
1718     }
1719   }
1720 
1721   /**
1722    * Convert from ZKUtilOp to ZKOp
1723    */
1724   private static Op toZooKeeperOp(ZooKeeperWatcher zkw, ZKUtilOp op)
1725   throws UnsupportedOperationException {
1726     if(op == null) return null;
1727 
1728     if (op instanceof CreateAndFailSilent) {
1729       CreateAndFailSilent cafs = (CreateAndFailSilent)op;
1730       return Op.create(cafs.getPath(), cafs.getData(), createACL(zkw, cafs.getPath()),
1731         CreateMode.PERSISTENT);
1732     } else if (op instanceof DeleteNodeFailSilent) {
1733       DeleteNodeFailSilent dnfs = (DeleteNodeFailSilent)op;
1734       return Op.delete(dnfs.getPath(), -1);
1735     } else if (op instanceof SetData) {
1736       SetData sd = (SetData)op;
1737       return Op.setData(sd.getPath(), sd.getData(), -1);
1738     } else {
1739       throw new UnsupportedOperationException("Unexpected ZKUtilOp type: "
1740         + op.getClass().getName());
1741     }
1742   }
1743 
1744   /**
1745    * If hbase.zookeeper.useMulti is true, use ZooKeeper's multi-update functionality.
1746    * Otherwise, run the list of operations sequentially.
1747    *
1748    * If all of the following are true:
1749    * - runSequentialOnMultiFailure is true
1750    * - hbase.zookeeper.useMulti is true
1751    * - on calling multi, we get a ZooKeeper exception that can be handled by a sequential call(*)
1752    * Then:
1753    * - we retry the operations one-by-one (sequentially)
1754    *
1755    * Note *: an example is receiving a NodeExistsException from a "create" call.  Without multi,
1756    * a user could call "createAndFailSilent" to ensure that a node exists if they don't care who
1757    * actually created the node (i.e. the NodeExistsException from ZooKeeper is caught).
1758    * This will cause all operations in the multi to fail, however, because
1759    * the NodeExistsException that zk.create throws will fail the multi transaction.
1760    * In this case, if the previous conditions hold, the commands are run sequentially, which should
1761    * result in the correct final state, but means that the operations will not run atomically.
1762    *
1763    * @throws KeeperException
1764    */
1765   public static void multiOrSequential(ZooKeeperWatcher zkw, List<ZKUtilOp> ops,
1766       boolean runSequentialOnMultiFailure) throws KeeperException {
1767     if (ops == null) return;
1768     boolean useMulti = zkw.getConfiguration().getBoolean(HConstants.ZOOKEEPER_USEMULTI, false);
1769 
1770     if (useMulti) {
1771       List<Op> zkOps = new LinkedList<Op>();
1772       for (ZKUtilOp op : ops) {
1773         zkOps.add(toZooKeeperOp(zkw, op));
1774       }
1775       try {
1776         zkw.getRecoverableZooKeeper().multi(zkOps);
1777       } catch (KeeperException ke) {
1778        switch (ke.code()) {
1779          case NODEEXISTS:
1780          case NONODE:
1781          case BADVERSION:
1782          case NOAUTH:
1783            // if we get an exception that could be solved by running sequentially
1784            // (and the client asked us to), then break out and run sequentially
1785            if (runSequentialOnMultiFailure) {
1786              LOG.info("On call to ZK.multi, received exception: " + ke.toString() + "."
1787                + "  Attempting to run operations sequentially because"
1788                + " runSequentialOnMultiFailure is: " + runSequentialOnMultiFailure + ".");
1789              processSequentially(zkw, ops);
1790              break;
1791            }
1792           default:
1793             throw ke;
1794         }
1795       } catch (InterruptedException ie) {
1796         zkw.interruptedException(ie);
1797       }
1798     } else {
1799       // run sequentially
1800       processSequentially(zkw, ops);
1801     }
1802 
1803   }
1804 
1805   private static void processSequentially(ZooKeeperWatcher zkw, List<ZKUtilOp> ops)
1806       throws KeeperException, NoNodeException {
1807     for (ZKUtilOp op : ops) {
1808       if (op instanceof CreateAndFailSilent) {
1809         createAndFailSilent(zkw, (CreateAndFailSilent) op);
1810       } else if (op instanceof DeleteNodeFailSilent) {
1811         deleteNodeFailSilent(zkw, (DeleteNodeFailSilent) op);
1812       } else if (op instanceof SetData) {
1813         setData(zkw, (SetData) op);
1814       } else {
1815         throw new UnsupportedOperationException("Unexpected ZKUtilOp type: "
1816             + op.getClass().getName());
1817       }
1818     }
1819   }
1820 
1821   //
1822   // ZooKeeper cluster information
1823   //
1824 
1825   /** @return String dump of everything in ZooKeeper. */
1826   public static String dump(ZooKeeperWatcher zkw) {
1827     StringBuilder sb = new StringBuilder();
1828     try {
1829       sb.append("HBase is rooted at ").append(zkw.baseZNode);
1830       sb.append("\nActive master address: ");
1831       try {
1832         sb.append(MasterAddressTracker.getMasterAddress(zkw));
1833       } catch (IOException e) {
1834         sb.append("<<FAILED LOOKUP: " + e.getMessage() + ">>");
1835       }
1836       sb.append("\nBackup master addresses:");
1837       for (String child : listChildrenNoWatch(zkw,
1838                                               zkw.backupMasterAddressesZNode)) {
1839         sb.append("\n ").append(child);
1840       }
1841       sb.append("\nRegion server holding hbase:meta: "
1842         + new MetaTableLocator().getMetaRegionLocation(zkw));
1843       Configuration conf = HBaseConfiguration.create();
1844       int numMetaReplicas = conf.getInt(HConstants.META_REPLICAS_NUM,
1845                HConstants.DEFAULT_META_REPLICA_NUM);
1846       for (int i = 1; i < numMetaReplicas; i++) {
1847         sb.append("\nRegion server holding hbase:meta, replicaId " + i + " "
1848                     + new MetaTableLocator().getMetaRegionLocation(zkw, i));
1849       }
1850       sb.append("\nRegion servers:");
1851       for (String child : listChildrenNoWatch(zkw, zkw.rsZNode)) {
1852         sb.append("\n ").append(child);
1853       }
1854       try {
1855         getReplicationZnodesDump(zkw, sb);
1856       } catch (KeeperException ke) {
1857         LOG.warn("Couldn't get the replication znode dump", ke);
1858       }
1859       sb.append("\nQuorum Server Statistics:");
1860       String[] servers = zkw.getQuorum().split(",");
1861       for (String server : servers) {
1862         sb.append("\n ").append(server);
1863         try {
1864           String[] stat = getServerStats(server, ZKUtil.zkDumpConnectionTimeOut);
1865 
1866           if (stat == null) {
1867             sb.append("[Error] invalid quorum server: " + server);
1868             break;
1869           }
1870 
1871           for (String s : stat) {
1872             sb.append("\n  ").append(s);
1873           }
1874         } catch (Exception e) {
1875           sb.append("\n  ERROR: ").append(e.getMessage());
1876         }
1877       }
1878     } catch (KeeperException ke) {
1879       sb.append("\nFATAL ZooKeeper Exception!\n");
1880       sb.append("\n" + ke.getMessage());
1881     }
1882     return sb.toString();
1883   }
1884 
1885   /**
1886    * Appends replication znodes to the passed StringBuilder.
1887    * @param zkw
1888    * @param sb
1889    * @throws KeeperException
1890    */
1891   private static void getReplicationZnodesDump(ZooKeeperWatcher zkw, StringBuilder sb)
1892       throws KeeperException {
1893     String replicationZNodeName = zkw.getConfiguration().get("zookeeper.znode.replication",
1894       "replication");
1895     String replicationZnode = joinZNode(zkw.baseZNode, replicationZNodeName);
1896     if (ZKUtil.checkExists(zkw, replicationZnode) == -1) return;
1897     // do a ls -r on this znode
1898     sb.append("\n").append(replicationZnode).append(": ");
1899     List<String> children = ZKUtil.listChildrenNoWatch(zkw, replicationZnode);
1900     for (String child : children) {
1901       String znode = joinZNode(replicationZnode, child);
1902       if (child.equals(zkw.getConfiguration().get("zookeeper.znode.replication.peers", "peers"))) {
1903         appendPeersZnodes(zkw, znode, sb);
1904       } else if (child.equals(zkw.getConfiguration().
1905           get("zookeeper.znode.replication.rs", "rs"))) {
1906         appendRSZnodes(zkw, znode, sb);
1907       }
1908     }
1909   }
1910 
1911   private static void appendRSZnodes(ZooKeeperWatcher zkw, String znode, StringBuilder sb)
1912       throws KeeperException {
1913     List<String> stack = new LinkedList<String>();
1914     stack.add(znode);
1915     do {
1916       String znodeToProcess = stack.remove(stack.size() - 1);
1917       sb.append("\n").append(znodeToProcess).append(": ");
1918       byte[] data;
1919       try {
1920         data = ZKUtil.getData(zkw, znodeToProcess);
1921       } catch (InterruptedException e) {
1922         zkw.interruptedException(e);
1923         return;
1924       }
1925       if (data != null && data.length > 0) { // log position
1926         long position = 0;
1927         try {
1928           position = ZKUtil.parseWALPositionFrom(ZKUtil.getData(zkw, znodeToProcess));
1929           sb.append(position);
1930         } catch (DeserializationException ignored) {
1931         } catch (InterruptedException e) {
1932           zkw.interruptedException(e);
1933           return;
1934         }
1935       }
1936       for (String zNodeChild : ZKUtil.listChildrenNoWatch(zkw, znodeToProcess)) {
1937         stack.add(ZKUtil.joinZNode(znodeToProcess, zNodeChild));
1938       }
1939     } while (stack.size() > 0);
1940   }
1941 
1942   private static void appendPeersZnodes(ZooKeeperWatcher zkw, String peersZnode,
1943     StringBuilder sb) throws KeeperException {
1944     int pblen = ProtobufUtil.lengthOfPBMagic();
1945     sb.append("\n").append(peersZnode).append(": ");
1946     for (String peerIdZnode : ZKUtil.listChildrenNoWatch(zkw, peersZnode)) {
1947       String znodeToProcess = ZKUtil.joinZNode(peersZnode, peerIdZnode);
1948       byte[] data;
1949       try {
1950         data = ZKUtil.getData(zkw, znodeToProcess);
1951       } catch (InterruptedException e) {
1952         zkw.interruptedException(e);
1953         return;
1954       }
1955       // parse the data of the above peer znode.
1956       try {
1957       String clusterKey = ZooKeeperProtos.ReplicationPeer.newBuilder().
1958         mergeFrom(data, pblen, data.length - pblen).getClusterkey();
1959       sb.append("\n").append(znodeToProcess).append(": ").append(clusterKey);
1960       // add the peer-state.
1961       appendPeerState(zkw, znodeToProcess, sb);
1962       } catch (InvalidProtocolBufferException ipbe) {
1963         LOG.warn("Got Exception while parsing peer: " + znodeToProcess, ipbe);
1964       }
1965     }
1966   }
1967 
1968   private static void appendPeerState(ZooKeeperWatcher zkw, String znodeToProcess,
1969       StringBuilder sb) throws KeeperException, InvalidProtocolBufferException {
1970     String peerState = zkw.getConfiguration().get("zookeeper.znode.replication.peers.state",
1971       "peer-state");
1972     int pblen = ProtobufUtil.lengthOfPBMagic();
1973     for (String child : ZKUtil.listChildrenNoWatch(zkw, znodeToProcess)) {
1974       if (!child.equals(peerState)) continue;
1975       String peerStateZnode = ZKUtil.joinZNode(znodeToProcess, child);
1976       sb.append("\n").append(peerStateZnode).append(": ");
1977       byte[] peerStateData;
1978       try {
1979         peerStateData = ZKUtil.getData(zkw, peerStateZnode);
1980         sb.append(ZooKeeperProtos.ReplicationState.newBuilder()
1981             .mergeFrom(peerStateData, pblen, peerStateData.length - pblen).getState().name());
1982       } catch (InterruptedException e) {
1983         zkw.interruptedException(e);
1984         return;
1985       }
1986     }
1987   }
1988 
1989   /**
1990    * Gets the statistics from the given server.
1991    *
1992    * @param server  The server to get the statistics from.
1993    * @param timeout  The socket timeout to use.
1994    * @return The array of response strings.
1995    * @throws IOException When the socket communication fails.
1996    */
1997   public static String[] getServerStats(String server, int timeout)
1998   throws IOException {
1999     String[] sp = server.split(":");
2000     if (sp == null || sp.length == 0) {
2001       return null;
2002     }
2003 
2004     String host = sp[0];
2005     int port = sp.length > 1 ? Integer.parseInt(sp[1])
2006         : HConstants.DEFAULT_ZOOKEPER_CLIENT_PORT;
2007 
2008     Socket socket = new Socket();
2009     InetSocketAddress sockAddr = new InetSocketAddress(host, port);
2010     socket.connect(sockAddr, timeout);
2011 
2012     socket.setSoTimeout(timeout);
2013     PrintWriter out = new PrintWriter(socket.getOutputStream(), true);
2014     BufferedReader in = new BufferedReader(new InputStreamReader(
2015       socket.getInputStream()));
2016     out.println("stat");
2017     out.flush();
2018     ArrayList<String> res = new ArrayList<String>();
2019     while (true) {
2020       String line = in.readLine();
2021       if (line != null) {
2022         res.add(line);
2023       } else {
2024         break;
2025       }
2026     }
2027     socket.close();
2028     return res.toArray(new String[res.size()]);
2029   }
2030 
2031   private static void logRetrievedMsg(final ZooKeeperWatcher zkw,
2032       final String znode, final byte [] data, final boolean watcherSet) {
2033     if (!LOG.isTraceEnabled()) return;
2034     LOG.trace(zkw.prefix("Retrieved " + ((data == null)? 0: data.length) +
2035       " byte(s) of data from znode " + znode +
2036       (watcherSet? " and set watcher; ": "; data=") +
2037       (data == null? "null": data.length == 0? "empty": (
2038           znode.startsWith(ZooKeeperWatcher.META_ZNODE_PREFIX)?
2039             getServerNameOrEmptyString(data):
2040           znode.startsWith(zkw.backupMasterAddressesZNode)?
2041             getServerNameOrEmptyString(data):
2042           StringUtils.abbreviate(Bytes.toStringBinary(data), 32)))));
2043   }
2044 
2045   private static String getServerNameOrEmptyString(final byte [] data) {
2046     try {
2047       return ServerName.parseFrom(data).toString();
2048     } catch (DeserializationException e) {
2049       return "";
2050     }
2051   }
2052 
2053   /**
2054    * Waits for HBase installation's base (parent) znode to become available.
2055    * @throws IOException on ZK errors
2056    */
2057   public static void waitForBaseZNode(Configuration conf) throws IOException {
2058     LOG.info("Waiting until the base znode is available");
2059     String parentZNode = conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT,
2060         HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT);
2061     ZooKeeper zk = new ZooKeeper(ZKConfig.getZKQuorumServersString(conf),
2062         conf.getInt(HConstants.ZK_SESSION_TIMEOUT,
2063         HConstants.DEFAULT_ZK_SESSION_TIMEOUT), EmptyWatcher.instance);
2064 
2065     final int maxTimeMs = 10000;
2066     final int maxNumAttempts = maxTimeMs / HConstants.SOCKET_RETRY_WAIT_MS;
2067 
2068     KeeperException keeperEx = null;
2069     try {
2070       try {
2071         for (int attempt = 0; attempt < maxNumAttempts; ++attempt) {
2072           try {
2073             if (zk.exists(parentZNode, false) != null) {
2074               LOG.info("Parent znode exists: " + parentZNode);
2075               keeperEx = null;
2076               break;
2077             }
2078           } catch (KeeperException e) {
2079             keeperEx = e;
2080           }
2081           Threads.sleepWithoutInterrupt(HConstants.SOCKET_RETRY_WAIT_MS);
2082         }
2083       } finally {
2084         zk.close();
2085       }
2086     } catch (InterruptedException ex) {
2087       Thread.currentThread().interrupt();
2088     }
2089 
2090     if (keeperEx != null) {
2091       throw new IOException(keeperEx);
2092     }
2093   }
2094 
2095   /**
2096    * Convert a {@link DeserializationException} to a more palatable {@link KeeperException}.
2097    * Used when can't let a {@link DeserializationException} out w/o changing public API.
2098    * @param e Exception to convert
2099    * @return Converted exception
2100    */
2101   public static KeeperException convert(final DeserializationException e) {
2102     KeeperException ke = new KeeperException.DataInconsistencyException();
2103     ke.initCause(e);
2104     return ke;
2105   }
2106 
2107   /**
2108    * Recursively print the current state of ZK (non-transactional)
2109    * @param root name of the root directory in zk to print
2110    */
2111   public static void logZKTree(ZooKeeperWatcher zkw, String root) {
2112     if (!LOG.isDebugEnabled()) return;
2113     LOG.debug("Current zk system:");
2114     String prefix = "|-";
2115     LOG.debug(prefix + root);
2116     try {
2117       logZKTree(zkw, root, prefix);
2118     } catch (KeeperException e) {
2119       throw new RuntimeException(e);
2120     }
2121   }
2122 
2123   /**
2124    * Helper method to print the current state of the ZK tree.
2125    * @see #logZKTree(ZooKeeperWatcher, String)
2126    * @throws KeeperException if an unexpected exception occurs
2127    */
2128   protected static void logZKTree(ZooKeeperWatcher zkw, String root, String prefix)
2129       throws KeeperException {
2130     List<String> children = ZKUtil.listChildrenNoWatch(zkw, root);
2131     if (children == null) return;
2132     for (String child : children) {
2133       LOG.debug(prefix + child);
2134       String node = ZKUtil.joinZNode(root.equals("/") ? "" : root, child);
2135       logZKTree(zkw, node, prefix + "---");
2136     }
2137   }
2138 
2139   /**
2140    * @param position
2141    * @return Serialized protobuf of <code>position</code> with pb magic prefix prepended suitable
2142    *         for use as content of an wal position in a replication queue.
2143    */
2144   public static byte[] positionToByteArray(final long position) {
2145     byte[] bytes = ZooKeeperProtos.ReplicationHLogPosition.newBuilder().setPosition(position)
2146         .build().toByteArray();
2147     return ProtobufUtil.prependPBMagic(bytes);
2148   }
2149 
2150   /**
2151    * @param bytes - Content of a WAL position znode.
2152    * @return long - The current WAL position.
2153    * @throws DeserializationException
2154    */
2155   public static long parseWALPositionFrom(final byte[] bytes) throws DeserializationException {
2156     if (bytes == null) {
2157       throw new DeserializationException("Unable to parse null WAL position.");
2158     }
2159     if (ProtobufUtil.isPBMagicPrefix(bytes)) {
2160       int pblen = ProtobufUtil.lengthOfPBMagic();
2161       ZooKeeperProtos.ReplicationHLogPosition.Builder builder =
2162           ZooKeeperProtos.ReplicationHLogPosition.newBuilder();
2163       ZooKeeperProtos.ReplicationHLogPosition position;
2164       try {
2165         position = builder.mergeFrom(bytes, pblen, bytes.length - pblen).build();
2166       } catch (InvalidProtocolBufferException e) {
2167         throw new DeserializationException(e);
2168       }
2169       return position.getPosition();
2170     } else {
2171       if (bytes.length > 0) {
2172         return Bytes.toLong(bytes);
2173       }
2174       return 0;
2175     }
2176   }
2177 
2178   /**
2179    * @param regionLastFlushedSequenceId the flushed sequence id of a region which is the min of its
2180    *          store max seq ids
2181    * @param storeSequenceIds column family to sequence Id map
2182    * @return Serialized protobuf of <code>RegionSequenceIds</code> with pb magic prefix prepended
2183    *         suitable for use to filter wal edits in distributedLogReplay mode
2184    */
2185   public static byte[] regionSequenceIdsToByteArray(final Long regionLastFlushedSequenceId,
2186       final Map<byte[], Long> storeSequenceIds) {
2187     ClusterStatusProtos.RegionStoreSequenceIds.Builder regionSequenceIdsBuilder =
2188         ClusterStatusProtos.RegionStoreSequenceIds.newBuilder();
2189     ClusterStatusProtos.StoreSequenceId.Builder storeSequenceIdBuilder =
2190         ClusterStatusProtos.StoreSequenceId.newBuilder();
2191     if (storeSequenceIds != null) {
2192       for (Map.Entry<byte[], Long> e : storeSequenceIds.entrySet()){
2193         byte[] columnFamilyName = e.getKey();
2194         Long curSeqId = e.getValue();
2195         storeSequenceIdBuilder.setFamilyName(ByteStringer.wrap(columnFamilyName));
2196         storeSequenceIdBuilder.setSequenceId(curSeqId);
2197         regionSequenceIdsBuilder.addStoreSequenceId(storeSequenceIdBuilder.build());
2198         storeSequenceIdBuilder.clear();
2199       }
2200     }
2201     regionSequenceIdsBuilder.setLastFlushedSequenceId(regionLastFlushedSequenceId);
2202     byte[] result = regionSequenceIdsBuilder.build().toByteArray();
2203     return ProtobufUtil.prependPBMagic(result);
2204   }
2205 
2206   /**
2207    * @param bytes Content of serialized data of RegionStoreSequenceIds
2208    * @return a RegionStoreSequenceIds object
2209    * @throws DeserializationException
2210    */
2211   public static RegionStoreSequenceIds parseRegionStoreSequenceIds(final byte[] bytes)
2212       throws DeserializationException {
2213     if (bytes == null || !ProtobufUtil.isPBMagicPrefix(bytes)) {
2214       throw new DeserializationException("Unable to parse RegionStoreSequenceIds.");
2215     }
2216     RegionStoreSequenceIds.Builder regionSequenceIdsBuilder =
2217         ClusterStatusProtos.RegionStoreSequenceIds.newBuilder();
2218     int pblen = ProtobufUtil.lengthOfPBMagic();
2219     RegionStoreSequenceIds storeIds = null;
2220     try {
2221       storeIds = regionSequenceIdsBuilder.mergeFrom(bytes, pblen, bytes.length - pblen).build();
2222     } catch (InvalidProtocolBufferException e) {
2223       throw new DeserializationException(e);
2224     }
2225     return storeIds;
2226   }
2227 }