View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.zookeeper;
20  
21  import java.io.BufferedReader;
22  import java.io.IOException;
23  import java.io.InputStreamReader;
24  import java.io.PrintWriter;
25  import java.net.InetSocketAddress;
26  import java.net.Socket;
27  import java.util.ArrayList;
28  import java.util.Arrays;
29  import java.util.Deque;
30  import java.util.HashMap;
31  import java.util.LinkedList;
32  import java.util.List;
33  import java.util.Map;
34  
35  import javax.security.auth.login.AppConfigurationEntry;
36  import javax.security.auth.login.AppConfigurationEntry.LoginModuleControlFlag;
37  
38  import org.apache.commons.lang.StringUtils;
39  import org.apache.commons.logging.Log;
40  import org.apache.commons.logging.LogFactory;
41  import org.apache.hadoop.conf.Configuration;
42  import org.apache.hadoop.hbase.AuthUtil;
43  import org.apache.hadoop.hbase.HBaseConfiguration;
44  import org.apache.hadoop.hbase.HConstants;
45  import org.apache.hadoop.hbase.ServerName;
46  import org.apache.hadoop.hbase.classification.InterfaceAudience;
47  import org.apache.hadoop.hbase.exceptions.DeserializationException;
48  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
49  import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos;
50  import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionStoreSequenceIds;
51  import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
52  import org.apache.hadoop.hbase.security.Superusers;
53  import org.apache.hadoop.hbase.util.ByteStringer;
54  import org.apache.hadoop.hbase.util.Bytes;
55  import org.apache.hadoop.hbase.util.Threads;
56  import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp.CreateAndFailSilent;
57  import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp.DeleteNodeFailSilent;
58  import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp.SetData;
59  import org.apache.hadoop.security.SecurityUtil;
60  import org.apache.hadoop.security.authentication.util.KerberosUtil;
61  import org.apache.zookeeper.AsyncCallback;
62  import org.apache.zookeeper.CreateMode;
63  import org.apache.zookeeper.KeeperException;
64  import org.apache.zookeeper.KeeperException.NoNodeException;
65  import org.apache.zookeeper.Op;
66  import org.apache.zookeeper.Watcher;
67  import org.apache.zookeeper.ZooDefs.Ids;
68  import org.apache.zookeeper.ZooDefs.Perms;
69  import org.apache.zookeeper.ZooKeeper;
70  import org.apache.zookeeper.client.ZooKeeperSaslClient;
71  import org.apache.zookeeper.data.ACL;
72  import org.apache.zookeeper.data.Id;
73  import org.apache.zookeeper.data.Stat;
74  import org.apache.zookeeper.proto.CreateRequest;
75  import org.apache.zookeeper.proto.DeleteRequest;
76  import org.apache.zookeeper.proto.SetDataRequest;
77  import org.apache.zookeeper.server.ZooKeeperSaslServer;
78  
79  import com.google.protobuf.InvalidProtocolBufferException;
80  
81  /**
82   * Internal HBase utility class for ZooKeeper.
83   *
84   * <p>Contains only static methods and constants.
85   *
86   * <p>Methods all throw {@link KeeperException} if there is an unexpected
87   * zookeeper exception, so callers of these methods must handle appropriately.
88   * If ZK is required for the operation, the server will need to be aborted.
89   */
90  @InterfaceAudience.Private
91  public class ZKUtil {
92    private static final Log LOG = LogFactory.getLog(ZKUtil.class);
93  
94    // TODO: Replace this with ZooKeeper constant when ZOOKEEPER-277 is resolved.
95    public static final char ZNODE_PATH_SEPARATOR = '/';
96    private static int zkDumpConnectionTimeOut;
97  
98    /**
99     * Creates a new connection to ZooKeeper, pulling settings and ensemble config
100    * from the specified configuration object using methods from {@link ZKConfig}.
101    *
102    * Sets the connection status monitoring watcher to the specified watcher.
103    *
104    * @param conf configuration to pull ensemble and other settings from
105    * @param watcher watcher to monitor connection changes
106    * @return connection to zookeeper
107    * @throws IOException if unable to connect to zk or config problem
108    */
109   public static RecoverableZooKeeper connect(Configuration conf, Watcher watcher)
110   throws IOException {
111     String ensemble = ZKConfig.getZKQuorumServersString(conf);
112     return connect(conf, ensemble, watcher);
113   }
114 
115   public static RecoverableZooKeeper connect(Configuration conf, String ensemble,
116       Watcher watcher)
117   throws IOException {
118     return connect(conf, ensemble, watcher, null);
119   }
120 
121   public static RecoverableZooKeeper connect(Configuration conf, String ensemble,
122       Watcher watcher, final String identifier)
123   throws IOException {
124     if(ensemble == null) {
125       throw new IOException("Unable to determine ZooKeeper ensemble");
126     }
127     int timeout = conf.getInt(HConstants.ZK_SESSION_TIMEOUT,
128         HConstants.DEFAULT_ZK_SESSION_TIMEOUT);
129     if (LOG.isTraceEnabled()) {
130       LOG.trace(identifier + " opening connection to ZooKeeper ensemble=" + ensemble);
131     }
132     int retry = conf.getInt("zookeeper.recovery.retry", 3);
133     int retryIntervalMillis =
134       conf.getInt("zookeeper.recovery.retry.intervalmill", 1000);
135     zkDumpConnectionTimeOut = conf.getInt("zookeeper.dump.connection.timeout",
136         1000);
137     return new RecoverableZooKeeper(ensemble, timeout, watcher,
138         retry, retryIntervalMillis, identifier);
139   }
140 
141   /**
142    * Log in the current zookeeper server process using the given configuration
143    * keys for the credential file and login principal.
144    *
145    * <p><strong>This is only applicable when running on secure hbase</strong>
146    * On regular HBase (without security features), this will safely be ignored.
147    * </p>
148    *
149    * @param conf The configuration data to use
150    * @param keytabFileKey Property key used to configure the path to the credential file
151    * @param userNameKey Property key used to configure the login principal
152    * @param hostname Current hostname to use in any credentials
153    * @throws IOException underlying exception from SecurityUtil.login() call
154    */
155   public static void loginServer(Configuration conf, String keytabFileKey,
156       String userNameKey, String hostname) throws IOException {
157     login(conf, keytabFileKey, userNameKey, hostname,
158           ZooKeeperSaslServer.LOGIN_CONTEXT_NAME_KEY,
159           JaasConfiguration.SERVER_KEYTAB_KERBEROS_CONFIG_NAME);
160   }
161 
162   /**
163    * Log in the current zookeeper client using the given configuration
164    * keys for the credential file and login principal.
165    *
166    * <p><strong>This is only applicable when running on secure hbase</strong>
167    * On regular HBase (without security features), this will safely be ignored.
168    * </p>
169    *
170    * @param conf The configuration data to use
171    * @param keytabFileKey Property key used to configure the path to the credential file
172    * @param userNameKey Property key used to configure the login principal
173    * @param hostname Current hostname to use in any credentials
174    * @throws IOException underlying exception from SecurityUtil.login() call
175    */
176   public static void loginClient(Configuration conf, String keytabFileKey,
177       String userNameKey, String hostname) throws IOException {
178     login(conf, keytabFileKey, userNameKey, hostname,
179           ZooKeeperSaslClient.LOGIN_CONTEXT_NAME_KEY,
180           JaasConfiguration.CLIENT_KEYTAB_KERBEROS_CONFIG_NAME);
181   }
182 
183   /**
184    * Log in the current process using the given configuration keys for the
185    * credential file and login principal.
186    *
187    * <p><strong>This is only applicable when running on secure hbase</strong>
188    * On regular HBase (without security features), this will safely be ignored.
189    * </p>
190    *
191    * @param conf The configuration data to use
192    * @param keytabFileKey Property key used to configure the path to the credential file
193    * @param userNameKey Property key used to configure the login principal
194    * @param hostname Current hostname to use in any credentials
195    * @param loginContextProperty property name to expose the entry name
196    * @param loginContextName jaas entry name
197    * @throws IOException underlying exception from SecurityUtil.login() call
198    */
199   private static void login(Configuration conf, String keytabFileKey,
200       String userNameKey, String hostname,
201       String loginContextProperty, String loginContextName)
202       throws IOException {
203     if (!isSecureZooKeeper(conf))
204       return;
205 
206     // User has specified a jaas.conf, keep this one as the good one.
207     // HBASE_OPTS="-Djava.security.auth.login.config=jaas.conf"
208     if (System.getProperty("java.security.auth.login.config") != null)
209       return;
210 
211     // No keytab specified, no auth
212     String keytabFilename = conf.get(keytabFileKey);
213     if (keytabFilename == null) {
214       LOG.warn("no keytab specified for: " + keytabFileKey);
215       return;
216     }
217 
218     String principalConfig = conf.get(userNameKey, System.getProperty("user.name"));
219     String principalName = SecurityUtil.getServerPrincipal(principalConfig, hostname);
220 
221     // Initialize the "jaas.conf" for keyTab/principal,
222     // If keyTab is not specified use the Ticket Cache.
223     // and set the zookeeper login context name.
224     JaasConfiguration jaasConf = new JaasConfiguration(loginContextName,
225         principalName, keytabFilename);
226     javax.security.auth.login.Configuration.setConfiguration(jaasConf);
227     System.setProperty(loginContextProperty, loginContextName);
228   }
229 
230   /**
231    * A JAAS configuration that defines the login modules that we want to use for login.
232    */
233   private static class JaasConfiguration extends javax.security.auth.login.Configuration {
234     private static final String SERVER_KEYTAB_KERBEROS_CONFIG_NAME =
235       "zookeeper-server-keytab-kerberos";
236     private static final String CLIENT_KEYTAB_KERBEROS_CONFIG_NAME =
237       "zookeeper-client-keytab-kerberos";
238 
239     private static final Map<String, String> BASIC_JAAS_OPTIONS =
240       new HashMap<String,String>();
241     static {
242       String jaasEnvVar = System.getenv("HBASE_JAAS_DEBUG");
243       if (jaasEnvVar != null && "true".equalsIgnoreCase(jaasEnvVar)) {
244         BASIC_JAAS_OPTIONS.put("debug", "true");
245       }
246     }
247 
248     private static final Map<String,String> KEYTAB_KERBEROS_OPTIONS =
249       new HashMap<String,String>();
250     static {
251       KEYTAB_KERBEROS_OPTIONS.put("doNotPrompt", "true");
252       KEYTAB_KERBEROS_OPTIONS.put("storeKey", "true");
253       KEYTAB_KERBEROS_OPTIONS.put("refreshKrb5Config", "true");
254       KEYTAB_KERBEROS_OPTIONS.putAll(BASIC_JAAS_OPTIONS);
255     }
256 
257     private static final AppConfigurationEntry KEYTAB_KERBEROS_LOGIN =
258       new AppConfigurationEntry(KerberosUtil.getKrb5LoginModuleName(),
259                                 LoginModuleControlFlag.REQUIRED,
260                                 KEYTAB_KERBEROS_OPTIONS);
261 
262     private static final AppConfigurationEntry[] KEYTAB_KERBEROS_CONF =
263       new AppConfigurationEntry[]{KEYTAB_KERBEROS_LOGIN};
264 
265     private javax.security.auth.login.Configuration baseConfig;
266     private final String loginContextName;
267     private final boolean useTicketCache;
268     private final String keytabFile;
269     private final String principal;
270 
271     public JaasConfiguration(String loginContextName, String principal) {
272       this(loginContextName, principal, null, true);
273     }
274 
275     public JaasConfiguration(String loginContextName, String principal, String keytabFile) {
276       this(loginContextName, principal, keytabFile, keytabFile == null || keytabFile.length() == 0);
277     }
278 
279     private JaasConfiguration(String loginContextName, String principal,
280                              String keytabFile, boolean useTicketCache) {
281       try {
282         this.baseConfig = javax.security.auth.login.Configuration.getConfiguration();
283       } catch (SecurityException e) {
284         this.baseConfig = null;
285       }
286       this.loginContextName = loginContextName;
287       this.useTicketCache = useTicketCache;
288       this.keytabFile = keytabFile;
289       this.principal = principal;
290       LOG.info("JaasConfiguration loginContextName=" + loginContextName +
291                " principal=" + principal + " useTicketCache=" + useTicketCache +
292                " keytabFile=" + keytabFile);
293     }
294 
295     @Override
296     public AppConfigurationEntry[] getAppConfigurationEntry(String appName) {
297       if (loginContextName.equals(appName)) {
298         if (!useTicketCache) {
299           KEYTAB_KERBEROS_OPTIONS.put("keyTab", keytabFile);
300           KEYTAB_KERBEROS_OPTIONS.put("useKeyTab", "true");
301         }
302         KEYTAB_KERBEROS_OPTIONS.put("principal", principal);
303         KEYTAB_KERBEROS_OPTIONS.put("useTicketCache", useTicketCache ? "true" : "false");
304         return KEYTAB_KERBEROS_CONF;
305       }
306       if (baseConfig != null) return baseConfig.getAppConfigurationEntry(appName);
307       return(null);
308     }
309   }
310 
311   //
312   // Helper methods
313   //
314 
315   /**
316    * Join the prefix znode name with the suffix znode name to generate a proper
317    * full znode name.
318    *
319    * Assumes prefix does not end with slash and suffix does not begin with it.
320    *
321    * @param prefix beginning of znode name
322    * @param suffix ending of znode name
323    * @return result of properly joining prefix with suffix
324    */
325   public static String joinZNode(String prefix, String suffix) {
326     return prefix + ZNODE_PATH_SEPARATOR + suffix;
327   }
328 
329   /**
330    * Returns the full path of the immediate parent of the specified node.
331    * @param node path to get parent of
332    * @return parent of path, null if passed the root node or an invalid node
333    */
334   public static String getParent(String node) {
335     int idx = node.lastIndexOf(ZNODE_PATH_SEPARATOR);
336     return idx <= 0 ? null : node.substring(0, idx);
337   }
338 
339   /**
340    * Get the name of the current node from the specified fully-qualified path.
341    * @param path fully-qualified path
342    * @return name of the current node
343    */
344   public static String getNodeName(String path) {
345     return path.substring(path.lastIndexOf("/")+1);
346   }
347 
348   //
349   // Existence checks and watches
350   //
351 
352   /**
353    * Watch the specified znode for delete/create/change events.  The watcher is
354    * set whether or not the node exists.  If the node already exists, the method
355    * returns true.  If the node does not exist, the method returns false.
356    *
357    * @param zkw zk reference
358    * @param znode path of node to watch
359    * @return true if znode exists, false if does not exist or error
360    * @throws KeeperException if unexpected zookeeper exception
361    */
362   public static boolean watchAndCheckExists(ZooKeeperWatcher zkw, String znode)
363   throws KeeperException {
364     try {
365       Stat s = zkw.getRecoverableZooKeeper().exists(znode, zkw);
366       boolean exists = s != null ? true : false;
367       if (exists) {
368         LOG.debug(zkw.prefix("Set watcher on existing znode=" + znode));
369       } else {
370         LOG.debug(zkw.prefix("Set watcher on znode that does not yet exist, " + znode));
371       }
372       return exists;
373     } catch (KeeperException e) {
374       LOG.warn(zkw.prefix("Unable to set watcher on znode " + znode), e);
375       zkw.keeperException(e);
376       return false;
377     } catch (InterruptedException e) {
378       LOG.warn(zkw.prefix("Unable to set watcher on znode " + znode), e);
379       zkw.interruptedException(e);
380       return false;
381     }
382   }
383 
384   /**
385    * Watch the specified znode, but only if exists. Useful when watching
386    * for deletions. Uses .getData() (and handles NoNodeException) instead
387    * of .exists() to accomplish this, as .getData() will only set a watch if
388    * the znode exists.
389    * @param zkw zk reference
390    * @param znode path of node to watch
391    * @return true if the watch is set, false if node does not exists
392    * @throws KeeperException if unexpected zookeeper exception
393    */
394   public static boolean setWatchIfNodeExists(ZooKeeperWatcher zkw, String znode)
395       throws KeeperException {
396     try {
397       zkw.getRecoverableZooKeeper().getData(znode, true, null);
398       return true;
399     } catch (NoNodeException e) {
400       return false;
401     } catch (InterruptedException e) {
402       LOG.warn(zkw.prefix("Unable to set watcher on znode " + znode), e);
403       zkw.interruptedException(e);
404       return false;
405     }
406   }
407 
408   /**
409    * Check if the specified node exists.  Sets no watches.
410    *
411    * @param zkw zk reference
412    * @param znode path of node to watch
413    * @return version of the node if it exists, -1 if does not exist
414    * @throws KeeperException if unexpected zookeeper exception
415    */
416   public static int checkExists(ZooKeeperWatcher zkw, String znode)
417   throws KeeperException {
418     try {
419       Stat s = zkw.getRecoverableZooKeeper().exists(znode, null);
420       return s != null ? s.getVersion() : -1;
421     } catch (KeeperException e) {
422       LOG.warn(zkw.prefix("Unable to set watcher on znode (" + znode + ")"), e);
423       zkw.keeperException(e);
424       return -1;
425     } catch (InterruptedException e) {
426       LOG.warn(zkw.prefix("Unable to set watcher on znode (" + znode + ")"), e);
427       zkw.interruptedException(e);
428       return -1;
429     }
430   }
431 
432   //
433   // Znode listings
434   //
435 
436   /**
437    * Lists the children znodes of the specified znode.  Also sets a watch on
438    * the specified znode which will capture a NodeDeleted event on the specified
439    * znode as well as NodeChildrenChanged if any children of the specified znode
440    * are created or deleted.
441    *
442    * Returns null if the specified node does not exist.  Otherwise returns a
443    * list of children of the specified node.  If the node exists but it has no
444    * children, an empty list will be returned.
445    *
446    * @param zkw zk reference
447    * @param znode path of node to list and watch children of
448    * @return list of children of the specified node, an empty list if the node
449    *          exists but has no children, and null if the node does not exist
450    * @throws KeeperException if unexpected zookeeper exception
451    */
452   public static List<String> listChildrenAndWatchForNewChildren(
453       ZooKeeperWatcher zkw, String znode)
454   throws KeeperException {
455     try {
456       List<String> children = zkw.getRecoverableZooKeeper().getChildren(znode, zkw);
457       return children;
458     } catch(KeeperException.NoNodeException ke) {
459       LOG.debug(zkw.prefix("Unable to list children of znode " + znode + " " +
460           "because node does not exist (not an error)"));
461       return null;
462     } catch (KeeperException e) {
463       LOG.warn(zkw.prefix("Unable to list children of znode " + znode + " "), e);
464       zkw.keeperException(e);
465       return null;
466     } catch (InterruptedException e) {
467       LOG.warn(zkw.prefix("Unable to list children of znode " + znode + " "), e);
468       zkw.interruptedException(e);
469       return null;
470     }
471   }
472 
473   /**
474    * List all the children of the specified znode, setting a watch for children
475    * changes and also setting a watch on every individual child in order to get
476    * the NodeCreated and NodeDeleted events.
477    * @param zkw zookeeper reference
478    * @param znode node to get children of and watch
479    * @return list of znode names, null if the node doesn't exist
480    * @throws KeeperException
481    */
482   public static List<String> listChildrenAndWatchThem(ZooKeeperWatcher zkw,
483       String znode) throws KeeperException {
484     List<String> children = listChildrenAndWatchForNewChildren(zkw, znode);
485     if (children == null) {
486       return null;
487     }
488     for (String child : children) {
489       watchAndCheckExists(zkw, joinZNode(znode, child));
490     }
491     return children;
492   }
493 
494   /**
495    * Lists the children of the specified znode without setting any watches.
496    *
497    * Sets no watches at all, this method is best effort.
498    *
499    * Returns an empty list if the node has no children.  Returns null if the
500    * parent node itself does not exist.
501    *
502    * @param zkw zookeeper reference
503    * @param znode node to get children
504    * @return list of data of children of specified znode, empty if no children,
505    *         null if parent does not exist
506    * @throws KeeperException if unexpected zookeeper exception
507    */
508   public static List<String> listChildrenNoWatch(ZooKeeperWatcher zkw, String znode)
509   throws KeeperException {
510     List<String> children = null;
511     try {
512       // List the children without watching
513       children = zkw.getRecoverableZooKeeper().getChildren(znode, null);
514     } catch(KeeperException.NoNodeException nne) {
515       return null;
516     } catch(InterruptedException ie) {
517       zkw.interruptedException(ie);
518     }
519     return children;
520   }
521 
522   /**
523    * Simple class to hold a node path and node data.
524    * @deprecated Unused
525    */
526   @Deprecated
527   public static class NodeAndData {
528     private String node;
529     private byte [] data;
530     public NodeAndData(String node, byte [] data) {
531       this.node = node;
532       this.data = data;
533     }
534     public String getNode() {
535       return node;
536     }
537     public byte [] getData() {
538       return data;
539     }
540     @Override
541     public String toString() {
542       return node;
543     }
544     public boolean isEmpty() {
545       return (data == null || data.length == 0);
546     }
547   }
548 
549   /**
550    * Checks if the specified znode has any children.  Sets no watches.
551    *
552    * Returns true if the node exists and has children.  Returns false if the
553    * node does not exist or if the node does not have any children.
554    *
555    * Used during master initialization to determine if the master is a
556    * failed-over-to master or the first master during initial cluster startup.
557    * If the directory for regionserver ephemeral nodes is empty then this is
558    * a cluster startup, if not then it is not cluster startup.
559    *
560    * @param zkw zk reference
561    * @param znode path of node to check for children of
562    * @return true if node has children, false if not or node does not exist
563    * @throws KeeperException if unexpected zookeeper exception
564    */
565   public static boolean nodeHasChildren(ZooKeeperWatcher zkw, String znode)
566   throws KeeperException {
567     try {
568       return !zkw.getRecoverableZooKeeper().getChildren(znode, null).isEmpty();
569     } catch(KeeperException.NoNodeException ke) {
570       LOG.debug(zkw.prefix("Unable to list children of znode " + znode + " " +
571       "because node does not exist (not an error)"));
572       return false;
573     } catch (KeeperException e) {
574       LOG.warn(zkw.prefix("Unable to list children of znode " + znode), e);
575       zkw.keeperException(e);
576       return false;
577     } catch (InterruptedException e) {
578       LOG.warn(zkw.prefix("Unable to list children of znode " + znode), e);
579       zkw.interruptedException(e);
580       return false;
581     }
582   }
583 
584   /**
585    * Get the number of children of the specified node.
586    *
587    * If the node does not exist or has no children, returns 0.
588    *
589    * Sets no watches at all.
590    *
591    * @param zkw zk reference
592    * @param znode path of node to count children of
593    * @return number of children of specified node, 0 if none or parent does not
594    *         exist
595    * @throws KeeperException if unexpected zookeeper exception
596    */
597   public static int getNumberOfChildren(ZooKeeperWatcher zkw, String znode)
598   throws KeeperException {
599     try {
600       Stat stat = zkw.getRecoverableZooKeeper().exists(znode, null);
601       return stat == null ? 0 : stat.getNumChildren();
602     } catch(KeeperException e) {
603       LOG.warn(zkw.prefix("Unable to get children of node " + znode));
604       zkw.keeperException(e);
605     } catch(InterruptedException e) {
606       zkw.interruptedException(e);
607     }
608     return 0;
609   }
610 
611   //
612   // Data retrieval
613   //
614 
615   /**
616    * Get znode data. Does not set a watcher.
617    * @return ZNode data, null if the node does not exist or if there is an
618    *  error.
619    */
620   public static byte [] getData(ZooKeeperWatcher zkw, String znode)
621       throws KeeperException, InterruptedException {
622     try {
623       byte [] data = zkw.getRecoverableZooKeeper().getData(znode, null, null);
624       logRetrievedMsg(zkw, znode, data, false);
625       return data;
626     } catch (KeeperException.NoNodeException e) {
627       LOG.debug(zkw.prefix("Unable to get data of znode " + znode + " " +
628           "because node does not exist (not an error)"));
629       return null;
630     } catch (KeeperException e) {
631       LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
632       zkw.keeperException(e);
633       return null;
634     }
635   }
636 
637   /**
638    * Get the data at the specified znode and set a watch.
639    *
640    * Returns the data and sets a watch if the node exists.  Returns null and no
641    * watch is set if the node does not exist or there is an exception.
642    *
643    * @param zkw zk reference
644    * @param znode path of node
645    * @return data of the specified znode, or null
646    * @throws KeeperException if unexpected zookeeper exception
647    */
648   public static byte [] getDataAndWatch(ZooKeeperWatcher zkw, String znode)
649   throws KeeperException {
650     return getDataInternal(zkw, znode, null, true);
651   }
652 
653   /**
654    * Get the data at the specified znode and set a watch.
655    *
656    * Returns the data and sets a watch if the node exists.  Returns null and no
657    * watch is set if the node does not exist or there is an exception.
658    *
659    * @param zkw zk reference
660    * @param znode path of node
661    * @param stat object to populate the version of the znode
662    * @return data of the specified znode, or null
663    * @throws KeeperException if unexpected zookeeper exception
664    */
665   public static byte[] getDataAndWatch(ZooKeeperWatcher zkw, String znode,
666       Stat stat) throws KeeperException {
667     return getDataInternal(zkw, znode, stat, true);
668   }
669 
670   private static byte[] getDataInternal(ZooKeeperWatcher zkw, String znode, Stat stat,
671       boolean watcherSet)
672       throws KeeperException {
673     try {
674       byte [] data = zkw.getRecoverableZooKeeper().getData(znode, zkw, stat);
675       logRetrievedMsg(zkw, znode, data, watcherSet);
676       return data;
677     } catch (KeeperException.NoNodeException e) {
678       // This log can get pretty annoying when we cycle on 100ms waits.
679       // Enable trace if you really want to see it.
680       LOG.trace(zkw.prefix("Unable to get data of znode " + znode + " " +
681         "because node does not exist (not an error)"));
682       return null;
683     } catch (KeeperException e) {
684       LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
685       zkw.keeperException(e);
686       return null;
687     } catch (InterruptedException e) {
688       LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
689       zkw.interruptedException(e);
690       return null;
691     }
692   }
693 
694   /**
695    * Get the data at the specified znode without setting a watch.
696    *
697    * Returns the data if the node exists.  Returns null if the node does not
698    * exist.
699    *
700    * Sets the stats of the node in the passed Stat object.  Pass a null stat if
701    * not interested.
702    *
703    * @param zkw zk reference
704    * @param znode path of node
705    * @param stat node status to get if node exists
706    * @return data of the specified znode, or null if node does not exist
707    * @throws KeeperException if unexpected zookeeper exception
708    */
709   public static byte [] getDataNoWatch(ZooKeeperWatcher zkw, String znode,
710       Stat stat)
711   throws KeeperException {
712     try {
713       byte [] data = zkw.getRecoverableZooKeeper().getData(znode, null, stat);
714       logRetrievedMsg(zkw, znode, data, false);
715       return data;
716     } catch (KeeperException.NoNodeException e) {
717       LOG.debug(zkw.prefix("Unable to get data of znode " + znode + " " +
718           "because node does not exist (not necessarily an error)"));
719       return null;
720     } catch (KeeperException e) {
721       LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
722       zkw.keeperException(e);
723       return null;
724     } catch (InterruptedException e) {
725       LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
726       zkw.interruptedException(e);
727       return null;
728     }
729   }
730 
731   /**
732    * Returns the date of child znodes of the specified znode.  Also sets a watch on
733    * the specified znode which will capture a NodeDeleted event on the specified
734    * znode as well as NodeChildrenChanged if any children of the specified znode
735    * are created or deleted.
736    *
737    * Returns null if the specified node does not exist.  Otherwise returns a
738    * list of children of the specified node.  If the node exists but it has no
739    * children, an empty list will be returned.
740    *
741    * @param zkw zk reference
742    * @param baseNode path of node to list and watch children of
743    * @return list of data of children of the specified node, an empty list if the node
744    *          exists but has no children, and null if the node does not exist
745    * @throws KeeperException if unexpected zookeeper exception
746    * @deprecated Unused
747    */
748   public static List<NodeAndData> getChildDataAndWatchForNewChildren(
749       ZooKeeperWatcher zkw, String baseNode) throws KeeperException {
750     List<String> nodes =
751       ZKUtil.listChildrenAndWatchForNewChildren(zkw, baseNode);
752     if (nodes != null) {
753       List<NodeAndData> newNodes = new ArrayList<NodeAndData>();
754       for (String node : nodes) {
755         String nodePath = ZKUtil.joinZNode(baseNode, node);
756         byte[] data = ZKUtil.getDataAndWatch(zkw, nodePath);
757         newNodes.add(new NodeAndData(nodePath, data));
758       }
759       return newNodes;
760     }
761     return null;
762   }
763 
764   /**
765    * Update the data of an existing node with the expected version to have the
766    * specified data.
767    *
768    * Throws an exception if there is a version mismatch or some other problem.
769    *
770    * Sets no watches under any conditions.
771    *
772    * @param zkw zk reference
773    * @param znode
774    * @param data
775    * @param expectedVersion
776    * @throws KeeperException if unexpected zookeeper exception
777    * @throws KeeperException.BadVersionException if version mismatch
778    * @deprecated Unused
779    */
780   public static void updateExistingNodeData(ZooKeeperWatcher zkw, String znode,
781       byte [] data, int expectedVersion)
782   throws KeeperException {
783     try {
784       zkw.getRecoverableZooKeeper().setData(znode, data, expectedVersion);
785     } catch(InterruptedException ie) {
786       zkw.interruptedException(ie);
787     }
788   }
789 
790   //
791   // Data setting
792   //
793 
794   /**
795    * Sets the data of the existing znode to be the specified data.  Ensures that
796    * the current data has the specified expected version.
797    *
798    * <p>If the node does not exist, a {@link NoNodeException} will be thrown.
799    *
800    * <p>If their is a version mismatch, method returns null.
801    *
802    * <p>No watches are set but setting data will trigger other watchers of this
803    * node.
804    *
805    * <p>If there is another problem, a KeeperException will be thrown.
806    *
807    * @param zkw zk reference
808    * @param znode path of node
809    * @param data data to set for node
810    * @param expectedVersion version expected when setting data
811    * @return true if data set, false if version mismatch
812    * @throws KeeperException if unexpected zookeeper exception
813    */
814   public static boolean setData(ZooKeeperWatcher zkw, String znode,
815       byte [] data, int expectedVersion)
816   throws KeeperException, KeeperException.NoNodeException {
817     try {
818       return zkw.getRecoverableZooKeeper().setData(znode, data, expectedVersion) != null;
819     } catch (InterruptedException e) {
820       zkw.interruptedException(e);
821       return false;
822     }
823   }
824 
825   /**
826    * Set data into node creating node if it doesn't yet exist.
827    * Does not set watch.
828    *
829    * @param zkw zk reference
830    * @param znode path of node
831    * @param data data to set for node
832    * @throws KeeperException
833    */
834   public static void createSetData(final ZooKeeperWatcher zkw, final String znode,
835       final byte [] data)
836   throws KeeperException {
837     if (checkExists(zkw, znode) == -1) {
838       ZKUtil.createWithParents(zkw, znode, data);
839     } else {
840       ZKUtil.setData(zkw, znode, data);
841     }
842   }
843 
844   /**
845    * Sets the data of the existing znode to be the specified data.  The node
846    * must exist but no checks are done on the existing data or version.
847    *
848    * <p>If the node does not exist, a {@link NoNodeException} will be thrown.
849    *
850    * <p>No watches are set but setting data will trigger other watchers of this
851    * node.
852    *
853    * <p>If there is another problem, a KeeperException will be thrown.
854    *
855    * @param zkw zk reference
856    * @param znode path of node
857    * @param data data to set for node
858    * @throws KeeperException if unexpected zookeeper exception
859    */
860   public static void setData(ZooKeeperWatcher zkw, String znode, byte [] data)
861   throws KeeperException, KeeperException.NoNodeException {
862     setData(zkw, (SetData)ZKUtilOp.setData(znode, data));
863   }
864 
865   private static void setData(ZooKeeperWatcher zkw, SetData setData)
866   throws KeeperException, KeeperException.NoNodeException {
867     SetDataRequest sd = (SetDataRequest)toZooKeeperOp(zkw, setData).toRequestRecord();
868     setData(zkw, sd.getPath(), sd.getData(), sd.getVersion());
869   }
870 
871   /**
872    * Returns whether or not secure authentication is enabled
873    * (whether <code>hbase.security.authentication</code> is set to
874    * <code>kerberos</code>.
875    */
876   public static boolean isSecureZooKeeper(Configuration conf) {
877     // Detection for embedded HBase client with jaas configuration
878     // defined for third party programs.
879     try {
880       javax.security.auth.login.Configuration testConfig =
881           javax.security.auth.login.Configuration.getConfiguration();
882       if (testConfig.getAppConfigurationEntry("Client") == null
883           && testConfig.getAppConfigurationEntry(
884             JaasConfiguration.CLIENT_KEYTAB_KERBEROS_CONFIG_NAME) == null
885           && testConfig.getAppConfigurationEntry(
886               JaasConfiguration.SERVER_KEYTAB_KERBEROS_CONFIG_NAME) == null
887           && conf.get(HConstants.ZK_CLIENT_KERBEROS_PRINCIPAL) == null
888           && conf.get(HConstants.ZK_SERVER_KERBEROS_PRINCIPAL) == null) {
889               
890         return false;
891       }
892     } catch(Exception e) {
893       // No Jaas configuration defined.
894       return false;
895     }
896 
897     // Master & RSs uses hbase.zookeeper.client.*
898     return "kerberos".equalsIgnoreCase(conf.get("hbase.security.authentication"));
899   }
900 
901   private static ArrayList<ACL> createACL(ZooKeeperWatcher zkw, String node) {
902     return createACL(zkw, node, isSecureZooKeeper(zkw.getConfiguration()));
903   }
904 
905   public static ArrayList<ACL> createACL(ZooKeeperWatcher zkw, String node,
906     boolean isSecureZooKeeper) {
907     if (!node.startsWith(zkw.baseZNode)) {
908       return Ids.OPEN_ACL_UNSAFE;
909     }
910     if (isSecureZooKeeper) {
911       ArrayList<ACL> acls = new ArrayList<ACL>();
912       // add permission to hbase supper user
913       String[] superUsers = zkw.getConfiguration().getStrings(Superusers.SUPERUSER_CONF_KEY);
914       if (superUsers != null) {
915         List<String> groups = new ArrayList<String>();
916         for (String user : superUsers) {
917           if (user.startsWith(AuthUtil.GROUP_PREFIX)) {
918             // TODO: Set node ACL for groups when ZK supports this feature
919             groups.add(user);
920           } else {
921             acls.add(new ACL(Perms.ALL, new Id("auth", user)));
922           }
923         }
924         if (!groups.isEmpty()) {
925           LOG.warn("Znode ACL setting for group " + groups
926               + " is skipped, Zookeeper doesn't support this feature presently.");
927         }
928       }
929       // Certain znodes are accessed directly by the client,
930       // so they must be readable by non-authenticated clients
931       if (zkw.isClientReadable(node)) {
932         acls.addAll(Ids.CREATOR_ALL_ACL);
933         acls.addAll(Ids.READ_ACL_UNSAFE);
934       } else {
935         acls.addAll(Ids.CREATOR_ALL_ACL);
936       }
937       return acls;
938     } else {
939       return Ids.OPEN_ACL_UNSAFE;
940     }
941   }
942 
943   //
944   // Node creation
945   //
946 
947   /**
948    *
949    * Set the specified znode to be an ephemeral node carrying the specified
950    * data.
951    *
952    * If the node is created successfully, a watcher is also set on the node.
953    *
954    * If the node is not created successfully because it already exists, this
955    * method will also set a watcher on the node.
956    *
957    * If there is another problem, a KeeperException will be thrown.
958    *
959    * @param zkw zk reference
960    * @param znode path of node
961    * @param data data of node
962    * @return true if node created, false if not, watch set in both cases
963    * @throws KeeperException if unexpected zookeeper exception
964    */
965   public static boolean createEphemeralNodeAndWatch(ZooKeeperWatcher zkw,
966       String znode, byte [] data)
967   throws KeeperException {
968     boolean ret = true;
969     try {
970       zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode),
971           CreateMode.EPHEMERAL);
972     } catch (KeeperException.NodeExistsException nee) {
973       ret = false;
974     } catch (InterruptedException e) {
975       LOG.info("Interrupted", e);
976       Thread.currentThread().interrupt();
977     }
978     if(!watchAndCheckExists(zkw, znode)) {
979       // It did exist but now it doesn't, try again
980       return createEphemeralNodeAndWatch(zkw, znode, data);
981     }
982     return ret;
983   }
984 
985   /**
986    * Creates the specified znode to be a persistent node carrying the specified
987    * data.
988    *
989    * Returns true if the node was successfully created, false if the node
990    * already existed.
991    *
992    * If the node is created successfully, a watcher is also set on the node.
993    *
994    * If the node is not created successfully because it already exists, this
995    * method will also set a watcher on the node but return false.
996    *
997    * If there is another problem, a KeeperException will be thrown.
998    *
999    * @param zkw zk reference
1000    * @param znode path of node
1001    * @param data data of node
1002    * @return true if node created, false if not, watch set in both cases
1003    * @throws KeeperException if unexpected zookeeper exception
1004    */
1005   public static boolean createNodeIfNotExistsAndWatch(
1006       ZooKeeperWatcher zkw, String znode, byte [] data)
1007   throws KeeperException {
1008     boolean ret = true;
1009     try {
1010       zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode),
1011           CreateMode.PERSISTENT);
1012     } catch (KeeperException.NodeExistsException nee) {
1013       ret = false;
1014     } catch (InterruptedException e) {
1015       zkw.interruptedException(e);
1016       return false;
1017     }
1018     try {
1019       zkw.getRecoverableZooKeeper().exists(znode, zkw);
1020     } catch (InterruptedException e) {
1021       zkw.interruptedException(e);
1022       return false;
1023     }
1024     return ret;
1025   }
1026 
1027   /**
1028    * Creates the specified znode with the specified data but does not watch it.
1029    *
1030    * Returns the znode of the newly created node
1031    *
1032    * If there is another problem, a KeeperException will be thrown.
1033    *
1034    * @param zkw zk reference
1035    * @param znode path of node
1036    * @param data data of node
1037    * @param createMode specifying whether the node to be created is ephemeral and/or sequential
1038    * @return true name of the newly created znode or null
1039    * @throws KeeperException if unexpected zookeeper exception
1040    */
1041   public static String createNodeIfNotExistsNoWatch(ZooKeeperWatcher zkw, String znode,
1042       byte[] data, CreateMode createMode) throws KeeperException {
1043 
1044     String createdZNode = null;
1045     try {
1046       createdZNode = zkw.getRecoverableZooKeeper().create(znode, data,
1047           createACL(zkw, znode), createMode);
1048     } catch (KeeperException.NodeExistsException nee) {
1049       return znode;
1050     } catch (InterruptedException e) {
1051       zkw.interruptedException(e);
1052       return null;
1053     }
1054     return createdZNode;
1055   }
1056 
1057   /**
1058    * Creates the specified node with the specified data and watches it.
1059    *
1060    * <p>Throws an exception if the node already exists.
1061    *
1062    * <p>The node created is persistent and open access.
1063    *
1064    * <p>Returns the version number of the created node if successful.
1065    *
1066    * @param zkw zk reference
1067    * @param znode path of node to create
1068    * @param data data of node to create
1069    * @return version of node created
1070    * @throws KeeperException if unexpected zookeeper exception
1071    * @throws KeeperException.NodeExistsException if node already exists
1072    */
1073   public static int createAndWatch(ZooKeeperWatcher zkw,
1074       String znode, byte [] data)
1075   throws KeeperException, KeeperException.NodeExistsException {
1076     try {
1077       zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode),
1078           CreateMode.PERSISTENT);
1079       Stat stat = zkw.getRecoverableZooKeeper().exists(znode, zkw);
1080       if (stat == null){
1081         // Likely a race condition. Someone deleted the znode.
1082         throw KeeperException.create(KeeperException.Code.SYSTEMERROR,
1083             "ZK.exists returned null (i.e.: znode does not exist) for znode=" + znode);
1084       }
1085      return stat.getVersion();
1086     } catch (InterruptedException e) {
1087       zkw.interruptedException(e);
1088       return -1;
1089     }
1090   }
1091 
1092   /**
1093    * Async creates the specified node with the specified data.
1094    *
1095    * <p>Throws an exception if the node already exists.
1096    *
1097    * <p>The node created is persistent and open access.
1098    *
1099    * @param zkw zk reference
1100    * @param znode path of node to create
1101    * @param data data of node to create
1102    * @param cb
1103    * @param ctx
1104    */
1105   public static void asyncCreate(ZooKeeperWatcher zkw,
1106       String znode, byte [] data, final AsyncCallback.StringCallback cb,
1107       final Object ctx) {
1108     zkw.getRecoverableZooKeeper().getZooKeeper().create(znode, data,
1109         createACL(zkw, znode), CreateMode.PERSISTENT, cb, ctx);
1110   }
1111 
1112   /**
1113    * Creates the specified node, iff the node does not exist.  Does not set a
1114    * watch and fails silently if the node already exists.
1115    *
1116    * The node created is persistent and open access.
1117    *
1118    * @param zkw zk reference
1119    * @param znode path of node
1120    * @throws KeeperException if unexpected zookeeper exception
1121    */
1122   public static void createAndFailSilent(ZooKeeperWatcher zkw,
1123       String znode) throws KeeperException {
1124     createAndFailSilent(zkw, znode, new byte[0]);
1125   }
1126 
1127   /**
1128    * Creates the specified node containing specified data, iff the node does not exist.  Does
1129    * not set a watch and fails silently if the node already exists.
1130    *
1131    * The node created is persistent and open access.
1132    *
1133    * @param zkw zk reference
1134    * @param znode path of node
1135    * @param data a byte array data to store in the znode
1136    * @throws KeeperException if unexpected zookeeper exception
1137    */
1138   public static void createAndFailSilent(ZooKeeperWatcher zkw,
1139       String znode, byte[] data)
1140   throws KeeperException {
1141     createAndFailSilent(zkw,
1142         (CreateAndFailSilent)ZKUtilOp.createAndFailSilent(znode, data));
1143   }
1144 
1145   private static void createAndFailSilent(ZooKeeperWatcher zkw, CreateAndFailSilent cafs)
1146   throws KeeperException {
1147     CreateRequest create = (CreateRequest)toZooKeeperOp(zkw, cafs).toRequestRecord();
1148     String znode = create.getPath();
1149     try {
1150       RecoverableZooKeeper zk = zkw.getRecoverableZooKeeper();
1151       if (zk.exists(znode, false) == null) {
1152         zk.create(znode, create.getData(), create.getAcl(), CreateMode.fromFlag(create.getFlags()));
1153       }
1154     } catch(KeeperException.NodeExistsException nee) {
1155     } catch(KeeperException.NoAuthException nee){
1156       try {
1157         if (null == zkw.getRecoverableZooKeeper().exists(znode, false)) {
1158           // If we failed to create the file and it does not already exist.
1159           throw(nee);
1160         }
1161       } catch (InterruptedException ie) {
1162         zkw.interruptedException(ie);
1163       }
1164     } catch(InterruptedException ie) {
1165       zkw.interruptedException(ie);
1166     }
1167   }
1168 
1169   /**
1170    * Creates the specified node and all parent nodes required for it to exist.
1171    *
1172    * No watches are set and no errors are thrown if the node already exists.
1173    *
1174    * The nodes created are persistent and open access.
1175    *
1176    * @param zkw zk reference
1177    * @param znode path of node
1178    * @throws KeeperException if unexpected zookeeper exception
1179    */
1180   public static void createWithParents(ZooKeeperWatcher zkw, String znode)
1181   throws KeeperException {
1182     createWithParents(zkw, znode, new byte[0]);
1183   }
1184 
1185   /**
1186    * Creates the specified node and all parent nodes required for it to exist.  The creation of
1187    * parent znodes is not atomic with the leafe znode creation but the data is written atomically
1188    * when the leaf node is created.
1189    *
1190    * No watches are set and no errors are thrown if the node already exists.
1191    *
1192    * The nodes created are persistent and open access.
1193    *
1194    * @param zkw zk reference
1195    * @param znode path of node
1196    * @throws KeeperException if unexpected zookeeper exception
1197    */
1198   public static void createWithParents(ZooKeeperWatcher zkw, String znode, byte[] data)
1199   throws KeeperException {
1200     try {
1201       if(znode == null) {
1202         return;
1203       }
1204       zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode),
1205           CreateMode.PERSISTENT);
1206     } catch(KeeperException.NodeExistsException nee) {
1207       return;
1208     } catch(KeeperException.NoNodeException nne) {
1209       createWithParents(zkw, getParent(znode));
1210       createWithParents(zkw, znode, data);
1211     } catch(InterruptedException ie) {
1212       zkw.interruptedException(ie);
1213     }
1214   }
1215 
1216   //
1217   // Deletes
1218   //
1219 
1220   /**
1221    * Delete the specified node.  Sets no watches.  Throws all exceptions.
1222    */
1223   public static void deleteNode(ZooKeeperWatcher zkw, String node)
1224   throws KeeperException {
1225     deleteNode(zkw, node, -1);
1226   }
1227 
1228   /**
1229    * Delete the specified node with the specified version.  Sets no watches.
1230    * Throws all exceptions.
1231    */
1232   public static boolean deleteNode(ZooKeeperWatcher zkw, String node,
1233       int version)
1234   throws KeeperException {
1235     try {
1236       zkw.getRecoverableZooKeeper().delete(node, version);
1237       return true;
1238     } catch(KeeperException.BadVersionException bve) {
1239       return false;
1240     } catch(InterruptedException ie) {
1241       zkw.interruptedException(ie);
1242       return false;
1243     }
1244   }
1245 
1246   /**
1247    * Deletes the specified node.  Fails silent if the node does not exist.
1248    * @param zkw
1249    * @param node
1250    * @throws KeeperException
1251    */
1252   public static void deleteNodeFailSilent(ZooKeeperWatcher zkw, String node)
1253   throws KeeperException {
1254     deleteNodeFailSilent(zkw,
1255       (DeleteNodeFailSilent)ZKUtilOp.deleteNodeFailSilent(node));
1256   }
1257 
1258   private static void deleteNodeFailSilent(ZooKeeperWatcher zkw,
1259       DeleteNodeFailSilent dnfs) throws KeeperException {
1260     DeleteRequest delete = (DeleteRequest)toZooKeeperOp(zkw, dnfs).toRequestRecord();
1261     try {
1262       zkw.getRecoverableZooKeeper().delete(delete.getPath(), delete.getVersion());
1263     } catch(KeeperException.NoNodeException nne) {
1264     } catch(InterruptedException ie) {
1265       zkw.interruptedException(ie);
1266     }
1267   }
1268 
1269 
1270   /**
1271    * Delete the specified node and all of it's children.
1272    * <p>
1273    * If the node does not exist, just returns.
1274    * <p>
1275    * Sets no watches. Throws all exceptions besides dealing with deletion of
1276    * children.
1277    */
1278   public static void deleteNodeRecursively(ZooKeeperWatcher zkw, String node)
1279   throws KeeperException {
1280     deleteNodeRecursivelyMultiOrSequential(zkw, true, node);
1281   }
1282 
1283   /**
1284    * Delete all the children of the specified node but not the node itself.
1285    *
1286    * Sets no watches.  Throws all exceptions besides dealing with deletion of
1287    * children.
1288    *
1289    * If hbase.zookeeper.useMulti is true, use ZooKeeper's multi-update functionality.
1290    * Otherwise, run the list of operations sequentially.
1291    *
1292    * @throws KeeperException
1293    */
1294   public static void deleteChildrenRecursively(ZooKeeperWatcher zkw, String node)
1295       throws KeeperException {
1296     deleteChildrenRecursivelyMultiOrSequential(zkw, true, node);
1297   }
1298 
1299   /**
1300    * Delete all the children of the specified node but not the node itself. This
1301    * will first traverse the znode tree for listing the children and then delete
1302    * these znodes using multi-update api or sequential based on the specified
1303    * configurations.
1304    * <p>
1305    * Sets no watches. Throws all exceptions besides dealing with deletion of
1306    * children.
1307    * <p>
1308    * If hbase.zookeeper.useMulti is true, use ZooKeeper's multi-update
1309    * functionality. Otherwise, run the list of operations sequentially.
1310    * <p>
1311    * If all of the following are true:
1312    * <ul>
1313    * <li>runSequentialOnMultiFailure is true
1314    * <li>hbase.zookeeper.useMulti is true
1315    * </ul>
1316    * on calling multi, we get a ZooKeeper exception that can be handled by a
1317    * sequential call(*), we retry the operations one-by-one (sequentially).
1318    *
1319    * @param zkw
1320    *          - zk reference
1321    * @param runSequentialOnMultiFailure
1322    *          - if true when we get a ZooKeeper exception that could retry the
1323    *          operations one-by-one (sequentially)
1324    * @param pathRoots
1325    *          - path of the parent node(s)
1326    * @throws KeeperException.NotEmptyException
1327    *           if node has children while deleting
1328    * @throws KeeperException
1329    *           if unexpected ZooKeeper exception
1330    * @throws IllegalArgumentException
1331    *           if an invalid path is specified
1332    */
1333   public static void deleteChildrenRecursivelyMultiOrSequential(
1334       ZooKeeperWatcher zkw, boolean runSequentialOnMultiFailure,
1335       String... pathRoots) throws KeeperException {
1336     if (pathRoots == null || pathRoots.length <= 0) {
1337       LOG.warn("Given path is not valid!");
1338       return;
1339     }
1340     List<ZKUtilOp> ops = new ArrayList<ZKUtil.ZKUtilOp>();
1341     for (String eachRoot : pathRoots) {
1342       List<String> children = listChildrenBFSNoWatch(zkw, eachRoot);
1343       // Delete the leaves first and eventually get rid of the root
1344       for (int i = children.size() - 1; i >= 0; --i) {
1345         ops.add(ZKUtilOp.deleteNodeFailSilent(children.get(i)));
1346       }
1347     }
1348     // atleast one element should exist
1349     if (ops.size() > 0) {
1350       multiOrSequential(zkw, ops, runSequentialOnMultiFailure);
1351     }
1352   }
1353 
1354   /**
1355    * Delete the specified node and its children. This traverse the
1356    * znode tree for listing the children and then delete
1357    * these znodes including the parent using multi-update api or
1358    * sequential based on the specified configurations.
1359    * <p>
1360    * Sets no watches. Throws all exceptions besides dealing with deletion of
1361    * children.
1362    * <p>
1363    * If hbase.zookeeper.useMulti is true, use ZooKeeper's multi-update
1364    * functionality. Otherwise, run the list of operations sequentially.
1365    * <p>
1366    * If all of the following are true:
1367    * <ul>
1368    * <li>runSequentialOnMultiFailure is true
1369    * <li>hbase.zookeeper.useMulti is true
1370    * </ul>
1371    * on calling multi, we get a ZooKeeper exception that can be handled by a
1372    * sequential call(*), we retry the operations one-by-one (sequentially).
1373    *
1374    * @param zkw
1375    *          - zk reference
1376    * @param runSequentialOnMultiFailure
1377    *          - if true when we get a ZooKeeper exception that could retry the
1378    *          operations one-by-one (sequentially)
1379    * @param pathRoots
1380    *          - path of the parent node(s)
1381    * @throws KeeperException.NotEmptyException
1382    *           if node has children while deleting
1383    * @throws KeeperException
1384    *           if unexpected ZooKeeper exception
1385    * @throws IllegalArgumentException
1386    *           if an invalid path is specified
1387    */
1388   public static void deleteNodeRecursivelyMultiOrSequential(ZooKeeperWatcher zkw,
1389       boolean runSequentialOnMultiFailure, String... pathRoots) throws KeeperException {
1390     if (pathRoots == null || pathRoots.length <= 0) {
1391       LOG.warn("Given path is not valid!");
1392       return;
1393     }
1394     List<ZKUtilOp> ops = new ArrayList<ZKUtil.ZKUtilOp>();
1395     for (String eachRoot : pathRoots) {
1396       // Zookeeper Watches are one time triggers; When children of parent nodes are deleted
1397       // recursively, must set another watch, get notified of delete node
1398       List<String> children = listChildrenBFSAndWatchThem(zkw, eachRoot);
1399       // Delete the leaves first and eventually get rid of the root
1400       for (int i = children.size() - 1; i >= 0; --i) {
1401         ops.add(ZKUtilOp.deleteNodeFailSilent(children.get(i)));
1402       }
1403       try {
1404         if (zkw.getRecoverableZooKeeper().exists(eachRoot, zkw) != null) {
1405           ops.add(ZKUtilOp.deleteNodeFailSilent(eachRoot));
1406         }
1407       } catch (InterruptedException e) {
1408         zkw.interruptedException(e);
1409       }
1410     }
1411     // atleast one element should exist
1412     if (ops.size() > 0) {
1413       multiOrSequential(zkw, ops, runSequentialOnMultiFailure);
1414     }
1415   }
1416 
1417   /**
1418    * BFS Traversal of all the children under path, with the entries in the list,
1419    * in the same order as that of the traversal. Lists all the children without
1420    * setting any watches.
1421    *
1422    * @param zkw
1423    *          - zk reference
1424    * @param znode
1425    *          - path of node
1426    * @return list of children znodes under the path
1427    * @throws KeeperException
1428    *           if unexpected ZooKeeper exception
1429    */
1430   private static List<String> listChildrenBFSNoWatch(ZooKeeperWatcher zkw,
1431       final String znode) throws KeeperException {
1432     Deque<String> queue = new LinkedList<String>();
1433     List<String> tree = new ArrayList<String>();
1434     queue.add(znode);
1435     while (true) {
1436       String node = queue.pollFirst();
1437       if (node == null) {
1438         break;
1439       }
1440       List<String> children = listChildrenNoWatch(zkw, node);
1441       if (children == null) {
1442         continue;
1443       }
1444       for (final String child : children) {
1445         final String childPath = node + "/" + child;
1446         queue.add(childPath);
1447         tree.add(childPath);
1448       }
1449     }
1450     return tree;
1451   }
1452 
1453   /**
1454    * BFS Traversal of all the children under path, with the entries in the list,
1455    * in the same order as that of the traversal.
1456    * Lists all the children and set watches on to them.
1457    *
1458    * @param zkw
1459    *          - zk reference
1460    * @param znode
1461    *          - path of node
1462    * @return list of children znodes under the path
1463    * @throws KeeperException
1464    *           if unexpected ZooKeeper exception
1465    */
1466   private static List<String> listChildrenBFSAndWatchThem(ZooKeeperWatcher zkw, final String znode)
1467       throws KeeperException {
1468     Deque<String> queue = new LinkedList<String>();
1469     List<String> tree = new ArrayList<String>();
1470     queue.add(znode);
1471     while (true) {
1472       String node = queue.pollFirst();
1473       if (node == null) {
1474         break;
1475       }
1476       List<String> children = listChildrenAndWatchThem(zkw, node);
1477       if (children == null) {
1478         continue;
1479       }
1480       for (final String child : children) {
1481         final String childPath = node + "/" + child;
1482         queue.add(childPath);
1483         tree.add(childPath);
1484       }
1485     }
1486     return tree;
1487   }
1488 
1489   /**
1490    * Represents an action taken by ZKUtil, e.g. createAndFailSilent.
1491    * These actions are higher-level than ZKOp actions, which represent
1492    * individual actions in the ZooKeeper API, like create.
1493    */
1494   public abstract static class ZKUtilOp {
1495     private String path;
1496 
1497     private ZKUtilOp(String path) {
1498       this.path = path;
1499     }
1500 
1501     /**
1502      * @return a createAndFailSilent ZKUtilOp
1503      */
1504     public static ZKUtilOp createAndFailSilent(String path, byte[] data) {
1505       return new CreateAndFailSilent(path, data);
1506     }
1507 
1508     /**
1509      * @return a deleteNodeFailSilent ZKUtilOP
1510      */
1511     public static ZKUtilOp deleteNodeFailSilent(String path) {
1512       return new DeleteNodeFailSilent(path);
1513     }
1514 
1515     /**
1516      * @return a setData ZKUtilOp
1517      */
1518     public static ZKUtilOp setData(String path, byte [] data) {
1519       return new SetData(path, data);
1520     }
1521 
1522     /**
1523      * @return path to znode where the ZKOp will occur
1524      */
1525     public String getPath() {
1526       return path;
1527     }
1528 
1529     /**
1530      * ZKUtilOp representing createAndFailSilent in ZooKeeper
1531      * (attempt to create node, ignore error if already exists)
1532      */
1533     public static class CreateAndFailSilent extends ZKUtilOp {
1534       private byte [] data;
1535 
1536       private CreateAndFailSilent(String path, byte [] data) {
1537         super(path);
1538         this.data = data;
1539       }
1540 
1541       public byte[] getData() {
1542         return data;
1543       }
1544 
1545       @Override
1546       public boolean equals(Object o) {
1547         if (this == o) return true;
1548         if (!(o instanceof CreateAndFailSilent)) return false;
1549 
1550         CreateAndFailSilent op = (CreateAndFailSilent) o;
1551         return getPath().equals(op.getPath()) && Arrays.equals(data, op.data);
1552       }
1553 
1554       @Override
1555       public int hashCode() {
1556         int ret = 17 + getPath().hashCode() * 31;
1557         return ret * 31 + Bytes.hashCode(data);
1558       }
1559     }
1560 
1561     /**
1562      * ZKUtilOp representing deleteNodeFailSilent in ZooKeeper
1563      * (attempt to delete node, ignore error if node doesn't exist)
1564      */
1565     public static class DeleteNodeFailSilent extends ZKUtilOp {
1566       private DeleteNodeFailSilent(String path) {
1567         super(path);
1568       }
1569 
1570       @Override
1571       public boolean equals(Object o) {
1572         if (this == o) return true;
1573         if (!(o instanceof DeleteNodeFailSilent)) return false;
1574 
1575         return super.equals(o);
1576       }
1577 
1578       @Override
1579       public int hashCode() {
1580         return getPath().hashCode();
1581       }
1582     }
1583 
1584     /**
1585      * ZKUtilOp representing setData in ZooKeeper
1586      */
1587     public static class SetData extends ZKUtilOp {
1588       private byte [] data;
1589 
1590       private SetData(String path, byte [] data) {
1591         super(path);
1592         this.data = data;
1593       }
1594 
1595       public byte[] getData() {
1596         return data;
1597       }
1598 
1599       @Override
1600       public boolean equals(Object o) {
1601         if (this == o) return true;
1602         if (!(o instanceof SetData)) return false;
1603 
1604         SetData op = (SetData) o;
1605         return getPath().equals(op.getPath()) && Arrays.equals(data, op.data);
1606       }
1607 
1608       @Override
1609       public int hashCode() {
1610         int ret = getPath().hashCode();
1611         return ret * 31 + Bytes.hashCode(data);
1612       }
1613     }
1614   }
1615 
1616   /**
1617    * Convert from ZKUtilOp to ZKOp
1618    */
1619   private static Op toZooKeeperOp(ZooKeeperWatcher zkw, ZKUtilOp op)
1620   throws UnsupportedOperationException {
1621     if(op == null) return null;
1622 
1623     if (op instanceof CreateAndFailSilent) {
1624       CreateAndFailSilent cafs = (CreateAndFailSilent)op;
1625       return Op.create(cafs.getPath(), cafs.getData(), createACL(zkw, cafs.getPath()),
1626         CreateMode.PERSISTENT);
1627     } else if (op instanceof DeleteNodeFailSilent) {
1628       DeleteNodeFailSilent dnfs = (DeleteNodeFailSilent)op;
1629       return Op.delete(dnfs.getPath(), -1);
1630     } else if (op instanceof SetData) {
1631       SetData sd = (SetData)op;
1632       return Op.setData(sd.getPath(), sd.getData(), -1);
1633     } else {
1634       throw new UnsupportedOperationException("Unexpected ZKUtilOp type: "
1635         + op.getClass().getName());
1636     }
1637   }
1638 
1639   /**
1640    * If hbase.zookeeper.useMulti is true, use ZooKeeper's multi-update functionality.
1641    * Otherwise, run the list of operations sequentially.
1642    *
1643    * If all of the following are true:
1644    * - runSequentialOnMultiFailure is true
1645    * - hbase.zookeeper.useMulti is true
1646    * - on calling multi, we get a ZooKeeper exception that can be handled by a sequential call(*)
1647    * Then:
1648    * - we retry the operations one-by-one (sequentially)
1649    *
1650    * Note *: an example is receiving a NodeExistsException from a "create" call.  Without multi,
1651    * a user could call "createAndFailSilent" to ensure that a node exists if they don't care who
1652    * actually created the node (i.e. the NodeExistsException from ZooKeeper is caught).
1653    * This will cause all operations in the multi to fail, however, because
1654    * the NodeExistsException that zk.create throws will fail the multi transaction.
1655    * In this case, if the previous conditions hold, the commands are run sequentially, which should
1656    * result in the correct final state, but means that the operations will not run atomically.
1657    *
1658    * @throws KeeperException
1659    */
1660   public static void multiOrSequential(ZooKeeperWatcher zkw, List<ZKUtilOp> ops,
1661       boolean runSequentialOnMultiFailure) throws KeeperException {
1662     if (ops == null) return;
1663     boolean useMulti = zkw.getConfiguration().getBoolean(HConstants.ZOOKEEPER_USEMULTI, false);
1664 
1665     if (useMulti) {
1666       List<Op> zkOps = new LinkedList<Op>();
1667       for (ZKUtilOp op : ops) {
1668         zkOps.add(toZooKeeperOp(zkw, op));
1669       }
1670       try {
1671         zkw.getRecoverableZooKeeper().multi(zkOps);
1672       } catch (KeeperException ke) {
1673        switch (ke.code()) {
1674          case NODEEXISTS:
1675          case NONODE:
1676          case BADVERSION:
1677          case NOAUTH:
1678            // if we get an exception that could be solved by running sequentially
1679            // (and the client asked us to), then break out and run sequentially
1680            if (runSequentialOnMultiFailure) {
1681              LOG.info("On call to ZK.multi, received exception: " + ke.toString() + "."
1682                + "  Attempting to run operations sequentially because"
1683                + " runSequentialOnMultiFailure is: " + runSequentialOnMultiFailure + ".");
1684              processSequentially(zkw, ops);
1685              break;
1686            }
1687           default:
1688             throw ke;
1689         }
1690       } catch (InterruptedException ie) {
1691         zkw.interruptedException(ie);
1692       }
1693     } else {
1694       // run sequentially
1695       processSequentially(zkw, ops);
1696     }
1697 
1698   }
1699 
1700   private static void processSequentially(ZooKeeperWatcher zkw, List<ZKUtilOp> ops)
1701       throws KeeperException, NoNodeException {
1702     for (ZKUtilOp op : ops) {
1703       if (op instanceof CreateAndFailSilent) {
1704         createAndFailSilent(zkw, (CreateAndFailSilent) op);
1705       } else if (op instanceof DeleteNodeFailSilent) {
1706         deleteNodeFailSilent(zkw, (DeleteNodeFailSilent) op);
1707       } else if (op instanceof SetData) {
1708         setData(zkw, (SetData) op);
1709       } else {
1710         throw new UnsupportedOperationException("Unexpected ZKUtilOp type: "
1711             + op.getClass().getName());
1712       }
1713     }
1714   }
1715 
1716   //
1717   // ZooKeeper cluster information
1718   //
1719 
1720   /** @return String dump of everything in ZooKeeper. */
1721   public static String dump(ZooKeeperWatcher zkw) {
1722     StringBuilder sb = new StringBuilder();
1723     try {
1724       sb.append("HBase is rooted at ").append(zkw.baseZNode);
1725       sb.append("\nActive master address: ");
1726       try {
1727         sb.append(MasterAddressTracker.getMasterAddress(zkw));
1728       } catch (IOException e) {
1729         sb.append("<<FAILED LOOKUP: " + e.getMessage() + ">>");
1730       }
1731       sb.append("\nBackup master addresses:");
1732       for (String child : listChildrenNoWatch(zkw,
1733                                               zkw.backupMasterAddressesZNode)) {
1734         sb.append("\n ").append(child);
1735       }
1736       sb.append("\nRegion server holding hbase:meta: "
1737         + new MetaTableLocator().getMetaRegionLocation(zkw));
1738       Configuration conf = HBaseConfiguration.create();
1739       int numMetaReplicas = conf.getInt(HConstants.META_REPLICAS_NUM,
1740                HConstants.DEFAULT_META_REPLICA_NUM);
1741       for (int i = 1; i < numMetaReplicas; i++) {
1742         sb.append("\nRegion server holding hbase:meta, replicaId " + i + " "
1743                     + new MetaTableLocator().getMetaRegionLocation(zkw, i));
1744       }
1745       sb.append("\nRegion servers:");
1746       for (String child : listChildrenNoWatch(zkw, zkw.rsZNode)) {
1747         sb.append("\n ").append(child);
1748       }
1749       try {
1750         getReplicationZnodesDump(zkw, sb);
1751       } catch (KeeperException ke) {
1752         LOG.warn("Couldn't get the replication znode dump", ke);
1753       }
1754       sb.append("\nQuorum Server Statistics:");
1755       String[] servers = zkw.getQuorum().split(",");
1756       for (String server : servers) {
1757         sb.append("\n ").append(server);
1758         try {
1759           String[] stat = getServerStats(server, ZKUtil.zkDumpConnectionTimeOut);
1760 
1761           if (stat == null) {
1762             sb.append("[Error] invalid quorum server: " + server);
1763             break;
1764           }
1765 
1766           for (String s : stat) {
1767             sb.append("\n  ").append(s);
1768           }
1769         } catch (Exception e) {
1770           sb.append("\n  ERROR: ").append(e.getMessage());
1771         }
1772       }
1773     } catch (KeeperException ke) {
1774       sb.append("\nFATAL ZooKeeper Exception!\n");
1775       sb.append("\n" + ke.getMessage());
1776     }
1777     return sb.toString();
1778   }
1779 
1780   /**
1781    * Appends replication znodes to the passed StringBuilder.
1782    * @param zkw
1783    * @param sb
1784    * @throws KeeperException
1785    */
1786   private static void getReplicationZnodesDump(ZooKeeperWatcher zkw, StringBuilder sb)
1787       throws KeeperException {
1788     String replicationZNodeName = zkw.getConfiguration().get("zookeeper.znode.replication",
1789       "replication");
1790     String replicationZnode = joinZNode(zkw.baseZNode, replicationZNodeName);
1791     if (ZKUtil.checkExists(zkw, replicationZnode) == -1) return;
1792     // do a ls -r on this znode
1793     sb.append("\n").append(replicationZnode).append(": ");
1794     List<String> children = ZKUtil.listChildrenNoWatch(zkw, replicationZnode);
1795     for (String child : children) {
1796       String znode = joinZNode(replicationZnode, child);
1797       if (child.equals(zkw.getConfiguration().get("zookeeper.znode.replication.peers", "peers"))) {
1798         appendPeersZnodes(zkw, znode, sb);
1799       } else if (child.equals(zkw.getConfiguration().
1800           get("zookeeper.znode.replication.rs", "rs"))) {
1801         appendRSZnodes(zkw, znode, sb);
1802       }
1803     }
1804   }
1805 
1806   private static void appendRSZnodes(ZooKeeperWatcher zkw, String znode, StringBuilder sb)
1807       throws KeeperException {
1808     List<String> stack = new LinkedList<String>();
1809     stack.add(znode);
1810     do {
1811       String znodeToProcess = stack.remove(stack.size() - 1);
1812       sb.append("\n").append(znodeToProcess).append(": ");
1813       byte[] data;
1814       try {
1815         data = ZKUtil.getData(zkw, znodeToProcess);
1816       } catch (InterruptedException e) {
1817         zkw.interruptedException(e);
1818         return;
1819       }
1820       if (data != null && data.length > 0) { // log position
1821         long position = 0;
1822         try {
1823           position = ZKUtil.parseWALPositionFrom(ZKUtil.getData(zkw, znodeToProcess));
1824           sb.append(position);
1825         } catch (DeserializationException ignored) {
1826         } catch (InterruptedException e) {
1827           zkw.interruptedException(e);
1828           return;
1829         }
1830       }
1831       for (String zNodeChild : ZKUtil.listChildrenNoWatch(zkw, znodeToProcess)) {
1832         stack.add(ZKUtil.joinZNode(znodeToProcess, zNodeChild));
1833       }
1834     } while (stack.size() > 0);
1835   }
1836 
1837   private static void appendPeersZnodes(ZooKeeperWatcher zkw, String peersZnode,
1838     StringBuilder sb) throws KeeperException {
1839     int pblen = ProtobufUtil.lengthOfPBMagic();
1840     sb.append("\n").append(peersZnode).append(": ");
1841     for (String peerIdZnode : ZKUtil.listChildrenNoWatch(zkw, peersZnode)) {
1842       String znodeToProcess = ZKUtil.joinZNode(peersZnode, peerIdZnode);
1843       byte[] data;
1844       try {
1845         data = ZKUtil.getData(zkw, znodeToProcess);
1846       } catch (InterruptedException e) {
1847         zkw.interruptedException(e);
1848         return;
1849       }
1850       // parse the data of the above peer znode.
1851       try {
1852         ZooKeeperProtos.ReplicationPeer.Builder builder =
1853           ZooKeeperProtos.ReplicationPeer.newBuilder();
1854         ProtobufUtil.mergeFrom(builder, data, pblen, data.length - pblen);
1855         String clusterKey = builder.getClusterkey();
1856         sb.append("\n").append(znodeToProcess).append(": ").append(clusterKey);
1857         // add the peer-state.
1858         appendPeerState(zkw, znodeToProcess, sb);
1859       } catch (IOException ipbe) {
1860         LOG.warn("Got Exception while parsing peer: " + znodeToProcess, ipbe);
1861       }
1862     }
1863   }
1864 
1865   private static void appendPeerState(ZooKeeperWatcher zkw, String znodeToProcess,
1866       StringBuilder sb) throws KeeperException, InvalidProtocolBufferException {
1867     String peerState = zkw.getConfiguration().get("zookeeper.znode.replication.peers.state",
1868       "peer-state");
1869     int pblen = ProtobufUtil.lengthOfPBMagic();
1870     for (String child : ZKUtil.listChildrenNoWatch(zkw, znodeToProcess)) {
1871       if (!child.equals(peerState)) continue;
1872       String peerStateZnode = ZKUtil.joinZNode(znodeToProcess, child);
1873       sb.append("\n").append(peerStateZnode).append(": ");
1874       byte[] peerStateData;
1875       try {
1876         peerStateData = ZKUtil.getData(zkw, peerStateZnode);
1877         ZooKeeperProtos.ReplicationState.Builder builder =
1878             ZooKeeperProtos.ReplicationState.newBuilder();
1879         ProtobufUtil.mergeFrom(builder, peerStateData, pblen, peerStateData.length - pblen);
1880         sb.append(builder.getState().name());
1881       } catch (IOException ipbe) {
1882         LOG.warn("Got Exception while parsing peer: " + znodeToProcess, ipbe);
1883       } catch (InterruptedException e) {
1884         zkw.interruptedException(e);
1885         return;
1886       }
1887     }
1888   }
1889 
1890   /**
1891    * Gets the statistics from the given server.
1892    *
1893    * @param server  The server to get the statistics from.
1894    * @param timeout  The socket timeout to use.
1895    * @return The array of response strings.
1896    * @throws IOException When the socket communication fails.
1897    */
1898   public static String[] getServerStats(String server, int timeout)
1899   throws IOException {
1900     String[] sp = server.split(":");
1901     if (sp == null || sp.length == 0) {
1902       return null;
1903     }
1904 
1905     String host = sp[0];
1906     int port = sp.length > 1 ? Integer.parseInt(sp[1])
1907         : HConstants.DEFAULT_ZOOKEPER_CLIENT_PORT;
1908 
1909     Socket socket = new Socket();
1910     InetSocketAddress sockAddr = new InetSocketAddress(host, port);
1911     socket.connect(sockAddr, timeout);
1912 
1913     socket.setSoTimeout(timeout);
1914     PrintWriter out = new PrintWriter(socket.getOutputStream(), true);
1915     BufferedReader in = new BufferedReader(new InputStreamReader(
1916       socket.getInputStream()));
1917     out.println("stat");
1918     out.flush();
1919     ArrayList<String> res = new ArrayList<String>();
1920     while (true) {
1921       String line = in.readLine();
1922       if (line != null) {
1923         res.add(line);
1924       } else {
1925         break;
1926       }
1927     }
1928     socket.close();
1929     return res.toArray(new String[res.size()]);
1930   }
1931 
1932   private static void logRetrievedMsg(final ZooKeeperWatcher zkw,
1933       final String znode, final byte [] data, final boolean watcherSet) {
1934     if (!LOG.isTraceEnabled()) return;
1935     LOG.trace(zkw.prefix("Retrieved " + ((data == null)? 0: data.length) +
1936       " byte(s) of data from znode " + znode +
1937       (watcherSet? " and set watcher; ": "; data=") +
1938       (data == null? "null": data.length == 0? "empty": (
1939           znode.startsWith(zkw.assignmentZNode)?
1940             ZKAssign.toString(data): // We should not be doing this reaching into another class
1941           znode.startsWith(ZooKeeperWatcher.META_ZNODE_PREFIX)?
1942             getServerNameOrEmptyString(data):
1943           znode.startsWith(zkw.backupMasterAddressesZNode)?
1944             getServerNameOrEmptyString(data):
1945           StringUtils.abbreviate(Bytes.toStringBinary(data), 32)))));
1946   }
1947 
1948   private static String getServerNameOrEmptyString(final byte [] data) {
1949     try {
1950       return ServerName.parseFrom(data).toString();
1951     } catch (DeserializationException e) {
1952       return "";
1953     }
1954   }
1955 
1956   /**
1957    * Waits for HBase installation's base (parent) znode to become available.
1958    * @throws IOException on ZK errors
1959    */
1960   public static void waitForBaseZNode(Configuration conf) throws IOException {
1961     LOG.info("Waiting until the base znode is available");
1962     String parentZNode = conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT,
1963         HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT);
1964     ZooKeeper zk = new ZooKeeper(ZKConfig.getZKQuorumServersString(conf),
1965         conf.getInt(HConstants.ZK_SESSION_TIMEOUT,
1966         HConstants.DEFAULT_ZK_SESSION_TIMEOUT), EmptyWatcher.instance);
1967 
1968     final int maxTimeMs = 10000;
1969     final int maxNumAttempts = maxTimeMs / HConstants.SOCKET_RETRY_WAIT_MS;
1970 
1971     KeeperException keeperEx = null;
1972     try {
1973       try {
1974         for (int attempt = 0; attempt < maxNumAttempts; ++attempt) {
1975           try {
1976             if (zk.exists(parentZNode, false) != null) {
1977               LOG.info("Parent znode exists: " + parentZNode);
1978               keeperEx = null;
1979               break;
1980             }
1981           } catch (KeeperException e) {
1982             keeperEx = e;
1983           }
1984           Threads.sleepWithoutInterrupt(HConstants.SOCKET_RETRY_WAIT_MS);
1985         }
1986       } finally {
1987         zk.close();
1988       }
1989     } catch (InterruptedException ex) {
1990       Thread.currentThread().interrupt();
1991     }
1992 
1993     if (keeperEx != null) {
1994       throw new IOException(keeperEx);
1995     }
1996   }
1997 
1998 
1999   public static byte[] blockUntilAvailable(
2000     final ZooKeeperWatcher zkw, final String znode, final long timeout)
2001     throws InterruptedException {
2002     if (timeout < 0) throw new IllegalArgumentException();
2003     if (zkw == null) throw new IllegalArgumentException();
2004     if (znode == null) throw new IllegalArgumentException();
2005 
2006     byte[] data = null;
2007     boolean finished = false;
2008     final long endTime = System.currentTimeMillis() + timeout;
2009     while (!finished) {
2010       try {
2011         data = ZKUtil.getData(zkw, znode);
2012       } catch(KeeperException e) {
2013         if (e instanceof KeeperException.SessionExpiredException
2014             || e instanceof KeeperException.AuthFailedException) {
2015           // non-recoverable errors so stop here
2016           throw new InterruptedException("interrupted due to " + e);
2017         }
2018         LOG.warn("Unexpected exception handling blockUntilAvailable", e);
2019       }
2020 
2021       if (data == null && (System.currentTimeMillis() +
2022         HConstants.SOCKET_RETRY_WAIT_MS < endTime)) {
2023         Thread.sleep(HConstants.SOCKET_RETRY_WAIT_MS);
2024       } else {
2025         finished = true;
2026       }
2027     }
2028 
2029     return data;
2030   }
2031 
2032 
2033   /**
2034    * Convert a {@link DeserializationException} to a more palatable {@link KeeperException}.
2035    * Used when can't let a {@link DeserializationException} out w/o changing public API.
2036    * @param e Exception to convert
2037    * @return Converted exception
2038    */
2039   public static KeeperException convert(final DeserializationException e) {
2040     KeeperException ke = new KeeperException.DataInconsistencyException();
2041     ke.initCause(e);
2042     return ke;
2043   }
2044 
2045   /**
2046    * Recursively print the current state of ZK (non-transactional)
2047    * @param root name of the root directory in zk to print
2048    */
2049   public static void logZKTree(ZooKeeperWatcher zkw, String root) {
2050     if (!LOG.isDebugEnabled()) return;
2051     LOG.debug("Current zk system:");
2052     String prefix = "|-";
2053     LOG.debug(prefix + root);
2054     try {
2055       logZKTree(zkw, root, prefix);
2056     } catch (KeeperException e) {
2057       throw new RuntimeException(e);
2058     }
2059   }
2060 
2061   /**
2062    * Helper method to print the current state of the ZK tree.
2063    * @see #logZKTree(ZooKeeperWatcher, String)
2064    * @throws KeeperException if an unexpected exception occurs
2065    */
2066   protected static void logZKTree(ZooKeeperWatcher zkw, String root, String prefix)
2067       throws KeeperException {
2068     List<String> children = ZKUtil.listChildrenNoWatch(zkw, root);
2069     if (children == null) return;
2070     for (String child : children) {
2071       LOG.debug(prefix + child);
2072       String node = ZKUtil.joinZNode(root.equals("/") ? "" : root, child);
2073       logZKTree(zkw, node, prefix + "---");
2074     }
2075   }
2076 
2077   /**
2078    * @param position
2079    * @return Serialized protobuf of <code>position</code> with pb magic prefix prepended suitable
2080    *         for use as content of an wal position in a replication queue.
2081    */
2082   public static byte[] positionToByteArray(final long position) {
2083     byte[] bytes = ZooKeeperProtos.ReplicationHLogPosition.newBuilder().setPosition(position)
2084         .build().toByteArray();
2085     return ProtobufUtil.prependPBMagic(bytes);
2086   }
2087 
2088   /**
2089    * @param bytes - Content of a WAL position znode.
2090    * @return long - The current WAL position.
2091    * @throws DeserializationException
2092    */
2093   public static long parseWALPositionFrom(final byte[] bytes) throws DeserializationException {
2094     if (bytes == null) {
2095       throw new DeserializationException("Unable to parse null WAL position.");
2096     }
2097     if (ProtobufUtil.isPBMagicPrefix(bytes)) {
2098       int pblen = ProtobufUtil.lengthOfPBMagic();
2099       ZooKeeperProtos.ReplicationHLogPosition.Builder builder =
2100           ZooKeeperProtos.ReplicationHLogPosition.newBuilder();
2101       ZooKeeperProtos.ReplicationHLogPosition position;
2102       try {
2103         ProtobufUtil.mergeFrom(builder, bytes, pblen, bytes.length - pblen);
2104         position = builder.build();
2105       } catch (IOException e) {
2106         throw new DeserializationException(e);
2107       }
2108       return position.getPosition();
2109     } else {
2110       if (bytes.length > 0) {
2111         return Bytes.toLong(bytes);
2112       }
2113       return 0;
2114     }
2115   }
2116 
2117   /**
2118    * @param regionLastFlushedSequenceId the flushed sequence id of a region which is the min of its
2119    *          store max seq ids
2120    * @param storeSequenceIds column family to sequence Id map
2121    * @return Serialized protobuf of <code>RegionSequenceIds</code> with pb magic prefix prepended
2122    *         suitable for use to filter wal edits in distributedLogReplay mode
2123    */
2124   public static byte[] regionSequenceIdsToByteArray(final Long regionLastFlushedSequenceId,
2125       final Map<byte[], Long> storeSequenceIds) {
2126     ClusterStatusProtos.RegionStoreSequenceIds.Builder regionSequenceIdsBuilder =
2127         ClusterStatusProtos.RegionStoreSequenceIds.newBuilder();
2128     ClusterStatusProtos.StoreSequenceId.Builder storeSequenceIdBuilder =
2129         ClusterStatusProtos.StoreSequenceId.newBuilder();
2130     if (storeSequenceIds != null) {
2131       for (Map.Entry<byte[], Long> e : storeSequenceIds.entrySet()){
2132         byte[] columnFamilyName = e.getKey();
2133         Long curSeqId = e.getValue();
2134         storeSequenceIdBuilder.setFamilyName(ByteStringer.wrap(columnFamilyName));
2135         storeSequenceIdBuilder.setSequenceId(curSeqId);
2136         regionSequenceIdsBuilder.addStoreSequenceId(storeSequenceIdBuilder.build());
2137         storeSequenceIdBuilder.clear();
2138       }
2139     }
2140     regionSequenceIdsBuilder.setLastFlushedSequenceId(regionLastFlushedSequenceId);
2141     byte[] result = regionSequenceIdsBuilder.build().toByteArray();
2142     return ProtobufUtil.prependPBMagic(result);
2143   }
2144 
2145   /**
2146    * @param bytes Content of serialized data of RegionStoreSequenceIds
2147    * @return a RegionStoreSequenceIds object
2148    * @throws DeserializationException
2149    */
2150   public static RegionStoreSequenceIds parseRegionStoreSequenceIds(final byte[] bytes)
2151       throws DeserializationException {
2152     if (bytes == null || !ProtobufUtil.isPBMagicPrefix(bytes)) {
2153       throw new DeserializationException("Unable to parse RegionStoreSequenceIds.");
2154     }
2155     RegionStoreSequenceIds.Builder regionSequenceIdsBuilder =
2156         ClusterStatusProtos.RegionStoreSequenceIds.newBuilder();
2157     int pblen = ProtobufUtil.lengthOfPBMagic();
2158     RegionStoreSequenceIds storeIds = null;
2159     try {
2160       ProtobufUtil.mergeFrom(regionSequenceIdsBuilder, bytes, pblen, bytes.length - pblen);
2161       storeIds = regionSequenceIdsBuilder.build();
2162     } catch (IOException e) {
2163       throw new DeserializationException(e);
2164     }
2165     return storeIds;
2166   }
2167 }