View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.zookeeper;
20  
21  import java.io.BufferedReader;
22  import java.io.IOException;
23  import java.io.InputStreamReader;
24  import java.io.PrintWriter;
25  import java.net.InetSocketAddress;
26  import java.net.Socket;
27  import java.util.ArrayList;
28  import java.util.Arrays;
29  import java.util.Deque;
30  import java.util.HashMap;
31  import java.util.LinkedList;
32  import java.util.List;
33  import java.util.Map;
34  
35  import javax.security.auth.login.AppConfigurationEntry;
36  import javax.security.auth.login.AppConfigurationEntry.LoginModuleControlFlag;
37  
38  import org.apache.commons.lang.StringUtils;
39  import org.apache.commons.logging.Log;
40  import org.apache.commons.logging.LogFactory;
41  import org.apache.hadoop.conf.Configuration;
42  import org.apache.hadoop.hbase.AuthUtil;
43  import org.apache.hadoop.hbase.HBaseConfiguration;
44  import org.apache.hadoop.hbase.HConstants;
45  import org.apache.hadoop.hbase.ServerName;
46  import org.apache.hadoop.hbase.classification.InterfaceAudience;
47  import org.apache.hadoop.hbase.exceptions.DeserializationException;
48  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
49  import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos;
50  import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionStoreSequenceIds;
51  import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
52  import org.apache.hadoop.hbase.replication.ReplicationStateZKBase;
53  import org.apache.hadoop.hbase.security.Superusers;
54  import org.apache.hadoop.hbase.util.ByteStringer;
55  import org.apache.hadoop.hbase.util.Bytes;
56  import org.apache.hadoop.hbase.util.Threads;
57  import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp.CreateAndFailSilent;
58  import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp.DeleteNodeFailSilent;
59  import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp.SetData;
60  import org.apache.hadoop.security.SecurityUtil;
61  import org.apache.hadoop.security.authentication.util.KerberosUtil;
62  import org.apache.zookeeper.AsyncCallback;
63  import org.apache.zookeeper.CreateMode;
64  import org.apache.zookeeper.KeeperException;
65  import org.apache.zookeeper.KeeperException.NoNodeException;
66  import org.apache.zookeeper.Op;
67  import org.apache.zookeeper.Watcher;
68  import org.apache.zookeeper.ZooDefs.Ids;
69  import org.apache.zookeeper.ZooDefs.Perms;
70  import org.apache.zookeeper.ZooKeeper;
71  import org.apache.zookeeper.client.ZooKeeperSaslClient;
72  import org.apache.zookeeper.data.ACL;
73  import org.apache.zookeeper.data.Id;
74  import org.apache.zookeeper.data.Stat;
75  import org.apache.zookeeper.proto.CreateRequest;
76  import org.apache.zookeeper.proto.DeleteRequest;
77  import org.apache.zookeeper.proto.SetDataRequest;
78  import org.apache.zookeeper.server.ZooKeeperSaslServer;
79  
80  import com.google.protobuf.InvalidProtocolBufferException;
81  
82  /**
83   * Internal HBase utility class for ZooKeeper.
84   *
85   * <p>Contains only static methods and constants.
86   *
87   * <p>Methods all throw {@link KeeperException} if there is an unexpected
88   * zookeeper exception, so callers of these methods must handle appropriately.
89   * If ZK is required for the operation, the server will need to be aborted.
90   */
91  @InterfaceAudience.Private
92  public class ZKUtil {
93    private static final Log LOG = LogFactory.getLog(ZKUtil.class);
94  
95    // TODO: Replace this with ZooKeeper constant when ZOOKEEPER-277 is resolved.
96    public static final char ZNODE_PATH_SEPARATOR = '/';
97    private static int zkDumpConnectionTimeOut;
98  
99    /**
100    * Creates a new connection to ZooKeeper, pulling settings and ensemble config
101    * from the specified configuration object using methods from {@link ZKConfig}.
102    *
103    * Sets the connection status monitoring watcher to the specified watcher.
104    *
105    * @param conf configuration to pull ensemble and other settings from
106    * @param watcher watcher to monitor connection changes
107    * @return connection to zookeeper
108    * @throws IOException if unable to connect to zk or config problem
109    */
110   public static RecoverableZooKeeper connect(Configuration conf, Watcher watcher)
111   throws IOException {
112     String ensemble = ZKConfig.getZKQuorumServersString(conf);
113     return connect(conf, ensemble, watcher);
114   }
115 
116   public static RecoverableZooKeeper connect(Configuration conf, String ensemble,
117       Watcher watcher)
118   throws IOException {
119     return connect(conf, ensemble, watcher, null);
120   }
121 
122   public static RecoverableZooKeeper connect(Configuration conf, String ensemble,
123       Watcher watcher, final String identifier)
124   throws IOException {
125     if(ensemble == null) {
126       throw new IOException("Unable to determine ZooKeeper ensemble");
127     }
128     int timeout = conf.getInt(HConstants.ZK_SESSION_TIMEOUT,
129         HConstants.DEFAULT_ZK_SESSION_TIMEOUT);
130     if (LOG.isTraceEnabled()) {
131       LOG.trace(identifier + " opening connection to ZooKeeper ensemble=" + ensemble);
132     }
133     int retry = conf.getInt("zookeeper.recovery.retry", 3);
134     int retryIntervalMillis =
135       conf.getInt("zookeeper.recovery.retry.intervalmill", 1000);
136     zkDumpConnectionTimeOut = conf.getInt("zookeeper.dump.connection.timeout",
137         1000);
138     return new RecoverableZooKeeper(ensemble, timeout, watcher,
139         retry, retryIntervalMillis, identifier);
140   }
141 
142   /**
143    * Log in the current zookeeper server process using the given configuration
144    * keys for the credential file and login principal.
145    *
146    * <p><strong>This is only applicable when running on secure hbase</strong>
147    * On regular HBase (without security features), this will safely be ignored.
148    * </p>
149    *
150    * @param conf The configuration data to use
151    * @param keytabFileKey Property key used to configure the path to the credential file
152    * @param userNameKey Property key used to configure the login principal
153    * @param hostname Current hostname to use in any credentials
154    * @throws IOException underlying exception from SecurityUtil.login() call
155    */
156   public static void loginServer(Configuration conf, String keytabFileKey,
157       String userNameKey, String hostname) throws IOException {
158     login(conf, keytabFileKey, userNameKey, hostname,
159           ZooKeeperSaslServer.LOGIN_CONTEXT_NAME_KEY,
160           JaasConfiguration.SERVER_KEYTAB_KERBEROS_CONFIG_NAME);
161   }
162 
163   /**
164    * Log in the current zookeeper client using the given configuration
165    * keys for the credential file and login principal.
166    *
167    * <p><strong>This is only applicable when running on secure hbase</strong>
168    * On regular HBase (without security features), this will safely be ignored.
169    * </p>
170    *
171    * @param conf The configuration data to use
172    * @param keytabFileKey Property key used to configure the path to the credential file
173    * @param userNameKey Property key used to configure the login principal
174    * @param hostname Current hostname to use in any credentials
175    * @throws IOException underlying exception from SecurityUtil.login() call
176    */
177   public static void loginClient(Configuration conf, String keytabFileKey,
178       String userNameKey, String hostname) throws IOException {
179     login(conf, keytabFileKey, userNameKey, hostname,
180           ZooKeeperSaslClient.LOGIN_CONTEXT_NAME_KEY,
181           JaasConfiguration.CLIENT_KEYTAB_KERBEROS_CONFIG_NAME);
182   }
183 
184   /**
185    * Log in the current process using the given configuration keys for the
186    * credential file and login principal.
187    *
188    * <p><strong>This is only applicable when running on secure hbase</strong>
189    * On regular HBase (without security features), this will safely be ignored.
190    * </p>
191    *
192    * @param conf The configuration data to use
193    * @param keytabFileKey Property key used to configure the path to the credential file
194    * @param userNameKey Property key used to configure the login principal
195    * @param hostname Current hostname to use in any credentials
196    * @param loginContextProperty property name to expose the entry name
197    * @param loginContextName jaas entry name
198    * @throws IOException underlying exception from SecurityUtil.login() call
199    */
200   private static void login(Configuration conf, String keytabFileKey,
201       String userNameKey, String hostname,
202       String loginContextProperty, String loginContextName)
203       throws IOException {
204     if (!isSecureZooKeeper(conf))
205       return;
206 
207     // User has specified a jaas.conf, keep this one as the good one.
208     // HBASE_OPTS="-Djava.security.auth.login.config=jaas.conf"
209     if (System.getProperty("java.security.auth.login.config") != null)
210       return;
211 
212     // No keytab specified, no auth
213     String keytabFilename = conf.get(keytabFileKey);
214     if (keytabFilename == null) {
215       LOG.warn("no keytab specified for: " + keytabFileKey);
216       return;
217     }
218 
219     String principalConfig = conf.get(userNameKey, System.getProperty("user.name"));
220     String principalName = SecurityUtil.getServerPrincipal(principalConfig, hostname);
221 
222     // Initialize the "jaas.conf" for keyTab/principal,
223     // If keyTab is not specified use the Ticket Cache.
224     // and set the zookeeper login context name.
225     JaasConfiguration jaasConf = new JaasConfiguration(loginContextName,
226         principalName, keytabFilename);
227     javax.security.auth.login.Configuration.setConfiguration(jaasConf);
228     System.setProperty(loginContextProperty, loginContextName);
229   }
230 
231   /**
232    * A JAAS configuration that defines the login modules that we want to use for login.
233    */
234   private static class JaasConfiguration extends javax.security.auth.login.Configuration {
235     private static final String SERVER_KEYTAB_KERBEROS_CONFIG_NAME =
236       "zookeeper-server-keytab-kerberos";
237     private static final String CLIENT_KEYTAB_KERBEROS_CONFIG_NAME =
238       "zookeeper-client-keytab-kerberos";
239 
240     private static final Map<String, String> BASIC_JAAS_OPTIONS =
241       new HashMap<String,String>();
242     static {
243       String jaasEnvVar = System.getenv("HBASE_JAAS_DEBUG");
244       if (jaasEnvVar != null && "true".equalsIgnoreCase(jaasEnvVar)) {
245         BASIC_JAAS_OPTIONS.put("debug", "true");
246       }
247     }
248 
249     private static final Map<String,String> KEYTAB_KERBEROS_OPTIONS =
250       new HashMap<String,String>();
251     static {
252       KEYTAB_KERBEROS_OPTIONS.put("doNotPrompt", "true");
253       KEYTAB_KERBEROS_OPTIONS.put("storeKey", "true");
254       KEYTAB_KERBEROS_OPTIONS.put("refreshKrb5Config", "true");
255       KEYTAB_KERBEROS_OPTIONS.putAll(BASIC_JAAS_OPTIONS);
256     }
257 
258     private static final AppConfigurationEntry KEYTAB_KERBEROS_LOGIN =
259       new AppConfigurationEntry(KerberosUtil.getKrb5LoginModuleName(),
260                                 LoginModuleControlFlag.REQUIRED,
261                                 KEYTAB_KERBEROS_OPTIONS);
262 
263     private static final AppConfigurationEntry[] KEYTAB_KERBEROS_CONF =
264       new AppConfigurationEntry[]{KEYTAB_KERBEROS_LOGIN};
265 
266     private javax.security.auth.login.Configuration baseConfig;
267     private final String loginContextName;
268     private final boolean useTicketCache;
269     private final String keytabFile;
270     private final String principal;
271 
272     public JaasConfiguration(String loginContextName, String principal, String keytabFile) {
273       this(loginContextName, principal, keytabFile, keytabFile == null || keytabFile.length() == 0);
274     }
275 
276     private JaasConfiguration(String loginContextName, String principal,
277                              String keytabFile, boolean useTicketCache) {
278       try {
279         this.baseConfig = javax.security.auth.login.Configuration.getConfiguration();
280       } catch (SecurityException e) {
281         this.baseConfig = null;
282       }
283       this.loginContextName = loginContextName;
284       this.useTicketCache = useTicketCache;
285       this.keytabFile = keytabFile;
286       this.principal = principal;
287       LOG.info("JaasConfiguration loginContextName=" + loginContextName +
288                " principal=" + principal + " useTicketCache=" + useTicketCache +
289                " keytabFile=" + keytabFile);
290     }
291 
292     @Override
293     public AppConfigurationEntry[] getAppConfigurationEntry(String appName) {
294       if (loginContextName.equals(appName)) {
295         if (!useTicketCache) {
296           KEYTAB_KERBEROS_OPTIONS.put("keyTab", keytabFile);
297           KEYTAB_KERBEROS_OPTIONS.put("useKeyTab", "true");
298         }
299         KEYTAB_KERBEROS_OPTIONS.put("principal", principal);
300         KEYTAB_KERBEROS_OPTIONS.put("useTicketCache", useTicketCache ? "true" : "false");
301         return KEYTAB_KERBEROS_CONF;
302       }
303       if (baseConfig != null) return baseConfig.getAppConfigurationEntry(appName);
304       return(null);
305     }
306   }
307 
308   //
309   // Helper methods
310   //
311 
312   /**
313    * Join the prefix znode name with the suffix znode name to generate a proper
314    * full znode name.
315    *
316    * Assumes prefix does not end with slash and suffix does not begin with it.
317    *
318    * @param prefix beginning of znode name
319    * @param suffix ending of znode name
320    * @return result of properly joining prefix with suffix
321    */
322   public static String joinZNode(String prefix, String suffix) {
323     return prefix + ZNODE_PATH_SEPARATOR + suffix;
324   }
325 
326   /**
327    * Returns the full path of the immediate parent of the specified node.
328    * @param node path to get parent of
329    * @return parent of path, null if passed the root node or an invalid node
330    */
331   public static String getParent(String node) {
332     int idx = node.lastIndexOf(ZNODE_PATH_SEPARATOR);
333     return idx <= 0 ? null : node.substring(0, idx);
334   }
335 
336   /**
337    * Get the name of the current node from the specified fully-qualified path.
338    * @param path fully-qualified path
339    * @return name of the current node
340    */
341   public static String getNodeName(String path) {
342     return path.substring(path.lastIndexOf("/")+1);
343   }
344 
345   //
346   // Existence checks and watches
347   //
348 
349   /**
350    * Watch the specified znode for delete/create/change events.  The watcher is
351    * set whether or not the node exists.  If the node already exists, the method
352    * returns true.  If the node does not exist, the method returns false.
353    *
354    * @param zkw zk reference
355    * @param znode path of node to watch
356    * @return true if znode exists, false if does not exist or error
357    * @throws KeeperException if unexpected zookeeper exception
358    */
359   public static boolean watchAndCheckExists(ZooKeeperWatcher zkw, String znode)
360   throws KeeperException {
361     try {
362       Stat s = zkw.getRecoverableZooKeeper().exists(znode, zkw);
363       boolean exists = s != null ? true : false;
364       if (exists) {
365         LOG.debug(zkw.prefix("Set watcher on existing znode=" + znode));
366       } else {
367         LOG.debug(zkw.prefix("Set watcher on znode that does not yet exist, " + znode));
368       }
369       return exists;
370     } catch (KeeperException e) {
371       LOG.warn(zkw.prefix("Unable to set watcher on znode " + znode), e);
372       zkw.keeperException(e);
373       return false;
374     } catch (InterruptedException e) {
375       LOG.warn(zkw.prefix("Unable to set watcher on znode " + znode), e);
376       zkw.interruptedException(e);
377       return false;
378     }
379   }
380 
381   /**
382    * Watch the specified znode, but only if exists. Useful when watching
383    * for deletions. Uses .getData() (and handles NoNodeException) instead
384    * of .exists() to accomplish this, as .getData() will only set a watch if
385    * the znode exists.
386    * @param zkw zk reference
387    * @param znode path of node to watch
388    * @return true if the watch is set, false if node does not exists
389    * @throws KeeperException if unexpected zookeeper exception
390    */
391   public static boolean setWatchIfNodeExists(ZooKeeperWatcher zkw, String znode)
392       throws KeeperException {
393     try {
394       zkw.getRecoverableZooKeeper().getData(znode, true, null);
395       return true;
396     } catch (NoNodeException e) {
397       return false;
398     } catch (InterruptedException e) {
399       LOG.warn(zkw.prefix("Unable to set watcher on znode " + znode), e);
400       zkw.interruptedException(e);
401       return false;
402     }
403   }
404 
405   /**
406    * Check if the specified node exists.  Sets no watches.
407    *
408    * @param zkw zk reference
409    * @param znode path of node to watch
410    * @return version of the node if it exists, -1 if does not exist
411    * @throws KeeperException if unexpected zookeeper exception
412    */
413   public static int checkExists(ZooKeeperWatcher zkw, String znode)
414   throws KeeperException {
415     try {
416       Stat s = zkw.getRecoverableZooKeeper().exists(znode, null);
417       return s != null ? s.getVersion() : -1;
418     } catch (KeeperException e) {
419       LOG.warn(zkw.prefix("Unable to set watcher on znode (" + znode + ")"), e);
420       zkw.keeperException(e);
421       return -1;
422     } catch (InterruptedException e) {
423       LOG.warn(zkw.prefix("Unable to set watcher on znode (" + znode + ")"), e);
424       zkw.interruptedException(e);
425       return -1;
426     }
427   }
428 
429   //
430   // Znode listings
431   //
432 
433   /**
434    * Lists the children znodes of the specified znode.  Also sets a watch on
435    * the specified znode which will capture a NodeDeleted event on the specified
436    * znode as well as NodeChildrenChanged if any children of the specified znode
437    * are created or deleted.
438    *
439    * Returns null if the specified node does not exist.  Otherwise returns a
440    * list of children of the specified node.  If the node exists but it has no
441    * children, an empty list will be returned.
442    *
443    * @param zkw zk reference
444    * @param znode path of node to list and watch children of
445    * @return list of children of the specified node, an empty list if the node
446    *          exists but has no children, and null if the node does not exist
447    * @throws KeeperException if unexpected zookeeper exception
448    */
449   public static List<String> listChildrenAndWatchForNewChildren(
450       ZooKeeperWatcher zkw, String znode)
451   throws KeeperException {
452     try {
453       List<String> children = zkw.getRecoverableZooKeeper().getChildren(znode, zkw);
454       return children;
455     } catch(KeeperException.NoNodeException ke) {
456       LOG.debug(zkw.prefix("Unable to list children of znode " + znode + " " +
457           "because node does not exist (not an error)"));
458       return null;
459     } catch (KeeperException e) {
460       LOG.warn(zkw.prefix("Unable to list children of znode " + znode + " "), e);
461       zkw.keeperException(e);
462       return null;
463     } catch (InterruptedException e) {
464       LOG.warn(zkw.prefix("Unable to list children of znode " + znode + " "), e);
465       zkw.interruptedException(e);
466       return null;
467     }
468   }
469 
470   /**
471    * List all the children of the specified znode, setting a watch for children
472    * changes and also setting a watch on every individual child in order to get
473    * the NodeCreated and NodeDeleted events.
474    * @param zkw zookeeper reference
475    * @param znode node to get children of and watch
476    * @return list of znode names, null if the node doesn't exist
477    * @throws KeeperException
478    */
479   public static List<String> listChildrenAndWatchThem(ZooKeeperWatcher zkw,
480       String znode) throws KeeperException {
481     List<String> children = listChildrenAndWatchForNewChildren(zkw, znode);
482     if (children == null) {
483       return null;
484     }
485     for (String child : children) {
486       watchAndCheckExists(zkw, joinZNode(znode, child));
487     }
488     return children;
489   }
490 
491   /**
492    * Lists the children of the specified znode without setting any watches.
493    *
494    * Sets no watches at all, this method is best effort.
495    *
496    * Returns an empty list if the node has no children.  Returns null if the
497    * parent node itself does not exist.
498    *
499    * @param zkw zookeeper reference
500    * @param znode node to get children
501    * @return list of data of children of specified znode, empty if no children,
502    *         null if parent does not exist
503    * @throws KeeperException if unexpected zookeeper exception
504    */
505   public static List<String> listChildrenNoWatch(ZooKeeperWatcher zkw, String znode)
506   throws KeeperException {
507     List<String> children = null;
508     try {
509       // List the children without watching
510       children = zkw.getRecoverableZooKeeper().getChildren(znode, null);
511     } catch(KeeperException.NoNodeException nne) {
512       return null;
513     } catch(InterruptedException ie) {
514       zkw.interruptedException(ie);
515     }
516     return children;
517   }
518 
519   /**
520    * Simple class to hold a node path and node data.
521    * @deprecated Unused
522    */
523   @Deprecated
524   public static class NodeAndData {
525     private String node;
526     private byte [] data;
527     public NodeAndData(String node, byte [] data) {
528       this.node = node;
529       this.data = data;
530     }
531     public String getNode() {
532       return node;
533     }
534     public byte [] getData() {
535       return data;
536     }
537     @Override
538     public String toString() {
539       return node;
540     }
541     public boolean isEmpty() {
542       return (data == null || data.length == 0);
543     }
544   }
545 
546   /**
547    * Checks if the specified znode has any children.  Sets no watches.
548    *
549    * Returns true if the node exists and has children.  Returns false if the
550    * node does not exist or if the node does not have any children.
551    *
552    * Used during master initialization to determine if the master is a
553    * failed-over-to master or the first master during initial cluster startup.
554    * If the directory for regionserver ephemeral nodes is empty then this is
555    * a cluster startup, if not then it is not cluster startup.
556    *
557    * @param zkw zk reference
558    * @param znode path of node to check for children of
559    * @return true if node has children, false if not or node does not exist
560    * @throws KeeperException if unexpected zookeeper exception
561    */
562   public static boolean nodeHasChildren(ZooKeeperWatcher zkw, String znode)
563   throws KeeperException {
564     try {
565       return !zkw.getRecoverableZooKeeper().getChildren(znode, null).isEmpty();
566     } catch(KeeperException.NoNodeException ke) {
567       LOG.debug(zkw.prefix("Unable to list children of znode " + znode + " " +
568       "because node does not exist (not an error)"));
569       return false;
570     } catch (KeeperException e) {
571       LOG.warn(zkw.prefix("Unable to list children of znode " + znode), e);
572       zkw.keeperException(e);
573       return false;
574     } catch (InterruptedException e) {
575       LOG.warn(zkw.prefix("Unable to list children of znode " + znode), e);
576       zkw.interruptedException(e);
577       return false;
578     }
579   }
580 
581   /**
582    * Get the number of children of the specified node.
583    *
584    * If the node does not exist or has no children, returns 0.
585    *
586    * Sets no watches at all.
587    *
588    * @param zkw zk reference
589    * @param znode path of node to count children of
590    * @return number of children of specified node, 0 if none or parent does not
591    *         exist
592    * @throws KeeperException if unexpected zookeeper exception
593    */
594   public static int getNumberOfChildren(ZooKeeperWatcher zkw, String znode)
595   throws KeeperException {
596     try {
597       Stat stat = zkw.getRecoverableZooKeeper().exists(znode, null);
598       return stat == null ? 0 : stat.getNumChildren();
599     } catch(KeeperException e) {
600       LOG.warn(zkw.prefix("Unable to get children of node " + znode));
601       zkw.keeperException(e);
602     } catch(InterruptedException e) {
603       zkw.interruptedException(e);
604     }
605     return 0;
606   }
607 
608   //
609   // Data retrieval
610   //
611 
612   /**
613    * Get znode data. Does not set a watcher.
614    * @return ZNode data, null if the node does not exist or if there is an
615    *  error.
616    */
617   public static byte [] getData(ZooKeeperWatcher zkw, String znode)
618       throws KeeperException, InterruptedException {
619     try {
620       byte [] data = zkw.getRecoverableZooKeeper().getData(znode, null, null);
621       logRetrievedMsg(zkw, znode, data, false);
622       return data;
623     } catch (KeeperException.NoNodeException e) {
624       LOG.debug(zkw.prefix("Unable to get data of znode " + znode + " " +
625           "because node does not exist (not an error)"));
626       return null;
627     } catch (KeeperException e) {
628       LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
629       zkw.keeperException(e);
630       return null;
631     }
632   }
633 
634   /**
635    * Get the data at the specified znode and set a watch.
636    *
637    * Returns the data and sets a watch if the node exists.  Returns null and no
638    * watch is set if the node does not exist or there is an exception.
639    *
640    * @param zkw zk reference
641    * @param znode path of node
642    * @return data of the specified znode, or null
643    * @throws KeeperException if unexpected zookeeper exception
644    */
645   public static byte [] getDataAndWatch(ZooKeeperWatcher zkw, String znode)
646   throws KeeperException {
647     return getDataInternal(zkw, znode, null, true);
648   }
649 
650   /**
651    * Get the data at the specified znode and set a watch.
652    *
653    * Returns the data and sets a watch if the node exists.  Returns null and no
654    * watch is set if the node does not exist or there is an exception.
655    *
656    * @param zkw zk reference
657    * @param znode path of node
658    * @param stat object to populate the version of the znode
659    * @return data of the specified znode, or null
660    * @throws KeeperException if unexpected zookeeper exception
661    */
662   public static byte[] getDataAndWatch(ZooKeeperWatcher zkw, String znode,
663       Stat stat) throws KeeperException {
664     return getDataInternal(zkw, znode, stat, true);
665   }
666 
667   private static byte[] getDataInternal(ZooKeeperWatcher zkw, String znode, Stat stat,
668       boolean watcherSet)
669       throws KeeperException {
670     try {
671       byte [] data = zkw.getRecoverableZooKeeper().getData(znode, zkw, stat);
672       logRetrievedMsg(zkw, znode, data, watcherSet);
673       return data;
674     } catch (KeeperException.NoNodeException e) {
675       // This log can get pretty annoying when we cycle on 100ms waits.
676       // Enable trace if you really want to see it.
677       LOG.trace(zkw.prefix("Unable to get data of znode " + znode + " " +
678         "because node does not exist (not an error)"));
679       return null;
680     } catch (KeeperException e) {
681       LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
682       zkw.keeperException(e);
683       return null;
684     } catch (InterruptedException e) {
685       LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
686       zkw.interruptedException(e);
687       return null;
688     }
689   }
690 
691   /**
692    * Get the data at the specified znode without setting a watch.
693    *
694    * Returns the data if the node exists.  Returns null if the node does not
695    * exist.
696    *
697    * Sets the stats of the node in the passed Stat object.  Pass a null stat if
698    * not interested.
699    *
700    * @param zkw zk reference
701    * @param znode path of node
702    * @param stat node status to get if node exists
703    * @return data of the specified znode, or null if node does not exist
704    * @throws KeeperException if unexpected zookeeper exception
705    */
706   public static byte [] getDataNoWatch(ZooKeeperWatcher zkw, String znode,
707       Stat stat)
708   throws KeeperException {
709     try {
710       byte [] data = zkw.getRecoverableZooKeeper().getData(znode, null, stat);
711       logRetrievedMsg(zkw, znode, data, false);
712       return data;
713     } catch (KeeperException.NoNodeException e) {
714       LOG.debug(zkw.prefix("Unable to get data of znode " + znode + " " +
715           "because node does not exist (not necessarily an error)"));
716       return null;
717     } catch (KeeperException e) {
718       LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
719       zkw.keeperException(e);
720       return null;
721     } catch (InterruptedException e) {
722       LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
723       zkw.interruptedException(e);
724       return null;
725     }
726   }
727 
728   /**
729    * Returns the date of child znodes of the specified znode.  Also sets a watch on
730    * the specified znode which will capture a NodeDeleted event on the specified
731    * znode as well as NodeChildrenChanged if any children of the specified znode
732    * are created or deleted.
733    *
734    * Returns null if the specified node does not exist.  Otherwise returns a
735    * list of children of the specified node.  If the node exists but it has no
736    * children, an empty list will be returned.
737    *
738    * @param zkw zk reference
739    * @param baseNode path of node to list and watch children of
740    * @return list of data of children of the specified node, an empty list if the node
741    *          exists but has no children, and null if the node does not exist
742    * @throws KeeperException if unexpected zookeeper exception
743    * @deprecated Unused
744    */
745   @Deprecated
746   public static List<NodeAndData> getChildDataAndWatchForNewChildren(
747       ZooKeeperWatcher zkw, String baseNode) throws KeeperException {
748     List<String> nodes =
749       ZKUtil.listChildrenAndWatchForNewChildren(zkw, baseNode);
750     if (nodes != null) {
751       List<NodeAndData> newNodes = new ArrayList<NodeAndData>();
752       for (String node : nodes) {
753         String nodePath = ZKUtil.joinZNode(baseNode, node);
754         byte[] data = ZKUtil.getDataAndWatch(zkw, nodePath);
755         newNodes.add(new NodeAndData(nodePath, data));
756       }
757       return newNodes;
758     }
759     return null;
760   }
761 
762   /**
763    * Update the data of an existing node with the expected version to have the
764    * specified data.
765    *
766    * Throws an exception if there is a version mismatch or some other problem.
767    *
768    * Sets no watches under any conditions.
769    *
770    * @param zkw zk reference
771    * @param znode
772    * @param data
773    * @param expectedVersion
774    * @throws KeeperException if unexpected zookeeper exception
775    * @throws KeeperException.BadVersionException if version mismatch
776    * @deprecated Unused
777    */
778   @Deprecated
779   public static void updateExistingNodeData(ZooKeeperWatcher zkw, String znode,
780       byte [] data, int expectedVersion)
781   throws KeeperException {
782     try {
783       zkw.getRecoverableZooKeeper().setData(znode, data, expectedVersion);
784     } catch(InterruptedException ie) {
785       zkw.interruptedException(ie);
786     }
787   }
788 
789   //
790   // Data setting
791   //
792 
793   /**
794    * Sets the data of the existing znode to be the specified data.  Ensures that
795    * the current data has the specified expected version.
796    *
797    * <p>If the node does not exist, a {@link NoNodeException} will be thrown.
798    *
799    * <p>If their is a version mismatch, method returns null.
800    *
801    * <p>No watches are set but setting data will trigger other watchers of this
802    * node.
803    *
804    * <p>If there is another problem, a KeeperException will be thrown.
805    *
806    * @param zkw zk reference
807    * @param znode path of node
808    * @param data data to set for node
809    * @param expectedVersion version expected when setting data
810    * @return true if data set, false if version mismatch
811    * @throws KeeperException if unexpected zookeeper exception
812    */
813   public static boolean setData(ZooKeeperWatcher zkw, String znode,
814       byte [] data, int expectedVersion)
815   throws KeeperException, KeeperException.NoNodeException {
816     try {
817       return zkw.getRecoverableZooKeeper().setData(znode, data, expectedVersion) != null;
818     } catch (InterruptedException e) {
819       zkw.interruptedException(e);
820       return false;
821     }
822   }
823 
824   /**
825    * Set data into node creating node if it doesn't yet exist.
826    * Does not set watch.
827    *
828    * @param zkw zk reference
829    * @param znode path of node
830    * @param data data to set for node
831    * @throws KeeperException
832    */
833   public static void createSetData(final ZooKeeperWatcher zkw, final String znode,
834       final byte [] data)
835   throws KeeperException {
836     if (checkExists(zkw, znode) == -1) {
837       ZKUtil.createWithParents(zkw, znode, data);
838     } else {
839       ZKUtil.setData(zkw, znode, data);
840     }
841   }
842 
843   /**
844    * Sets the data of the existing znode to be the specified data.  The node
845    * must exist but no checks are done on the existing data or version.
846    *
847    * <p>If the node does not exist, a {@link NoNodeException} will be thrown.
848    *
849    * <p>No watches are set but setting data will trigger other watchers of this
850    * node.
851    *
852    * <p>If there is another problem, a KeeperException will be thrown.
853    *
854    * @param zkw zk reference
855    * @param znode path of node
856    * @param data data to set for node
857    * @throws KeeperException if unexpected zookeeper exception
858    */
859   public static void setData(ZooKeeperWatcher zkw, String znode, byte [] data)
860   throws KeeperException, KeeperException.NoNodeException {
861     setData(zkw, (SetData)ZKUtilOp.setData(znode, data));
862   }
863 
864   private static void setData(ZooKeeperWatcher zkw, SetData setData)
865   throws KeeperException, KeeperException.NoNodeException {
866     SetDataRequest sd = (SetDataRequest)toZooKeeperOp(zkw, setData).toRequestRecord();
867     setData(zkw, sd.getPath(), sd.getData(), sd.getVersion());
868   }
869 
870   /**
871    * Returns whether or not secure authentication is enabled
872    * (whether <code>hbase.security.authentication</code> is set to
873    * <code>kerberos</code>.
874    */
875   public static boolean isSecureZooKeeper(Configuration conf) {
876     // Detection for embedded HBase client with jaas configuration
877     // defined for third party programs.
878     try {
879       javax.security.auth.login.Configuration testConfig =
880           javax.security.auth.login.Configuration.getConfiguration();
881       if (testConfig.getAppConfigurationEntry("Client") == null
882           && testConfig.getAppConfigurationEntry(
883             JaasConfiguration.CLIENT_KEYTAB_KERBEROS_CONFIG_NAME) == null
884           && testConfig.getAppConfigurationEntry(
885               JaasConfiguration.SERVER_KEYTAB_KERBEROS_CONFIG_NAME) == null
886           && conf.get(HConstants.ZK_CLIENT_KERBEROS_PRINCIPAL) == null
887           && conf.get(HConstants.ZK_SERVER_KERBEROS_PRINCIPAL) == null) {
888 
889         return false;
890       }
891     } catch(Exception e) {
892       // No Jaas configuration defined.
893       return false;
894     }
895 
896     // Master & RSs uses hbase.zookeeper.client.*
897     return "kerberos".equalsIgnoreCase(conf.get("hbase.security.authentication"));
898   }
899 
900   private static ArrayList<ACL> createACL(ZooKeeperWatcher zkw, String node) {
901     return createACL(zkw, node, isSecureZooKeeper(zkw.getConfiguration()));
902   }
903 
904   public static ArrayList<ACL> createACL(ZooKeeperWatcher zkw, String node,
905     boolean isSecureZooKeeper) {
906     if (!node.startsWith(zkw.baseZNode)) {
907       return Ids.OPEN_ACL_UNSAFE;
908     }
909     if (isSecureZooKeeper) {
910       ArrayList<ACL> acls = new ArrayList<ACL>();
911       // add permission to hbase supper user
912       String[] superUsers = zkw.getConfiguration().getStrings(Superusers.SUPERUSER_CONF_KEY);
913       if (superUsers != null) {
914         List<String> groups = new ArrayList<String>();
915         for (String user : superUsers) {
916           if (user.startsWith(AuthUtil.GROUP_PREFIX)) {
917             // TODO: Set node ACL for groups when ZK supports this feature
918             groups.add(user);
919           } else {
920             acls.add(new ACL(Perms.ALL, new Id("auth", user)));
921           }
922         }
923         if (!groups.isEmpty()) {
924           LOG.warn("Znode ACL setting for group " + groups
925               + " is skipped, Zookeeper doesn't support this feature presently.");
926         }
927       }
928       // Certain znodes are accessed directly by the client,
929       // so they must be readable by non-authenticated clients
930       if (zkw.isClientReadable(node)) {
931         acls.addAll(Ids.CREATOR_ALL_ACL);
932         acls.addAll(Ids.READ_ACL_UNSAFE);
933       } else {
934         acls.addAll(Ids.CREATOR_ALL_ACL);
935       }
936       return acls;
937     } else {
938       return Ids.OPEN_ACL_UNSAFE;
939     }
940   }
941 
942   //
943   // Node creation
944   //
945 
946   /**
947    *
948    * Set the specified znode to be an ephemeral node carrying the specified
949    * data.
950    *
951    * If the node is created successfully, a watcher is also set on the node.
952    *
953    * If the node is not created successfully because it already exists, this
954    * method will also set a watcher on the node.
955    *
956    * If there is another problem, a KeeperException will be thrown.
957    *
958    * @param zkw zk reference
959    * @param znode path of node
960    * @param data data of node
961    * @return true if node created, false if not, watch set in both cases
962    * @throws KeeperException if unexpected zookeeper exception
963    */
964   public static boolean createEphemeralNodeAndWatch(ZooKeeperWatcher zkw,
965       String znode, byte [] data)
966   throws KeeperException {
967     boolean ret = true;
968     try {
969       zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode),
970           CreateMode.EPHEMERAL);
971     } catch (KeeperException.NodeExistsException nee) {
972       ret = false;
973     } catch (InterruptedException e) {
974       LOG.info("Interrupted", e);
975       Thread.currentThread().interrupt();
976     }
977     if(!watchAndCheckExists(zkw, znode)) {
978       // It did exist but now it doesn't, try again
979       return createEphemeralNodeAndWatch(zkw, znode, data);
980     }
981     return ret;
982   }
983 
984   /**
985    * Creates the specified znode to be a persistent node carrying the specified
986    * data.
987    *
988    * Returns true if the node was successfully created, false if the node
989    * already existed.
990    *
991    * If the node is created successfully, a watcher is also set on the node.
992    *
993    * If the node is not created successfully because it already exists, this
994    * method will also set a watcher on the node but return false.
995    *
996    * If there is another problem, a KeeperException will be thrown.
997    *
998    * @param zkw zk reference
999    * @param znode path of node
1000    * @param data data of node
1001    * @return true if node created, false if not, watch set in both cases
1002    * @throws KeeperException if unexpected zookeeper exception
1003    */
1004   public static boolean createNodeIfNotExistsAndWatch(
1005       ZooKeeperWatcher zkw, String znode, byte [] data)
1006   throws KeeperException {
1007     boolean ret = true;
1008     try {
1009       zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode),
1010           CreateMode.PERSISTENT);
1011     } catch (KeeperException.NodeExistsException nee) {
1012       ret = false;
1013     } catch (InterruptedException e) {
1014       zkw.interruptedException(e);
1015       return false;
1016     }
1017     try {
1018       zkw.getRecoverableZooKeeper().exists(znode, zkw);
1019     } catch (InterruptedException e) {
1020       zkw.interruptedException(e);
1021       return false;
1022     }
1023     return ret;
1024   }
1025 
1026   /**
1027    * Creates the specified znode with the specified data but does not watch it.
1028    *
1029    * Returns the znode of the newly created node
1030    *
1031    * If there is another problem, a KeeperException will be thrown.
1032    *
1033    * @param zkw zk reference
1034    * @param znode path of node
1035    * @param data data of node
1036    * @param createMode specifying whether the node to be created is ephemeral and/or sequential
1037    * @return true name of the newly created znode or null
1038    * @throws KeeperException if unexpected zookeeper exception
1039    */
1040   public static String createNodeIfNotExistsNoWatch(ZooKeeperWatcher zkw, String znode,
1041       byte[] data, CreateMode createMode) throws KeeperException {
1042 
1043     String createdZNode = null;
1044     try {
1045       createdZNode = zkw.getRecoverableZooKeeper().create(znode, data,
1046           createACL(zkw, znode), createMode);
1047     } catch (KeeperException.NodeExistsException nee) {
1048       return znode;
1049     } catch (InterruptedException e) {
1050       zkw.interruptedException(e);
1051       return null;
1052     }
1053     return createdZNode;
1054   }
1055 
1056   /**
1057    * Creates the specified node with the specified data and watches it.
1058    *
1059    * <p>Throws an exception if the node already exists.
1060    *
1061    * <p>The node created is persistent and open access.
1062    *
1063    * <p>Returns the version number of the created node if successful.
1064    *
1065    * @param zkw zk reference
1066    * @param znode path of node to create
1067    * @param data data of node to create
1068    * @return version of node created
1069    * @throws KeeperException if unexpected zookeeper exception
1070    * @throws KeeperException.NodeExistsException if node already exists
1071    */
1072   public static int createAndWatch(ZooKeeperWatcher zkw,
1073       String znode, byte [] data)
1074   throws KeeperException, KeeperException.NodeExistsException {
1075     try {
1076       zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode),
1077           CreateMode.PERSISTENT);
1078       Stat stat = zkw.getRecoverableZooKeeper().exists(znode, zkw);
1079       if (stat == null){
1080         // Likely a race condition. Someone deleted the znode.
1081         throw KeeperException.create(KeeperException.Code.SYSTEMERROR,
1082             "ZK.exists returned null (i.e.: znode does not exist) for znode=" + znode);
1083       }
1084      return stat.getVersion();
1085     } catch (InterruptedException e) {
1086       zkw.interruptedException(e);
1087       return -1;
1088     }
1089   }
1090 
1091   /**
1092    * Async creates the specified node with the specified data.
1093    *
1094    * <p>Throws an exception if the node already exists.
1095    *
1096    * <p>The node created is persistent and open access.
1097    *
1098    * @param zkw zk reference
1099    * @param znode path of node to create
1100    * @param data data of node to create
1101    * @param cb
1102    * @param ctx
1103    */
1104   public static void asyncCreate(ZooKeeperWatcher zkw,
1105       String znode, byte [] data, final AsyncCallback.StringCallback cb,
1106       final Object ctx) {
1107     zkw.getRecoverableZooKeeper().getZooKeeper().create(znode, data,
1108         createACL(zkw, znode), CreateMode.PERSISTENT, cb, ctx);
1109   }
1110 
1111   /**
1112    * Creates the specified node, iff the node does not exist.  Does not set a
1113    * watch and fails silently if the node already exists.
1114    *
1115    * The node created is persistent and open access.
1116    *
1117    * @param zkw zk reference
1118    * @param znode path of node
1119    * @throws KeeperException if unexpected zookeeper exception
1120    */
1121   public static void createAndFailSilent(ZooKeeperWatcher zkw,
1122       String znode) throws KeeperException {
1123     createAndFailSilent(zkw, znode, new byte[0]);
1124   }
1125 
1126   /**
1127    * Creates the specified node containing specified data, iff the node does not exist.  Does
1128    * not set a watch and fails silently if the node already exists.
1129    *
1130    * The node created is persistent and open access.
1131    *
1132    * @param zkw zk reference
1133    * @param znode path of node
1134    * @param data a byte array data to store in the znode
1135    * @throws KeeperException if unexpected zookeeper exception
1136    */
1137   public static void createAndFailSilent(ZooKeeperWatcher zkw,
1138       String znode, byte[] data)
1139   throws KeeperException {
1140     createAndFailSilent(zkw,
1141         (CreateAndFailSilent)ZKUtilOp.createAndFailSilent(znode, data));
1142   }
1143 
1144   private static void createAndFailSilent(ZooKeeperWatcher zkw, CreateAndFailSilent cafs)
1145   throws KeeperException {
1146     CreateRequest create = (CreateRequest)toZooKeeperOp(zkw, cafs).toRequestRecord();
1147     String znode = create.getPath();
1148     try {
1149       RecoverableZooKeeper zk = zkw.getRecoverableZooKeeper();
1150       if (zk.exists(znode, false) == null) {
1151         zk.create(znode, create.getData(), create.getAcl(), CreateMode.fromFlag(create.getFlags()));
1152       }
1153     } catch(KeeperException.NodeExistsException nee) {
1154     } catch(KeeperException.NoAuthException nee){
1155       try {
1156         if (null == zkw.getRecoverableZooKeeper().exists(znode, false)) {
1157           // If we failed to create the file and it does not already exist.
1158           throw(nee);
1159         }
1160       } catch (InterruptedException ie) {
1161         zkw.interruptedException(ie);
1162       }
1163     } catch(InterruptedException ie) {
1164       zkw.interruptedException(ie);
1165     }
1166   }
1167 
1168   /**
1169    * Creates the specified node and all parent nodes required for it to exist.
1170    *
1171    * No watches are set and no errors are thrown if the node already exists.
1172    *
1173    * The nodes created are persistent and open access.
1174    *
1175    * @param zkw zk reference
1176    * @param znode path of node
1177    * @throws KeeperException if unexpected zookeeper exception
1178    */
1179   public static void createWithParents(ZooKeeperWatcher zkw, String znode)
1180   throws KeeperException {
1181     createWithParents(zkw, znode, new byte[0]);
1182   }
1183 
1184   /**
1185    * Creates the specified node and all parent nodes required for it to exist.  The creation of
1186    * parent znodes is not atomic with the leafe znode creation but the data is written atomically
1187    * when the leaf node is created.
1188    *
1189    * No watches are set and no errors are thrown if the node already exists.
1190    *
1191    * The nodes created are persistent and open access.
1192    *
1193    * @param zkw zk reference
1194    * @param znode path of node
1195    * @throws KeeperException if unexpected zookeeper exception
1196    */
1197   public static void createWithParents(ZooKeeperWatcher zkw, String znode, byte[] data)
1198   throws KeeperException {
1199     try {
1200       if(znode == null) {
1201         return;
1202       }
1203       zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode),
1204           CreateMode.PERSISTENT);
1205     } catch(KeeperException.NodeExistsException nee) {
1206       return;
1207     } catch(KeeperException.NoNodeException nne) {
1208       createWithParents(zkw, getParent(znode));
1209       createWithParents(zkw, znode, data);
1210     } catch(InterruptedException ie) {
1211       zkw.interruptedException(ie);
1212     }
1213   }
1214 
1215   //
1216   // Deletes
1217   //
1218 
1219   /**
1220    * Delete the specified node.  Sets no watches.  Throws all exceptions.
1221    */
1222   public static void deleteNode(ZooKeeperWatcher zkw, String node)
1223   throws KeeperException {
1224     deleteNode(zkw, node, -1);
1225   }
1226 
1227   /**
1228    * Delete the specified node with the specified version.  Sets no watches.
1229    * Throws all exceptions.
1230    */
1231   public static boolean deleteNode(ZooKeeperWatcher zkw, String node,
1232       int version)
1233   throws KeeperException {
1234     try {
1235       zkw.getRecoverableZooKeeper().delete(node, version);
1236       return true;
1237     } catch(KeeperException.BadVersionException bve) {
1238       return false;
1239     } catch(InterruptedException ie) {
1240       zkw.interruptedException(ie);
1241       return false;
1242     }
1243   }
1244 
1245   /**
1246    * Deletes the specified node.  Fails silent if the node does not exist.
1247    * @param zkw
1248    * @param node
1249    * @throws KeeperException
1250    */
1251   public static void deleteNodeFailSilent(ZooKeeperWatcher zkw, String node)
1252   throws KeeperException {
1253     deleteNodeFailSilent(zkw,
1254       (DeleteNodeFailSilent)ZKUtilOp.deleteNodeFailSilent(node));
1255   }
1256 
1257   private static void deleteNodeFailSilent(ZooKeeperWatcher zkw,
1258       DeleteNodeFailSilent dnfs) throws KeeperException {
1259     DeleteRequest delete = (DeleteRequest)toZooKeeperOp(zkw, dnfs).toRequestRecord();
1260     try {
1261       zkw.getRecoverableZooKeeper().delete(delete.getPath(), delete.getVersion());
1262     } catch(KeeperException.NoNodeException nne) {
1263     } catch(InterruptedException ie) {
1264       zkw.interruptedException(ie);
1265     }
1266   }
1267 
1268 
1269   /**
1270    * Delete the specified node and all of it's children.
1271    * <p>
1272    * If the node does not exist, just returns.
1273    * <p>
1274    * Sets no watches. Throws all exceptions besides dealing with deletion of
1275    * children.
1276    */
1277   public static void deleteNodeRecursively(ZooKeeperWatcher zkw, String node)
1278   throws KeeperException {
1279     deleteNodeRecursivelyMultiOrSequential(zkw, true, node);
1280   }
1281 
1282   /**
1283    * Delete all the children of the specified node but not the node itself.
1284    *
1285    * Sets no watches.  Throws all exceptions besides dealing with deletion of
1286    * children.
1287    *
1288    * If hbase.zookeeper.useMulti is true, use ZooKeeper's multi-update functionality.
1289    * Otherwise, run the list of operations sequentially.
1290    *
1291    * @throws KeeperException
1292    */
1293   public static void deleteChildrenRecursively(ZooKeeperWatcher zkw, String node)
1294       throws KeeperException {
1295     deleteChildrenRecursivelyMultiOrSequential(zkw, true, node);
1296   }
1297 
1298   /**
1299    * Delete all the children of the specified node but not the node itself. This
1300    * will first traverse the znode tree for listing the children and then delete
1301    * these znodes using multi-update api or sequential based on the specified
1302    * configurations.
1303    * <p>
1304    * Sets no watches. Throws all exceptions besides dealing with deletion of
1305    * children.
1306    * <p>
1307    * If hbase.zookeeper.useMulti is true, use ZooKeeper's multi-update
1308    * functionality. Otherwise, run the list of operations sequentially.
1309    * <p>
1310    * If all of the following are true:
1311    * <ul>
1312    * <li>runSequentialOnMultiFailure is true
1313    * <li>hbase.zookeeper.useMulti is true
1314    * </ul>
1315    * on calling multi, we get a ZooKeeper exception that can be handled by a
1316    * sequential call(*), we retry the operations one-by-one (sequentially).
1317    *
1318    * @param zkw
1319    *          - zk reference
1320    * @param runSequentialOnMultiFailure
1321    *          - if true when we get a ZooKeeper exception that could retry the
1322    *          operations one-by-one (sequentially)
1323    * @param pathRoots
1324    *          - path of the parent node(s)
1325    * @throws KeeperException.NotEmptyException
1326    *           if node has children while deleting
1327    * @throws KeeperException
1328    *           if unexpected ZooKeeper exception
1329    * @throws IllegalArgumentException
1330    *           if an invalid path is specified
1331    */
1332   public static void deleteChildrenRecursivelyMultiOrSequential(
1333       ZooKeeperWatcher zkw, boolean runSequentialOnMultiFailure,
1334       String... pathRoots) throws KeeperException {
1335     if (pathRoots == null || pathRoots.length <= 0) {
1336       LOG.warn("Given path is not valid!");
1337       return;
1338     }
1339     List<ZKUtilOp> ops = new ArrayList<ZKUtil.ZKUtilOp>();
1340     for (String eachRoot : pathRoots) {
1341       List<String> children = listChildrenBFSNoWatch(zkw, eachRoot);
1342       // Delete the leaves first and eventually get rid of the root
1343       for (int i = children.size() - 1; i >= 0; --i) {
1344         ops.add(ZKUtilOp.deleteNodeFailSilent(children.get(i)));
1345       }
1346     }
1347     // atleast one element should exist
1348     if (ops.size() > 0) {
1349       multiOrSequential(zkw, ops, runSequentialOnMultiFailure);
1350     }
1351   }
1352 
1353   /**
1354    * Delete the specified node and its children. This traverse the
1355    * znode tree for listing the children and then delete
1356    * these znodes including the parent using multi-update api or
1357    * sequential based on the specified configurations.
1358    * <p>
1359    * Sets no watches. Throws all exceptions besides dealing with deletion of
1360    * children.
1361    * <p>
1362    * If hbase.zookeeper.useMulti is true, use ZooKeeper's multi-update
1363    * functionality. Otherwise, run the list of operations sequentially.
1364    * <p>
1365    * If all of the following are true:
1366    * <ul>
1367    * <li>runSequentialOnMultiFailure is true
1368    * <li>hbase.zookeeper.useMulti is true
1369    * </ul>
1370    * on calling multi, we get a ZooKeeper exception that can be handled by a
1371    * sequential call(*), we retry the operations one-by-one (sequentially).
1372    *
1373    * @param zkw
1374    *          - zk reference
1375    * @param runSequentialOnMultiFailure
1376    *          - if true when we get a ZooKeeper exception that could retry the
1377    *          operations one-by-one (sequentially)
1378    * @param pathRoots
1379    *          - path of the parent node(s)
1380    * @throws KeeperException.NotEmptyException
1381    *           if node has children while deleting
1382    * @throws KeeperException
1383    *           if unexpected ZooKeeper exception
1384    * @throws IllegalArgumentException
1385    *           if an invalid path is specified
1386    */
1387   public static void deleteNodeRecursivelyMultiOrSequential(ZooKeeperWatcher zkw,
1388       boolean runSequentialOnMultiFailure, String... pathRoots) throws KeeperException {
1389     if (pathRoots == null || pathRoots.length <= 0) {
1390       LOG.warn("Given path is not valid!");
1391       return;
1392     }
1393     List<ZKUtilOp> ops = new ArrayList<ZKUtil.ZKUtilOp>();
1394     for (String eachRoot : pathRoots) {
1395       // Zookeeper Watches are one time triggers; When children of parent nodes are deleted
1396       // recursively, must set another watch, get notified of delete node
1397       List<String> children = listChildrenBFSAndWatchThem(zkw, eachRoot);
1398       // Delete the leaves first and eventually get rid of the root
1399       for (int i = children.size() - 1; i >= 0; --i) {
1400         ops.add(ZKUtilOp.deleteNodeFailSilent(children.get(i)));
1401       }
1402       try {
1403         if (zkw.getRecoverableZooKeeper().exists(eachRoot, zkw) != null) {
1404           ops.add(ZKUtilOp.deleteNodeFailSilent(eachRoot));
1405         }
1406       } catch (InterruptedException e) {
1407         zkw.interruptedException(e);
1408       }
1409     }
1410     // atleast one element should exist
1411     if (ops.size() > 0) {
1412       multiOrSequential(zkw, ops, runSequentialOnMultiFailure);
1413     }
1414   }
1415 
1416   /**
1417    * BFS Traversal of all the children under path, with the entries in the list,
1418    * in the same order as that of the traversal. Lists all the children without
1419    * setting any watches.
1420    *
1421    * @param zkw
1422    *          - zk reference
1423    * @param znode
1424    *          - path of node
1425    * @return list of children znodes under the path
1426    * @throws KeeperException
1427    *           if unexpected ZooKeeper exception
1428    */
1429   private static List<String> listChildrenBFSNoWatch(ZooKeeperWatcher zkw,
1430       final String znode) throws KeeperException {
1431     Deque<String> queue = new LinkedList<String>();
1432     List<String> tree = new ArrayList<String>();
1433     queue.add(znode);
1434     while (true) {
1435       String node = queue.pollFirst();
1436       if (node == null) {
1437         break;
1438       }
1439       List<String> children = listChildrenNoWatch(zkw, node);
1440       if (children == null) {
1441         continue;
1442       }
1443       for (final String child : children) {
1444         final String childPath = node + "/" + child;
1445         queue.add(childPath);
1446         tree.add(childPath);
1447       }
1448     }
1449     return tree;
1450   }
1451 
1452   /**
1453    * BFS Traversal of all the children under path, with the entries in the list,
1454    * in the same order as that of the traversal.
1455    * Lists all the children and set watches on to them.
1456    *
1457    * @param zkw
1458    *          - zk reference
1459    * @param znode
1460    *          - path of node
1461    * @return list of children znodes under the path
1462    * @throws KeeperException
1463    *           if unexpected ZooKeeper exception
1464    */
1465   private static List<String> listChildrenBFSAndWatchThem(ZooKeeperWatcher zkw, final String znode)
1466       throws KeeperException {
1467     Deque<String> queue = new LinkedList<String>();
1468     List<String> tree = new ArrayList<String>();
1469     queue.add(znode);
1470     while (true) {
1471       String node = queue.pollFirst();
1472       if (node == null) {
1473         break;
1474       }
1475       List<String> children = listChildrenAndWatchThem(zkw, node);
1476       if (children == null) {
1477         continue;
1478       }
1479       for (final String child : children) {
1480         final String childPath = node + "/" + child;
1481         queue.add(childPath);
1482         tree.add(childPath);
1483       }
1484     }
1485     return tree;
1486   }
1487 
1488   /**
1489    * Represents an action taken by ZKUtil, e.g. createAndFailSilent.
1490    * These actions are higher-level than ZKOp actions, which represent
1491    * individual actions in the ZooKeeper API, like create.
1492    */
1493   public abstract static class ZKUtilOp {
1494     private String path;
1495 
1496     private ZKUtilOp(String path) {
1497       this.path = path;
1498     }
1499 
1500     /**
1501      * @return a createAndFailSilent ZKUtilOp
1502      */
1503     public static ZKUtilOp createAndFailSilent(String path, byte[] data) {
1504       return new CreateAndFailSilent(path, data);
1505     }
1506 
1507     /**
1508      * @return a deleteNodeFailSilent ZKUtilOP
1509      */
1510     public static ZKUtilOp deleteNodeFailSilent(String path) {
1511       return new DeleteNodeFailSilent(path);
1512     }
1513 
1514     /**
1515      * @return a setData ZKUtilOp
1516      */
1517     public static ZKUtilOp setData(String path, byte [] data) {
1518       return new SetData(path, data);
1519     }
1520 
1521     /**
1522      * @return path to znode where the ZKOp will occur
1523      */
1524     public String getPath() {
1525       return path;
1526     }
1527 
1528     /**
1529      * ZKUtilOp representing createAndFailSilent in ZooKeeper
1530      * (attempt to create node, ignore error if already exists)
1531      */
1532     public static class CreateAndFailSilent extends ZKUtilOp {
1533       private byte [] data;
1534 
1535       private CreateAndFailSilent(String path, byte [] data) {
1536         super(path);
1537         this.data = data;
1538       }
1539 
1540       public byte[] getData() {
1541         return data;
1542       }
1543 
1544       @Override
1545       public boolean equals(Object o) {
1546         if (this == o) return true;
1547         if (!(o instanceof CreateAndFailSilent)) return false;
1548 
1549         CreateAndFailSilent op = (CreateAndFailSilent) o;
1550         return getPath().equals(op.getPath()) && Arrays.equals(data, op.data);
1551       }
1552 
1553       @Override
1554       public int hashCode() {
1555         int ret = 17 + getPath().hashCode() * 31;
1556         return ret * 31 + Bytes.hashCode(data);
1557       }
1558     }
1559 
1560     /**
1561      * ZKUtilOp representing deleteNodeFailSilent in ZooKeeper
1562      * (attempt to delete node, ignore error if node doesn't exist)
1563      */
1564     public static class DeleteNodeFailSilent extends ZKUtilOp {
1565       private DeleteNodeFailSilent(String path) {
1566         super(path);
1567       }
1568 
1569       @Override
1570       public boolean equals(Object o) {
1571         if (this == o) return true;
1572         if (!(o instanceof DeleteNodeFailSilent)) return false;
1573 
1574         return super.equals(o);
1575       }
1576 
1577       @Override
1578       public int hashCode() {
1579         return getPath().hashCode();
1580       }
1581     }
1582 
1583     /**
1584      * ZKUtilOp representing setData in ZooKeeper
1585      */
1586     public static class SetData extends ZKUtilOp {
1587       private byte [] data;
1588 
1589       private SetData(String path, byte [] data) {
1590         super(path);
1591         this.data = data;
1592       }
1593 
1594       public byte[] getData() {
1595         return data;
1596       }
1597 
1598       @Override
1599       public boolean equals(Object o) {
1600         if (this == o) return true;
1601         if (!(o instanceof SetData)) return false;
1602 
1603         SetData op = (SetData) o;
1604         return getPath().equals(op.getPath()) && Arrays.equals(data, op.data);
1605       }
1606 
1607       @Override
1608       public int hashCode() {
1609         int ret = getPath().hashCode();
1610         return ret * 31 + Bytes.hashCode(data);
1611       }
1612     }
1613   }
1614 
1615   /**
1616    * Convert from ZKUtilOp to ZKOp
1617    */
1618   private static Op toZooKeeperOp(ZooKeeperWatcher zkw, ZKUtilOp op)
1619   throws UnsupportedOperationException {
1620     if(op == null) return null;
1621 
1622     if (op instanceof CreateAndFailSilent) {
1623       CreateAndFailSilent cafs = (CreateAndFailSilent)op;
1624       return Op.create(cafs.getPath(), cafs.getData(), createACL(zkw, cafs.getPath()),
1625         CreateMode.PERSISTENT);
1626     } else if (op instanceof DeleteNodeFailSilent) {
1627       DeleteNodeFailSilent dnfs = (DeleteNodeFailSilent)op;
1628       return Op.delete(dnfs.getPath(), -1);
1629     } else if (op instanceof SetData) {
1630       SetData sd = (SetData)op;
1631       return Op.setData(sd.getPath(), sd.getData(), -1);
1632     } else {
1633       throw new UnsupportedOperationException("Unexpected ZKUtilOp type: "
1634         + op.getClass().getName());
1635     }
1636   }
1637 
1638   /**
1639    * If hbase.zookeeper.useMulti is true, use ZooKeeper's multi-update functionality.
1640    * Otherwise, run the list of operations sequentially.
1641    *
1642    * If all of the following are true:
1643    * - runSequentialOnMultiFailure is true
1644    * - hbase.zookeeper.useMulti is true
1645    * - on calling multi, we get a ZooKeeper exception that can be handled by a sequential call(*)
1646    * Then:
1647    * - we retry the operations one-by-one (sequentially)
1648    *
1649    * Note *: an example is receiving a NodeExistsException from a "create" call.  Without multi,
1650    * a user could call "createAndFailSilent" to ensure that a node exists if they don't care who
1651    * actually created the node (i.e. the NodeExistsException from ZooKeeper is caught).
1652    * This will cause all operations in the multi to fail, however, because
1653    * the NodeExistsException that zk.create throws will fail the multi transaction.
1654    * In this case, if the previous conditions hold, the commands are run sequentially, which should
1655    * result in the correct final state, but means that the operations will not run atomically.
1656    *
1657    * @throws KeeperException
1658    */
1659   public static void multiOrSequential(ZooKeeperWatcher zkw, List<ZKUtilOp> ops,
1660       boolean runSequentialOnMultiFailure) throws KeeperException {
1661     if (ops == null) return;
1662     boolean useMulti = zkw.getConfiguration().getBoolean(HConstants.ZOOKEEPER_USEMULTI, false);
1663 
1664     if (useMulti) {
1665       List<Op> zkOps = new LinkedList<Op>();
1666       for (ZKUtilOp op : ops) {
1667         zkOps.add(toZooKeeperOp(zkw, op));
1668       }
1669       try {
1670         zkw.getRecoverableZooKeeper().multi(zkOps);
1671       } catch (KeeperException ke) {
1672        switch (ke.code()) {
1673          case NODEEXISTS:
1674          case NONODE:
1675          case BADVERSION:
1676          case NOAUTH:
1677            // if we get an exception that could be solved by running sequentially
1678            // (and the client asked us to), then break out and run sequentially
1679            if (runSequentialOnMultiFailure) {
1680              LOG.info("On call to ZK.multi, received exception: " + ke.toString() + "."
1681                + "  Attempting to run operations sequentially because"
1682                + " runSequentialOnMultiFailure is: " + runSequentialOnMultiFailure + ".");
1683              processSequentially(zkw, ops);
1684              break;
1685            }
1686           default:
1687             throw ke;
1688         }
1689       } catch (InterruptedException ie) {
1690         zkw.interruptedException(ie);
1691       }
1692     } else {
1693       // run sequentially
1694       processSequentially(zkw, ops);
1695     }
1696 
1697   }
1698 
1699   private static void processSequentially(ZooKeeperWatcher zkw, List<ZKUtilOp> ops)
1700       throws KeeperException, NoNodeException {
1701     for (ZKUtilOp op : ops) {
1702       if (op instanceof CreateAndFailSilent) {
1703         createAndFailSilent(zkw, (CreateAndFailSilent) op);
1704       } else if (op instanceof DeleteNodeFailSilent) {
1705         deleteNodeFailSilent(zkw, (DeleteNodeFailSilent) op);
1706       } else if (op instanceof SetData) {
1707         setData(zkw, (SetData) op);
1708       } else {
1709         throw new UnsupportedOperationException("Unexpected ZKUtilOp type: "
1710             + op.getClass().getName());
1711       }
1712     }
1713   }
1714 
1715   //
1716   // ZooKeeper cluster information
1717   //
1718 
1719   /** @return String dump of everything in ZooKeeper. */
1720   public static String dump(ZooKeeperWatcher zkw) {
1721     StringBuilder sb = new StringBuilder();
1722     try {
1723       sb.append("HBase is rooted at ").append(zkw.baseZNode);
1724       sb.append("\nActive master address: ");
1725       try {
1726         sb.append(MasterAddressTracker.getMasterAddress(zkw));
1727       } catch (IOException e) {
1728         sb.append("<<FAILED LOOKUP: " + e.getMessage() + ">>");
1729       }
1730       sb.append("\nBackup master addresses:");
1731       for (String child : listChildrenNoWatch(zkw,
1732                                               zkw.backupMasterAddressesZNode)) {
1733         sb.append("\n ").append(child);
1734       }
1735       sb.append("\nRegion server holding hbase:meta: "
1736         + new MetaTableLocator().getMetaRegionLocation(zkw));
1737       Configuration conf = HBaseConfiguration.create();
1738       int numMetaReplicas = conf.getInt(HConstants.META_REPLICAS_NUM,
1739                HConstants.DEFAULT_META_REPLICA_NUM);
1740       for (int i = 1; i < numMetaReplicas; i++) {
1741         sb.append("\nRegion server holding hbase:meta, replicaId " + i + " "
1742                     + new MetaTableLocator().getMetaRegionLocation(zkw, i));
1743       }
1744       sb.append("\nRegion servers:");
1745       for (String child : listChildrenNoWatch(zkw, zkw.rsZNode)) {
1746         sb.append("\n ").append(child);
1747       }
1748       try {
1749         getReplicationZnodesDump(zkw, sb);
1750       } catch (KeeperException ke) {
1751         LOG.warn("Couldn't get the replication znode dump", ke);
1752       }
1753       sb.append("\nQuorum Server Statistics:");
1754       String[] servers = zkw.getQuorum().split(",");
1755       for (String server : servers) {
1756         sb.append("\n ").append(server);
1757         try {
1758           String[] stat = getServerStats(server, ZKUtil.zkDumpConnectionTimeOut);
1759 
1760           if (stat == null) {
1761             sb.append("[Error] invalid quorum server: " + server);
1762             break;
1763           }
1764 
1765           for (String s : stat) {
1766             sb.append("\n  ").append(s);
1767           }
1768         } catch (Exception e) {
1769           sb.append("\n  ERROR: ").append(e.getMessage());
1770         }
1771       }
1772     } catch (KeeperException ke) {
1773       sb.append("\nFATAL ZooKeeper Exception!\n");
1774       sb.append("\n" + ke.getMessage());
1775     }
1776     return sb.toString();
1777   }
1778 
1779   /**
1780    * Appends replication znodes to the passed StringBuilder.
1781    * @param zkw
1782    * @param sb
1783    * @throws KeeperException
1784    */
1785   private static void getReplicationZnodesDump(ZooKeeperWatcher zkw, StringBuilder sb)
1786       throws KeeperException {
1787     String replicationZNodeName = zkw.getConfiguration().get("zookeeper.znode.replication",
1788       "replication");
1789     String replicationZnode = joinZNode(zkw.baseZNode, replicationZNodeName);
1790     if (ZKUtil.checkExists(zkw, replicationZnode) == -1) return;
1791     // do a ls -r on this znode
1792     sb.append("\n").append(replicationZnode).append(": ");
1793     List<String> children = ZKUtil.listChildrenNoWatch(zkw, replicationZnode);
1794     for (String child : children) {
1795       String znode = joinZNode(replicationZnode, child);
1796       if (child.equals(zkw.getConfiguration().get("zookeeper.znode.replication.peers", "peers"))) {
1797         appendPeersZnodes(zkw, znode, sb);
1798       } else if (child.equals(zkw.getConfiguration().
1799           get("zookeeper.znode.replication.rs", "rs"))) {
1800         appendRSZnodes(zkw, znode, sb);
1801       } else if (child.equals(zkw.getConfiguration().get(
1802         ReplicationStateZKBase.ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_KEY,
1803         ReplicationStateZKBase.ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_DEFAULT))) {
1804         appendHFileRefsZnodes(zkw, znode, sb);
1805       }
1806     }
1807   }
1808 
1809   private static void appendHFileRefsZnodes(ZooKeeperWatcher zkw, String hfileRefsZnode,
1810       StringBuilder sb) throws KeeperException {
1811     sb.append("\n").append(hfileRefsZnode).append(": ");
1812     for (String peerIdZnode : ZKUtil.listChildrenNoWatch(zkw, hfileRefsZnode)) {
1813       String znodeToProcess = ZKUtil.joinZNode(hfileRefsZnode, peerIdZnode);
1814       sb.append("\n").append(znodeToProcess).append(": ");
1815       List<String> peerHFileRefsZnodes = ZKUtil.listChildrenNoWatch(zkw, znodeToProcess);
1816       int size = peerHFileRefsZnodes.size();
1817       for (int i = 0; i < size; i++) {
1818         sb.append(peerHFileRefsZnodes.get(i));
1819         if (i != size - 1) {
1820           sb.append(", ");
1821         }
1822       }
1823     }
1824   }
1825 
1826   private static void appendRSZnodes(ZooKeeperWatcher zkw, String znode, StringBuilder sb)
1827       throws KeeperException {
1828     List<String> stack = new LinkedList<String>();
1829     stack.add(znode);
1830     do {
1831       String znodeToProcess = stack.remove(stack.size() - 1);
1832       sb.append("\n").append(znodeToProcess).append(": ");
1833       byte[] data;
1834       try {
1835         data = ZKUtil.getData(zkw, znodeToProcess);
1836       } catch (InterruptedException e) {
1837         zkw.interruptedException(e);
1838         return;
1839       }
1840       if (data != null && data.length > 0) { // log position
1841         long position = 0;
1842         try {
1843           position = ZKUtil.parseWALPositionFrom(ZKUtil.getData(zkw, znodeToProcess));
1844           sb.append(position);
1845         } catch (DeserializationException ignored) {
1846         } catch (InterruptedException e) {
1847           zkw.interruptedException(e);
1848           return;
1849         }
1850       }
1851       for (String zNodeChild : ZKUtil.listChildrenNoWatch(zkw, znodeToProcess)) {
1852         stack.add(ZKUtil.joinZNode(znodeToProcess, zNodeChild));
1853       }
1854     } while (stack.size() > 0);
1855   }
1856 
1857   private static void appendPeersZnodes(ZooKeeperWatcher zkw, String peersZnode,
1858     StringBuilder sb) throws KeeperException {
1859     int pblen = ProtobufUtil.lengthOfPBMagic();
1860     sb.append("\n").append(peersZnode).append(": ");
1861     for (String peerIdZnode : ZKUtil.listChildrenNoWatch(zkw, peersZnode)) {
1862       String znodeToProcess = ZKUtil.joinZNode(peersZnode, peerIdZnode);
1863       byte[] data;
1864       try {
1865         data = ZKUtil.getData(zkw, znodeToProcess);
1866       } catch (InterruptedException e) {
1867         zkw.interruptedException(e);
1868         return;
1869       }
1870       // parse the data of the above peer znode.
1871       try {
1872         ZooKeeperProtos.ReplicationPeer.Builder builder =
1873           ZooKeeperProtos.ReplicationPeer.newBuilder();
1874         ProtobufUtil.mergeFrom(builder, data, pblen, data.length - pblen);
1875         String clusterKey = builder.getClusterkey();
1876         sb.append("\n").append(znodeToProcess).append(": ").append(clusterKey);
1877         // add the peer-state.
1878         appendPeerState(zkw, znodeToProcess, sb);
1879       } catch (IOException ipbe) {
1880         LOG.warn("Got Exception while parsing peer: " + znodeToProcess, ipbe);
1881       }
1882     }
1883   }
1884 
1885   private static void appendPeerState(ZooKeeperWatcher zkw, String znodeToProcess,
1886       StringBuilder sb) throws KeeperException, InvalidProtocolBufferException {
1887     String peerState = zkw.getConfiguration().get("zookeeper.znode.replication.peers.state",
1888       "peer-state");
1889     int pblen = ProtobufUtil.lengthOfPBMagic();
1890     for (String child : ZKUtil.listChildrenNoWatch(zkw, znodeToProcess)) {
1891       if (!child.equals(peerState)) continue;
1892       String peerStateZnode = ZKUtil.joinZNode(znodeToProcess, child);
1893       sb.append("\n").append(peerStateZnode).append(": ");
1894       byte[] peerStateData;
1895       try {
1896         peerStateData = ZKUtil.getData(zkw, peerStateZnode);
1897         ZooKeeperProtos.ReplicationState.Builder builder =
1898             ZooKeeperProtos.ReplicationState.newBuilder();
1899         ProtobufUtil.mergeFrom(builder, peerStateData, pblen, peerStateData.length - pblen);
1900         sb.append(builder.getState().name());
1901       } catch (IOException ipbe) {
1902         LOG.warn("Got Exception while parsing peer: " + znodeToProcess, ipbe);
1903       } catch (InterruptedException e) {
1904         zkw.interruptedException(e);
1905         return;
1906       }
1907     }
1908   }
1909 
1910   /**
1911    * Gets the statistics from the given server.
1912    *
1913    * @param server  The server to get the statistics from.
1914    * @param timeout  The socket timeout to use.
1915    * @return The array of response strings.
1916    * @throws IOException When the socket communication fails.
1917    */
1918   public static String[] getServerStats(String server, int timeout)
1919   throws IOException {
1920     String[] sp = server.split(":");
1921     if (sp == null || sp.length == 0) {
1922       return null;
1923     }
1924 
1925     String host = sp[0];
1926     int port = sp.length > 1 ? Integer.parseInt(sp[1])
1927         : HConstants.DEFAULT_ZOOKEPER_CLIENT_PORT;
1928 
1929     Socket socket = new Socket();
1930     InetSocketAddress sockAddr = new InetSocketAddress(host, port);
1931     socket.connect(sockAddr, timeout);
1932 
1933     socket.setSoTimeout(timeout);
1934     PrintWriter out = new PrintWriter(socket.getOutputStream(), true);
1935     BufferedReader in = new BufferedReader(new InputStreamReader(
1936       socket.getInputStream()));
1937     out.println("stat");
1938     out.flush();
1939     ArrayList<String> res = new ArrayList<String>();
1940     while (true) {
1941       String line = in.readLine();
1942       if (line != null) {
1943         res.add(line);
1944       } else {
1945         break;
1946       }
1947     }
1948     socket.close();
1949     return res.toArray(new String[res.size()]);
1950   }
1951 
1952   private static void logRetrievedMsg(final ZooKeeperWatcher zkw,
1953       final String znode, final byte [] data, final boolean watcherSet) {
1954     if (!LOG.isTraceEnabled()) return;
1955     LOG.trace(zkw.prefix("Retrieved " + ((data == null)? 0: data.length) +
1956       " byte(s) of data from znode " + znode +
1957       (watcherSet? " and set watcher; ": "; data=") +
1958       (data == null? "null": data.length == 0? "empty": (
1959           znode.startsWith(ZooKeeperWatcher.META_ZNODE_PREFIX)?
1960             getServerNameOrEmptyString(data):
1961           znode.startsWith(zkw.backupMasterAddressesZNode)?
1962             getServerNameOrEmptyString(data):
1963           StringUtils.abbreviate(Bytes.toStringBinary(data), 32)))));
1964   }
1965 
1966   private static String getServerNameOrEmptyString(final byte [] data) {
1967     try {
1968       return ServerName.parseFrom(data).toString();
1969     } catch (DeserializationException e) {
1970       return "";
1971     }
1972   }
1973 
1974   /**
1975    * Waits for HBase installation's base (parent) znode to become available.
1976    * @throws IOException on ZK errors
1977    */
1978   public static void waitForBaseZNode(Configuration conf) throws IOException {
1979     LOG.info("Waiting until the base znode is available");
1980     String parentZNode = conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT,
1981         HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT);
1982     ZooKeeper zk = new ZooKeeper(ZKConfig.getZKQuorumServersString(conf),
1983         conf.getInt(HConstants.ZK_SESSION_TIMEOUT,
1984         HConstants.DEFAULT_ZK_SESSION_TIMEOUT), EmptyWatcher.instance);
1985 
1986     final int maxTimeMs = 10000;
1987     final int maxNumAttempts = maxTimeMs / HConstants.SOCKET_RETRY_WAIT_MS;
1988 
1989     KeeperException keeperEx = null;
1990     try {
1991       try {
1992         for (int attempt = 0; attempt < maxNumAttempts; ++attempt) {
1993           try {
1994             if (zk.exists(parentZNode, false) != null) {
1995               LOG.info("Parent znode exists: " + parentZNode);
1996               keeperEx = null;
1997               break;
1998             }
1999           } catch (KeeperException e) {
2000             keeperEx = e;
2001           }
2002           Threads.sleepWithoutInterrupt(HConstants.SOCKET_RETRY_WAIT_MS);
2003         }
2004       } finally {
2005         zk.close();
2006       }
2007     } catch (InterruptedException ex) {
2008       Thread.currentThread().interrupt();
2009     }
2010 
2011     if (keeperEx != null) {
2012       throw new IOException(keeperEx);
2013     }
2014   }
2015 
2016   /**
2017    * Convert a {@link DeserializationException} to a more palatable {@link KeeperException}.
2018    * Used when can't let a {@link DeserializationException} out w/o changing public API.
2019    * @param e Exception to convert
2020    * @return Converted exception
2021    */
2022   public static KeeperException convert(final DeserializationException e) {
2023     KeeperException ke = new KeeperException.DataInconsistencyException();
2024     ke.initCause(e);
2025     return ke;
2026   }
2027 
2028   /**
2029    * Recursively print the current state of ZK (non-transactional)
2030    * @param root name of the root directory in zk to print
2031    */
2032   public static void logZKTree(ZooKeeperWatcher zkw, String root) {
2033     if (!LOG.isDebugEnabled()) return;
2034     LOG.debug("Current zk system:");
2035     String prefix = "|-";
2036     LOG.debug(prefix + root);
2037     try {
2038       logZKTree(zkw, root, prefix);
2039     } catch (KeeperException e) {
2040       throw new RuntimeException(e);
2041     }
2042   }
2043 
2044   /**
2045    * Helper method to print the current state of the ZK tree.
2046    * @see #logZKTree(ZooKeeperWatcher, String)
2047    * @throws KeeperException if an unexpected exception occurs
2048    */
2049   protected static void logZKTree(ZooKeeperWatcher zkw, String root, String prefix)
2050       throws KeeperException {
2051     List<String> children = ZKUtil.listChildrenNoWatch(zkw, root);
2052     if (children == null) return;
2053     for (String child : children) {
2054       LOG.debug(prefix + child);
2055       String node = ZKUtil.joinZNode(root.equals("/") ? "" : root, child);
2056       logZKTree(zkw, node, prefix + "---");
2057     }
2058   }
2059 
2060   /**
2061    * @param position
2062    * @return Serialized protobuf of <code>position</code> with pb magic prefix prepended suitable
2063    *         for use as content of an wal position in a replication queue.
2064    */
2065   public static byte[] positionToByteArray(final long position) {
2066     byte[] bytes = ZooKeeperProtos.ReplicationHLogPosition.newBuilder().setPosition(position)
2067         .build().toByteArray();
2068     return ProtobufUtil.prependPBMagic(bytes);
2069   }
2070 
2071   /**
2072    * @param bytes - Content of a WAL position znode.
2073    * @return long - The current WAL position.
2074    * @throws DeserializationException
2075    */
2076   public static long parseWALPositionFrom(final byte[] bytes) throws DeserializationException {
2077     if (bytes == null) {
2078       throw new DeserializationException("Unable to parse null WAL position.");
2079     }
2080     if (ProtobufUtil.isPBMagicPrefix(bytes)) {
2081       int pblen = ProtobufUtil.lengthOfPBMagic();
2082       ZooKeeperProtos.ReplicationHLogPosition.Builder builder =
2083           ZooKeeperProtos.ReplicationHLogPosition.newBuilder();
2084       ZooKeeperProtos.ReplicationHLogPosition position;
2085       try {
2086         ProtobufUtil.mergeFrom(builder, bytes, pblen, bytes.length - pblen);
2087         position = builder.build();
2088       } catch (IOException e) {
2089         throw new DeserializationException(e);
2090       }
2091       return position.getPosition();
2092     } else {
2093       if (bytes.length > 0) {
2094         return Bytes.toLong(bytes);
2095       }
2096       return 0;
2097     }
2098   }
2099 
2100   /**
2101    * @param regionLastFlushedSequenceId the flushed sequence id of a region which is the min of its
2102    *          store max seq ids
2103    * @param storeSequenceIds column family to sequence Id map
2104    * @return Serialized protobuf of <code>RegionSequenceIds</code> with pb magic prefix prepended
2105    *         suitable for use to filter wal edits in distributedLogReplay mode
2106    */
2107   public static byte[] regionSequenceIdsToByteArray(final Long regionLastFlushedSequenceId,
2108       final Map<byte[], Long> storeSequenceIds) {
2109     ClusterStatusProtos.RegionStoreSequenceIds.Builder regionSequenceIdsBuilder =
2110         ClusterStatusProtos.RegionStoreSequenceIds.newBuilder();
2111     ClusterStatusProtos.StoreSequenceId.Builder storeSequenceIdBuilder =
2112         ClusterStatusProtos.StoreSequenceId.newBuilder();
2113     if (storeSequenceIds != null) {
2114       for (Map.Entry<byte[], Long> e : storeSequenceIds.entrySet()){
2115         byte[] columnFamilyName = e.getKey();
2116         Long curSeqId = e.getValue();
2117         storeSequenceIdBuilder.setFamilyName(ByteStringer.wrap(columnFamilyName));
2118         storeSequenceIdBuilder.setSequenceId(curSeqId);
2119         regionSequenceIdsBuilder.addStoreSequenceId(storeSequenceIdBuilder.build());
2120         storeSequenceIdBuilder.clear();
2121       }
2122     }
2123     regionSequenceIdsBuilder.setLastFlushedSequenceId(regionLastFlushedSequenceId);
2124     byte[] result = regionSequenceIdsBuilder.build().toByteArray();
2125     return ProtobufUtil.prependPBMagic(result);
2126   }
2127 
2128   /**
2129    * @param bytes Content of serialized data of RegionStoreSequenceIds
2130    * @return a RegionStoreSequenceIds object
2131    * @throws DeserializationException
2132    */
2133   public static RegionStoreSequenceIds parseRegionStoreSequenceIds(final byte[] bytes)
2134       throws DeserializationException {
2135     if (bytes == null || !ProtobufUtil.isPBMagicPrefix(bytes)) {
2136       throw new DeserializationException("Unable to parse RegionStoreSequenceIds.");
2137     }
2138     RegionStoreSequenceIds.Builder regionSequenceIdsBuilder =
2139         ClusterStatusProtos.RegionStoreSequenceIds.newBuilder();
2140     int pblen = ProtobufUtil.lengthOfPBMagic();
2141     RegionStoreSequenceIds storeIds = null;
2142     try {
2143       ProtobufUtil.mergeFrom(regionSequenceIdsBuilder, bytes, pblen, bytes.length - pblen);
2144       storeIds = regionSequenceIdsBuilder.build();
2145     } catch (IOException e) {
2146       throw new DeserializationException(e);
2147     }
2148     return storeIds;
2149   }
2150 }