001/*
002 *
003 * Licensed to the Apache Software Foundation (ASF) under one
004 * or more contributor license agreements.  See the NOTICE file
005 * distributed with this work for additional information
006 * regarding copyright ownership.  The ASF licenses this file
007 * to you under the Apache License, Version 2.0 (the
008 * "License"); you may not use this file except in compliance
009 * with the License.  You may obtain a copy of the License at
010 *
011 *     http://www.apache.org/licenses/LICENSE-2.0
012 *
013 * Unless required by applicable law or agreed to in writing, software
014 * distributed under the License is distributed on an "AS IS" BASIS,
015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
016 * See the License for the specific language governing permissions and
017 * limitations under the License.
018 */
019package org.apache.hadoop.hbase.zookeeper;
020
021import java.io.BufferedReader;
022import java.io.BufferedWriter;
023import java.io.IOException;
024import java.io.InputStreamReader;
025import java.io.OutputStreamWriter;
026import java.io.PrintWriter;
027import java.net.InetSocketAddress;
028import java.net.Socket;
029import java.nio.charset.StandardCharsets;
030import java.util.ArrayList;
031import java.util.Arrays;
032import java.util.Collections;
033import java.util.Deque;
034import java.util.HashMap;
035import java.util.Iterator;
036import java.util.LinkedList;
037import java.util.List;
038import java.util.Map;
039import java.util.stream.Collectors;
040import javax.security.auth.login.AppConfigurationEntry;
041import javax.security.auth.login.AppConfigurationEntry.LoginModuleControlFlag;
042import org.apache.commons.lang3.StringUtils;
043import org.apache.hadoop.conf.Configuration;
044import org.apache.hadoop.hbase.AuthUtil;
045import org.apache.hadoop.hbase.HConstants;
046import org.apache.hadoop.hbase.exceptions.DeserializationException;
047import org.apache.hadoop.hbase.security.Superusers;
048import org.apache.hadoop.hbase.util.Bytes;
049import org.apache.hadoop.hbase.util.Threads;
050import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp.CreateAndFailSilent;
051import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp.DeleteNodeFailSilent;
052import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp.SetData;
053import org.apache.hadoop.security.SecurityUtil;
054import org.apache.hadoop.security.UserGroupInformation;
055import org.apache.hadoop.security.authentication.util.KerberosUtil;
056import org.apache.yetus.audience.InterfaceAudience;
057import org.apache.zookeeper.AsyncCallback;
058import org.apache.zookeeper.CreateMode;
059import org.apache.zookeeper.KeeperException;
060import org.apache.zookeeper.KeeperException.NoNodeException;
061import org.apache.zookeeper.Op;
062import org.apache.zookeeper.Watcher;
063import org.apache.zookeeper.ZooDefs.Ids;
064import org.apache.zookeeper.ZooDefs.Perms;
065import org.apache.zookeeper.ZooKeeper;
066import org.apache.zookeeper.client.ZooKeeperSaslClient;
067import org.apache.zookeeper.data.ACL;
068import org.apache.zookeeper.data.Id;
069import org.apache.zookeeper.data.Stat;
070import org.apache.zookeeper.proto.CreateRequest;
071import org.apache.zookeeper.proto.DeleteRequest;
072import org.apache.zookeeper.proto.SetDataRequest;
073import org.apache.zookeeper.server.ZooKeeperSaslServer;
074import org.slf4j.Logger;
075import org.slf4j.LoggerFactory;
076
077import org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException;
078
079import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
080import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos;
081
082/**
083 * Internal HBase utility class for ZooKeeper.
084 *
085 * <p>Contains only static methods and constants.
086 *
087 * <p>Methods all throw {@link KeeperException} if there is an unexpected
088 * zookeeper exception, so callers of these methods must handle appropriately.
089 * If ZK is required for the operation, the server will need to be aborted.
090 */
091@InterfaceAudience.Private
092public final class ZKUtil {
093  private static final Logger LOG = LoggerFactory.getLogger(ZKUtil.class);
094
095  private static int zkDumpConnectionTimeOut;
096
097  private ZKUtil() {
098  }
099
100  /**
101   * Creates a new connection to ZooKeeper, pulling settings and ensemble config
102   * from the specified configuration object using methods from {@link ZKConfig}.
103   *
104   * Sets the connection status monitoring watcher to the specified watcher.
105   *
106   * @param conf configuration to pull ensemble and other settings from
107   * @param watcher watcher to monitor connection changes
108   * @return connection to zookeeper
109   * @throws IOException if unable to connect to zk or config problem
110   */
111  public static RecoverableZooKeeper connect(Configuration conf, Watcher watcher)
112    throws IOException {
113    String ensemble = ZKConfig.getZKQuorumServersString(conf);
114    return connect(conf, ensemble, watcher);
115  }
116
117  public static RecoverableZooKeeper connect(Configuration conf, String ensemble,
118      Watcher watcher)
119    throws IOException {
120    return connect(conf, ensemble, watcher, null);
121  }
122
123  public static RecoverableZooKeeper connect(Configuration conf, String ensemble,
124      Watcher watcher, final String identifier)
125    throws IOException {
126    if(ensemble == null) {
127      throw new IOException("Unable to determine ZooKeeper ensemble");
128    }
129    int timeout = conf.getInt(HConstants.ZK_SESSION_TIMEOUT,
130        HConstants.DEFAULT_ZK_SESSION_TIMEOUT);
131    if (LOG.isTraceEnabled()) {
132      LOG.trace("{} opening connection to ZooKeeper ensemble={}", identifier, ensemble);
133    }
134    int retry = conf.getInt("zookeeper.recovery.retry", 3);
135    int retryIntervalMillis =
136      conf.getInt("zookeeper.recovery.retry.intervalmill", 1000);
137    int maxSleepTime = conf.getInt("zookeeper.recovery.retry.maxsleeptime", 60000);
138    zkDumpConnectionTimeOut = conf.getInt("zookeeper.dump.connection.timeout",
139        1000);
140    int multiMaxSize = conf.getInt("zookeeper.multi.max.size", 1024*1024);
141    return new RecoverableZooKeeper(ensemble, timeout, watcher,
142        retry, retryIntervalMillis, maxSleepTime, identifier, multiMaxSize);
143  }
144
145  /**
146   * Log in the current zookeeper server process using the given configuration
147   * keys for the credential file and login principal.
148   *
149   * <p><strong>This is only applicable when running on secure hbase</strong>
150   * On regular HBase (without security features), this will safely be ignored.
151   * </p>
152   *
153   * @param conf The configuration data to use
154   * @param keytabFileKey Property key used to configure the path to the credential file
155   * @param userNameKey Property key used to configure the login principal
156   * @param hostname Current hostname to use in any credentials
157   * @throws IOException underlying exception from SecurityUtil.login() call
158   */
159  public static void loginServer(Configuration conf, String keytabFileKey,
160      String userNameKey, String hostname) throws IOException {
161    login(conf, keytabFileKey, userNameKey, hostname,
162          ZooKeeperSaslServer.LOGIN_CONTEXT_NAME_KEY,
163          JaasConfiguration.SERVER_KEYTAB_KERBEROS_CONFIG_NAME);
164  }
165
166  /**
167   * Log in the current zookeeper client using the given configuration
168   * keys for the credential file and login principal.
169   *
170   * <p><strong>This is only applicable when running on secure hbase</strong>
171   * On regular HBase (without security features), this will safely be ignored.
172   * </p>
173   *
174   * @param conf The configuration data to use
175   * @param keytabFileKey Property key used to configure the path to the credential file
176   * @param userNameKey Property key used to configure the login principal
177   * @param hostname Current hostname to use in any credentials
178   * @throws IOException underlying exception from SecurityUtil.login() call
179   */
180  public static void loginClient(Configuration conf, String keytabFileKey,
181      String userNameKey, String hostname) throws IOException {
182    login(conf, keytabFileKey, userNameKey, hostname,
183          ZooKeeperSaslClient.LOGIN_CONTEXT_NAME_KEY,
184          JaasConfiguration.CLIENT_KEYTAB_KERBEROS_CONFIG_NAME);
185  }
186
187  /**
188   * Log in the current process using the given configuration keys for the
189   * credential file and login principal.
190   *
191   * <p><strong>This is only applicable when running on secure hbase</strong>
192   * On regular HBase (without security features), this will safely be ignored.
193   * </p>
194   *
195   * @param conf The configuration data to use
196   * @param keytabFileKey Property key used to configure the path to the credential file
197   * @param userNameKey Property key used to configure the login principal
198   * @param hostname Current hostname to use in any credentials
199   * @param loginContextProperty property name to expose the entry name
200   * @param loginContextName jaas entry name
201   * @throws IOException underlying exception from SecurityUtil.login() call
202   */
203  private static void login(Configuration conf, String keytabFileKey,
204      String userNameKey, String hostname,
205      String loginContextProperty, String loginContextName)
206      throws IOException {
207    if (!isSecureZooKeeper(conf)) {
208      return;
209    }
210
211    // User has specified a jaas.conf, keep this one as the good one.
212    // HBASE_OPTS="-Djava.security.auth.login.config=jaas.conf"
213    if (System.getProperty("java.security.auth.login.config") != null) {
214      return;
215    }
216
217    // No keytab specified, no auth
218    String keytabFilename = conf.get(keytabFileKey);
219    if (keytabFilename == null) {
220      LOG.warn("no keytab specified for: {}", keytabFileKey);
221      return;
222    }
223
224    String principalConfig = conf.get(userNameKey, System.getProperty("user.name"));
225    String principalName = SecurityUtil.getServerPrincipal(principalConfig, hostname);
226
227    // Initialize the "jaas.conf" for keyTab/principal,
228    // If keyTab is not specified use the Ticket Cache.
229    // and set the zookeeper login context name.
230    JaasConfiguration jaasConf = new JaasConfiguration(loginContextName,
231        principalName, keytabFilename);
232    javax.security.auth.login.Configuration.setConfiguration(jaasConf);
233    System.setProperty(loginContextProperty, loginContextName);
234  }
235
236  /**
237   * A JAAS configuration that defines the login modules that we want to use for login.
238   */
239  private static class JaasConfiguration extends javax.security.auth.login.Configuration {
240    private static final String SERVER_KEYTAB_KERBEROS_CONFIG_NAME =
241      "zookeeper-server-keytab-kerberos";
242    private static final String CLIENT_KEYTAB_KERBEROS_CONFIG_NAME =
243      "zookeeper-client-keytab-kerberos";
244
245    private static final Map<String, String> BASIC_JAAS_OPTIONS = new HashMap<>();
246    static {
247      String jaasEnvVar = System.getenv("HBASE_JAAS_DEBUG");
248      if ("true".equalsIgnoreCase(jaasEnvVar)) {
249        BASIC_JAAS_OPTIONS.put("debug", "true");
250      }
251    }
252
253    private static final Map<String,String> KEYTAB_KERBEROS_OPTIONS = new HashMap<>();
254    static {
255      KEYTAB_KERBEROS_OPTIONS.put("doNotPrompt", "true");
256      KEYTAB_KERBEROS_OPTIONS.put("storeKey", "true");
257      KEYTAB_KERBEROS_OPTIONS.put("refreshKrb5Config", "true");
258      KEYTAB_KERBEROS_OPTIONS.putAll(BASIC_JAAS_OPTIONS);
259    }
260
261    private static final AppConfigurationEntry KEYTAB_KERBEROS_LOGIN =
262      new AppConfigurationEntry(KerberosUtil.getKrb5LoginModuleName(),
263                                LoginModuleControlFlag.REQUIRED,
264                                KEYTAB_KERBEROS_OPTIONS);
265
266    private static final AppConfigurationEntry[] KEYTAB_KERBEROS_CONF =
267      new AppConfigurationEntry[]{KEYTAB_KERBEROS_LOGIN};
268
269    private javax.security.auth.login.Configuration baseConfig;
270    private final String loginContextName;
271    private final boolean useTicketCache;
272    private final String keytabFile;
273    private final String principal;
274
275    public JaasConfiguration(String loginContextName, String principal, String keytabFile) {
276      this(loginContextName, principal, keytabFile, keytabFile == null || keytabFile.length() == 0);
277    }
278
279    private JaasConfiguration(String loginContextName, String principal,
280                             String keytabFile, boolean useTicketCache) {
281      try {
282        this.baseConfig = javax.security.auth.login.Configuration.getConfiguration();
283      } catch (SecurityException e) {
284        this.baseConfig = null;
285      }
286      this.loginContextName = loginContextName;
287      this.useTicketCache = useTicketCache;
288      this.keytabFile = keytabFile;
289      this.principal = principal;
290      LOG.info("JaasConfiguration loginContextName={} principal={} useTicketCache={} keytabFile={}",
291        loginContextName, principal, useTicketCache, keytabFile);
292    }
293
294    @Override
295    public AppConfigurationEntry[] getAppConfigurationEntry(String appName) {
296      if (loginContextName.equals(appName)) {
297        if (!useTicketCache) {
298          KEYTAB_KERBEROS_OPTIONS.put("keyTab", keytabFile);
299          KEYTAB_KERBEROS_OPTIONS.put("useKeyTab", "true");
300        }
301        KEYTAB_KERBEROS_OPTIONS.put("principal", principal);
302        KEYTAB_KERBEROS_OPTIONS.put("useTicketCache", useTicketCache ? "true" : "false");
303        return KEYTAB_KERBEROS_CONF;
304      }
305
306      if (baseConfig != null) {
307        return baseConfig.getAppConfigurationEntry(appName);
308      }
309
310      return(null);
311    }
312  }
313
314  //
315  // Helper methods
316  //
317  /**
318   * Returns the full path of the immediate parent of the specified node.
319   * @param node path to get parent of
320   * @return parent of path, null if passed the root node or an invalid node
321   */
322  public static String getParent(String node) {
323    int idx = node.lastIndexOf(ZNodePaths.ZNODE_PATH_SEPARATOR);
324    return idx <= 0 ? null : node.substring(0, idx);
325  }
326
327  /**
328   * Get the name of the current node from the specified fully-qualified path.
329   * @param path fully-qualified path
330   * @return name of the current node
331   */
332  public static String getNodeName(String path) {
333    return path.substring(path.lastIndexOf("/")+1);
334  }
335
336  //
337  // Existence checks and watches
338  //
339
340  /**
341   * Watch the specified znode for delete/create/change events.  The watcher is
342   * set whether or not the node exists.  If the node already exists, the method
343   * returns true.  If the node does not exist, the method returns false.
344   *
345   * @param zkw zk reference
346   * @param znode path of node to watch
347   * @return true if znode exists, false if does not exist or error
348   * @throws KeeperException if unexpected zookeeper exception
349   */
350  public static boolean watchAndCheckExists(ZKWatcher zkw, String znode)
351    throws KeeperException {
352    try {
353      Stat s = zkw.getRecoverableZooKeeper().exists(znode, zkw);
354      boolean exists = s != null;
355      if (exists) {
356        LOG.debug(zkw.prefix("Set watcher on existing znode=" + znode));
357      } else {
358        LOG.debug(zkw.prefix("Set watcher on znode that does not yet exist, " + znode));
359      }
360      return exists;
361    } catch (KeeperException e) {
362      LOG.warn(zkw.prefix("Unable to set watcher on znode " + znode), e);
363      zkw.keeperException(e);
364      return false;
365    } catch (InterruptedException e) {
366      LOG.warn(zkw.prefix("Unable to set watcher on znode " + znode), e);
367      zkw.interruptedException(e);
368      return false;
369    }
370  }
371
372  /**
373   * Watch the specified znode, but only if exists. Useful when watching
374   * for deletions. Uses .getData() (and handles NoNodeException) instead
375   * of .exists() to accomplish this, as .getData() will only set a watch if
376   * the znode exists.
377   * @param zkw zk reference
378   * @param znode path of node to watch
379   * @return true if the watch is set, false if node does not exists
380   * @throws KeeperException if unexpected zookeeper exception
381   */
382  public static boolean setWatchIfNodeExists(ZKWatcher zkw, String znode)
383      throws KeeperException {
384    try {
385      zkw.getRecoverableZooKeeper().getData(znode, true, null);
386      return true;
387    } catch (NoNodeException e) {
388      return false;
389    } catch (InterruptedException e) {
390      LOG.warn(zkw.prefix("Unable to set watcher on znode " + znode), e);
391      zkw.interruptedException(e);
392      return false;
393    }
394  }
395
396  /**
397   * Check if the specified node exists.  Sets no watches.
398   *
399   * @param zkw zk reference
400   * @param znode path of node to watch
401   * @return version of the node if it exists, -1 if does not exist
402   * @throws KeeperException if unexpected zookeeper exception
403   */
404  public static int checkExists(ZKWatcher zkw, String znode)
405    throws KeeperException {
406    try {
407      Stat s = zkw.getRecoverableZooKeeper().exists(znode, null);
408      return s != null ? s.getVersion() : -1;
409    } catch (KeeperException e) {
410      LOG.warn(zkw.prefix("Unable to set watcher on znode (" + znode + ")"), e);
411      zkw.keeperException(e);
412      return -1;
413    } catch (InterruptedException e) {
414      LOG.warn(zkw.prefix("Unable to set watcher on znode (" + znode + ")"), e);
415      zkw.interruptedException(e);
416      return -1;
417    }
418  }
419
420  //
421  // Znode listings
422  //
423
424  /**
425   * Lists the children znodes of the specified znode.  Also sets a watch on
426   * the specified znode which will capture a NodeDeleted event on the specified
427   * znode as well as NodeChildrenChanged if any children of the specified znode
428   * are created or deleted.
429   *
430   * Returns null if the specified node does not exist.  Otherwise returns a
431   * list of children of the specified node.  If the node exists but it has no
432   * children, an empty list will be returned.
433   *
434   * @param zkw zk reference
435   * @param znode path of node to list and watch children of
436   * @return list of children of the specified node, an empty list if the node
437   *          exists but has no children, and null if the node does not exist
438   * @throws KeeperException if unexpected zookeeper exception
439   */
440  public static List<String> listChildrenAndWatchForNewChildren(
441          ZKWatcher zkw, String znode)
442    throws KeeperException {
443    try {
444      return zkw.getRecoverableZooKeeper().getChildren(znode, zkw);
445    } catch(KeeperException.NoNodeException ke) {
446      LOG.debug(zkw.prefix("Unable to list children of znode " + znode + " " +
447          "because node does not exist (not an error)"));
448      return null;
449    } catch (KeeperException e) {
450      LOG.warn(zkw.prefix("Unable to list children of znode " + znode + " "), e);
451      zkw.keeperException(e);
452      return null;
453    } catch (InterruptedException e) {
454      LOG.warn(zkw.prefix("Unable to list children of znode " + znode + " "), e);
455      zkw.interruptedException(e);
456      return null;
457    }
458  }
459
460  /**
461   * List all the children of the specified znode, setting a watch for children
462   * changes and also setting a watch on every individual child in order to get
463   * the NodeCreated and NodeDeleted events.
464   * @param zkw zookeeper reference
465   * @param znode node to get children of and watch
466   * @return list of znode names, null if the node doesn't exist
467   * @throws KeeperException if a ZooKeeper operation fails
468   */
469  public static List<String> listChildrenAndWatchThem(ZKWatcher zkw,
470      String znode) throws KeeperException {
471    List<String> children = listChildrenAndWatchForNewChildren(zkw, znode);
472    if (children == null) {
473      return null;
474    }
475    for (String child : children) {
476      watchAndCheckExists(zkw, ZNodePaths.joinZNode(znode, child));
477    }
478    return children;
479  }
480
481  /**
482   * Lists the children of the specified znode without setting any watches.
483   *
484   * Sets no watches at all, this method is best effort.
485   *
486   * Returns an empty list if the node has no children.  Returns null if the
487   * parent node itself does not exist.
488   *
489   * @param zkw zookeeper reference
490   * @param znode node to get children
491   * @return list of data of children of specified znode, empty if no children,
492   *         null if parent does not exist
493   * @throws KeeperException if unexpected zookeeper exception
494   */
495  public static List<String> listChildrenNoWatch(ZKWatcher zkw, String znode)
496    throws KeeperException {
497    List<String> children = null;
498    try {
499      // List the children without watching
500      children = zkw.getRecoverableZooKeeper().getChildren(znode, null);
501    } catch(KeeperException.NoNodeException nne) {
502      return null;
503    } catch(InterruptedException ie) {
504      zkw.interruptedException(ie);
505    }
506    return children;
507  }
508
509  /**
510   * Simple class to hold a node path and node data.
511   * @deprecated Unused
512   */
513  @Deprecated
514  public static class NodeAndData {
515    private String node;
516    private byte [] data;
517    public NodeAndData(String node, byte [] data) {
518      this.node = node;
519      this.data = data;
520    }
521    public String getNode() {
522      return node;
523    }
524    public byte [] getData() {
525      return data;
526    }
527    @Override
528    public String toString() {
529      return node;
530    }
531    public boolean isEmpty() {
532      return (data == null || data.length == 0);
533    }
534  }
535
536  /**
537   * Checks if the specified znode has any children.  Sets no watches.
538   *
539   * Returns true if the node exists and has children.  Returns false if the
540   * node does not exist or if the node does not have any children.
541   *
542   * Used during master initialization to determine if the master is a
543   * failed-over-to master or the first master during initial cluster startup.
544   * If the directory for regionserver ephemeral nodes is empty then this is
545   * a cluster startup, if not then it is not cluster startup.
546   *
547   * @param zkw zk reference
548   * @param znode path of node to check for children of
549   * @return true if node has children, false if not or node does not exist
550   * @throws KeeperException if unexpected zookeeper exception
551   */
552  public static boolean nodeHasChildren(ZKWatcher zkw, String znode)
553    throws KeeperException {
554    try {
555      return !zkw.getRecoverableZooKeeper().getChildren(znode, null).isEmpty();
556    } catch(KeeperException.NoNodeException ke) {
557      LOG.debug(zkw.prefix("Unable to list children of znode " + znode +
558              " because node does not exist (not an error)"));
559      return false;
560    } catch (KeeperException e) {
561      LOG.warn(zkw.prefix("Unable to list children of znode " + znode), e);
562      zkw.keeperException(e);
563      return false;
564    } catch (InterruptedException e) {
565      LOG.warn(zkw.prefix("Unable to list children of znode " + znode), e);
566      zkw.interruptedException(e);
567      return false;
568    }
569  }
570
571  /**
572   * Get the number of children of the specified node.
573   *
574   * If the node does not exist or has no children, returns 0.
575   *
576   * Sets no watches at all.
577   *
578   * @param zkw zk reference
579   * @param znode path of node to count children of
580   * @return number of children of specified node, 0 if none or parent does not
581   *         exist
582   * @throws KeeperException if unexpected zookeeper exception
583   */
584  public static int getNumberOfChildren(ZKWatcher zkw, String znode)
585    throws KeeperException {
586    try {
587      Stat stat = zkw.getRecoverableZooKeeper().exists(znode, null);
588      return stat == null ? 0 : stat.getNumChildren();
589    } catch(KeeperException e) {
590      LOG.warn(zkw.prefix("Unable to get children of node " + znode));
591      zkw.keeperException(e);
592    } catch(InterruptedException e) {
593      zkw.interruptedException(e);
594    }
595    return 0;
596  }
597
598  //
599  // Data retrieval
600  //
601
602  /**
603   * Get znode data. Does not set a watcher.
604   *
605   * @return ZNode data, null if the node does not exist or if there is an error.
606   */
607  public static byte [] getData(ZKWatcher zkw, String znode)
608      throws KeeperException, InterruptedException {
609    try {
610      byte [] data = zkw.getRecoverableZooKeeper().getData(znode, null, null);
611      logRetrievedMsg(zkw, znode, data, false);
612      return data;
613    } catch (KeeperException.NoNodeException e) {
614      LOG.debug(zkw.prefix("Unable to get data of znode " + znode + " " +
615          "because node does not exist (not an error)"));
616      return null;
617    } catch (KeeperException e) {
618      LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
619      zkw.keeperException(e);
620      return null;
621    }
622  }
623
624  /**
625   * Get the data at the specified znode and set a watch.
626   *
627   * Returns the data and sets a watch if the node exists.  Returns null and no
628   * watch is set if the node does not exist or there is an exception.
629   *
630   * @param zkw zk reference
631   * @param znode path of node
632   * @return data of the specified znode, or null
633   * @throws KeeperException if unexpected zookeeper exception
634   */
635  public static byte[] getDataAndWatch(ZKWatcher zkw, String znode) throws KeeperException {
636    return getDataInternal(zkw, znode, null, true, true);
637  }
638
639  /**
640   * Get the data at the specified znode and set a watch.
641   * Returns the data and sets a watch if the node exists.  Returns null and no
642   * watch is set if the node does not exist or there is an exception.
643   *
644   * @param zkw              zk reference
645   * @param znode            path of node
646   * @param throwOnInterrupt if false then just interrupt the thread, do not throw exception
647   * @return data of the specified znode, or null
648   * @throws KeeperException if unexpected zookeeper exception
649   */
650  public static byte[] getDataAndWatch(ZKWatcher zkw, String znode, boolean throwOnInterrupt)
651    throws KeeperException {
652    return getDataInternal(zkw, znode, null, true, throwOnInterrupt);
653  }
654
655  /**
656   * Get the data at the specified znode and set a watch.
657   *
658   * Returns the data and sets a watch if the node exists.  Returns null and no
659   * watch is set if the node does not exist or there is an exception.
660   *
661   * @param zkw zk reference
662   * @param znode path of node
663   * @param stat object to populate the version of the znode
664   * @return data of the specified znode, or null
665   * @throws KeeperException if unexpected zookeeper exception
666   */
667  public static byte[] getDataAndWatch(ZKWatcher zkw, String znode,
668                                       Stat stat) throws KeeperException {
669    return getDataInternal(zkw, znode, stat, true, true);
670  }
671
672  private static byte[] getDataInternal(ZKWatcher zkw, String znode, Stat stat, boolean watcherSet,
673    boolean throwOnInterrupt)
674      throws KeeperException {
675    try {
676      byte [] data = zkw.getRecoverableZooKeeper().getData(znode, zkw, stat);
677      logRetrievedMsg(zkw, znode, data, watcherSet);
678      return data;
679    } catch (KeeperException.NoNodeException e) {
680      // This log can get pretty annoying when we cycle on 100ms waits.
681      // Enable trace if you really want to see it.
682      LOG.trace(zkw.prefix("Unable to get data of znode " + znode + " " +
683        "because node does not exist (not an error)"));
684      return null;
685    } catch (KeeperException e) {
686      LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
687      zkw.keeperException(e);
688      return null;
689    } catch (InterruptedException e) {
690      LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
691      if (throwOnInterrupt) {
692        zkw.interruptedException(e);
693      } else {
694        zkw.interruptedExceptionNoThrow(e, true);
695      }
696      return null;
697    }
698  }
699
700  /**
701   * Get the data at the specified znode without setting a watch.
702   *
703   * Returns the data if the node exists.  Returns null if the node does not
704   * exist.
705   *
706   * Sets the stats of the node in the passed Stat object.  Pass a null stat if
707   * not interested.
708   *
709   * @param zkw zk reference
710   * @param znode path of node
711   * @param stat node status to get if node exists
712   * @return data of the specified znode, or null if node does not exist
713   * @throws KeeperException if unexpected zookeeper exception
714   */
715  public static byte [] getDataNoWatch(ZKWatcher zkw, String znode,
716                                       Stat stat)
717    throws KeeperException {
718    try {
719      byte [] data = zkw.getRecoverableZooKeeper().getData(znode, null, stat);
720      logRetrievedMsg(zkw, znode, data, false);
721      return data;
722    } catch (KeeperException.NoNodeException e) {
723      LOG.debug(zkw.prefix("Unable to get data of znode " + znode + " " +
724          "because node does not exist (not necessarily an error)"));
725      return null;
726    } catch (KeeperException e) {
727      LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
728      zkw.keeperException(e);
729      return null;
730    } catch (InterruptedException e) {
731      LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
732      zkw.interruptedException(e);
733      return null;
734    }
735  }
736
737  /**
738   * Returns the date of child znodes of the specified znode.  Also sets a watch on
739   * the specified znode which will capture a NodeDeleted event on the specified
740   * znode as well as NodeChildrenChanged if any children of the specified znode
741   * are created or deleted.
742   *
743   * Returns null if the specified node does not exist.  Otherwise returns a
744   * list of children of the specified node.  If the node exists but it has no
745   * children, an empty list will be returned.
746   *
747   * @param zkw zk reference
748   * @param baseNode path of node to list and watch children of
749   * @return list of data of children of the specified node, an empty list if the node
750   *          exists but has no children, and null if the node does not exist
751   * @throws KeeperException if unexpected zookeeper exception
752   * @deprecated Unused
753   */
754  @Deprecated
755  public static List<NodeAndData> getChildDataAndWatchForNewChildren(ZKWatcher zkw, String baseNode)
756    throws KeeperException {
757    return getChildDataAndWatchForNewChildren(zkw, baseNode, true);
758  }
759
760  /**
761   * Returns the date of child znodes of the specified znode.  Also sets a watch on
762   * the specified znode which will capture a NodeDeleted event on the specified
763   * znode as well as NodeChildrenChanged if any children of the specified znode
764   * are created or deleted.
765   *
766   * Returns null if the specified node does not exist.  Otherwise returns a
767   * list of children of the specified node.  If the node exists but it has no
768   * children, an empty list will be returned.
769   *
770   * @param zkw zk reference
771   * @param baseNode path of node to list and watch children of
772   * @param throwOnInterrupt if true then just interrupt the thread, do not throw exception
773   * @return list of data of children of the specified node, an empty list if the node
774   *          exists but has no children, and null if the node does not exist
775   * @throws KeeperException if unexpected zookeeper exception
776   * @deprecated Unused
777   */
778  @Deprecated
779  public static List<NodeAndData> getChildDataAndWatchForNewChildren(
780          ZKWatcher zkw, String baseNode, boolean throwOnInterrupt) throws KeeperException {
781    List<String> nodes =
782      ZKUtil.listChildrenAndWatchForNewChildren(zkw, baseNode);
783    if (nodes != null) {
784      List<NodeAndData> newNodes = new ArrayList<>();
785      for (String node : nodes) {
786        if (Thread.interrupted()) {
787          // Partial data should not be processed. Cancel processing by sending empty list.
788          return Collections.emptyList();
789        }
790        String nodePath = ZNodePaths.joinZNode(baseNode, node);
791        byte[] data = ZKUtil.getDataAndWatch(zkw, nodePath, throwOnInterrupt);
792        newNodes.add(new NodeAndData(nodePath, data));
793      }
794      return newNodes;
795    }
796    return null;
797  }
798
799  /**
800   * Update the data of an existing node with the expected version to have the
801   * specified data.
802   *
803   * Throws an exception if there is a version mismatch or some other problem.
804   *
805   * Sets no watches under any conditions.
806   *
807   * @param zkw zk reference
808   * @param znode the path to the ZNode
809   * @param data the data to store in ZooKeeper
810   * @param expectedVersion the expected version
811   * @throws KeeperException if unexpected zookeeper exception
812   * @throws KeeperException.BadVersionException if version mismatch
813   * @deprecated Unused
814   */
815  @Deprecated
816  public static void updateExistingNodeData(ZKWatcher zkw, String znode, byte[] data,
817      int expectedVersion) throws KeeperException {
818    try {
819      zkw.getRecoverableZooKeeper().setData(znode, data, expectedVersion);
820    } catch(InterruptedException ie) {
821      zkw.interruptedException(ie);
822    }
823  }
824
825  //
826  // Data setting
827  //
828
829  /**
830   * Sets the data of the existing znode to be the specified data.  Ensures that
831   * the current data has the specified expected version.
832   *
833   * <p>If the node does not exist, a {@link NoNodeException} will be thrown.
834   *
835   * <p>If their is a version mismatch, method returns null.
836   *
837   * <p>No watches are set but setting data will trigger other watchers of this
838   * node.
839   *
840   * <p>If there is another problem, a KeeperException will be thrown.
841   *
842   * @param zkw zk reference
843   * @param znode path of node
844   * @param data data to set for node
845   * @param expectedVersion version expected when setting data
846   * @return true if data set, false if version mismatch
847   * @throws KeeperException if unexpected zookeeper exception
848   */
849  public static boolean setData(ZKWatcher zkw, String znode,
850                                byte [] data, int expectedVersion)
851    throws KeeperException, KeeperException.NoNodeException {
852    try {
853      return zkw.getRecoverableZooKeeper().setData(znode, data, expectedVersion) != null;
854    } catch (InterruptedException e) {
855      zkw.interruptedException(e);
856      return false;
857    }
858  }
859
860  /**
861   * Set data into node creating node if it doesn't yet exist.
862   * Does not set watch.
863   *
864   * @param zkw zk reference
865   * @param znode path of node
866   * @param data data to set for node
867   * @throws KeeperException if a ZooKeeper operation fails
868   */
869  public static void createSetData(final ZKWatcher zkw, final String znode, final byte [] data)
870          throws KeeperException {
871    if (checkExists(zkw, znode) == -1) {
872      ZKUtil.createWithParents(zkw, znode, data);
873    } else {
874      ZKUtil.setData(zkw, znode, data);
875    }
876  }
877
878  /**
879   * Sets the data of the existing znode to be the specified data.  The node
880   * must exist but no checks are done on the existing data or version.
881   *
882   * <p>If the node does not exist, a {@link NoNodeException} will be thrown.
883   *
884   * <p>No watches are set but setting data will trigger other watchers of this
885   * node.
886   *
887   * <p>If there is another problem, a KeeperException will be thrown.
888   *
889   * @param zkw zk reference
890   * @param znode path of node
891   * @param data data to set for node
892   * @throws KeeperException if unexpected zookeeper exception
893   */
894  public static void setData(ZKWatcher zkw, String znode, byte [] data)
895    throws KeeperException, KeeperException.NoNodeException {
896    setData(zkw, (SetData)ZKUtilOp.setData(znode, data));
897  }
898
899  private static void setData(ZKWatcher zkw, SetData setData)
900    throws KeeperException, KeeperException.NoNodeException {
901    SetDataRequest sd = (SetDataRequest)toZooKeeperOp(zkw, setData).toRequestRecord();
902    setData(zkw, sd.getPath(), sd.getData(), sd.getVersion());
903  }
904
905  /**
906   * Returns whether or not secure authentication is enabled
907   * (whether <code>hbase.security.authentication</code> is set to
908   * <code>kerberos</code>.
909   */
910  public static boolean isSecureZooKeeper(Configuration conf) {
911    // Detection for embedded HBase client with jaas configuration
912    // defined for third party programs.
913    try {
914      javax.security.auth.login.Configuration testConfig =
915          javax.security.auth.login.Configuration.getConfiguration();
916      if (testConfig.getAppConfigurationEntry("Client") == null
917          && testConfig.getAppConfigurationEntry(
918            JaasConfiguration.CLIENT_KEYTAB_KERBEROS_CONFIG_NAME) == null
919          && testConfig.getAppConfigurationEntry(
920              JaasConfiguration.SERVER_KEYTAB_KERBEROS_CONFIG_NAME) == null
921          && conf.get(HConstants.ZK_CLIENT_KERBEROS_PRINCIPAL) == null
922          && conf.get(HConstants.ZK_SERVER_KERBEROS_PRINCIPAL) == null) {
923
924        return false;
925      }
926    } catch(Exception e) {
927      // No Jaas configuration defined.
928      return false;
929    }
930
931    // Master & RSs uses hbase.zookeeper.client.*
932    return "kerberos".equalsIgnoreCase(conf.get("hbase.security.authentication"));
933  }
934
935  private static ArrayList<ACL> createACL(ZKWatcher zkw, String node) {
936    return createACL(zkw, node, isSecureZooKeeper(zkw.getConfiguration()));
937  }
938
939  public static ArrayList<ACL> createACL(ZKWatcher zkw, String node,
940                                         boolean isSecureZooKeeper) {
941    if (!node.startsWith(zkw.getZNodePaths().baseZNode)) {
942      return Ids.OPEN_ACL_UNSAFE;
943    }
944    if (isSecureZooKeeper) {
945      ArrayList<ACL> acls = new ArrayList<>();
946      // add permission to hbase supper user
947      String[] superUsers = zkw.getConfiguration().getStrings(Superusers.SUPERUSER_CONF_KEY);
948      String hbaseUser = null;
949      try {
950        hbaseUser = UserGroupInformation.getCurrentUser().getShortUserName();
951      } catch (IOException e) {
952        LOG.debug("Could not acquire current User.", e);
953      }
954      if (superUsers != null) {
955        List<String> groups = new ArrayList<>();
956        for (String user : superUsers) {
957          if (AuthUtil.isGroupPrincipal(user)) {
958            // TODO: Set node ACL for groups when ZK supports this feature
959            groups.add(user);
960          } else {
961            if(!user.equals(hbaseUser)) {
962              acls.add(new ACL(Perms.ALL, new Id("sasl", user)));
963            }
964          }
965        }
966        if (!groups.isEmpty()) {
967          LOG.warn("Znode ACL setting for group {} is skipped, ZooKeeper doesn't support this " +
968            "feature presently.", groups);
969        }
970      }
971      // Certain znodes are accessed directly by the client,
972      // so they must be readable by non-authenticated clients
973      if (zkw.getZNodePaths().isClientReadable(node)) {
974        acls.addAll(Ids.CREATOR_ALL_ACL);
975        acls.addAll(Ids.READ_ACL_UNSAFE);
976      } else {
977        acls.addAll(Ids.CREATOR_ALL_ACL);
978      }
979      return acls;
980    } else {
981      return Ids.OPEN_ACL_UNSAFE;
982    }
983  }
984
985  //
986  // Node creation
987  //
988
989  /**
990   *
991   * Set the specified znode to be an ephemeral node carrying the specified
992   * data.
993   *
994   * If the node is created successfully, a watcher is also set on the node.
995   *
996   * If the node is not created successfully because it already exists, this
997   * method will also set a watcher on the node.
998   *
999   * If there is another problem, a KeeperException will be thrown.
1000   *
1001   * @param zkw zk reference
1002   * @param znode path of node
1003   * @param data data of node
1004   * @return true if node created, false if not, watch set in both cases
1005   * @throws KeeperException if unexpected zookeeper exception
1006   */
1007  public static boolean createEphemeralNodeAndWatch(ZKWatcher zkw,
1008      String znode, byte [] data)
1009    throws KeeperException {
1010    boolean ret = true;
1011    try {
1012      zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode),
1013          CreateMode.EPHEMERAL);
1014    } catch (KeeperException.NodeExistsException nee) {
1015      ret = false;
1016    } catch (InterruptedException e) {
1017      LOG.info("Interrupted", e);
1018      Thread.currentThread().interrupt();
1019    }
1020    if(!watchAndCheckExists(zkw, znode)) {
1021      // It did exist but now it doesn't, try again
1022      return createEphemeralNodeAndWatch(zkw, znode, data);
1023    }
1024    return ret;
1025  }
1026
1027  /**
1028   * Creates the specified znode to be a persistent node carrying the specified
1029   * data.
1030   *
1031   * Returns true if the node was successfully created, false if the node
1032   * already existed.
1033   *
1034   * If the node is created successfully, a watcher is also set on the node.
1035   *
1036   * If the node is not created successfully because it already exists, this
1037   * method will also set a watcher on the node but return false.
1038   *
1039   * If there is another problem, a KeeperException will be thrown.
1040   *
1041   * @param zkw zk reference
1042   * @param znode path of node
1043   * @param data data of node
1044   * @return true if node created, false if not, watch set in both cases
1045   * @throws KeeperException if unexpected zookeeper exception
1046   */
1047  public static boolean createNodeIfNotExistsAndWatch(
1048          ZKWatcher zkw, String znode, byte [] data)
1049    throws KeeperException {
1050    boolean ret = true;
1051    try {
1052      zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode),
1053          CreateMode.PERSISTENT);
1054    } catch (KeeperException.NodeExistsException nee) {
1055      ret = false;
1056    } catch (InterruptedException e) {
1057      zkw.interruptedException(e);
1058      return false;
1059    }
1060    try {
1061      zkw.getRecoverableZooKeeper().exists(znode, zkw);
1062    } catch (InterruptedException e) {
1063      zkw.interruptedException(e);
1064      return false;
1065    }
1066    return ret;
1067  }
1068
1069  /**
1070   * Creates the specified znode with the specified data but does not watch it.
1071   *
1072   * Returns the znode of the newly created node
1073   *
1074   * If there is another problem, a KeeperException will be thrown.
1075   *
1076   * @param zkw zk reference
1077   * @param znode path of node
1078   * @param data data of node
1079   * @param createMode specifying whether the node to be created is ephemeral and/or sequential
1080   * @return true name of the newly created znode or null
1081   * @throws KeeperException if unexpected zookeeper exception
1082   */
1083  public static String createNodeIfNotExistsNoWatch(ZKWatcher zkw, String znode, byte[] data,
1084      CreateMode createMode) throws KeeperException {
1085    String createdZNode = null;
1086    try {
1087      createdZNode = zkw.getRecoverableZooKeeper().create(znode, data,
1088          createACL(zkw, znode), createMode);
1089    } catch (KeeperException.NodeExistsException nee) {
1090      return znode;
1091    } catch (InterruptedException e) {
1092      zkw.interruptedException(e);
1093      return null;
1094    }
1095    return createdZNode;
1096  }
1097
1098  /**
1099   * Creates the specified node with the specified data and watches it.
1100   *
1101   * <p>Throws an exception if the node already exists.
1102   *
1103   * <p>The node created is persistent and open access.
1104   *
1105   * <p>Returns the version number of the created node if successful.
1106   *
1107   * @param zkw zk reference
1108   * @param znode path of node to create
1109   * @param data data of node to create
1110   * @return version of node created
1111   * @throws KeeperException if unexpected zookeeper exception
1112   * @throws KeeperException.NodeExistsException if node already exists
1113   */
1114  public static int createAndWatch(ZKWatcher zkw,
1115      String znode, byte [] data)
1116    throws KeeperException, KeeperException.NodeExistsException {
1117    try {
1118      zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode),
1119          CreateMode.PERSISTENT);
1120      Stat stat = zkw.getRecoverableZooKeeper().exists(znode, zkw);
1121      if (stat == null){
1122        // Likely a race condition. Someone deleted the znode.
1123        throw KeeperException.create(KeeperException.Code.SYSTEMERROR,
1124            "ZK.exists returned null (i.e.: znode does not exist) for znode=" + znode);
1125      }
1126
1127      return stat.getVersion();
1128    } catch (InterruptedException e) {
1129      zkw.interruptedException(e);
1130      return -1;
1131    }
1132  }
1133
1134  /**
1135   * Async creates the specified node with the specified data.
1136   *
1137   * <p>Throws an exception if the node already exists.
1138   *
1139   * <p>The node created is persistent and open access.
1140   *
1141   * @param zkw zk reference
1142   * @param znode path of node to create
1143   * @param data data of node to create
1144   * @param cb the callback to use for the creation
1145   * @param ctx the context to use for the creation
1146   */
1147  public static void asyncCreate(ZKWatcher zkw,
1148      String znode, byte [] data, final AsyncCallback.StringCallback cb,
1149      final Object ctx) {
1150    zkw.getRecoverableZooKeeper().getZooKeeper().create(znode, data,
1151        createACL(zkw, znode), CreateMode.PERSISTENT, cb, ctx);
1152  }
1153
1154  /**
1155   * Creates the specified node, iff the node does not exist.  Does not set a
1156   * watch and fails silently if the node already exists.
1157   *
1158   * The node created is persistent and open access.
1159   *
1160   * @param zkw zk reference
1161   * @param znode path of node
1162   * @throws KeeperException if unexpected zookeeper exception
1163   */
1164  public static void createAndFailSilent(ZKWatcher zkw,
1165      String znode) throws KeeperException {
1166    createAndFailSilent(zkw, znode, new byte[0]);
1167  }
1168
1169  /**
1170   * Creates the specified node containing specified data, iff the node does not exist.  Does
1171   * not set a watch and fails silently if the node already exists.
1172   *
1173   * The node created is persistent and open access.
1174   *
1175   * @param zkw zk reference
1176   * @param znode path of node
1177   * @param data a byte array data to store in the znode
1178   * @throws KeeperException if unexpected zookeeper exception
1179   */
1180  public static void createAndFailSilent(ZKWatcher zkw,
1181      String znode, byte[] data)
1182    throws KeeperException {
1183    createAndFailSilent(zkw,
1184        (CreateAndFailSilent)ZKUtilOp.createAndFailSilent(znode, data));
1185  }
1186
1187  private static void createAndFailSilent(ZKWatcher zkw, CreateAndFailSilent cafs)
1188    throws KeeperException {
1189    CreateRequest create = (CreateRequest)toZooKeeperOp(zkw, cafs).toRequestRecord();
1190    String znode = create.getPath();
1191    try {
1192      RecoverableZooKeeper zk = zkw.getRecoverableZooKeeper();
1193      if (zk.exists(znode, false) == null) {
1194        zk.create(znode, create.getData(), create.getAcl(), CreateMode.fromFlag(create.getFlags()));
1195      }
1196    } catch(KeeperException.NodeExistsException nee) {
1197    } catch(KeeperException.NoAuthException nee){
1198      try {
1199        if (null == zkw.getRecoverableZooKeeper().exists(znode, false)) {
1200          // If we failed to create the file and it does not already exist.
1201          throw(nee);
1202        }
1203      } catch (InterruptedException ie) {
1204        zkw.interruptedException(ie);
1205      }
1206    } catch(InterruptedException ie) {
1207      zkw.interruptedException(ie);
1208    }
1209  }
1210
1211  /**
1212   * Creates the specified node and all parent nodes required for it to exist.
1213   *
1214   * No watches are set and no errors are thrown if the node already exists.
1215   *
1216   * The nodes created are persistent and open access.
1217   *
1218   * @param zkw zk reference
1219   * @param znode path of node
1220   * @throws KeeperException if unexpected zookeeper exception
1221   */
1222  public static void createWithParents(ZKWatcher zkw, String znode)
1223    throws KeeperException {
1224    createWithParents(zkw, znode, new byte[0]);
1225  }
1226
1227  /**
1228   * Creates the specified node and all parent nodes required for it to exist.  The creation of
1229   * parent znodes is not atomic with the leafe znode creation but the data is written atomically
1230   * when the leaf node is created.
1231   *
1232   * No watches are set and no errors are thrown if the node already exists.
1233   *
1234   * The nodes created are persistent and open access.
1235   *
1236   * @param zkw zk reference
1237   * @param znode path of node
1238   * @throws KeeperException if unexpected zookeeper exception
1239   */
1240  public static void createWithParents(ZKWatcher zkw, String znode, byte[] data)
1241    throws KeeperException {
1242    try {
1243      if(znode == null) {
1244        return;
1245      }
1246      zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode),
1247          CreateMode.PERSISTENT);
1248    } catch(KeeperException.NodeExistsException nee) {
1249      return;
1250    } catch(KeeperException.NoNodeException nne) {
1251      createWithParents(zkw, getParent(znode));
1252      createWithParents(zkw, znode, data);
1253    } catch(InterruptedException ie) {
1254      zkw.interruptedException(ie);
1255    }
1256  }
1257
1258  //
1259  // Deletes
1260  //
1261
1262  /**
1263   * Delete the specified node.  Sets no watches.  Throws all exceptions.
1264   */
1265  public static void deleteNode(ZKWatcher zkw, String node)
1266    throws KeeperException {
1267    deleteNode(zkw, node, -1);
1268  }
1269
1270  /**
1271   * Delete the specified node with the specified version.  Sets no watches.
1272   * Throws all exceptions.
1273   */
1274  public static boolean deleteNode(ZKWatcher zkw, String node,
1275                                   int version)
1276    throws KeeperException {
1277    try {
1278      zkw.getRecoverableZooKeeper().delete(node, version);
1279      return true;
1280    } catch(KeeperException.BadVersionException bve) {
1281      return false;
1282    } catch(InterruptedException ie) {
1283      zkw.interruptedException(ie);
1284      return false;
1285    }
1286  }
1287
1288  /**
1289   * Deletes the specified node.  Fails silent if the node does not exist.
1290   *
1291   * @param zkw reference to the {@link ZKWatcher} which also contains configuration and operation
1292   * @param node the node to delete
1293   * @throws KeeperException if a ZooKeeper operation fails
1294   */
1295  public static void deleteNodeFailSilent(ZKWatcher zkw, String node)
1296    throws KeeperException {
1297    deleteNodeFailSilent(zkw,
1298      (DeleteNodeFailSilent)ZKUtilOp.deleteNodeFailSilent(node));
1299  }
1300
1301  private static void deleteNodeFailSilent(ZKWatcher zkw,
1302      DeleteNodeFailSilent dnfs) throws KeeperException {
1303    DeleteRequest delete = (DeleteRequest)toZooKeeperOp(zkw, dnfs).toRequestRecord();
1304    try {
1305      zkw.getRecoverableZooKeeper().delete(delete.getPath(), delete.getVersion());
1306    } catch(KeeperException.NoNodeException nne) {
1307    } catch(InterruptedException ie) {
1308      zkw.interruptedException(ie);
1309    }
1310  }
1311
1312
1313  /**
1314   * Delete the specified node and all of it's children.
1315   * <p>
1316   * If the node does not exist, just returns.
1317   * <p>
1318   * Sets no watches. Throws all exceptions besides dealing with deletion of
1319   * children.
1320   */
1321  public static void deleteNodeRecursively(ZKWatcher zkw, String node)
1322    throws KeeperException {
1323    deleteNodeRecursivelyMultiOrSequential(zkw, true, node);
1324  }
1325
1326  /**
1327   * Delete all the children of the specified node but not the node itself.
1328   *
1329   * Sets no watches.  Throws all exceptions besides dealing with deletion of
1330   * children.
1331   *
1332   * @throws KeeperException if a ZooKeeper operation fails
1333   */
1334  public static void deleteChildrenRecursively(ZKWatcher zkw, String node)
1335      throws KeeperException {
1336    deleteChildrenRecursivelyMultiOrSequential(zkw, true, node);
1337  }
1338
1339  /**
1340   * Delete all the children of the specified node but not the node itself. This
1341   * will first traverse the znode tree for listing the children and then delete
1342   * these znodes using multi-update api or sequential based on the specified
1343   * configurations.
1344   * <p>
1345   * Sets no watches. Throws all exceptions besides dealing with deletion of
1346   * children.
1347   * <p>
1348   * If the following is true:
1349   * <ul>
1350   * <li>runSequentialOnMultiFailure is true
1351   * </ul>
1352   * on calling multi, we get a ZooKeeper exception that can be handled by a
1353   * sequential call(*), we retry the operations one-by-one (sequentially).
1354   *
1355   * @param zkw
1356   *          - zk reference
1357   * @param runSequentialOnMultiFailure
1358   *          - if true when we get a ZooKeeper exception that could retry the
1359   *          operations one-by-one (sequentially)
1360   * @param pathRoots
1361   *          - path of the parent node(s)
1362   * @throws KeeperException.NotEmptyException
1363   *           if node has children while deleting
1364   * @throws KeeperException
1365   *           if unexpected ZooKeeper exception
1366   * @throws IllegalArgumentException
1367   *           if an invalid path is specified
1368   */
1369  public static void deleteChildrenRecursivelyMultiOrSequential(
1370          ZKWatcher zkw, boolean runSequentialOnMultiFailure,
1371          String... pathRoots) throws KeeperException {
1372    if (pathRoots == null || pathRoots.length <= 0) {
1373      LOG.warn("Given path is not valid!");
1374      return;
1375    }
1376    List<ZKUtilOp> ops = new ArrayList<>();
1377    for (String eachRoot : pathRoots) {
1378      List<String> children = listChildrenBFSNoWatch(zkw, eachRoot);
1379      // Delete the leaves first and eventually get rid of the root
1380      for (int i = children.size() - 1; i >= 0; --i) {
1381        ops.add(ZKUtilOp.deleteNodeFailSilent(children.get(i)));
1382      }
1383    }
1384    submitBatchedMultiOrSequential(zkw, runSequentialOnMultiFailure, ops);
1385  }
1386
1387  /**
1388   * Delete the specified node and its children. This traverse the
1389   * znode tree for listing the children and then delete
1390   * these znodes including the parent using multi-update api or
1391   * sequential based on the specified configurations.
1392   * <p>
1393   * Sets no watches. Throws all exceptions besides dealing with deletion of
1394   * children.
1395   * <p>
1396   * If the following is true:
1397   * <ul>
1398   * <li>runSequentialOnMultiFailure is true
1399   * </ul>
1400   * on calling multi, we get a ZooKeeper exception that can be handled by a
1401   * sequential call(*), we retry the operations one-by-one (sequentially).
1402   *
1403   * @param zkw
1404   *          - zk reference
1405   * @param runSequentialOnMultiFailure
1406   *          - if true when we get a ZooKeeper exception that could retry the
1407   *          operations one-by-one (sequentially)
1408   * @param pathRoots
1409   *          - path of the parent node(s)
1410   * @throws KeeperException.NotEmptyException
1411   *           if node has children while deleting
1412   * @throws KeeperException
1413   *           if unexpected ZooKeeper exception
1414   * @throws IllegalArgumentException
1415   *           if an invalid path is specified
1416   */
1417  public static void deleteNodeRecursivelyMultiOrSequential(ZKWatcher zkw,
1418      boolean runSequentialOnMultiFailure, String... pathRoots) throws KeeperException {
1419    if (pathRoots == null || pathRoots.length <= 0) {
1420      LOG.warn("Given path is not valid!");
1421      return;
1422    }
1423    List<ZKUtilOp> ops = new ArrayList<>();
1424    for (String eachRoot : pathRoots) {
1425      // ZooKeeper Watches are one time triggers; When children of parent nodes are deleted
1426      // recursively, must set another watch, get notified of delete node
1427      List<String> children = listChildrenBFSAndWatchThem(zkw, eachRoot);
1428      // Delete the leaves first and eventually get rid of the root
1429      for (int i = children.size() - 1; i >= 0; --i) {
1430        ops.add(ZKUtilOp.deleteNodeFailSilent(children.get(i)));
1431      }
1432      try {
1433        if (zkw.getRecoverableZooKeeper().exists(eachRoot, zkw) != null) {
1434          ops.add(ZKUtilOp.deleteNodeFailSilent(eachRoot));
1435        }
1436      } catch (InterruptedException e) {
1437        zkw.interruptedException(e);
1438      }
1439    }
1440    submitBatchedMultiOrSequential(zkw, runSequentialOnMultiFailure, ops);
1441  }
1442
1443  /**
1444   * Chunks the provided {@code ops} when their approximate size exceeds the the configured limit.
1445   * Take caution that this can ONLY be used for operations where atomicity is not important,
1446   * e.g. deletions. It must not be used when atomicity of the operations is critical.
1447   *
1448   * @param zkw reference to the {@link ZKWatcher} which contains configuration and constants
1449   * @param runSequentialOnMultiFailure if true when we get a ZooKeeper exception that could
1450   *        retry the operations one-by-one (sequentially)
1451   * @param ops list of ZKUtilOp {@link ZKUtilOp} to partition while submitting batched multi
1452   *        or sequential
1453   * @throws KeeperException unexpected ZooKeeper Exception / Zookeeper unreachable
1454   */
1455  private static void submitBatchedMultiOrSequential(ZKWatcher zkw,
1456      boolean runSequentialOnMultiFailure, List<ZKUtilOp> ops) throws KeeperException {
1457    // at least one element should exist
1458    if (ops.isEmpty()) {
1459      return;
1460    }
1461    final int maxMultiSize = zkw.getRecoverableZooKeeper().getMaxMultiSizeLimit();
1462    // Batch up the items to over smashing through jute.maxbuffer with too many Ops.
1463    final List<List<ZKUtilOp>> batchedOps = partitionOps(ops, maxMultiSize);
1464    // Would use forEach() but have to handle KeeperException
1465    for (List<ZKUtilOp> batch : batchedOps) {
1466      multiOrSequential(zkw, batch, runSequentialOnMultiFailure);
1467    }
1468  }
1469
1470  /**
1471   * Partition the list of {@code ops} by size (using {@link #estimateSize(ZKUtilOp)}).
1472   */
1473  static List<List<ZKUtilOp>> partitionOps(List<ZKUtilOp> ops, int maxPartitionSize) {
1474    List<List<ZKUtilOp>> partitionedOps = new ArrayList<>();
1475    List<ZKUtilOp> currentPartition = new ArrayList<>();
1476    int currentPartitionSize = 0;
1477    partitionedOps.add(currentPartition);
1478    Iterator<ZKUtilOp> iter = ops.iterator();
1479    while (iter.hasNext()) {
1480      ZKUtilOp currentOp = iter.next();
1481      int currentOpSize = estimateSize(currentOp);
1482
1483      // Roll a new partition if necessary
1484      // If the current partition is empty, put the element in there anyways.
1485      // We can roll a new partition if we get another element
1486      if (!currentPartition.isEmpty() && currentOpSize + currentPartitionSize > maxPartitionSize) {
1487        currentPartition = new ArrayList<>();
1488        partitionedOps.add(currentPartition);
1489        currentPartitionSize = 0;
1490      }
1491
1492      // Add the current op to the partition
1493      currentPartition.add(currentOp);
1494      // And record its size
1495      currentPartitionSize += currentOpSize;
1496    }
1497    return partitionedOps;
1498  }
1499
1500  static int estimateSize(ZKUtilOp op) {
1501    return Bytes.toBytes(op.getPath()).length;
1502  }
1503
1504  /**
1505   * BFS Traversal of all the children under path, with the entries in the list,
1506   * in the same order as that of the traversal. Lists all the children without
1507   * setting any watches.
1508   *
1509   * @param zkw
1510   *          - zk reference
1511   * @param znode
1512   *          - path of node
1513   * @return list of children znodes under the path
1514   * @throws KeeperException
1515   *           if unexpected ZooKeeper exception
1516   */
1517  private static List<String> listChildrenBFSNoWatch(ZKWatcher zkw,
1518      final String znode) throws KeeperException {
1519    Deque<String> queue = new LinkedList<>();
1520    List<String> tree = new ArrayList<>();
1521    queue.add(znode);
1522    while (true) {
1523      String node = queue.pollFirst();
1524      if (node == null) {
1525        break;
1526      }
1527      List<String> children = listChildrenNoWatch(zkw, node);
1528      if (children == null) {
1529        continue;
1530      }
1531      for (final String child : children) {
1532        final String childPath = node + "/" + child;
1533        queue.add(childPath);
1534        tree.add(childPath);
1535      }
1536    }
1537    return tree;
1538  }
1539
1540  /**
1541   * BFS Traversal of all the children under path, with the entries in the list,
1542   * in the same order as that of the traversal.
1543   * Lists all the children and set watches on to them.
1544   *
1545   * @param zkw
1546   *          - zk reference
1547   * @param znode
1548   *          - path of node
1549   * @return list of children znodes under the path
1550   * @throws KeeperException
1551   *           if unexpected ZooKeeper exception
1552   */
1553  private static List<String> listChildrenBFSAndWatchThem(ZKWatcher zkw, final String znode)
1554      throws KeeperException {
1555    Deque<String> queue = new LinkedList<>();
1556    List<String> tree = new ArrayList<>();
1557    queue.add(znode);
1558    while (true) {
1559      String node = queue.pollFirst();
1560      if (node == null) {
1561        break;
1562      }
1563      List<String> children = listChildrenAndWatchThem(zkw, node);
1564      if (children == null) {
1565        continue;
1566      }
1567      for (final String child : children) {
1568        final String childPath = node + "/" + child;
1569        queue.add(childPath);
1570        tree.add(childPath);
1571      }
1572    }
1573    return tree;
1574  }
1575
1576  /**
1577   * Represents an action taken by ZKUtil, e.g. createAndFailSilent.
1578   * These actions are higher-level than ZKOp actions, which represent
1579   * individual actions in the ZooKeeper API, like create.
1580   */
1581  public abstract static class ZKUtilOp {
1582    private String path;
1583
1584    @Override public String toString() {
1585      return this.getClass().getSimpleName() + ", path=" + this.path;
1586    }
1587
1588    private ZKUtilOp(String path) {
1589      this.path = path;
1590    }
1591
1592    /**
1593     * @return a createAndFailSilent ZKUtilOp
1594     */
1595    public static ZKUtilOp createAndFailSilent(String path, byte[] data) {
1596      return new CreateAndFailSilent(path, data);
1597    }
1598
1599    /**
1600     * @return a deleteNodeFailSilent ZKUtilOP
1601     */
1602    public static ZKUtilOp deleteNodeFailSilent(String path) {
1603      return new DeleteNodeFailSilent(path);
1604    }
1605
1606    /**
1607     * @return a setData ZKUtilOp
1608     */
1609    public static ZKUtilOp setData(String path, byte[] data) {
1610      return new SetData(path, data);
1611    }
1612
1613    /**
1614     * @return a setData ZKUtilOp
1615     */
1616    public static ZKUtilOp setData(String path, byte[] data, int version) {
1617      return new SetData(path, data, version);
1618    }
1619
1620    /**
1621     * @return path to znode where the ZKOp will occur
1622     */
1623    public String getPath() {
1624      return path;
1625    }
1626
1627    /**
1628     * ZKUtilOp representing createAndFailSilent in ZooKeeper
1629     * (attempt to create node, ignore error if already exists)
1630     */
1631    public static final class CreateAndFailSilent extends ZKUtilOp {
1632      private byte [] data;
1633
1634      private CreateAndFailSilent(String path, byte [] data) {
1635        super(path);
1636        this.data = data;
1637      }
1638
1639      public byte[] getData() {
1640        return data;
1641      }
1642
1643      @Override
1644      public boolean equals(Object o) {
1645        if (this == o) {
1646          return true;
1647        }
1648        if (!(o instanceof CreateAndFailSilent)) {
1649          return false;
1650        }
1651
1652        CreateAndFailSilent op = (CreateAndFailSilent) o;
1653        return getPath().equals(op.getPath()) && Arrays.equals(data, op.data);
1654      }
1655
1656      @Override
1657      public int hashCode() {
1658        int ret = 17 + getPath().hashCode() * 31;
1659        return ret * 31 + Bytes.hashCode(data);
1660      }
1661    }
1662
1663    /**
1664     * ZKUtilOp representing deleteNodeFailSilent in ZooKeeper
1665     * (attempt to delete node, ignore error if node doesn't exist)
1666     */
1667    public static final class DeleteNodeFailSilent extends ZKUtilOp {
1668      private DeleteNodeFailSilent(String path) {
1669        super(path);
1670      }
1671
1672      @Override
1673      public boolean equals(Object o) {
1674        if (this == o) {
1675          return true;
1676        }
1677        if (!(o instanceof DeleteNodeFailSilent)) {
1678          return false;
1679        }
1680
1681        return super.equals(o);
1682      }
1683
1684      @Override
1685      public int hashCode() {
1686        return getPath().hashCode();
1687      }
1688    }
1689
1690    /**
1691     * ZKUtilOp representing setData in ZooKeeper
1692     */
1693    public static final class SetData extends ZKUtilOp {
1694      private byte[] data;
1695      private int version = -1;
1696
1697      private SetData(String path, byte[] data) {
1698        super(path);
1699        this.data = data;
1700      }
1701
1702      private SetData(String path, byte[] data, int version) {
1703        super(path);
1704        this.data = data;
1705        this.version = version;
1706      }
1707
1708      public byte[] getData() {
1709        return data;
1710      }
1711
1712      public int getVersion() {
1713        return version;
1714      }
1715
1716      @Override
1717      public boolean equals(Object o) {
1718        if (this == o) {
1719          return true;
1720        }
1721        if (!(o instanceof SetData)) {
1722          return false;
1723        }
1724
1725        SetData op = (SetData) o;
1726        return getPath().equals(op.getPath()) && Arrays.equals(data, op.data)
1727            && getVersion() == op.getVersion();
1728      }
1729
1730      @Override
1731      public int hashCode() {
1732        int ret = getPath().hashCode();
1733        ret = ret * 31 + Bytes.hashCode(data);
1734        return ret * 31 + Integer.hashCode(version);
1735      }
1736    }
1737  }
1738
1739  /**
1740   * Convert from ZKUtilOp to ZKOp
1741   */
1742  private static Op toZooKeeperOp(ZKWatcher zkw, ZKUtilOp op) throws UnsupportedOperationException {
1743    if(op == null) {
1744      return null;
1745    }
1746
1747    if (op instanceof CreateAndFailSilent) {
1748      CreateAndFailSilent cafs = (CreateAndFailSilent)op;
1749      return Op.create(cafs.getPath(), cafs.getData(), createACL(zkw, cafs.getPath()),
1750        CreateMode.PERSISTENT);
1751    } else if (op instanceof DeleteNodeFailSilent) {
1752      DeleteNodeFailSilent dnfs = (DeleteNodeFailSilent)op;
1753      return Op.delete(dnfs.getPath(), -1);
1754    } else if (op instanceof SetData) {
1755      SetData sd = (SetData) op;
1756      return Op.setData(sd.getPath(), sd.getData(), sd.getVersion());
1757    } else {
1758      throw new UnsupportedOperationException("Unexpected ZKUtilOp type: "
1759        + op.getClass().getName());
1760    }
1761  }
1762
1763  /**
1764   * Use ZooKeeper's multi-update functionality.
1765   *
1766   * If all of the following are true:
1767   * - runSequentialOnMultiFailure is true
1768   * - on calling multi, we get a ZooKeeper exception that can be handled by a sequential call(*)
1769   * Then:
1770   * - we retry the operations one-by-one (sequentially)
1771   *
1772   * Note *: an example is receiving a NodeExistsException from a "create" call.  Without multi,
1773   * a user could call "createAndFailSilent" to ensure that a node exists if they don't care who
1774   * actually created the node (i.e. the NodeExistsException from ZooKeeper is caught).
1775   * This will cause all operations in the multi to fail, however, because
1776   * the NodeExistsException that zk.create throws will fail the multi transaction.
1777   * In this case, if the previous conditions hold, the commands are run sequentially, which should
1778   * result in the correct final state, but means that the operations will not run atomically.
1779   *
1780   * @throws KeeperException if a ZooKeeper operation fails
1781   */
1782  public static void multiOrSequential(ZKWatcher zkw, List<ZKUtilOp> ops,
1783                                       boolean runSequentialOnMultiFailure) throws KeeperException {
1784    if (zkw.getConfiguration().get("hbase.zookeeper.useMulti") != null) {
1785      LOG.warn("hbase.zookeeper.useMulti is deprecated. Default to true always.");
1786    }
1787    if (ops == null) {
1788      return;
1789    }
1790
1791    List<Op> zkOps = new LinkedList<>();
1792    for (ZKUtilOp op : ops) {
1793      zkOps.add(toZooKeeperOp(zkw, op));
1794    }
1795    try {
1796      zkw.getRecoverableZooKeeper().multi(zkOps);
1797    } catch (KeeperException ke) {
1798      switch (ke.code()) {
1799        case NODEEXISTS:
1800        case NONODE:
1801        case BADVERSION:
1802        case NOAUTH:
1803        case NOTEMPTY:
1804          // if we get an exception that could be solved by running sequentially
1805          // (and the client asked us to), then break out and run sequentially
1806          if (runSequentialOnMultiFailure) {
1807            LOG.info("multi exception: {}; running operations sequentially " +
1808              "(runSequentialOnMultiFailure=true); {}", ke.toString(),
1809              ops.stream().map(o -> o.toString()).collect(Collectors.joining(",")));
1810            processSequentially(zkw, ops);
1811            break;
1812          }
1813        default:
1814          throw ke;
1815      }
1816    } catch (InterruptedException ie) {
1817      zkw.interruptedException(ie);
1818    }
1819  }
1820
1821  private static void processSequentially(ZKWatcher zkw, List<ZKUtilOp> ops)
1822      throws KeeperException, NoNodeException {
1823    for (ZKUtilOp op : ops) {
1824      if (op instanceof CreateAndFailSilent) {
1825        createAndFailSilent(zkw, (CreateAndFailSilent) op);
1826      } else if (op instanceof DeleteNodeFailSilent) {
1827        deleteNodeFailSilent(zkw, (DeleteNodeFailSilent) op);
1828      } else if (op instanceof SetData) {
1829        setData(zkw, (SetData) op);
1830      } else {
1831        throw new UnsupportedOperationException("Unexpected ZKUtilOp type: "
1832            + op.getClass().getName());
1833      }
1834    }
1835  }
1836
1837  //
1838  // ZooKeeper cluster information
1839  //
1840
1841  /** @return String dump of everything in ZooKeeper. */
1842  public static String dump(ZKWatcher zkw) {
1843    StringBuilder sb = new StringBuilder();
1844    try {
1845      sb.append("HBase is rooted at ").append(zkw.getZNodePaths().baseZNode);
1846      sb.append("\nActive master address: ");
1847      try {
1848        sb.append(MasterAddressTracker.getMasterAddress(zkw));
1849      } catch (IOException e) {
1850        sb.append("<<FAILED LOOKUP: " + e.getMessage() + ">>");
1851      }
1852      sb.append("\nBackup master addresses:");
1853      final List<String> backupMasterChildrenNoWatchList = listChildrenNoWatch(zkw,
1854              zkw.getZNodePaths().backupMasterAddressesZNode);
1855      if (backupMasterChildrenNoWatchList != null) {
1856        for (String child : backupMasterChildrenNoWatchList) {
1857          sb.append("\n ").append(child);
1858        }
1859      }
1860      sb.append("\nRegion server holding hbase:meta: "
1861        + MetaTableLocator.getMetaRegionLocation(zkw));
1862      int numMetaReplicas = zkw.getMetaReplicaNodes().size();
1863      for (int i = 1; i < numMetaReplicas; i++) {
1864        sb.append("\nRegion server holding hbase:meta, replicaId " + i + " "
1865                    + MetaTableLocator.getMetaRegionLocation(zkw, i));
1866      }
1867      sb.append("\nRegion servers:");
1868      final List<String> rsChildrenNoWatchList =
1869              listChildrenNoWatch(zkw, zkw.getZNodePaths().rsZNode);
1870      if (rsChildrenNoWatchList != null) {
1871        for (String child : rsChildrenNoWatchList) {
1872          sb.append("\n ").append(child);
1873        }
1874      }
1875      try {
1876        getReplicationZnodesDump(zkw, sb);
1877      } catch (KeeperException ke) {
1878        LOG.warn("Couldn't get the replication znode dump", ke);
1879      }
1880      sb.append("\nQuorum Server Statistics:");
1881      String[] servers = zkw.getQuorum().split(",");
1882      for (String server : servers) {
1883        sb.append("\n ").append(server);
1884        try {
1885          String[] stat = getServerStats(server, ZKUtil.zkDumpConnectionTimeOut);
1886
1887          if (stat == null) {
1888            sb.append("[Error] invalid quorum server: " + server);
1889            break;
1890          }
1891
1892          for (String s : stat) {
1893            sb.append("\n  ").append(s);
1894          }
1895        } catch (Exception e) {
1896          sb.append("\n  ERROR: ").append(e.getMessage());
1897        }
1898      }
1899    } catch (KeeperException ke) {
1900      sb.append("\nFATAL ZooKeeper Exception!\n");
1901      sb.append("\n" + ke.getMessage());
1902    }
1903    return sb.toString();
1904  }
1905
1906  /**
1907   * Appends replication znodes to the passed StringBuilder.
1908   *
1909   * @param zkw reference to the {@link ZKWatcher} which also contains configuration and operation
1910   * @param sb the {@link StringBuilder} to append to
1911   * @throws KeeperException if a ZooKeeper operation fails
1912   */
1913  private static void getReplicationZnodesDump(ZKWatcher zkw, StringBuilder sb)
1914      throws KeeperException {
1915    String replicationZnode = zkw.getZNodePaths().replicationZNode;
1916
1917    if (ZKUtil.checkExists(zkw, replicationZnode) == -1) {
1918      return;
1919    }
1920
1921    // do a ls -r on this znode
1922    sb.append("\n").append(replicationZnode).append(": ");
1923    List<String> children = ZKUtil.listChildrenNoWatch(zkw, replicationZnode);
1924    if (children != null) {
1925      Collections.sort(children);
1926      for (String child : children) {
1927        String zNode = ZNodePaths.joinZNode(replicationZnode, child);
1928        if (zNode.equals(zkw.getZNodePaths().peersZNode)) {
1929          appendPeersZnodes(zkw, zNode, sb);
1930        } else if (zNode.equals(zkw.getZNodePaths().queuesZNode)) {
1931          appendRSZnodes(zkw, zNode, sb);
1932        } else if (zNode.equals(zkw.getZNodePaths().hfileRefsZNode)) {
1933          appendHFileRefsZNodes(zkw, zNode, sb);
1934        }
1935      }
1936    }
1937  }
1938
1939  private static void appendHFileRefsZNodes(ZKWatcher zkw, String hFileRefsZNode,
1940                                            StringBuilder sb) throws KeeperException {
1941    sb.append("\n").append(hFileRefsZNode).append(": ");
1942    final List<String> hFileRefChildrenNoWatchList =
1943            ZKUtil.listChildrenNoWatch(zkw, hFileRefsZNode);
1944    if (hFileRefChildrenNoWatchList != null) {
1945      for (String peerIdZNode : hFileRefChildrenNoWatchList) {
1946        String zNodeToProcess = ZNodePaths.joinZNode(hFileRefsZNode, peerIdZNode);
1947        sb.append("\n").append(zNodeToProcess).append(": ");
1948        List<String> peerHFileRefsZNodes = ZKUtil.listChildrenNoWatch(zkw, zNodeToProcess);
1949        if (peerHFileRefsZNodes != null) {
1950          sb.append(String.join(", ", peerHFileRefsZNodes));
1951        }
1952      }
1953    }
1954  }
1955
1956  /**
1957   * Returns a string with replication znodes and position of the replication log
1958   * @param zkw reference to the {@link ZKWatcher} which also contains configuration and operation
1959   * @return aq string of replication znodes and log positions
1960   */
1961  public static String getReplicationZnodesDump(ZKWatcher zkw) throws KeeperException {
1962    StringBuilder sb = new StringBuilder();
1963    getReplicationZnodesDump(zkw, sb);
1964    return sb.toString();
1965  }
1966
1967  private static void appendRSZnodes(ZKWatcher zkw, String znode, StringBuilder sb)
1968      throws KeeperException {
1969    List<String> stack = new LinkedList<>();
1970    stack.add(znode);
1971    do {
1972      String znodeToProcess = stack.remove(stack.size() - 1);
1973      sb.append("\n").append(znodeToProcess).append(": ");
1974      byte[] data;
1975      try {
1976        data = ZKUtil.getData(zkw, znodeToProcess);
1977      } catch (InterruptedException e) {
1978        zkw.interruptedException(e);
1979        return;
1980      }
1981      if (data != null && data.length > 0) { // log position
1982        long position = 0;
1983        try {
1984          position = ZKUtil.parseWALPositionFrom(ZKUtil.getData(zkw, znodeToProcess));
1985          sb.append(position);
1986        } catch (DeserializationException ignored) {
1987        } catch (InterruptedException e) {
1988          zkw.interruptedException(e);
1989          return;
1990        }
1991      }
1992      for (String zNodeChild : ZKUtil.listChildrenNoWatch(zkw, znodeToProcess)) {
1993        stack.add(ZNodePaths.joinZNode(znodeToProcess, zNodeChild));
1994      }
1995    } while (stack.size() > 0);
1996  }
1997
1998  private static void appendPeersZnodes(ZKWatcher zkw, String peersZnode,
1999                                        StringBuilder sb) throws KeeperException {
2000    int pblen = ProtobufUtil.lengthOfPBMagic();
2001    sb.append("\n").append(peersZnode).append(": ");
2002    for (String peerIdZnode : ZKUtil.listChildrenNoWatch(zkw, peersZnode)) {
2003      String znodeToProcess = ZNodePaths.joinZNode(peersZnode, peerIdZnode);
2004      byte[] data;
2005      try {
2006        data = ZKUtil.getData(zkw, znodeToProcess);
2007      } catch (InterruptedException e) {
2008        zkw.interruptedException(e);
2009        return;
2010      }
2011      // parse the data of the above peer znode.
2012      try {
2013        ReplicationProtos.ReplicationPeer.Builder builder =
2014          ReplicationProtos.ReplicationPeer.newBuilder();
2015        ProtobufUtil.mergeFrom(builder, data, pblen, data.length - pblen);
2016        String clusterKey = builder.getClusterkey();
2017        sb.append("\n").append(znodeToProcess).append(": ").append(clusterKey);
2018        // add the peer-state.
2019        appendPeerState(zkw, znodeToProcess, sb);
2020      } catch (IOException ipbe) {
2021        LOG.warn("Got Exception while parsing peer: " + znodeToProcess, ipbe);
2022      }
2023    }
2024  }
2025
2026  private static void appendPeerState(ZKWatcher zkw, String znodeToProcess, StringBuilder sb)
2027          throws KeeperException, InvalidProtocolBufferException {
2028    String peerState = zkw.getConfiguration().get("zookeeper.znode.replication.peers.state",
2029      "peer-state");
2030    int pblen = ProtobufUtil.lengthOfPBMagic();
2031    for (String child : ZKUtil.listChildrenNoWatch(zkw, znodeToProcess)) {
2032      if (!child.equals(peerState)) {
2033        continue;
2034      }
2035
2036      String peerStateZnode = ZNodePaths.joinZNode(znodeToProcess, child);
2037      sb.append("\n").append(peerStateZnode).append(": ");
2038      byte[] peerStateData;
2039      try {
2040        peerStateData = ZKUtil.getData(zkw, peerStateZnode);
2041        ReplicationProtos.ReplicationState.Builder builder =
2042            ReplicationProtos.ReplicationState.newBuilder();
2043        ProtobufUtil.mergeFrom(builder, peerStateData, pblen, peerStateData.length - pblen);
2044        sb.append(builder.getState().name());
2045      } catch (IOException ipbe) {
2046        LOG.warn("Got Exception while parsing peer: " + znodeToProcess, ipbe);
2047      } catch (InterruptedException e) {
2048        zkw.interruptedException(e);
2049        return;
2050      }
2051    }
2052  }
2053
2054  /**
2055   * Gets the statistics from the given server.
2056   *
2057   * @param server  The server to get the statistics from.
2058   * @param timeout  The socket timeout to use.
2059   * @return The array of response strings.
2060   * @throws IOException When the socket communication fails.
2061   */
2062  private static String[] getServerStats(String server, int timeout)
2063    throws IOException {
2064    String[] sp = server.split(":");
2065    if (sp.length == 0) {
2066      return null;
2067    }
2068
2069    String host = sp[0];
2070    int port = sp.length > 1 ? Integer.parseInt(sp[1])
2071        : HConstants.DEFAULT_ZOOKEEPER_CLIENT_PORT;
2072
2073    InetSocketAddress sockAddr = new InetSocketAddress(host, port);
2074    try (Socket socket = new Socket()) {
2075      socket.connect(sockAddr, timeout);
2076
2077      socket.setSoTimeout(timeout);
2078      try (PrintWriter out = new PrintWriter(new BufferedWriter(
2079          new OutputStreamWriter(socket.getOutputStream(), StandardCharsets.UTF_8)), true);
2080          BufferedReader in = new BufferedReader(
2081              new InputStreamReader(socket.getInputStream(), StandardCharsets.UTF_8))) {
2082        out.println("stat");
2083        out.flush();
2084        ArrayList<String> res = new ArrayList<>();
2085        while (true) {
2086          String line = in.readLine();
2087          if (line != null) {
2088            res.add(line);
2089          } else {
2090            break;
2091          }
2092        }
2093        return res.toArray(new String[res.size()]);
2094      }
2095    }
2096  }
2097
2098  private static void logRetrievedMsg(final ZKWatcher zkw,
2099      final String znode, final byte [] data, final boolean watcherSet) {
2100    if (!LOG.isTraceEnabled()) {
2101      return;
2102    }
2103
2104    LOG.trace(zkw.prefix("Retrieved " + ((data == null)? 0: data.length) +
2105      " byte(s) of data from znode " + znode +
2106      (watcherSet? " and set watcher; ": "; data=") +
2107      (data == null? "null": data.length == 0? "empty": (
2108          zkw.getZNodePaths().isMetaZNodePath(znode)?
2109            getServerNameOrEmptyString(data):
2110          znode.startsWith(zkw.getZNodePaths().backupMasterAddressesZNode)?
2111            getServerNameOrEmptyString(data):
2112          StringUtils.abbreviate(Bytes.toStringBinary(data), 32)))));
2113  }
2114
2115  private static String getServerNameOrEmptyString(final byte [] data) {
2116    try {
2117      return ProtobufUtil.parseServerNameFrom(data).toString();
2118    } catch (DeserializationException e) {
2119      return "";
2120    }
2121  }
2122
2123  /**
2124   * Waits for HBase installation's base (parent) znode to become available.
2125   * @throws IOException on ZK errors
2126   */
2127  public static void waitForBaseZNode(Configuration conf) throws IOException {
2128    LOG.info("Waiting until the base znode is available");
2129    String parentZNode = conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT,
2130        HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT);
2131    ZooKeeper zk = new ZooKeeper(ZKConfig.getZKQuorumServersString(conf),
2132        conf.getInt(HConstants.ZK_SESSION_TIMEOUT,
2133        HConstants.DEFAULT_ZK_SESSION_TIMEOUT), EmptyWatcher.instance);
2134
2135    final int maxTimeMs = 10000;
2136    final int maxNumAttempts = maxTimeMs / HConstants.SOCKET_RETRY_WAIT_MS;
2137
2138    KeeperException keeperEx = null;
2139    try {
2140      try {
2141        for (int attempt = 0; attempt < maxNumAttempts; ++attempt) {
2142          try {
2143            if (zk.exists(parentZNode, false) != null) {
2144              LOG.info("Parent znode exists: {}", parentZNode);
2145              keeperEx = null;
2146              break;
2147            }
2148          } catch (KeeperException e) {
2149            keeperEx = e;
2150          }
2151          Threads.sleepWithoutInterrupt(HConstants.SOCKET_RETRY_WAIT_MS);
2152        }
2153      } finally {
2154        zk.close();
2155      }
2156    } catch (InterruptedException ex) {
2157      Thread.currentThread().interrupt();
2158    }
2159
2160    if (keeperEx != null) {
2161      throw new IOException(keeperEx);
2162    }
2163  }
2164
2165  /**
2166   * Convert a {@link DeserializationException} to a more palatable {@link KeeperException}.
2167   * Used when can't let a {@link DeserializationException} out w/o changing public API.
2168   * @param e Exception to convert
2169   * @return Converted exception
2170   */
2171  public static KeeperException convert(final DeserializationException e) {
2172    KeeperException ke = new KeeperException.DataInconsistencyException();
2173    ke.initCause(e);
2174    return ke;
2175  }
2176
2177  /**
2178   * Recursively print the current state of ZK (non-transactional)
2179   * @param root name of the root directory in zk to print
2180   */
2181  public static void logZKTree(ZKWatcher zkw, String root) {
2182    if (!LOG.isDebugEnabled()) {
2183      return;
2184    }
2185
2186    LOG.debug("Current zk system:");
2187    String prefix = "|-";
2188    LOG.debug(prefix + root);
2189    try {
2190      logZKTree(zkw, root, prefix);
2191    } catch (KeeperException e) {
2192      throw new RuntimeException(e);
2193    }
2194  }
2195
2196  /**
2197   * Helper method to print the current state of the ZK tree.
2198   * @see #logZKTree(ZKWatcher, String)
2199   * @throws KeeperException if an unexpected exception occurs
2200   */
2201  private static void logZKTree(ZKWatcher zkw, String root, String prefix)
2202      throws KeeperException {
2203    List<String> children = ZKUtil.listChildrenNoWatch(zkw, root);
2204
2205    if (children == null) {
2206      return;
2207    }
2208
2209    for (String child : children) {
2210      LOG.debug(prefix + child);
2211      String node = ZNodePaths.joinZNode(root.equals("/") ? "" : root, child);
2212      logZKTree(zkw, node, prefix + "---");
2213    }
2214  }
2215
2216  /**
2217   * @param position the position to serialize
2218   * @return Serialized protobuf of <code>position</code> with pb magic prefix prepended suitable
2219   *         for use as content of an wal position in a replication queue.
2220   */
2221  public static byte[] positionToByteArray(final long position) {
2222    byte[] bytes = ReplicationProtos.ReplicationHLogPosition.newBuilder().setPosition(position)
2223        .build().toByteArray();
2224    return ProtobufUtil.prependPBMagic(bytes);
2225  }
2226
2227  /**
2228   * @param bytes - Content of a WAL position znode.
2229   * @return long - The current WAL position.
2230   * @throws DeserializationException if the WAL position cannot be parsed
2231   */
2232  public static long parseWALPositionFrom(final byte[] bytes) throws DeserializationException {
2233    if (bytes == null) {
2234      throw new DeserializationException("Unable to parse null WAL position.");
2235    }
2236    if (ProtobufUtil.isPBMagicPrefix(bytes)) {
2237      int pblen = ProtobufUtil.lengthOfPBMagic();
2238      ReplicationProtos.ReplicationHLogPosition.Builder builder =
2239          ReplicationProtos.ReplicationHLogPosition.newBuilder();
2240      ReplicationProtos.ReplicationHLogPosition position;
2241      try {
2242        ProtobufUtil.mergeFrom(builder, bytes, pblen, bytes.length - pblen);
2243        position = builder.build();
2244      } catch (IOException e) {
2245        throw new DeserializationException(e);
2246      }
2247      return position.getPosition();
2248    } else {
2249      if (bytes.length > 0) {
2250        return Bytes.toLong(bytes);
2251      }
2252      return 0;
2253    }
2254  }
2255}