View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.zookeeper;
20  
21  import java.io.Closeable;
22  import java.io.IOException;
23  import java.util.ArrayList;
24  import java.util.HashMap;
25  import java.util.List;
26  import java.util.Map;
27  import java.util.concurrent.CopyOnWriteArrayList;
28  import java.util.concurrent.CountDownLatch;
29  import java.util.regex.Matcher;
30  import java.util.regex.Pattern;
31  
32  import org.apache.commons.logging.Log;
33  import org.apache.commons.logging.LogFactory;
34  import org.apache.hadoop.conf.Configuration;
35  import org.apache.hadoop.hbase.Abortable;
36  import org.apache.hadoop.hbase.AuthUtil;
37  import org.apache.hadoop.hbase.HConstants;
38  import org.apache.hadoop.hbase.HRegionInfo;
39  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
40  import org.apache.hadoop.hbase.classification.InterfaceAudience;
41  import org.apache.hadoop.hbase.security.Superusers;
42  import org.apache.hadoop.security.UserGroupInformation;
43  import org.apache.zookeeper.KeeperException;
44  import org.apache.zookeeper.WatchedEvent;
45  import org.apache.zookeeper.Watcher;
46  import org.apache.zookeeper.ZooDefs.Ids;
47  import org.apache.zookeeper.ZooDefs.Perms;
48  import org.apache.zookeeper.data.ACL;
49  import org.apache.zookeeper.data.Id;
50  import org.apache.zookeeper.data.Stat;
51  
52  /**
53   * Acts as the single ZooKeeper Watcher.  One instance of this is instantiated
54   * for each Master, RegionServer, and client process.
55   *
56   * <p>This is the only class that implements {@link Watcher}.  Other internal
57   * classes which need to be notified of ZooKeeper events must register with
58   * the local instance of this watcher via {@link #registerListener}.
59   *
60   * <p>This class also holds and manages the connection to ZooKeeper.  Code to
61   * deal with connection related events and exceptions are handled here.
62   */
63  @InterfaceAudience.Private
64  public class ZooKeeperWatcher implements Watcher, Abortable, Closeable {
65    private static final Log LOG = LogFactory.getLog(ZooKeeperWatcher.class);
66  
67    // Identifier for this watcher (for logging only).  It is made of the prefix
68    // passed on construction and the zookeeper sessionid.
69    private String prefix;
70    private String identifier;
71  
72    // zookeeper quorum
73    private String quorum;
74  
75    // zookeeper connection
76    private RecoverableZooKeeper recoverableZooKeeper;
77  
78    // abortable in case of zk failure
79    protected Abortable abortable;
80    // Used if abortable is null
81    private boolean aborted = false;
82  
83    // listeners to be notified
84    private final List<ZooKeeperListener> listeners =
85      new CopyOnWriteArrayList<ZooKeeperListener>();
86  
87    // Used by ZKUtil:waitForZKConnectionIfAuthenticating to wait for SASL
88    // negotiation to complete
89    public CountDownLatch saslLatch = new CountDownLatch(1);
90  
91    // node names
92  
93    // base znode for this cluster
94    public String baseZNode;
95    //znodes containing the locations of the servers hosting the meta replicas
96    private Map<Integer,String> metaReplicaZnodes = new HashMap<Integer, String>();
97    // znode containing ephemeral nodes of the regionservers
98    public String rsZNode;
99    // znode containing ephemeral nodes of the draining regionservers
100   public String drainingZNode;
101   // znode of currently active master
102   private String masterAddressZNode;
103   // znode of this master in backup master directory, if not the active master
104   public String backupMasterAddressesZNode;
105   // znode containing the current cluster state
106   public String clusterStateZNode;
107   // znode used for table disabling/enabling
108   @Deprecated
109   public String tableZNode;
110   // znode containing the unique cluster ID
111   public String clusterIdZNode;
112   // znode used for log splitting work assignment
113   public String splitLogZNode;
114   // znode containing the state of the load balancer
115   public String balancerZNode;
116   // znode containing the state of region normalizer
117   private String regionNormalizerZNode;
118   // znode containing the state of all switches, currently there are split and merge child node.
119   private String switchZNode;
120   // znode containing the lock for the switches
121   private String switchLockZNode;
122   // znode containing the lock for the tables
123   public String tableLockZNode;
124   // znode containing the state of recovering regions
125   public String recoveringRegionsZNode;
126   // znode containing namespace descriptors
127   public static String namespaceZNode = "namespace";
128 
129   public final static String META_ZNODE_PREFIX = "meta-region-server";
130 
131   private final Configuration conf;
132 
133   private final Exception constructorCaller;
134 
135   /* A pattern that matches a Kerberos name, borrowed from Hadoop's KerberosName */
136   private static final Pattern NAME_PATTERN = Pattern.compile("([^/@]*)(/([^/@]*))?@([^/@]*)");
137 
138   /**
139    * Instantiate a ZooKeeper connection and watcher.
140    * @param identifier string that is passed to RecoverableZookeeper to be used as
141    * identifier for this instance. Use null for default.
142    * @throws IOException
143    * @throws ZooKeeperConnectionException
144    */
145   public ZooKeeperWatcher(Configuration conf, String identifier,
146       Abortable abortable) throws ZooKeeperConnectionException, IOException {
147     this(conf, identifier, abortable, false);
148   }
149 
150   /**
151    * Instantiate a ZooKeeper connection and watcher.
152    * @param conf
153    * @param identifier string that is passed to RecoverableZookeeper to be used as identifier for
154    *          this instance. Use null for default.
155    * @param abortable Can be null if there is on error there is no host to abort: e.g. client
156    *          context.
157    * @param canCreateBaseZNode
158    * @throws IOException
159    * @throws ZooKeeperConnectionException
160    */
161   public ZooKeeperWatcher(Configuration conf, String identifier,
162       Abortable abortable, boolean canCreateBaseZNode)
163   throws IOException, ZooKeeperConnectionException {
164     this.conf = conf;
165     // Capture a stack trace now.  Will print it out later if problem so we can
166     // distingush amongst the myriad ZKWs.
167     try {
168       throw new Exception("ZKW CONSTRUCTOR STACK TRACE FOR DEBUGGING");
169     } catch (Exception e) {
170       this.constructorCaller = e;
171     }
172     this.quorum = ZKConfig.getZKQuorumServersString(conf);
173     this.prefix = identifier;
174     // Identifier will get the sessionid appended later below down when we
175     // handle the syncconnect event.
176     this.identifier = identifier + "0x0";
177     this.abortable = abortable;
178     setNodeNames(conf);
179     this.recoverableZooKeeper = ZKUtil.connect(conf, quorum, this, identifier);
180     if (canCreateBaseZNode) {
181       createBaseZNodes();
182     }
183   }
184 
185   private void createBaseZNodes() throws ZooKeeperConnectionException {
186     try {
187       // Create all the necessary "directories" of znodes
188       ZKUtil.createWithParents(this, baseZNode);
189       ZKUtil.createAndFailSilent(this, rsZNode);
190       ZKUtil.createAndFailSilent(this, drainingZNode);
191       ZKUtil.createAndFailSilent(this, tableZNode);
192       ZKUtil.createAndFailSilent(this, splitLogZNode);
193       ZKUtil.createAndFailSilent(this, backupMasterAddressesZNode);
194       ZKUtil.createAndFailSilent(this, tableLockZNode);
195       ZKUtil.createAndFailSilent(this, recoveringRegionsZNode);
196     } catch (KeeperException e) {
197       throw new ZooKeeperConnectionException(
198           prefix("Unexpected KeeperException creating base node"), e);
199     }
200   }
201 
202   /** Returns whether the znode is supposed to be readable by the client
203    * and DOES NOT contain sensitive information (world readable).*/
204   public boolean isClientReadable(String node) {
205     // Developer notice: These znodes are world readable. DO NOT add more znodes here UNLESS
206     // all clients need to access this data to work. Using zk for sharing data to clients (other
207     // than service lookup case is not a recommended design pattern.
208     return
209         node.equals(baseZNode) ||
210         isAnyMetaReplicaZnode(node) ||
211         node.equals(getMasterAddressZNode()) ||
212         node.equals(clusterIdZNode)||
213         node.equals(rsZNode) ||
214         // /hbase/table and /hbase/table/foo is allowed, /hbase/table-lock is not
215         node.equals(tableZNode) ||
216         node.startsWith(tableZNode + "/");
217   }
218 
219   /**
220    * On master start, we check the znode ACLs under the root directory and set the ACLs properly
221    * if needed. If the cluster goes from an unsecure setup to a secure setup, this step is needed
222    * so that the existing znodes created with open permissions are now changed with restrictive
223    * perms.
224    */
225   public void checkAndSetZNodeAcls() {
226     if (!ZKUtil.isSecureZooKeeper(getConfiguration())) {
227       LOG.info("not a secure deployment, proceeding");
228       return;
229     }
230 
231     // Check the base znodes permission first. Only do the recursion if base znode's perms are not
232     // correct.
233     try {
234       List<ACL> actualAcls = recoverableZooKeeper.getAcl(baseZNode, new Stat());
235 
236       if (!isBaseZnodeAclSetup(actualAcls)) {
237         LOG.info("setting znode ACLs");
238         setZnodeAclsRecursive(baseZNode);
239       }
240     } catch(KeeperException.NoNodeException nne) {
241       return;
242     } catch(InterruptedException ie) {
243       interruptedException(ie);
244     } catch (IOException|KeeperException e) {
245       LOG.warn("Received exception while checking and setting zookeeper ACLs", e);
246     }
247   }
248 
249   /**
250    * Set the znode perms recursively. This will do post-order recursion, so that baseZnode ACLs
251    * will be set last in case the master fails in between.
252    * @param znode
253    */
254   private void setZnodeAclsRecursive(String znode) throws KeeperException, InterruptedException {
255     List<String> children = recoverableZooKeeper.getChildren(znode, false);
256 
257     for (String child : children) {
258       setZnodeAclsRecursive(ZKUtil.joinZNode(znode, child));
259     }
260     List<ACL> acls = ZKUtil.createACL(this, znode, true);
261     LOG.info("Setting ACLs for znode:" + znode + " , acl:" + acls);
262     recoverableZooKeeper.setAcl(znode, acls, -1);
263   }
264 
265   /**
266    * Checks whether the ACLs returned from the base znode (/hbase) is set for secure setup.
267    * @param acls acls from zookeeper
268    * @return whether ACLs are set for the base znode
269    * @throws IOException
270    */
271   private boolean isBaseZnodeAclSetup(List<ACL> acls) throws IOException {
272     if (LOG.isDebugEnabled()) {
273       LOG.debug("Checking znode ACLs");
274     }
275     String[] superUsers = conf.getStrings(Superusers.SUPERUSER_CONF_KEY);
276     // Check whether ACL set for all superusers
277     if (superUsers != null && !checkACLForSuperUsers(superUsers, acls)) {
278       return false;
279     }
280 
281     // this assumes that current authenticated user is the same as zookeeper client user
282     // configured via JAAS
283     String hbaseUser = UserGroupInformation.getCurrentUser().getShortUserName();
284 
285     if (acls.isEmpty()) {
286       if (LOG.isDebugEnabled()) {
287         LOG.debug("ACL is empty");
288       }
289       return false;
290     }
291 
292     for (ACL acl : acls) {
293       int perms = acl.getPerms();
294       Id id = acl.getId();
295       // We should only set at most 3 possible ACLs for 3 Ids. One for everyone, one for superuser
296       // and one for the hbase user
297       if (Ids.ANYONE_ID_UNSAFE.equals(id)) {
298         if (perms != Perms.READ) {
299           if (LOG.isDebugEnabled()) {
300             LOG.debug(String.format("permissions for '%s' are not correct: have 0x%x, want 0x%x",
301               id, perms, Perms.READ));
302           }
303           return false;
304         }
305       } else if (superUsers != null && isSuperUserId(superUsers, id)) {
306         if (perms != Perms.ALL) {
307           if (LOG.isDebugEnabled()) {
308             LOG.debug(String.format("permissions for '%s' are not correct: have 0x%x, want 0x%x",
309               id, perms, Perms.ALL));
310           }
311           return false;
312         }
313       } else if ("sasl".equals(id.getScheme())) {
314         String name = id.getId();
315         // If ZooKeeper recorded the Kerberos full name in the ACL, use only the shortname
316         Matcher match = NAME_PATTERN.matcher(name);
317         if (match.matches()) {
318           name = match.group(1);
319         }
320         if (name.equals(hbaseUser)) {
321           if (perms != Perms.ALL) {
322             if (LOG.isDebugEnabled()) {
323               LOG.debug(String.format("permissions for '%s' are not correct: have 0x%x, want 0x%x",
324                 id, perms, Perms.ALL));
325             }
326             return false;
327           }
328         } else {
329           if (LOG.isDebugEnabled()) {
330             LOG.debug("Unexpected shortname in SASL ACL: " + id);
331           }
332           return false;
333         }
334       } else {
335         if (LOG.isDebugEnabled()) {
336           LOG.debug("unexpected ACL id '" + id + "'");
337         }
338         return false;
339       }
340     }
341     return true;
342   }
343 
344   /*
345    * Validate whether ACL set for all superusers.
346    */
347   private boolean checkACLForSuperUsers(String[] superUsers, List<ACL> acls) {
348     for (String user : superUsers) {
349       boolean hasAccess = false;
350       // TODO: Validate super group members also when ZK supports setting node ACL for groups.
351       if (!user.startsWith(AuthUtil.GROUP_PREFIX)) {
352         for (ACL acl : acls) {
353           if (user.equals(acl.getId().getId())) {
354             if (acl.getPerms() == Perms.ALL) {
355               hasAccess = true;
356             } else {
357               if (LOG.isDebugEnabled()) {
358                 LOG.debug(String.format(
359                   "superuser '%s' does not have correct permissions: have 0x%x, want 0x%x",
360                   acl.getId().getId(), acl.getPerms(), Perms.ALL));
361               }
362             }
363             break;
364           }
365         }
366         if (!hasAccess) {
367           return false;
368         }
369       }
370     }
371     return true;
372   }
373 
374   /*
375    * Validate whether ACL ID is superuser.
376    */
377   public static boolean isSuperUserId(String[] superUsers, Id id) {
378     for (String user : superUsers) {
379       // TODO: Validate super group members also when ZK supports setting node ACL for groups.
380       if (!user.startsWith(AuthUtil.GROUP_PREFIX) && new Id("sasl", user).equals(id)) {
381         return true;
382       }
383     }
384     return false;
385   }
386 
387   @Override
388   public String toString() {
389     return this.identifier + ", quorum=" + quorum + ", baseZNode=" + baseZNode;
390   }
391 
392   /**
393    * Adds this instance's identifier as a prefix to the passed <code>str</code>
394    * @param str String to amend.
395    * @return A new string with this instance's identifier as prefix: e.g.
396    * if passed 'hello world', the returned string could be
397    */
398   public String prefix(final String str) {
399     return this.toString() + " " + str;
400   }
401 
402   /**
403    * Set the local variable node names using the specified configuration.
404    */
405   private void setNodeNames(Configuration conf) {
406     baseZNode = conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT,
407         HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT);
408     metaReplicaZnodes.put(0, ZKUtil.joinZNode(baseZNode,
409            conf.get("zookeeper.znode.metaserver", "meta-region-server")));
410     int numMetaReplicas = conf.getInt(HConstants.META_REPLICAS_NUM,
411             HConstants.DEFAULT_META_REPLICA_NUM);
412     for (int i = 1; i < numMetaReplicas; i++) {
413       String str = ZKUtil.joinZNode(baseZNode,
414         conf.get("zookeeper.znode.metaserver", "meta-region-server") + "-" + i);
415       metaReplicaZnodes.put(i, str);
416     }
417     rsZNode = ZKUtil.joinZNode(baseZNode,
418         conf.get("zookeeper.znode.rs", "rs"));
419     drainingZNode = ZKUtil.joinZNode(baseZNode,
420         conf.get("zookeeper.znode.draining.rs", "draining"));
421     masterAddressZNode = ZKUtil.joinZNode(baseZNode,
422         conf.get("zookeeper.znode.master", "master"));
423     backupMasterAddressesZNode = ZKUtil.joinZNode(baseZNode,
424         conf.get("zookeeper.znode.backup.masters", "backup-masters"));
425     clusterStateZNode = ZKUtil.joinZNode(baseZNode,
426         conf.get("zookeeper.znode.state", "running"));
427     tableZNode = ZKUtil.joinZNode(baseZNode,
428         conf.get("zookeeper.znode.tableEnableDisable", "table"));
429     clusterIdZNode = ZKUtil.joinZNode(baseZNode,
430         conf.get("zookeeper.znode.clusterId", "hbaseid"));
431     splitLogZNode = ZKUtil.joinZNode(baseZNode,
432         conf.get("zookeeper.znode.splitlog", HConstants.SPLIT_LOGDIR_NAME));
433     balancerZNode = ZKUtil.joinZNode(baseZNode,
434         conf.get("zookeeper.znode.balancer", "balancer"));
435     regionNormalizerZNode = ZKUtil.joinZNode(baseZNode,
436       conf.get("zookeeper.znode.regionNormalizer", "normalizer"));
437     switchZNode = ZKUtil.joinZNode(baseZNode, conf.get("zookeeper.znode.switch", "switch"));
438     switchLockZNode = ZKUtil.joinZNode(switchZNode, "locks");
439     tableLockZNode = ZKUtil.joinZNode(baseZNode,
440         conf.get("zookeeper.znode.tableLock", "table-lock"));
441     recoveringRegionsZNode = ZKUtil.joinZNode(baseZNode,
442         conf.get("zookeeper.znode.recovering.regions", "recovering-regions"));
443     namespaceZNode = ZKUtil.joinZNode(baseZNode,
444         conf.get("zookeeper.znode.namespace", "namespace"));
445   }
446 
447   /**
448    * Is the znode of any meta replica
449    * @param node
450    * @return true or false
451    */
452   public boolean isAnyMetaReplicaZnode(String node) {
453     if (metaReplicaZnodes.values().contains(node)) {
454       return true;
455     }
456     return false;
457   }
458 
459   /**
460    * Is it the default meta replica's znode
461    * @param node
462    * @return true or false
463    */
464   public boolean isDefaultMetaReplicaZnode(String node) {
465     if (getZNodeForReplica(HRegionInfo.DEFAULT_REPLICA_ID).equals(node)) {
466       return true;
467     }
468     return false;
469   }
470 
471   /**
472    * Get the znodes corresponding to the meta replicas from ZK
473    * @return list of znodes
474    * @throws KeeperException
475    */
476   public List<String> getMetaReplicaNodes() throws KeeperException {
477     List<String> childrenOfBaseNode = ZKUtil.listChildrenNoWatch(this, baseZNode);
478     List<String> metaReplicaNodes = new ArrayList<String>(2);
479     String pattern = conf.get("zookeeper.znode.metaserver","meta-region-server");
480     for (String child : childrenOfBaseNode) {
481       if (child.startsWith(pattern)) metaReplicaNodes.add(child);
482     }
483     return metaReplicaNodes;
484   }
485 
486   /**
487    * Get the znode string corresponding to a replicaId
488    * @param replicaId
489    * @return znode
490    */
491   public String getZNodeForReplica(int replicaId) {
492     String str = metaReplicaZnodes.get(replicaId);
493     // return a newly created path but don't update the cache of paths
494     // This is mostly needed for tests that attempt to create meta replicas
495     // from outside the master
496     if (str == null) {
497       str = ZKUtil.joinZNode(baseZNode,
498           conf.get("zookeeper.znode.metaserver", "meta-region-server") + "-" + replicaId);
499     }
500     return str;
501   }
502 
503   /**
504    * Parse the meta replicaId from the passed znode
505    * @param znode
506    * @return replicaId
507    */
508   public int getMetaReplicaIdFromZnode(String znode) {
509     String pattern = conf.get("zookeeper.znode.metaserver","meta-region-server");
510     if (znode.equals(pattern)) return HRegionInfo.DEFAULT_REPLICA_ID;
511     // the non-default replicas are of the pattern meta-region-server-<replicaId>
512     String nonDefaultPattern = pattern + "-";
513     return Integer.parseInt(znode.substring(nonDefaultPattern.length()));
514   }
515 
516   /**
517    * Register the specified listener to receive ZooKeeper events.
518    * @param listener
519    */
520   public void registerListener(ZooKeeperListener listener) {
521     listeners.add(listener);
522   }
523 
524   /**
525    * Register the specified listener to receive ZooKeeper events and add it as
526    * the first in the list of current listeners.
527    * @param listener
528    */
529   public void registerListenerFirst(ZooKeeperListener listener) {
530     listeners.add(0, listener);
531   }
532 
533   public void unregisterListener(ZooKeeperListener listener) {
534     listeners.remove(listener);
535   }
536 
537   /**
538    * Clean all existing listeners
539    */
540   public void unregisterAllListeners() {
541     listeners.clear();
542   }
543 
544   /**
545    * Get a copy of current registered listeners
546    */
547   public List<ZooKeeperListener> getListeners() {
548     return new ArrayList<ZooKeeperListener>(listeners);
549   }
550 
551   /**
552    * @return The number of currently registered listeners
553    */
554   public int getNumberOfListeners() {
555     return listeners.size();
556   }
557 
558   /**
559    * Get the connection to ZooKeeper.
560    * @return connection reference to zookeeper
561    */
562   public RecoverableZooKeeper getRecoverableZooKeeper() {
563     return recoverableZooKeeper;
564   }
565 
566   public void reconnectAfterExpiration() throws IOException, KeeperException, InterruptedException {
567     recoverableZooKeeper.reconnectAfterExpiration();
568   }
569 
570   /**
571    * Get the quorum address of this instance.
572    * @return quorum string of this zookeeper connection instance
573    */
574   public String getQuorum() {
575     return quorum;
576   }
577 
578   /**
579    * @return the base znode of this zookeeper connection instance.
580    */
581   public String getBaseZNode() {
582     return baseZNode;
583   }
584 
585   /**
586    * Method called from ZooKeeper for events and connection status.
587    * <p>
588    * Valid events are passed along to listeners.  Connection status changes
589    * are dealt with locally.
590    */
591   @Override
592   public void process(WatchedEvent event) {
593     LOG.debug(prefix("Received ZooKeeper Event, " +
594         "type=" + event.getType() + ", " +
595         "state=" + event.getState() + ", " +
596         "path=" + event.getPath()));
597 
598     switch(event.getType()) {
599 
600       // If event type is NONE, this is a connection status change
601       case None: {
602         connectionEvent(event);
603         break;
604       }
605 
606       // Otherwise pass along to the listeners
607 
608       case NodeCreated: {
609         for(ZooKeeperListener listener : listeners) {
610           listener.nodeCreated(event.getPath());
611         }
612         break;
613       }
614 
615       case NodeDeleted: {
616         for(ZooKeeperListener listener : listeners) {
617           listener.nodeDeleted(event.getPath());
618         }
619         break;
620       }
621 
622       case NodeDataChanged: {
623         for(ZooKeeperListener listener : listeners) {
624           listener.nodeDataChanged(event.getPath());
625         }
626         break;
627       }
628 
629       case NodeChildrenChanged: {
630         for(ZooKeeperListener listener : listeners) {
631           listener.nodeChildrenChanged(event.getPath());
632         }
633         break;
634       }
635     }
636   }
637 
638   // Connection management
639 
640   /**
641    * Called when there is a connection-related event via the Watcher callback.
642    * <p>
643    * If Disconnected or Expired, this should shutdown the cluster. But, since
644    * we send a KeeperException.SessionExpiredException along with the abort
645    * call, it's possible for the Abortable to catch it and try to create a new
646    * session with ZooKeeper. This is what the client does in HCM.
647    * <p>
648    * @param event
649    */
650   private void connectionEvent(WatchedEvent event) {
651     switch(event.getState()) {
652       case SyncConnected:
653         // Now, this callback can be invoked before the this.zookeeper is set.
654         // Wait a little while.
655         long finished = System.currentTimeMillis() +
656           this.conf.getLong("hbase.zookeeper.watcher.sync.connected.wait", 2000);
657         while (System.currentTimeMillis() < finished) {
658           try {
659             Thread.sleep(1);
660           } catch (InterruptedException e) {
661             LOG.warn("Interrupted while sleeping");
662             throw new RuntimeException("Interrupted while waiting for" +
663                 " recoverableZooKeeper is set");
664           }
665           if (this.recoverableZooKeeper != null) break;
666         }
667 
668         if (this.recoverableZooKeeper == null) {
669           LOG.error("ZK is null on connection event -- see stack trace " +
670             "for the stack trace when constructor was called on this zkw",
671             this.constructorCaller);
672           throw new NullPointerException("ZK is null");
673         }
674         this.identifier = this.prefix + "-0x" +
675           Long.toHexString(this.recoverableZooKeeper.getSessionId());
676         // Update our identifier.  Otherwise ignore.
677         LOG.debug(this.identifier + " connected");
678         break;
679 
680       // Abort the server if Disconnected or Expired
681       case Disconnected:
682         LOG.debug(prefix("Received Disconnected from ZooKeeper, ignoring"));
683         break;
684 
685       case Expired:
686         String msg = prefix(this.identifier + " received expired from " +
687           "ZooKeeper, aborting");
688         // TODO: One thought is to add call to ZooKeeperListener so say,
689         // ZooKeeperNodeTracker can zero out its data values.
690         if (this.abortable != null) {
691           this.abortable.abort(msg, new KeeperException.SessionExpiredException());
692         }
693         break;
694 
695       case ConnectedReadOnly:
696       case SaslAuthenticated:
697       case AuthFailed:
698         break;
699 
700       default:
701         throw new IllegalStateException("Received event is not valid: " + event.getState());
702     }
703   }
704 
705   /**
706    * Forces a synchronization of this ZooKeeper client connection.
707    * <p>
708    * Executing this method before running other methods will ensure that the
709    * subsequent operations are up-to-date and consistent as of the time that
710    * the sync is complete.
711    * <p>
712    * This is used for compareAndSwap type operations where we need to read the
713    * data of an existing node and delete or transition that node, utilizing the
714    * previously read version and data.  We want to ensure that the version read
715    * is up-to-date from when we begin the operation.
716    */
717   public void sync(String path) throws KeeperException {
718     this.recoverableZooKeeper.sync(path, null, null);
719   }
720 
721   /**
722    * Handles KeeperExceptions in client calls.
723    * <p>
724    * This may be temporary but for now this gives one place to deal with these.
725    * <p>
726    * TODO: Currently this method rethrows the exception to let the caller handle
727    * <p>
728    * @param ke
729    * @throws KeeperException
730    */
731   public void keeperException(KeeperException ke)
732   throws KeeperException {
733     LOG.error(prefix("Received unexpected KeeperException, re-throwing exception"), ke);
734     throw ke;
735   }
736 
737   /**
738    * Handles InterruptedExceptions in client calls.
739    * <p>
740    * This may be temporary but for now this gives one place to deal with these.
741    * <p>
742    * TODO: Currently, this method does nothing.
743    *       Is this ever expected to happen?  Do we abort or can we let it run?
744    *       Maybe this should be logged as WARN?  It shouldn't happen?
745    * <p>
746    * @param ie
747    */
748   public void interruptedException(InterruptedException ie) {
749     LOG.debug(prefix("Received InterruptedException, doing nothing here"), ie);
750     // At least preserver interrupt.
751     Thread.currentThread().interrupt();
752     // no-op
753   }
754 
755   /**
756    * Close the connection to ZooKeeper.
757    *
758    */
759   @Override
760   public void close() {
761     try {
762       if (recoverableZooKeeper != null) {
763         recoverableZooKeeper.close();
764       }
765     } catch (InterruptedException e) {
766       Thread.currentThread().interrupt();
767     }
768   }
769 
770   public Configuration getConfiguration() {
771     return conf;
772   }
773 
774   @Override
775   public void abort(String why, Throwable e) {
776     if (this.abortable != null) this.abortable.abort(why, e);
777     else this.aborted = true;
778   }
779 
780   @Override
781   public boolean isAborted() {
782     return this.abortable == null? this.aborted: this.abortable.isAborted();
783   }
784 
785   /**
786    * @return Path to the currently active master.
787    */
788   public String getMasterAddressZNode() {
789     return this.masterAddressZNode;
790   }
791 
792   /**
793    * @return ZooKeeper znode for region normalizer state
794    */
795   public String getRegionNormalizerZNode() {
796     return regionNormalizerZNode;
797   }
798 
799   /**
800    *  @return ZK node for switch
801    * */
802   public String getSwitchZNode() {
803     return switchZNode;
804   }
805 
806   /**
807    *  @return ZK node for switchLock node.
808    * */
809   public String getSwitchLockZNode() {
810     return switchLockZNode;
811   }
812 }