001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.master; 019 020import static org.junit.Assert.assertFalse; 021import static org.junit.Assert.assertNotNull; 022import static org.junit.Assert.assertTrue; 023 024import java.io.IOException; 025import java.util.concurrent.Semaphore; 026import org.apache.hadoop.conf.Configuration; 027import org.apache.hadoop.fs.FileSystem; 028import org.apache.hadoop.hbase.ChoreService; 029import org.apache.hadoop.hbase.CoordinatedStateManager; 030import org.apache.hadoop.hbase.HBaseClassTestRule; 031import org.apache.hadoop.hbase.HBaseTestingUtility; 032import org.apache.hadoop.hbase.Server; 033import org.apache.hadoop.hbase.ServerName; 034import org.apache.hadoop.hbase.client.ClusterConnection; 035import org.apache.hadoop.hbase.client.Connection; 036import org.apache.hadoop.hbase.monitoring.MonitoredTask; 037import org.apache.hadoop.hbase.testclassification.MasterTests; 038import org.apache.hadoop.hbase.testclassification.MediumTests; 039import org.apache.hadoop.hbase.zookeeper.ClusterStatusTracker; 040import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker; 041import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; 042import org.apache.hadoop.hbase.zookeeper.ZKListener; 043import org.apache.hadoop.hbase.zookeeper.ZKUtil; 044import org.apache.hadoop.hbase.zookeeper.ZKWatcher; 045import org.apache.zookeeper.KeeperException; 046import org.junit.AfterClass; 047import org.junit.BeforeClass; 048import org.junit.ClassRule; 049import org.junit.Test; 050import org.junit.experimental.categories.Category; 051import org.mockito.Mockito; 052import org.slf4j.Logger; 053import org.slf4j.LoggerFactory; 054 055/** 056 * Test the {@link ActiveMasterManager}. 057 */ 058@Category({MasterTests.class, MediumTests.class}) 059public class TestActiveMasterManager { 060 061 @ClassRule 062 public static final HBaseClassTestRule CLASS_RULE = 063 HBaseClassTestRule.forClass(TestActiveMasterManager.class); 064 065 private final static Logger LOG = LoggerFactory.getLogger(TestActiveMasterManager.class); 066 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); 067 068 @BeforeClass 069 public static void setUpBeforeClass() throws Exception { 070 TEST_UTIL.startMiniZKCluster(); 071 } 072 073 @AfterClass 074 public static void tearDownAfterClass() throws Exception { 075 TEST_UTIL.shutdownMiniZKCluster(); 076 } 077 078 @Test public void testRestartMaster() throws IOException, KeeperException { 079 ZKWatcher zk = new ZKWatcher(TEST_UTIL.getConfiguration(), 080 "testActiveMasterManagerFromZK", null, true); 081 try { 082 ZKUtil.deleteNode(zk, zk.znodePaths.masterAddressZNode); 083 ZKUtil.deleteNode(zk, zk.znodePaths.clusterStateZNode); 084 } catch(KeeperException.NoNodeException nne) {} 085 086 // Create the master node with a dummy address 087 ServerName master = ServerName.valueOf("localhost", 1, System.currentTimeMillis()); 088 // Should not have a master yet 089 DummyMaster dummyMaster = new DummyMaster(zk,master); 090 ClusterStatusTracker clusterStatusTracker = 091 dummyMaster.getClusterStatusTracker(); 092 ActiveMasterManager activeMasterManager = 093 dummyMaster.getActiveMasterManager(); 094 assertFalse(activeMasterManager.clusterHasActiveMaster.get()); 095 096 // First test becoming the active master uninterrupted 097 MonitoredTask status = Mockito.mock(MonitoredTask.class); 098 clusterStatusTracker.setClusterUp(); 099 100 activeMasterManager.blockUntilBecomingActiveMaster(100, status); 101 assertTrue(activeMasterManager.clusterHasActiveMaster.get()); 102 assertMaster(zk, master); 103 104 // Now pretend master restart 105 DummyMaster secondDummyMaster = new DummyMaster(zk,master); 106 ActiveMasterManager secondActiveMasterManager = 107 secondDummyMaster.getActiveMasterManager(); 108 assertFalse(secondActiveMasterManager.clusterHasActiveMaster.get()); 109 activeMasterManager.blockUntilBecomingActiveMaster(100, status); 110 assertTrue(activeMasterManager.clusterHasActiveMaster.get()); 111 assertMaster(zk, master); 112 } 113 114 /** 115 * Unit tests that uses ZooKeeper but does not use the master-side methods 116 * but rather acts directly on ZK. 117 * @throws Exception 118 */ 119 @Test 120 public void testActiveMasterManagerFromZK() throws Exception { 121 ZKWatcher zk = new ZKWatcher(TEST_UTIL.getConfiguration(), 122 "testActiveMasterManagerFromZK", null, true); 123 try { 124 ZKUtil.deleteNode(zk, zk.znodePaths.masterAddressZNode); 125 ZKUtil.deleteNode(zk, zk.znodePaths.clusterStateZNode); 126 } catch(KeeperException.NoNodeException nne) {} 127 128 // Create the master node with a dummy address 129 ServerName firstMasterAddress = 130 ServerName.valueOf("localhost", 1, System.currentTimeMillis()); 131 ServerName secondMasterAddress = 132 ServerName.valueOf("localhost", 2, System.currentTimeMillis()); 133 134 // Should not have a master yet 135 DummyMaster ms1 = new DummyMaster(zk,firstMasterAddress); 136 ActiveMasterManager activeMasterManager = 137 ms1.getActiveMasterManager(); 138 assertFalse(activeMasterManager.clusterHasActiveMaster.get()); 139 140 // First test becoming the active master uninterrupted 141 ClusterStatusTracker clusterStatusTracker = 142 ms1.getClusterStatusTracker(); 143 clusterStatusTracker.setClusterUp(); 144 activeMasterManager.blockUntilBecomingActiveMaster(100, 145 Mockito.mock(MonitoredTask.class)); 146 assertTrue(activeMasterManager.clusterHasActiveMaster.get()); 147 assertMaster(zk, firstMasterAddress); 148 149 // New manager will now try to become the active master in another thread 150 WaitToBeMasterThread t = new WaitToBeMasterThread(zk, secondMasterAddress); 151 t.start(); 152 // Wait for this guy to figure out there is another active master 153 // Wait for 1 second at most 154 int sleeps = 0; 155 while(!t.manager.clusterHasActiveMaster.get() && sleeps < 100) { 156 Thread.sleep(10); 157 sleeps++; 158 } 159 160 // Both should see that there is an active master 161 assertTrue(activeMasterManager.clusterHasActiveMaster.get()); 162 assertTrue(t.manager.clusterHasActiveMaster.get()); 163 // But secondary one should not be the active master 164 assertFalse(t.isActiveMaster); 165 166 // Close the first server and delete it's master node 167 ms1.stop("stopping first server"); 168 169 // Use a listener to capture when the node is actually deleted 170 NodeDeletionListener listener = new NodeDeletionListener(zk, zk.znodePaths.masterAddressZNode); 171 zk.registerListener(listener); 172 173 LOG.info("Deleting master node"); 174 ZKUtil.deleteNode(zk, zk.znodePaths.masterAddressZNode); 175 176 // Wait for the node to be deleted 177 LOG.info("Waiting for active master manager to be notified"); 178 listener.waitForDeletion(); 179 LOG.info("Master node deleted"); 180 181 // Now we expect the secondary manager to have and be the active master 182 // Wait for 1 second at most 183 sleeps = 0; 184 while(!t.isActiveMaster && sleeps < 100) { 185 Thread.sleep(10); 186 sleeps++; 187 } 188 LOG.debug("Slept " + sleeps + " times"); 189 190 assertTrue(t.manager.clusterHasActiveMaster.get()); 191 assertTrue(t.isActiveMaster); 192 193 LOG.info("Deleting master node"); 194 195 ZKUtil.deleteNode(zk, zk.znodePaths.masterAddressZNode); 196 } 197 198 /** 199 * Assert there is an active master and that it has the specified address. 200 * @param zk single Zookeeper watcher 201 * @param expectedAddress the expected address of the master 202 * @throws KeeperException unexpected Zookeeper exception 203 * @throws IOException if an IO problem is encountered 204 */ 205 private void assertMaster(ZKWatcher zk, 206 ServerName expectedAddress) 207 throws KeeperException, IOException { 208 ServerName readAddress = MasterAddressTracker.getMasterAddress(zk); 209 assertNotNull(readAddress); 210 assertTrue(expectedAddress.equals(readAddress)); 211 } 212 213 public static class WaitToBeMasterThread extends Thread { 214 215 ActiveMasterManager manager; 216 DummyMaster dummyMaster; 217 boolean isActiveMaster; 218 219 public WaitToBeMasterThread(ZKWatcher zk, ServerName address) { 220 this.dummyMaster = new DummyMaster(zk,address); 221 this.manager = this.dummyMaster.getActiveMasterManager(); 222 isActiveMaster = false; 223 } 224 225 @Override 226 public void run() { 227 manager.blockUntilBecomingActiveMaster(100, 228 Mockito.mock(MonitoredTask.class)); 229 LOG.info("Second master has become the active master!"); 230 isActiveMaster = true; 231 } 232 } 233 234 public static class NodeDeletionListener extends ZKListener { 235 private static final Logger LOG = LoggerFactory.getLogger(NodeDeletionListener.class); 236 237 private Semaphore lock; 238 private String node; 239 240 public NodeDeletionListener(ZKWatcher watcher, String node) { 241 super(watcher); 242 lock = new Semaphore(0); 243 this.node = node; 244 } 245 246 @Override 247 public void nodeDeleted(String path) { 248 if(path.equals(node)) { 249 LOG.debug("nodeDeleted(" + path + ")"); 250 lock.release(); 251 } 252 } 253 254 public void waitForDeletion() throws InterruptedException { 255 lock.acquire(); 256 } 257 } 258 259 /** 260 * Dummy Master Implementation. 261 */ 262 public static class DummyMaster implements Server { 263 private volatile boolean stopped; 264 private ClusterStatusTracker clusterStatusTracker; 265 private ActiveMasterManager activeMasterManager; 266 267 public DummyMaster(ZKWatcher zk, ServerName master) { 268 this.clusterStatusTracker = 269 new ClusterStatusTracker(zk, this); 270 clusterStatusTracker.start(); 271 272 this.activeMasterManager = 273 new ActiveMasterManager(zk, master, this); 274 zk.registerListener(activeMasterManager); 275 } 276 277 @Override 278 public void abort(final String msg, final Throwable t) {} 279 280 @Override 281 public boolean isAborted() { 282 return false; 283 } 284 285 @Override 286 public Configuration getConfiguration() { 287 return null; 288 } 289 290 @Override 291 public ZKWatcher getZooKeeper() { 292 return null; 293 } 294 295 @Override 296 public CoordinatedStateManager getCoordinatedStateManager() { 297 return null; 298 } 299 300 @Override 301 public ServerName getServerName() { 302 return null; 303 } 304 305 @Override 306 public boolean isStopped() { 307 return this.stopped; 308 } 309 310 @Override 311 public void stop(String why) { 312 this.stopped = true; 313 } 314 315 @Override 316 public ClusterConnection getConnection() { 317 return null; 318 } 319 320 @Override 321 public MetaTableLocator getMetaTableLocator() { 322 return null; 323 } 324 325 public ClusterStatusTracker getClusterStatusTracker() { 326 return clusterStatusTracker; 327 } 328 329 public ActiveMasterManager getActiveMasterManager() { 330 return activeMasterManager; 331 } 332 333 @Override 334 public ChoreService getChoreService() { 335 return null; 336 } 337 338 @Override 339 public ClusterConnection getClusterConnection() { 340 // TODO Auto-generated method stub 341 return null; 342 } 343 344 @Override 345 public FileSystem getFileSystem() { 346 return null; 347 } 348 349 @Override 350 public boolean isStopping() { 351 return false; 352 } 353 354 @Override 355 public Connection createConnection(Configuration conf) throws IOException { 356 return null; 357 } 358 } 359}