001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertNotNull;
023import static org.junit.Assert.assertTrue;
024
025import java.io.IOException;
026import java.io.InterruptedIOException;
027import java.util.ArrayList;
028import java.util.List;
029import java.util.concurrent.Semaphore;
030import org.apache.hadoop.conf.Configuration;
031import org.apache.hadoop.hbase.HBaseClassTestRule;
032import org.apache.hadoop.hbase.HBaseTestingUtil;
033import org.apache.hadoop.hbase.ServerName;
034import org.apache.hadoop.hbase.monitoring.MonitoredTask;
035import org.apache.hadoop.hbase.testclassification.MasterTests;
036import org.apache.hadoop.hbase.testclassification.MediumTests;
037import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
038import org.apache.hadoop.hbase.util.MockServer;
039import org.apache.hadoop.hbase.zookeeper.ClusterStatusTracker;
040import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker;
041import org.apache.hadoop.hbase.zookeeper.ZKListener;
042import org.apache.hadoop.hbase.zookeeper.ZKUtil;
043import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
044import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
045import org.apache.zookeeper.KeeperException;
046import org.junit.AfterClass;
047import org.junit.BeforeClass;
048import org.junit.ClassRule;
049import org.junit.Test;
050import org.junit.experimental.categories.Category;
051import org.mockito.Mockito;
052import org.slf4j.Logger;
053import org.slf4j.LoggerFactory;
054
055/**
056 * Test the {@link ActiveMasterManager}.
057 */
058@Category({ MasterTests.class, MediumTests.class })
059public class TestActiveMasterManager {
060
061  @ClassRule
062  public static final HBaseClassTestRule CLASS_RULE =
063    HBaseClassTestRule.forClass(TestActiveMasterManager.class);
064
065  private final static Logger LOG = LoggerFactory.getLogger(TestActiveMasterManager.class);
066  private final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
067
068  @BeforeClass
069  public static void setUpBeforeClass() throws Exception {
070    TEST_UTIL.startMiniZKCluster();
071  }
072
073  @AfterClass
074  public static void tearDownAfterClass() throws Exception {
075    TEST_UTIL.shutdownMiniZKCluster();
076  }
077
078  @Test
079  public void testRestartMaster() throws IOException, KeeperException {
080    try (ZKWatcher zk =
081      new ZKWatcher(TEST_UTIL.getConfiguration(), "testActiveMasterManagerFromZK", null, true)) {
082      try {
083        ZKUtil.deleteNode(zk, zk.getZNodePaths().masterAddressZNode);
084        ZKUtil.deleteNode(zk, zk.getZNodePaths().clusterStateZNode);
085      } catch (KeeperException.NoNodeException nne) {
086      }
087
088      // Create the master node with a dummy address
089      ServerName master = ServerName.valueOf("localhost", 1, EnvironmentEdgeManager.currentTime());
090      // Should not have a master yet
091      DummyMaster dummyMaster = new DummyMaster(zk, master);
092      ClusterStatusTracker clusterStatusTracker = dummyMaster.getClusterStatusTracker();
093      ActiveMasterManager activeMasterManager = dummyMaster.getActiveMasterManager();
094      assertFalse(activeMasterManager.clusterHasActiveMaster.get());
095      assertFalse(activeMasterManager.getActiveMasterServerName().isPresent());
096
097      // First test becoming the active master uninterrupted
098      MonitoredTask status = Mockito.mock(MonitoredTask.class);
099      clusterStatusTracker.setClusterUp();
100
101      activeMasterManager.blockUntilBecomingActiveMaster(100, status);
102      assertTrue(activeMasterManager.clusterHasActiveMaster.get());
103      assertMaster(zk, master);
104      assertMaster(zk, activeMasterManager.getActiveMasterServerName().get());
105
106      // Now pretend master restart
107      DummyMaster secondDummyMaster = new DummyMaster(zk, master);
108      ActiveMasterManager secondActiveMasterManager = secondDummyMaster.getActiveMasterManager();
109      assertFalse(secondActiveMasterManager.clusterHasActiveMaster.get());
110      activeMasterManager.blockUntilBecomingActiveMaster(100, status);
111      assertTrue(activeMasterManager.clusterHasActiveMaster.get());
112      assertMaster(zk, master);
113      assertMaster(zk, activeMasterManager.getActiveMasterServerName().get());
114      assertMaster(zk, secondActiveMasterManager.getActiveMasterServerName().get());
115    }
116  }
117
118  /**
119   * Unit tests that uses ZooKeeper but does not use the master-side methods but rather acts
120   * directly on ZK. n
121   */
122  @Test
123  public void testActiveMasterManagerFromZK() throws Exception {
124    try (ZKWatcher zk =
125      new ZKWatcher(TEST_UTIL.getConfiguration(), "testActiveMasterManagerFromZK", null, true)) {
126      try {
127        ZKUtil.deleteNode(zk, zk.getZNodePaths().masterAddressZNode);
128        ZKUtil.deleteNode(zk, zk.getZNodePaths().clusterStateZNode);
129      } catch (KeeperException.NoNodeException nne) {
130      }
131
132      // Create the master node with a dummy address
133      ServerName firstMasterAddress =
134        ServerName.valueOf("localhost", 1, EnvironmentEdgeManager.currentTime());
135      ServerName secondMasterAddress =
136        ServerName.valueOf("localhost", 2, EnvironmentEdgeManager.currentTime());
137
138      // Should not have a master yet
139      DummyMaster ms1 = new DummyMaster(zk, firstMasterAddress);
140      ActiveMasterManager activeMasterManager = ms1.getActiveMasterManager();
141      assertFalse(activeMasterManager.clusterHasActiveMaster.get());
142      assertFalse(activeMasterManager.getActiveMasterServerName().isPresent());
143
144      // First test becoming the active master uninterrupted
145      ClusterStatusTracker clusterStatusTracker = ms1.getClusterStatusTracker();
146      clusterStatusTracker.setClusterUp();
147      activeMasterManager.blockUntilBecomingActiveMaster(100, Mockito.mock(MonitoredTask.class));
148      assertTrue(activeMasterManager.clusterHasActiveMaster.get());
149      assertMaster(zk, firstMasterAddress);
150      assertMaster(zk, activeMasterManager.getActiveMasterServerName().get());
151
152      // New manager will now try to become the active master in another thread
153      WaitToBeMasterThread t = new WaitToBeMasterThread(zk, secondMasterAddress);
154      t.start();
155      // Wait for this guy to figure out there is another active master
156      // Wait for 1 second at most
157      int sleeps = 0;
158      while (!t.manager.clusterHasActiveMaster.get() && sleeps < 100) {
159        Thread.sleep(10);
160        sleeps++;
161      }
162
163      // Both should see that there is an active master
164      assertTrue(activeMasterManager.clusterHasActiveMaster.get());
165      assertTrue(t.manager.clusterHasActiveMaster.get());
166      // But secondary one should not be the active master
167      assertFalse(t.isActiveMaster);
168      // Verify the active master ServerName is populated in standby master.
169      assertEquals(firstMasterAddress, t.manager.getActiveMasterServerName().get());
170
171      // Close the first server and delete it's master node
172      ms1.stop("stopping first server");
173
174      // Use a listener to capture when the node is actually deleted
175      NodeDeletionListener listener =
176        new NodeDeletionListener(zk, zk.getZNodePaths().masterAddressZNode);
177      zk.registerListener(listener);
178
179      LOG.info("Deleting master node");
180      ZKUtil.deleteNode(zk, zk.getZNodePaths().masterAddressZNode);
181
182      // Wait for the node to be deleted
183      LOG.info("Waiting for active master manager to be notified");
184      listener.waitForDeletion();
185      LOG.info("Master node deleted");
186
187      // Now we expect the secondary manager to have and be the active master
188      // Wait for 1 second at most
189      sleeps = 0;
190      while (!t.isActiveMaster && sleeps < 100) {
191        Thread.sleep(10);
192        sleeps++;
193      }
194      LOG.debug("Slept " + sleeps + " times");
195
196      assertTrue(t.manager.clusterHasActiveMaster.get());
197      assertTrue(t.isActiveMaster);
198      assertEquals(secondMasterAddress, t.manager.getActiveMasterServerName().get());
199
200      LOG.info("Deleting master node");
201
202      ZKUtil.deleteNode(zk, zk.getZNodePaths().masterAddressZNode);
203    }
204  }
205
206  @Test
207  public void testBackupMasterUpdates() throws Exception {
208    Configuration conf = TEST_UTIL.getConfiguration();
209    try (ZKWatcher zk = new ZKWatcher(conf, "testBackupMasterUpdates", null, true)) {
210      ServerName sn1 = ServerName.valueOf("localhost", 1, -1);
211      DummyMaster master1 = new DummyMaster(zk, sn1);
212      ActiveMasterManager activeMasterManager = master1.getActiveMasterManager();
213      activeMasterManager.blockUntilBecomingActiveMaster(100, Mockito.mock(MonitoredTask.class));
214      assertEquals(sn1, activeMasterManager.getActiveMasterServerName().get());
215      assertEquals(0, activeMasterManager.getBackupMasters().size());
216      // Add backup masters
217      List<String> backupZNodes = new ArrayList<>();
218      for (int i = 1; i <= 10; i++) {
219        ServerName backupSn = ServerName.valueOf("localhost", 1000 + i, -1);
220        String backupZn =
221          ZNodePaths.joinZNode(zk.getZNodePaths().backupMasterAddressesZNode, backupSn.toString());
222        backupZNodes.add(backupZn);
223        MasterAddressTracker.setMasterAddress(zk, backupZn, backupSn, 1234);
224        TEST_UTIL.waitFor(10000,
225          () -> activeMasterManager.getBackupMasters().size() == backupZNodes.size());
226      }
227      // Remove backup masters
228      int numBackups = backupZNodes.size();
229      for (String backupZNode : backupZNodes) {
230        ZKUtil.deleteNode(zk, backupZNode);
231        final int currentBackups = --numBackups;
232        TEST_UTIL.waitFor(10000,
233          () -> activeMasterManager.getBackupMasters().size() == currentBackups);
234      }
235    }
236  }
237
238  /**
239   * Assert there is an active master and that it has the specified address.
240   * @param zk              single Zookeeper watcher
241   * @param expectedAddress the expected address of the master
242   * @throws KeeperException unexpected Zookeeper exception
243   * @throws IOException     if an IO problem is encountered
244   */
245  private void assertMaster(ZKWatcher zk, ServerName expectedAddress)
246    throws KeeperException, IOException {
247    ServerName readAddress = MasterAddressTracker.getMasterAddress(zk);
248    assertNotNull(readAddress);
249    assertEquals(expectedAddress, readAddress);
250  }
251
252  public static class WaitToBeMasterThread extends Thread {
253
254    ActiveMasterManager manager;
255    DummyMaster dummyMaster;
256    boolean isActiveMaster;
257
258    public WaitToBeMasterThread(ZKWatcher zk, ServerName address) throws InterruptedIOException {
259      this.dummyMaster = new DummyMaster(zk, address);
260      this.manager = this.dummyMaster.getActiveMasterManager();
261      isActiveMaster = false;
262    }
263
264    @Override
265    public void run() {
266      manager.blockUntilBecomingActiveMaster(100, Mockito.mock(MonitoredTask.class));
267      LOG.info("Second master has become the active master!");
268      isActiveMaster = true;
269    }
270  }
271
272  public static class NodeDeletionListener extends ZKListener {
273    private static final Logger LOG = LoggerFactory.getLogger(NodeDeletionListener.class);
274
275    private Semaphore lock;
276    private String node;
277
278    public NodeDeletionListener(ZKWatcher watcher, String node) {
279      super(watcher);
280      lock = new Semaphore(0);
281      this.node = node;
282    }
283
284    @Override
285    public void nodeDeleted(String path) {
286      if (path.equals(node)) {
287        LOG.debug("nodeDeleted(" + path + ")");
288        lock.release();
289      }
290    }
291
292    public void waitForDeletion() throws InterruptedException {
293      lock.acquire();
294    }
295  }
296
297  /**
298   * Dummy Master Implementation.
299   */
300  public static class DummyMaster extends MockServer {
301    private ClusterStatusTracker clusterStatusTracker;
302    private ActiveMasterManager activeMasterManager;
303
304    public DummyMaster(ZKWatcher zk, ServerName master) throws InterruptedIOException {
305      this.clusterStatusTracker = new ClusterStatusTracker(zk, this);
306      clusterStatusTracker.start();
307
308      this.activeMasterManager = new ActiveMasterManager(zk, master, this);
309      zk.registerListener(activeMasterManager);
310    }
311
312    public ClusterStatusTracker getClusterStatusTracker() {
313      return clusterStatusTracker;
314    }
315
316    public ActiveMasterManager getActiveMasterManager() {
317      return activeMasterManager;
318    }
319  }
320}