001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNotEquals;
022import static org.junit.Assert.assertNotNull;
023
024import java.io.IOException;
025import java.util.List;
026import java.util.concurrent.TimeUnit;
027import org.apache.hadoop.conf.Configuration;
028import org.apache.hadoop.hbase.ClusterMetrics;
029import org.apache.hadoop.hbase.HBaseClassTestRule;
030import org.apache.hadoop.hbase.HBaseConfiguration;
031import org.apache.hadoop.hbase.HBaseTestingUtility;
032import org.apache.hadoop.hbase.LocalHBaseCluster;
033import org.apache.hadoop.hbase.MiniHBaseCluster;
034import org.apache.hadoop.hbase.StartMiniClusterOption;
035import org.apache.hadoop.hbase.Waiter;
036import org.apache.hadoop.hbase.testclassification.LargeTests;
037import org.apache.hadoop.hbase.testclassification.MasterTests;
038import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
039import org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient;
040import org.junit.Before;
041import org.junit.ClassRule;
042import org.junit.Test;
043import org.junit.experimental.categories.Category;
044import org.slf4j.Logger;
045import org.slf4j.LoggerFactory;
046
047import org.apache.hbase.thirdparty.org.apache.commons.collections4.CollectionUtils;
048
049@Category({ MasterTests.class, LargeTests.class })
050public class TestMasterShutdown {
051  private static final Logger LOG = LoggerFactory.getLogger(TestMasterShutdown.class);
052
053  @ClassRule
054  public static final HBaseClassTestRule CLASS_RULE =
055    HBaseClassTestRule.forClass(TestMasterShutdown.class);
056
057  private HBaseTestingUtility htu;
058
059  @Before
060  public void shutdownCluster() throws IOException {
061    if (htu != null) {
062      // an extra check in case the test cluster was not terminated after HBaseClassTestRule's
063      // Timeout interrupted the test thread.
064      LOG.warn("found non-null TestingUtility -- previous test did not terminate cleanly.");
065      htu.shutdownMiniCluster();
066    }
067  }
068
069  /**
070   * Simple test of shutdown.
071   * <p>
072   * Starts with three masters. Tells the active master to shutdown the cluster. Verifies that all
073   * masters are properly shutdown.
074   */
075  @Test
076  public void testMasterShutdown() throws Exception {
077    // Create config to use for this cluster
078    Configuration conf = HBaseConfiguration.create();
079
080    // Start the cluster
081    try {
082      htu = new HBaseTestingUtility(conf);
083      StartMiniClusterOption option =
084        StartMiniClusterOption.builder().numMasters(3).numRegionServers(1).numDataNodes(1).build();
085      final MiniHBaseCluster cluster = htu.startMiniCluster(option);
086
087      // wait for all master thread to spawn and start their run loop.
088      final long thirtySeconds = TimeUnit.SECONDS.toMillis(30);
089      final long oneSecond = TimeUnit.SECONDS.toMillis(1);
090      assertNotEquals(-1, htu.waitFor(thirtySeconds, oneSecond, () -> {
091        final List<MasterThread> masterThreads = cluster.getMasterThreads();
092        return masterThreads != null && masterThreads.size() >= 3
093          && masterThreads.stream().allMatch(Thread::isAlive);
094      }));
095
096      // find the active master
097      final HMaster active = cluster.getMaster();
098      assertNotNull(active);
099
100      // make sure the other two are backup masters
101      ClusterMetrics status = active.getClusterMetrics();
102      assertEquals(2, status.getBackupMasterNames().size());
103
104      // tell the active master to shutdown the cluster
105      active.shutdown();
106      assertNotEquals(-1, htu.waitFor(thirtySeconds, oneSecond,
107        () -> CollectionUtils.isEmpty(cluster.getLiveMasterThreads())));
108      assertNotEquals(-1, htu.waitFor(thirtySeconds, oneSecond,
109        () -> CollectionUtils.isEmpty(cluster.getLiveRegionServerThreads())));
110    } finally {
111      if (htu != null) {
112        htu.shutdownMiniCluster();
113        htu = null;
114      }
115    }
116  }
117
118  /**
119   * This test appears to be an intentional race between a thread that issues a shutdown RPC to the
120   * master, while the master is concurrently realizing it cannot initialize because there are no
121   * region servers available to it. The expected behavior is that master initialization is
122   * interruptable via said shutdown RPC.
123   */
124  @Test
125  public void testMasterShutdownBeforeStartingAnyRegionServer() throws Exception {
126    LocalHBaseCluster hbaseCluster = null;
127    try {
128      htu =
129        new HBaseTestingUtility(createMasterShutdownBeforeStartingAnyRegionServerConfiguration());
130
131      // configure a cluster with
132      final StartMiniClusterOption options = StartMiniClusterOption.builder().numDataNodes(1)
133        .numMasters(1).numRegionServers(0).masterClass(HMaster.class)
134        .rsClass(MiniHBaseCluster.MiniHBaseClusterRegionServer.class).createRootDir(true).build();
135
136      // Can't simply `htu.startMiniCluster(options)` because that method waits for the master to
137      // start completely. However, this test's premise is that a partially started master should
138      // still respond to a shutdown RPC. So instead, we manage each component lifecycle
139      // independently.
140      // I think it's not worth refactoring HTU's helper methods just for this class.
141      htu.startMiniDFSCluster(options.getNumDataNodes());
142      htu.startMiniZKCluster(options.getNumZkServers());
143      htu.createRootDir();
144      hbaseCluster = new LocalHBaseCluster(htu.getConfiguration(), options.getNumMasters(),
145        options.getNumRegionServers(), options.getMasterClass(), options.getRsClass());
146      final MasterThread masterThread = hbaseCluster.getMasters().get(0);
147      masterThread.start();
148      // Switching to master registry exacerbated a race in the master bootstrap that can result
149      // in a lost shutdown command (HBASE-8422, HBASE-23836). The race is essentially because
150      // the server manager in HMaster is not initialized by the time shutdown() RPC (below) is
151      // made to the master. The suspected reason as to why it was uncommon before HBASE-18095
152      // is because the connection creation with ZK registry is so slow that by then the server
153      // manager is usually init'ed in time for the RPC to be made. For now, adding an explicit
154      // wait() in the test, waiting for the server manager to become available.
155      final long timeout = TimeUnit.MINUTES.toMillis(10);
156      assertNotEquals("Timeout waiting for server manager to become available.", -1,
157        Waiter.waitFor(htu.getConfiguration(), timeout,
158          () -> masterThread.getMaster().getServerManager() != null));
159      htu.getConnection().getAdmin().shutdown();
160      masterThread.join();
161    } finally {
162      if (hbaseCluster != null) {
163        hbaseCluster.shutdown();
164      }
165      if (htu != null) {
166        htu.shutdownMiniCluster();
167        htu = null;
168      }
169    }
170  }
171
172  /**
173   * Create a cluster configuration suitable for
174   * {@link #testMasterShutdownBeforeStartingAnyRegionServer()}.
175   */
176  private static Configuration createMasterShutdownBeforeStartingAnyRegionServerConfiguration() {
177    final Configuration conf = HBaseConfiguration.create();
178    // make sure the master will wait forever in the absence of a RS.
179    conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 1);
180    // don't need a long write pipeline for this test.
181    conf.setInt("dfs.replication", 1);
182    return conf;
183  }
184
185  /**
186   * Create a new {@link Configuration} based on {@code baseConf} that has ZooKeeper connection
187   * settings tuned very aggressively. The resulting client is used within a retry loop, so there's
188   * no value in having the client itself do the retries. We want to iterate on the base
189   * configuration because we're waiting for the mini-cluster to start and set it's ZK client port.
190   * @return a new, configured {@link Configuration} instance.
191   */
192  private static Configuration createResponsiveZkConfig(final Configuration baseConf) {
193    final Configuration conf = HBaseConfiguration.create(baseConf);
194    conf.setInt(ReadOnlyZKClient.RECOVERY_RETRY, 3);
195    conf.setInt(ReadOnlyZKClient.RECOVERY_RETRY_INTERVAL_MILLIS, 100);
196    return conf;
197  }
198}