001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNotEquals;
022import static org.junit.Assert.assertNotNull;
023import java.io.IOException;
024import java.util.List;
025import java.util.concurrent.TimeUnit;
026import org.apache.hadoop.conf.Configuration;
027import org.apache.hadoop.hbase.ClusterMetrics;
028import org.apache.hadoop.hbase.HBaseClassTestRule;
029import org.apache.hadoop.hbase.HBaseConfiguration;
030import org.apache.hadoop.hbase.HBaseTestingUtility;
031import org.apache.hadoop.hbase.LocalHBaseCluster;
032import org.apache.hadoop.hbase.MiniHBaseCluster;
033import org.apache.hadoop.hbase.StartMiniClusterOption;
034import org.apache.hadoop.hbase.Waiter;
035import org.apache.hadoop.hbase.testclassification.LargeTests;
036import org.apache.hadoop.hbase.testclassification.MasterTests;
037import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
038import org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient;
039import org.junit.Before;
040import org.junit.ClassRule;
041import org.junit.Test;
042import org.junit.experimental.categories.Category;
043import org.slf4j.Logger;
044import org.slf4j.LoggerFactory;
045import org.apache.hbase.thirdparty.org.apache.commons.collections4.CollectionUtils;
046
047@Category({MasterTests.class, LargeTests.class})
048public class TestMasterShutdown {
049  private static final Logger LOG = LoggerFactory.getLogger(TestMasterShutdown.class);
050
051  @ClassRule
052  public static final HBaseClassTestRule CLASS_RULE =
053      HBaseClassTestRule.forClass(TestMasterShutdown.class);
054
055  private HBaseTestingUtility htu;
056
057  @Before
058  public void shutdownCluster() throws IOException {
059    if (htu != null) {
060      // an extra check in case the test cluster was not terminated after HBaseClassTestRule's
061      // Timeout interrupted the test thread.
062      LOG.warn("found non-null TestingUtility -- previous test did not terminate cleanly.");
063      htu.shutdownMiniCluster();
064    }
065  }
066
067  /**
068   * Simple test of shutdown.
069   * <p>
070   * Starts with three masters.  Tells the active master to shutdown the cluster.
071   * Verifies that all masters are properly shutdown.
072   */
073  @Test
074  public void testMasterShutdown() throws Exception {
075    // Create config to use for this cluster
076    Configuration conf = HBaseConfiguration.create();
077
078    // Start the cluster
079    try {
080      htu = new HBaseTestingUtility(conf);
081      StartMiniClusterOption option = StartMiniClusterOption.builder()
082        .numMasters(3)
083        .numRegionServers(1)
084        .numDataNodes(1)
085        .build();
086      final MiniHBaseCluster cluster = htu.startMiniCluster(option);
087
088      // wait for all master thread to spawn and start their run loop.
089      final long thirtySeconds = TimeUnit.SECONDS.toMillis(30);
090      final long oneSecond = TimeUnit.SECONDS.toMillis(1);
091      assertNotEquals(-1, htu.waitFor(thirtySeconds, oneSecond, () -> {
092        final List<MasterThread> masterThreads = cluster.getMasterThreads();
093        return masterThreads != null
094          && masterThreads.size() >= 3
095          && masterThreads.stream().allMatch(Thread::isAlive);
096      }));
097
098      // find the active master
099      final HMaster active = cluster.getMaster();
100      assertNotNull(active);
101
102      // make sure the other two are backup masters
103      ClusterMetrics status = active.getClusterMetrics();
104      assertEquals(2, status.getBackupMasterNames().size());
105
106      // tell the active master to shutdown the cluster
107      active.shutdown();
108      assertNotEquals(-1, htu.waitFor(thirtySeconds, oneSecond,
109        () -> CollectionUtils.isEmpty(cluster.getLiveMasterThreads())));
110      assertNotEquals(-1, htu.waitFor(thirtySeconds, oneSecond,
111        () -> CollectionUtils.isEmpty(cluster.getLiveRegionServerThreads())));
112    } finally {
113      if (htu != null) {
114        htu.shutdownMiniCluster();
115        htu = null;
116      }
117    }
118  }
119
120  /**
121   * This test appears to be an intentional race between a thread that issues a shutdown RPC to the
122   * master, while the master is concurrently realizing it cannot initialize because there are no
123   * region servers available to it. The expected behavior is that master initialization is
124   * interruptable via said shutdown RPC.
125   */
126  @Test
127  public void testMasterShutdownBeforeStartingAnyRegionServer() throws Exception {
128    LocalHBaseCluster hbaseCluster = null;
129    try {
130      htu =  new HBaseTestingUtility(
131        createMasterShutdownBeforeStartingAnyRegionServerConfiguration());
132
133      // configure a cluster with
134      final StartMiniClusterOption options = StartMiniClusterOption.builder()
135        .numDataNodes(1)
136        .numMasters(1)
137        .numRegionServers(0)
138        .masterClass(HMaster.class)
139        .rsClass(MiniHBaseCluster.MiniHBaseClusterRegionServer.class)
140        .createRootDir(true)
141        .build();
142
143      // Can't simply `htu.startMiniCluster(options)` because that method waits for the master to
144      // start completely. However, this test's premise is that a partially started master should
145      // still respond to a shutdown RPC. So instead, we manage each component lifecycle
146      // independently.
147      // I think it's not worth refactoring HTU's helper methods just for this class.
148      htu.startMiniDFSCluster(options.getNumDataNodes());
149      htu.startMiniZKCluster(options.getNumZkServers());
150      htu.createRootDir();
151      hbaseCluster = new LocalHBaseCluster(htu.getConfiguration(), options.getNumMasters(),
152        options.getNumRegionServers(), options.getMasterClass(), options.getRsClass());
153      final MasterThread masterThread = hbaseCluster.getMasters().get(0);
154      masterThread.start();
155      // Switching to master registry exacerbated a race in the master bootstrap that can result
156      // in a lost shutdown command (HBASE-8422, HBASE-23836). The race is essentially because
157      // the server manager in HMaster is not initialized by the time shutdown() RPC (below) is
158      // made to the master. The suspected reason as to why it was uncommon before HBASE-18095
159      // is because the connection creation with ZK registry is so slow that by then the server
160      // manager is usually init'ed in time for the RPC to be made. For now, adding an explicit
161      // wait() in the test, waiting for the server manager to become available.
162      final long timeout = TimeUnit.MINUTES.toMillis(10);
163      assertNotEquals("Timeout waiting for server manager to become available.",
164        -1, Waiter.waitFor(htu.getConfiguration(), timeout,
165          () -> masterThread.getMaster().getServerManager() != null));
166      htu.getConnection().getAdmin().shutdown();
167      masterThread.join();
168    } finally {
169      if (hbaseCluster != null) {
170        hbaseCluster.shutdown();
171      }
172      if (htu != null) {
173        htu.shutdownMiniCluster();
174        htu = null;
175      }
176    }
177  }
178
179  /**
180   * Create a cluster configuration suitable for
181   * {@link #testMasterShutdownBeforeStartingAnyRegionServer()}.
182   */
183  private static Configuration createMasterShutdownBeforeStartingAnyRegionServerConfiguration() {
184    final Configuration conf = HBaseConfiguration.create();
185    // make sure the master will wait forever in the absence of a RS.
186    conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 1);
187    // don't need a long write pipeline for this test.
188    conf.setInt("dfs.replication", 1);
189    return conf;
190  }
191
192  /**
193   * Create a new {@link Configuration} based on {@code baseConf} that has ZooKeeper connection
194   * settings tuned very aggressively. The resulting client is used within a retry loop, so there's
195   * no value in having the client itself do the retries. We want to iterate on the base
196   * configuration because we're waiting for the mini-cluster to start and set it's ZK client port.
197   *
198   * @return a new, configured {@link Configuration} instance.
199   */
200  private static Configuration createResponsiveZkConfig(final Configuration baseConf) {
201    final Configuration conf = HBaseConfiguration.create(baseConf);
202    conf.setInt(ReadOnlyZKClient.RECOVERY_RETRY, 3);
203    conf.setInt(ReadOnlyZKClient.RECOVERY_RETRY_INTERVAL_MILLIS, 100);
204    return conf;
205  }
206}