001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.master; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertNotEquals; 022import static org.junit.Assert.assertNotNull; 023import java.io.IOException; 024import java.util.List; 025import java.util.concurrent.TimeUnit; 026import org.apache.hadoop.conf.Configuration; 027import org.apache.hadoop.hbase.ClusterMetrics; 028import org.apache.hadoop.hbase.HBaseClassTestRule; 029import org.apache.hadoop.hbase.HBaseConfiguration; 030import org.apache.hadoop.hbase.HBaseTestingUtility; 031import org.apache.hadoop.hbase.LocalHBaseCluster; 032import org.apache.hadoop.hbase.MiniHBaseCluster; 033import org.apache.hadoop.hbase.StartMiniClusterOption; 034import org.apache.hadoop.hbase.Waiter; 035import org.apache.hadoop.hbase.testclassification.LargeTests; 036import org.apache.hadoop.hbase.testclassification.MasterTests; 037import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread; 038import org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient; 039import org.junit.Before; 040import org.junit.ClassRule; 041import org.junit.Test; 042import org.junit.experimental.categories.Category; 043import org.slf4j.Logger; 044import org.slf4j.LoggerFactory; 045import org.apache.hbase.thirdparty.org.apache.commons.collections4.CollectionUtils; 046 047@Category({MasterTests.class, LargeTests.class}) 048public class TestMasterShutdown { 049 private static final Logger LOG = LoggerFactory.getLogger(TestMasterShutdown.class); 050 051 @ClassRule 052 public static final HBaseClassTestRule CLASS_RULE = 053 HBaseClassTestRule.forClass(TestMasterShutdown.class); 054 055 private HBaseTestingUtility htu; 056 057 @Before 058 public void shutdownCluster() throws IOException { 059 if (htu != null) { 060 // an extra check in case the test cluster was not terminated after HBaseClassTestRule's 061 // Timeout interrupted the test thread. 062 LOG.warn("found non-null TestingUtility -- previous test did not terminate cleanly."); 063 htu.shutdownMiniCluster(); 064 } 065 } 066 067 /** 068 * Simple test of shutdown. 069 * <p> 070 * Starts with three masters. Tells the active master to shutdown the cluster. 071 * Verifies that all masters are properly shutdown. 072 */ 073 @Test 074 public void testMasterShutdown() throws Exception { 075 // Create config to use for this cluster 076 Configuration conf = HBaseConfiguration.create(); 077 078 // Start the cluster 079 try { 080 htu = new HBaseTestingUtility(conf); 081 StartMiniClusterOption option = StartMiniClusterOption.builder() 082 .numMasters(3) 083 .numRegionServers(1) 084 .numDataNodes(1) 085 .build(); 086 final MiniHBaseCluster cluster = htu.startMiniCluster(option); 087 088 // wait for all master thread to spawn and start their run loop. 089 final long thirtySeconds = TimeUnit.SECONDS.toMillis(30); 090 final long oneSecond = TimeUnit.SECONDS.toMillis(1); 091 assertNotEquals(-1, htu.waitFor(thirtySeconds, oneSecond, () -> { 092 final List<MasterThread> masterThreads = cluster.getMasterThreads(); 093 return masterThreads != null 094 && masterThreads.size() >= 3 095 && masterThreads.stream().allMatch(Thread::isAlive); 096 })); 097 098 // find the active master 099 final HMaster active = cluster.getMaster(); 100 assertNotNull(active); 101 102 // make sure the other two are backup masters 103 ClusterMetrics status = active.getClusterMetrics(); 104 assertEquals(2, status.getBackupMasterNames().size()); 105 106 // tell the active master to shutdown the cluster 107 active.shutdown(); 108 assertNotEquals(-1, htu.waitFor(thirtySeconds, oneSecond, 109 () -> CollectionUtils.isEmpty(cluster.getLiveMasterThreads()))); 110 assertNotEquals(-1, htu.waitFor(thirtySeconds, oneSecond, 111 () -> CollectionUtils.isEmpty(cluster.getLiveRegionServerThreads()))); 112 } finally { 113 if (htu != null) { 114 htu.shutdownMiniCluster(); 115 htu = null; 116 } 117 } 118 } 119 120 /** 121 * This test appears to be an intentional race between a thread that issues a shutdown RPC to the 122 * master, while the master is concurrently realizing it cannot initialize because there are no 123 * region servers available to it. The expected behavior is that master initialization is 124 * interruptable via said shutdown RPC. 125 */ 126 @Test 127 public void testMasterShutdownBeforeStartingAnyRegionServer() throws Exception { 128 LocalHBaseCluster hbaseCluster = null; 129 try { 130 htu = new HBaseTestingUtility( 131 createMasterShutdownBeforeStartingAnyRegionServerConfiguration()); 132 133 // configure a cluster with 134 final StartMiniClusterOption options = StartMiniClusterOption.builder() 135 .numDataNodes(1) 136 .numMasters(1) 137 .numRegionServers(0) 138 .masterClass(HMaster.class) 139 .rsClass(MiniHBaseCluster.MiniHBaseClusterRegionServer.class) 140 .createRootDir(true) 141 .build(); 142 143 // Can't simply `htu.startMiniCluster(options)` because that method waits for the master to 144 // start completely. However, this test's premise is that a partially started master should 145 // still respond to a shutdown RPC. So instead, we manage each component lifecycle 146 // independently. 147 // I think it's not worth refactoring HTU's helper methods just for this class. 148 htu.startMiniDFSCluster(options.getNumDataNodes()); 149 htu.startMiniZKCluster(options.getNumZkServers()); 150 htu.createRootDir(); 151 hbaseCluster = new LocalHBaseCluster(htu.getConfiguration(), options.getNumMasters(), 152 options.getNumRegionServers(), options.getMasterClass(), options.getRsClass()); 153 final MasterThread masterThread = hbaseCluster.getMasters().get(0); 154 masterThread.start(); 155 // Switching to master registry exacerbated a race in the master bootstrap that can result 156 // in a lost shutdown command (HBASE-8422, HBASE-23836). The race is essentially because 157 // the server manager in HMaster is not initialized by the time shutdown() RPC (below) is 158 // made to the master. The suspected reason as to why it was uncommon before HBASE-18095 159 // is because the connection creation with ZK registry is so slow that by then the server 160 // manager is usually init'ed in time for the RPC to be made. For now, adding an explicit 161 // wait() in the test, waiting for the server manager to become available. 162 final long timeout = TimeUnit.MINUTES.toMillis(10); 163 assertNotEquals("Timeout waiting for server manager to become available.", 164 -1, Waiter.waitFor(htu.getConfiguration(), timeout, 165 () -> masterThread.getMaster().getServerManager() != null)); 166 htu.getConnection().getAdmin().shutdown(); 167 masterThread.join(); 168 } finally { 169 if (hbaseCluster != null) { 170 hbaseCluster.shutdown(); 171 } 172 if (htu != null) { 173 htu.shutdownMiniCluster(); 174 htu = null; 175 } 176 } 177 } 178 179 /** 180 * Create a cluster configuration suitable for 181 * {@link #testMasterShutdownBeforeStartingAnyRegionServer()}. 182 */ 183 private static Configuration createMasterShutdownBeforeStartingAnyRegionServerConfiguration() { 184 final Configuration conf = HBaseConfiguration.create(); 185 // make sure the master will wait forever in the absence of a RS. 186 conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 1); 187 // don't need a long write pipeline for this test. 188 conf.setInt("dfs.replication", 1); 189 return conf; 190 } 191 192 /** 193 * Create a new {@link Configuration} based on {@code baseConf} that has ZooKeeper connection 194 * settings tuned very aggressively. The resulting client is used within a retry loop, so there's 195 * no value in having the client itself do the retries. We want to iterate on the base 196 * configuration because we're waiting for the mini-cluster to start and set it's ZK client port. 197 * 198 * @return a new, configured {@link Configuration} instance. 199 */ 200 private static Configuration createResponsiveZkConfig(final Configuration baseConf) { 201 final Configuration conf = HBaseConfiguration.create(baseConf); 202 conf.setInt(ReadOnlyZKClient.RECOVERY_RETRY, 3); 203 conf.setInt(ReadOnlyZKClient.RECOVERY_RETRY_INTERVAL_MILLIS, 100); 204 return conf; 205 } 206}