001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.master; 019 020import static org.junit.Assert.assertFalse; 021import static org.junit.Assert.assertTrue; 022 023import java.io.IOException; 024import java.net.InetAddress; 025import java.net.UnknownHostException; 026import org.apache.hadoop.conf.Configuration; 027import org.apache.hadoop.hbase.Abortable; 028import org.apache.hadoop.hbase.HBaseClassTestRule; 029import org.apache.hadoop.hbase.HBaseTestingUtility; 030import org.apache.hadoop.hbase.HConstants; 031import org.apache.hadoop.hbase.HRegionInfo; 032import org.apache.hadoop.hbase.MetaMockingUtil; 033import org.apache.hadoop.hbase.ServerLoad; 034import org.apache.hadoop.hbase.ServerMetricsBuilder; 035import org.apache.hadoop.hbase.ServerName; 036import org.apache.hadoop.hbase.TableName; 037import org.apache.hadoop.hbase.ZooKeeperConnectionException; 038import org.apache.hadoop.hbase.client.ClusterConnection; 039import org.apache.hadoop.hbase.client.HConnectionTestingUtility; 040import org.apache.hadoop.hbase.client.Result; 041import org.apache.hadoop.hbase.testclassification.MasterTests; 042import org.apache.hadoop.hbase.testclassification.MediumTests; 043import org.apache.hadoop.hbase.util.FSUtils; 044import org.apache.hadoop.hbase.util.Threads; 045import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; 046import org.apache.hadoop.hbase.zookeeper.ZKUtil; 047import org.apache.hadoop.hbase.zookeeper.ZKWatcher; 048import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; 049import org.apache.zookeeper.KeeperException; 050import org.junit.After; 051import org.junit.AfterClass; 052import org.junit.BeforeClass; 053import org.junit.ClassRule; 054import org.junit.Ignore; 055import org.junit.Rule; 056import org.junit.Test; 057import org.junit.experimental.categories.Category; 058import org.junit.rules.TestName; 059import org.mockito.Mockito; 060import org.slf4j.Logger; 061import org.slf4j.LoggerFactory; 062 063import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; 064import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionServerReportRequest; 065 066/** 067 * Standup the master and fake it to test various aspects of master function. 068 * Does NOT spin up a mini hbase nor mini dfs cluster testing master (it does 069 * put up a zk cluster but this is usually pretty fast compared). Also, should 070 * be possible to inject faults at points difficult to get at in cluster context. 071 * TODO: Speed up the zk connection by Master. It pauses 5 seconds establishing 072 * session. 073 */ 074@Category({MasterTests.class, MediumTests.class}) 075public class TestMasterNoCluster { 076 077 @ClassRule 078 public static final HBaseClassTestRule CLASS_RULE = 079 HBaseClassTestRule.forClass(TestMasterNoCluster.class); 080 081 private static final Logger LOG = LoggerFactory.getLogger(TestMasterNoCluster.class); 082 private static final HBaseTestingUtility TESTUTIL = new HBaseTestingUtility(); 083 084 @Rule 085 public TestName name = new TestName(); 086 087 @BeforeClass 088 public static void setUpBeforeClass() throws Exception { 089 Configuration c = TESTUTIL.getConfiguration(); 090 // We use local filesystem. Set it so it writes into the testdir. 091 FSUtils.setRootDir(c, TESTUTIL.getDataTestDir()); 092 DefaultMetricsSystem.setMiniClusterMode(true); 093 // Startup a mini zk cluster. 094 TESTUTIL.startMiniZKCluster(); 095 } 096 097 @AfterClass 098 public static void tearDownAfterClass() throws Exception { 099 TESTUTIL.shutdownMiniZKCluster(); 100 } 101 102 @After 103 public void tearDown() 104 throws KeeperException, ZooKeeperConnectionException, IOException { 105 // Make sure zk is clean before we run the next test. 106 ZKWatcher zkw = new ZKWatcher(TESTUTIL.getConfiguration(), 107 "@Before", new Abortable() { 108 @Override 109 public void abort(String why, Throwable e) { 110 throw new RuntimeException(why, e); 111 } 112 113 @Override 114 public boolean isAborted() { 115 return false; 116 } 117 }); 118 ZKUtil.deleteNodeRecursively(zkw, zkw.znodePaths.baseZNode); 119 zkw.close(); 120 } 121 122 /** 123 * Test starting master then stopping it before its fully up. 124 * @throws IOException 125 * @throws KeeperException 126 * @throws InterruptedException 127 */ 128 @Test 129 public void testStopDuringStart() 130 throws IOException, KeeperException, InterruptedException { 131 HMaster master = new HMaster(TESTUTIL.getConfiguration()); 132 master.start(); 133 // Immediately have it stop. We used hang in assigning meta. 134 master.stopMaster(); 135 master.join(); 136 } 137 138 /** 139 * Test master failover. 140 * Start up three fake regionservers and a master. 141 * @throws IOException 142 * @throws KeeperException 143 * @throws InterruptedException 144 * @throws org.apache.hbase.thirdparty.com.google.protobuf.ServiceException 145 */ 146 @Ignore @Test // Disabled since HBASE-18511. Reenable when master can carry regions. 147 public void testFailover() throws Exception { 148 final long now = System.currentTimeMillis(); 149 // Names for our three servers. Make the port numbers match hostname. 150 // Will come in use down in the server when we need to figure how to respond. 151 final ServerName sn0 = ServerName.valueOf("0.example.org", 0, now); 152 final ServerName sn1 = ServerName.valueOf("1.example.org", 1, now); 153 final ServerName sn2 = ServerName.valueOf("2.example.org", 2, now); 154 final ServerName [] sns = new ServerName [] {sn0, sn1, sn2}; 155 // Put up the mock servers 156 final Configuration conf = TESTUTIL.getConfiguration(); 157 final MockRegionServer rs0 = new MockRegionServer(conf, sn0); 158 final MockRegionServer rs1 = new MockRegionServer(conf, sn1); 159 final MockRegionServer rs2 = new MockRegionServer(conf, sn2); 160 // Put some data into the servers. Make it look like sn0 has the metaH 161 // Put data into sn2 so it looks like it has a few regions for a table named 't'. 162 MetaTableLocator.setMetaLocation(rs0.getZooKeeper(), 163 rs0.getServerName(), RegionState.State.OPEN); 164 final TableName tableName = TableName.valueOf(name.getMethodName()); 165 Result [] results = new Result [] { 166 MetaMockingUtil.getMetaTableRowResult( 167 new HRegionInfo(tableName, HConstants.EMPTY_START_ROW, HBaseTestingUtility.KEYS[1]), 168 rs2.getServerName()), 169 MetaMockingUtil.getMetaTableRowResult( 170 new HRegionInfo(tableName, HBaseTestingUtility.KEYS[1], HBaseTestingUtility.KEYS[2]), 171 rs2.getServerName()), 172 MetaMockingUtil.getMetaTableRowResult(new HRegionInfo(tableName, HBaseTestingUtility.KEYS[2], 173 HConstants.EMPTY_END_ROW), 174 rs2.getServerName()) 175 }; 176 rs1.setNextResults(HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), results); 177 178 // Create master. Subclass to override a few methods so we can insert mocks 179 // and get notification on transitions. We need to fake out any rpcs the 180 // master does opening/closing regions. Also need to fake out the address 181 // of the 'remote' mocked up regionservers. 182 // Insert a mock for the connection, use TESTUTIL.getConfiguration rather than 183 // the conf from the master; the conf will already have an ClusterConnection 184 // associate so the below mocking of a connection will fail. 185 final ClusterConnection mockedConnection = HConnectionTestingUtility.getMockedConnectionAndDecorate( 186 TESTUTIL.getConfiguration(), rs0, rs0, rs0.getServerName(), 187 HRegionInfo.FIRST_META_REGIONINFO); 188 HMaster master = new HMaster(conf) { 189 @Override 190 InetAddress getRemoteInetAddress(final int port, final long serverStartCode) 191 throws UnknownHostException { 192 // Return different address dependent on port passed. 193 if (port > sns.length) { 194 return super.getRemoteInetAddress(port, serverStartCode); 195 } 196 ServerName sn = sns[port]; 197 return InetAddress.getByAddress(sn.getHostname(), 198 new byte [] {10, 0, 0, (byte)sn.getPort()}); 199 } 200 201 @Override 202 protected void initClusterSchemaService() throws IOException, InterruptedException {} 203 204 @Override 205 protected ServerManager createServerManager(MasterServices master) throws IOException { 206 ServerManager sm = super.createServerManager(master); 207 // Spy on the created servermanager 208 ServerManager spy = Mockito.spy(sm); 209 return spy; 210 } 211 212 @Override 213 public ClusterConnection getConnection() { 214 return mockedConnection; 215 } 216 217 @Override 218 public ClusterConnection getClusterConnection() { 219 return mockedConnection; 220 } 221 }; 222 master.start(); 223 224 try { 225 // Wait till master is up ready for RPCs. 226 while (!master.serviceStarted) Threads.sleep(10); 227 // Fake master that there are regionservers out there. Report in. 228 for (int i = 0; i < sns.length; i++) { 229 RegionServerReportRequest.Builder request = RegionServerReportRequest.newBuilder();; 230 ServerName sn = ServerName.parseVersionedServerName(sns[i].getVersionedBytes()); 231 request.setServer(ProtobufUtil.toServerName(sn)); 232 request.setLoad(ServerMetricsBuilder.toServerLoad(ServerMetricsBuilder.of(sn))); 233 master.getMasterRpcServices().regionServerReport(null, request.build()); 234 } 235 // Master should now come up. 236 while (!master.isInitialized()) { 237 Threads.sleep(100); 238 } 239 assertTrue(master.isInitialized()); 240 } finally { 241 rs0.stop("Test is done"); 242 rs1.stop("Test is done"); 243 rs2.stop("Test is done"); 244 master.stopMaster(); 245 master.join(); 246 } 247 } 248 249 @Ignore @Test // Disabled since HBASE-18511. Reenable when master can carry regions. 250 public void testNotPullingDeadRegionServerFromZK() 251 throws IOException, KeeperException, InterruptedException { 252 final Configuration conf = TESTUTIL.getConfiguration(); 253 final ServerName newServer = ServerName.valueOf("test.sample", 1, 101); 254 final ServerName deadServer = ServerName.valueOf("test.sample", 1, 100); 255 final MockRegionServer rs0 = new MockRegionServer(conf, newServer); 256 257 HMaster master = new HMaster(conf) { 258 @Override 259 protected MasterMetaBootstrap createMetaBootstrap() { 260 return new MasterMetaBootstrap(this) { 261 @Override 262 protected void assignMetaReplicas() 263 throws IOException, InterruptedException, KeeperException { 264 // Nothing to do. 265 } 266 }; 267 } 268 269 @Override 270 protected void initClusterSchemaService() throws IOException, InterruptedException {} 271 272 @Override 273 protected void initializeZKBasedSystemTrackers() throws IOException, InterruptedException, 274 KeeperException { 275 super.initializeZKBasedSystemTrackers(); 276 // Record a newer server in server manager at first 277 getServerManager().recordNewServerWithLock(newServer, 278 new ServerLoad(ServerMetricsBuilder.of(newServer))); 279 } 280 281 @Override 282 public ClusterConnection getConnection() { 283 // Insert a mock for the connection, use TESTUTIL.getConfiguration rather than 284 // the conf from the master; the conf will already have a Connection 285 // associate so the below mocking of a connection will fail. 286 try { 287 return HConnectionTestingUtility.getMockedConnectionAndDecorate( 288 TESTUTIL.getConfiguration(), rs0, rs0, rs0.getServerName(), 289 HRegionInfo.FIRST_META_REGIONINFO); 290 } catch (IOException e) { 291 return null; 292 } 293 } 294 }; 295 master.start(); 296 297 try { 298 // Wait till master is initialized. 299 while (!master.isInitialized()) Threads.sleep(10); 300 LOG.info("Master is initialized"); 301 302 assertFalse("The dead server should not be pulled in", 303 master.getServerManager().isServerOnline(deadServer)); 304 } finally { 305 master.stopMaster(); 306 master.join(); 307 } 308 } 309}