001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.master; 019 020import static org.junit.Assert.assertFalse; 021import static org.junit.Assert.assertTrue; 022 023import java.io.IOException; 024import java.net.InetAddress; 025import java.net.UnknownHostException; 026import org.apache.hadoop.conf.Configuration; 027import org.apache.hadoop.hbase.Abortable; 028import org.apache.hadoop.hbase.HBaseClassTestRule; 029import org.apache.hadoop.hbase.HBaseTestingUtility; 030import org.apache.hadoop.hbase.HConstants; 031import org.apache.hadoop.hbase.HRegionInfo; 032import org.apache.hadoop.hbase.MetaMockingUtil; 033import org.apache.hadoop.hbase.ServerLoad; 034import org.apache.hadoop.hbase.ServerMetricsBuilder; 035import org.apache.hadoop.hbase.ServerName; 036import org.apache.hadoop.hbase.TableName; 037import org.apache.hadoop.hbase.ZooKeeperConnectionException; 038import org.apache.hadoop.hbase.client.ClusterConnection; 039import org.apache.hadoop.hbase.client.HConnectionTestingUtility; 040import org.apache.hadoop.hbase.client.Result; 041import org.apache.hadoop.hbase.replication.ReplicationException; 042import org.apache.hadoop.hbase.testclassification.MasterTests; 043import org.apache.hadoop.hbase.testclassification.MediumTests; 044import org.apache.hadoop.hbase.util.FSUtils; 045import org.apache.hadoop.hbase.util.Threads; 046import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; 047import org.apache.hadoop.hbase.zookeeper.ZKUtil; 048import org.apache.hadoop.hbase.zookeeper.ZKWatcher; 049import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; 050import org.apache.zookeeper.KeeperException; 051import org.junit.After; 052import org.junit.AfterClass; 053import org.junit.Assert; 054import org.junit.BeforeClass; 055import org.junit.ClassRule; 056import org.junit.Ignore; 057import org.junit.Rule; 058import org.junit.Test; 059import org.junit.experimental.categories.Category; 060import org.junit.rules.TestName; 061import org.mockito.Mockito; 062import org.slf4j.Logger; 063import org.slf4j.LoggerFactory; 064 065import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; 066import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionServerReportRequest; 067 068/** 069 * Standup the master and fake it to test various aspects of master function. 070 * Does NOT spin up a mini hbase nor mini dfs cluster testing master (it does 071 * put up a zk cluster but this is usually pretty fast compared). Also, should 072 * be possible to inject faults at points difficult to get at in cluster context. 073 * TODO: Speed up the zk connection by Master. It pauses 5 seconds establishing 074 * session. 075 */ 076@Category({MasterTests.class, MediumTests.class}) 077public class TestMasterNoCluster { 078 079 @ClassRule 080 public static final HBaseClassTestRule CLASS_RULE = 081 HBaseClassTestRule.forClass(TestMasterNoCluster.class); 082 083 private static final Logger LOG = LoggerFactory.getLogger(TestMasterNoCluster.class); 084 private static final HBaseTestingUtility TESTUTIL = new HBaseTestingUtility(); 085 086 @Rule 087 public TestName name = new TestName(); 088 089 @BeforeClass 090 public static void setUpBeforeClass() throws Exception { 091 Configuration c = TESTUTIL.getConfiguration(); 092 // We use local filesystem. Set it so it writes into the testdir. 093 FSUtils.setRootDir(c, TESTUTIL.getDataTestDir()); 094 DefaultMetricsSystem.setMiniClusterMode(true); 095 // Startup a mini zk cluster. 096 TESTUTIL.startMiniZKCluster(); 097 } 098 099 @AfterClass 100 public static void tearDownAfterClass() throws Exception { 101 TESTUTIL.shutdownMiniZKCluster(); 102 } 103 104 @After 105 public void tearDown() 106 throws KeeperException, ZooKeeperConnectionException, IOException { 107 // Make sure zk is clean before we run the next test. 108 ZKWatcher zkw = new ZKWatcher(TESTUTIL.getConfiguration(), 109 "@Before", new Abortable() { 110 @Override 111 public void abort(String why, Throwable e) { 112 throw new RuntimeException(why, e); 113 } 114 115 @Override 116 public boolean isAborted() { 117 return false; 118 } 119 }); 120 ZKUtil.deleteNodeRecursively(zkw, zkw.getZNodePaths().baseZNode); 121 zkw.close(); 122 } 123 124 /** 125 * Test starting master then stopping it before its fully up. 126 * @throws IOException 127 * @throws KeeperException 128 * @throws InterruptedException 129 */ 130 @Test 131 public void testStopDuringStart() 132 throws IOException, KeeperException, InterruptedException { 133 HMaster master = new HMaster(TESTUTIL.getConfiguration()); 134 master.start(); 135 // Immediately have it stop. We used hang in assigning meta. 136 master.stopMaster(); 137 master.join(); 138 } 139 140 /** 141 * Test master failover. 142 * Start up three fake regionservers and a master. 143 * @throws IOException 144 * @throws KeeperException 145 * @throws InterruptedException 146 * @throws org.apache.hbase.thirdparty.com.google.protobuf.ServiceException 147 */ 148 @Ignore @Test // Disabled since HBASE-18511. Reenable when master can carry regions. 149 public void testFailover() throws Exception { 150 final long now = System.currentTimeMillis(); 151 // Names for our three servers. Make the port numbers match hostname. 152 // Will come in use down in the server when we need to figure how to respond. 153 final ServerName sn0 = ServerName.valueOf("0.example.org", 0, now); 154 final ServerName sn1 = ServerName.valueOf("1.example.org", 1, now); 155 final ServerName sn2 = ServerName.valueOf("2.example.org", 2, now); 156 final ServerName [] sns = new ServerName [] {sn0, sn1, sn2}; 157 // Put up the mock servers 158 final Configuration conf = TESTUTIL.getConfiguration(); 159 final MockRegionServer rs0 = new MockRegionServer(conf, sn0); 160 final MockRegionServer rs1 = new MockRegionServer(conf, sn1); 161 final MockRegionServer rs2 = new MockRegionServer(conf, sn2); 162 // Put some data into the servers. Make it look like sn0 has the metaH 163 // Put data into sn2 so it looks like it has a few regions for a table named 't'. 164 MetaTableLocator.setMetaLocation(rs0.getZooKeeper(), 165 rs0.getServerName(), RegionState.State.OPEN); 166 final TableName tableName = TableName.valueOf(name.getMethodName()); 167 Result [] results = new Result [] { 168 MetaMockingUtil.getMetaTableRowResult( 169 new HRegionInfo(tableName, HConstants.EMPTY_START_ROW, HBaseTestingUtility.KEYS[1]), 170 rs2.getServerName()), 171 MetaMockingUtil.getMetaTableRowResult( 172 new HRegionInfo(tableName, HBaseTestingUtility.KEYS[1], HBaseTestingUtility.KEYS[2]), 173 rs2.getServerName()), 174 MetaMockingUtil.getMetaTableRowResult(new HRegionInfo(tableName, HBaseTestingUtility.KEYS[2], 175 HConstants.EMPTY_END_ROW), 176 rs2.getServerName()) 177 }; 178 rs1.setNextResults(HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), results); 179 180 // Create master. Subclass to override a few methods so we can insert mocks 181 // and get notification on transitions. We need to fake out any rpcs the 182 // master does opening/closing regions. Also need to fake out the address 183 // of the 'remote' mocked up regionservers. 184 // Insert a mock for the connection, use TESTUTIL.getConfiguration rather than 185 // the conf from the master; the conf will already have an ClusterConnection 186 // associate so the below mocking of a connection will fail. 187 final ClusterConnection mockedConnection = HConnectionTestingUtility.getMockedConnectionAndDecorate( 188 TESTUTIL.getConfiguration(), rs0, rs0, rs0.getServerName(), 189 HRegionInfo.FIRST_META_REGIONINFO); 190 HMaster master = new HMaster(conf) { 191 @Override 192 InetAddress getRemoteInetAddress(final int port, final long serverStartCode) 193 throws UnknownHostException { 194 // Return different address dependent on port passed. 195 if (port > sns.length) { 196 return super.getRemoteInetAddress(port, serverStartCode); 197 } 198 ServerName sn = sns[port]; 199 return InetAddress.getByAddress(sn.getHostname(), 200 new byte [] {10, 0, 0, (byte)sn.getPort()}); 201 } 202 203 @Override 204 protected void initClusterSchemaService() throws IOException, InterruptedException {} 205 206 @Override 207 protected ServerManager createServerManager(MasterServices master) throws IOException { 208 ServerManager sm = super.createServerManager(master); 209 // Spy on the created servermanager 210 ServerManager spy = Mockito.spy(sm); 211 return spy; 212 } 213 214 @Override 215 public ClusterConnection getConnection() { 216 return mockedConnection; 217 } 218 219 @Override 220 public ClusterConnection getClusterConnection() { 221 return mockedConnection; 222 } 223 }; 224 master.start(); 225 226 try { 227 // Wait till master is up ready for RPCs. 228 while (!master.serviceStarted) Threads.sleep(10); 229 // Fake master that there are regionservers out there. Report in. 230 for (int i = 0; i < sns.length; i++) { 231 RegionServerReportRequest.Builder request = RegionServerReportRequest.newBuilder();; 232 ServerName sn = ServerName.parseVersionedServerName(sns[i].getVersionedBytes()); 233 request.setServer(ProtobufUtil.toServerName(sn)); 234 request.setLoad(ServerMetricsBuilder.toServerLoad(ServerMetricsBuilder.of(sn))); 235 master.getMasterRpcServices().regionServerReport(null, request.build()); 236 } 237 // Master should now come up. 238 while (!master.isInitialized()) { 239 Threads.sleep(100); 240 } 241 assertTrue(master.isInitialized()); 242 } finally { 243 rs0.stop("Test is done"); 244 rs1.stop("Test is done"); 245 rs2.stop("Test is done"); 246 master.stopMaster(); 247 master.join(); 248 } 249 } 250 251 @Ignore @Test // Disabled since HBASE-18511. Reenable when master can carry regions. 252 public void testNotPullingDeadRegionServerFromZK() 253 throws IOException, KeeperException, InterruptedException { 254 final Configuration conf = TESTUTIL.getConfiguration(); 255 final ServerName newServer = ServerName.valueOf("test.sample", 1, 101); 256 final ServerName deadServer = ServerName.valueOf("test.sample", 1, 100); 257 final MockRegionServer rs0 = new MockRegionServer(conf, newServer); 258 259 HMaster master = new HMaster(conf) { 260 @Override 261 protected MasterMetaBootstrap createMetaBootstrap() { 262 return new MasterMetaBootstrap(this) { 263 @Override 264 protected void assignMetaReplicas() 265 throws IOException, InterruptedException, KeeperException { 266 // Nothing to do. 267 } 268 }; 269 } 270 271 @Override 272 protected void initClusterSchemaService() throws IOException, InterruptedException {} 273 274 @Override 275 protected void initializeZKBasedSystemTrackers() throws IOException, InterruptedException, 276 KeeperException, ReplicationException { 277 super.initializeZKBasedSystemTrackers(); 278 // Record a newer server in server manager at first 279 getServerManager().recordNewServerWithLock(newServer, 280 new ServerLoad(ServerMetricsBuilder.of(newServer))); 281 } 282 283 @Override 284 public ClusterConnection getConnection() { 285 // Insert a mock for the connection, use TESTUTIL.getConfiguration rather than 286 // the conf from the master; the conf will already have a Connection 287 // associate so the below mocking of a connection will fail. 288 try { 289 return HConnectionTestingUtility.getMockedConnectionAndDecorate( 290 TESTUTIL.getConfiguration(), rs0, rs0, rs0.getServerName(), 291 HRegionInfo.FIRST_META_REGIONINFO); 292 } catch (IOException e) { 293 return null; 294 } 295 } 296 }; 297 master.start(); 298 299 try { 300 // Wait till master is initialized. 301 while (!master.isInitialized()) Threads.sleep(10); 302 LOG.info("Master is initialized"); 303 304 assertFalse("The dead server should not be pulled in", 305 master.getServerManager().isServerOnline(deadServer)); 306 } finally { 307 master.stopMaster(); 308 master.join(); 309 } 310 } 311 312 @Test 313 public void testMasterInitWithSameClientServerZKQuorum() throws Exception { 314 Configuration conf = new Configuration(TESTUTIL.getConfiguration()); 315 conf.set(HConstants.CLIENT_ZOOKEEPER_QUORUM, HConstants.LOCALHOST); 316 conf.setInt(HConstants.CLIENT_ZOOKEEPER_CLIENT_PORT, TESTUTIL.getZkCluster().getClientPort()); 317 HMaster master = new HMaster(conf); 318 master.start(); 319 // the master will abort due to IllegalArgumentException so we should finish within 60 seconds 320 master.join(); 321 } 322 323 @Test 324 public void testMasterInitWithObserverModeClientZKQuorum() throws Exception { 325 Configuration conf = new Configuration(TESTUTIL.getConfiguration()); 326 Assert.assertFalse(Boolean.getBoolean(HConstants.CLIENT_ZOOKEEPER_OBSERVER_MODE)); 327 // set client ZK to some non-existing address and make sure server won't access client ZK 328 // (server start should not be affected) 329 conf.set(HConstants.CLIENT_ZOOKEEPER_QUORUM, HConstants.LOCALHOST); 330 conf.setInt(HConstants.CLIENT_ZOOKEEPER_CLIENT_PORT, 331 TESTUTIL.getZkCluster().getClientPort() + 1); 332 // settings to allow us not to start additional RS 333 conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 1); 334 conf.setBoolean(LoadBalancer.TABLES_ON_MASTER, true); 335 // main setting for this test case 336 conf.setBoolean(HConstants.CLIENT_ZOOKEEPER_OBSERVER_MODE, true); 337 HMaster master = new HMaster(conf); 338 master.start(); 339 while (!master.isInitialized()) { 340 Threads.sleep(200); 341 } 342 Assert.assertNull(master.metaLocationSyncer); 343 Assert.assertNull(master.masterAddressSyncer); 344 master.stopMaster(); 345 master.join(); 346 } 347}