001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.master; 019 020import static org.junit.Assert.assertFalse; 021import static org.junit.Assert.assertTrue; 022import static org.apache.hadoop.hbase.HConstants.ZOOKEEPER_QUORUM; 023 024import java.io.IOException; 025import java.net.InetAddress; 026import java.net.UnknownHostException; 027import org.apache.hadoop.conf.Configuration; 028import org.apache.hadoop.hbase.Abortable; 029import org.apache.hadoop.hbase.HBaseClassTestRule; 030import org.apache.hadoop.hbase.HBaseTestingUtility; 031import org.apache.hadoop.hbase.HConstants; 032import org.apache.hadoop.hbase.HRegionInfo; 033import org.apache.hadoop.hbase.MetaMockingUtil; 034import org.apache.hadoop.hbase.ServerLoad; 035import org.apache.hadoop.hbase.ServerMetricsBuilder; 036import org.apache.hadoop.hbase.ServerName; 037import org.apache.hadoop.hbase.TableName; 038import org.apache.hadoop.hbase.Waiter; 039import org.apache.hadoop.hbase.ZooKeeperConnectionException; 040import org.apache.hadoop.hbase.client.ClusterConnection; 041import org.apache.hadoop.hbase.client.HConnectionTestingUtility; 042import org.apache.hadoop.hbase.client.Result; 043import org.apache.hadoop.hbase.replication.ReplicationException; 044import org.apache.hadoop.hbase.testclassification.MasterTests; 045import org.apache.hadoop.hbase.testclassification.MediumTests; 046import org.apache.hadoop.hbase.util.CommonFSUtils; 047import org.apache.hadoop.hbase.util.Threads; 048import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; 049import org.apache.hadoop.hbase.zookeeper.ZKUtil; 050import org.apache.hadoop.hbase.zookeeper.ZKWatcher; 051import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; 052import org.apache.zookeeper.KeeperException; 053import org.junit.After; 054import org.junit.AfterClass; 055import org.junit.Assert; 056import org.junit.BeforeClass; 057import org.junit.ClassRule; 058import org.junit.Ignore; 059import org.junit.Rule; 060import org.junit.Test; 061import org.junit.experimental.categories.Category; 062import org.junit.rules.TestName; 063import org.mockito.Mockito; 064import org.slf4j.Logger; 065import org.slf4j.LoggerFactory; 066 067import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; 068import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionServerReportRequest; 069 070/** 071 * Standup the master and fake it to test various aspects of master function. 072 * Does NOT spin up a mini hbase nor mini dfs cluster testing master (it does 073 * put up a zk cluster but this is usually pretty fast compared). Also, should 074 * be possible to inject faults at points difficult to get at in cluster context. 075 * TODO: Speed up the zk connection by Master. It pauses 5 seconds establishing 076 * session. 077 */ 078@Category({MasterTests.class, MediumTests.class}) 079public class TestMasterNoCluster { 080 081 @ClassRule 082 public static final HBaseClassTestRule CLASS_RULE = 083 HBaseClassTestRule.forClass(TestMasterNoCluster.class); 084 085 private static final Logger LOG = LoggerFactory.getLogger(TestMasterNoCluster.class); 086 private static final HBaseTestingUtility TESTUTIL = new HBaseTestingUtility(); 087 088 @Rule 089 public TestName name = new TestName(); 090 091 @BeforeClass 092 public static void setUpBeforeClass() throws Exception { 093 Configuration c = TESTUTIL.getConfiguration(); 094 // We use local filesystem. Set it so it writes into the testdir. 095 CommonFSUtils.setRootDir(c, TESTUTIL.getDataTestDir()); 096 DefaultMetricsSystem.setMiniClusterMode(true); 097 // Startup a mini zk cluster. 098 TESTUTIL.startMiniZKCluster(); 099 } 100 101 @AfterClass 102 public static void tearDownAfterClass() throws Exception { 103 TESTUTIL.shutdownMiniZKCluster(); 104 } 105 106 @After 107 public void tearDown() 108 throws KeeperException, ZooKeeperConnectionException, IOException { 109 // Make sure zk is clean before we run the next test. 110 ZKWatcher zkw = new ZKWatcher(TESTUTIL.getConfiguration(), 111 "@Before", new Abortable() { 112 @Override 113 public void abort(String why, Throwable e) { 114 throw new RuntimeException(why, e); 115 } 116 117 @Override 118 public boolean isAborted() { 119 return false; 120 } 121 }); 122 // Before fails sometimes so retry. 123 try { 124 TESTUTIL.waitFor(10000, new Waiter.Predicate<Exception>() { 125 @Override public boolean evaluate() throws Exception { 126 try { 127 ZKUtil.deleteNodeRecursively(zkw, zkw.getZNodePaths().baseZNode); 128 return true; 129 } catch (KeeperException.NotEmptyException e) { 130 LOG.info("Failed delete, retrying", e); 131 } 132 return false; 133 } 134 }); 135 } catch (Exception e) { 136 LOG.info("Failed zk clear", e); 137 } 138 zkw.close(); 139 } 140 141 /** 142 * Test starting master then stopping it before its fully up. 143 * @throws IOException 144 * @throws KeeperException 145 * @throws InterruptedException 146 */ 147 @Test 148 public void testStopDuringStart() 149 throws IOException, KeeperException, InterruptedException { 150 HMaster master = new HMaster(TESTUTIL.getConfiguration()); 151 master.start(); 152 // Immediately have it stop. We used hang in assigning meta. 153 master.stopMaster(); 154 master.join(); 155 } 156 157 /** 158 * Test master failover. 159 * Start up three fake regionservers and a master. 160 * @throws IOException 161 * @throws KeeperException 162 * @throws InterruptedException 163 * @throws org.apache.hbase.thirdparty.com.google.protobuf.ServiceException 164 */ 165 @Ignore @Test // Disabled since HBASE-18511. Reenable when master can carry regions. 166 public void testFailover() throws Exception { 167 final long now = System.currentTimeMillis(); 168 // Names for our three servers. Make the port numbers match hostname. 169 // Will come in use down in the server when we need to figure how to respond. 170 final ServerName sn0 = ServerName.valueOf("0.example.org", 0, now); 171 final ServerName sn1 = ServerName.valueOf("1.example.org", 1, now); 172 final ServerName sn2 = ServerName.valueOf("2.example.org", 2, now); 173 final ServerName [] sns = new ServerName [] {sn0, sn1, sn2}; 174 // Put up the mock servers 175 final Configuration conf = TESTUTIL.getConfiguration(); 176 final MockRegionServer rs0 = new MockRegionServer(conf, sn0); 177 final MockRegionServer rs1 = new MockRegionServer(conf, sn1); 178 final MockRegionServer rs2 = new MockRegionServer(conf, sn2); 179 // Put some data into the servers. Make it look like sn0 has the metaH 180 // Put data into sn2 so it looks like it has a few regions for a table named 't'. 181 MetaTableLocator.setMetaLocation(rs0.getZooKeeper(), 182 rs0.getServerName(), RegionState.State.OPEN); 183 final TableName tableName = TableName.valueOf(name.getMethodName()); 184 Result [] results = new Result [] { 185 MetaMockingUtil.getMetaTableRowResult( 186 new HRegionInfo(tableName, HConstants.EMPTY_START_ROW, HBaseTestingUtility.KEYS[1]), 187 rs2.getServerName()), 188 MetaMockingUtil.getMetaTableRowResult( 189 new HRegionInfo(tableName, HBaseTestingUtility.KEYS[1], HBaseTestingUtility.KEYS[2]), 190 rs2.getServerName()), 191 MetaMockingUtil.getMetaTableRowResult(new HRegionInfo(tableName, HBaseTestingUtility.KEYS[2], 192 HConstants.EMPTY_END_ROW), 193 rs2.getServerName()) 194 }; 195 rs1.setNextResults(HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), results); 196 197 // Create master. Subclass to override a few methods so we can insert mocks 198 // and get notification on transitions. We need to fake out any rpcs the 199 // master does opening/closing regions. Also need to fake out the address 200 // of the 'remote' mocked up regionservers. 201 // Insert a mock for the connection, use TESTUTIL.getConfiguration rather than 202 // the conf from the master; the conf will already have an ClusterConnection 203 // associate so the below mocking of a connection will fail. 204 final ClusterConnection mockedConnection = HConnectionTestingUtility.getMockedConnectionAndDecorate( 205 TESTUTIL.getConfiguration(), rs0, rs0, rs0.getServerName(), 206 HRegionInfo.FIRST_META_REGIONINFO); 207 HMaster master = new HMaster(conf) { 208 @Override 209 InetAddress getRemoteInetAddress(final int port, final long serverStartCode) 210 throws UnknownHostException { 211 // Return different address dependent on port passed. 212 if (port > sns.length) { 213 return super.getRemoteInetAddress(port, serverStartCode); 214 } 215 ServerName sn = sns[port]; 216 return InetAddress.getByAddress(sn.getHostname(), 217 new byte [] {10, 0, 0, (byte)sn.getPort()}); 218 } 219 220 @Override 221 protected void initClusterSchemaService() throws IOException, InterruptedException {} 222 223 @Override 224 protected ServerManager createServerManager(MasterServices master) throws IOException { 225 ServerManager sm = super.createServerManager(master); 226 // Spy on the created servermanager 227 ServerManager spy = Mockito.spy(sm); 228 return spy; 229 } 230 231 @Override 232 public ClusterConnection getConnection() { 233 return mockedConnection; 234 } 235 236 @Override 237 public ClusterConnection getClusterConnection() { 238 return mockedConnection; 239 } 240 }; 241 master.start(); 242 243 try { 244 // Wait till master is up ready for RPCs. 245 while (!master.serviceStarted) Threads.sleep(10); 246 // Fake master that there are regionservers out there. Report in. 247 for (int i = 0; i < sns.length; i++) { 248 RegionServerReportRequest.Builder request = RegionServerReportRequest.newBuilder();; 249 ServerName sn = ServerName.parseVersionedServerName(sns[i].getVersionedBytes()); 250 request.setServer(ProtobufUtil.toServerName(sn)); 251 request.setLoad(ServerMetricsBuilder.toServerLoad(ServerMetricsBuilder.of(sn))); 252 master.getMasterRpcServices().regionServerReport(null, request.build()); 253 } 254 // Master should now come up. 255 while (!master.isInitialized()) { 256 Threads.sleep(100); 257 } 258 assertTrue(master.isInitialized()); 259 } finally { 260 rs0.stop("Test is done"); 261 rs1.stop("Test is done"); 262 rs2.stop("Test is done"); 263 master.stopMaster(); 264 master.join(); 265 } 266 } 267 268 @Ignore @Test // Disabled since HBASE-18511. Reenable when master can carry regions. 269 public void testNotPullingDeadRegionServerFromZK() 270 throws IOException, KeeperException, InterruptedException { 271 final Configuration conf = TESTUTIL.getConfiguration(); 272 final ServerName newServer = ServerName.valueOf("test.sample", 1, 101); 273 final ServerName deadServer = ServerName.valueOf("test.sample", 1, 100); 274 final MockRegionServer rs0 = new MockRegionServer(conf, newServer); 275 276 HMaster master = new HMaster(conf) { 277 @Override 278 protected MasterMetaBootstrap createMetaBootstrap() { 279 return new MasterMetaBootstrap(this) { 280 @Override 281 protected void assignMetaReplicas() 282 throws IOException, InterruptedException, KeeperException { 283 // Nothing to do. 284 } 285 }; 286 } 287 288 @Override 289 protected void initClusterSchemaService() throws IOException, InterruptedException {} 290 291 @Override 292 protected void initializeZKBasedSystemTrackers() throws IOException, InterruptedException, 293 KeeperException, ReplicationException { 294 super.initializeZKBasedSystemTrackers(); 295 // Record a newer server in server manager at first 296 getServerManager().recordNewServerWithLock(newServer, 297 new ServerLoad(ServerMetricsBuilder.of(newServer))); 298 } 299 300 @Override 301 public ClusterConnection getConnection() { 302 // Insert a mock for the connection, use TESTUTIL.getConfiguration rather than 303 // the conf from the master; the conf will already have a Connection 304 // associate so the below mocking of a connection will fail. 305 try { 306 return HConnectionTestingUtility.getMockedConnectionAndDecorate( 307 TESTUTIL.getConfiguration(), rs0, rs0, rs0.getServerName(), 308 HRegionInfo.FIRST_META_REGIONINFO); 309 } catch (IOException e) { 310 return null; 311 } 312 } 313 }; 314 master.start(); 315 316 try { 317 // Wait till master is initialized. 318 while (!master.isInitialized()) Threads.sleep(10); 319 LOG.info("Master is initialized"); 320 321 assertFalse("The dead server should not be pulled in", 322 master.getServerManager().isServerOnline(deadServer)); 323 } finally { 324 master.stopMaster(); 325 master.join(); 326 } 327 } 328 329 @Test 330 public void testMasterInitWithSameClientServerZKQuorum() throws Exception { 331 Configuration conf = new Configuration(TESTUTIL.getConfiguration()); 332 conf.set(HConstants.CLIENT_ZOOKEEPER_QUORUM, conf.get(ZOOKEEPER_QUORUM)); 333 conf.setInt(HConstants.CLIENT_ZOOKEEPER_CLIENT_PORT, TESTUTIL.getZkCluster().getClientPort()); 334 HMaster master = new HMaster(conf); 335 master.start(); 336 // the master will abort due to IllegalArgumentException so we should finish within 60 seconds 337 master.join(); 338 } 339 340 @Test 341 public void testMasterInitWithObserverModeClientZKQuorum() throws Exception { 342 Configuration conf = new Configuration(TESTUTIL.getConfiguration()); 343 Assert.assertFalse(Boolean.getBoolean(HConstants.CLIENT_ZOOKEEPER_OBSERVER_MODE)); 344 // set client ZK to some non-existing address and make sure server won't access client ZK 345 // (server start should not be affected) 346 conf.set(HConstants.CLIENT_ZOOKEEPER_QUORUM, HConstants.LOCALHOST); 347 conf.setInt(HConstants.CLIENT_ZOOKEEPER_CLIENT_PORT, 348 TESTUTIL.getZkCluster().getClientPort() + 1); 349 // settings to allow us not to start additional RS 350 conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 1); 351 conf.setBoolean(LoadBalancer.TABLES_ON_MASTER, true); 352 // main setting for this test case 353 conf.setBoolean(HConstants.CLIENT_ZOOKEEPER_OBSERVER_MODE, true); 354 HMaster master = new HMaster(conf); 355 master.start(); 356 while (!master.isInitialized()) { 357 Threads.sleep(200); 358 } 359 Assert.assertNull(master.metaLocationSyncer); 360 Assert.assertNull(master.masterAddressSyncer); 361 master.stopMaster(); 362 master.join(); 363 } 364}