001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master;
019
020import static org.junit.Assert.assertFalse;
021import static org.junit.Assert.assertTrue;
022
023import java.io.IOException;
024import java.net.InetAddress;
025import java.net.UnknownHostException;
026import org.apache.hadoop.conf.Configuration;
027import org.apache.hadoop.hbase.Abortable;
028import org.apache.hadoop.hbase.HBaseClassTestRule;
029import org.apache.hadoop.hbase.HBaseTestingUtility;
030import org.apache.hadoop.hbase.HConstants;
031import org.apache.hadoop.hbase.HRegionInfo;
032import org.apache.hadoop.hbase.MetaMockingUtil;
033import org.apache.hadoop.hbase.ServerLoad;
034import org.apache.hadoop.hbase.ServerMetricsBuilder;
035import org.apache.hadoop.hbase.ServerName;
036import org.apache.hadoop.hbase.TableName;
037import org.apache.hadoop.hbase.ZooKeeperConnectionException;
038import org.apache.hadoop.hbase.client.ClusterConnection;
039import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
040import org.apache.hadoop.hbase.client.Result;
041import org.apache.hadoop.hbase.replication.ReplicationException;
042import org.apache.hadoop.hbase.testclassification.MasterTests;
043import org.apache.hadoop.hbase.testclassification.MediumTests;
044import org.apache.hadoop.hbase.util.FSUtils;
045import org.apache.hadoop.hbase.util.Threads;
046import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
047import org.apache.hadoop.hbase.zookeeper.ZKUtil;
048import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
049import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
050import org.apache.zookeeper.KeeperException;
051import org.junit.After;
052import org.junit.AfterClass;
053import org.junit.Assert;
054import org.junit.BeforeClass;
055import org.junit.ClassRule;
056import org.junit.Ignore;
057import org.junit.Rule;
058import org.junit.Test;
059import org.junit.experimental.categories.Category;
060import org.junit.rules.TestName;
061import org.mockito.Mockito;
062import org.slf4j.Logger;
063import org.slf4j.LoggerFactory;
064
065import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
066import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionServerReportRequest;
067
068/**
069 * Standup the master and fake it to test various aspects of master function.
070 * Does NOT spin up a mini hbase nor mini dfs cluster testing master (it does
071 * put up a zk cluster but this is usually pretty fast compared).  Also, should
072 * be possible to inject faults at points difficult to get at in cluster context.
073 * TODO: Speed up the zk connection by Master.  It pauses 5 seconds establishing
074 * session.
075 */
076@Category({MasterTests.class, MediumTests.class})
077public class TestMasterNoCluster {
078
079  @ClassRule
080  public static final HBaseClassTestRule CLASS_RULE =
081      HBaseClassTestRule.forClass(TestMasterNoCluster.class);
082
083  private static final Logger LOG = LoggerFactory.getLogger(TestMasterNoCluster.class);
084  private static final HBaseTestingUtility TESTUTIL = new HBaseTestingUtility();
085
086  @Rule
087  public TestName name = new TestName();
088
089  @BeforeClass
090  public static void setUpBeforeClass() throws Exception {
091    Configuration c = TESTUTIL.getConfiguration();
092    // We use local filesystem.  Set it so it writes into the testdir.
093    FSUtils.setRootDir(c, TESTUTIL.getDataTestDir());
094    DefaultMetricsSystem.setMiniClusterMode(true);
095    // Startup a mini zk cluster.
096    TESTUTIL.startMiniZKCluster();
097  }
098
099  @AfterClass
100  public static void tearDownAfterClass() throws Exception {
101    TESTUTIL.shutdownMiniZKCluster();
102  }
103
104  @After
105  public void tearDown()
106  throws KeeperException, ZooKeeperConnectionException, IOException {
107    // Make sure zk is clean before we run the next test.
108    ZKWatcher zkw = new ZKWatcher(TESTUTIL.getConfiguration(),
109        "@Before", new Abortable() {
110      @Override
111      public void abort(String why, Throwable e) {
112        throw new RuntimeException(why, e);
113      }
114
115      @Override
116      public boolean isAborted() {
117        return false;
118      }
119    });
120    ZKUtil.deleteNodeRecursively(zkw, zkw.getZNodePaths().baseZNode);
121    zkw.close();
122  }
123
124  /**
125   * Test starting master then stopping it before its fully up.
126   * @throws IOException
127   * @throws KeeperException
128   * @throws InterruptedException
129   */
130  @Test
131  public void testStopDuringStart()
132  throws IOException, KeeperException, InterruptedException {
133    HMaster master = new HMaster(TESTUTIL.getConfiguration());
134    master.start();
135    // Immediately have it stop.  We used hang in assigning meta.
136    master.stopMaster();
137    master.join();
138  }
139
140  /**
141   * Test master failover.
142   * Start up three fake regionservers and a master.
143   * @throws IOException
144   * @throws KeeperException
145   * @throws InterruptedException
146   * @throws org.apache.hbase.thirdparty.com.google.protobuf.ServiceException
147   */
148  @Ignore @Test // Disabled since HBASE-18511. Reenable when master can carry regions.
149  public void testFailover() throws Exception {
150    final long now = System.currentTimeMillis();
151    // Names for our three servers.  Make the port numbers match hostname.
152    // Will come in use down in the server when we need to figure how to respond.
153    final ServerName sn0 = ServerName.valueOf("0.example.org", 0, now);
154    final ServerName sn1 = ServerName.valueOf("1.example.org", 1, now);
155    final ServerName sn2 = ServerName.valueOf("2.example.org", 2, now);
156    final ServerName [] sns = new ServerName [] {sn0, sn1, sn2};
157    // Put up the mock servers
158    final Configuration conf = TESTUTIL.getConfiguration();
159    final MockRegionServer rs0 = new MockRegionServer(conf, sn0);
160    final MockRegionServer rs1 = new MockRegionServer(conf, sn1);
161    final MockRegionServer rs2 = new MockRegionServer(conf, sn2);
162    // Put some data into the servers.  Make it look like sn0 has the metaH
163    // Put data into sn2 so it looks like it has a few regions for a table named 't'.
164    MetaTableLocator.setMetaLocation(rs0.getZooKeeper(),
165      rs0.getServerName(), RegionState.State.OPEN);
166    final TableName tableName = TableName.valueOf(name.getMethodName());
167    Result [] results = new Result [] {
168      MetaMockingUtil.getMetaTableRowResult(
169        new HRegionInfo(tableName, HConstants.EMPTY_START_ROW, HBaseTestingUtility.KEYS[1]),
170        rs2.getServerName()),
171      MetaMockingUtil.getMetaTableRowResult(
172        new HRegionInfo(tableName, HBaseTestingUtility.KEYS[1], HBaseTestingUtility.KEYS[2]),
173        rs2.getServerName()),
174      MetaMockingUtil.getMetaTableRowResult(new HRegionInfo(tableName, HBaseTestingUtility.KEYS[2],
175          HConstants.EMPTY_END_ROW),
176        rs2.getServerName())
177    };
178    rs1.setNextResults(HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), results);
179
180    // Create master.  Subclass to override a few methods so we can insert mocks
181    // and get notification on transitions.  We need to fake out any rpcs the
182    // master does opening/closing regions.  Also need to fake out the address
183    // of the 'remote' mocked up regionservers.
184    // Insert a mock for the connection, use TESTUTIL.getConfiguration rather than
185    // the conf from the master; the conf will already have an ClusterConnection
186    // associate so the below mocking of a connection will fail.
187    final ClusterConnection mockedConnection = HConnectionTestingUtility.getMockedConnectionAndDecorate(
188        TESTUTIL.getConfiguration(), rs0, rs0, rs0.getServerName(),
189        HRegionInfo.FIRST_META_REGIONINFO);
190    HMaster master = new HMaster(conf) {
191      @Override
192      InetAddress getRemoteInetAddress(final int port, final long serverStartCode)
193      throws UnknownHostException {
194        // Return different address dependent on port passed.
195        if (port > sns.length) {
196          return super.getRemoteInetAddress(port, serverStartCode);
197        }
198        ServerName sn = sns[port];
199        return InetAddress.getByAddress(sn.getHostname(),
200          new byte [] {10, 0, 0, (byte)sn.getPort()});
201      }
202
203      @Override
204      protected void initClusterSchemaService() throws IOException, InterruptedException {}
205
206      @Override
207      protected ServerManager createServerManager(MasterServices master) throws IOException {
208        ServerManager sm = super.createServerManager(master);
209        // Spy on the created servermanager
210        ServerManager spy = Mockito.spy(sm);
211        return spy;
212      }
213
214      @Override
215      public ClusterConnection getConnection() {
216        return mockedConnection;
217      }
218
219      @Override
220      public ClusterConnection getClusterConnection() {
221        return mockedConnection;
222      }
223    };
224    master.start();
225
226    try {
227      // Wait till master is up ready for RPCs.
228      while (!master.serviceStarted) Threads.sleep(10);
229      // Fake master that there are regionservers out there.  Report in.
230      for (int i = 0; i < sns.length; i++) {
231        RegionServerReportRequest.Builder request = RegionServerReportRequest.newBuilder();;
232        ServerName sn = ServerName.parseVersionedServerName(sns[i].getVersionedBytes());
233        request.setServer(ProtobufUtil.toServerName(sn));
234        request.setLoad(ServerMetricsBuilder.toServerLoad(ServerMetricsBuilder.of(sn)));
235        master.getMasterRpcServices().regionServerReport(null, request.build());
236      }
237       // Master should now come up.
238      while (!master.isInitialized()) {
239        Threads.sleep(100);
240      }
241      assertTrue(master.isInitialized());
242    } finally {
243      rs0.stop("Test is done");
244      rs1.stop("Test is done");
245      rs2.stop("Test is done");
246      master.stopMaster();
247      master.join();
248    }
249  }
250
251  @Ignore @Test // Disabled since HBASE-18511. Reenable when master can carry regions.
252  public void testNotPullingDeadRegionServerFromZK()
253      throws IOException, KeeperException, InterruptedException {
254    final Configuration conf = TESTUTIL.getConfiguration();
255    final ServerName newServer = ServerName.valueOf("test.sample", 1, 101);
256    final ServerName deadServer = ServerName.valueOf("test.sample", 1, 100);
257    final MockRegionServer rs0 = new MockRegionServer(conf, newServer);
258
259    HMaster master = new HMaster(conf) {
260      @Override
261      protected MasterMetaBootstrap createMetaBootstrap() {
262        return new MasterMetaBootstrap(this) {
263          @Override
264          protected void assignMetaReplicas()
265              throws IOException, InterruptedException, KeeperException {
266            // Nothing to do.
267          }
268        };
269      }
270
271      @Override
272      protected void initClusterSchemaService() throws IOException, InterruptedException {}
273
274      @Override
275      protected void initializeZKBasedSystemTrackers() throws IOException, InterruptedException,
276          KeeperException, ReplicationException {
277        super.initializeZKBasedSystemTrackers();
278        // Record a newer server in server manager at first
279        getServerManager().recordNewServerWithLock(newServer,
280          new ServerLoad(ServerMetricsBuilder.of(newServer)));
281      }
282
283      @Override
284      public ClusterConnection getConnection() {
285        // Insert a mock for the connection, use TESTUTIL.getConfiguration rather than
286        // the conf from the master; the conf will already have a Connection
287        // associate so the below mocking of a connection will fail.
288        try {
289          return HConnectionTestingUtility.getMockedConnectionAndDecorate(
290            TESTUTIL.getConfiguration(), rs0, rs0, rs0.getServerName(),
291            HRegionInfo.FIRST_META_REGIONINFO);
292        } catch (IOException e) {
293          return null;
294        }
295      }
296    };
297    master.start();
298
299    try {
300      // Wait till master is initialized.
301      while (!master.isInitialized()) Threads.sleep(10);
302      LOG.info("Master is initialized");
303
304      assertFalse("The dead server should not be pulled in",
305        master.getServerManager().isServerOnline(deadServer));
306    } finally {
307      master.stopMaster();
308      master.join();
309    }
310  }
311
312  @Test
313  public void testMasterInitWithSameClientServerZKQuorum() throws Exception {
314    Configuration conf = new Configuration(TESTUTIL.getConfiguration());
315    conf.set(HConstants.CLIENT_ZOOKEEPER_QUORUM, HConstants.LOCALHOST);
316    conf.setInt(HConstants.CLIENT_ZOOKEEPER_CLIENT_PORT, TESTUTIL.getZkCluster().getClientPort());
317    HMaster master = new HMaster(conf);
318    master.start();
319    // the master will abort due to IllegalArgumentException so we should finish within 60 seconds
320    master.join();
321  }
322
323  @Test
324  public void testMasterInitWithObserverModeClientZKQuorum() throws Exception {
325    Configuration conf = new Configuration(TESTUTIL.getConfiguration());
326    Assert.assertFalse(Boolean.getBoolean(HConstants.CLIENT_ZOOKEEPER_OBSERVER_MODE));
327    // set client ZK to some non-existing address and make sure server won't access client ZK
328    // (server start should not be affected)
329    conf.set(HConstants.CLIENT_ZOOKEEPER_QUORUM, HConstants.LOCALHOST);
330    conf.setInt(HConstants.CLIENT_ZOOKEEPER_CLIENT_PORT,
331      TESTUTIL.getZkCluster().getClientPort() + 1);
332    // settings to allow us not to start additional RS
333    conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 1);
334    conf.setBoolean(LoadBalancer.TABLES_ON_MASTER, true);
335    // main setting for this test case
336    conf.setBoolean(HConstants.CLIENT_ZOOKEEPER_OBSERVER_MODE, true);
337    HMaster master = new HMaster(conf);
338    master.start();
339    while (!master.isInitialized()) {
340      Threads.sleep(200);
341    }
342    Assert.assertNull(master.metaLocationSyncer);
343    Assert.assertNull(master.masterAddressSyncer);
344    master.stopMaster();
345    master.join();
346  }
347}