001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master;
019
020import static org.apache.hadoop.hbase.HConstants.ZOOKEEPER_QUORUM;
021import static org.junit.Assert.assertTrue;
022
023import java.io.IOException;
024import java.net.InetAddress;
025import java.net.UnknownHostException;
026import org.apache.hadoop.conf.Configuration;
027import org.apache.hadoop.hbase.Abortable;
028import org.apache.hadoop.hbase.HBaseClassTestRule;
029import org.apache.hadoop.hbase.HBaseTestingUtility;
030import org.apache.hadoop.hbase.HConstants;
031import org.apache.hadoop.hbase.HRegionInfo;
032import org.apache.hadoop.hbase.MetaMockingUtil;
033import org.apache.hadoop.hbase.ServerMetricsBuilder;
034import org.apache.hadoop.hbase.ServerName;
035import org.apache.hadoop.hbase.TableName;
036import org.apache.hadoop.hbase.Waiter;
037import org.apache.hadoop.hbase.ZooKeeperConnectionException;
038import org.apache.hadoop.hbase.client.ClusterConnection;
039import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
040import org.apache.hadoop.hbase.client.Result;
041import org.apache.hadoop.hbase.testclassification.MasterTests;
042import org.apache.hadoop.hbase.testclassification.MediumTests;
043import org.apache.hadoop.hbase.util.CommonFSUtils;
044import org.apache.hadoop.hbase.util.Threads;
045import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
046import org.apache.hadoop.hbase.zookeeper.ZKUtil;
047import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
048import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
049import org.apache.zookeeper.KeeperException;
050import org.junit.After;
051import org.junit.AfterClass;
052import org.junit.Assert;
053import org.junit.BeforeClass;
054import org.junit.ClassRule;
055import org.junit.Ignore;
056import org.junit.Rule;
057import org.junit.Test;
058import org.junit.experimental.categories.Category;
059import org.junit.rules.TestName;
060import org.mockito.Mockito;
061import org.slf4j.Logger;
062import org.slf4j.LoggerFactory;
063
064import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
065import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionServerReportRequest;
066
067/**
068 * Standup the master and fake it to test various aspects of master function.
069 * Does NOT spin up a mini hbase nor mini dfs cluster testing master (it does
070 * put up a zk cluster but this is usually pretty fast compared).  Also, should
071 * be possible to inject faults at points difficult to get at in cluster context.
072 * TODO: Speed up the zk connection by Master.  It pauses 5 seconds establishing
073 * session.
074 */
075@Category({MasterTests.class, MediumTests.class})
076public class TestMasterNoCluster {
077
078  @ClassRule
079  public static final HBaseClassTestRule CLASS_RULE =
080      HBaseClassTestRule.forClass(TestMasterNoCluster.class);
081
082  private static final Logger LOG = LoggerFactory.getLogger(TestMasterNoCluster.class);
083  private static final HBaseTestingUtility TESTUTIL = new HBaseTestingUtility();
084
085  @Rule
086  public TestName name = new TestName();
087
088  @BeforeClass
089  public static void setUpBeforeClass() throws Exception {
090    Configuration c = TESTUTIL.getConfiguration();
091    // We use local filesystem.  Set it so it writes into the testdir.
092    CommonFSUtils.setRootDir(c, TESTUTIL.getDataTestDir());
093    DefaultMetricsSystem.setMiniClusterMode(true);
094    // Startup a mini zk cluster.
095    TESTUTIL.startMiniZKCluster();
096  }
097
098  @AfterClass
099  public static void tearDownAfterClass() throws Exception {
100    TESTUTIL.shutdownMiniZKCluster();
101  }
102
103  @After
104  public void tearDown()
105  throws KeeperException, ZooKeeperConnectionException, IOException {
106    // Make sure zk is clean before we run the next test.
107    ZKWatcher zkw = new ZKWatcher(TESTUTIL.getConfiguration(),
108        "@Before", new Abortable() {
109      @Override
110      public void abort(String why, Throwable e) {
111        throw new RuntimeException(why, e);
112      }
113
114      @Override
115      public boolean isAborted() {
116        return false;
117      }
118    });
119    // Before fails sometimes so retry.
120    try {
121      TESTUTIL.waitFor(10000, new Waiter.Predicate<Exception>() {
122        @Override public boolean evaluate() throws Exception {
123          try {
124            ZKUtil.deleteNodeRecursively(zkw, zkw.getZNodePaths().baseZNode);
125            return true;
126          } catch (KeeperException.NotEmptyException e) {
127            LOG.info("Failed delete, retrying", e);
128          }
129          return false;
130        }
131      });
132    } catch (Exception e) {
133      LOG.info("Failed zk clear", e);
134    }
135    zkw.close();
136  }
137
138  /**
139   * Test starting master then stopping it before its fully up.
140   * @throws IOException
141   * @throws KeeperException
142   * @throws InterruptedException
143   */
144  @Test
145  public void testStopDuringStart()
146  throws IOException, KeeperException, InterruptedException {
147    HMaster master = new HMaster(TESTUTIL.getConfiguration());
148    master.start();
149    // Immediately have it stop.  We used hang in assigning meta.
150    master.stopMaster();
151    master.join();
152  }
153
154  /**
155   * Test master failover.
156   * Start up three fake regionservers and a master.
157   * @throws IOException
158   * @throws KeeperException
159   * @throws InterruptedException
160   * @throws org.apache.hbase.thirdparty.com.google.protobuf.ServiceException
161   */
162  @Ignore @Test // Disabled since HBASE-18511. Reenable when master can carry regions.
163  public void testFailover() throws Exception {
164    final long now = System.currentTimeMillis();
165    // Names for our three servers.  Make the port numbers match hostname.
166    // Will come in use down in the server when we need to figure how to respond.
167    final ServerName sn0 = ServerName.valueOf("0.example.org", 0, now);
168    final ServerName sn1 = ServerName.valueOf("1.example.org", 1, now);
169    final ServerName sn2 = ServerName.valueOf("2.example.org", 2, now);
170    final ServerName [] sns = new ServerName [] {sn0, sn1, sn2};
171    // Put up the mock servers
172    final Configuration conf = TESTUTIL.getConfiguration();
173    final MockRegionServer rs0 = new MockRegionServer(conf, sn0);
174    final MockRegionServer rs1 = new MockRegionServer(conf, sn1);
175    final MockRegionServer rs2 = new MockRegionServer(conf, sn2);
176    // Put some data into the servers.  Make it look like sn0 has the metaH
177    // Put data into sn2 so it looks like it has a few regions for a table named 't'.
178    MetaTableLocator.setMetaLocation(rs0.getZooKeeper(),
179      rs0.getServerName(), RegionState.State.OPEN);
180    final TableName tableName = TableName.valueOf(name.getMethodName());
181    Result [] results = new Result [] {
182      MetaMockingUtil.getMetaTableRowResult(
183        new HRegionInfo(tableName, HConstants.EMPTY_START_ROW, HBaseTestingUtility.KEYS[1]),
184        rs2.getServerName()),
185      MetaMockingUtil.getMetaTableRowResult(
186        new HRegionInfo(tableName, HBaseTestingUtility.KEYS[1], HBaseTestingUtility.KEYS[2]),
187        rs2.getServerName()),
188      MetaMockingUtil.getMetaTableRowResult(new HRegionInfo(tableName, HBaseTestingUtility.KEYS[2],
189          HConstants.EMPTY_END_ROW),
190        rs2.getServerName())
191    };
192    rs1.setNextResults(HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), results);
193
194    // Create master.  Subclass to override a few methods so we can insert mocks
195    // and get notification on transitions.  We need to fake out any rpcs the
196    // master does opening/closing regions.  Also need to fake out the address
197    // of the 'remote' mocked up regionservers.
198    // Insert a mock for the connection, use TESTUTIL.getConfiguration rather than
199    // the conf from the master; the conf will already have an ClusterConnection
200    // associate so the below mocking of a connection will fail.
201    final ClusterConnection mockedConnection = HConnectionTestingUtility.getMockedConnectionAndDecorate(
202        TESTUTIL.getConfiguration(), rs0, rs0, rs0.getServerName(),
203        HRegionInfo.FIRST_META_REGIONINFO);
204    HMaster master = new HMaster(conf) {
205      @Override
206      InetAddress getRemoteInetAddress(final int port, final long serverStartCode)
207      throws UnknownHostException {
208        // Return different address dependent on port passed.
209        if (port > sns.length) {
210          return super.getRemoteInetAddress(port, serverStartCode);
211        }
212        ServerName sn = sns[port];
213        return InetAddress.getByAddress(sn.getHostname(),
214          new byte [] {10, 0, 0, (byte)sn.getPort()});
215      }
216
217      @Override
218      protected void initClusterSchemaService() throws IOException, InterruptedException {}
219
220      @Override
221      protected ServerManager createServerManager(MasterServices master) throws IOException {
222        ServerManager sm = super.createServerManager(master);
223        // Spy on the created servermanager
224        ServerManager spy = Mockito.spy(sm);
225        return spy;
226      }
227
228      @Override
229      public ClusterConnection getConnection() {
230        return mockedConnection;
231      }
232
233      @Override
234      public ClusterConnection getClusterConnection() {
235        return mockedConnection;
236      }
237    };
238    master.start();
239
240    try {
241      // Wait till master is up ready for RPCs.
242      while (!master.serviceStarted) Threads.sleep(10);
243      // Fake master that there are regionservers out there.  Report in.
244      for (int i = 0; i < sns.length; i++) {
245        RegionServerReportRequest.Builder request = RegionServerReportRequest.newBuilder();;
246        ServerName sn = ServerName.parseVersionedServerName(sns[i].getVersionedBytes());
247        request.setServer(ProtobufUtil.toServerName(sn));
248        request.setLoad(ServerMetricsBuilder.toServerLoad(ServerMetricsBuilder.of(sn)));
249        master.getMasterRpcServices().regionServerReport(null, request.build());
250      }
251       // Master should now come up.
252      while (!master.isInitialized()) {
253        Threads.sleep(100);
254      }
255      assertTrue(master.isInitialized());
256    } finally {
257      rs0.stop("Test is done");
258      rs1.stop("Test is done");
259      rs2.stop("Test is done");
260      master.stopMaster();
261      master.join();
262    }
263  }
264
265  @Test
266  public void testMasterInitWithSameClientServerZKQuorum() throws Exception {
267    Configuration conf = new Configuration(TESTUTIL.getConfiguration());
268    conf.set(HConstants.CLIENT_ZOOKEEPER_QUORUM, conf.get(ZOOKEEPER_QUORUM));
269    conf.setInt(HConstants.CLIENT_ZOOKEEPER_CLIENT_PORT, TESTUTIL.getZkCluster().getClientPort());
270    HMaster master = new HMaster(conf);
271    master.start();
272    // the master will abort due to IllegalArgumentException so we should finish within 60 seconds
273    master.join();
274  }
275
276  @Test
277  public void testMasterInitWithObserverModeClientZKQuorum() throws Exception {
278    Configuration conf = new Configuration(TESTUTIL.getConfiguration());
279    Assert.assertFalse(Boolean.getBoolean(HConstants.CLIENT_ZOOKEEPER_OBSERVER_MODE));
280    // set client ZK to some non-existing address and make sure server won't access client ZK
281    // (server start should not be affected)
282    conf.set(HConstants.CLIENT_ZOOKEEPER_QUORUM, HConstants.LOCALHOST);
283    conf.setInt(HConstants.CLIENT_ZOOKEEPER_CLIENT_PORT,
284      TESTUTIL.getZkCluster().getClientPort() + 1);
285    // settings to allow us not to start additional RS
286    conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 1);
287    conf.setBoolean(LoadBalancer.TABLES_ON_MASTER, true);
288    // main setting for this test case
289    conf.setBoolean(HConstants.CLIENT_ZOOKEEPER_OBSERVER_MODE, true);
290    HMaster master = new HMaster(conf);
291    master.start();
292    while (!master.isInitialized()) {
293      Threads.sleep(200);
294    }
295    Assert.assertNull(master.getMetaLocationSyncer());
296    Assert.assertNull(master.masterAddressSyncer);
297    master.stopMaster();
298    master.join();
299  }
300}