001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master;
019
020import static org.junit.Assert.assertFalse;
021import static org.junit.Assert.assertTrue;
022import static org.apache.hadoop.hbase.HConstants.ZOOKEEPER_QUORUM;
023
024import java.io.IOException;
025import java.net.InetAddress;
026import java.net.UnknownHostException;
027import org.apache.hadoop.conf.Configuration;
028import org.apache.hadoop.hbase.Abortable;
029import org.apache.hadoop.hbase.HBaseClassTestRule;
030import org.apache.hadoop.hbase.HBaseTestingUtility;
031import org.apache.hadoop.hbase.HConstants;
032import org.apache.hadoop.hbase.HRegionInfo;
033import org.apache.hadoop.hbase.MetaMockingUtil;
034import org.apache.hadoop.hbase.ServerLoad;
035import org.apache.hadoop.hbase.ServerMetricsBuilder;
036import org.apache.hadoop.hbase.ServerName;
037import org.apache.hadoop.hbase.TableName;
038import org.apache.hadoop.hbase.Waiter;
039import org.apache.hadoop.hbase.ZooKeeperConnectionException;
040import org.apache.hadoop.hbase.client.ClusterConnection;
041import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
042import org.apache.hadoop.hbase.client.Result;
043import org.apache.hadoop.hbase.replication.ReplicationException;
044import org.apache.hadoop.hbase.testclassification.MasterTests;
045import org.apache.hadoop.hbase.testclassification.MediumTests;
046import org.apache.hadoop.hbase.util.CommonFSUtils;
047import org.apache.hadoop.hbase.util.Threads;
048import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
049import org.apache.hadoop.hbase.zookeeper.ZKUtil;
050import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
051import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
052import org.apache.zookeeper.KeeperException;
053import org.junit.After;
054import org.junit.AfterClass;
055import org.junit.Assert;
056import org.junit.BeforeClass;
057import org.junit.ClassRule;
058import org.junit.Ignore;
059import org.junit.Rule;
060import org.junit.Test;
061import org.junit.experimental.categories.Category;
062import org.junit.rules.TestName;
063import org.mockito.Mockito;
064import org.slf4j.Logger;
065import org.slf4j.LoggerFactory;
066
067import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
068import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionServerReportRequest;
069
070/**
071 * Standup the master and fake it to test various aspects of master function.
072 * Does NOT spin up a mini hbase nor mini dfs cluster testing master (it does
073 * put up a zk cluster but this is usually pretty fast compared).  Also, should
074 * be possible to inject faults at points difficult to get at in cluster context.
075 * TODO: Speed up the zk connection by Master.  It pauses 5 seconds establishing
076 * session.
077 */
078@Category({MasterTests.class, MediumTests.class})
079public class TestMasterNoCluster {
080
081  @ClassRule
082  public static final HBaseClassTestRule CLASS_RULE =
083      HBaseClassTestRule.forClass(TestMasterNoCluster.class);
084
085  private static final Logger LOG = LoggerFactory.getLogger(TestMasterNoCluster.class);
086  private static final HBaseTestingUtility TESTUTIL = new HBaseTestingUtility();
087
088  @Rule
089  public TestName name = new TestName();
090
091  @BeforeClass
092  public static void setUpBeforeClass() throws Exception {
093    Configuration c = TESTUTIL.getConfiguration();
094    // We use local filesystem.  Set it so it writes into the testdir.
095    CommonFSUtils.setRootDir(c, TESTUTIL.getDataTestDir());
096    DefaultMetricsSystem.setMiniClusterMode(true);
097    // Startup a mini zk cluster.
098    TESTUTIL.startMiniZKCluster();
099  }
100
101  @AfterClass
102  public static void tearDownAfterClass() throws Exception {
103    TESTUTIL.shutdownMiniZKCluster();
104  }
105
106  @After
107  public void tearDown()
108  throws KeeperException, ZooKeeperConnectionException, IOException {
109    // Make sure zk is clean before we run the next test.
110    ZKWatcher zkw = new ZKWatcher(TESTUTIL.getConfiguration(),
111        "@Before", new Abortable() {
112      @Override
113      public void abort(String why, Throwable e) {
114        throw new RuntimeException(why, e);
115      }
116
117      @Override
118      public boolean isAborted() {
119        return false;
120      }
121    });
122    // Before fails sometimes so retry.
123    try {
124      TESTUTIL.waitFor(10000, new Waiter.Predicate<Exception>() {
125        @Override public boolean evaluate() throws Exception {
126          try {
127            ZKUtil.deleteNodeRecursively(zkw, zkw.getZNodePaths().baseZNode);
128            return true;
129          } catch (KeeperException.NotEmptyException e) {
130            LOG.info("Failed delete, retrying", e);
131          }
132          return false;
133        }
134      });
135    } catch (Exception e) {
136      LOG.info("Failed zk clear", e);
137    }
138    zkw.close();
139  }
140
141  /**
142   * Test starting master then stopping it before its fully up.
143   * @throws IOException
144   * @throws KeeperException
145   * @throws InterruptedException
146   */
147  @Test
148  public void testStopDuringStart()
149  throws IOException, KeeperException, InterruptedException {
150    HMaster master = new HMaster(TESTUTIL.getConfiguration());
151    master.start();
152    // Immediately have it stop.  We used hang in assigning meta.
153    master.stopMaster();
154    master.join();
155  }
156
157  /**
158   * Test master failover.
159   * Start up three fake regionservers and a master.
160   * @throws IOException
161   * @throws KeeperException
162   * @throws InterruptedException
163   * @throws org.apache.hbase.thirdparty.com.google.protobuf.ServiceException
164   */
165  @Ignore @Test // Disabled since HBASE-18511. Reenable when master can carry regions.
166  public void testFailover() throws Exception {
167    final long now = System.currentTimeMillis();
168    // Names for our three servers.  Make the port numbers match hostname.
169    // Will come in use down in the server when we need to figure how to respond.
170    final ServerName sn0 = ServerName.valueOf("0.example.org", 0, now);
171    final ServerName sn1 = ServerName.valueOf("1.example.org", 1, now);
172    final ServerName sn2 = ServerName.valueOf("2.example.org", 2, now);
173    final ServerName [] sns = new ServerName [] {sn0, sn1, sn2};
174    // Put up the mock servers
175    final Configuration conf = TESTUTIL.getConfiguration();
176    final MockRegionServer rs0 = new MockRegionServer(conf, sn0);
177    final MockRegionServer rs1 = new MockRegionServer(conf, sn1);
178    final MockRegionServer rs2 = new MockRegionServer(conf, sn2);
179    // Put some data into the servers.  Make it look like sn0 has the metaH
180    // Put data into sn2 so it looks like it has a few regions for a table named 't'.
181    MetaTableLocator.setMetaLocation(rs0.getZooKeeper(),
182      rs0.getServerName(), RegionState.State.OPEN);
183    final TableName tableName = TableName.valueOf(name.getMethodName());
184    Result [] results = new Result [] {
185      MetaMockingUtil.getMetaTableRowResult(
186        new HRegionInfo(tableName, HConstants.EMPTY_START_ROW, HBaseTestingUtility.KEYS[1]),
187        rs2.getServerName()),
188      MetaMockingUtil.getMetaTableRowResult(
189        new HRegionInfo(tableName, HBaseTestingUtility.KEYS[1], HBaseTestingUtility.KEYS[2]),
190        rs2.getServerName()),
191      MetaMockingUtil.getMetaTableRowResult(new HRegionInfo(tableName, HBaseTestingUtility.KEYS[2],
192          HConstants.EMPTY_END_ROW),
193        rs2.getServerName())
194    };
195    rs1.setNextResults(HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), results);
196
197    // Create master.  Subclass to override a few methods so we can insert mocks
198    // and get notification on transitions.  We need to fake out any rpcs the
199    // master does opening/closing regions.  Also need to fake out the address
200    // of the 'remote' mocked up regionservers.
201    // Insert a mock for the connection, use TESTUTIL.getConfiguration rather than
202    // the conf from the master; the conf will already have an ClusterConnection
203    // associate so the below mocking of a connection will fail.
204    final ClusterConnection mockedConnection = HConnectionTestingUtility.getMockedConnectionAndDecorate(
205        TESTUTIL.getConfiguration(), rs0, rs0, rs0.getServerName(),
206        HRegionInfo.FIRST_META_REGIONINFO);
207    HMaster master = new HMaster(conf) {
208      @Override
209      InetAddress getRemoteInetAddress(final int port, final long serverStartCode)
210      throws UnknownHostException {
211        // Return different address dependent on port passed.
212        if (port > sns.length) {
213          return super.getRemoteInetAddress(port, serverStartCode);
214        }
215        ServerName sn = sns[port];
216        return InetAddress.getByAddress(sn.getHostname(),
217          new byte [] {10, 0, 0, (byte)sn.getPort()});
218      }
219
220      @Override
221      protected void initClusterSchemaService() throws IOException, InterruptedException {}
222
223      @Override
224      protected ServerManager createServerManager(MasterServices master) throws IOException {
225        ServerManager sm = super.createServerManager(master);
226        // Spy on the created servermanager
227        ServerManager spy = Mockito.spy(sm);
228        return spy;
229      }
230
231      @Override
232      public ClusterConnection getConnection() {
233        return mockedConnection;
234      }
235
236      @Override
237      public ClusterConnection getClusterConnection() {
238        return mockedConnection;
239      }
240    };
241    master.start();
242
243    try {
244      // Wait till master is up ready for RPCs.
245      while (!master.serviceStarted) Threads.sleep(10);
246      // Fake master that there are regionservers out there.  Report in.
247      for (int i = 0; i < sns.length; i++) {
248        RegionServerReportRequest.Builder request = RegionServerReportRequest.newBuilder();;
249        ServerName sn = ServerName.parseVersionedServerName(sns[i].getVersionedBytes());
250        request.setServer(ProtobufUtil.toServerName(sn));
251        request.setLoad(ServerMetricsBuilder.toServerLoad(ServerMetricsBuilder.of(sn)));
252        master.getMasterRpcServices().regionServerReport(null, request.build());
253      }
254       // Master should now come up.
255      while (!master.isInitialized()) {
256        Threads.sleep(100);
257      }
258      assertTrue(master.isInitialized());
259    } finally {
260      rs0.stop("Test is done");
261      rs1.stop("Test is done");
262      rs2.stop("Test is done");
263      master.stopMaster();
264      master.join();
265    }
266  }
267
268  @Ignore @Test // Disabled since HBASE-18511. Reenable when master can carry regions.
269  public void testNotPullingDeadRegionServerFromZK()
270      throws IOException, KeeperException, InterruptedException {
271    final Configuration conf = TESTUTIL.getConfiguration();
272    final ServerName newServer = ServerName.valueOf("test.sample", 1, 101);
273    final ServerName deadServer = ServerName.valueOf("test.sample", 1, 100);
274    final MockRegionServer rs0 = new MockRegionServer(conf, newServer);
275
276    HMaster master = new HMaster(conf) {
277      @Override
278      protected MasterMetaBootstrap createMetaBootstrap() {
279        return new MasterMetaBootstrap(this) {
280          @Override
281          protected void assignMetaReplicas()
282              throws IOException, InterruptedException, KeeperException {
283            // Nothing to do.
284          }
285        };
286      }
287
288      @Override
289      protected void initClusterSchemaService() throws IOException, InterruptedException {}
290
291      @Override
292      protected void initializeZKBasedSystemTrackers() throws IOException, InterruptedException,
293          KeeperException, ReplicationException {
294        super.initializeZKBasedSystemTrackers();
295        // Record a newer server in server manager at first
296        getServerManager().recordNewServerWithLock(newServer,
297          new ServerLoad(ServerMetricsBuilder.of(newServer)));
298      }
299
300      @Override
301      public ClusterConnection getConnection() {
302        // Insert a mock for the connection, use TESTUTIL.getConfiguration rather than
303        // the conf from the master; the conf will already have a Connection
304        // associate so the below mocking of a connection will fail.
305        try {
306          return HConnectionTestingUtility.getMockedConnectionAndDecorate(
307            TESTUTIL.getConfiguration(), rs0, rs0, rs0.getServerName(),
308            HRegionInfo.FIRST_META_REGIONINFO);
309        } catch (IOException e) {
310          return null;
311        }
312      }
313    };
314    master.start();
315
316    try {
317      // Wait till master is initialized.
318      while (!master.isInitialized()) Threads.sleep(10);
319      LOG.info("Master is initialized");
320
321      assertFalse("The dead server should not be pulled in",
322        master.getServerManager().isServerOnline(deadServer));
323    } finally {
324      master.stopMaster();
325      master.join();
326    }
327  }
328
329  @Test
330  public void testMasterInitWithSameClientServerZKQuorum() throws Exception {
331    Configuration conf = new Configuration(TESTUTIL.getConfiguration());
332    conf.set(HConstants.CLIENT_ZOOKEEPER_QUORUM, conf.get(ZOOKEEPER_QUORUM));
333    conf.setInt(HConstants.CLIENT_ZOOKEEPER_CLIENT_PORT, TESTUTIL.getZkCluster().getClientPort());
334    HMaster master = new HMaster(conf);
335    master.start();
336    // the master will abort due to IllegalArgumentException so we should finish within 60 seconds
337    master.join();
338  }
339
340  @Test
341  public void testMasterInitWithObserverModeClientZKQuorum() throws Exception {
342    Configuration conf = new Configuration(TESTUTIL.getConfiguration());
343    Assert.assertFalse(Boolean.getBoolean(HConstants.CLIENT_ZOOKEEPER_OBSERVER_MODE));
344    // set client ZK to some non-existing address and make sure server won't access client ZK
345    // (server start should not be affected)
346    conf.set(HConstants.CLIENT_ZOOKEEPER_QUORUM, HConstants.LOCALHOST);
347    conf.setInt(HConstants.CLIENT_ZOOKEEPER_CLIENT_PORT,
348      TESTUTIL.getZkCluster().getClientPort() + 1);
349    // settings to allow us not to start additional RS
350    conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 1);
351    conf.setBoolean(LoadBalancer.TABLES_ON_MASTER, true);
352    // main setting for this test case
353    conf.setBoolean(HConstants.CLIENT_ZOOKEEPER_OBSERVER_MODE, true);
354    HMaster master = new HMaster(conf);
355    master.start();
356    while (!master.isInitialized()) {
357      Threads.sleep(200);
358    }
359    Assert.assertNull(master.metaLocationSyncer);
360    Assert.assertNull(master.masterAddressSyncer);
361    master.stopMaster();
362    master.join();
363  }
364}