001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNotEquals;
022import static org.junit.Assert.assertNotNull;
023import static org.junit.Assert.assertTrue;
024
025import java.io.IOException;
026import java.util.List;
027import org.apache.hadoop.hbase.HBaseClassTestRule;
028import org.apache.hadoop.hbase.HBaseTestingUtility;
029import org.apache.hadoop.hbase.HConstants;
030import org.apache.hadoop.hbase.MetaTableAccessor;
031import org.apache.hadoop.hbase.MiniHBaseCluster;
032import org.apache.hadoop.hbase.ServerName;
033import org.apache.hadoop.hbase.TableExistsException;
034import org.apache.hadoop.hbase.TableName;
035import org.apache.hadoop.hbase.client.RegionInfo;
036import org.apache.hadoop.hbase.testclassification.LargeTests;
037import org.apache.hadoop.hbase.testclassification.MasterTests;
038import org.apache.hadoop.hbase.util.Bytes;
039import org.apache.hadoop.hbase.util.JVMClusterUtil;
040import org.apache.hadoop.hbase.util.Threads;
041import org.junit.After;
042import org.junit.ClassRule;
043import org.junit.Test;
044import org.junit.experimental.categories.Category;
045import org.slf4j.Logger;
046import org.slf4j.LoggerFactory;
047
048@Category({ MasterTests.class, LargeTests.class })
049public class TestRestartCluster {
050
051  @ClassRule
052  public static final HBaseClassTestRule CLASS_RULE =
053      HBaseClassTestRule.forClass(TestRestartCluster.class);
054
055  private static final Logger LOG = LoggerFactory.getLogger(TestRestartCluster.class);
056  private HBaseTestingUtility UTIL = new HBaseTestingUtility();
057
058  private static final TableName[] TABLES = {
059      TableName.valueOf("restartTableOne"),
060      TableName.valueOf("restartTableTwo"),
061      TableName.valueOf("restartTableThree")
062  };
063  private static final byte[] FAMILY = Bytes.toBytes("family");
064
065  @After public void tearDown() throws Exception {
066    UTIL.shutdownMiniCluster();
067  }
068
069  @Test
070  public void testClusterRestart() throws Exception {
071    UTIL.startMiniCluster(3);
072    while (!UTIL.getMiniHBaseCluster().getMaster().isInitialized()) {
073      Threads.sleep(1);
074    }
075    LOG.info("\n\nCreating tables");
076    for(TableName TABLE : TABLES) {
077      UTIL.createTable(TABLE, FAMILY);
078    }
079    for(TableName TABLE : TABLES) {
080      UTIL.waitTableEnabled(TABLE);
081    }
082
083    List<RegionInfo> allRegions = MetaTableAccessor.getAllRegions(UTIL.getConnection(), false);
084    assertEquals(4, allRegions.size());
085
086    LOG.info("\n\nShutting down cluster");
087    UTIL.shutdownMiniHBaseCluster();
088
089    LOG.info("\n\nSleeping a bit");
090    Thread.sleep(2000);
091
092    LOG.info("\n\nStarting cluster the second time");
093    UTIL.restartHBaseCluster(3);
094
095    // Need to use a new 'Configuration' so we make a new Connection.
096    // Otherwise we're reusing an Connection that has gone stale because
097    // the shutdown of the cluster also called shut of the connection.
098    allRegions = MetaTableAccessor.getAllRegions(UTIL.getConnection(), false);
099    assertEquals(4, allRegions.size());
100    LOG.info("\n\nWaiting for tables to be available");
101    for(TableName TABLE: TABLES) {
102      try {
103        UTIL.createTable(TABLE, FAMILY);
104        assertTrue("Able to create table that should already exist", false);
105      } catch(TableExistsException tee) {
106        LOG.info("Table already exists as expected");
107      }
108      UTIL.waitTableAvailable(TABLE);
109    }
110  }
111
112  /**
113   * This tests retaining assignments on a cluster restart
114   */
115  @Test
116  public void testRoundRobinAssignmentOnRestart() throws Exception {
117    final int regionNum = 10;
118    final int rsNum = 2;
119    UTIL.startMiniCluster(rsNum);
120    // Turn off balancer
121    UTIL.getMiniHBaseCluster().getMaster().getMasterRpcServices().synchronousBalanceSwitch(false);
122    LOG.info("\n\nCreating tables");
123    for (TableName TABLE : TABLES) {
124      UTIL.createMultiRegionTable(TABLE, FAMILY, regionNum);
125    }
126    // Wait until all regions are assigned
127    for (TableName TABLE : TABLES) {
128      UTIL.waitTableEnabled(TABLE);
129    }
130    UTIL.waitUntilNoRegionsInTransition(120000);
131
132    MiniHBaseCluster cluster = UTIL.getHBaseCluster();
133    List<JVMClusterUtil.RegionServerThread> threads = cluster.getLiveRegionServerThreads();
134    assertEquals(rsNum, threads.size());
135
136    ServerName testServer = threads.get(0).getRegionServer().getServerName();
137    int port = testServer.getPort();
138    List<RegionInfo> regionInfos =
139        cluster.getMaster().getAssignmentManager().getRegionStates().getServerNode(testServer)
140            .getRegionInfoList();
141    LOG.debug("RegionServer {} has {} regions", testServer, regionInfos.size());
142    assertTrue(regionInfos.size() >= (TABLES.length * regionNum / rsNum));
143
144    // Restart 1 regionserver
145    cluster.stopRegionServer(testServer);
146    cluster.waitForRegionServerToStop(testServer, 60000);
147    cluster.getConf().setInt(HConstants.REGIONSERVER_PORT, port);
148    cluster.startRegionServer();
149
150    HMaster master = UTIL.getMiniHBaseCluster().getMaster();
151    List<ServerName> localServers = master.getServerManager().getOnlineServersList();
152    ServerName newTestServer = null;
153    for (ServerName serverName : localServers) {
154      if (serverName.getAddress().equals(testServer.getAddress())) {
155        newTestServer = serverName;
156        break;
157      }
158    }
159    assertNotNull(newTestServer);
160
161    // Wait until all regions are assigned
162    for (TableName TABLE : TABLES) {
163      UTIL.waitTableAvailable(TABLE);
164    }
165    UTIL.waitUntilNoRegionsInTransition(60000);
166
167    List<RegionInfo> newRegionInfos =
168        cluster.getMaster().getAssignmentManager().getRegionStates().getServerNode(newTestServer)
169            .getRegionInfoList();
170    LOG.debug("RegionServer {} has {} regions", newTestServer, newRegionInfos.size());
171    assertTrue("Should not retain all regions when restart",
172        newRegionInfos.size() < regionInfos.size());
173  }
174
175  @Test
176  public void testNewStartedRegionServerVersion() throws Exception {
177    UTIL.startMiniCluster(1);
178
179    // Start 3 new region server
180    Thread t = new Thread(() -> {
181      for (int i = 0; i < 3; i++) {
182        try {
183          JVMClusterUtil.RegionServerThread newRS = UTIL.getMiniHBaseCluster().startRegionServer();
184          newRS.waitForServerOnline();
185        } catch (IOException e) {
186          LOG.error("Failed to start a new RS", e);
187        }
188      }
189    });
190    t.start();
191
192    HMaster master = UTIL.getMiniHBaseCluster().getMaster();
193    while (t.isAlive()) {
194      List<ServerName> serverNames = master.getServerManager().getOnlineServersList();
195      for (ServerName serverName : serverNames) {
196        assertNotEquals(0, master.getServerManager().getVersionNumber(serverName));
197      }
198      Thread.sleep(100);
199    }
200  }
201}