001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNotEquals;
022import static org.junit.Assert.assertTrue;
023
024import java.io.IOException;
025import java.util.List;
026import java.util.Map;
027import org.apache.hadoop.hbase.HBaseClassTestRule;
028import org.apache.hadoop.hbase.HBaseTestingUtility;
029import org.apache.hadoop.hbase.HConstants;
030import org.apache.hadoop.hbase.MetaTableAccessor;
031import org.apache.hadoop.hbase.MiniHBaseCluster;
032import org.apache.hadoop.hbase.ServerName;
033import org.apache.hadoop.hbase.TableExistsException;
034import org.apache.hadoop.hbase.TableName;
035import org.apache.hadoop.hbase.client.RegionInfo;
036import org.apache.hadoop.hbase.testclassification.LargeTests;
037import org.apache.hadoop.hbase.testclassification.MasterTests;
038import org.apache.hadoop.hbase.util.Bytes;
039import org.apache.hadoop.hbase.util.JVMClusterUtil;
040import org.apache.hadoop.hbase.util.Threads;
041import org.junit.After;
042import org.junit.ClassRule;
043import org.junit.Test;
044import org.junit.experimental.categories.Category;
045import org.slf4j.Logger;
046import org.slf4j.LoggerFactory;
047
048@Category({ MasterTests.class, LargeTests.class })
049public class TestRestartCluster {
050
051  @ClassRule
052  public static final HBaseClassTestRule CLASS_RULE =
053      HBaseClassTestRule.forClass(TestRestartCluster.class);
054
055  private static final Logger LOG = LoggerFactory.getLogger(TestRestartCluster.class);
056  private HBaseTestingUtility UTIL = new HBaseTestingUtility();
057
058  private static final TableName[] TABLES = {
059      TableName.valueOf("restartTableOne"),
060      TableName.valueOf("restartTableTwo"),
061      TableName.valueOf("restartTableThree")
062  };
063  private static final byte[] FAMILY = Bytes.toBytes("family");
064
065  @After public void tearDown() throws Exception {
066    UTIL.shutdownMiniCluster();
067  }
068
069  @Test
070  public void testClusterRestart() throws Exception {
071    UTIL.startMiniCluster(3);
072    while (!UTIL.getMiniHBaseCluster().getMaster().isInitialized()) {
073      Threads.sleep(1);
074    }
075    LOG.info("\n\nCreating tables");
076    for(TableName TABLE : TABLES) {
077      UTIL.createTable(TABLE, FAMILY);
078    }
079    for(TableName TABLE : TABLES) {
080      UTIL.waitTableEnabled(TABLE);
081    }
082
083    List<RegionInfo> allRegions = MetaTableAccessor.getAllRegions(UTIL.getConnection(), false);
084    assertEquals(4, allRegions.size());
085
086    LOG.info("\n\nShutting down cluster");
087    UTIL.shutdownMiniHBaseCluster();
088
089    LOG.info("\n\nSleeping a bit");
090    Thread.sleep(2000);
091
092    LOG.info("\n\nStarting cluster the second time");
093    UTIL.restartHBaseCluster(3);
094
095    // Need to use a new 'Configuration' so we make a new Connection.
096    // Otherwise we're reusing an Connection that has gone stale because
097    // the shutdown of the cluster also called shut of the connection.
098    allRegions = MetaTableAccessor.getAllRegions(UTIL.getConnection(), false);
099    assertEquals(4, allRegions.size());
100    LOG.info("\n\nWaiting for tables to be available");
101    for(TableName TABLE: TABLES) {
102      try {
103        UTIL.createTable(TABLE, FAMILY);
104        assertTrue("Able to create table that should already exist", false);
105      } catch(TableExistsException tee) {
106        LOG.info("Table already exists as expected");
107      }
108      UTIL.waitTableAvailable(TABLE);
109    }
110  }
111
112  /**
113   * This tests retaining assignments on a cluster restart
114   */
115  @Test
116  public void testRetainAssignmentOnRestart() throws Exception {
117    UTIL.startMiniCluster(2);
118    // Turn off balancer
119    UTIL.getMiniHBaseCluster().getMaster().getMasterRpcServices().synchronousBalanceSwitch(false);
120    LOG.info("\n\nCreating tables");
121    for (TableName TABLE : TABLES) {
122      UTIL.createTable(TABLE, FAMILY);
123    }
124    for (TableName TABLE : TABLES) {
125      UTIL.waitTableEnabled(TABLE);
126    }
127
128    HMaster master = UTIL.getMiniHBaseCluster().getMaster();
129    UTIL.waitUntilNoRegionsInTransition(120000);
130
131    // We don't have to use SnapshotOfRegionAssignmentFromMeta.
132    // We use it here because AM used to use it to load all user region placements
133    SnapshotOfRegionAssignmentFromMeta snapshot = new SnapshotOfRegionAssignmentFromMeta(
134      master.getConnection());
135    snapshot.initialize();
136    Map<RegionInfo, ServerName> regionToRegionServerMap
137      = snapshot.getRegionToRegionServerMap();
138
139    MiniHBaseCluster cluster = UTIL.getHBaseCluster();
140    List<JVMClusterUtil.RegionServerThread> threads = cluster.getLiveRegionServerThreads();
141    assertEquals(2, threads.size());
142    int[] rsPorts = new int[3];
143    for (int i = 0; i < 2; i++) {
144      rsPorts[i] = threads.get(i).getRegionServer().getServerName().getPort();
145    }
146    rsPorts[2] = cluster.getMaster().getServerName().getPort();
147    for (ServerName serverName: regionToRegionServerMap.values()) {
148      boolean found = false; // Test only, no need to optimize
149      for (int k = 0; k < 3 && !found; k++) {
150        found = serverName.getPort() == rsPorts[k];
151      }
152      assertTrue(found);
153    }
154
155    LOG.info("\n\nShutting down HBase cluster");
156    cluster.stopMaster(0);
157    cluster.shutdown();
158    cluster.waitUntilShutDown();
159
160    LOG.info("\n\nSleeping a bit");
161    Thread.sleep(2000);
162
163    LOG.info("\n\nStarting cluster the second time with the same ports");
164    try {
165      cluster.getConf().setInt(
166          ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 3);
167      master = cluster.startMaster().getMaster();
168      for (int i = 0; i < 3; i++) {
169        cluster.getConf().setInt(HConstants.REGIONSERVER_PORT, rsPorts[i]);
170        cluster.startRegionServer();
171      }
172    } finally {
173      // Reset region server port so as not to conflict with other tests
174      cluster.getConf().setInt(HConstants.REGIONSERVER_PORT, 0);
175      cluster.getConf().setInt(
176        ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 2);
177    }
178
179    // Make sure live regionservers are on the same host/port
180    List<ServerName> localServers = master.getServerManager().getOnlineServersList();
181    assertEquals(3, localServers.size());
182    for (int i = 0; i < 3; i++) {
183      boolean found = false;
184      for (ServerName serverName: localServers) {
185        if (serverName.getPort() == rsPorts[i]) {
186          found = true;
187          break;
188        }
189      }
190      assertTrue(found);
191    }
192
193    // Wait till master is initialized and all regions are assigned
194    for (TableName TABLE : TABLES) {
195      UTIL.waitTableAvailable(TABLE);
196    }
197
198    snapshot = new SnapshotOfRegionAssignmentFromMeta(master.getConnection());
199    snapshot.initialize();
200    Map<RegionInfo, ServerName> newRegionToRegionServerMap =
201      snapshot.getRegionToRegionServerMap();
202    assertEquals(regionToRegionServerMap.size(), newRegionToRegionServerMap.size());
203    for (Map.Entry<RegionInfo, ServerName> entry : newRegionToRegionServerMap.entrySet()) {
204      if (TableName.NAMESPACE_TABLE_NAME.equals(entry.getKey().getTable())) {
205        continue;
206      }
207      ServerName oldServer = regionToRegionServerMap.get(entry.getKey());
208      ServerName currentServer = entry.getValue();
209      LOG.info(
210        "Key=" + entry.getKey() + " oldServer=" + oldServer + ", currentServer=" + currentServer);
211      assertEquals(entry.getKey().toString(), oldServer.getAddress(), currentServer.getAddress());
212      assertNotEquals(oldServer.getStartcode(), currentServer.getStartcode());
213    }
214  }
215
216  @Test
217  public void testNewStartedRegionServerVersion() throws Exception {
218    UTIL.startMiniCluster(1);
219
220    // Start 3 new region server
221    Thread t = new Thread(() -> {
222      for (int i = 0; i < 3; i++) {
223        try {
224          JVMClusterUtil.RegionServerThread newRS = UTIL.getMiniHBaseCluster().startRegionServer();
225          newRS.waitForServerOnline();
226        } catch (IOException e) {
227          LOG.error("Failed to start a new RS", e);
228        }
229      }
230    });
231    t.start();
232
233    HMaster master = UTIL.getMiniHBaseCluster().getMaster();
234    while (t.isAlive()) {
235      List<ServerName> serverNames = master.getServerManager().getOnlineServersList();
236      for (ServerName serverName : serverNames) {
237        assertNotEquals(0, master.getServerManager().getServerVersion(serverName));
238      }
239      Thread.sleep(100);
240    }
241  }
242}