001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.master; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertNotEquals; 022import static org.junit.Assert.assertTrue; 023 024import java.io.IOException; 025import java.util.List; 026import java.util.Map; 027import org.apache.hadoop.hbase.HBaseClassTestRule; 028import org.apache.hadoop.hbase.HBaseTestingUtility; 029import org.apache.hadoop.hbase.HConstants; 030import org.apache.hadoop.hbase.MetaTableAccessor; 031import org.apache.hadoop.hbase.MiniHBaseCluster; 032import org.apache.hadoop.hbase.ServerName; 033import org.apache.hadoop.hbase.TableExistsException; 034import org.apache.hadoop.hbase.TableName; 035import org.apache.hadoop.hbase.client.RegionInfo; 036import org.apache.hadoop.hbase.testclassification.LargeTests; 037import org.apache.hadoop.hbase.testclassification.MasterTests; 038import org.apache.hadoop.hbase.util.Bytes; 039import org.apache.hadoop.hbase.util.JVMClusterUtil; 040import org.apache.hadoop.hbase.util.Threads; 041import org.junit.After; 042import org.junit.ClassRule; 043import org.junit.Test; 044import org.junit.experimental.categories.Category; 045import org.slf4j.Logger; 046import org.slf4j.LoggerFactory; 047 048@Category({ MasterTests.class, LargeTests.class }) 049public class TestRestartCluster { 050 051 @ClassRule 052 public static final HBaseClassTestRule CLASS_RULE = 053 HBaseClassTestRule.forClass(TestRestartCluster.class); 054 055 private static final Logger LOG = LoggerFactory.getLogger(TestRestartCluster.class); 056 private HBaseTestingUtility UTIL = new HBaseTestingUtility(); 057 058 private static final TableName[] TABLES = { 059 TableName.valueOf("restartTableOne"), 060 TableName.valueOf("restartTableTwo"), 061 TableName.valueOf("restartTableThree") 062 }; 063 private static final byte[] FAMILY = Bytes.toBytes("family"); 064 065 @After public void tearDown() throws Exception { 066 UTIL.shutdownMiniCluster(); 067 } 068 069 @Test 070 public void testClusterRestart() throws Exception { 071 UTIL.startMiniCluster(3); 072 while (!UTIL.getMiniHBaseCluster().getMaster().isInitialized()) { 073 Threads.sleep(1); 074 } 075 LOG.info("\n\nCreating tables"); 076 for(TableName TABLE : TABLES) { 077 UTIL.createTable(TABLE, FAMILY); 078 } 079 for(TableName TABLE : TABLES) { 080 UTIL.waitTableEnabled(TABLE); 081 } 082 083 List<RegionInfo> allRegions = MetaTableAccessor.getAllRegions(UTIL.getConnection(), false); 084 assertEquals(4, allRegions.size()); 085 086 LOG.info("\n\nShutting down cluster"); 087 UTIL.shutdownMiniHBaseCluster(); 088 089 LOG.info("\n\nSleeping a bit"); 090 Thread.sleep(2000); 091 092 LOG.info("\n\nStarting cluster the second time"); 093 UTIL.restartHBaseCluster(3); 094 095 // Need to use a new 'Configuration' so we make a new Connection. 096 // Otherwise we're reusing an Connection that has gone stale because 097 // the shutdown of the cluster also called shut of the connection. 098 allRegions = MetaTableAccessor.getAllRegions(UTIL.getConnection(), false); 099 assertEquals(4, allRegions.size()); 100 LOG.info("\n\nWaiting for tables to be available"); 101 for(TableName TABLE: TABLES) { 102 try { 103 UTIL.createTable(TABLE, FAMILY); 104 assertTrue("Able to create table that should already exist", false); 105 } catch(TableExistsException tee) { 106 LOG.info("Table already exists as expected"); 107 } 108 UTIL.waitTableAvailable(TABLE); 109 } 110 } 111 112 /** 113 * This tests retaining assignments on a cluster restart 114 */ 115 @Test 116 public void testRetainAssignmentOnRestart() throws Exception { 117 UTIL.startMiniCluster(2); 118 // Turn off balancer 119 UTIL.getMiniHBaseCluster().getMaster().getMasterRpcServices().synchronousBalanceSwitch(false); 120 LOG.info("\n\nCreating tables"); 121 for (TableName TABLE : TABLES) { 122 UTIL.createTable(TABLE, FAMILY); 123 } 124 for (TableName TABLE : TABLES) { 125 UTIL.waitTableEnabled(TABLE); 126 } 127 128 HMaster master = UTIL.getMiniHBaseCluster().getMaster(); 129 UTIL.waitUntilNoRegionsInTransition(120000); 130 131 // We don't have to use SnapshotOfRegionAssignmentFromMeta. 132 // We use it here because AM used to use it to load all user region placements 133 SnapshotOfRegionAssignmentFromMeta snapshot = new SnapshotOfRegionAssignmentFromMeta( 134 master.getConnection()); 135 snapshot.initialize(); 136 Map<RegionInfo, ServerName> regionToRegionServerMap 137 = snapshot.getRegionToRegionServerMap(); 138 139 MiniHBaseCluster cluster = UTIL.getHBaseCluster(); 140 List<JVMClusterUtil.RegionServerThread> threads = cluster.getLiveRegionServerThreads(); 141 assertEquals(2, threads.size()); 142 int[] rsPorts = new int[3]; 143 for (int i = 0; i < 2; i++) { 144 rsPorts[i] = threads.get(i).getRegionServer().getServerName().getPort(); 145 } 146 rsPorts[2] = cluster.getMaster().getServerName().getPort(); 147 for (ServerName serverName: regionToRegionServerMap.values()) { 148 boolean found = false; // Test only, no need to optimize 149 for (int k = 0; k < 3 && !found; k++) { 150 found = serverName.getPort() == rsPorts[k]; 151 } 152 assertTrue(found); 153 } 154 155 LOG.info("\n\nShutting down HBase cluster"); 156 cluster.stopMaster(0); 157 cluster.shutdown(); 158 cluster.waitUntilShutDown(); 159 160 LOG.info("\n\nSleeping a bit"); 161 Thread.sleep(2000); 162 163 LOG.info("\n\nStarting cluster the second time with the same ports"); 164 try { 165 cluster.getConf().setInt( 166 ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 3); 167 master = cluster.startMaster().getMaster(); 168 for (int i = 0; i < 3; i++) { 169 cluster.getConf().setInt(HConstants.REGIONSERVER_PORT, rsPorts[i]); 170 cluster.startRegionServer(); 171 } 172 } finally { 173 // Reset region server port so as not to conflict with other tests 174 cluster.getConf().setInt(HConstants.REGIONSERVER_PORT, 0); 175 cluster.getConf().setInt( 176 ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 2); 177 } 178 179 // Make sure live regionservers are on the same host/port 180 List<ServerName> localServers = master.getServerManager().getOnlineServersList(); 181 assertEquals(3, localServers.size()); 182 for (int i = 0; i < 3; i++) { 183 boolean found = false; 184 for (ServerName serverName: localServers) { 185 if (serverName.getPort() == rsPorts[i]) { 186 found = true; 187 break; 188 } 189 } 190 assertTrue(found); 191 } 192 193 // Wait till master is initialized and all regions are assigned 194 for (TableName TABLE : TABLES) { 195 UTIL.waitTableAvailable(TABLE); 196 } 197 198 snapshot = new SnapshotOfRegionAssignmentFromMeta(master.getConnection()); 199 snapshot.initialize(); 200 Map<RegionInfo, ServerName> newRegionToRegionServerMap = 201 snapshot.getRegionToRegionServerMap(); 202 assertEquals(regionToRegionServerMap.size(), newRegionToRegionServerMap.size()); 203 for (Map.Entry<RegionInfo, ServerName> entry : newRegionToRegionServerMap.entrySet()) { 204 if (TableName.NAMESPACE_TABLE_NAME.equals(entry.getKey().getTable())) { 205 continue; 206 } 207 ServerName oldServer = regionToRegionServerMap.get(entry.getKey()); 208 ServerName currentServer = entry.getValue(); 209 LOG.info( 210 "Key=" + entry.getKey() + " oldServer=" + oldServer + ", currentServer=" + currentServer); 211 assertEquals(entry.getKey().toString(), oldServer.getAddress(), currentServer.getAddress()); 212 assertNotEquals(oldServer.getStartcode(), currentServer.getStartcode()); 213 } 214 } 215 216 @Test 217 public void testNewStartedRegionServerVersion() throws Exception { 218 UTIL.startMiniCluster(1); 219 220 // Start 3 new region server 221 Thread t = new Thread(() -> { 222 for (int i = 0; i < 3; i++) { 223 try { 224 JVMClusterUtil.RegionServerThread newRS = UTIL.getMiniHBaseCluster().startRegionServer(); 225 newRS.waitForServerOnline(); 226 } catch (IOException e) { 227 LOG.error("Failed to start a new RS", e); 228 } 229 } 230 }); 231 t.start(); 232 233 HMaster master = UTIL.getMiniHBaseCluster().getMaster(); 234 while (t.isAlive()) { 235 List<ServerName> serverNames = master.getServerManager().getOnlineServersList(); 236 for (ServerName serverName : serverNames) { 237 assertNotEquals(0, master.getServerManager().getServerVersion(serverName)); 238 } 239 Thread.sleep(100); 240 } 241 } 242}