001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.rsgroup; 019 020import static org.apache.hadoop.hbase.util.Threads.sleep; 021import static org.junit.jupiter.api.Assertions.assertEquals; 022import static org.junit.jupiter.api.Assertions.assertTrue; 023 024import java.util.ArrayList; 025import java.util.HashSet; 026import java.util.List; 027import java.util.Map; 028import java.util.Set; 029import org.apache.hadoop.hbase.HConstants; 030import org.apache.hadoop.hbase.NamespaceDescriptor; 031import org.apache.hadoop.hbase.ServerName; 032import org.apache.hadoop.hbase.TableName; 033import org.apache.hadoop.hbase.Waiter; 034import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; 035import org.apache.hadoop.hbase.client.RegionInfo; 036import org.apache.hadoop.hbase.client.Table; 037import org.apache.hadoop.hbase.client.TableDescriptor; 038import org.apache.hadoop.hbase.client.TableDescriptorBuilder; 039import org.apache.hadoop.hbase.ipc.MetaRWQueueRpcExecutor; 040import org.apache.hadoop.hbase.net.Address; 041import org.apache.hadoop.hbase.testclassification.LargeTests; 042import org.apache.hadoop.hbase.testclassification.RSGroupTests; 043import org.apache.hadoop.hbase.util.Bytes; 044import org.apache.hadoop.hbase.util.JVMClusterUtil; 045import org.junit.jupiter.api.AfterAll; 046import org.junit.jupiter.api.AfterEach; 047import org.junit.jupiter.api.BeforeAll; 048import org.junit.jupiter.api.BeforeEach; 049import org.junit.jupiter.api.Tag; 050import org.junit.jupiter.api.Test; 051import org.junit.jupiter.api.TestInfo; 052import org.slf4j.Logger; 053import org.slf4j.LoggerFactory; 054 055import org.apache.hbase.thirdparty.com.google.common.collect.Sets; 056 057@Tag(RSGroupTests.TAG) 058@Tag(LargeTests.TAG) 059public class TestRSGroupsKillRS extends TestRSGroupsBase { 060 061 private static final Logger LOG = LoggerFactory.getLogger(TestRSGroupsKillRS.class); 062 063 @BeforeAll 064 public static void setUp() throws Exception { 065 // avoid all the handlers blocked when meta is offline, and regionServerReport can not be 066 // processed which causes dead lock. 067 TEST_UTIL.getConfiguration().setInt(HConstants.REGION_SERVER_HIGH_PRIORITY_HANDLER_COUNT, 10); 068 TEST_UTIL.getConfiguration() 069 .setFloat(MetaRWQueueRpcExecutor.META_CALL_QUEUE_READ_SHARE_CONF_KEY, 0.5f); 070 setUpTestBeforeClass(); 071 } 072 073 @AfterAll 074 public static void tearDown() throws Exception { 075 tearDownAfterClass(); 076 } 077 078 @BeforeEach 079 public void beforeMethod(TestInfo testInfo) throws Exception { 080 setUpBeforeMethod(testInfo); 081 } 082 083 @AfterEach 084 public void afterMethod() throws Exception { 085 tearDownAfterMethod(); 086 } 087 088 @Test 089 public void testKillRS() throws Exception { 090 RSGroupInfo appInfo = addGroup("appInfo", 1); 091 final TableName tableName = 092 TableName.valueOf(TABLE_PREFIX + "_ns", getNameWithoutIndex(name.getMethodName())); 093 ADMIN.createNamespace(NamespaceDescriptor.create(tableName.getNamespaceAsString()) 094 .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, appInfo.getName()).build()); 095 final TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName) 096 .setColumnFamily(ColumnFamilyDescriptorBuilder.of("f")).build(); 097 ADMIN.createTable(desc); 098 // wait for created table to be assigned 099 TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() { 100 @Override 101 public boolean evaluate() throws Exception { 102 return getTableRegionMap().get(desc.getTableName()) != null; 103 } 104 }); 105 106 ServerName targetServer = getServerName(appInfo.getServers().iterator().next()); 107 assertEquals(1, ADMIN.getRegions(targetServer).size()); 108 109 try { 110 // stopping may cause an exception 111 // due to the connection loss 112 ADMIN.stopRegionServer(targetServer.getAddress().toString()); 113 } catch (Exception e) { 114 } 115 // wait until the server is actually down 116 TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() { 117 @Override 118 public boolean evaluate() throws Exception { 119 return !CLUSTER.getClusterMetrics().getLiveServerMetrics().containsKey(targetServer); 120 } 121 }); 122 // there is only one rs in the group and we killed it, so the region can not be online, until 123 // later we add new servers to it. 124 TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() { 125 @Override 126 public boolean evaluate() throws Exception { 127 return !CLUSTER.getClusterMetrics().getRegionStatesInTransition().isEmpty(); 128 } 129 }); 130 Set<Address> newServers = Sets.newHashSet(); 131 newServers.add(ADMIN.getRSGroup(RSGroupInfo.DEFAULT_GROUP).getServers().iterator().next()); 132 ADMIN.moveServersToRSGroup(newServers, appInfo.getName()); 133 134 // Make sure all the table's regions get reassigned 135 // disabling the table guarantees no conflicting assign/unassign (ie SSH) happens 136 ADMIN.disableTable(tableName); 137 ADMIN.enableTable(tableName); 138 139 // wait for region to be assigned 140 TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() { 141 @Override 142 public boolean evaluate() throws Exception { 143 return CLUSTER.getClusterMetrics().getRegionStatesInTransition().isEmpty(); 144 } 145 }); 146 147 ServerName targetServer1 = getServerName(newServers.iterator().next()); 148 assertEquals(1, ADMIN.getRegions(targetServer1).size()); 149 assertEquals(tableName, ADMIN.getRegions(targetServer1).get(0).getTable()); 150 } 151 152 @Test 153 public void testKillAllRSInGroup() throws Exception { 154 // create a rsgroup and move two regionservers to it 155 String groupName = "my_group"; 156 int groupRSCount = 2; 157 addGroup(groupName, groupRSCount); 158 159 // create a table, and move it to my_group 160 Table t = TEST_UTIL.createMultiRegionTable(tableName, Bytes.toBytes("f"), 5); 161 TEST_UTIL.loadTable(t, Bytes.toBytes("f")); 162 Set<TableName> toAddTables = new HashSet<>(); 163 toAddTables.add(tableName); 164 ADMIN.setRSGroup(toAddTables, groupName); 165 assertTrue( 166 ADMIN.getConfiguredNamespacesAndTablesInRSGroup(groupName).getSecond().contains(tableName)); 167 TEST_UTIL.waitTableAvailable(tableName, 30000); 168 169 // check my_group servers and table regions 170 Set<Address> servers = ADMIN.getRSGroup(groupName).getServers(); 171 assertEquals(2, servers.size()); 172 LOG.debug("group servers {}", servers); 173 for (RegionInfo tr : MASTER.getAssignmentManager().getRegionStates() 174 .getRegionsOfTable(tableName)) { 175 assertTrue(servers.contains(MASTER.getAssignmentManager().getRegionStates() 176 .getRegionAssignments().get(tr).getAddress())); 177 } 178 179 // Move a region, to ensure there exists a region whose 'lastHost' is in my_group 180 // ('lastHost' of other regions are in 'default' group) 181 // and check if all table regions are online 182 List<ServerName> gsn = new ArrayList<>(); 183 for (Address addr : servers) { 184 gsn.add(getServerName(addr)); 185 } 186 assertEquals(2, gsn.size()); 187 for (Map.Entry<RegionInfo, ServerName> entry : MASTER.getAssignmentManager().getRegionStates() 188 .getRegionAssignments().entrySet()) { 189 if (entry.getKey().getTable().equals(tableName)) { 190 LOG.debug("move region {} from {} to {}", entry.getKey().getRegionNameAsString(), 191 entry.getValue(), gsn.get(1 - gsn.indexOf(entry.getValue()))); 192 TEST_UTIL.moveRegionAndWait(entry.getKey(), gsn.get(1 - gsn.indexOf(entry.getValue()))); 193 break; 194 } 195 } 196 TEST_UTIL.waitTableAvailable(tableName, 30000); 197 198 // case 1: stop all the regionservers in my_group, and restart a regionserver in my_group, 199 // and then check if all table regions are online 200 for (Address addr : ADMIN.getRSGroup(groupName).getServers()) { 201 TEST_UTIL.getMiniHBaseCluster().stopRegionServer(getServerName(addr)); 202 } 203 // better wait for a while for region reassign 204 sleep(10000); 205 assertEquals(NUM_SLAVES_BASE - gsn.size(), 206 TEST_UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().size()); 207 TEST_UTIL.getMiniHBaseCluster().startRegionServer(gsn.get(0).getHostname(), 208 gsn.get(0).getPort()); 209 assertEquals(NUM_SLAVES_BASE - gsn.size() + 1, 210 TEST_UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().size()); 211 TEST_UTIL.waitTableAvailable(tableName, 30000); 212 213 // case 2: stop all the regionservers in my_group, and move another 214 // regionserver(from the 'default' group) to my_group, 215 // and then check if all table regions are online 216 for (JVMClusterUtil.RegionServerThread rst : TEST_UTIL.getMiniHBaseCluster() 217 .getLiveRegionServerThreads()) { 218 if (rst.getRegionServer().getServerName().getAddress().equals(gsn.get(0).getAddress())) { 219 TEST_UTIL.getMiniHBaseCluster().stopRegionServer(rst.getRegionServer().getServerName()); 220 break; 221 } 222 } 223 sleep(10000); 224 assertEquals(NUM_SLAVES_BASE - gsn.size(), 225 TEST_UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().size()); 226 ServerName newServer = MASTER.getServerManager().getOnlineServersList().get(0); 227 ADMIN.moveServersToRSGroup(Sets.newHashSet(newServer.getAddress()), groupName); 228 // wait and check if table regions are online 229 TEST_UTIL.waitTableAvailable(tableName, 30000); 230 } 231 232}