001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.rsgroup;
019
020import static org.apache.hadoop.hbase.util.Threads.sleep;
021import static org.junit.jupiter.api.Assertions.assertEquals;
022import static org.junit.jupiter.api.Assertions.assertTrue;
023
024import java.util.ArrayList;
025import java.util.HashSet;
026import java.util.List;
027import java.util.Map;
028import java.util.Set;
029import org.apache.hadoop.hbase.HConstants;
030import org.apache.hadoop.hbase.NamespaceDescriptor;
031import org.apache.hadoop.hbase.ServerName;
032import org.apache.hadoop.hbase.TableName;
033import org.apache.hadoop.hbase.Waiter;
034import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
035import org.apache.hadoop.hbase.client.RegionInfo;
036import org.apache.hadoop.hbase.client.Table;
037import org.apache.hadoop.hbase.client.TableDescriptor;
038import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
039import org.apache.hadoop.hbase.ipc.MetaRWQueueRpcExecutor;
040import org.apache.hadoop.hbase.net.Address;
041import org.apache.hadoop.hbase.testclassification.LargeTests;
042import org.apache.hadoop.hbase.testclassification.RSGroupTests;
043import org.apache.hadoop.hbase.util.Bytes;
044import org.apache.hadoop.hbase.util.JVMClusterUtil;
045import org.junit.jupiter.api.AfterAll;
046import org.junit.jupiter.api.AfterEach;
047import org.junit.jupiter.api.BeforeAll;
048import org.junit.jupiter.api.BeforeEach;
049import org.junit.jupiter.api.Tag;
050import org.junit.jupiter.api.Test;
051import org.junit.jupiter.api.TestInfo;
052import org.slf4j.Logger;
053import org.slf4j.LoggerFactory;
054
055import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
056
057@Tag(RSGroupTests.TAG)
058@Tag(LargeTests.TAG)
059public class TestRSGroupsKillRS extends TestRSGroupsBase {
060
061  private static final Logger LOG = LoggerFactory.getLogger(TestRSGroupsKillRS.class);
062
063  @BeforeAll
064  public static void setUp() throws Exception {
065    // avoid all the handlers blocked when meta is offline, and regionServerReport can not be
066    // processed which causes dead lock.
067    TEST_UTIL.getConfiguration().setInt(HConstants.REGION_SERVER_HIGH_PRIORITY_HANDLER_COUNT, 10);
068    TEST_UTIL.getConfiguration()
069      .setFloat(MetaRWQueueRpcExecutor.META_CALL_QUEUE_READ_SHARE_CONF_KEY, 0.5f);
070    setUpTestBeforeClass();
071  }
072
073  @AfterAll
074  public static void tearDown() throws Exception {
075    tearDownAfterClass();
076  }
077
078  @BeforeEach
079  public void beforeMethod(TestInfo testInfo) throws Exception {
080    setUpBeforeMethod(testInfo);
081  }
082
083  @AfterEach
084  public void afterMethod() throws Exception {
085    tearDownAfterMethod();
086  }
087
088  @Test
089  public void testKillRS() throws Exception {
090    RSGroupInfo appInfo = addGroup("appInfo", 1);
091    final TableName tableName =
092      TableName.valueOf(TABLE_PREFIX + "_ns", getNameWithoutIndex(name.getMethodName()));
093    ADMIN.createNamespace(NamespaceDescriptor.create(tableName.getNamespaceAsString())
094      .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, appInfo.getName()).build());
095    final TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName)
096      .setColumnFamily(ColumnFamilyDescriptorBuilder.of("f")).build();
097    ADMIN.createTable(desc);
098    // wait for created table to be assigned
099    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
100      @Override
101      public boolean evaluate() throws Exception {
102        return getTableRegionMap().get(desc.getTableName()) != null;
103      }
104    });
105
106    ServerName targetServer = getServerName(appInfo.getServers().iterator().next());
107    assertEquals(1, ADMIN.getRegions(targetServer).size());
108
109    try {
110      // stopping may cause an exception
111      // due to the connection loss
112      ADMIN.stopRegionServer(targetServer.getAddress().toString());
113    } catch (Exception e) {
114    }
115    // wait until the server is actually down
116    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
117      @Override
118      public boolean evaluate() throws Exception {
119        return !CLUSTER.getClusterMetrics().getLiveServerMetrics().containsKey(targetServer);
120      }
121    });
122    // there is only one rs in the group and we killed it, so the region can not be online, until
123    // later we add new servers to it.
124    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
125      @Override
126      public boolean evaluate() throws Exception {
127        return !CLUSTER.getClusterMetrics().getRegionStatesInTransition().isEmpty();
128      }
129    });
130    Set<Address> newServers = Sets.newHashSet();
131    newServers.add(ADMIN.getRSGroup(RSGroupInfo.DEFAULT_GROUP).getServers().iterator().next());
132    ADMIN.moveServersToRSGroup(newServers, appInfo.getName());
133
134    // Make sure all the table's regions get reassigned
135    // disabling the table guarantees no conflicting assign/unassign (ie SSH) happens
136    ADMIN.disableTable(tableName);
137    ADMIN.enableTable(tableName);
138
139    // wait for region to be assigned
140    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
141      @Override
142      public boolean evaluate() throws Exception {
143        return CLUSTER.getClusterMetrics().getRegionStatesInTransition().isEmpty();
144      }
145    });
146
147    ServerName targetServer1 = getServerName(newServers.iterator().next());
148    assertEquals(1, ADMIN.getRegions(targetServer1).size());
149    assertEquals(tableName, ADMIN.getRegions(targetServer1).get(0).getTable());
150  }
151
152  @Test
153  public void testKillAllRSInGroup() throws Exception {
154    // create a rsgroup and move two regionservers to it
155    String groupName = "my_group";
156    int groupRSCount = 2;
157    addGroup(groupName, groupRSCount);
158
159    // create a table, and move it to my_group
160    Table t = TEST_UTIL.createMultiRegionTable(tableName, Bytes.toBytes("f"), 5);
161    TEST_UTIL.loadTable(t, Bytes.toBytes("f"));
162    Set<TableName> toAddTables = new HashSet<>();
163    toAddTables.add(tableName);
164    ADMIN.setRSGroup(toAddTables, groupName);
165    assertTrue(
166      ADMIN.getConfiguredNamespacesAndTablesInRSGroup(groupName).getSecond().contains(tableName));
167    TEST_UTIL.waitTableAvailable(tableName, 30000);
168
169    // check my_group servers and table regions
170    Set<Address> servers = ADMIN.getRSGroup(groupName).getServers();
171    assertEquals(2, servers.size());
172    LOG.debug("group servers {}", servers);
173    for (RegionInfo tr : MASTER.getAssignmentManager().getRegionStates()
174      .getRegionsOfTable(tableName)) {
175      assertTrue(servers.contains(MASTER.getAssignmentManager().getRegionStates()
176        .getRegionAssignments().get(tr).getAddress()));
177    }
178
179    // Move a region, to ensure there exists a region whose 'lastHost' is in my_group
180    // ('lastHost' of other regions are in 'default' group)
181    // and check if all table regions are online
182    List<ServerName> gsn = new ArrayList<>();
183    for (Address addr : servers) {
184      gsn.add(getServerName(addr));
185    }
186    assertEquals(2, gsn.size());
187    for (Map.Entry<RegionInfo, ServerName> entry : MASTER.getAssignmentManager().getRegionStates()
188      .getRegionAssignments().entrySet()) {
189      if (entry.getKey().getTable().equals(tableName)) {
190        LOG.debug("move region {} from {} to {}", entry.getKey().getRegionNameAsString(),
191          entry.getValue(), gsn.get(1 - gsn.indexOf(entry.getValue())));
192        TEST_UTIL.moveRegionAndWait(entry.getKey(), gsn.get(1 - gsn.indexOf(entry.getValue())));
193        break;
194      }
195    }
196    TEST_UTIL.waitTableAvailable(tableName, 30000);
197
198    // case 1: stop all the regionservers in my_group, and restart a regionserver in my_group,
199    // and then check if all table regions are online
200    for (Address addr : ADMIN.getRSGroup(groupName).getServers()) {
201      TEST_UTIL.getMiniHBaseCluster().stopRegionServer(getServerName(addr));
202    }
203    // better wait for a while for region reassign
204    sleep(10000);
205    assertEquals(NUM_SLAVES_BASE - gsn.size(),
206      TEST_UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().size());
207    TEST_UTIL.getMiniHBaseCluster().startRegionServer(gsn.get(0).getHostname(),
208      gsn.get(0).getPort());
209    assertEquals(NUM_SLAVES_BASE - gsn.size() + 1,
210      TEST_UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().size());
211    TEST_UTIL.waitTableAvailable(tableName, 30000);
212
213    // case 2: stop all the regionservers in my_group, and move another
214    // regionserver(from the 'default' group) to my_group,
215    // and then check if all table regions are online
216    for (JVMClusterUtil.RegionServerThread rst : TEST_UTIL.getMiniHBaseCluster()
217      .getLiveRegionServerThreads()) {
218      if (rst.getRegionServer().getServerName().getAddress().equals(gsn.get(0).getAddress())) {
219        TEST_UTIL.getMiniHBaseCluster().stopRegionServer(rst.getRegionServer().getServerName());
220        break;
221      }
222    }
223    sleep(10000);
224    assertEquals(NUM_SLAVES_BASE - gsn.size(),
225      TEST_UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().size());
226    ServerName newServer = MASTER.getServerManager().getOnlineServersList().get(0);
227    ADMIN.moveServersToRSGroup(Sets.newHashSet(newServer.getAddress()), groupName);
228    // wait and check if table regions are online
229    TEST_UTIL.waitTableAvailable(tableName, 30000);
230  }
231
232}