001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.rsgroup; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertFalse; 022import static org.junit.Assert.assertTrue; 023import static org.junit.Assert.fail; 024 025import java.io.IOException; 026import java.util.ArrayList; 027import java.util.EnumSet; 028import java.util.Iterator; 029import java.util.List; 030import java.util.Map; 031import java.util.Set; 032import org.apache.hadoop.hbase.ClusterMetrics.Option; 033import org.apache.hadoop.hbase.HBaseClassTestRule; 034import org.apache.hadoop.hbase.ServerName; 035import org.apache.hadoop.hbase.TableName; 036import org.apache.hadoop.hbase.Waiter; 037import org.apache.hadoop.hbase.client.RegionInfo; 038import org.apache.hadoop.hbase.constraint.ConstraintException; 039import org.apache.hadoop.hbase.net.Address; 040import org.apache.hadoop.hbase.testclassification.MediumTests; 041import org.apache.hadoop.hbase.util.Bytes; 042import org.junit.After; 043import org.junit.AfterClass; 044import org.junit.Assert; 045import org.junit.Before; 046import org.junit.BeforeClass; 047import org.junit.ClassRule; 048import org.junit.Test; 049import org.junit.experimental.categories.Category; 050import org.slf4j.Logger; 051import org.slf4j.LoggerFactory; 052 053import org.apache.hbase.thirdparty.com.google.common.collect.Sets; 054 055@Category({ MediumTests.class }) 056public class TestRSGroupsAdmin2 extends TestRSGroupsBase { 057 058 @ClassRule 059 public static final HBaseClassTestRule CLASS_RULE = 060 HBaseClassTestRule.forClass(TestRSGroupsAdmin2.class); 061 062 protected static final Logger LOG = LoggerFactory.getLogger(TestRSGroupsAdmin2.class); 063 064 @BeforeClass 065 public static void setUp() throws Exception { 066 setUpTestBeforeClass(); 067 } 068 069 @AfterClass 070 public static void tearDown() throws Exception { 071 tearDownAfterClass(); 072 } 073 074 @Before 075 public void beforeMethod() throws Exception { 076 setUpBeforeMethod(); 077 } 078 079 @After 080 public void afterMethod() throws Exception { 081 tearDownAfterMethod(); 082 } 083 084 @Test 085 public void testRegionMove() throws Exception { 086 final RSGroupInfo newGroup = addGroup(getGroupName(name.getMethodName()), 1); 087 final byte[] familyNameBytes = Bytes.toBytes("f"); 088 // All the regions created below will be assigned to the default group. 089 TEST_UTIL.createMultiRegionTable(tableName, familyNameBytes, 6); 090 TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() { 091 @Override 092 public boolean evaluate() throws Exception { 093 List<String> regions = getTableRegionMap().get(tableName); 094 if (regions == null) { 095 return false; 096 } 097 098 return getTableRegionMap().get(tableName).size() >= 6; 099 } 100 }); 101 102 // get target region to move 103 Map<ServerName, List<String>> assignMap = getTableServerRegionMap().get(tableName); 104 String targetRegion = null; 105 for (ServerName server : assignMap.keySet()) { 106 targetRegion = assignMap.get(server).size() > 0 ? assignMap.get(server).get(0) : null; 107 if (targetRegion != null) { 108 break; 109 } 110 } 111 // get server which is not a member of new group 112 ServerName tmpTargetServer = null; 113 for (ServerName server : admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)) 114 .getLiveServerMetrics().keySet()) { 115 if (!newGroup.containsServer(server.getAddress())) { 116 tmpTargetServer = server; 117 break; 118 } 119 } 120 final ServerName targetServer = tmpTargetServer; 121 // move target server to group 122 rsGroupAdmin.moveServers(Sets.newHashSet(targetServer.getAddress()), newGroup.getName()); 123 TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() { 124 @Override 125 public boolean evaluate() throws Exception { 126 return admin.getRegions(targetServer).size() <= 0; 127 } 128 }); 129 130 // Lets move this region to the new group. 131 TEST_UTIL.getAdmin() 132 .move(Bytes.toBytes(RegionInfo.encodeRegionName(Bytes.toBytes(targetRegion))), targetServer); 133 TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() { 134 @Override 135 public boolean evaluate() throws Exception { 136 return getTableRegionMap().get(tableName) != null && 137 getTableRegionMap().get(tableName).size() == 6 && 138 admin.getClusterMetrics(EnumSet.of(Option.REGIONS_IN_TRANSITION)) 139 .getRegionStatesInTransition().size() < 1; 140 } 141 }); 142 143 // verify that targetServer didn't open it 144 for (RegionInfo region : admin.getRegions(targetServer)) { 145 if (targetRegion.equals(region.getRegionNameAsString())) { 146 fail("Target server opened region"); 147 } 148 } 149 } 150 151 @Test 152 public void testRegionServerMove() throws IOException, InterruptedException { 153 int initNumGroups = rsGroupAdmin.listRSGroups().size(); 154 RSGroupInfo appInfo = addGroup(getGroupName(name.getMethodName()), 1); 155 RSGroupInfo adminInfo = addGroup(getGroupName(name.getMethodName()), 1); 156 RSGroupInfo dInfo = rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP); 157 Assert.assertEquals(initNumGroups + 2, rsGroupAdmin.listRSGroups().size()); 158 assertEquals(1, adminInfo.getServers().size()); 159 assertEquals(1, appInfo.getServers().size()); 160 assertEquals(getNumServers() - 2, dInfo.getServers().size()); 161 rsGroupAdmin.moveServers(appInfo.getServers(), RSGroupInfo.DEFAULT_GROUP); 162 rsGroupAdmin.removeRSGroup(appInfo.getName()); 163 rsGroupAdmin.moveServers(adminInfo.getServers(), RSGroupInfo.DEFAULT_GROUP); 164 rsGroupAdmin.removeRSGroup(adminInfo.getName()); 165 Assert.assertEquals(rsGroupAdmin.listRSGroups().size(), initNumGroups); 166 } 167 168 @Test 169 public void testMoveServers() throws Exception { 170 // create groups and assign servers 171 addGroup("bar", 3); 172 rsGroupAdmin.addRSGroup("foo"); 173 174 RSGroupInfo barGroup = rsGroupAdmin.getRSGroupInfo("bar"); 175 RSGroupInfo fooGroup = rsGroupAdmin.getRSGroupInfo("foo"); 176 assertEquals(3, barGroup.getServers().size()); 177 assertEquals(0, fooGroup.getServers().size()); 178 179 // test fail bogus server move 180 try { 181 rsGroupAdmin.moveServers(Sets.newHashSet(Address.fromString("foo:9999")), "foo"); 182 fail("Bogus servers shouldn't have been successfully moved."); 183 } catch (IOException ex) { 184 String exp = "Source RSGroup for server foo:9999 does not exist."; 185 String msg = "Expected '" + exp + "' in exception message: "; 186 assertTrue(msg + " " + ex.getMessage(), ex.getMessage().contains(exp)); 187 } 188 189 // test success case 190 LOG.info("moving servers " + barGroup.getServers() + " to group foo"); 191 rsGroupAdmin.moveServers(barGroup.getServers(), fooGroup.getName()); 192 193 barGroup = rsGroupAdmin.getRSGroupInfo("bar"); 194 fooGroup = rsGroupAdmin.getRSGroupInfo("foo"); 195 assertEquals(0, barGroup.getServers().size()); 196 assertEquals(3, fooGroup.getServers().size()); 197 198 LOG.info("moving servers " + fooGroup.getServers() + " to group default"); 199 rsGroupAdmin.moveServers(fooGroup.getServers(), RSGroupInfo.DEFAULT_GROUP); 200 201 TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() { 202 @Override 203 public boolean evaluate() throws Exception { 204 return getNumServers() == rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP) 205 .getServers().size(); 206 } 207 }); 208 209 fooGroup = rsGroupAdmin.getRSGroupInfo("foo"); 210 assertEquals(0, fooGroup.getServers().size()); 211 212 // test group removal 213 LOG.info("Remove group " + barGroup.getName()); 214 rsGroupAdmin.removeRSGroup(barGroup.getName()); 215 Assert.assertEquals(null, rsGroupAdmin.getRSGroupInfo(barGroup.getName())); 216 LOG.info("Remove group " + fooGroup.getName()); 217 rsGroupAdmin.removeRSGroup(fooGroup.getName()); 218 Assert.assertEquals(null, rsGroupAdmin.getRSGroupInfo(fooGroup.getName())); 219 } 220 221 @Test 222 public void testRemoveServers() throws Exception { 223 LOG.info("testRemoveServers"); 224 final RSGroupInfo newGroup = addGroup(getGroupName(name.getMethodName()), 3); 225 Iterator<Address> iterator = newGroup.getServers().iterator(); 226 ServerName targetServer = getServerName(iterator.next()); 227 228 // remove online servers 229 try { 230 rsGroupAdmin.removeServers(Sets.newHashSet(targetServer.getAddress())); 231 fail("Online servers shouldn't have been successfully removed."); 232 } catch (IOException ex) { 233 String exp = 234 "Server " + targetServer.getAddress() + " is an online server, not allowed to remove."; 235 String msg = "Expected '" + exp + "' in exception message: "; 236 assertTrue(msg + " " + ex.getMessage(), ex.getMessage().contains(exp)); 237 } 238 assertTrue(newGroup.getServers().contains(targetServer.getAddress())); 239 240 // remove dead servers 241 NUM_DEAD_SERVERS = cluster.getClusterMetrics().getDeadServerNames().size(); 242 try { 243 // stopping may cause an exception 244 // due to the connection loss 245 LOG.info("stopping server " + targetServer.getServerName()); 246 admin.stopRegionServer(targetServer.getAddress().toString()); 247 NUM_DEAD_SERVERS++; 248 } catch (Exception e) { 249 } 250 251 // wait for stopped regionserver to dead server list 252 TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() { 253 @Override 254 public boolean evaluate() throws Exception { 255 return !master.getServerManager().areDeadServersInProgress() && 256 cluster.getClusterMetrics().getDeadServerNames().size() == NUM_DEAD_SERVERS; 257 } 258 }); 259 260 try { 261 rsGroupAdmin.removeServers(Sets.newHashSet(targetServer.getAddress())); 262 fail("Dead servers shouldn't have been successfully removed."); 263 } catch (IOException ex) { 264 String exp = "Server " + targetServer.getAddress() + " is on the dead servers list," + 265 " Maybe it will come back again, not allowed to remove."; 266 String msg = "Expected '" + exp + "' in exception message: "; 267 assertTrue(msg + " " + ex.getMessage(), ex.getMessage().contains(exp)); 268 } 269 assertTrue(newGroup.getServers().contains(targetServer.getAddress())); 270 271 // remove decommissioned servers 272 List<ServerName> serversToDecommission = new ArrayList<>(); 273 targetServer = getServerName(iterator.next()); 274 assertTrue(master.getServerManager().getOnlineServers().containsKey(targetServer)); 275 serversToDecommission.add(targetServer); 276 277 admin.decommissionRegionServers(serversToDecommission, true); 278 assertEquals(1, admin.listDecommissionedRegionServers().size()); 279 280 assertTrue(newGroup.getServers().contains(targetServer.getAddress())); 281 rsGroupAdmin.removeServers(Sets.newHashSet(targetServer.getAddress())); 282 Set<Address> newGroupServers = rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getServers(); 283 assertFalse(newGroupServers.contains(targetServer.getAddress())); 284 assertEquals(2, newGroupServers.size()); 285 286 assertTrue(observer.preRemoveServersCalled); 287 assertTrue(observer.postRemoveServersCalled); 288 } 289 290 @Test 291 public void testMoveServersAndTables() throws Exception { 292 LOG.info("testMoveServersAndTables"); 293 final RSGroupInfo newGroup = addGroup(getGroupName(name.getMethodName()), 1); 294 // create table 295 final byte[] familyNameBytes = Bytes.toBytes("f"); 296 TEST_UTIL.createMultiRegionTable(tableName, familyNameBytes, 5); 297 TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() { 298 @Override 299 public boolean evaluate() throws Exception { 300 List<String> regions = getTableRegionMap().get(tableName); 301 if (regions == null) { 302 return false; 303 } 304 305 return getTableRegionMap().get(tableName).size() >= 5; 306 } 307 }); 308 309 // get server which is not a member of new group 310 ServerName targetServer = null; 311 for (ServerName server : admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)) 312 .getLiveServerMetrics().keySet()) { 313 if (!newGroup.containsServer(server.getAddress()) && 314 !rsGroupAdmin.getRSGroupInfo("master").containsServer(server.getAddress())) { 315 targetServer = server; 316 break; 317 } 318 } 319 320 LOG.debug("Print group info : " + rsGroupAdmin.listRSGroups()); 321 int oldDefaultGroupServerSize = 322 rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size(); 323 int oldDefaultGroupTableSize = 324 rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getTables().size(); 325 326 // test fail bogus server move 327 try { 328 rsGroupAdmin.moveServersAndTables(Sets.newHashSet(Address.fromString("foo:9999")), 329 Sets.newHashSet(tableName), newGroup.getName()); 330 fail("Bogus servers shouldn't have been successfully moved."); 331 } catch (IOException ex) { 332 String exp = "Source RSGroup for server foo:9999 does not exist."; 333 String msg = "Expected '" + exp + "' in exception message: "; 334 assertTrue(msg + " " + ex.getMessage(), ex.getMessage().contains(exp)); 335 } 336 337 // test fail server move 338 try { 339 rsGroupAdmin.moveServersAndTables(Sets.newHashSet(targetServer.getAddress()), 340 Sets.newHashSet(tableName), RSGroupInfo.DEFAULT_GROUP); 341 fail("servers shouldn't have been successfully moved."); 342 } catch (IOException ex) { 343 String exp = "Target RSGroup " + RSGroupInfo.DEFAULT_GROUP + " is same as source " + 344 RSGroupInfo.DEFAULT_GROUP + " RSGroup."; 345 String msg = "Expected '" + exp + "' in exception message: "; 346 assertTrue(msg + " " + ex.getMessage(), ex.getMessage().contains(exp)); 347 } 348 349 // verify default group info 350 Assert.assertEquals(oldDefaultGroupServerSize, 351 rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size()); 352 Assert.assertEquals(oldDefaultGroupTableSize, 353 rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getTables().size()); 354 355 // verify new group info 356 Assert.assertEquals(1, rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getServers().size()); 357 Assert.assertEquals(0, rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getTables().size()); 358 359 // get all region to move targetServer 360 List<String> regionList = getTableRegionMap().get(tableName); 361 for (String region : regionList) { 362 // Lets move this region to the targetServer 363 TEST_UTIL.getAdmin().move(Bytes.toBytes(RegionInfo.encodeRegionName(Bytes.toBytes(region))), 364 targetServer); 365 } 366 367 TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() { 368 @Override 369 public boolean evaluate() throws Exception { 370 return getTableRegionMap().get(tableName) != null && 371 getTableRegionMap().get(tableName).size() == 5 && 372 getTableServerRegionMap().get(tableName).size() == 1 && 373 admin.getClusterMetrics(EnumSet.of(Option.REGIONS_IN_TRANSITION)) 374 .getRegionStatesInTransition().size() < 1; 375 } 376 }); 377 378 // verify that all region move to targetServer 379 Assert.assertEquals(5, getTableServerRegionMap().get(tableName).get(targetServer).size()); 380 381 // move targetServer and table to newGroup 382 LOG.info("moving server and table to newGroup"); 383 rsGroupAdmin.moveServersAndTables(Sets.newHashSet(targetServer.getAddress()), 384 Sets.newHashSet(tableName), newGroup.getName()); 385 386 // verify group change 387 Assert.assertEquals(newGroup.getName(), 388 rsGroupAdmin.getRSGroupInfoOfTable(tableName).getName()); 389 390 // verify servers' not exist in old group 391 Set<Address> defaultServers = 392 rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers(); 393 assertFalse(defaultServers.contains(targetServer.getAddress())); 394 395 // verify servers' exist in new group 396 Set<Address> newGroupServers = rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getServers(); 397 assertTrue(newGroupServers.contains(targetServer.getAddress())); 398 399 // verify tables' not exist in old group 400 Set<TableName> defaultTables = 401 rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getTables(); 402 assertFalse(defaultTables.contains(tableName)); 403 404 // verify tables' exist in new group 405 Set<TableName> newGroupTables = rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getTables(); 406 assertTrue(newGroupTables.contains(tableName)); 407 408 // verify that all region still assgin on targetServer 409 Assert.assertEquals(5, getTableServerRegionMap().get(tableName).get(targetServer).size()); 410 411 assertTrue(observer.preMoveServersAndTables); 412 assertTrue(observer.postMoveServersAndTables); 413 } 414 415 @Test 416 public void testMoveServersFromDefaultGroup() throws Exception { 417 // create groups and assign servers 418 rsGroupAdmin.addRSGroup("foo"); 419 420 RSGroupInfo fooGroup = rsGroupAdmin.getRSGroupInfo("foo"); 421 assertEquals(0, fooGroup.getServers().size()); 422 RSGroupInfo defaultGroup = rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP); 423 424 // test remove all servers from default 425 try { 426 rsGroupAdmin.moveServers(defaultGroup.getServers(), fooGroup.getName()); 427 fail(RSGroupAdminServer.KEEP_ONE_SERVER_IN_DEFAULT_ERROR_MESSAGE); 428 } catch (ConstraintException ex) { 429 assertTrue( 430 ex.getMessage().contains(RSGroupAdminServer.KEEP_ONE_SERVER_IN_DEFAULT_ERROR_MESSAGE)); 431 } 432 433 // test success case, remove one server from default ,keep at least one server 434 if (defaultGroup.getServers().size() > 1) { 435 Address serverInDefaultGroup = defaultGroup.getServers().iterator().next(); 436 LOG.info("moving server " + serverInDefaultGroup + " from group default to group " + 437 fooGroup.getName()); 438 rsGroupAdmin.moveServers(Sets.newHashSet(serverInDefaultGroup), fooGroup.getName()); 439 } 440 441 fooGroup = rsGroupAdmin.getRSGroupInfo("foo"); 442 LOG.info("moving servers " + fooGroup.getServers() + " to group default"); 443 rsGroupAdmin.moveServers(fooGroup.getServers(), RSGroupInfo.DEFAULT_GROUP); 444 445 TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() { 446 @Override 447 public boolean evaluate() throws Exception { 448 return getNumServers() == rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP) 449 .getServers().size(); 450 } 451 }); 452 453 fooGroup = rsGroupAdmin.getRSGroupInfo("foo"); 454 assertEquals(0, fooGroup.getServers().size()); 455 456 // test group removal 457 LOG.info("Remove group " + fooGroup.getName()); 458 rsGroupAdmin.removeRSGroup(fooGroup.getName()); 459 Assert.assertEquals(null, rsGroupAdmin.getRSGroupInfo(fooGroup.getName())); 460 } 461 462}