001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.rsgroup; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertFalse; 022import static org.junit.Assert.assertNull; 023import static org.junit.Assert.assertTrue; 024import static org.junit.Assert.fail; 025 026import java.io.IOException; 027import java.security.SecureRandom; 028import java.util.ArrayList; 029import java.util.EnumSet; 030import java.util.Iterator; 031import java.util.LinkedList; 032import java.util.List; 033import java.util.Map; 034import java.util.Set; 035import java.util.TreeMap; 036import org.apache.hadoop.hbase.ClusterMetrics; 037import org.apache.hadoop.hbase.ClusterMetrics.Option; 038import org.apache.hadoop.hbase.HBaseCluster; 039import org.apache.hadoop.hbase.HBaseTestingUtility; 040import org.apache.hadoop.hbase.HColumnDescriptor; 041import org.apache.hadoop.hbase.HTableDescriptor; 042import org.apache.hadoop.hbase.NamespaceDescriptor; 043import org.apache.hadoop.hbase.ServerName; 044import org.apache.hadoop.hbase.TableName; 045import org.apache.hadoop.hbase.TableNotFoundException; 046import org.apache.hadoop.hbase.Waiter; 047import org.apache.hadoop.hbase.client.Admin; 048import org.apache.hadoop.hbase.client.ClusterConnection; 049import org.apache.hadoop.hbase.client.RegionInfo; 050import org.apache.hadoop.hbase.constraint.ConstraintException; 051import org.apache.hadoop.hbase.master.HMaster; 052import org.apache.hadoop.hbase.net.Address; 053import org.apache.hadoop.hbase.util.Bytes; 054import org.junit.Assert; 055import org.junit.Before; 056import org.junit.Rule; 057import org.junit.Test; 058import org.junit.rules.TestName; 059import org.slf4j.Logger; 060import org.slf4j.LoggerFactory; 061 062import org.apache.hbase.thirdparty.com.google.common.collect.Lists; 063import org.apache.hbase.thirdparty.com.google.common.collect.Maps; 064import org.apache.hbase.thirdparty.com.google.common.collect.Sets; 065 066import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; 067import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos; 068import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetServerInfoRequest; 069 070public abstract class TestRSGroupsBase { 071 protected static final Logger LOG = LoggerFactory.getLogger(TestRSGroupsBase.class); 072 @Rule 073 public TestName name = new TestName(); 074 075 //shared 076 protected final static String groupPrefix = "Group"; 077 protected final static String tablePrefix = "Group"; 078 protected final static SecureRandom rand = new SecureRandom(); 079 080 //shared, cluster type specific 081 protected static HBaseTestingUtility TEST_UTIL; 082 protected static Admin admin; 083 protected static HBaseCluster cluster; 084 protected static RSGroupAdmin rsGroupAdmin; 085 protected static HMaster master; 086 087 public final static long WAIT_TIMEOUT = 60000*5; 088 public final static int NUM_SLAVES_BASE = 4; //number of slaves for the smallest cluster 089 public static int NUM_DEAD_SERVERS = 0; 090 091 // Per test variables 092 TableName tableName; 093 @Before 094 public void setup() { 095 LOG.info(name.getMethodName()); 096 tableName = TableName.valueOf(tablePrefix + "_" + name.getMethodName()); 097 } 098 099 protected RSGroupInfo addGroup(String groupName, int serverCount) 100 throws IOException, InterruptedException { 101 return RSGroupTestingUtil.addRSGroup(rsGroupAdmin, groupName, serverCount); 102 } 103 104 protected void removeGroup(String groupName) throws IOException { 105 RSGroupInfo RSGroupInfo = rsGroupAdmin.getRSGroupInfo(groupName); 106 rsGroupAdmin.moveTables(RSGroupInfo.getTables(), RSGroupInfo.DEFAULT_GROUP); 107 rsGroupAdmin.moveServers(RSGroupInfo.getServers(), RSGroupInfo.DEFAULT_GROUP); 108 rsGroupAdmin.removeRSGroup(groupName); 109 } 110 111 protected void deleteTableIfNecessary() throws IOException { 112 for (HTableDescriptor desc : TEST_UTIL.getAdmin().listTables(tablePrefix+".*")) { 113 TEST_UTIL.deleteTable(desc.getTableName()); 114 } 115 } 116 117 protected void deleteNamespaceIfNecessary() throws IOException { 118 for (NamespaceDescriptor desc : TEST_UTIL.getAdmin().listNamespaceDescriptors()) { 119 if(desc.getName().startsWith(tablePrefix)) { 120 admin.deleteNamespace(desc.getName()); 121 } 122 } 123 } 124 125 protected void deleteGroups() throws IOException { 126 RSGroupAdmin groupAdmin = 127 new RSGroupAdminClient(TEST_UTIL.getConnection()); 128 for(RSGroupInfo group: groupAdmin.listRSGroups()) { 129 if(!group.getName().equals(RSGroupInfo.DEFAULT_GROUP)) { 130 groupAdmin.moveTables(group.getTables(), RSGroupInfo.DEFAULT_GROUP); 131 groupAdmin.moveServers(group.getServers(), RSGroupInfo.DEFAULT_GROUP); 132 groupAdmin.removeRSGroup(group.getName()); 133 } 134 } 135 } 136 137 protected Map<TableName, List<String>> getTableRegionMap() throws IOException { 138 Map<TableName, List<String>> map = Maps.newTreeMap(); 139 Map<TableName, Map<ServerName, List<String>>> tableServerRegionMap 140 = getTableServerRegionMap(); 141 for(TableName tableName : tableServerRegionMap.keySet()) { 142 if(!map.containsKey(tableName)) { 143 map.put(tableName, new LinkedList<>()); 144 } 145 for(List<String> subset: tableServerRegionMap.get(tableName).values()) { 146 map.get(tableName).addAll(subset); 147 } 148 } 149 return map; 150 } 151 152 protected Map<TableName, Map<ServerName, List<String>>> getTableServerRegionMap() 153 throws IOException { 154 Map<TableName, Map<ServerName, List<String>>> map = Maps.newTreeMap(); 155 Admin admin = TEST_UTIL.getAdmin(); 156 ClusterMetrics metrics = 157 admin.getClusterMetrics(EnumSet.of(ClusterMetrics.Option.LIVE_SERVERS)); 158 for (ServerName serverName : metrics.getLiveServerMetrics().keySet()) { 159 for (RegionInfo region : admin.getRegions(serverName)) { 160 TableName tableName = region.getTable(); 161 map.computeIfAbsent(tableName, k -> new TreeMap<>()) 162 .computeIfAbsent(serverName, k -> new ArrayList<>()).add(region.getRegionNameAsString()); 163 } 164 } 165 return map; 166 } 167 168 @Test 169 public void testBogusArgs() throws Exception { 170 assertNull(rsGroupAdmin.getRSGroupInfoOfTable(TableName.valueOf("nonexistent"))); 171 assertNull(rsGroupAdmin.getRSGroupOfServer(Address.fromParts("bogus",123))); 172 assertNull(rsGroupAdmin.getRSGroupInfo("bogus")); 173 174 try { 175 rsGroupAdmin.removeRSGroup("bogus"); 176 fail("Expected removing bogus group to fail"); 177 } catch(ConstraintException ex) { 178 //expected 179 } 180 181 try { 182 rsGroupAdmin.moveTables(Sets.newHashSet(TableName.valueOf("bogustable")), "bogus"); 183 fail("Expected move with bogus group to fail"); 184 } catch(ConstraintException|TableNotFoundException ex) { 185 //expected 186 } 187 188 try { 189 rsGroupAdmin.moveServers(Sets.newHashSet(Address.fromParts("bogus",123)), "bogus"); 190 fail("Expected move with bogus group to fail"); 191 } catch(ConstraintException ex) { 192 //expected 193 } 194 195 try { 196 admin.setBalancerRunning(true,true); 197 rsGroupAdmin.balanceRSGroup("bogus"); 198 admin.setBalancerRunning(false,true); 199 fail("Expected move with bogus group to fail"); 200 } catch(ConstraintException ex) { 201 //expected 202 } 203 } 204 205 @Test 206 public void testCreateMultiRegion() throws IOException { 207 byte[] end = {1,3,5,7,9}; 208 byte[] start = {0,2,4,6,8}; 209 byte[][] f = {Bytes.toBytes("f")}; 210 TEST_UTIL.createTable(tableName, f,1,start,end,10); 211 } 212 213 @Test 214 public void testCreateAndDrop() throws Exception { 215 TEST_UTIL.createTable(tableName, Bytes.toBytes("cf")); 216 //wait for created table to be assigned 217 TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() { 218 @Override 219 public boolean evaluate() throws Exception { 220 return getTableRegionMap().get(tableName) != null; 221 } 222 }); 223 TEST_UTIL.deleteTable(tableName); 224 } 225 226 227 @Test 228 public void testSimpleRegionServerMove() throws IOException, 229 InterruptedException { 230 int initNumGroups = rsGroupAdmin.listRSGroups().size(); 231 RSGroupInfo appInfo = addGroup(getGroupName(name.getMethodName()), 1); 232 RSGroupInfo adminInfo = addGroup(getGroupName(name.getMethodName()), 1); 233 RSGroupInfo dInfo = rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP); 234 Assert.assertEquals(initNumGroups + 2, rsGroupAdmin.listRSGroups().size()); 235 assertEquals(1, adminInfo.getServers().size()); 236 assertEquals(1, appInfo.getServers().size()); 237 assertEquals(getNumServers() - 2, dInfo.getServers().size()); 238 rsGroupAdmin.moveServers(appInfo.getServers(), 239 RSGroupInfo.DEFAULT_GROUP); 240 rsGroupAdmin.removeRSGroup(appInfo.getName()); 241 rsGroupAdmin.moveServers(adminInfo.getServers(), 242 RSGroupInfo.DEFAULT_GROUP); 243 rsGroupAdmin.removeRSGroup(adminInfo.getName()); 244 Assert.assertEquals(rsGroupAdmin.listRSGroups().size(), initNumGroups); 245 } 246 247 // return the real number of region servers, excluding the master embedded region server in 2.0+ 248 public int getNumServers() throws IOException { 249 ClusterMetrics status = 250 admin.getClusterMetrics(EnumSet.of(Option.MASTER, Option.LIVE_SERVERS)); 251 ServerName masterName = status.getMasterName(); 252 int count = 0; 253 for (ServerName sn : status.getLiveServerMetrics().keySet()) { 254 if (!sn.equals(masterName)) { 255 count++; 256 } 257 } 258 return count; 259 } 260 261 @Test 262 public void testMoveServers() throws Exception { 263 //create groups and assign servers 264 addGroup("bar", 3); 265 rsGroupAdmin.addRSGroup("foo"); 266 267 RSGroupInfo barGroup = rsGroupAdmin.getRSGroupInfo("bar"); 268 RSGroupInfo fooGroup = rsGroupAdmin.getRSGroupInfo("foo"); 269 assertEquals(3, barGroup.getServers().size()); 270 assertEquals(0, fooGroup.getServers().size()); 271 272 //test fail bogus server move 273 try { 274 rsGroupAdmin.moveServers(Sets.newHashSet(Address.fromString("foo:9999")),"foo"); 275 fail("Bogus servers shouldn't have been successfully moved."); 276 } catch(IOException ex) { 277 String exp = "Source RSGroup for server foo:9999 does not exist."; 278 String msg = "Expected '"+exp+"' in exception message: "; 279 assertTrue(msg+" "+ex.getMessage(), ex.getMessage().contains(exp)); 280 } 281 282 //test success case 283 LOG.info("moving servers "+barGroup.getServers()+" to group foo"); 284 rsGroupAdmin.moveServers(barGroup.getServers(), fooGroup.getName()); 285 286 barGroup = rsGroupAdmin.getRSGroupInfo("bar"); 287 fooGroup = rsGroupAdmin.getRSGroupInfo("foo"); 288 assertEquals(0,barGroup.getServers().size()); 289 assertEquals(3,fooGroup.getServers().size()); 290 291 LOG.info("moving servers "+fooGroup.getServers()+" to group default"); 292 rsGroupAdmin.moveServers(fooGroup.getServers(), RSGroupInfo.DEFAULT_GROUP); 293 294 TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() { 295 @Override 296 public boolean evaluate() throws Exception { 297 return getNumServers() == 298 rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size(); 299 } 300 }); 301 302 fooGroup = rsGroupAdmin.getRSGroupInfo("foo"); 303 assertEquals(0,fooGroup.getServers().size()); 304 305 //test group removal 306 LOG.info("Remove group "+barGroup.getName()); 307 rsGroupAdmin.removeRSGroup(barGroup.getName()); 308 Assert.assertEquals(null, rsGroupAdmin.getRSGroupInfo(barGroup.getName())); 309 LOG.info("Remove group "+fooGroup.getName()); 310 rsGroupAdmin.removeRSGroup(fooGroup.getName()); 311 Assert.assertEquals(null, rsGroupAdmin.getRSGroupInfo(fooGroup.getName())); 312 } 313 314 @Test 315 public void testMoveServersFromDefaultGroup() throws Exception { 316 //create groups and assign servers 317 rsGroupAdmin.addRSGroup("foo"); 318 319 RSGroupInfo fooGroup = rsGroupAdmin.getRSGroupInfo("foo"); 320 assertEquals(0, fooGroup.getServers().size()); 321 RSGroupInfo defaultGroup = rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP); 322 323 //test remove all servers from default 324 try { 325 rsGroupAdmin.moveServers(defaultGroup.getServers(), fooGroup.getName()); 326 fail(RSGroupAdminServer.KEEP_ONE_SERVER_IN_DEFAULT_ERROR_MESSAGE); 327 } catch (ConstraintException ex) { 328 assertTrue(ex.getMessage().contains(RSGroupAdminServer 329 .KEEP_ONE_SERVER_IN_DEFAULT_ERROR_MESSAGE)); 330 } 331 332 //test success case, remove one server from default ,keep at least one server 333 if (defaultGroup.getServers().size() > 1) { 334 Address serverInDefaultGroup = defaultGroup.getServers().iterator().next(); 335 LOG.info("moving server " + serverInDefaultGroup + " from group default to group " + 336 fooGroup.getName()); 337 rsGroupAdmin.moveServers(Sets.newHashSet(serverInDefaultGroup), fooGroup.getName()); 338 } 339 340 fooGroup = rsGroupAdmin.getRSGroupInfo("foo"); 341 LOG.info("moving servers " + fooGroup.getServers() + " to group default"); 342 rsGroupAdmin.moveServers(fooGroup.getServers(), RSGroupInfo.DEFAULT_GROUP); 343 344 TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() { 345 @Override 346 public boolean evaluate() throws Exception { 347 return getNumServers() == 348 rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size(); 349 } 350 }); 351 352 fooGroup = rsGroupAdmin.getRSGroupInfo("foo"); 353 assertEquals(0, fooGroup.getServers().size()); 354 355 //test group removal 356 LOG.info("Remove group " + fooGroup.getName()); 357 rsGroupAdmin.removeRSGroup(fooGroup.getName()); 358 Assert.assertEquals(null, rsGroupAdmin.getRSGroupInfo(fooGroup.getName())); 359 } 360 361 @Test 362 public void testTableMoveTruncateAndDrop() throws Exception { 363 final byte[] familyNameBytes = Bytes.toBytes("f"); 364 String newGroupName = getGroupName(name.getMethodName()); 365 final RSGroupInfo newGroup = addGroup(newGroupName, 2); 366 367 TEST_UTIL.createMultiRegionTable(tableName, familyNameBytes, 5); 368 TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() { 369 @Override 370 public boolean evaluate() throws Exception { 371 List<String> regions = getTableRegionMap().get(tableName); 372 if (regions == null) { 373 return false; 374 } 375 376 return getTableRegionMap().get(tableName).size() >= 5; 377 } 378 }); 379 380 RSGroupInfo tableGrp = rsGroupAdmin.getRSGroupInfoOfTable(tableName); 381 assertTrue(tableGrp.getName().equals(RSGroupInfo.DEFAULT_GROUP)); 382 383 //change table's group 384 LOG.info("Moving table "+tableName+" to "+newGroup.getName()); 385 rsGroupAdmin.moveTables(Sets.newHashSet(tableName), newGroup.getName()); 386 387 //verify group change 388 Assert.assertEquals(newGroup.getName(), 389 rsGroupAdmin.getRSGroupInfoOfTable(tableName).getName()); 390 391 TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() { 392 @Override 393 public boolean evaluate() throws Exception { 394 Map<ServerName, List<String>> serverMap = getTableServerRegionMap().get(tableName); 395 int count = 0; 396 if (serverMap != null) { 397 for (ServerName rs : serverMap.keySet()) { 398 if (newGroup.containsServer(rs.getAddress())) { 399 count += serverMap.get(rs).size(); 400 } 401 } 402 } 403 return count == 5; 404 } 405 }); 406 407 //test truncate 408 admin.disableTable(tableName); 409 admin.truncateTable(tableName, true); 410 Assert.assertEquals(1, rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getTables().size()); 411 Assert.assertEquals(tableName, rsGroupAdmin.getRSGroupInfo( 412 newGroup.getName()).getTables().first()); 413 414 //verify removed table is removed from group 415 TEST_UTIL.deleteTable(tableName); 416 Assert.assertEquals(0, rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getTables().size()); 417 } 418 419 @Test 420 public void testGroupBalance() throws Exception { 421 LOG.info(name.getMethodName()); 422 String newGroupName = getGroupName(name.getMethodName()); 423 final RSGroupInfo newGroup = addGroup(newGroupName, 3); 424 425 final TableName tableName = TableName.valueOf(tablePrefix+"_ns", name.getMethodName()); 426 admin.createNamespace( 427 NamespaceDescriptor.create(tableName.getNamespaceAsString()) 428 .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, newGroupName).build()); 429 final byte[] familyNameBytes = Bytes.toBytes("f"); 430 final HTableDescriptor desc = new HTableDescriptor(tableName); 431 desc.addFamily(new HColumnDescriptor("f")); 432 byte [] startKey = Bytes.toBytes("aaaaa"); 433 byte [] endKey = Bytes.toBytes("zzzzz"); 434 admin.createTable(desc, startKey, endKey, 6); 435 TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() { 436 @Override 437 public boolean evaluate() throws Exception { 438 List<String> regions = getTableRegionMap().get(tableName); 439 if (regions == null) { 440 return false; 441 } 442 return regions.size() >= 6; 443 } 444 }); 445 446 //make assignment uneven, move all regions to one server 447 Map<ServerName,List<String>> assignMap = 448 getTableServerRegionMap().get(tableName); 449 final ServerName first = assignMap.entrySet().iterator().next().getKey(); 450 for(RegionInfo region: admin.getTableRegions(tableName)) { 451 if(!assignMap.get(first).contains(region.getRegionNameAsString())) { 452 admin.move(region.getEncodedNameAsBytes(), Bytes.toBytes(first.getServerName())); 453 } 454 } 455 TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() { 456 @Override 457 public boolean evaluate() throws Exception { 458 Map<ServerName, List<String>> map = getTableServerRegionMap().get(tableName); 459 if (map == null) { 460 return true; 461 } 462 List<String> regions = map.get(first); 463 if (regions == null) { 464 return true; 465 } 466 return regions.size() >= 6; 467 } 468 }); 469 470 //balance the other group and make sure it doesn't affect the new group 471 admin.setBalancerRunning(true,true); 472 rsGroupAdmin.balanceRSGroup(RSGroupInfo.DEFAULT_GROUP); 473 assertEquals(6, getTableServerRegionMap().get(tableName).get(first).size()); 474 475 //disable balance, balancer will not be run and return false 476 admin.setBalancerRunning(false,true); 477 assertFalse(rsGroupAdmin.balanceRSGroup(newGroupName)); 478 assertEquals(6, getTableServerRegionMap().get(tableName).get(first).size()); 479 480 //enable balance 481 admin.setBalancerRunning(true,true); 482 rsGroupAdmin.balanceRSGroup(newGroupName); 483 TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() { 484 @Override 485 public boolean evaluate() throws Exception { 486 for (List<String> regions : getTableServerRegionMap().get(tableName).values()) { 487 if (2 != regions.size()) { 488 return false; 489 } 490 } 491 return true; 492 } 493 }); 494 admin.setBalancerRunning(false,true); 495 } 496 497 @Test 498 public void testRegionMove() throws Exception { 499 final RSGroupInfo newGroup = addGroup(getGroupName(name.getMethodName()), 1); 500 final byte[] familyNameBytes = Bytes.toBytes("f"); 501 // All the regions created below will be assigned to the default group. 502 TEST_UTIL.createMultiRegionTable(tableName, familyNameBytes, 6); 503 TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() { 504 @Override 505 public boolean evaluate() throws Exception { 506 List<String> regions = getTableRegionMap().get(tableName); 507 if (regions == null) { 508 return false; 509 } 510 return getTableRegionMap().get(tableName).size() >= 6; 511 } 512 }); 513 514 //get target region to move 515 Map<ServerName,List<String>> assignMap = 516 getTableServerRegionMap().get(tableName); 517 String targetRegion = null; 518 for(ServerName server : assignMap.keySet()) { 519 targetRegion = assignMap.get(server).size() > 0 ? assignMap.get(server).get(0) : null; 520 if(targetRegion != null) { 521 break; 522 } 523 } 524 //get server which is not a member of new group 525 ServerName targetServer = null; 526 for (ServerName server : admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)) 527 .getLiveServerMetrics().keySet()) { 528 if (!newGroup.containsServer(server.getAddress())) { 529 targetServer = server; 530 break; 531 } 532 } 533 534 final AdminProtos.AdminService.BlockingInterface targetRS = 535 ((ClusterConnection) admin.getConnection()).getAdmin(targetServer); 536 537 //move target server to group 538 rsGroupAdmin.moveServers(Sets.newHashSet(targetServer.getAddress()), 539 newGroup.getName()); 540 TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() { 541 @Override 542 public boolean evaluate() throws Exception { 543 return ProtobufUtil.getOnlineRegions(targetRS).size() <= 0; 544 } 545 }); 546 547 // Lets move this region to the new group. 548 TEST_UTIL.getAdmin().move(Bytes.toBytes(RegionInfo.encodeRegionName( 549 Bytes.toBytes(targetRegion))), Bytes.toBytes(targetServer.getServerName())); 550 TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() { 551 @Override 552 public boolean evaluate() throws Exception { 553 return 554 getTableRegionMap().get(tableName) != null && 555 getTableRegionMap().get(tableName).size() == 6 && 556 admin.getClusterMetrics(EnumSet.of(Option.REGIONS_IN_TRANSITION)) 557 .getRegionStatesInTransition().size() < 1; 558 } 559 }); 560 561 //verify that targetServer didn't open it 562 for (RegionInfo region: ProtobufUtil.getOnlineRegions(targetRS)) { 563 if (targetRegion.equals(region.getRegionNameAsString())) { 564 fail("Target server opened region"); 565 } 566 } 567 } 568 569 @Test 570 public void testFailRemoveGroup() throws IOException, InterruptedException { 571 int initNumGroups = rsGroupAdmin.listRSGroups().size(); 572 addGroup("bar", 3); 573 TEST_UTIL.createTable(tableName, Bytes.toBytes("f")); 574 rsGroupAdmin.moveTables(Sets.newHashSet(tableName), "bar"); 575 RSGroupInfo barGroup = rsGroupAdmin.getRSGroupInfo("bar"); 576 //group is not empty therefore it should fail 577 try { 578 rsGroupAdmin.removeRSGroup(barGroup.getName()); 579 fail("Expected remove group to fail"); 580 } catch(IOException e) { 581 } 582 //group cannot lose all it's servers therefore it should fail 583 try { 584 rsGroupAdmin.moveServers(barGroup.getServers(), RSGroupInfo.DEFAULT_GROUP); 585 fail("Expected move servers to fail"); 586 } catch(IOException e) { 587 } 588 589 rsGroupAdmin.moveTables(barGroup.getTables(), RSGroupInfo.DEFAULT_GROUP); 590 try { 591 rsGroupAdmin.removeRSGroup(barGroup.getName()); 592 fail("Expected move servers to fail"); 593 } catch(IOException e) { 594 } 595 596 rsGroupAdmin.moveServers(barGroup.getServers(), RSGroupInfo.DEFAULT_GROUP); 597 rsGroupAdmin.removeRSGroup(barGroup.getName()); 598 599 Assert.assertEquals(initNumGroups, rsGroupAdmin.listRSGroups().size()); 600 } 601 602 @Test 603 public void testKillRS() throws Exception { 604 RSGroupInfo appInfo = addGroup("appInfo", 1); 605 606 final TableName tableName = TableName.valueOf(tablePrefix+"_ns", name.getMethodName()); 607 admin.createNamespace( 608 NamespaceDescriptor.create(tableName.getNamespaceAsString()) 609 .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, appInfo.getName()).build()); 610 final HTableDescriptor desc = new HTableDescriptor(tableName); 611 desc.addFamily(new HColumnDescriptor("f")); 612 admin.createTable(desc); 613 //wait for created table to be assigned 614 TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() { 615 @Override 616 public boolean evaluate() throws Exception { 617 return getTableRegionMap().get(desc.getTableName()) != null; 618 } 619 }); 620 621 ServerName targetServer = ServerName.parseServerName( 622 appInfo.getServers().iterator().next().toString()); 623 AdminProtos.AdminService.BlockingInterface targetRS = 624 ((ClusterConnection) admin.getConnection()).getAdmin(targetServer); 625 RegionInfo targetRegion = ProtobufUtil.getOnlineRegions(targetRS).get(0); 626 Assert.assertEquals(1, ProtobufUtil.getOnlineRegions(targetRS).size()); 627 628 try { 629 //stopping may cause an exception 630 //due to the connection loss 631 targetRS.stopServer(null, 632 AdminProtos.StopServerRequest.newBuilder().setReason("Die").build()); 633 } catch(Exception e) { 634 } 635 assertFalse(cluster.getClusterMetrics().getLiveServerMetrics().containsKey(targetServer)); 636 637 //wait for created table to be assigned 638 TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() { 639 @Override 640 public boolean evaluate() throws Exception { 641 return cluster.getClusterMetrics().getRegionStatesInTransition().isEmpty(); 642 } 643 }); 644 Set<Address> newServers = Sets.newHashSet(); 645 newServers.add( 646 rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().iterator().next()); 647 rsGroupAdmin.moveServers(newServers, appInfo.getName()); 648 649 //Make sure all the table's regions get reassigned 650 //disabling the table guarantees no conflicting assign/unassign (ie SSH) happens 651 admin.disableTable(tableName); 652 admin.enableTable(tableName); 653 654 //wait for region to be assigned 655 TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() { 656 @Override 657 public boolean evaluate() throws Exception { 658 return cluster.getClusterMetrics().getRegionStatesInTransition().isEmpty(); 659 } 660 }); 661 662 targetServer = ServerName.parseServerName( 663 newServers.iterator().next().toString()); 664 targetRS = 665 ((ClusterConnection) admin.getConnection()).getAdmin(targetServer); 666 Assert.assertEquals(1, ProtobufUtil.getOnlineRegions(targetRS).size()); 667 Assert.assertEquals(tableName, 668 ProtobufUtil.getOnlineRegions(targetRS).get(0).getTable()); 669 } 670 671 @Test 672 public void testValidGroupNames() throws IOException { 673 String[] badNames = {"foo*","foo@","-"}; 674 String[] goodNames = {"foo_123"}; 675 676 for(String entry: badNames) { 677 try { 678 rsGroupAdmin.addRSGroup(entry); 679 fail("Expected a constraint exception for: "+entry); 680 } catch(ConstraintException ex) { 681 //expected 682 } 683 } 684 685 for(String entry: goodNames) { 686 rsGroupAdmin.addRSGroup(entry); 687 } 688 } 689 690 private String getGroupName(String baseName) { 691 return groupPrefix+"_"+baseName+"_"+rand.nextInt(Integer.MAX_VALUE); 692 } 693 694 @Test 695 public void testMultiTableMove() throws Exception { 696 final TableName tableNameA = TableName.valueOf(tablePrefix + name.getMethodName() + "A"); 697 final TableName tableNameB = TableName.valueOf(tablePrefix + name.getMethodName() + "B"); 698 final byte[] familyNameBytes = Bytes.toBytes("f"); 699 String newGroupName = getGroupName(name.getMethodName()); 700 final RSGroupInfo newGroup = addGroup(newGroupName, 1); 701 702 TEST_UTIL.createTable(tableNameA, familyNameBytes); 703 TEST_UTIL.createTable(tableNameB, familyNameBytes); 704 TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() { 705 @Override 706 public boolean evaluate() throws Exception { 707 List<String> regionsA = getTableRegionMap().get(tableNameA); 708 if (regionsA == null) { 709 return false; 710 } 711 712 List<String> regionsB = getTableRegionMap().get(tableNameB); 713 if (regionsB == null) { 714 return false; 715 } 716 717 return getTableRegionMap().get(tableNameA).size() >= 1 718 && getTableRegionMap().get(tableNameB).size() >= 1; 719 } 720 }); 721 722 RSGroupInfo tableGrpA = rsGroupAdmin.getRSGroupInfoOfTable(tableNameA); 723 assertTrue(tableGrpA.getName().equals(RSGroupInfo.DEFAULT_GROUP)); 724 725 RSGroupInfo tableGrpB = rsGroupAdmin.getRSGroupInfoOfTable(tableNameB); 726 assertTrue(tableGrpB.getName().equals(RSGroupInfo.DEFAULT_GROUP)); 727 //change table's group 728 LOG.info("Moving table [" + tableNameA + "," + tableNameB + "] to " + newGroup.getName()); 729 rsGroupAdmin.moveTables(Sets.newHashSet(tableNameA, tableNameB), newGroup.getName()); 730 731 //verify group change 732 Assert.assertEquals(newGroup.getName(), 733 rsGroupAdmin.getRSGroupInfoOfTable(tableNameA).getName()); 734 735 Assert.assertEquals(newGroup.getName(), 736 rsGroupAdmin.getRSGroupInfoOfTable(tableNameB).getName()); 737 738 //verify tables' not exist in old group 739 Set<TableName> DefaultTables = rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP) 740 .getTables(); 741 assertFalse(DefaultTables.contains(tableNameA)); 742 assertFalse(DefaultTables.contains(tableNameB)); 743 744 //verify tables' exist in new group 745 Set<TableName> newGroupTables = rsGroupAdmin.getRSGroupInfo(newGroupName).getTables(); 746 assertTrue(newGroupTables.contains(tableNameA)); 747 assertTrue(newGroupTables.contains(tableNameB)); 748 } 749 750 @Test 751 public void testDisabledTableMove() throws Exception { 752 final byte[] familyNameBytes = Bytes.toBytes("f"); 753 String newGroupName = getGroupName(name.getMethodName()); 754 final RSGroupInfo newGroup = addGroup(newGroupName, 2); 755 756 TEST_UTIL.createMultiRegionTable(tableName, familyNameBytes, 5); 757 TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() { 758 @Override 759 public boolean evaluate() throws Exception { 760 List<String> regions = getTableRegionMap().get(tableName); 761 if (regions == null) { 762 return false; 763 } 764 return getTableRegionMap().get(tableName).size() >= 5; 765 } 766 }); 767 768 RSGroupInfo tableGrp = rsGroupAdmin.getRSGroupInfoOfTable(tableName); 769 assertTrue(tableGrp.getName().equals(RSGroupInfo.DEFAULT_GROUP)); 770 771 //test disable table 772 admin.disableTable(tableName); 773 774 //change table's group 775 LOG.info("Moving table "+ tableName + " to " + newGroup.getName()); 776 rsGroupAdmin.moveTables(Sets.newHashSet(tableName), newGroup.getName()); 777 778 //verify group change 779 Assert.assertEquals(newGroup.getName(), 780 rsGroupAdmin.getRSGroupInfoOfTable(tableName).getName()); 781 } 782 783 @Test 784 public void testNonExistentTableMove() throws Exception { 785 TableName tableName = TableName.valueOf(tablePrefix + name.getMethodName()); 786 787 RSGroupInfo tableGrp = rsGroupAdmin.getRSGroupInfoOfTable(tableName); 788 assertNull(tableGrp); 789 790 //test if table exists already. 791 boolean exist = admin.tableExists(tableName); 792 assertFalse(exist); 793 794 LOG.info("Moving table "+ tableName + " to " + RSGroupInfo.DEFAULT_GROUP); 795 try { 796 rsGroupAdmin.moveTables(Sets.newHashSet(tableName), RSGroupInfo.DEFAULT_GROUP); 797 fail("Table " + tableName + " shouldn't have been successfully moved."); 798 } catch(IOException ex) { 799 assertTrue(ex instanceof TableNotFoundException); 800 } 801 802 try { 803 rsGroupAdmin.moveServersAndTables( 804 Sets.newHashSet(Address.fromParts("bogus",123)), 805 Sets.newHashSet(tableName), RSGroupInfo.DEFAULT_GROUP); 806 fail("Table " + tableName + " shouldn't have been successfully moved."); 807 } catch(IOException ex) { 808 assertTrue(ex instanceof TableNotFoundException); 809 } 810 //verify group change 811 assertNull(rsGroupAdmin.getRSGroupInfoOfTable(tableName)); 812 } 813 814 @Test 815 public void testMoveServersAndTables() throws Exception { 816 LOG.info("testMoveServersAndTables"); 817 final RSGroupInfo newGroup = addGroup(getGroupName(name.getMethodName()), 1); 818 //create table 819 final byte[] familyNameBytes = Bytes.toBytes("f"); 820 TEST_UTIL.createMultiRegionTable(tableName, familyNameBytes, 5); 821 TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() { 822 @Override 823 public boolean evaluate() throws Exception { 824 List<String> regions = getTableRegionMap().get(tableName); 825 if (regions == null) { 826 return false; 827 } 828 return getTableRegionMap().get(tableName).size() >= 5; 829 } 830 }); 831 832 //get server which is not a member of new group 833 ServerName targetServer = null; 834 for(ServerName server : admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)) 835 .getLiveServerMetrics().keySet()) { 836 if(!newGroup.containsServer(server.getAddress()) && 837 !rsGroupAdmin.getRSGroupInfo("master").containsServer(server.getAddress())) { 838 targetServer = server; 839 break; 840 } 841 } 842 843 LOG.debug("Print group info : " + rsGroupAdmin.listRSGroups()); 844 int oldDefaultGroupServerSize = 845 rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size(); 846 int oldDefaultGroupTableSize = 847 rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getTables().size(); 848 849 //test fail bogus server move 850 try { 851 rsGroupAdmin.moveServersAndTables(Sets.newHashSet(Address.fromString("foo:9999")), 852 Sets.newHashSet(tableName), newGroup.getName()); 853 fail("Bogus servers shouldn't have been successfully moved."); 854 } catch(IOException ex) { 855 String exp = "Source RSGroup for server foo:9999 does not exist."; 856 String msg = "Expected '" + exp + "' in exception message: "; 857 assertTrue(msg + " " + ex.getMessage(), ex.getMessage().contains(exp)); 858 } 859 860 //test fail server move 861 try { 862 rsGroupAdmin.moveServersAndTables(Sets.newHashSet(targetServer.getAddress()), 863 Sets.newHashSet(tableName), RSGroupInfo.DEFAULT_GROUP); 864 fail("servers shouldn't have been successfully moved."); 865 } catch(IOException ex) { 866 String exp = "Target RSGroup " + RSGroupInfo.DEFAULT_GROUP + 867 " is same as source " + RSGroupInfo.DEFAULT_GROUP + " RSGroup."; 868 String msg = "Expected '" + exp + "' in exception message: "; 869 assertTrue(msg + " " + ex.getMessage(), ex.getMessage().contains(exp)); 870 } 871 872 //verify default group info 873 Assert.assertEquals(oldDefaultGroupServerSize, 874 rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size()); 875 Assert.assertEquals(oldDefaultGroupTableSize, 876 rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getTables().size()); 877 878 //verify new group info 879 Assert.assertEquals(1, 880 rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getServers().size()); 881 Assert.assertEquals(0, 882 rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getTables().size()); 883 884 //get all region to move targetServer 885 List<String> regionList = getTableRegionMap().get(tableName); 886 for(String region : regionList) { 887 // Lets move this region to the targetServer 888 TEST_UTIL.getAdmin().move(Bytes.toBytes(RegionInfo.encodeRegionName(Bytes.toBytes(region))), 889 Bytes.toBytes(targetServer.getServerName())); 890 } 891 892 TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() { 893 @Override 894 public boolean evaluate() throws Exception { 895 return getTableRegionMap().get(tableName) != null && 896 getTableRegionMap().get(tableName).size() == 5 && 897 getTableServerRegionMap().get(tableName).size() == 1 && 898 admin.getClusterMetrics(EnumSet.of(Option.REGIONS_IN_TRANSITION)) 899 .getRegionStatesInTransition().size() < 1; 900 } 901 }); 902 903 //verify that all region move to targetServer 904 Assert.assertEquals(5, getTableServerRegionMap().get(tableName).get(targetServer).size()); 905 906 //move targetServer and table to newGroup 907 LOG.info("moving server and table to newGroup"); 908 rsGroupAdmin.moveServersAndTables(Sets.newHashSet(targetServer.getAddress()), 909 Sets.newHashSet(tableName), newGroup.getName()); 910 911 //verify group change 912 Assert.assertEquals(newGroup.getName(), 913 rsGroupAdmin.getRSGroupInfoOfTable(tableName).getName()); 914 915 //verify servers' not exist in old group 916 Set<Address> defaultServers = rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP) 917 .getServers(); 918 assertFalse(defaultServers.contains(targetServer.getAddress())); 919 920 //verify servers' exist in new group 921 Set<Address> newGroupServers = rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getServers(); 922 assertTrue(newGroupServers.contains(targetServer.getAddress())); 923 924 //verify tables' not exist in old group 925 Set<TableName> defaultTables = rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP) 926 .getTables(); 927 assertFalse(defaultTables.contains(tableName)); 928 929 //verify tables' exist in new group 930 Set<TableName> newGroupTables = rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getTables(); 931 assertTrue(newGroupTables.contains(tableName)); 932 933 //verify that all region still assgin on targetServer 934 Assert.assertEquals(5, getTableServerRegionMap().get(tableName).get(targetServer).size()); 935 } 936 937 @Test 938 public void testClearDeadServers() throws Exception { 939 LOG.info("testClearDeadServers"); 940 final RSGroupInfo newGroup = addGroup(getGroupName(name.getMethodName()), 3); 941 NUM_DEAD_SERVERS = cluster.getClusterMetrics().getDeadServerNames().size(); 942 943 ServerName targetServer = ServerName.parseServerName( 944 newGroup.getServers().iterator().next().toString()); 945 AdminProtos.AdminService.BlockingInterface targetRS = 946 ((ClusterConnection) admin.getConnection()).getAdmin(targetServer); 947 try { 948 targetServer = ProtobufUtil.toServerName(targetRS.getServerInfo(null, 949 GetServerInfoRequest.newBuilder().build()).getServerInfo().getServerName()); 950 //stopping may cause an exception 951 //due to the connection loss 952 targetRS.stopServer(null, 953 AdminProtos.StopServerRequest.newBuilder().setReason("Die").build()); 954 NUM_DEAD_SERVERS ++; 955 } catch(Exception e) { 956 } 957 //wait for stopped regionserver to dead server list 958 TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() { 959 @Override 960 public boolean evaluate() throws Exception { 961 return !master.getServerManager().areDeadServersInProgress() 962 && cluster.getClusterMetrics().getDeadServerNames().size() == NUM_DEAD_SERVERS; 963 } 964 }); 965 assertFalse(cluster.getClusterMetrics().getLiveServerMetrics().containsKey(targetServer)); 966 assertTrue(cluster.getClusterMetrics().getDeadServerNames().contains(targetServer)); 967 assertTrue(newGroup.getServers().contains(targetServer.getAddress())); 968 969 //clear dead servers list 970 List<ServerName> notClearedServers = admin.clearDeadServers(Lists.newArrayList(targetServer)); 971 assertEquals(0, notClearedServers.size()); 972 973 Set<Address> newGroupServers = rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getServers(); 974 assertFalse(newGroupServers.contains(targetServer.getAddress())); 975 assertEquals(2, newGroupServers.size()); 976 } 977 978 @Test 979 public void testRemoveServers() throws Exception { 980 LOG.info("testRemoveServers"); 981 final RSGroupInfo newGroup = addGroup(getGroupName(name.getMethodName()), 3); 982 Iterator<Address> iterator = newGroup.getServers().iterator(); 983 ServerName targetServer = ServerName.parseServerName(iterator.next().toString()); 984 985 // remove online servers 986 try { 987 rsGroupAdmin.removeServers(Sets.newHashSet(targetServer.getAddress())); 988 fail("Online servers shouldn't have been successfully removed."); 989 } catch(IOException ex) { 990 String exp = "Server " + targetServer.getAddress() 991 + " is an online server, not allowed to remove."; 992 String msg = "Expected '" + exp + "' in exception message: "; 993 assertTrue(msg + " " + ex.getMessage(), ex.getMessage().contains(exp)); 994 } 995 assertTrue(newGroup.getServers().contains(targetServer.getAddress())); 996 997 // remove dead servers 998 NUM_DEAD_SERVERS = cluster.getClusterMetrics().getDeadServerNames().size(); 999 AdminProtos.AdminService.BlockingInterface targetRS = 1000 ((ClusterConnection) admin.getConnection()).getAdmin(targetServer); 1001 try { 1002 targetServer = ProtobufUtil.toServerName(targetRS.getServerInfo(null, 1003 GetServerInfoRequest.newBuilder().build()).getServerInfo().getServerName()); 1004 //stopping may cause an exception 1005 //due to the connection loss 1006 LOG.info("stopping server " + targetServer.getHostAndPort()); 1007 targetRS.stopServer(null, 1008 AdminProtos.StopServerRequest.newBuilder().setReason("Die").build()); 1009 NUM_DEAD_SERVERS ++; 1010 } catch(Exception e) { 1011 } 1012 1013 //wait for stopped regionserver to dead server list 1014 TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() { 1015 @Override 1016 public boolean evaluate() throws Exception { 1017 return !master.getServerManager().areDeadServersInProgress() 1018 && cluster.getClusterMetrics().getDeadServerNames().size() == NUM_DEAD_SERVERS; 1019 } 1020 }); 1021 1022 try { 1023 rsGroupAdmin.removeServers(Sets.newHashSet(targetServer.getAddress())); 1024 fail("Dead servers shouldn't have been successfully removed."); 1025 } catch(IOException ex) { 1026 String exp = "Server " + targetServer.getAddress() + " is on the dead servers list," 1027 + " Maybe it will come back again, not allowed to remove."; 1028 String msg = "Expected '" + exp + "' in exception message: "; 1029 assertTrue(msg + " " + ex.getMessage(), ex.getMessage().contains(exp)); 1030 } 1031 assertTrue(newGroup.getServers().contains(targetServer.getAddress())); 1032 1033 // remove decommissioned servers 1034 List<ServerName> serversToDecommission = new ArrayList<>(); 1035 targetServer = ServerName.parseServerName(iterator.next().toString()); 1036 targetRS = ((ClusterConnection) admin.getConnection()).getAdmin(targetServer); 1037 targetServer = ProtobufUtil.toServerName(targetRS.getServerInfo(null, 1038 GetServerInfoRequest.newBuilder().build()).getServerInfo().getServerName()); 1039 assertTrue(master.getServerManager().getOnlineServers().containsKey(targetServer)); 1040 serversToDecommission.add(targetServer); 1041 1042 admin.decommissionRegionServers(serversToDecommission, true); 1043 assertEquals(1, admin.listDecommissionedRegionServers().size()); 1044 1045 assertTrue(newGroup.getServers().contains(targetServer.getAddress())); 1046 rsGroupAdmin.removeServers(Sets.newHashSet(targetServer.getAddress())); 1047 Set<Address> newGroupServers = rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getServers(); 1048 assertFalse(newGroupServers.contains(targetServer.getAddress())); 1049 assertEquals(2, newGroupServers.size()); 1050 } 1051 1052 @Test 1053 public void testCreateWhenRsgroupNoOnlineServers() throws Exception { 1054 LOG.info("testCreateWhenRsgroupNoOnlineServers"); 1055 1056 // set rsgroup has no online servers and test create table 1057 final RSGroupInfo appInfo = addGroup("appInfo", 1); 1058 Iterator<Address> iterator = appInfo.getServers().iterator(); 1059 List<ServerName> serversToDecommission = new ArrayList<>(); 1060 ServerName targetServer = ServerName.parseServerName(iterator.next().toString()); 1061 AdminProtos.AdminService.BlockingInterface targetRS = 1062 ((ClusterConnection) admin.getConnection()).getAdmin(targetServer); 1063 targetServer = ProtobufUtil.toServerName( 1064 targetRS.getServerInfo(null, GetServerInfoRequest.newBuilder().build()).getServerInfo() 1065 .getServerName()); 1066 assertTrue(master.getServerManager().getOnlineServers().containsKey(targetServer)); 1067 serversToDecommission.add(targetServer); 1068 admin.decommissionRegionServers(serversToDecommission, true); 1069 assertEquals(1, admin.listDecommissionedRegionServers().size()); 1070 1071 final TableName tableName = TableName.valueOf(tablePrefix + "_ns", name.getMethodName()); 1072 admin.createNamespace(NamespaceDescriptor.create(tableName.getNamespaceAsString()) 1073 .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, appInfo.getName()).build()); 1074 final HTableDescriptor desc = new HTableDescriptor(tableName); 1075 desc.addFamily(new HColumnDescriptor("f")); 1076 try { 1077 admin.createTable(desc); 1078 fail("Shouldn't create table successfully!"); 1079 } catch (Exception e) { 1080 LOG.debug("create table error", e); 1081 } 1082 1083 // recommission and test create table 1084 admin.recommissionRegionServer(targetServer, null); 1085 assertEquals(0, admin.listDecommissionedRegionServers().size()); 1086 admin.createTable(desc); 1087 // wait for created table to be assigned 1088 TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() { 1089 @Override public boolean evaluate() throws Exception { 1090 return getTableRegionMap().get(desc.getTableName()) != null; 1091 } 1092 }); 1093 } 1094}