Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.rsgroup;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertNull;
023import static org.junit.Assert.assertTrue;
024import static org.junit.Assert.fail;
025
026import java.io.IOException;
027import java.security.SecureRandom;
028import java.util.ArrayList;
029import java.util.EnumSet;
030import java.util.Iterator;
031import java.util.LinkedList;
032import java.util.List;
033import java.util.Map;
034import java.util.Set;
035import java.util.TreeMap;
036import org.apache.hadoop.hbase.ClusterMetrics;
037import org.apache.hadoop.hbase.ClusterMetrics.Option;
038import org.apache.hadoop.hbase.HBaseCluster;
039import org.apache.hadoop.hbase.HBaseTestingUtility;
040import org.apache.hadoop.hbase.HColumnDescriptor;
041import org.apache.hadoop.hbase.HTableDescriptor;
042import org.apache.hadoop.hbase.NamespaceDescriptor;
043import org.apache.hadoop.hbase.ServerName;
044import org.apache.hadoop.hbase.TableName;
045import org.apache.hadoop.hbase.TableNotFoundException;
046import org.apache.hadoop.hbase.Waiter;
047import org.apache.hadoop.hbase.client.Admin;
048import org.apache.hadoop.hbase.client.ClusterConnection;
049import org.apache.hadoop.hbase.client.RegionInfo;
050import org.apache.hadoop.hbase.constraint.ConstraintException;
051import org.apache.hadoop.hbase.master.HMaster;
052import org.apache.hadoop.hbase.net.Address;
053import org.apache.hadoop.hbase.util.Bytes;
054import org.junit.Assert;
055import org.junit.Before;
056import org.junit.Rule;
057import org.junit.Test;
058import org.junit.rules.TestName;
059import org.slf4j.Logger;
060import org.slf4j.LoggerFactory;
061
062import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
063import org.apache.hbase.thirdparty.com.google.common.collect.Maps;
064import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
065
066import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
067import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos;
068import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetServerInfoRequest;
069
070public abstract class TestRSGroupsBase {
071  protected static final Logger LOG = LoggerFactory.getLogger(TestRSGroupsBase.class);
072  @Rule
073  public TestName name = new TestName();
074
075  //shared
076  protected final static String groupPrefix = "Group";
077  protected final static String tablePrefix = "Group";
078  protected final static SecureRandom rand = new SecureRandom();
079
080  //shared, cluster type specific
081  protected static HBaseTestingUtility TEST_UTIL;
082  protected static Admin admin;
083  protected static HBaseCluster cluster;
084  protected static RSGroupAdmin rsGroupAdmin;
085  protected static HMaster master;
086
087  public final static long WAIT_TIMEOUT = 60000*5;
088  public final static int NUM_SLAVES_BASE = 4; //number of slaves for the smallest cluster
089  public static int NUM_DEAD_SERVERS = 0;
090
091  // Per test variables
092  TableName tableName;
093  @Before
094  public void setup() {
095    LOG.info(name.getMethodName());
096    tableName = TableName.valueOf(tablePrefix + "_" + name.getMethodName());
097  }
098
099  protected RSGroupInfo addGroup(String groupName, int serverCount)
100      throws IOException, InterruptedException {
101    return RSGroupTestingUtil.addRSGroup(rsGroupAdmin, groupName, serverCount);
102  }
103
104  protected void removeGroup(String groupName) throws IOException {
105    RSGroupInfo RSGroupInfo = rsGroupAdmin.getRSGroupInfo(groupName);
106    rsGroupAdmin.moveTables(RSGroupInfo.getTables(), RSGroupInfo.DEFAULT_GROUP);
107    rsGroupAdmin.moveServers(RSGroupInfo.getServers(), RSGroupInfo.DEFAULT_GROUP);
108    rsGroupAdmin.removeRSGroup(groupName);
109  }
110
111  protected void deleteTableIfNecessary() throws IOException {
112    for (HTableDescriptor desc : TEST_UTIL.getAdmin().listTables(tablePrefix+".*")) {
113      TEST_UTIL.deleteTable(desc.getTableName());
114    }
115  }
116
117  protected void deleteNamespaceIfNecessary() throws IOException {
118    for (NamespaceDescriptor desc : TEST_UTIL.getAdmin().listNamespaceDescriptors()) {
119      if(desc.getName().startsWith(tablePrefix)) {
120        admin.deleteNamespace(desc.getName());
121      }
122    }
123  }
124
125  protected void deleteGroups() throws IOException {
126    RSGroupAdmin groupAdmin =
127        new RSGroupAdminClient(TEST_UTIL.getConnection());
128    for(RSGroupInfo group: groupAdmin.listRSGroups()) {
129      if(!group.getName().equals(RSGroupInfo.DEFAULT_GROUP)) {
130        groupAdmin.moveTables(group.getTables(), RSGroupInfo.DEFAULT_GROUP);
131        groupAdmin.moveServers(group.getServers(), RSGroupInfo.DEFAULT_GROUP);
132        groupAdmin.removeRSGroup(group.getName());
133      }
134    }
135  }
136
137  protected Map<TableName, List<String>> getTableRegionMap() throws IOException {
138    Map<TableName, List<String>> map = Maps.newTreeMap();
139    Map<TableName, Map<ServerName, List<String>>> tableServerRegionMap
140        = getTableServerRegionMap();
141    for(TableName tableName : tableServerRegionMap.keySet()) {
142      if(!map.containsKey(tableName)) {
143        map.put(tableName, new LinkedList<>());
144      }
145      for(List<String> subset: tableServerRegionMap.get(tableName).values()) {
146        map.get(tableName).addAll(subset);
147      }
148    }
149    return map;
150  }
151
152  protected Map<TableName, Map<ServerName, List<String>>> getTableServerRegionMap()
153    throws IOException {
154    Map<TableName, Map<ServerName, List<String>>> map = Maps.newTreeMap();
155    Admin admin = TEST_UTIL.getAdmin();
156    ClusterMetrics metrics =
157      admin.getClusterMetrics(EnumSet.of(ClusterMetrics.Option.LIVE_SERVERS));
158    for (ServerName serverName : metrics.getLiveServerMetrics().keySet()) {
159      for (RegionInfo region : admin.getRegions(serverName)) {
160        TableName tableName = region.getTable();
161        map.computeIfAbsent(tableName, k -> new TreeMap<>())
162          .computeIfAbsent(serverName, k -> new ArrayList<>()).add(region.getRegionNameAsString());
163      }
164    }
165    return map;
166  }
167
168  @Test
169  public void testBogusArgs() throws Exception {
170    assertNull(rsGroupAdmin.getRSGroupInfoOfTable(TableName.valueOf("nonexistent")));
171    assertNull(rsGroupAdmin.getRSGroupOfServer(Address.fromParts("bogus",123)));
172    assertNull(rsGroupAdmin.getRSGroupInfo("bogus"));
173
174    try {
175      rsGroupAdmin.removeRSGroup("bogus");
176      fail("Expected removing bogus group to fail");
177    } catch(ConstraintException ex) {
178      //expected
179    }
180
181    try {
182      rsGroupAdmin.moveTables(Sets.newHashSet(TableName.valueOf("bogustable")), "bogus");
183      fail("Expected move with bogus group to fail");
184    } catch(ConstraintException|TableNotFoundException ex) {
185      //expected
186    }
187
188    try {
189      rsGroupAdmin.moveServers(Sets.newHashSet(Address.fromParts("bogus",123)), "bogus");
190      fail("Expected move with bogus group to fail");
191    } catch(ConstraintException ex) {
192      //expected
193    }
194
195    try {
196      admin.setBalancerRunning(true,true);
197      rsGroupAdmin.balanceRSGroup("bogus");
198      admin.setBalancerRunning(false,true);
199      fail("Expected move with bogus group to fail");
200    } catch(ConstraintException ex) {
201      //expected
202    }
203  }
204
205  @Test
206  public void testCreateMultiRegion() throws IOException {
207    byte[] end = {1,3,5,7,9};
208    byte[] start = {0,2,4,6,8};
209    byte[][] f = {Bytes.toBytes("f")};
210    TEST_UTIL.createTable(tableName, f,1,start,end,10);
211  }
212
213  @Test
214  public void testCreateAndDrop() throws Exception {
215    TEST_UTIL.createTable(tableName, Bytes.toBytes("cf"));
216    //wait for created table to be assigned
217    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
218      @Override
219      public boolean evaluate() throws Exception {
220        return getTableRegionMap().get(tableName) != null;
221      }
222    });
223    TEST_UTIL.deleteTable(tableName);
224  }
225
226
227  @Test
228  public void testSimpleRegionServerMove() throws IOException,
229      InterruptedException {
230    int initNumGroups = rsGroupAdmin.listRSGroups().size();
231    RSGroupInfo appInfo = addGroup(getGroupName(name.getMethodName()), 1);
232    RSGroupInfo adminInfo = addGroup(getGroupName(name.getMethodName()), 1);
233    RSGroupInfo dInfo = rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP);
234    Assert.assertEquals(initNumGroups + 2, rsGroupAdmin.listRSGroups().size());
235    assertEquals(1, adminInfo.getServers().size());
236    assertEquals(1, appInfo.getServers().size());
237    assertEquals(getNumServers() - 2, dInfo.getServers().size());
238    rsGroupAdmin.moveServers(appInfo.getServers(),
239        RSGroupInfo.DEFAULT_GROUP);
240    rsGroupAdmin.removeRSGroup(appInfo.getName());
241    rsGroupAdmin.moveServers(adminInfo.getServers(),
242        RSGroupInfo.DEFAULT_GROUP);
243    rsGroupAdmin.removeRSGroup(adminInfo.getName());
244    Assert.assertEquals(rsGroupAdmin.listRSGroups().size(), initNumGroups);
245  }
246
247  // return the real number of region servers, excluding the master embedded region server in 2.0+
248  public int getNumServers() throws IOException {
249    ClusterMetrics status =
250        admin.getClusterMetrics(EnumSet.of(Option.MASTER, Option.LIVE_SERVERS));
251    ServerName masterName = status.getMasterName();
252    int count = 0;
253    for (ServerName sn : status.getLiveServerMetrics().keySet()) {
254      if (!sn.equals(masterName)) {
255        count++;
256      }
257    }
258    return count;
259  }
260
261  @Test
262  public void testMoveServers() throws Exception {
263    //create groups and assign servers
264    addGroup("bar", 3);
265    rsGroupAdmin.addRSGroup("foo");
266
267    RSGroupInfo barGroup = rsGroupAdmin.getRSGroupInfo("bar");
268    RSGroupInfo fooGroup = rsGroupAdmin.getRSGroupInfo("foo");
269    assertEquals(3, barGroup.getServers().size());
270    assertEquals(0, fooGroup.getServers().size());
271
272    //test fail bogus server move
273    try {
274      rsGroupAdmin.moveServers(Sets.newHashSet(Address.fromString("foo:9999")),"foo");
275      fail("Bogus servers shouldn't have been successfully moved.");
276    } catch(IOException ex) {
277      String exp = "Source RSGroup for server foo:9999 does not exist.";
278      String msg = "Expected '"+exp+"' in exception message: ";
279      assertTrue(msg+" "+ex.getMessage(), ex.getMessage().contains(exp));
280    }
281
282    //test success case
283    LOG.info("moving servers "+barGroup.getServers()+" to group foo");
284    rsGroupAdmin.moveServers(barGroup.getServers(), fooGroup.getName());
285
286    barGroup = rsGroupAdmin.getRSGroupInfo("bar");
287    fooGroup = rsGroupAdmin.getRSGroupInfo("foo");
288    assertEquals(0,barGroup.getServers().size());
289    assertEquals(3,fooGroup.getServers().size());
290
291    LOG.info("moving servers "+fooGroup.getServers()+" to group default");
292    rsGroupAdmin.moveServers(fooGroup.getServers(), RSGroupInfo.DEFAULT_GROUP);
293
294    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
295      @Override
296      public boolean evaluate() throws Exception {
297        return getNumServers() ==
298        rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size();
299      }
300    });
301
302    fooGroup = rsGroupAdmin.getRSGroupInfo("foo");
303    assertEquals(0,fooGroup.getServers().size());
304
305    //test group removal
306    LOG.info("Remove group "+barGroup.getName());
307    rsGroupAdmin.removeRSGroup(barGroup.getName());
308    Assert.assertEquals(null, rsGroupAdmin.getRSGroupInfo(barGroup.getName()));
309    LOG.info("Remove group "+fooGroup.getName());
310    rsGroupAdmin.removeRSGroup(fooGroup.getName());
311    Assert.assertEquals(null, rsGroupAdmin.getRSGroupInfo(fooGroup.getName()));
312  }
313
314  @Test
315  public void testMoveServersFromDefaultGroup() throws Exception {
316    //create groups and assign servers
317    rsGroupAdmin.addRSGroup("foo");
318
319    RSGroupInfo fooGroup = rsGroupAdmin.getRSGroupInfo("foo");
320    assertEquals(0, fooGroup.getServers().size());
321    RSGroupInfo defaultGroup = rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP);
322
323    //test remove all servers from default
324    try {
325      rsGroupAdmin.moveServers(defaultGroup.getServers(), fooGroup.getName());
326      fail(RSGroupAdminServer.KEEP_ONE_SERVER_IN_DEFAULT_ERROR_MESSAGE);
327    } catch (ConstraintException ex) {
328      assertTrue(ex.getMessage().contains(RSGroupAdminServer
329              .KEEP_ONE_SERVER_IN_DEFAULT_ERROR_MESSAGE));
330    }
331
332    //test success case, remove one server from default ,keep at least one server
333    if (defaultGroup.getServers().size() > 1) {
334      Address serverInDefaultGroup = defaultGroup.getServers().iterator().next();
335      LOG.info("moving server " + serverInDefaultGroup + " from group default to group " +
336              fooGroup.getName());
337      rsGroupAdmin.moveServers(Sets.newHashSet(serverInDefaultGroup), fooGroup.getName());
338    }
339
340    fooGroup = rsGroupAdmin.getRSGroupInfo("foo");
341    LOG.info("moving servers " + fooGroup.getServers() + " to group default");
342    rsGroupAdmin.moveServers(fooGroup.getServers(), RSGroupInfo.DEFAULT_GROUP);
343
344    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
345      @Override
346      public boolean evaluate() throws Exception {
347        return getNumServers() ==
348                rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size();
349      }
350    });
351
352    fooGroup = rsGroupAdmin.getRSGroupInfo("foo");
353    assertEquals(0, fooGroup.getServers().size());
354
355    //test group removal
356    LOG.info("Remove group " + fooGroup.getName());
357    rsGroupAdmin.removeRSGroup(fooGroup.getName());
358    Assert.assertEquals(null, rsGroupAdmin.getRSGroupInfo(fooGroup.getName()));
359  }
360
361  @Test
362  public void testTableMoveTruncateAndDrop() throws Exception {
363    final byte[] familyNameBytes = Bytes.toBytes("f");
364    String newGroupName = getGroupName(name.getMethodName());
365    final RSGroupInfo newGroup = addGroup(newGroupName, 2);
366
367    TEST_UTIL.createMultiRegionTable(tableName, familyNameBytes, 5);
368    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
369      @Override
370      public boolean evaluate() throws Exception {
371        List<String> regions = getTableRegionMap().get(tableName);
372        if (regions == null) {
373          return false;
374        }
375
376        return getTableRegionMap().get(tableName).size() >= 5;
377      }
378    });
379
380    RSGroupInfo tableGrp = rsGroupAdmin.getRSGroupInfoOfTable(tableName);
381    assertTrue(tableGrp.getName().equals(RSGroupInfo.DEFAULT_GROUP));
382
383    //change table's group
384    LOG.info("Moving table "+tableName+" to "+newGroup.getName());
385    rsGroupAdmin.moveTables(Sets.newHashSet(tableName), newGroup.getName());
386
387    //verify group change
388    Assert.assertEquals(newGroup.getName(),
389        rsGroupAdmin.getRSGroupInfoOfTable(tableName).getName());
390
391    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
392      @Override
393      public boolean evaluate() throws Exception {
394        Map<ServerName, List<String>> serverMap = getTableServerRegionMap().get(tableName);
395        int count = 0;
396        if (serverMap != null) {
397          for (ServerName rs : serverMap.keySet()) {
398            if (newGroup.containsServer(rs.getAddress())) {
399              count += serverMap.get(rs).size();
400            }
401          }
402        }
403        return count == 5;
404      }
405    });
406
407    //test truncate
408    admin.disableTable(tableName);
409    admin.truncateTable(tableName, true);
410    Assert.assertEquals(1, rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getTables().size());
411    Assert.assertEquals(tableName, rsGroupAdmin.getRSGroupInfo(
412        newGroup.getName()).getTables().first());
413
414    //verify removed table is removed from group
415    TEST_UTIL.deleteTable(tableName);
416    Assert.assertEquals(0, rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getTables().size());
417  }
418
419  @Test
420  public void testGroupBalance() throws Exception {
421    LOG.info(name.getMethodName());
422    String newGroupName = getGroupName(name.getMethodName());
423    final RSGroupInfo newGroup = addGroup(newGroupName, 3);
424
425    final TableName tableName = TableName.valueOf(tablePrefix+"_ns", name.getMethodName());
426    admin.createNamespace(
427        NamespaceDescriptor.create(tableName.getNamespaceAsString())
428            .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, newGroupName).build());
429    final byte[] familyNameBytes = Bytes.toBytes("f");
430    final HTableDescriptor desc = new HTableDescriptor(tableName);
431    desc.addFamily(new HColumnDescriptor("f"));
432    byte [] startKey = Bytes.toBytes("aaaaa");
433    byte [] endKey = Bytes.toBytes("zzzzz");
434    admin.createTable(desc, startKey, endKey, 6);
435    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
436      @Override
437      public boolean evaluate() throws Exception {
438        List<String> regions = getTableRegionMap().get(tableName);
439        if (regions == null) {
440          return false;
441        }
442        return regions.size() >= 6;
443      }
444    });
445
446    //make assignment uneven, move all regions to one server
447    Map<ServerName,List<String>> assignMap =
448        getTableServerRegionMap().get(tableName);
449    final ServerName first = assignMap.entrySet().iterator().next().getKey();
450    for(RegionInfo region: admin.getTableRegions(tableName)) {
451      if(!assignMap.get(first).contains(region.getRegionNameAsString())) {
452        admin.move(region.getEncodedNameAsBytes(), Bytes.toBytes(first.getServerName()));
453      }
454    }
455    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
456      @Override
457      public boolean evaluate() throws Exception {
458        Map<ServerName, List<String>> map = getTableServerRegionMap().get(tableName);
459        if (map == null) {
460          return true;
461        }
462        List<String> regions = map.get(first);
463        if (regions == null) {
464          return true;
465        }
466        return regions.size() >= 6;
467      }
468    });
469
470    //balance the other group and make sure it doesn't affect the new group
471    admin.setBalancerRunning(true,true);
472    rsGroupAdmin.balanceRSGroup(RSGroupInfo.DEFAULT_GROUP);
473    assertEquals(6, getTableServerRegionMap().get(tableName).get(first).size());
474
475    //disable balance, balancer will not be run and return false
476    admin.setBalancerRunning(false,true);
477    assertFalse(rsGroupAdmin.balanceRSGroup(newGroupName));
478    assertEquals(6, getTableServerRegionMap().get(tableName).get(first).size());
479
480    //enable balance
481    admin.setBalancerRunning(true,true);
482    rsGroupAdmin.balanceRSGroup(newGroupName);
483    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
484      @Override
485      public boolean evaluate() throws Exception {
486        for (List<String> regions : getTableServerRegionMap().get(tableName).values()) {
487          if (2 != regions.size()) {
488            return false;
489          }
490        }
491        return true;
492      }
493    });
494    admin.setBalancerRunning(false,true);
495  }
496
497  @Test
498  public void testRegionMove() throws Exception {
499    final RSGroupInfo newGroup = addGroup(getGroupName(name.getMethodName()), 1);
500    final byte[] familyNameBytes = Bytes.toBytes("f");
501    // All the regions created below will be assigned to the default group.
502    TEST_UTIL.createMultiRegionTable(tableName, familyNameBytes, 6);
503    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
504      @Override
505      public boolean evaluate() throws Exception {
506        List<String> regions = getTableRegionMap().get(tableName);
507        if (regions == null) {
508          return false;
509        }
510        return getTableRegionMap().get(tableName).size() >= 6;
511      }
512    });
513
514    //get target region to move
515    Map<ServerName,List<String>> assignMap =
516        getTableServerRegionMap().get(tableName);
517    String targetRegion = null;
518    for(ServerName server : assignMap.keySet()) {
519      targetRegion = assignMap.get(server).size() > 0 ? assignMap.get(server).get(0) : null;
520      if(targetRegion != null) {
521        break;
522      }
523    }
524    //get server which is not a member of new group
525    ServerName targetServer = null;
526    for (ServerName server : admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS))
527                                  .getLiveServerMetrics().keySet()) {
528      if (!newGroup.containsServer(server.getAddress())) {
529        targetServer = server;
530        break;
531      }
532    }
533
534    final AdminProtos.AdminService.BlockingInterface targetRS =
535      ((ClusterConnection) admin.getConnection()).getAdmin(targetServer);
536
537    //move target server to group
538    rsGroupAdmin.moveServers(Sets.newHashSet(targetServer.getAddress()),
539        newGroup.getName());
540    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
541      @Override
542      public boolean evaluate() throws Exception {
543        return ProtobufUtil.getOnlineRegions(targetRS).size() <= 0;
544      }
545    });
546
547    // Lets move this region to the new group.
548    TEST_UTIL.getAdmin().move(Bytes.toBytes(RegionInfo.encodeRegionName(
549        Bytes.toBytes(targetRegion))), Bytes.toBytes(targetServer.getServerName()));
550    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
551      @Override
552      public boolean evaluate() throws Exception {
553        return
554            getTableRegionMap().get(tableName) != null &&
555                getTableRegionMap().get(tableName).size() == 6 &&
556                admin.getClusterMetrics(EnumSet.of(Option.REGIONS_IN_TRANSITION))
557                     .getRegionStatesInTransition().size() < 1;
558      }
559    });
560
561    //verify that targetServer didn't open it
562    for (RegionInfo region: ProtobufUtil.getOnlineRegions(targetRS)) {
563      if (targetRegion.equals(region.getRegionNameAsString())) {
564        fail("Target server opened region");
565      }
566    }
567  }
568
569  @Test
570  public void testFailRemoveGroup() throws IOException, InterruptedException {
571    int initNumGroups = rsGroupAdmin.listRSGroups().size();
572    addGroup("bar", 3);
573    TEST_UTIL.createTable(tableName, Bytes.toBytes("f"));
574    rsGroupAdmin.moveTables(Sets.newHashSet(tableName), "bar");
575    RSGroupInfo barGroup = rsGroupAdmin.getRSGroupInfo("bar");
576    //group is not empty therefore it should fail
577    try {
578      rsGroupAdmin.removeRSGroup(barGroup.getName());
579      fail("Expected remove group to fail");
580    } catch(IOException e) {
581    }
582    //group cannot lose all it's servers therefore it should fail
583    try {
584      rsGroupAdmin.moveServers(barGroup.getServers(), RSGroupInfo.DEFAULT_GROUP);
585      fail("Expected move servers to fail");
586    } catch(IOException e) {
587    }
588
589    rsGroupAdmin.moveTables(barGroup.getTables(), RSGroupInfo.DEFAULT_GROUP);
590    try {
591      rsGroupAdmin.removeRSGroup(barGroup.getName());
592      fail("Expected move servers to fail");
593    } catch(IOException e) {
594    }
595
596    rsGroupAdmin.moveServers(barGroup.getServers(), RSGroupInfo.DEFAULT_GROUP);
597    rsGroupAdmin.removeRSGroup(barGroup.getName());
598
599    Assert.assertEquals(initNumGroups, rsGroupAdmin.listRSGroups().size());
600  }
601
602  @Test
603  public void testKillRS() throws Exception {
604    RSGroupInfo appInfo = addGroup("appInfo", 1);
605
606    final TableName tableName = TableName.valueOf(tablePrefix+"_ns", name.getMethodName());
607    admin.createNamespace(
608        NamespaceDescriptor.create(tableName.getNamespaceAsString())
609            .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, appInfo.getName()).build());
610    final HTableDescriptor desc = new HTableDescriptor(tableName);
611    desc.addFamily(new HColumnDescriptor("f"));
612    admin.createTable(desc);
613    //wait for created table to be assigned
614    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
615      @Override
616      public boolean evaluate() throws Exception {
617        return getTableRegionMap().get(desc.getTableName()) != null;
618      }
619    });
620
621    ServerName targetServer = ServerName.parseServerName(
622        appInfo.getServers().iterator().next().toString());
623    AdminProtos.AdminService.BlockingInterface targetRS =
624      ((ClusterConnection) admin.getConnection()).getAdmin(targetServer);
625    RegionInfo targetRegion = ProtobufUtil.getOnlineRegions(targetRS).get(0);
626    Assert.assertEquals(1, ProtobufUtil.getOnlineRegions(targetRS).size());
627
628    try {
629      //stopping may cause an exception
630      //due to the connection loss
631      targetRS.stopServer(null,
632          AdminProtos.StopServerRequest.newBuilder().setReason("Die").build());
633    } catch(Exception e) {
634    }
635    assertFalse(cluster.getClusterMetrics().getLiveServerMetrics().containsKey(targetServer));
636
637    //wait for created table to be assigned
638    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
639      @Override
640      public boolean evaluate() throws Exception {
641        return cluster.getClusterMetrics().getRegionStatesInTransition().isEmpty();
642      }
643    });
644    Set<Address> newServers = Sets.newHashSet();
645    newServers.add(
646        rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().iterator().next());
647    rsGroupAdmin.moveServers(newServers, appInfo.getName());
648
649    //Make sure all the table's regions get reassigned
650    //disabling the table guarantees no conflicting assign/unassign (ie SSH) happens
651    admin.disableTable(tableName);
652    admin.enableTable(tableName);
653
654    //wait for region to be assigned
655    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
656      @Override
657      public boolean evaluate() throws Exception {
658        return cluster.getClusterMetrics().getRegionStatesInTransition().isEmpty();
659      }
660    });
661
662    targetServer = ServerName.parseServerName(
663        newServers.iterator().next().toString());
664    targetRS =
665      ((ClusterConnection) admin.getConnection()).getAdmin(targetServer);
666    Assert.assertEquals(1, ProtobufUtil.getOnlineRegions(targetRS).size());
667    Assert.assertEquals(tableName,
668        ProtobufUtil.getOnlineRegions(targetRS).get(0).getTable());
669  }
670
671  @Test
672  public void testValidGroupNames() throws IOException {
673    String[] badNames = {"foo*","foo@","-"};
674    String[] goodNames = {"foo_123"};
675
676    for(String entry: badNames) {
677      try {
678        rsGroupAdmin.addRSGroup(entry);
679        fail("Expected a constraint exception for: "+entry);
680      } catch(ConstraintException ex) {
681        //expected
682      }
683    }
684
685    for(String entry: goodNames) {
686      rsGroupAdmin.addRSGroup(entry);
687    }
688  }
689
690  private String getGroupName(String baseName) {
691    return groupPrefix+"_"+baseName+"_"+rand.nextInt(Integer.MAX_VALUE);
692  }
693
694  @Test
695  public void testMultiTableMove() throws Exception {
696    final TableName tableNameA = TableName.valueOf(tablePrefix + name.getMethodName() + "A");
697    final TableName tableNameB = TableName.valueOf(tablePrefix + name.getMethodName() + "B");
698    final byte[] familyNameBytes = Bytes.toBytes("f");
699    String newGroupName = getGroupName(name.getMethodName());
700    final RSGroupInfo newGroup = addGroup(newGroupName, 1);
701
702    TEST_UTIL.createTable(tableNameA, familyNameBytes);
703    TEST_UTIL.createTable(tableNameB, familyNameBytes);
704    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
705      @Override
706      public boolean evaluate() throws Exception {
707        List<String> regionsA = getTableRegionMap().get(tableNameA);
708        if (regionsA == null) {
709          return false;
710        }
711
712        List<String> regionsB = getTableRegionMap().get(tableNameB);
713        if (regionsB == null) {
714          return false;
715        }
716
717        return getTableRegionMap().get(tableNameA).size() >= 1
718                && getTableRegionMap().get(tableNameB).size() >= 1;
719      }
720    });
721
722    RSGroupInfo tableGrpA = rsGroupAdmin.getRSGroupInfoOfTable(tableNameA);
723    assertTrue(tableGrpA.getName().equals(RSGroupInfo.DEFAULT_GROUP));
724
725    RSGroupInfo tableGrpB = rsGroupAdmin.getRSGroupInfoOfTable(tableNameB);
726    assertTrue(tableGrpB.getName().equals(RSGroupInfo.DEFAULT_GROUP));
727    //change table's group
728    LOG.info("Moving table [" + tableNameA + "," + tableNameB + "] to " + newGroup.getName());
729    rsGroupAdmin.moveTables(Sets.newHashSet(tableNameA, tableNameB), newGroup.getName());
730
731    //verify group change
732    Assert.assertEquals(newGroup.getName(),
733            rsGroupAdmin.getRSGroupInfoOfTable(tableNameA).getName());
734
735    Assert.assertEquals(newGroup.getName(),
736            rsGroupAdmin.getRSGroupInfoOfTable(tableNameB).getName());
737
738    //verify tables' not exist in old group
739    Set<TableName> DefaultTables = rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP)
740        .getTables();
741    assertFalse(DefaultTables.contains(tableNameA));
742    assertFalse(DefaultTables.contains(tableNameB));
743
744    //verify tables' exist in new group
745    Set<TableName> newGroupTables = rsGroupAdmin.getRSGroupInfo(newGroupName).getTables();
746    assertTrue(newGroupTables.contains(tableNameA));
747    assertTrue(newGroupTables.contains(tableNameB));
748  }
749
750  @Test
751  public void testDisabledTableMove() throws Exception {
752    final byte[] familyNameBytes = Bytes.toBytes("f");
753    String newGroupName = getGroupName(name.getMethodName());
754    final RSGroupInfo newGroup = addGroup(newGroupName, 2);
755
756    TEST_UTIL.createMultiRegionTable(tableName, familyNameBytes, 5);
757    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
758      @Override
759      public boolean evaluate() throws Exception {
760        List<String> regions = getTableRegionMap().get(tableName);
761        if (regions == null) {
762          return false;
763        }
764        return getTableRegionMap().get(tableName).size() >= 5;
765      }
766    });
767
768    RSGroupInfo tableGrp = rsGroupAdmin.getRSGroupInfoOfTable(tableName);
769    assertTrue(tableGrp.getName().equals(RSGroupInfo.DEFAULT_GROUP));
770
771    //test disable table
772    admin.disableTable(tableName);
773
774    //change table's group
775    LOG.info("Moving table "+ tableName + " to " + newGroup.getName());
776    rsGroupAdmin.moveTables(Sets.newHashSet(tableName), newGroup.getName());
777
778    //verify group change
779    Assert.assertEquals(newGroup.getName(),
780        rsGroupAdmin.getRSGroupInfoOfTable(tableName).getName());
781  }
782
783  @Test
784  public void testNonExistentTableMove() throws Exception {
785    TableName tableName = TableName.valueOf(tablePrefix + name.getMethodName());
786
787    RSGroupInfo tableGrp = rsGroupAdmin.getRSGroupInfoOfTable(tableName);
788    assertNull(tableGrp);
789
790    //test if table exists already.
791    boolean exist = admin.tableExists(tableName);
792    assertFalse(exist);
793
794    LOG.info("Moving table "+ tableName + " to " + RSGroupInfo.DEFAULT_GROUP);
795    try {
796      rsGroupAdmin.moveTables(Sets.newHashSet(tableName), RSGroupInfo.DEFAULT_GROUP);
797      fail("Table " + tableName + " shouldn't have been successfully moved.");
798    } catch(IOException ex) {
799      assertTrue(ex instanceof TableNotFoundException);
800    }
801
802    try {
803      rsGroupAdmin.moveServersAndTables(
804          Sets.newHashSet(Address.fromParts("bogus",123)),
805          Sets.newHashSet(tableName), RSGroupInfo.DEFAULT_GROUP);
806      fail("Table " + tableName + " shouldn't have been successfully moved.");
807    } catch(IOException ex) {
808      assertTrue(ex instanceof TableNotFoundException);
809    }
810    //verify group change
811    assertNull(rsGroupAdmin.getRSGroupInfoOfTable(tableName));
812  }
813
814  @Test
815  public void testMoveServersAndTables() throws Exception {
816    LOG.info("testMoveServersAndTables");
817    final RSGroupInfo newGroup = addGroup(getGroupName(name.getMethodName()), 1);
818    //create table
819    final byte[] familyNameBytes = Bytes.toBytes("f");
820    TEST_UTIL.createMultiRegionTable(tableName, familyNameBytes, 5);
821    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
822      @Override
823      public boolean evaluate() throws Exception {
824        List<String> regions = getTableRegionMap().get(tableName);
825        if (regions == null) {
826          return false;
827        }
828        return getTableRegionMap().get(tableName).size() >= 5;
829      }
830    });
831
832    //get server which is not a member of new group
833    ServerName targetServer = null;
834    for(ServerName server : admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS))
835      .getLiveServerMetrics().keySet()) {
836      if(!newGroup.containsServer(server.getAddress()) &&
837           !rsGroupAdmin.getRSGroupInfo("master").containsServer(server.getAddress())) {
838        targetServer = server;
839        break;
840      }
841    }
842
843    LOG.debug("Print group info : " + rsGroupAdmin.listRSGroups());
844    int oldDefaultGroupServerSize =
845            rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size();
846    int oldDefaultGroupTableSize =
847            rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getTables().size();
848
849    //test fail bogus server move
850    try {
851      rsGroupAdmin.moveServersAndTables(Sets.newHashSet(Address.fromString("foo:9999")),
852              Sets.newHashSet(tableName), newGroup.getName());
853      fail("Bogus servers shouldn't have been successfully moved.");
854    } catch(IOException ex) {
855      String exp = "Source RSGroup for server foo:9999 does not exist.";
856      String msg = "Expected '" + exp + "' in exception message: ";
857      assertTrue(msg + " " + ex.getMessage(), ex.getMessage().contains(exp));
858    }
859
860    //test fail server move
861    try {
862      rsGroupAdmin.moveServersAndTables(Sets.newHashSet(targetServer.getAddress()),
863              Sets.newHashSet(tableName), RSGroupInfo.DEFAULT_GROUP);
864      fail("servers shouldn't have been successfully moved.");
865    } catch(IOException ex) {
866      String exp = "Target RSGroup " + RSGroupInfo.DEFAULT_GROUP +
867              " is same as source " + RSGroupInfo.DEFAULT_GROUP + " RSGroup.";
868      String msg = "Expected '" + exp + "' in exception message: ";
869      assertTrue(msg + " " + ex.getMessage(), ex.getMessage().contains(exp));
870    }
871
872    //verify default group info
873    Assert.assertEquals(oldDefaultGroupServerSize,
874            rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size());
875    Assert.assertEquals(oldDefaultGroupTableSize,
876            rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getTables().size());
877
878    //verify new group info
879    Assert.assertEquals(1,
880            rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getServers().size());
881    Assert.assertEquals(0,
882            rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getTables().size());
883
884    //get all region to move targetServer
885    List<String> regionList = getTableRegionMap().get(tableName);
886    for(String region : regionList) {
887      // Lets move this region to the targetServer
888      TEST_UTIL.getAdmin().move(Bytes.toBytes(RegionInfo.encodeRegionName(Bytes.toBytes(region))),
889              Bytes.toBytes(targetServer.getServerName()));
890    }
891
892    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
893      @Override
894      public boolean evaluate() throws Exception {
895        return getTableRegionMap().get(tableName) != null &&
896                getTableRegionMap().get(tableName).size() == 5 &&
897                getTableServerRegionMap().get(tableName).size() == 1 &&
898                admin.getClusterMetrics(EnumSet.of(Option.REGIONS_IN_TRANSITION))
899                     .getRegionStatesInTransition().size() < 1;
900      }
901    });
902
903    //verify that all region move to targetServer
904    Assert.assertEquals(5, getTableServerRegionMap().get(tableName).get(targetServer).size());
905
906    //move targetServer and table to newGroup
907    LOG.info("moving server and table to newGroup");
908    rsGroupAdmin.moveServersAndTables(Sets.newHashSet(targetServer.getAddress()),
909            Sets.newHashSet(tableName), newGroup.getName());
910
911    //verify group change
912    Assert.assertEquals(newGroup.getName(),
913            rsGroupAdmin.getRSGroupInfoOfTable(tableName).getName());
914
915    //verify servers' not exist in old group
916    Set<Address> defaultServers = rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP)
917        .getServers();
918    assertFalse(defaultServers.contains(targetServer.getAddress()));
919
920    //verify servers' exist in new group
921    Set<Address> newGroupServers = rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getServers();
922    assertTrue(newGroupServers.contains(targetServer.getAddress()));
923
924    //verify tables' not exist in old group
925    Set<TableName> defaultTables = rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP)
926        .getTables();
927    assertFalse(defaultTables.contains(tableName));
928
929    //verify tables' exist in new group
930    Set<TableName> newGroupTables = rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getTables();
931    assertTrue(newGroupTables.contains(tableName));
932
933    //verify that all region still assgin on targetServer
934    Assert.assertEquals(5, getTableServerRegionMap().get(tableName).get(targetServer).size());
935  }
936
937  @Test
938  public void testClearDeadServers() throws Exception {
939    LOG.info("testClearDeadServers");
940    final RSGroupInfo newGroup = addGroup(getGroupName(name.getMethodName()), 3);
941    NUM_DEAD_SERVERS = cluster.getClusterMetrics().getDeadServerNames().size();
942
943    ServerName targetServer = ServerName.parseServerName(
944        newGroup.getServers().iterator().next().toString());
945    AdminProtos.AdminService.BlockingInterface targetRS =
946        ((ClusterConnection) admin.getConnection()).getAdmin(targetServer);
947    try {
948      targetServer = ProtobufUtil.toServerName(targetRS.getServerInfo(null,
949          GetServerInfoRequest.newBuilder().build()).getServerInfo().getServerName());
950      //stopping may cause an exception
951      //due to the connection loss
952      targetRS.stopServer(null,
953          AdminProtos.StopServerRequest.newBuilder().setReason("Die").build());
954      NUM_DEAD_SERVERS ++;
955    } catch(Exception e) {
956    }
957    //wait for stopped regionserver to dead server list
958    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
959      @Override
960      public boolean evaluate() throws Exception {
961        return !master.getServerManager().areDeadServersInProgress()
962            && cluster.getClusterMetrics().getDeadServerNames().size() == NUM_DEAD_SERVERS;
963      }
964    });
965    assertFalse(cluster.getClusterMetrics().getLiveServerMetrics().containsKey(targetServer));
966    assertTrue(cluster.getClusterMetrics().getDeadServerNames().contains(targetServer));
967    assertTrue(newGroup.getServers().contains(targetServer.getAddress()));
968
969    //clear dead servers list
970    List<ServerName> notClearedServers = admin.clearDeadServers(Lists.newArrayList(targetServer));
971    assertEquals(0, notClearedServers.size());
972
973    Set<Address> newGroupServers = rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getServers();
974    assertFalse(newGroupServers.contains(targetServer.getAddress()));
975    assertEquals(2, newGroupServers.size());
976  }
977
978  @Test
979  public void testRemoveServers() throws Exception {
980    LOG.info("testRemoveServers");
981    final RSGroupInfo newGroup = addGroup(getGroupName(name.getMethodName()), 3);
982    Iterator<Address> iterator = newGroup.getServers().iterator();
983    ServerName targetServer = ServerName.parseServerName(iterator.next().toString());
984
985    // remove online servers
986    try {
987      rsGroupAdmin.removeServers(Sets.newHashSet(targetServer.getAddress()));
988      fail("Online servers shouldn't have been successfully removed.");
989    } catch(IOException ex) {
990      String exp = "Server " + targetServer.getAddress()
991          + " is an online server, not allowed to remove.";
992      String msg = "Expected '" + exp + "' in exception message: ";
993      assertTrue(msg + " " + ex.getMessage(), ex.getMessage().contains(exp));
994    }
995    assertTrue(newGroup.getServers().contains(targetServer.getAddress()));
996
997    // remove dead servers
998    NUM_DEAD_SERVERS = cluster.getClusterMetrics().getDeadServerNames().size();
999    AdminProtos.AdminService.BlockingInterface targetRS =
1000        ((ClusterConnection) admin.getConnection()).getAdmin(targetServer);
1001    try {
1002      targetServer = ProtobufUtil.toServerName(targetRS.getServerInfo(null,
1003          GetServerInfoRequest.newBuilder().build()).getServerInfo().getServerName());
1004      //stopping may cause an exception
1005      //due to the connection loss
1006      LOG.info("stopping server " + targetServer.getHostAndPort());
1007      targetRS.stopServer(null,
1008          AdminProtos.StopServerRequest.newBuilder().setReason("Die").build());
1009      NUM_DEAD_SERVERS ++;
1010    } catch(Exception e) {
1011    }
1012
1013    //wait for stopped regionserver to dead server list
1014    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
1015      @Override
1016      public boolean evaluate() throws Exception {
1017        return !master.getServerManager().areDeadServersInProgress()
1018            && cluster.getClusterMetrics().getDeadServerNames().size() == NUM_DEAD_SERVERS;
1019      }
1020    });
1021
1022    try {
1023      rsGroupAdmin.removeServers(Sets.newHashSet(targetServer.getAddress()));
1024      fail("Dead servers shouldn't have been successfully removed.");
1025    } catch(IOException ex) {
1026      String exp = "Server " + targetServer.getAddress() + " is on the dead servers list,"
1027          + " Maybe it will come back again, not allowed to remove.";
1028      String msg = "Expected '" + exp + "' in exception message: ";
1029      assertTrue(msg + " " + ex.getMessage(), ex.getMessage().contains(exp));
1030    }
1031    assertTrue(newGroup.getServers().contains(targetServer.getAddress()));
1032
1033    // remove decommissioned servers
1034    List<ServerName> serversToDecommission = new ArrayList<>();
1035    targetServer = ServerName.parseServerName(iterator.next().toString());
1036    targetRS = ((ClusterConnection) admin.getConnection()).getAdmin(targetServer);
1037    targetServer = ProtobufUtil.toServerName(targetRS.getServerInfo(null,
1038          GetServerInfoRequest.newBuilder().build()).getServerInfo().getServerName());
1039    assertTrue(master.getServerManager().getOnlineServers().containsKey(targetServer));
1040    serversToDecommission.add(targetServer);
1041
1042    admin.decommissionRegionServers(serversToDecommission, true);
1043    assertEquals(1, admin.listDecommissionedRegionServers().size());
1044
1045    assertTrue(newGroup.getServers().contains(targetServer.getAddress()));
1046    rsGroupAdmin.removeServers(Sets.newHashSet(targetServer.getAddress()));
1047    Set<Address> newGroupServers = rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getServers();
1048    assertFalse(newGroupServers.contains(targetServer.getAddress()));
1049    assertEquals(2, newGroupServers.size());
1050  }
1051
1052  @Test
1053  public void testCreateWhenRsgroupNoOnlineServers() throws Exception {
1054    LOG.info("testCreateWhenRsgroupNoOnlineServers");
1055
1056    // set rsgroup has no online servers and test create table
1057    final RSGroupInfo appInfo = addGroup("appInfo", 1);
1058    Iterator<Address> iterator = appInfo.getServers().iterator();
1059    List<ServerName> serversToDecommission = new ArrayList<>();
1060    ServerName targetServer = ServerName.parseServerName(iterator.next().toString());
1061    AdminProtos.AdminService.BlockingInterface targetRS =
1062        ((ClusterConnection) admin.getConnection()).getAdmin(targetServer);
1063    targetServer = ProtobufUtil.toServerName(
1064        targetRS.getServerInfo(null, GetServerInfoRequest.newBuilder().build()).getServerInfo()
1065            .getServerName());
1066    assertTrue(master.getServerManager().getOnlineServers().containsKey(targetServer));
1067    serversToDecommission.add(targetServer);
1068    admin.decommissionRegionServers(serversToDecommission, true);
1069    assertEquals(1, admin.listDecommissionedRegionServers().size());
1070
1071    final TableName tableName = TableName.valueOf(tablePrefix + "_ns", name.getMethodName());
1072    admin.createNamespace(NamespaceDescriptor.create(tableName.getNamespaceAsString())
1073        .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, appInfo.getName()).build());
1074    final HTableDescriptor desc = new HTableDescriptor(tableName);
1075    desc.addFamily(new HColumnDescriptor("f"));
1076    try {
1077      admin.createTable(desc);
1078      fail("Shouldn't create table successfully!");
1079    } catch (Exception e) {
1080      LOG.debug("create table error", e);
1081    }
1082
1083    // recommission and test create table
1084    admin.recommissionRegionServer(targetServer, null);
1085    assertEquals(0, admin.listDecommissionedRegionServers().size());
1086    admin.createTable(desc);
1087    // wait for created table to be assigned
1088    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
1089      @Override public boolean evaluate() throws Exception {
1090        return getTableRegionMap().get(desc.getTableName()) != null;
1091      }
1092    });
1093  }
1094}