Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.rsgroup;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertNull;
023import static org.junit.Assert.assertTrue;
024import static org.junit.Assert.fail;
025
026import java.io.IOException;
027import java.security.SecureRandom;
028import java.util.ArrayList;
029import java.util.EnumSet;
030import java.util.Iterator;
031import java.util.LinkedList;
032import java.util.List;
033import java.util.Map;
034import java.util.Set;
035import java.util.TreeMap;
036import org.apache.hadoop.hbase.ClusterMetrics;
037import org.apache.hadoop.hbase.ClusterMetrics.Option;
038import org.apache.hadoop.hbase.HBaseCluster;
039import org.apache.hadoop.hbase.HBaseTestingUtility;
040import org.apache.hadoop.hbase.HColumnDescriptor;
041import org.apache.hadoop.hbase.HTableDescriptor;
042import org.apache.hadoop.hbase.NamespaceDescriptor;
043import org.apache.hadoop.hbase.RegionMetrics;
044import org.apache.hadoop.hbase.ServerMetrics;
045import org.apache.hadoop.hbase.ServerName;
046import org.apache.hadoop.hbase.TableName;
047import org.apache.hadoop.hbase.TableNotFoundException;
048import org.apache.hadoop.hbase.Waiter;
049import org.apache.hadoop.hbase.client.Admin;
050import org.apache.hadoop.hbase.client.ClusterConnection;
051import org.apache.hadoop.hbase.client.RegionInfo;
052import org.apache.hadoop.hbase.constraint.ConstraintException;
053import org.apache.hadoop.hbase.master.HMaster;
054import org.apache.hadoop.hbase.net.Address;
055import org.apache.hadoop.hbase.util.Bytes;
056import org.junit.Assert;
057import org.junit.Before;
058import org.junit.Rule;
059import org.junit.Test;
060import org.junit.rules.TestName;
061import org.slf4j.Logger;
062import org.slf4j.LoggerFactory;
063
064import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
065import org.apache.hbase.thirdparty.com.google.common.collect.Maps;
066import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
067import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
068import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos;
069import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetServerInfoRequest;
070
071public abstract class TestRSGroupsBase {
072  protected static final Logger LOG = LoggerFactory.getLogger(TestRSGroupsBase.class);
073  @Rule
074  public TestName name = new TestName();
075
076  //shared
077  protected final static String groupPrefix = "Group";
078  protected final static String tablePrefix = "Group";
079  protected final static SecureRandom rand = new SecureRandom();
080
081  //shared, cluster type specific
082  protected static HBaseTestingUtility TEST_UTIL;
083  protected static Admin admin;
084  protected static HBaseCluster cluster;
085  protected static RSGroupAdmin rsGroupAdmin;
086  protected static HMaster master;
087
088  public final static long WAIT_TIMEOUT = 60000*5;
089  public final static int NUM_SLAVES_BASE = 4; //number of slaves for the smallest cluster
090  public static int NUM_DEAD_SERVERS = 0;
091
092  // Per test variables
093  TableName tableName;
094  @Before
095  public void setup() {
096    LOG.info(name.getMethodName());
097    tableName = TableName.valueOf(tablePrefix + "_" + name.getMethodName());
098  }
099
100  protected RSGroupInfo addGroup(String groupName, int serverCount)
101      throws IOException, InterruptedException {
102    return RSGroupTestingUtil.addRSGroup(rsGroupAdmin, groupName, serverCount);
103  }
104
105  void removeGroup(String groupName) throws IOException {
106    RSGroupInfo RSGroupInfo = rsGroupAdmin.getRSGroupInfo(groupName);
107    rsGroupAdmin.moveTables(RSGroupInfo.getTables(), RSGroupInfo.DEFAULT_GROUP);
108    rsGroupAdmin.moveServers(RSGroupInfo.getServers(), RSGroupInfo.DEFAULT_GROUP);
109    rsGroupAdmin.removeRSGroup(groupName);
110  }
111
112  protected void deleteTableIfNecessary() throws IOException {
113    for (HTableDescriptor desc : TEST_UTIL.getAdmin().listTables(tablePrefix+".*")) {
114      TEST_UTIL.deleteTable(desc.getTableName());
115    }
116  }
117
118  protected void deleteNamespaceIfNecessary() throws IOException {
119    for (NamespaceDescriptor desc : TEST_UTIL.getAdmin().listNamespaceDescriptors()) {
120      if(desc.getName().startsWith(tablePrefix)) {
121        admin.deleteNamespace(desc.getName());
122      }
123    }
124  }
125
126  protected void deleteGroups() throws IOException {
127    RSGroupAdmin groupAdmin =
128        new RSGroupAdminClient(TEST_UTIL.getConnection());
129    for(RSGroupInfo group: groupAdmin.listRSGroups()) {
130      if(!group.getName().equals(RSGroupInfo.DEFAULT_GROUP)) {
131        groupAdmin.moveTables(group.getTables(), RSGroupInfo.DEFAULT_GROUP);
132        groupAdmin.moveServers(group.getServers(), RSGroupInfo.DEFAULT_GROUP);
133        groupAdmin.removeRSGroup(group.getName());
134      }
135    }
136  }
137
138  public Map<TableName, List<String>> getTableRegionMap() throws IOException {
139    Map<TableName, List<String>> map = Maps.newTreeMap();
140    Map<TableName, Map<ServerName, List<String>>> tableServerRegionMap
141        = getTableServerRegionMap();
142    for(TableName tableName : tableServerRegionMap.keySet()) {
143      if(!map.containsKey(tableName)) {
144        map.put(tableName, new LinkedList<>());
145      }
146      for(List<String> subset: tableServerRegionMap.get(tableName).values()) {
147        map.get(tableName).addAll(subset);
148      }
149    }
150    return map;
151  }
152
153  public Map<TableName, Map<ServerName, List<String>>> getTableServerRegionMap()
154      throws IOException {
155    Map<TableName, Map<ServerName, List<String>>> map = Maps.newTreeMap();
156    ClusterMetrics status = TEST_UTIL.getHBaseClusterInterface().getClusterMetrics();
157    for (Map.Entry<ServerName, ServerMetrics> entry : status.getLiveServerMetrics().entrySet()) {
158      ServerName serverName = entry.getKey();
159      for(RegionMetrics rl : entry.getValue().getRegionMetrics().values()) {
160        TableName tableName = null;
161        try {
162          tableName = RegionInfo.getTable(rl.getRegionName());
163        } catch (IllegalArgumentException e) {
164          LOG.warn("Failed parse a table name from regionname=" +
165            Bytes.toStringBinary(rl.getRegionName()));
166          continue;
167        }
168        if(!map.containsKey(tableName)) {
169          map.put(tableName, new TreeMap<>());
170        }
171        if(!map.get(tableName).containsKey(serverName)) {
172          map.get(tableName).put(serverName, new LinkedList<>());
173        }
174        map.get(tableName).get(serverName).add(rl.getNameAsString());
175      }
176    }
177    return map;
178  }
179
180  @Test
181  public void testBogusArgs() throws Exception {
182    assertNull(rsGroupAdmin.getRSGroupInfoOfTable(TableName.valueOf("nonexistent")));
183    assertNull(rsGroupAdmin.getRSGroupOfServer(Address.fromParts("bogus",123)));
184    assertNull(rsGroupAdmin.getRSGroupInfo("bogus"));
185
186    try {
187      rsGroupAdmin.removeRSGroup("bogus");
188      fail("Expected removing bogus group to fail");
189    } catch(ConstraintException ex) {
190      //expected
191    }
192
193    try {
194      rsGroupAdmin.moveTables(Sets.newHashSet(TableName.valueOf("bogustable")), "bogus");
195      fail("Expected move with bogus group to fail");
196    } catch(ConstraintException|TableNotFoundException ex) {
197      //expected
198    }
199
200    try {
201      rsGroupAdmin.moveServers(Sets.newHashSet(Address.fromParts("bogus",123)), "bogus");
202      fail("Expected move with bogus group to fail");
203    } catch(ConstraintException ex) {
204      //expected
205    }
206
207    try {
208      admin.setBalancerRunning(true,true);
209      rsGroupAdmin.balanceRSGroup("bogus");
210      admin.setBalancerRunning(false,true);
211      fail("Expected move with bogus group to fail");
212    } catch(ConstraintException ex) {
213      //expected
214    }
215  }
216
217  @Test
218  public void testCreateMultiRegion() throws IOException {
219    byte[] end = {1,3,5,7,9};
220    byte[] start = {0,2,4,6,8};
221    byte[][] f = {Bytes.toBytes("f")};
222    TEST_UTIL.createTable(tableName, f,1,start,end,10);
223  }
224
225  @Test
226  public void testCreateAndDrop() throws Exception {
227    TEST_UTIL.createTable(tableName, Bytes.toBytes("cf"));
228    //wait for created table to be assigned
229    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
230      @Override
231      public boolean evaluate() throws Exception {
232        return getTableRegionMap().get(tableName) != null;
233      }
234    });
235    TEST_UTIL.deleteTable(tableName);
236  }
237
238
239  @Test
240  public void testSimpleRegionServerMove() throws IOException,
241      InterruptedException {
242    int initNumGroups = rsGroupAdmin.listRSGroups().size();
243    RSGroupInfo appInfo = addGroup(getGroupName(name.getMethodName()), 1);
244    RSGroupInfo adminInfo = addGroup(getGroupName(name.getMethodName()), 1);
245    RSGroupInfo dInfo = rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP);
246    Assert.assertEquals(initNumGroups + 2, rsGroupAdmin.listRSGroups().size());
247    assertEquals(1, adminInfo.getServers().size());
248    assertEquals(1, appInfo.getServers().size());
249    assertEquals(getNumServers() - 2, dInfo.getServers().size());
250    rsGroupAdmin.moveServers(appInfo.getServers(),
251        RSGroupInfo.DEFAULT_GROUP);
252    rsGroupAdmin.removeRSGroup(appInfo.getName());
253    rsGroupAdmin.moveServers(adminInfo.getServers(),
254        RSGroupInfo.DEFAULT_GROUP);
255    rsGroupAdmin.removeRSGroup(adminInfo.getName());
256    Assert.assertEquals(rsGroupAdmin.listRSGroups().size(), initNumGroups);
257  }
258
259  // return the real number of region servers, excluding the master embedded region server in 2.0+
260  public int getNumServers() throws IOException {
261    ClusterMetrics status =
262        admin.getClusterMetrics(EnumSet.of(Option.MASTER, Option.LIVE_SERVERS));
263    ServerName masterName = status.getMasterName();
264    int count = 0;
265    for (ServerName sn : status.getLiveServerMetrics().keySet()) {
266      if (!sn.equals(masterName)) {
267        count++;
268      }
269    }
270    return count;
271  }
272
273  @Test
274  public void testMoveServers() throws Exception {
275    //create groups and assign servers
276    addGroup("bar", 3);
277    rsGroupAdmin.addRSGroup("foo");
278
279    RSGroupInfo barGroup = rsGroupAdmin.getRSGroupInfo("bar");
280    RSGroupInfo fooGroup = rsGroupAdmin.getRSGroupInfo("foo");
281    assertEquals(3, barGroup.getServers().size());
282    assertEquals(0, fooGroup.getServers().size());
283
284    //test fail bogus server move
285    try {
286      rsGroupAdmin.moveServers(Sets.newHashSet(Address.fromString("foo:9999")),"foo");
287      fail("Bogus servers shouldn't have been successfully moved.");
288    } catch(IOException ex) {
289      String exp = "Source RSGroup for server foo:9999 does not exist.";
290      String msg = "Expected '"+exp+"' in exception message: ";
291      assertTrue(msg+" "+ex.getMessage(), ex.getMessage().contains(exp));
292    }
293
294    //test success case
295    LOG.info("moving servers "+barGroup.getServers()+" to group foo");
296    rsGroupAdmin.moveServers(barGroup.getServers(), fooGroup.getName());
297
298    barGroup = rsGroupAdmin.getRSGroupInfo("bar");
299    fooGroup = rsGroupAdmin.getRSGroupInfo("foo");
300    assertEquals(0,barGroup.getServers().size());
301    assertEquals(3,fooGroup.getServers().size());
302
303    LOG.info("moving servers "+fooGroup.getServers()+" to group default");
304    rsGroupAdmin.moveServers(fooGroup.getServers(), RSGroupInfo.DEFAULT_GROUP);
305
306    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
307      @Override
308      public boolean evaluate() throws Exception {
309        return getNumServers() ==
310        rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size();
311      }
312    });
313
314    fooGroup = rsGroupAdmin.getRSGroupInfo("foo");
315    assertEquals(0,fooGroup.getServers().size());
316
317    //test group removal
318    LOG.info("Remove group "+barGroup.getName());
319    rsGroupAdmin.removeRSGroup(barGroup.getName());
320    Assert.assertEquals(null, rsGroupAdmin.getRSGroupInfo(barGroup.getName()));
321    LOG.info("Remove group "+fooGroup.getName());
322    rsGroupAdmin.removeRSGroup(fooGroup.getName());
323    Assert.assertEquals(null, rsGroupAdmin.getRSGroupInfo(fooGroup.getName()));
324  }
325
326  @Test
327  public void testTableMoveTruncateAndDrop() throws Exception {
328    final byte[] familyNameBytes = Bytes.toBytes("f");
329    String newGroupName = getGroupName(name.getMethodName());
330    final RSGroupInfo newGroup = addGroup(newGroupName, 2);
331
332    TEST_UTIL.createMultiRegionTable(tableName, familyNameBytes, 5);
333    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
334      @Override
335      public boolean evaluate() throws Exception {
336        List<String> regions = getTableRegionMap().get(tableName);
337        if (regions == null) {
338          return false;
339        }
340
341        return getTableRegionMap().get(tableName).size() >= 5;
342      }
343    });
344
345    RSGroupInfo tableGrp = rsGroupAdmin.getRSGroupInfoOfTable(tableName);
346    assertTrue(tableGrp.getName().equals(RSGroupInfo.DEFAULT_GROUP));
347
348    //change table's group
349    LOG.info("Moving table "+tableName+" to "+newGroup.getName());
350    rsGroupAdmin.moveTables(Sets.newHashSet(tableName), newGroup.getName());
351
352    //verify group change
353    Assert.assertEquals(newGroup.getName(),
354        rsGroupAdmin.getRSGroupInfoOfTable(tableName).getName());
355
356    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
357      @Override
358      public boolean evaluate() throws Exception {
359        Map<ServerName, List<String>> serverMap = getTableServerRegionMap().get(tableName);
360        int count = 0;
361        if (serverMap != null) {
362          for (ServerName rs : serverMap.keySet()) {
363            if (newGroup.containsServer(rs.getAddress())) {
364              count += serverMap.get(rs).size();
365            }
366          }
367        }
368        return count == 5;
369      }
370    });
371
372    //test truncate
373    admin.disableTable(tableName);
374    admin.truncateTable(tableName, true);
375    Assert.assertEquals(1, rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getTables().size());
376    Assert.assertEquals(tableName, rsGroupAdmin.getRSGroupInfo(
377        newGroup.getName()).getTables().first());
378
379    //verify removed table is removed from group
380    TEST_UTIL.deleteTable(tableName);
381    Assert.assertEquals(0, rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getTables().size());
382  }
383
384  @Test
385  public void testGroupBalance() throws Exception {
386    LOG.info(name.getMethodName());
387    String newGroupName = getGroupName(name.getMethodName());
388    final RSGroupInfo newGroup = addGroup(newGroupName, 3);
389
390    final TableName tableName = TableName.valueOf(tablePrefix+"_ns", name.getMethodName());
391    admin.createNamespace(
392        NamespaceDescriptor.create(tableName.getNamespaceAsString())
393            .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, newGroupName).build());
394    final byte[] familyNameBytes = Bytes.toBytes("f");
395    final HTableDescriptor desc = new HTableDescriptor(tableName);
396    desc.addFamily(new HColumnDescriptor("f"));
397    byte [] startKey = Bytes.toBytes("aaaaa");
398    byte [] endKey = Bytes.toBytes("zzzzz");
399    admin.createTable(desc, startKey, endKey, 6);
400    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
401      @Override
402      public boolean evaluate() throws Exception {
403        List<String> regions = getTableRegionMap().get(tableName);
404        if (regions == null) {
405          return false;
406        }
407        return regions.size() >= 6;
408      }
409    });
410
411    //make assignment uneven, move all regions to one server
412    Map<ServerName,List<String>> assignMap =
413        getTableServerRegionMap().get(tableName);
414    final ServerName first = assignMap.entrySet().iterator().next().getKey();
415    for(RegionInfo region: admin.getTableRegions(tableName)) {
416      if(!assignMap.get(first).contains(region.getRegionNameAsString())) {
417        admin.move(region.getEncodedNameAsBytes(), Bytes.toBytes(first.getServerName()));
418      }
419    }
420    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
421      @Override
422      public boolean evaluate() throws Exception {
423        Map<ServerName, List<String>> map = getTableServerRegionMap().get(tableName);
424        if (map == null) {
425          return true;
426        }
427        List<String> regions = map.get(first);
428        if (regions == null) {
429          return true;
430        }
431        return regions.size() >= 6;
432      }
433    });
434
435    //balance the other group and make sure it doesn't affect the new group
436    admin.setBalancerRunning(true,true);
437    rsGroupAdmin.balanceRSGroup(RSGroupInfo.DEFAULT_GROUP);
438    assertEquals(6, getTableServerRegionMap().get(tableName).get(first).size());
439
440    //disable balance, balancer will not be run and return false
441    admin.setBalancerRunning(false,true);
442    assertFalse(rsGroupAdmin.balanceRSGroup(newGroupName));
443    assertEquals(6, getTableServerRegionMap().get(tableName).get(first).size());
444
445    //enable balance
446    admin.setBalancerRunning(true,true);
447    rsGroupAdmin.balanceRSGroup(newGroupName);
448    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
449      @Override
450      public boolean evaluate() throws Exception {
451        for (List<String> regions : getTableServerRegionMap().get(tableName).values()) {
452          if (2 != regions.size()) {
453            return false;
454          }
455        }
456        return true;
457      }
458    });
459    admin.setBalancerRunning(false,true);
460  }
461
462  @Test
463  public void testRegionMove() throws Exception {
464    final RSGroupInfo newGroup = addGroup(getGroupName(name.getMethodName()), 1);
465    final byte[] familyNameBytes = Bytes.toBytes("f");
466    // All the regions created below will be assigned to the default group.
467    TEST_UTIL.createMultiRegionTable(tableName, familyNameBytes, 6);
468    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
469      @Override
470      public boolean evaluate() throws Exception {
471        List<String> regions = getTableRegionMap().get(tableName);
472        if (regions == null) {
473          return false;
474        }
475        return getTableRegionMap().get(tableName).size() >= 6;
476      }
477    });
478
479    //get target region to move
480    Map<ServerName,List<String>> assignMap =
481        getTableServerRegionMap().get(tableName);
482    String targetRegion = null;
483    for(ServerName server : assignMap.keySet()) {
484      targetRegion = assignMap.get(server).size() > 0 ? assignMap.get(server).get(0) : null;
485      if(targetRegion != null) {
486        break;
487      }
488    }
489    //get server which is not a member of new group
490    ServerName targetServer = null;
491    for (ServerName server : admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS))
492                                  .getLiveServerMetrics().keySet()) {
493      if (!newGroup.containsServer(server.getAddress())) {
494        targetServer = server;
495        break;
496      }
497    }
498
499    final AdminProtos.AdminService.BlockingInterface targetRS =
500      ((ClusterConnection) admin.getConnection()).getAdmin(targetServer);
501
502    //move target server to group
503    rsGroupAdmin.moveServers(Sets.newHashSet(targetServer.getAddress()),
504        newGroup.getName());
505    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
506      @Override
507      public boolean evaluate() throws Exception {
508        return ProtobufUtil.getOnlineRegions(targetRS).size() <= 0;
509      }
510    });
511
512    // Lets move this region to the new group.
513    TEST_UTIL.getAdmin().move(Bytes.toBytes(RegionInfo.encodeRegionName(
514        Bytes.toBytes(targetRegion))), Bytes.toBytes(targetServer.getServerName()));
515    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
516      @Override
517      public boolean evaluate() throws Exception {
518        return
519            getTableRegionMap().get(tableName) != null &&
520                getTableRegionMap().get(tableName).size() == 6 &&
521                admin.getClusterMetrics(EnumSet.of(Option.REGIONS_IN_TRANSITION))
522                     .getRegionStatesInTransition().size() < 1;
523      }
524    });
525
526    //verify that targetServer didn't open it
527    for (RegionInfo region: ProtobufUtil.getOnlineRegions(targetRS)) {
528      if (targetRegion.equals(region.getRegionNameAsString())) {
529        fail("Target server opened region");
530      }
531    }
532  }
533
534  @Test
535  public void testFailRemoveGroup() throws IOException, InterruptedException {
536    int initNumGroups = rsGroupAdmin.listRSGroups().size();
537    addGroup("bar", 3);
538    TEST_UTIL.createTable(tableName, Bytes.toBytes("f"));
539    rsGroupAdmin.moveTables(Sets.newHashSet(tableName), "bar");
540    RSGroupInfo barGroup = rsGroupAdmin.getRSGroupInfo("bar");
541    //group is not empty therefore it should fail
542    try {
543      rsGroupAdmin.removeRSGroup(barGroup.getName());
544      fail("Expected remove group to fail");
545    } catch(IOException e) {
546    }
547    //group cannot lose all it's servers therefore it should fail
548    try {
549      rsGroupAdmin.moveServers(barGroup.getServers(), RSGroupInfo.DEFAULT_GROUP);
550      fail("Expected move servers to fail");
551    } catch(IOException e) {
552    }
553
554    rsGroupAdmin.moveTables(barGroup.getTables(), RSGroupInfo.DEFAULT_GROUP);
555    try {
556      rsGroupAdmin.removeRSGroup(barGroup.getName());
557      fail("Expected move servers to fail");
558    } catch(IOException e) {
559    }
560
561    rsGroupAdmin.moveServers(barGroup.getServers(), RSGroupInfo.DEFAULT_GROUP);
562    rsGroupAdmin.removeRSGroup(barGroup.getName());
563
564    Assert.assertEquals(initNumGroups, rsGroupAdmin.listRSGroups().size());
565  }
566
567  @Test
568  public void testKillRS() throws Exception {
569    RSGroupInfo appInfo = addGroup("appInfo", 1);
570
571    final TableName tableName = TableName.valueOf(tablePrefix+"_ns", name.getMethodName());
572    admin.createNamespace(
573        NamespaceDescriptor.create(tableName.getNamespaceAsString())
574            .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, appInfo.getName()).build());
575    final HTableDescriptor desc = new HTableDescriptor(tableName);
576    desc.addFamily(new HColumnDescriptor("f"));
577    admin.createTable(desc);
578    //wait for created table to be assigned
579    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
580      @Override
581      public boolean evaluate() throws Exception {
582        return getTableRegionMap().get(desc.getTableName()) != null;
583      }
584    });
585
586    ServerName targetServer = ServerName.parseServerName(
587        appInfo.getServers().iterator().next().toString());
588    AdminProtos.AdminService.BlockingInterface targetRS =
589      ((ClusterConnection) admin.getConnection()).getAdmin(targetServer);
590    RegionInfo targetRegion = ProtobufUtil.getOnlineRegions(targetRS).get(0);
591    Assert.assertEquals(1, ProtobufUtil.getOnlineRegions(targetRS).size());
592
593    try {
594      //stopping may cause an exception
595      //due to the connection loss
596      targetRS.stopServer(null,
597          AdminProtos.StopServerRequest.newBuilder().setReason("Die").build());
598    } catch(Exception e) {
599    }
600    assertFalse(cluster.getClusterMetrics().getLiveServerMetrics().containsKey(targetServer));
601
602    //wait for created table to be assigned
603    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
604      @Override
605      public boolean evaluate() throws Exception {
606        return cluster.getClusterMetrics().getRegionStatesInTransition().isEmpty();
607      }
608    });
609    Set<Address> newServers = Sets.newHashSet();
610    newServers.add(
611        rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().iterator().next());
612    rsGroupAdmin.moveServers(newServers, appInfo.getName());
613
614    //Make sure all the table's regions get reassigned
615    //disabling the table guarantees no conflicting assign/unassign (ie SSH) happens
616    admin.disableTable(tableName);
617    admin.enableTable(tableName);
618
619    //wait for region to be assigned
620    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
621      @Override
622      public boolean evaluate() throws Exception {
623        return cluster.getClusterMetrics().getRegionStatesInTransition().isEmpty();
624      }
625    });
626
627    targetServer = ServerName.parseServerName(
628        newServers.iterator().next().toString());
629    targetRS =
630      ((ClusterConnection) admin.getConnection()).getAdmin(targetServer);
631    Assert.assertEquals(1, ProtobufUtil.getOnlineRegions(targetRS).size());
632    Assert.assertEquals(tableName,
633        ProtobufUtil.getOnlineRegions(targetRS).get(0).getTable());
634  }
635
636  @Test
637  public void testValidGroupNames() throws IOException {
638    String[] badNames = {"foo*","foo@","-"};
639    String[] goodNames = {"foo_123"};
640
641    for(String entry: badNames) {
642      try {
643        rsGroupAdmin.addRSGroup(entry);
644        fail("Expected a constraint exception for: "+entry);
645      } catch(ConstraintException ex) {
646        //expected
647      }
648    }
649
650    for(String entry: goodNames) {
651      rsGroupAdmin.addRSGroup(entry);
652    }
653  }
654
655  private String getGroupName(String baseName) {
656    return groupPrefix+"_"+baseName+"_"+rand.nextInt(Integer.MAX_VALUE);
657  }
658
659  @Test
660  public void testMultiTableMove() throws Exception {
661    final TableName tableNameA = TableName.valueOf(tablePrefix + name.getMethodName() + "A");
662    final TableName tableNameB = TableName.valueOf(tablePrefix + name.getMethodName() + "B");
663    final byte[] familyNameBytes = Bytes.toBytes("f");
664    String newGroupName = getGroupName(name.getMethodName());
665    final RSGroupInfo newGroup = addGroup(newGroupName, 1);
666
667    TEST_UTIL.createTable(tableNameA, familyNameBytes);
668    TEST_UTIL.createTable(tableNameB, familyNameBytes);
669    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
670      @Override
671      public boolean evaluate() throws Exception {
672        List<String> regionsA = getTableRegionMap().get(tableNameA);
673        if (regionsA == null) {
674          return false;
675        }
676
677        List<String> regionsB = getTableRegionMap().get(tableNameB);
678        if (regionsB == null) {
679          return false;
680        }
681
682        return getTableRegionMap().get(tableNameA).size() >= 1
683                && getTableRegionMap().get(tableNameB).size() >= 1;
684      }
685    });
686
687    RSGroupInfo tableGrpA = rsGroupAdmin.getRSGroupInfoOfTable(tableNameA);
688    assertTrue(tableGrpA.getName().equals(RSGroupInfo.DEFAULT_GROUP));
689
690    RSGroupInfo tableGrpB = rsGroupAdmin.getRSGroupInfoOfTable(tableNameB);
691    assertTrue(tableGrpB.getName().equals(RSGroupInfo.DEFAULT_GROUP));
692    //change table's group
693    LOG.info("Moving table [" + tableNameA + "," + tableNameB + "] to " + newGroup.getName());
694    rsGroupAdmin.moveTables(Sets.newHashSet(tableNameA, tableNameB), newGroup.getName());
695
696    //verify group change
697    Assert.assertEquals(newGroup.getName(),
698            rsGroupAdmin.getRSGroupInfoOfTable(tableNameA).getName());
699
700    Assert.assertEquals(newGroup.getName(),
701            rsGroupAdmin.getRSGroupInfoOfTable(tableNameB).getName());
702
703    //verify tables' not exist in old group
704    Set<TableName> DefaultTables = rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP)
705        .getTables();
706    assertFalse(DefaultTables.contains(tableNameA));
707    assertFalse(DefaultTables.contains(tableNameB));
708
709    //verify tables' exist in new group
710    Set<TableName> newGroupTables = rsGroupAdmin.getRSGroupInfo(newGroupName).getTables();
711    assertTrue(newGroupTables.contains(tableNameA));
712    assertTrue(newGroupTables.contains(tableNameB));
713  }
714
715  @Test
716  public void testDisabledTableMove() throws Exception {
717    final byte[] familyNameBytes = Bytes.toBytes("f");
718    String newGroupName = getGroupName(name.getMethodName());
719    final RSGroupInfo newGroup = addGroup(newGroupName, 2);
720
721    TEST_UTIL.createMultiRegionTable(tableName, familyNameBytes, 5);
722    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
723      @Override
724      public boolean evaluate() throws Exception {
725        List<String> regions = getTableRegionMap().get(tableName);
726        if (regions == null) {
727          return false;
728        }
729        return getTableRegionMap().get(tableName).size() >= 5;
730      }
731    });
732
733    RSGroupInfo tableGrp = rsGroupAdmin.getRSGroupInfoOfTable(tableName);
734    assertTrue(tableGrp.getName().equals(RSGroupInfo.DEFAULT_GROUP));
735
736    //test disable table
737    admin.disableTable(tableName);
738
739    //change table's group
740    LOG.info("Moving table "+ tableName + " to " + newGroup.getName());
741    rsGroupAdmin.moveTables(Sets.newHashSet(tableName), newGroup.getName());
742
743    //verify group change
744    Assert.assertEquals(newGroup.getName(),
745        rsGroupAdmin.getRSGroupInfoOfTable(tableName).getName());
746  }
747
748  @Test
749  public void testNonExistentTableMove() throws Exception {
750    TableName tableName = TableName.valueOf(tablePrefix + name.getMethodName());
751
752    RSGroupInfo tableGrp = rsGroupAdmin.getRSGroupInfoOfTable(tableName);
753    assertNull(tableGrp);
754
755    //test if table exists already.
756    boolean exist = admin.tableExists(tableName);
757    assertFalse(exist);
758
759    LOG.info("Moving table "+ tableName + " to " + RSGroupInfo.DEFAULT_GROUP);
760    try {
761      rsGroupAdmin.moveTables(Sets.newHashSet(tableName), RSGroupInfo.DEFAULT_GROUP);
762      fail("Table " + tableName + " shouldn't have been successfully moved.");
763    } catch(IOException ex) {
764      assertTrue(ex instanceof TableNotFoundException);
765    }
766
767    try {
768      rsGroupAdmin.moveServersAndTables(
769          Sets.newHashSet(Address.fromParts("bogus",123)),
770          Sets.newHashSet(tableName), RSGroupInfo.DEFAULT_GROUP);
771      fail("Table " + tableName + " shouldn't have been successfully moved.");
772    } catch(IOException ex) {
773      assertTrue(ex instanceof TableNotFoundException);
774    }
775    //verify group change
776    assertNull(rsGroupAdmin.getRSGroupInfoOfTable(tableName));
777  }
778
779  @Test
780  public void testMoveServersAndTables() throws Exception {
781    LOG.info("testMoveServersAndTables");
782    final RSGroupInfo newGroup = addGroup(getGroupName(name.getMethodName()), 1);
783    //create table
784    final byte[] familyNameBytes = Bytes.toBytes("f");
785    TEST_UTIL.createMultiRegionTable(tableName, familyNameBytes, 5);
786    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
787      @Override
788      public boolean evaluate() throws Exception {
789        List<String> regions = getTableRegionMap().get(tableName);
790        if (regions == null) {
791          return false;
792        }
793        return getTableRegionMap().get(tableName).size() >= 5;
794      }
795    });
796
797    //get server which is not a member of new group
798    ServerName targetServer = null;
799    for(ServerName server : admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS))
800      .getLiveServerMetrics().keySet()) {
801      if(!newGroup.containsServer(server.getAddress()) &&
802           !rsGroupAdmin.getRSGroupInfo("master").containsServer(server.getAddress())) {
803        targetServer = server;
804        break;
805      }
806    }
807
808    LOG.debug("Print group info : " + rsGroupAdmin.listRSGroups());
809    int oldDefaultGroupServerSize =
810            rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size();
811    int oldDefaultGroupTableSize =
812            rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getTables().size();
813
814    //test fail bogus server move
815    try {
816      rsGroupAdmin.moveServersAndTables(Sets.newHashSet(Address.fromString("foo:9999")),
817              Sets.newHashSet(tableName), newGroup.getName());
818      fail("Bogus servers shouldn't have been successfully moved.");
819    } catch(IOException ex) {
820      String exp = "Source RSGroup for server foo:9999 does not exist.";
821      String msg = "Expected '" + exp + "' in exception message: ";
822      assertTrue(msg + " " + ex.getMessage(), ex.getMessage().contains(exp));
823    }
824
825    //test fail server move
826    try {
827      rsGroupAdmin.moveServersAndTables(Sets.newHashSet(targetServer.getAddress()),
828              Sets.newHashSet(tableName), RSGroupInfo.DEFAULT_GROUP);
829      fail("servers shouldn't have been successfully moved.");
830    } catch(IOException ex) {
831      String exp = "Target RSGroup " + RSGroupInfo.DEFAULT_GROUP +
832              " is same as source " + RSGroupInfo.DEFAULT_GROUP + " RSGroup.";
833      String msg = "Expected '" + exp + "' in exception message: ";
834      assertTrue(msg + " " + ex.getMessage(), ex.getMessage().contains(exp));
835    }
836
837    //verify default group info
838    Assert.assertEquals(oldDefaultGroupServerSize,
839            rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size());
840    Assert.assertEquals(oldDefaultGroupTableSize,
841            rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getTables().size());
842
843    //verify new group info
844    Assert.assertEquals(1,
845            rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getServers().size());
846    Assert.assertEquals(0,
847            rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getTables().size());
848
849    //get all region to move targetServer
850    List<String> regionList = getTableRegionMap().get(tableName);
851    for(String region : regionList) {
852      // Lets move this region to the targetServer
853      TEST_UTIL.getAdmin().move(Bytes.toBytes(RegionInfo.encodeRegionName(Bytes.toBytes(region))),
854              Bytes.toBytes(targetServer.getServerName()));
855    }
856
857    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
858      @Override
859      public boolean evaluate() throws Exception {
860        return getTableRegionMap().get(tableName) != null &&
861                getTableRegionMap().get(tableName).size() == 5 &&
862                getTableServerRegionMap().get(tableName).size() == 1 &&
863                admin.getClusterMetrics(EnumSet.of(Option.REGIONS_IN_TRANSITION))
864                     .getRegionStatesInTransition().size() < 1;
865      }
866    });
867
868    //verify that all region move to targetServer
869    Assert.assertEquals(5, getTableServerRegionMap().get(tableName).get(targetServer).size());
870
871    //move targetServer and table to newGroup
872    LOG.info("moving server and table to newGroup");
873    rsGroupAdmin.moveServersAndTables(Sets.newHashSet(targetServer.getAddress()),
874            Sets.newHashSet(tableName), newGroup.getName());
875
876    //verify group change
877    Assert.assertEquals(newGroup.getName(),
878            rsGroupAdmin.getRSGroupInfoOfTable(tableName).getName());
879
880    //verify servers' not exist in old group
881    Set<Address> defaultServers = rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP)
882        .getServers();
883    assertFalse(defaultServers.contains(targetServer.getAddress()));
884
885    //verify servers' exist in new group
886    Set<Address> newGroupServers = rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getServers();
887    assertTrue(newGroupServers.contains(targetServer.getAddress()));
888
889    //verify tables' not exist in old group
890    Set<TableName> defaultTables = rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP)
891        .getTables();
892    assertFalse(defaultTables.contains(tableName));
893
894    //verify tables' exist in new group
895    Set<TableName> newGroupTables = rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getTables();
896    assertTrue(newGroupTables.contains(tableName));
897
898    //verify that all region still assgin on targetServer
899    Assert.assertEquals(5, getTableServerRegionMap().get(tableName).get(targetServer).size());
900  }
901
902  @Test
903  public void testClearDeadServers() throws Exception {
904    LOG.info("testClearDeadServers");
905    final RSGroupInfo newGroup = addGroup(getGroupName(name.getMethodName()), 3);
906    NUM_DEAD_SERVERS = cluster.getClusterMetrics().getDeadServerNames().size();
907
908    ServerName targetServer = ServerName.parseServerName(
909        newGroup.getServers().iterator().next().toString());
910    AdminProtos.AdminService.BlockingInterface targetRS =
911        ((ClusterConnection) admin.getConnection()).getAdmin(targetServer);
912    try {
913      targetServer = ProtobufUtil.toServerName(targetRS.getServerInfo(null,
914          GetServerInfoRequest.newBuilder().build()).getServerInfo().getServerName());
915      //stopping may cause an exception
916      //due to the connection loss
917      targetRS.stopServer(null,
918          AdminProtos.StopServerRequest.newBuilder().setReason("Die").build());
919      NUM_DEAD_SERVERS ++;
920    } catch(Exception e) {
921    }
922    //wait for stopped regionserver to dead server list
923    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
924      @Override
925      public boolean evaluate() throws Exception {
926        return !master.getServerManager().areDeadServersInProgress()
927            && cluster.getClusterMetrics().getDeadServerNames().size() == NUM_DEAD_SERVERS;
928      }
929    });
930    assertFalse(cluster.getClusterMetrics().getLiveServerMetrics().containsKey(targetServer));
931    assertTrue(cluster.getClusterMetrics().getDeadServerNames().contains(targetServer));
932    assertTrue(newGroup.getServers().contains(targetServer.getAddress()));
933
934    //clear dead servers list
935    List<ServerName> notClearedServers = admin.clearDeadServers(Lists.newArrayList(targetServer));
936    assertEquals(0, notClearedServers.size());
937
938    Set<Address> newGroupServers = rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getServers();
939    assertFalse(newGroupServers.contains(targetServer.getAddress()));
940    assertEquals(2, newGroupServers.size());
941  }
942
943  @Test
944  public void testRemoveServers() throws Exception {
945    LOG.info("testRemoveServers");
946    final RSGroupInfo newGroup = addGroup(getGroupName(name.getMethodName()), 3);
947    Iterator<Address> iterator = newGroup.getServers().iterator();
948    ServerName targetServer = ServerName.parseServerName(iterator.next().toString());
949
950    // remove online servers
951    try {
952      rsGroupAdmin.removeServers(Sets.newHashSet(targetServer.getAddress()));
953      fail("Online servers shouldn't have been successfully removed.");
954    } catch(IOException ex) {
955      String exp = "Server " + targetServer.getAddress()
956          + " is an online server, not allowed to remove.";
957      String msg = "Expected '" + exp + "' in exception message: ";
958      assertTrue(msg + " " + ex.getMessage(), ex.getMessage().contains(exp));
959    }
960    assertTrue(newGroup.getServers().contains(targetServer.getAddress()));
961
962    // remove dead servers
963    NUM_DEAD_SERVERS = cluster.getClusterMetrics().getDeadServerNames().size();
964    AdminProtos.AdminService.BlockingInterface targetRS =
965        ((ClusterConnection) admin.getConnection()).getAdmin(targetServer);
966    try {
967      targetServer = ProtobufUtil.toServerName(targetRS.getServerInfo(null,
968          GetServerInfoRequest.newBuilder().build()).getServerInfo().getServerName());
969      //stopping may cause an exception
970      //due to the connection loss
971      LOG.info("stopping server " + targetServer.getHostAndPort());
972      targetRS.stopServer(null,
973          AdminProtos.StopServerRequest.newBuilder().setReason("Die").build());
974      NUM_DEAD_SERVERS ++;
975    } catch(Exception e) {
976    }
977
978    //wait for stopped regionserver to dead server list
979    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
980      @Override
981      public boolean evaluate() throws Exception {
982        return !master.getServerManager().areDeadServersInProgress()
983            && cluster.getClusterMetrics().getDeadServerNames().size() == NUM_DEAD_SERVERS;
984      }
985    });
986
987    try {
988      rsGroupAdmin.removeServers(Sets.newHashSet(targetServer.getAddress()));
989      fail("Dead servers shouldn't have been successfully removed.");
990    } catch(IOException ex) {
991      String exp = "Server " + targetServer.getAddress() + " is on the dead servers list,"
992          + " Maybe it will come back again, not allowed to remove.";
993      String msg = "Expected '" + exp + "' in exception message: ";
994      assertTrue(msg + " " + ex.getMessage(), ex.getMessage().contains(exp));
995    }
996    assertTrue(newGroup.getServers().contains(targetServer.getAddress()));
997
998    // remove decommissioned servers
999    List<ServerName> serversToDecommission = new ArrayList<>();
1000    targetServer = ServerName.parseServerName(iterator.next().toString());
1001    targetRS = ((ClusterConnection) admin.getConnection()).getAdmin(targetServer);
1002    targetServer = ProtobufUtil.toServerName(targetRS.getServerInfo(null,
1003          GetServerInfoRequest.newBuilder().build()).getServerInfo().getServerName());
1004    assertTrue(master.getServerManager().getOnlineServers().containsKey(targetServer));
1005    serversToDecommission.add(targetServer);
1006
1007    admin.decommissionRegionServers(serversToDecommission, true);
1008    assertEquals(1, admin.listDecommissionedRegionServers().size());
1009
1010    assertTrue(newGroup.getServers().contains(targetServer.getAddress()));
1011    rsGroupAdmin.removeServers(Sets.newHashSet(targetServer.getAddress()));
1012    Set<Address> newGroupServers = rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getServers();
1013    assertFalse(newGroupServers.contains(targetServer.getAddress()));
1014    assertEquals(2, newGroupServers.size());
1015  }
1016
1017  @Test
1018  public void testCreateWhenRsgroupNoOnlineServers() throws Exception {
1019    LOG.info("testCreateWhenRsgroupNoOnlineServers");
1020
1021    // set rsgroup has no online servers and test create table
1022    final RSGroupInfo appInfo = addGroup("appInfo", 1);
1023    Iterator<Address> iterator = appInfo.getServers().iterator();
1024    List<ServerName> serversToDecommission = new ArrayList<>();
1025    ServerName targetServer = ServerName.parseServerName(iterator.next().toString());
1026    AdminProtos.AdminService.BlockingInterface targetRS =
1027        ((ClusterConnection) admin.getConnection()).getAdmin(targetServer);
1028    targetServer = ProtobufUtil.toServerName(
1029        targetRS.getServerInfo(null, GetServerInfoRequest.newBuilder().build()).getServerInfo()
1030            .getServerName());
1031    assertTrue(master.getServerManager().getOnlineServers().containsKey(targetServer));
1032    serversToDecommission.add(targetServer);
1033    admin.decommissionRegionServers(serversToDecommission, true);
1034    assertEquals(1, admin.listDecommissionedRegionServers().size());
1035
1036    final TableName tableName = TableName.valueOf(tablePrefix + "_ns", name.getMethodName());
1037    admin.createNamespace(NamespaceDescriptor.create(tableName.getNamespaceAsString())
1038        .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, appInfo.getName()).build());
1039    final HTableDescriptor desc = new HTableDescriptor(tableName);
1040    desc.addFamily(new HColumnDescriptor("f"));
1041    try {
1042      admin.createTable(desc);
1043      fail("Shouldn't create table successfully!");
1044    } catch (Exception e) {
1045      LOG.debug("create table error", e);
1046    }
1047
1048    // recommission and test create table
1049    admin.recommissionRegionServer(targetServer, null);
1050    assertEquals(0, admin.listDecommissionedRegionServers().size());
1051    admin.createTable(desc);
1052    // wait for created table to be assigned
1053    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
1054      @Override public boolean evaluate() throws Exception {
1055        return getTableRegionMap().get(desc.getTableName()) != null;
1056      }
1057    });
1058  }
1059}