Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.rsgroup;
019
020import static org.apache.hadoop.hbase.util.Threads.sleep;
021import static org.junit.Assert.assertEquals;
022import static org.junit.Assert.assertFalse;
023import static org.junit.Assert.assertTrue;
024import static org.junit.Assert.fail;
025
026import java.io.IOException;
027import java.util.ArrayList;
028import java.util.EnumSet;
029import java.util.Iterator;
030import java.util.List;
031import java.util.Map;
032import java.util.Set;
033import java.util.concurrent.atomic.AtomicBoolean;
034import org.apache.hadoop.hbase.ClusterMetrics.Option;
035import org.apache.hadoop.hbase.HBaseClassTestRule;
036import org.apache.hadoop.hbase.ServerName;
037import org.apache.hadoop.hbase.TableName;
038import org.apache.hadoop.hbase.Waiter;
039import org.apache.hadoop.hbase.client.RegionInfo;
040import org.apache.hadoop.hbase.constraint.ConstraintException;
041import org.apache.hadoop.hbase.master.RegionState;
042import org.apache.hadoop.hbase.master.assignment.RegionStateNode;
043import org.apache.hadoop.hbase.net.Address;
044import org.apache.hadoop.hbase.testclassification.LargeTests;
045import org.apache.hadoop.hbase.util.Bytes;
046import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
047import org.junit.After;
048import org.junit.AfterClass;
049import org.junit.Assert;
050import org.junit.Before;
051import org.junit.BeforeClass;
052import org.junit.ClassRule;
053import org.junit.Test;
054import org.junit.experimental.categories.Category;
055import org.slf4j.Logger;
056import org.slf4j.LoggerFactory;
057
058import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
059
060@Category({ LargeTests.class })
061public class TestRSGroupsAdmin2 extends TestRSGroupsBase {
062
063  @ClassRule
064  public static final HBaseClassTestRule CLASS_RULE =
065    HBaseClassTestRule.forClass(TestRSGroupsAdmin2.class);
066
067  protected static final Logger LOG = LoggerFactory.getLogger(TestRSGroupsAdmin2.class);
068
069  @BeforeClass
070  public static void setUp() throws Exception {
071    setUpTestBeforeClass();
072  }
073
074  @AfterClass
075  public static void tearDown() throws Exception {
076    tearDownAfterClass();
077  }
078
079  @Before
080  public void beforeMethod() throws Exception {
081    setUpBeforeMethod();
082  }
083
084  @After
085  public void afterMethod() throws Exception {
086    tearDownAfterMethod();
087  }
088
089  @Test
090  public void testRegionMove() throws Exception {
091    final RSGroupInfo newGroup = addGroup(getGroupName(name.getMethodName()), 1);
092    final byte[] familyNameBytes = Bytes.toBytes("f");
093    // All the regions created below will be assigned to the default group.
094    TEST_UTIL.createMultiRegionTable(tableName, familyNameBytes, 6);
095    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
096      @Override
097      public boolean evaluate() throws Exception {
098        List<String> regions = getTableRegionMap().get(tableName);
099        if (regions == null) {
100          return false;
101        }
102
103        return getTableRegionMap().get(tableName).size() >= 6;
104      }
105    });
106
107    // get target region to move
108    Map<ServerName, List<String>> assignMap = getTableServerRegionMap().get(tableName);
109    String targetRegion = null;
110    for (ServerName server : assignMap.keySet()) {
111      targetRegion = assignMap.get(server).size() > 0 ? assignMap.get(server).get(0) : null;
112      if (targetRegion != null) {
113        break;
114      }
115    }
116    // get server which is not a member of new group
117    ServerName tmpTargetServer = null;
118    for (ServerName server : admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS))
119      .getLiveServerMetrics().keySet()) {
120      if (!newGroup.containsServer(server.getAddress())) {
121        tmpTargetServer = server;
122        break;
123      }
124    }
125    final ServerName targetServer = tmpTargetServer;
126    // move target server to group
127    rsGroupAdmin.moveServers(Sets.newHashSet(targetServer.getAddress()), newGroup.getName());
128    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
129      @Override
130      public boolean evaluate() throws Exception {
131        return admin.getRegions(targetServer).size() <= 0;
132      }
133    });
134
135    // Lets move this region to the new group.
136    TEST_UTIL.getAdmin()
137      .move(Bytes.toBytes(RegionInfo.encodeRegionName(Bytes.toBytes(targetRegion))), targetServer);
138    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
139      @Override
140      public boolean evaluate() throws Exception {
141        return getTableRegionMap().get(tableName) != null
142          && getTableRegionMap().get(tableName).size() == 6
143          && admin.getClusterMetrics(EnumSet.of(Option.REGIONS_IN_TRANSITION))
144            .getRegionStatesInTransition().size() < 1;
145      }
146    });
147
148    // verify that targetServer didn't open it
149    for (RegionInfo region : admin.getRegions(targetServer)) {
150      if (targetRegion.equals(region.getRegionNameAsString())) {
151        fail("Target server opened region");
152      }
153    }
154  }
155
156  @Test
157  public void testRegionServerMove() throws IOException, InterruptedException {
158    int initNumGroups = rsGroupAdmin.listRSGroups().size();
159    RSGroupInfo appInfo = addGroup(getGroupName(name.getMethodName()), 1);
160    RSGroupInfo adminInfo = addGroup(getGroupName(name.getMethodName()), 1);
161    RSGroupInfo dInfo = rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP);
162    Assert.assertEquals(initNumGroups + 2, rsGroupAdmin.listRSGroups().size());
163    assertEquals(1, adminInfo.getServers().size());
164    assertEquals(1, appInfo.getServers().size());
165    assertEquals(getNumServers() - 2, dInfo.getServers().size());
166    rsGroupAdmin.moveServers(appInfo.getServers(), RSGroupInfo.DEFAULT_GROUP);
167    rsGroupAdmin.removeRSGroup(appInfo.getName());
168    rsGroupAdmin.moveServers(adminInfo.getServers(), RSGroupInfo.DEFAULT_GROUP);
169    rsGroupAdmin.removeRSGroup(adminInfo.getName());
170    Assert.assertEquals(rsGroupAdmin.listRSGroups().size(), initNumGroups);
171  }
172
173  @Test
174  public void testMoveServers() throws Exception {
175    // create groups and assign servers
176    addGroup("bar", 3);
177    rsGroupAdmin.addRSGroup("foo");
178
179    RSGroupInfo barGroup = rsGroupAdmin.getRSGroupInfo("bar");
180    RSGroupInfo fooGroup = rsGroupAdmin.getRSGroupInfo("foo");
181    assertEquals(3, barGroup.getServers().size());
182    assertEquals(0, fooGroup.getServers().size());
183
184    // test fail bogus server move
185    try {
186      rsGroupAdmin.moveServers(Sets.newHashSet(Address.fromString("foo:9999")), "foo");
187      fail("Bogus servers shouldn't have been successfully moved.");
188    } catch (IOException ex) {
189      String exp = "Server foo:9999 is either offline or it does not exist.";
190      String msg = "Expected '" + exp + "' in exception message: ";
191      assertTrue(msg + " " + ex.getMessage(), ex.getMessage().contains(exp));
192    }
193
194    // test success case
195    LOG.info("moving servers " + barGroup.getServers() + " to group foo");
196    rsGroupAdmin.moveServers(barGroup.getServers(), fooGroup.getName());
197
198    barGroup = rsGroupAdmin.getRSGroupInfo("bar");
199    fooGroup = rsGroupAdmin.getRSGroupInfo("foo");
200    assertEquals(0, barGroup.getServers().size());
201    assertEquals(3, fooGroup.getServers().size());
202
203    LOG.info("moving servers " + fooGroup.getServers() + " to group default");
204    rsGroupAdmin.moveServers(fooGroup.getServers(), RSGroupInfo.DEFAULT_GROUP);
205
206    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
207      @Override
208      public boolean evaluate() throws Exception {
209        return getNumServers()
210            == rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size();
211      }
212    });
213
214    fooGroup = rsGroupAdmin.getRSGroupInfo("foo");
215    assertEquals(0, fooGroup.getServers().size());
216
217    // test group removal
218    LOG.info("Remove group " + barGroup.getName());
219    rsGroupAdmin.removeRSGroup(barGroup.getName());
220    Assert.assertEquals(null, rsGroupAdmin.getRSGroupInfo(barGroup.getName()));
221    LOG.info("Remove group " + fooGroup.getName());
222    rsGroupAdmin.removeRSGroup(fooGroup.getName());
223    Assert.assertEquals(null, rsGroupAdmin.getRSGroupInfo(fooGroup.getName()));
224  }
225
226  @Test
227  public void testRemoveServers() throws Exception {
228    LOG.info("testRemoveServers");
229    final RSGroupInfo newGroup = addGroup(getGroupName(name.getMethodName()), 3);
230    Iterator<Address> iterator = newGroup.getServers().iterator();
231    ServerName targetServer = getServerName(iterator.next());
232
233    // remove online servers
234    try {
235      rsGroupAdmin.removeServers(Sets.newHashSet(targetServer.getAddress()));
236      fail("Online servers shouldn't have been successfully removed.");
237    } catch (IOException ex) {
238      String exp =
239        "Server " + targetServer.getAddress() + " is an online server, not allowed to remove.";
240      String msg = "Expected '" + exp + "' in exception message: ";
241      assertTrue(msg + " " + ex.getMessage(), ex.getMessage().contains(exp));
242    }
243    assertTrue(newGroup.getServers().contains(targetServer.getAddress()));
244
245    // remove dead servers
246    NUM_DEAD_SERVERS = cluster.getClusterMetrics().getDeadServerNames().size();
247    try {
248      // stopping may cause an exception
249      // due to the connection loss
250      LOG.info("stopping server " + targetServer.getServerName());
251      admin.stopRegionServer(targetServer.getAddress().toString());
252      NUM_DEAD_SERVERS++;
253    } catch (Exception e) {
254    }
255
256    // wait for stopped regionserver to dead server list
257    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
258      @Override
259      public boolean evaluate() throws Exception {
260        return !master.getServerManager().areDeadServersInProgress()
261          && cluster.getClusterMetrics().getDeadServerNames().size() == NUM_DEAD_SERVERS;
262      }
263    });
264
265    try {
266      rsGroupAdmin.removeServers(Sets.newHashSet(targetServer.getAddress()));
267      fail("Dead servers shouldn't have been successfully removed.");
268    } catch (IOException ex) {
269      String exp = "Server " + targetServer.getAddress() + " is on the dead servers list,"
270        + " Maybe it will come back again, not allowed to remove.";
271      String msg = "Expected '" + exp + "' in exception message: ";
272      assertTrue(msg + " " + ex.getMessage(), ex.getMessage().contains(exp));
273    }
274    assertTrue(newGroup.getServers().contains(targetServer.getAddress()));
275
276    // remove decommissioned servers
277    List<ServerName> serversToDecommission = new ArrayList<>();
278    targetServer = getServerName(iterator.next());
279    assertTrue(master.getServerManager().getOnlineServers().containsKey(targetServer));
280    serversToDecommission.add(targetServer);
281
282    admin.decommissionRegionServers(serversToDecommission, true);
283    assertEquals(1, admin.listDecommissionedRegionServers().size());
284
285    assertTrue(newGroup.getServers().contains(targetServer.getAddress()));
286    rsGroupAdmin.removeServers(Sets.newHashSet(targetServer.getAddress()));
287    Set<Address> newGroupServers = rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getServers();
288    assertFalse(newGroupServers.contains(targetServer.getAddress()));
289    assertEquals(2, newGroupServers.size());
290
291    assertTrue(observer.preRemoveServersCalled);
292    assertTrue(observer.postRemoveServersCalled);
293  }
294
295  @Test
296  public void testMoveServersAndTables() throws Exception {
297    LOG.info("testMoveServersAndTables");
298    final RSGroupInfo newGroup = addGroup(getGroupName(name.getMethodName()), 1);
299    // create table
300    final byte[] familyNameBytes = Bytes.toBytes("f");
301    TEST_UTIL.createMultiRegionTable(tableName, familyNameBytes, 5);
302    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
303      @Override
304      public boolean evaluate() throws Exception {
305        List<String> regions = getTableRegionMap().get(tableName);
306        if (regions == null) {
307          return false;
308        }
309
310        return getTableRegionMap().get(tableName).size() >= 5;
311      }
312    });
313
314    // get server which is not a member of new group
315    ServerName targetServer = null;
316    for (ServerName server : admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS))
317      .getLiveServerMetrics().keySet()) {
318      if (
319        !newGroup.containsServer(server.getAddress())
320          && !rsGroupAdmin.getRSGroupInfo("master").containsServer(server.getAddress())
321      ) {
322        targetServer = server;
323        break;
324      }
325    }
326
327    LOG.debug("Print group info : " + rsGroupAdmin.listRSGroups());
328    int oldDefaultGroupServerSize =
329      rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size();
330    int oldDefaultGroupTableSize =
331      rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getTables().size();
332
333    // test fail bogus server move
334    try {
335      rsGroupAdmin.moveServersAndTables(Sets.newHashSet(Address.fromString("foo:9999")),
336        Sets.newHashSet(tableName), newGroup.getName());
337      fail("Bogus servers shouldn't have been successfully moved.");
338    } catch (IOException ex) {
339      String exp = "Server foo:9999 is either offline or it does not exist.";
340      String msg = "Expected '" + exp + "' in exception message: ";
341      assertTrue(msg + " " + ex.getMessage(), ex.getMessage().contains(exp));
342    }
343
344    // test move when src = dst
345    rsGroupAdmin.moveServersAndTables(Sets.newHashSet(targetServer.getAddress()),
346      Sets.newHashSet(tableName), RSGroupInfo.DEFAULT_GROUP);
347
348    // verify default group info
349    Assert.assertEquals(oldDefaultGroupServerSize,
350      rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size());
351    Assert.assertEquals(oldDefaultGroupTableSize,
352      rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getTables().size());
353
354    // verify new group info
355    Assert.assertEquals(1, rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getServers().size());
356    Assert.assertEquals(0, rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getTables().size());
357
358    // get all region to move targetServer
359    List<String> regionList = getTableRegionMap().get(tableName);
360    for (String region : regionList) {
361      // Lets move this region to the targetServer
362      TEST_UTIL.getAdmin().move(Bytes.toBytes(RegionInfo.encodeRegionName(Bytes.toBytes(region))),
363        targetServer);
364    }
365
366    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
367      @Override
368      public boolean evaluate() throws Exception {
369        return getTableRegionMap().get(tableName) != null
370          && getTableRegionMap().get(tableName).size() == 5
371          && getTableServerRegionMap().get(tableName).size() == 1
372          && admin.getClusterMetrics(EnumSet.of(Option.REGIONS_IN_TRANSITION))
373            .getRegionStatesInTransition().size() < 1;
374      }
375    });
376
377    // verify that all region move to targetServer
378    Assert.assertEquals(5, getTableServerRegionMap().get(tableName).get(targetServer).size());
379
380    // move targetServer and table to newGroup
381    LOG.info("moving server and table to newGroup");
382    rsGroupAdmin.moveServersAndTables(Sets.newHashSet(targetServer.getAddress()),
383      Sets.newHashSet(tableName), newGroup.getName());
384
385    // verify group change
386    Assert.assertEquals(newGroup.getName(),
387      rsGroupAdmin.getRSGroupInfoOfTable(tableName).getName());
388
389    // verify servers' not exist in old group
390    Set<Address> defaultServers =
391      rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers();
392    assertFalse(defaultServers.contains(targetServer.getAddress()));
393
394    // verify servers' exist in new group
395    Set<Address> newGroupServers = rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getServers();
396    assertTrue(newGroupServers.contains(targetServer.getAddress()));
397
398    // verify tables' not exist in old group
399    Set<TableName> defaultTables =
400      rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getTables();
401    assertFalse(defaultTables.contains(tableName));
402
403    // verify tables' exist in new group
404    Set<TableName> newGroupTables = rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getTables();
405    assertTrue(newGroupTables.contains(tableName));
406
407    // verify that all region still assgin on targetServer
408    Assert.assertEquals(5, getTableServerRegionMap().get(tableName).get(targetServer).size());
409
410    assertTrue(observer.preMoveServersAndTables);
411    assertTrue(observer.postMoveServersAndTables);
412  }
413
414  @Test
415  public void testMoveServersFromDefaultGroup() throws Exception {
416    // create groups and assign servers
417    rsGroupAdmin.addRSGroup("foo");
418
419    RSGroupInfo fooGroup = rsGroupAdmin.getRSGroupInfo("foo");
420    assertEquals(0, fooGroup.getServers().size());
421    RSGroupInfo defaultGroup = rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP);
422
423    // test remove all servers from default
424    try {
425      rsGroupAdmin.moveServers(defaultGroup.getServers(), fooGroup.getName());
426      fail(RSGroupAdminServer.KEEP_ONE_SERVER_IN_DEFAULT_ERROR_MESSAGE);
427    } catch (ConstraintException ex) {
428      assertTrue(
429        ex.getMessage().contains(RSGroupAdminServer.KEEP_ONE_SERVER_IN_DEFAULT_ERROR_MESSAGE));
430    }
431
432    // test success case, remove one server from default ,keep at least one server
433    if (defaultGroup.getServers().size() > 1) {
434      Address serverInDefaultGroup = defaultGroup.getServers().iterator().next();
435      LOG.info("moving server " + serverInDefaultGroup + " from group default to group "
436        + fooGroup.getName());
437      rsGroupAdmin.moveServers(Sets.newHashSet(serverInDefaultGroup), fooGroup.getName());
438    }
439
440    fooGroup = rsGroupAdmin.getRSGroupInfo("foo");
441    LOG.info("moving servers " + fooGroup.getServers() + " to group default");
442    rsGroupAdmin.moveServers(fooGroup.getServers(), RSGroupInfo.DEFAULT_GROUP);
443
444    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
445      @Override
446      public boolean evaluate() throws Exception {
447        return getNumServers()
448            == rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size();
449      }
450    });
451
452    fooGroup = rsGroupAdmin.getRSGroupInfo("foo");
453    assertEquals(0, fooGroup.getServers().size());
454
455    // test group removal
456    LOG.info("Remove group " + fooGroup.getName());
457    rsGroupAdmin.removeRSGroup(fooGroup.getName());
458    Assert.assertEquals(null, rsGroupAdmin.getRSGroupInfo(fooGroup.getName()));
459  }
460
461  @Test
462  public void testFailedMoveWhenMoveServer() throws Exception {
463    final RSGroupInfo newGroup = addGroup(getGroupName(name.getMethodName()), 1);
464    final byte[] familyNameBytes = Bytes.toBytes("f");
465    final int tableRegionCount = 10;
466    // All the regions created below will be assigned to the default group.
467    TEST_UTIL.createMultiRegionTable(tableName, familyNameBytes, tableRegionCount);
468    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
469      @Override
470      public boolean evaluate() throws Exception {
471        List<String> regions = getTableRegionMap().get(tableName);
472        if (regions == null) {
473          return false;
474        }
475        return getTableRegionMap().get(tableName).size() >= tableRegionCount;
476      }
477    });
478
479    // get target server to move, which should has more than one regions
480    // randomly set a region state to SPLITTING
481    Map<ServerName, List<String>> assignMap = getTableServerRegionMap().get(tableName);
482    String rregion = null;
483    ServerName toMoveServer = null;
484    for (ServerName server : assignMap.keySet()) {
485      rregion = assignMap.get(server).size() > 1 && !newGroup.containsServer(server.getAddress())
486        ? assignMap.get(server).get(0)
487        : null;
488      if (rregion != null) {
489        toMoveServer = server;
490        break;
491      }
492    }
493    assert toMoveServer != null;
494    RegionInfo ri = TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager()
495      .getRegionInfo(Bytes.toBytesBinary(rregion));
496    RegionStateNode rsn = TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager()
497      .getRegionStates().getRegionStateNode(ri);
498    rsn.setState(RegionState.State.SPLITTING);
499
500    // start thread to recover region state
501    final ServerName movedServer = toMoveServer;
502    final String sregion = rregion;
503    AtomicBoolean changed = new AtomicBoolean(false);
504    Thread t1 = new Thread(() -> {
505      LOG.debug("thread1 start running, will recover region state");
506      long current = EnvironmentEdgeManager.currentTime();
507      while (EnvironmentEdgeManager.currentTime() - current <= 50000) {
508        List<RegionInfo> regions = master.getAssignmentManager().getRegionsOnServer(movedServer);
509        LOG.debug("server region size is:{}", regions.size());
510        assert regions.size() >= 1;
511        // when there is exactly one region left, we can determine the move operation encountered
512        // exception caused by the strange region state.
513        if (regions.size() == 1) {
514          assertEquals(regions.get(0).getRegionNameAsString(), sregion);
515          rsn.setState(RegionState.State.OPEN);
516          LOG.info("set region {} state OPEN", sregion);
517          changed.set(true);
518          break;
519        }
520        sleep(5000);
521      }
522    });
523    t1.start();
524
525    // move target server to group
526    Thread t2 = new Thread(() -> {
527      LOG.info("thread2 start running, to move regions");
528      try {
529        rsGroupAdmin.moveServers(Sets.newHashSet(movedServer.getAddress()), newGroup.getName());
530      } catch (IOException e) {
531        LOG.error("move server error", e);
532      }
533    });
534    t2.start();
535
536    t1.join();
537    t2.join();
538
539    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
540      @Override
541      public boolean evaluate() {
542        if (changed.get()) {
543          return master.getAssignmentManager().getRegionsOnServer(movedServer).size() == 0
544            && !rsn.getRegionLocation().equals(movedServer);
545        }
546        return false;
547      }
548    });
549  }
550
551  @Test
552  public void testFailedMoveWhenMoveTable() throws Exception {
553    final RSGroupInfo newGroup = addGroup(getGroupName(name.getMethodName()), 1);
554    final byte[] familyNameBytes = Bytes.toBytes("f");
555    final int tableRegionCount = 5;
556    // All the regions created below will be assigned to the default group.
557    TEST_UTIL.createMultiRegionTable(tableName, familyNameBytes, tableRegionCount);
558    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
559      @Override
560      public boolean evaluate() throws Exception {
561        List<String> regions = getTableRegionMap().get(tableName);
562        if (regions == null) {
563          return false;
564        }
565        return getTableRegionMap().get(tableName).size() >= tableRegionCount;
566      }
567    });
568
569    // randomly set a region state to SPLITTING
570    Map<ServerName, List<String>> assignMap = getTableServerRegionMap().get(tableName);
571    String rregion = null;
572    ServerName srcServer = null;
573    for (ServerName server : assignMap.keySet()) {
574      rregion = assignMap.get(server).size() >= 1 && !newGroup.containsServer(server.getAddress())
575        ? assignMap.get(server).get(0)
576        : null;
577      if (rregion != null) {
578        srcServer = server;
579        break;
580      }
581    }
582    assert srcServer != null;
583    RegionInfo ri = TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager()
584      .getRegionInfo(Bytes.toBytesBinary(rregion));
585    RegionStateNode rsn = TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager()
586      .getRegionStates().getRegionStateNode(ri);
587    rsn.setState(RegionState.State.SPLITTING);
588
589    // move table to group
590    Thread t2 = new Thread(() -> {
591      LOG.info("thread2 start running, to move regions");
592      try {
593        rsGroupAdmin.moveTables(Sets.newHashSet(tableName), newGroup.getName());
594      } catch (IOException e) {
595        LOG.error("move server error", e);
596      }
597    });
598    t2.start();
599
600    // start thread to recover region state
601    final ServerName ss = srcServer;
602    final String sregion = rregion;
603    AtomicBoolean changed = new AtomicBoolean(false);
604    Thread t1 = new Thread(() -> {
605      LOG.info("thread1 start running, will recover region state");
606      long current = EnvironmentEdgeManager.currentTime();
607      while (EnvironmentEdgeManager.currentTime() - current <= 50000) {
608        List<RegionInfo> regions = master.getAssignmentManager().getRegionsOnServer(ss);
609        List<RegionInfo> tableRegions = new ArrayList<>();
610        for (RegionInfo regionInfo : regions) {
611          if (regionInfo.getTable().equals(tableName)) {
612            tableRegions.add(regionInfo);
613          }
614        }
615        LOG.debug("server table region size is:{}", tableRegions.size());
616        assert tableRegions.size() >= 1;
617        // when there is exactly one region left, we can determine the move operation encountered
618        // exception caused by the strange region state.
619        if (tableRegions.size() == 1) {
620          assertEquals(tableRegions.get(0).getRegionNameAsString(), sregion);
621          rsn.setState(RegionState.State.OPEN);
622          LOG.info("set region {} state OPEN", sregion);
623          changed.set(true);
624          break;
625        }
626        sleep(5000);
627      }
628    });
629    t1.start();
630
631    t1.join();
632    t2.join();
633
634    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
635      @Override
636      public boolean evaluate() {
637        if (changed.get()) {
638          boolean serverHasTableRegions = false;
639          for (RegionInfo regionInfo : master.getAssignmentManager().getRegionsOnServer(ss)) {
640            if (regionInfo.getTable().equals(tableName)) {
641              serverHasTableRegions = true;
642              break;
643            }
644          }
645          return !serverHasTableRegions && !rsn.getRegionLocation().equals(ss);
646        }
647        return false;
648      }
649    });
650  }
651
652  @Test
653  public void testMoveTablePerformance() throws Exception {
654    final RSGroupInfo newGroup = addGroup(getGroupName(name.getMethodName()), 1);
655    final byte[] familyNameBytes = Bytes.toBytes("f");
656    final int tableRegionCount = 100;
657    // All the regions created below will be assigned to the default group.
658    TEST_UTIL.createMultiRegionTable(tableName, familyNameBytes, tableRegionCount);
659    TEST_UTIL.waitFor(WAIT_TIMEOUT, (Waiter.Predicate<Exception>) () -> {
660      List<String> regions = getTableRegionMap().get(tableName);
661      if (regions == null) {
662        return false;
663      }
664      return getTableRegionMap().get(tableName).size() >= tableRegionCount;
665    });
666    long startTime = EnvironmentEdgeManager.currentTime();
667    rsGroupAdmin.moveServers(Sets.newHashSet(newGroup.getServers().iterator().next()),
668      newGroup.getName());
669    long timeTaken = EnvironmentEdgeManager.currentTime() - startTime;
670    String msg =
671      "Should not take mote than 15000 ms to move a table with 100 regions. Time taken  ="
672        + timeTaken + " ms";
673    // This test case is meant to be used for verifying the performance quickly by a developer.
674    // Moving 100 regions takes much less than 15000 ms. Given 15000 ms so test cases passes
675    // on all environment.
676    assertTrue(msg, timeTaken < 15000);
677    LOG.info("Time taken to move a table with 100 region is {} ms", timeTaken);
678  }
679}