Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.rsgroup;
019
020import static org.apache.hadoop.hbase.util.Threads.sleep;
021import static org.junit.Assert.assertEquals;
022import static org.junit.Assert.assertFalse;
023import static org.junit.Assert.assertTrue;
024import static org.junit.Assert.fail;
025
026import java.io.IOException;
027import java.util.ArrayList;
028import java.util.EnumSet;
029import java.util.Iterator;
030import java.util.List;
031import java.util.Map;
032import java.util.Set;
033import java.util.concurrent.atomic.AtomicBoolean;
034import org.apache.hadoop.hbase.ClusterMetrics.Option;
035import org.apache.hadoop.hbase.HBaseClassTestRule;
036import org.apache.hadoop.hbase.ServerName;
037import org.apache.hadoop.hbase.TableName;
038import org.apache.hadoop.hbase.Waiter;
039import org.apache.hadoop.hbase.client.RegionInfo;
040import org.apache.hadoop.hbase.constraint.ConstraintException;
041import org.apache.hadoop.hbase.master.RegionState;
042import org.apache.hadoop.hbase.master.assignment.RegionStateNode;
043import org.apache.hadoop.hbase.net.Address;
044import org.apache.hadoop.hbase.testclassification.LargeTests;
045import org.apache.hadoop.hbase.util.Bytes;
046import org.junit.After;
047import org.junit.AfterClass;
048import org.junit.Assert;
049import org.junit.Before;
050import org.junit.BeforeClass;
051import org.junit.ClassRule;
052import org.junit.Test;
053import org.junit.experimental.categories.Category;
054import org.slf4j.Logger;
055import org.slf4j.LoggerFactory;
056
057import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
058
059@Category({ LargeTests.class })
060public class TestRSGroupsAdmin2 extends TestRSGroupsBase {
061
062  @ClassRule
063  public static final HBaseClassTestRule CLASS_RULE =
064    HBaseClassTestRule.forClass(TestRSGroupsAdmin2.class);
065
066  protected static final Logger LOG = LoggerFactory.getLogger(TestRSGroupsAdmin2.class);
067
068  @BeforeClass
069  public static void setUp() throws Exception {
070    setUpTestBeforeClass();
071  }
072
073  @AfterClass
074  public static void tearDown() throws Exception {
075    tearDownAfterClass();
076  }
077
078  @Before
079  public void beforeMethod() throws Exception {
080    setUpBeforeMethod();
081  }
082
083  @After
084  public void afterMethod() throws Exception {
085    tearDownAfterMethod();
086  }
087
088  @Test
089  public void testRegionMove() throws Exception {
090    final RSGroupInfo newGroup = addGroup(getGroupName(name.getMethodName()), 1);
091    final byte[] familyNameBytes = Bytes.toBytes("f");
092    // All the regions created below will be assigned to the default group.
093    TEST_UTIL.createMultiRegionTable(tableName, familyNameBytes, 6);
094    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
095      @Override
096      public boolean evaluate() throws Exception {
097        List<String> regions = getTableRegionMap().get(tableName);
098        if (regions == null) {
099          return false;
100        }
101
102        return getTableRegionMap().get(tableName).size() >= 6;
103      }
104    });
105
106    // get target region to move
107    Map<ServerName, List<String>> assignMap = getTableServerRegionMap().get(tableName);
108    String targetRegion = null;
109    for (ServerName server : assignMap.keySet()) {
110      targetRegion = assignMap.get(server).size() > 0 ? assignMap.get(server).get(0) : null;
111      if (targetRegion != null) {
112        break;
113      }
114    }
115    // get server which is not a member of new group
116    ServerName tmpTargetServer = null;
117    for (ServerName server : admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS))
118      .getLiveServerMetrics().keySet()) {
119      if (!newGroup.containsServer(server.getAddress())) {
120        tmpTargetServer = server;
121        break;
122      }
123    }
124    final ServerName targetServer = tmpTargetServer;
125    // move target server to group
126    rsGroupAdmin.moveServers(Sets.newHashSet(targetServer.getAddress()), newGroup.getName());
127    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
128      @Override
129      public boolean evaluate() throws Exception {
130        return admin.getRegions(targetServer).size() <= 0;
131      }
132    });
133
134    // Lets move this region to the new group.
135    TEST_UTIL.getAdmin()
136      .move(Bytes.toBytes(RegionInfo.encodeRegionName(Bytes.toBytes(targetRegion))), targetServer);
137    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
138      @Override
139      public boolean evaluate() throws Exception {
140        return getTableRegionMap().get(tableName) != null &&
141          getTableRegionMap().get(tableName).size() == 6 &&
142          admin.getClusterMetrics(EnumSet.of(Option.REGIONS_IN_TRANSITION))
143            .getRegionStatesInTransition().size() < 1;
144      }
145    });
146
147    // verify that targetServer didn't open it
148    for (RegionInfo region : admin.getRegions(targetServer)) {
149      if (targetRegion.equals(region.getRegionNameAsString())) {
150        fail("Target server opened region");
151      }
152    }
153  }
154
155  @Test
156  public void testRegionServerMove() throws IOException, InterruptedException {
157    int initNumGroups = rsGroupAdmin.listRSGroups().size();
158    RSGroupInfo appInfo = addGroup(getGroupName(name.getMethodName()), 1);
159    RSGroupInfo adminInfo = addGroup(getGroupName(name.getMethodName()), 1);
160    RSGroupInfo dInfo = rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP);
161    Assert.assertEquals(initNumGroups + 2, rsGroupAdmin.listRSGroups().size());
162    assertEquals(1, adminInfo.getServers().size());
163    assertEquals(1, appInfo.getServers().size());
164    assertEquals(getNumServers() - 2, dInfo.getServers().size());
165    rsGroupAdmin.moveServers(appInfo.getServers(), RSGroupInfo.DEFAULT_GROUP);
166    rsGroupAdmin.removeRSGroup(appInfo.getName());
167    rsGroupAdmin.moveServers(adminInfo.getServers(), RSGroupInfo.DEFAULT_GROUP);
168    rsGroupAdmin.removeRSGroup(adminInfo.getName());
169    Assert.assertEquals(rsGroupAdmin.listRSGroups().size(), initNumGroups);
170  }
171
172  @Test
173  public void testMoveServers() throws Exception {
174    // create groups and assign servers
175    addGroup("bar", 3);
176    rsGroupAdmin.addRSGroup("foo");
177
178    RSGroupInfo barGroup = rsGroupAdmin.getRSGroupInfo("bar");
179    RSGroupInfo fooGroup = rsGroupAdmin.getRSGroupInfo("foo");
180    assertEquals(3, barGroup.getServers().size());
181    assertEquals(0, fooGroup.getServers().size());
182
183    // test fail bogus server move
184    try {
185      rsGroupAdmin.moveServers(Sets.newHashSet(Address.fromString("foo:9999")), "foo");
186      fail("Bogus servers shouldn't have been successfully moved.");
187    } catch (IOException ex) {
188      String exp = "Source RSGroup for server foo:9999 does not exist.";
189      String msg = "Expected '" + exp + "' in exception message: ";
190      assertTrue(msg + " " + ex.getMessage(), ex.getMessage().contains(exp));
191    }
192
193    // test success case
194    LOG.info("moving servers " + barGroup.getServers() + " to group foo");
195    rsGroupAdmin.moveServers(barGroup.getServers(), fooGroup.getName());
196
197    barGroup = rsGroupAdmin.getRSGroupInfo("bar");
198    fooGroup = rsGroupAdmin.getRSGroupInfo("foo");
199    assertEquals(0, barGroup.getServers().size());
200    assertEquals(3, fooGroup.getServers().size());
201
202    LOG.info("moving servers " + fooGroup.getServers() + " to group default");
203    rsGroupAdmin.moveServers(fooGroup.getServers(), RSGroupInfo.DEFAULT_GROUP);
204
205    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
206      @Override
207      public boolean evaluate() throws Exception {
208        return getNumServers() == rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP)
209          .getServers().size();
210      }
211    });
212
213    fooGroup = rsGroupAdmin.getRSGroupInfo("foo");
214    assertEquals(0, fooGroup.getServers().size());
215
216    // test group removal
217    LOG.info("Remove group " + barGroup.getName());
218    rsGroupAdmin.removeRSGroup(barGroup.getName());
219    Assert.assertEquals(null, rsGroupAdmin.getRSGroupInfo(barGroup.getName()));
220    LOG.info("Remove group " + fooGroup.getName());
221    rsGroupAdmin.removeRSGroup(fooGroup.getName());
222    Assert.assertEquals(null, rsGroupAdmin.getRSGroupInfo(fooGroup.getName()));
223  }
224
225  @Test
226  public void testRemoveServers() throws Exception {
227    LOG.info("testRemoveServers");
228    final RSGroupInfo newGroup = addGroup(getGroupName(name.getMethodName()), 3);
229    Iterator<Address> iterator = newGroup.getServers().iterator();
230    ServerName targetServer = getServerName(iterator.next());
231
232    // remove online servers
233    try {
234      rsGroupAdmin.removeServers(Sets.newHashSet(targetServer.getAddress()));
235      fail("Online servers shouldn't have been successfully removed.");
236    } catch (IOException ex) {
237      String exp =
238        "Server " + targetServer.getAddress() + " is an online server, not allowed to remove.";
239      String msg = "Expected '" + exp + "' in exception message: ";
240      assertTrue(msg + " " + ex.getMessage(), ex.getMessage().contains(exp));
241    }
242    assertTrue(newGroup.getServers().contains(targetServer.getAddress()));
243
244    // remove dead servers
245    NUM_DEAD_SERVERS = cluster.getClusterMetrics().getDeadServerNames().size();
246    try {
247      // stopping may cause an exception
248      // due to the connection loss
249      LOG.info("stopping server " + targetServer.getServerName());
250      admin.stopRegionServer(targetServer.getAddress().toString());
251      NUM_DEAD_SERVERS++;
252    } catch (Exception e) {
253    }
254
255    // wait for stopped regionserver to dead server list
256    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
257      @Override
258      public boolean evaluate() throws Exception {
259        return !master.getServerManager().areDeadServersInProgress() &&
260          cluster.getClusterMetrics().getDeadServerNames().size() == NUM_DEAD_SERVERS;
261      }
262    });
263
264    try {
265      rsGroupAdmin.removeServers(Sets.newHashSet(targetServer.getAddress()));
266      fail("Dead servers shouldn't have been successfully removed.");
267    } catch (IOException ex) {
268      String exp = "Server " + targetServer.getAddress() + " is on the dead servers list," +
269        " Maybe it will come back again, not allowed to remove.";
270      String msg = "Expected '" + exp + "' in exception message: ";
271      assertTrue(msg + " " + ex.getMessage(), ex.getMessage().contains(exp));
272    }
273    assertTrue(newGroup.getServers().contains(targetServer.getAddress()));
274
275    // remove decommissioned servers
276    List<ServerName> serversToDecommission = new ArrayList<>();
277    targetServer = getServerName(iterator.next());
278    assertTrue(master.getServerManager().getOnlineServers().containsKey(targetServer));
279    serversToDecommission.add(targetServer);
280
281    admin.decommissionRegionServers(serversToDecommission, true);
282    assertEquals(1, admin.listDecommissionedRegionServers().size());
283
284    assertTrue(newGroup.getServers().contains(targetServer.getAddress()));
285    rsGroupAdmin.removeServers(Sets.newHashSet(targetServer.getAddress()));
286    Set<Address> newGroupServers = rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getServers();
287    assertFalse(newGroupServers.contains(targetServer.getAddress()));
288    assertEquals(2, newGroupServers.size());
289
290    assertTrue(observer.preRemoveServersCalled);
291    assertTrue(observer.postRemoveServersCalled);
292  }
293
294  @Test
295  public void testMoveServersAndTables() throws Exception {
296    LOG.info("testMoveServersAndTables");
297    final RSGroupInfo newGroup = addGroup(getGroupName(name.getMethodName()), 1);
298    // create table
299    final byte[] familyNameBytes = Bytes.toBytes("f");
300    TEST_UTIL.createMultiRegionTable(tableName, familyNameBytes, 5);
301    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
302      @Override
303      public boolean evaluate() throws Exception {
304        List<String> regions = getTableRegionMap().get(tableName);
305        if (regions == null) {
306          return false;
307        }
308
309        return getTableRegionMap().get(tableName).size() >= 5;
310      }
311    });
312
313    // get server which is not a member of new group
314    ServerName targetServer = null;
315    for (ServerName server : admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS))
316      .getLiveServerMetrics().keySet()) {
317      if (!newGroup.containsServer(server.getAddress()) &&
318        !rsGroupAdmin.getRSGroupInfo("master").containsServer(server.getAddress())) {
319        targetServer = server;
320        break;
321      }
322    }
323
324    LOG.debug("Print group info : " + rsGroupAdmin.listRSGroups());
325    int oldDefaultGroupServerSize =
326      rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size();
327    int oldDefaultGroupTableSize =
328      rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getTables().size();
329
330    // test fail bogus server move
331    try {
332      rsGroupAdmin.moveServersAndTables(Sets.newHashSet(Address.fromString("foo:9999")),
333        Sets.newHashSet(tableName), newGroup.getName());
334      fail("Bogus servers shouldn't have been successfully moved.");
335    } catch (IOException ex) {
336      String exp = "Source RSGroup for server foo:9999 does not exist.";
337      String msg = "Expected '" + exp + "' in exception message: ";
338      assertTrue(msg + " " + ex.getMessage(), ex.getMessage().contains(exp));
339    }
340
341    // test move when src = dst
342    rsGroupAdmin.moveServersAndTables(Sets.newHashSet(targetServer.getAddress()),
343      Sets.newHashSet(tableName), RSGroupInfo.DEFAULT_GROUP);
344
345    // verify default group info
346    Assert.assertEquals(oldDefaultGroupServerSize,
347      rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size());
348    Assert.assertEquals(oldDefaultGroupTableSize,
349      rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getTables().size());
350
351    // verify new group info
352    Assert.assertEquals(1, rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getServers().size());
353    Assert.assertEquals(0, rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getTables().size());
354
355    // get all region to move targetServer
356    List<String> regionList = getTableRegionMap().get(tableName);
357    for (String region : regionList) {
358      // Lets move this region to the targetServer
359      TEST_UTIL.getAdmin().move(Bytes.toBytes(RegionInfo.encodeRegionName(Bytes.toBytes(region))),
360        targetServer);
361    }
362
363    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
364      @Override
365      public boolean evaluate() throws Exception {
366        return getTableRegionMap().get(tableName) != null &&
367          getTableRegionMap().get(tableName).size() == 5 &&
368          getTableServerRegionMap().get(tableName).size() == 1 &&
369          admin.getClusterMetrics(EnumSet.of(Option.REGIONS_IN_TRANSITION))
370            .getRegionStatesInTransition().size() < 1;
371      }
372    });
373
374    // verify that all region move to targetServer
375    Assert.assertEquals(5, getTableServerRegionMap().get(tableName).get(targetServer).size());
376
377    // move targetServer and table to newGroup
378    LOG.info("moving server and table to newGroup");
379    rsGroupAdmin.moveServersAndTables(Sets.newHashSet(targetServer.getAddress()),
380      Sets.newHashSet(tableName), newGroup.getName());
381
382    // verify group change
383    Assert.assertEquals(newGroup.getName(),
384      rsGroupAdmin.getRSGroupInfoOfTable(tableName).getName());
385
386    // verify servers' not exist in old group
387    Set<Address> defaultServers =
388      rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers();
389    assertFalse(defaultServers.contains(targetServer.getAddress()));
390
391    // verify servers' exist in new group
392    Set<Address> newGroupServers = rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getServers();
393    assertTrue(newGroupServers.contains(targetServer.getAddress()));
394
395    // verify tables' not exist in old group
396    Set<TableName> defaultTables =
397      rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getTables();
398    assertFalse(defaultTables.contains(tableName));
399
400    // verify tables' exist in new group
401    Set<TableName> newGroupTables = rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getTables();
402    assertTrue(newGroupTables.contains(tableName));
403
404    // verify that all region still assgin on targetServer
405    Assert.assertEquals(5, getTableServerRegionMap().get(tableName).get(targetServer).size());
406
407    assertTrue(observer.preMoveServersAndTables);
408    assertTrue(observer.postMoveServersAndTables);
409  }
410
411  @Test
412  public void testMoveServersFromDefaultGroup() throws Exception {
413    // create groups and assign servers
414    rsGroupAdmin.addRSGroup("foo");
415
416    RSGroupInfo fooGroup = rsGroupAdmin.getRSGroupInfo("foo");
417    assertEquals(0, fooGroup.getServers().size());
418    RSGroupInfo defaultGroup = rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP);
419
420    // test remove all servers from default
421    try {
422      rsGroupAdmin.moveServers(defaultGroup.getServers(), fooGroup.getName());
423      fail(RSGroupAdminServer.KEEP_ONE_SERVER_IN_DEFAULT_ERROR_MESSAGE);
424    } catch (ConstraintException ex) {
425      assertTrue(
426        ex.getMessage().contains(RSGroupAdminServer.KEEP_ONE_SERVER_IN_DEFAULT_ERROR_MESSAGE));
427    }
428
429    // test success case, remove one server from default ,keep at least one server
430    if (defaultGroup.getServers().size() > 1) {
431      Address serverInDefaultGroup = defaultGroup.getServers().iterator().next();
432      LOG.info("moving server " + serverInDefaultGroup + " from group default to group " +
433        fooGroup.getName());
434      rsGroupAdmin.moveServers(Sets.newHashSet(serverInDefaultGroup), fooGroup.getName());
435    }
436
437    fooGroup = rsGroupAdmin.getRSGroupInfo("foo");
438    LOG.info("moving servers " + fooGroup.getServers() + " to group default");
439    rsGroupAdmin.moveServers(fooGroup.getServers(), RSGroupInfo.DEFAULT_GROUP);
440
441    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
442      @Override
443      public boolean evaluate() throws Exception {
444        return getNumServers() == rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP)
445          .getServers().size();
446      }
447    });
448
449    fooGroup = rsGroupAdmin.getRSGroupInfo("foo");
450    assertEquals(0, fooGroup.getServers().size());
451
452    // test group removal
453    LOG.info("Remove group " + fooGroup.getName());
454    rsGroupAdmin.removeRSGroup(fooGroup.getName());
455    Assert.assertEquals(null, rsGroupAdmin.getRSGroupInfo(fooGroup.getName()));
456  }
457
458  @Test
459  public void testFailedMoveWhenMoveServer() throws Exception {
460    final RSGroupInfo newGroup = addGroup(getGroupName(name.getMethodName()), 1);
461    final byte[] familyNameBytes = Bytes.toBytes("f");
462    final int tableRegionCount = 10;
463    // All the regions created below will be assigned to the default group.
464    TEST_UTIL.createMultiRegionTable(tableName, familyNameBytes, tableRegionCount);
465    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
466      @Override
467      public boolean evaluate() throws Exception {
468        List<String> regions = getTableRegionMap().get(tableName);
469        if (regions == null) {
470          return false;
471        }
472        return getTableRegionMap().get(tableName).size() >= tableRegionCount;
473      }
474    });
475
476    // get target server to move, which should has more than one regions
477    // randomly set a region state to SPLITTING
478    Map<ServerName, List<String>> assignMap = getTableServerRegionMap().get(tableName);
479    String rregion = null;
480    ServerName toMoveServer = null;
481    for (ServerName server : assignMap.keySet()) {
482      rregion = assignMap.get(server).size() > 1 && !newGroup.containsServer(server.getAddress()) ?
483          assignMap.get(server).get(0) :
484          null;
485      if (rregion != null) {
486        toMoveServer = server;
487        break;
488      }
489    }
490    assert toMoveServer != null;
491    RegionInfo ri = TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().
492        getRegionInfo(Bytes.toBytesBinary(rregion));
493    RegionStateNode rsn =
494        TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates()
495            .getRegionStateNode(ri);
496    rsn.setState(RegionState.State.SPLITTING);
497
498    // start thread to recover region state
499    final ServerName movedServer = toMoveServer;
500    final String sregion = rregion;
501    AtomicBoolean changed = new AtomicBoolean(false);
502    Thread t1 = new Thread(() -> {
503      LOG.debug("thread1 start running, will recover region state");
504      long current = System.currentTimeMillis();
505      while (System.currentTimeMillis() - current <= 50000) {
506        List<RegionInfo> regions = master.getAssignmentManager().getRegionsOnServer(movedServer);
507        LOG.debug("server region size is:{}", regions.size());
508        assert regions.size() >= 1;
509        // when there is exactly one region left, we can determine the move operation encountered
510        // exception caused by the strange region state.
511        if (regions.size() == 1) {
512          assertEquals(regions.get(0).getRegionNameAsString(), sregion);
513          rsn.setState(RegionState.State.OPEN);
514          LOG.info("set region {} state OPEN", sregion);
515          changed.set(true);
516          break;
517        }
518        sleep(5000);
519      }
520    });
521    t1.start();
522
523    // move target server to group
524    Thread t2 = new Thread(() -> {
525      LOG.info("thread2 start running, to move regions");
526      try {
527        rsGroupAdmin.moveServers(Sets.newHashSet(movedServer.getAddress()), newGroup.getName());
528      } catch (IOException e) {
529        LOG.error("move server error", e);
530      }
531    });
532    t2.start();
533
534    t1.join();
535    t2.join();
536
537    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
538      @Override
539      public boolean evaluate() {
540        if (changed.get()) {
541          return master.getAssignmentManager().getRegionsOnServer(movedServer).size() == 0 && !rsn
542              .getRegionLocation().equals(movedServer);
543        }
544        return false;
545      }
546    });
547  }
548
549  @Test
550  public void testFailedMoveWhenMoveTable() throws Exception {
551    final RSGroupInfo newGroup = addGroup(getGroupName(name.getMethodName()), 1);
552    final byte[] familyNameBytes = Bytes.toBytes("f");
553    final int tableRegionCount = 5;
554    // All the regions created below will be assigned to the default group.
555    TEST_UTIL.createMultiRegionTable(tableName, familyNameBytes, tableRegionCount);
556    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
557      @Override
558      public boolean evaluate() throws Exception {
559        List<String> regions = getTableRegionMap().get(tableName);
560        if (regions == null) {
561          return false;
562        }
563        return getTableRegionMap().get(tableName).size() >= tableRegionCount;
564      }
565    });
566
567    // randomly set a region state to SPLITTING
568    Map<ServerName, List<String>> assignMap = getTableServerRegionMap().get(tableName);
569    String rregion = null;
570    ServerName srcServer = null;
571    for (ServerName server : assignMap.keySet()) {
572      rregion = assignMap.get(server).size() >= 1 && !newGroup.containsServer(server.getAddress()) ?
573          assignMap.get(server).get(0) :
574          null;
575      if (rregion != null) {
576        srcServer = server;
577        break;
578      }
579    }
580    assert srcServer != null;
581    RegionInfo ri = TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().
582        getRegionInfo(Bytes.toBytesBinary(rregion));
583    RegionStateNode rsn =
584        TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates()
585            .getRegionStateNode(ri);
586    rsn.setState(RegionState.State.SPLITTING);
587
588    // move table to group
589    Thread t2 = new Thread(() -> {
590      LOG.info("thread2 start running, to move regions");
591      try {
592        rsGroupAdmin.moveTables(Sets.newHashSet(tableName), newGroup.getName());
593      } catch (IOException e) {
594        LOG.error("move server error", e);
595      }
596    });
597    t2.start();
598
599    // start thread to recover region state
600    final ServerName ss = srcServer;
601    final String sregion = rregion;
602    AtomicBoolean changed = new AtomicBoolean(false);
603    Thread t1 = new Thread(() -> {
604      LOG.info("thread1 start running, will recover region state");
605      long current = System.currentTimeMillis();
606      while (System.currentTimeMillis() - current <= 50000) {
607        List<RegionInfo> regions = master.getAssignmentManager().getRegionsOnServer(ss);
608        List<RegionInfo> tableRegions = new ArrayList<>();
609        for (RegionInfo regionInfo : regions) {
610          if (regionInfo.getTable().equals(tableName)) {
611            tableRegions.add(regionInfo);
612          }
613        }
614        LOG.debug("server table region size is:{}", tableRegions.size());
615        assert tableRegions.size() >= 1;
616        // when there is exactly one region left, we can determine the move operation encountered
617        // exception caused by the strange region state.
618        if (tableRegions.size() == 1) {
619          assertEquals(tableRegions.get(0).getRegionNameAsString(), sregion);
620          rsn.setState(RegionState.State.OPEN);
621          LOG.info("set region {} state OPEN", sregion);
622          changed.set(true);
623          break;
624        }
625        sleep(5000);
626      }
627    });
628    t1.start();
629
630    t1.join();
631    t2.join();
632
633    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
634      @Override
635      public boolean evaluate() {
636        if (changed.get()) {
637          boolean serverHasTableRegions = false;
638          for (RegionInfo regionInfo : master.getAssignmentManager().getRegionsOnServer(ss)) {
639            if (regionInfo.getTable().equals(tableName)) {
640              serverHasTableRegions = true;
641              break;
642            }
643          }
644          return !serverHasTableRegions && !rsn.getRegionLocation().equals(ss);
645        }
646        return false;
647      }
648    });
649  }
650
651  @Test
652  public void testMoveTablePerformance() throws Exception {
653    final RSGroupInfo newGroup = addGroup(getGroupName(name.getMethodName()), 1);
654    final byte[] familyNameBytes = Bytes.toBytes("f");
655    final int tableRegionCount = 100;
656    // All the regions created below will be assigned to the default group.
657    TEST_UTIL.createMultiRegionTable(tableName, familyNameBytes, tableRegionCount);
658    TEST_UTIL.waitFor(WAIT_TIMEOUT, (Waiter.Predicate<Exception>) () -> {
659      List<String> regions = getTableRegionMap().get(tableName);
660      if (regions == null) {
661        return false;
662      }
663      return getTableRegionMap().get(tableName).size() >= tableRegionCount;
664    });
665    long startTime = System.currentTimeMillis();
666    rsGroupAdmin.moveTables(Sets.newHashSet(tableName), newGroup.getName());
667    long timeTaken = System.currentTimeMillis() - startTime;
668    String msg =
669      "Should not take mote than 15000 ms to move a table with 100 regions. Time taken  ="
670        + timeTaken + " ms";
671    // This test case is meant to be used for verifying the performance quickly by a developer.
672    // Moving 100 regions takes much less than 15000 ms. Given 15000 ms so test cases passes
673    // on all environment.
674    assertTrue(msg, timeTaken < 15000);
675    LOG.info("Time taken to move a table with 100 region is {} ms", timeTaken);
676  }
677}