Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.rsgroup;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertTrue;
023import static org.junit.Assert.fail;
024
025import java.io.IOException;
026import java.util.ArrayList;
027import java.util.EnumSet;
028import java.util.Iterator;
029import java.util.List;
030import java.util.Map;
031import java.util.Set;
032import org.apache.hadoop.hbase.ClusterMetrics.Option;
033import org.apache.hadoop.hbase.HBaseClassTestRule;
034import org.apache.hadoop.hbase.ServerName;
035import org.apache.hadoop.hbase.TableName;
036import org.apache.hadoop.hbase.Waiter;
037import org.apache.hadoop.hbase.client.RegionInfo;
038import org.apache.hadoop.hbase.constraint.ConstraintException;
039import org.apache.hadoop.hbase.net.Address;
040import org.apache.hadoop.hbase.testclassification.MediumTests;
041import org.apache.hadoop.hbase.util.Bytes;
042import org.junit.After;
043import org.junit.AfterClass;
044import org.junit.Assert;
045import org.junit.Before;
046import org.junit.BeforeClass;
047import org.junit.ClassRule;
048import org.junit.Test;
049import org.junit.experimental.categories.Category;
050import org.slf4j.Logger;
051import org.slf4j.LoggerFactory;
052
053import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
054
055@Category({ MediumTests.class })
056public class TestRSGroupsAdmin2 extends TestRSGroupsBase {
057
058  @ClassRule
059  public static final HBaseClassTestRule CLASS_RULE =
060    HBaseClassTestRule.forClass(TestRSGroupsAdmin2.class);
061
062  protected static final Logger LOG = LoggerFactory.getLogger(TestRSGroupsAdmin2.class);
063
064  @BeforeClass
065  public static void setUp() throws Exception {
066    setUpTestBeforeClass();
067  }
068
069  @AfterClass
070  public static void tearDown() throws Exception {
071    tearDownAfterClass();
072  }
073
074  @Before
075  public void beforeMethod() throws Exception {
076    setUpBeforeMethod();
077  }
078
079  @After
080  public void afterMethod() throws Exception {
081    tearDownAfterMethod();
082  }
083
084  @Test
085  public void testRegionMove() throws Exception {
086    final RSGroupInfo newGroup = addGroup(getGroupName(name.getMethodName()), 1);
087    final byte[] familyNameBytes = Bytes.toBytes("f");
088    // All the regions created below will be assigned to the default group.
089    TEST_UTIL.createMultiRegionTable(tableName, familyNameBytes, 6);
090    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
091      @Override
092      public boolean evaluate() throws Exception {
093        List<String> regions = getTableRegionMap().get(tableName);
094        if (regions == null) {
095          return false;
096        }
097
098        return getTableRegionMap().get(tableName).size() >= 6;
099      }
100    });
101
102    // get target region to move
103    Map<ServerName, List<String>> assignMap = getTableServerRegionMap().get(tableName);
104    String targetRegion = null;
105    for (ServerName server : assignMap.keySet()) {
106      targetRegion = assignMap.get(server).size() > 0 ? assignMap.get(server).get(0) : null;
107      if (targetRegion != null) {
108        break;
109      }
110    }
111    // get server which is not a member of new group
112    ServerName tmpTargetServer = null;
113    for (ServerName server : admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS))
114      .getLiveServerMetrics().keySet()) {
115      if (!newGroup.containsServer(server.getAddress())) {
116        tmpTargetServer = server;
117        break;
118      }
119    }
120    final ServerName targetServer = tmpTargetServer;
121    // move target server to group
122    rsGroupAdmin.moveServers(Sets.newHashSet(targetServer.getAddress()), newGroup.getName());
123    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
124      @Override
125      public boolean evaluate() throws Exception {
126        return admin.getRegions(targetServer).size() <= 0;
127      }
128    });
129
130    // Lets move this region to the new group.
131    TEST_UTIL.getAdmin()
132      .move(Bytes.toBytes(RegionInfo.encodeRegionName(Bytes.toBytes(targetRegion))), targetServer);
133    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
134      @Override
135      public boolean evaluate() throws Exception {
136        return getTableRegionMap().get(tableName) != null &&
137          getTableRegionMap().get(tableName).size() == 6 &&
138          admin.getClusterMetrics(EnumSet.of(Option.REGIONS_IN_TRANSITION))
139            .getRegionStatesInTransition().size() < 1;
140      }
141    });
142
143    // verify that targetServer didn't open it
144    for (RegionInfo region : admin.getRegions(targetServer)) {
145      if (targetRegion.equals(region.getRegionNameAsString())) {
146        fail("Target server opened region");
147      }
148    }
149  }
150
151  @Test
152  public void testRegionServerMove() throws IOException, InterruptedException {
153    int initNumGroups = rsGroupAdmin.listRSGroups().size();
154    RSGroupInfo appInfo = addGroup(getGroupName(name.getMethodName()), 1);
155    RSGroupInfo adminInfo = addGroup(getGroupName(name.getMethodName()), 1);
156    RSGroupInfo dInfo = rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP);
157    Assert.assertEquals(initNumGroups + 2, rsGroupAdmin.listRSGroups().size());
158    assertEquals(1, adminInfo.getServers().size());
159    assertEquals(1, appInfo.getServers().size());
160    assertEquals(getNumServers() - 2, dInfo.getServers().size());
161    rsGroupAdmin.moveServers(appInfo.getServers(), RSGroupInfo.DEFAULT_GROUP);
162    rsGroupAdmin.removeRSGroup(appInfo.getName());
163    rsGroupAdmin.moveServers(adminInfo.getServers(), RSGroupInfo.DEFAULT_GROUP);
164    rsGroupAdmin.removeRSGroup(adminInfo.getName());
165    Assert.assertEquals(rsGroupAdmin.listRSGroups().size(), initNumGroups);
166  }
167
168  @Test
169  public void testMoveServers() throws Exception {
170    // create groups and assign servers
171    addGroup("bar", 3);
172    rsGroupAdmin.addRSGroup("foo");
173
174    RSGroupInfo barGroup = rsGroupAdmin.getRSGroupInfo("bar");
175    RSGroupInfo fooGroup = rsGroupAdmin.getRSGroupInfo("foo");
176    assertEquals(3, barGroup.getServers().size());
177    assertEquals(0, fooGroup.getServers().size());
178
179    // test fail bogus server move
180    try {
181      rsGroupAdmin.moveServers(Sets.newHashSet(Address.fromString("foo:9999")), "foo");
182      fail("Bogus servers shouldn't have been successfully moved.");
183    } catch (IOException ex) {
184      String exp = "Source RSGroup for server foo:9999 does not exist.";
185      String msg = "Expected '" + exp + "' in exception message: ";
186      assertTrue(msg + " " + ex.getMessage(), ex.getMessage().contains(exp));
187    }
188
189    // test success case
190    LOG.info("moving servers " + barGroup.getServers() + " to group foo");
191    rsGroupAdmin.moveServers(barGroup.getServers(), fooGroup.getName());
192
193    barGroup = rsGroupAdmin.getRSGroupInfo("bar");
194    fooGroup = rsGroupAdmin.getRSGroupInfo("foo");
195    assertEquals(0, barGroup.getServers().size());
196    assertEquals(3, fooGroup.getServers().size());
197
198    LOG.info("moving servers " + fooGroup.getServers() + " to group default");
199    rsGroupAdmin.moveServers(fooGroup.getServers(), RSGroupInfo.DEFAULT_GROUP);
200
201    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
202      @Override
203      public boolean evaluate() throws Exception {
204        return getNumServers() == rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP)
205          .getServers().size();
206      }
207    });
208
209    fooGroup = rsGroupAdmin.getRSGroupInfo("foo");
210    assertEquals(0, fooGroup.getServers().size());
211
212    // test group removal
213    LOG.info("Remove group " + barGroup.getName());
214    rsGroupAdmin.removeRSGroup(barGroup.getName());
215    Assert.assertEquals(null, rsGroupAdmin.getRSGroupInfo(barGroup.getName()));
216    LOG.info("Remove group " + fooGroup.getName());
217    rsGroupAdmin.removeRSGroup(fooGroup.getName());
218    Assert.assertEquals(null, rsGroupAdmin.getRSGroupInfo(fooGroup.getName()));
219  }
220
221  @Test
222  public void testRemoveServers() throws Exception {
223    LOG.info("testRemoveServers");
224    final RSGroupInfo newGroup = addGroup(getGroupName(name.getMethodName()), 3);
225    Iterator<Address> iterator = newGroup.getServers().iterator();
226    ServerName targetServer = getServerName(iterator.next());
227
228    // remove online servers
229    try {
230      rsGroupAdmin.removeServers(Sets.newHashSet(targetServer.getAddress()));
231      fail("Online servers shouldn't have been successfully removed.");
232    } catch (IOException ex) {
233      String exp =
234        "Server " + targetServer.getAddress() + " is an online server, not allowed to remove.";
235      String msg = "Expected '" + exp + "' in exception message: ";
236      assertTrue(msg + " " + ex.getMessage(), ex.getMessage().contains(exp));
237    }
238    assertTrue(newGroup.getServers().contains(targetServer.getAddress()));
239
240    // remove dead servers
241    NUM_DEAD_SERVERS = cluster.getClusterMetrics().getDeadServerNames().size();
242    try {
243      // stopping may cause an exception
244      // due to the connection loss
245      LOG.info("stopping server " + targetServer.getServerName());
246      admin.stopRegionServer(targetServer.getAddress().toString());
247      NUM_DEAD_SERVERS++;
248    } catch (Exception e) {
249    }
250
251    // wait for stopped regionserver to dead server list
252    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
253      @Override
254      public boolean evaluate() throws Exception {
255        return !master.getServerManager().areDeadServersInProgress() &&
256          cluster.getClusterMetrics().getDeadServerNames().size() == NUM_DEAD_SERVERS;
257      }
258    });
259
260    try {
261      rsGroupAdmin.removeServers(Sets.newHashSet(targetServer.getAddress()));
262      fail("Dead servers shouldn't have been successfully removed.");
263    } catch (IOException ex) {
264      String exp = "Server " + targetServer.getAddress() + " is on the dead servers list," +
265        " Maybe it will come back again, not allowed to remove.";
266      String msg = "Expected '" + exp + "' in exception message: ";
267      assertTrue(msg + " " + ex.getMessage(), ex.getMessage().contains(exp));
268    }
269    assertTrue(newGroup.getServers().contains(targetServer.getAddress()));
270
271    // remove decommissioned servers
272    List<ServerName> serversToDecommission = new ArrayList<>();
273    targetServer = getServerName(iterator.next());
274    assertTrue(master.getServerManager().getOnlineServers().containsKey(targetServer));
275    serversToDecommission.add(targetServer);
276
277    admin.decommissionRegionServers(serversToDecommission, true);
278    assertEquals(1, admin.listDecommissionedRegionServers().size());
279
280    assertTrue(newGroup.getServers().contains(targetServer.getAddress()));
281    rsGroupAdmin.removeServers(Sets.newHashSet(targetServer.getAddress()));
282    Set<Address> newGroupServers = rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getServers();
283    assertFalse(newGroupServers.contains(targetServer.getAddress()));
284    assertEquals(2, newGroupServers.size());
285
286    assertTrue(observer.preRemoveServersCalled);
287    assertTrue(observer.postRemoveServersCalled);
288  }
289
290  @Test
291  public void testMoveServersAndTables() throws Exception {
292    LOG.info("testMoveServersAndTables");
293    final RSGroupInfo newGroup = addGroup(getGroupName(name.getMethodName()), 1);
294    // create table
295    final byte[] familyNameBytes = Bytes.toBytes("f");
296    TEST_UTIL.createMultiRegionTable(tableName, familyNameBytes, 5);
297    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
298      @Override
299      public boolean evaluate() throws Exception {
300        List<String> regions = getTableRegionMap().get(tableName);
301        if (regions == null) {
302          return false;
303        }
304
305        return getTableRegionMap().get(tableName).size() >= 5;
306      }
307    });
308
309    // get server which is not a member of new group
310    ServerName targetServer = null;
311    for (ServerName server : admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS))
312      .getLiveServerMetrics().keySet()) {
313      if (!newGroup.containsServer(server.getAddress()) &&
314        !rsGroupAdmin.getRSGroupInfo("master").containsServer(server.getAddress())) {
315        targetServer = server;
316        break;
317      }
318    }
319
320    LOG.debug("Print group info : " + rsGroupAdmin.listRSGroups());
321    int oldDefaultGroupServerSize =
322      rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size();
323    int oldDefaultGroupTableSize =
324      rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getTables().size();
325
326    // test fail bogus server move
327    try {
328      rsGroupAdmin.moveServersAndTables(Sets.newHashSet(Address.fromString("foo:9999")),
329        Sets.newHashSet(tableName), newGroup.getName());
330      fail("Bogus servers shouldn't have been successfully moved.");
331    } catch (IOException ex) {
332      String exp = "Source RSGroup for server foo:9999 does not exist.";
333      String msg = "Expected '" + exp + "' in exception message: ";
334      assertTrue(msg + " " + ex.getMessage(), ex.getMessage().contains(exp));
335    }
336
337    // test fail server move
338    try {
339      rsGroupAdmin.moveServersAndTables(Sets.newHashSet(targetServer.getAddress()),
340        Sets.newHashSet(tableName), RSGroupInfo.DEFAULT_GROUP);
341      fail("servers shouldn't have been successfully moved.");
342    } catch (IOException ex) {
343      String exp = "Target RSGroup " + RSGroupInfo.DEFAULT_GROUP + " is same as source " +
344        RSGroupInfo.DEFAULT_GROUP + " RSGroup.";
345      String msg = "Expected '" + exp + "' in exception message: ";
346      assertTrue(msg + " " + ex.getMessage(), ex.getMessage().contains(exp));
347    }
348
349    // verify default group info
350    Assert.assertEquals(oldDefaultGroupServerSize,
351      rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size());
352    Assert.assertEquals(oldDefaultGroupTableSize,
353      rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getTables().size());
354
355    // verify new group info
356    Assert.assertEquals(1, rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getServers().size());
357    Assert.assertEquals(0, rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getTables().size());
358
359    // get all region to move targetServer
360    List<String> regionList = getTableRegionMap().get(tableName);
361    for (String region : regionList) {
362      // Lets move this region to the targetServer
363      TEST_UTIL.getAdmin().move(Bytes.toBytes(RegionInfo.encodeRegionName(Bytes.toBytes(region))),
364        targetServer);
365    }
366
367    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
368      @Override
369      public boolean evaluate() throws Exception {
370        return getTableRegionMap().get(tableName) != null &&
371          getTableRegionMap().get(tableName).size() == 5 &&
372          getTableServerRegionMap().get(tableName).size() == 1 &&
373          admin.getClusterMetrics(EnumSet.of(Option.REGIONS_IN_TRANSITION))
374            .getRegionStatesInTransition().size() < 1;
375      }
376    });
377
378    // verify that all region move to targetServer
379    Assert.assertEquals(5, getTableServerRegionMap().get(tableName).get(targetServer).size());
380
381    // move targetServer and table to newGroup
382    LOG.info("moving server and table to newGroup");
383    rsGroupAdmin.moveServersAndTables(Sets.newHashSet(targetServer.getAddress()),
384      Sets.newHashSet(tableName), newGroup.getName());
385
386    // verify group change
387    Assert.assertEquals(newGroup.getName(),
388      rsGroupAdmin.getRSGroupInfoOfTable(tableName).getName());
389
390    // verify servers' not exist in old group
391    Set<Address> defaultServers =
392      rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers();
393    assertFalse(defaultServers.contains(targetServer.getAddress()));
394
395    // verify servers' exist in new group
396    Set<Address> newGroupServers = rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getServers();
397    assertTrue(newGroupServers.contains(targetServer.getAddress()));
398
399    // verify tables' not exist in old group
400    Set<TableName> defaultTables =
401      rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getTables();
402    assertFalse(defaultTables.contains(tableName));
403
404    // verify tables' exist in new group
405    Set<TableName> newGroupTables = rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getTables();
406    assertTrue(newGroupTables.contains(tableName));
407
408    // verify that all region still assgin on targetServer
409    Assert.assertEquals(5, getTableServerRegionMap().get(tableName).get(targetServer).size());
410
411    assertTrue(observer.preMoveServersAndTables);
412    assertTrue(observer.postMoveServersAndTables);
413  }
414
415  @Test
416  public void testMoveServersFromDefaultGroup() throws Exception {
417    // create groups and assign servers
418    rsGroupAdmin.addRSGroup("foo");
419
420    RSGroupInfo fooGroup = rsGroupAdmin.getRSGroupInfo("foo");
421    assertEquals(0, fooGroup.getServers().size());
422    RSGroupInfo defaultGroup = rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP);
423
424    // test remove all servers from default
425    try {
426      rsGroupAdmin.moveServers(defaultGroup.getServers(), fooGroup.getName());
427      fail(RSGroupAdminServer.KEEP_ONE_SERVER_IN_DEFAULT_ERROR_MESSAGE);
428    } catch (ConstraintException ex) {
429      assertTrue(
430        ex.getMessage().contains(RSGroupAdminServer.KEEP_ONE_SERVER_IN_DEFAULT_ERROR_MESSAGE));
431    }
432
433    // test success case, remove one server from default ,keep at least one server
434    if (defaultGroup.getServers().size() > 1) {
435      Address serverInDefaultGroup = defaultGroup.getServers().iterator().next();
436      LOG.info("moving server " + serverInDefaultGroup + " from group default to group " +
437        fooGroup.getName());
438      rsGroupAdmin.moveServers(Sets.newHashSet(serverInDefaultGroup), fooGroup.getName());
439    }
440
441    fooGroup = rsGroupAdmin.getRSGroupInfo("foo");
442    LOG.info("moving servers " + fooGroup.getServers() + " to group default");
443    rsGroupAdmin.moveServers(fooGroup.getServers(), RSGroupInfo.DEFAULT_GROUP);
444
445    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
446      @Override
447      public boolean evaluate() throws Exception {
448        return getNumServers() == rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP)
449          .getServers().size();
450      }
451    });
452
453    fooGroup = rsGroupAdmin.getRSGroupInfo("foo");
454    assertEquals(0, fooGroup.getServers().size());
455
456    // test group removal
457    LOG.info("Remove group " + fooGroup.getName());
458    rsGroupAdmin.removeRSGroup(fooGroup.getName());
459    Assert.assertEquals(null, rsGroupAdmin.getRSGroupInfo(fooGroup.getName()));
460  }
461
462}