Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.rsgroup;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertTrue;
023import static org.junit.Assert.fail;
024
025import java.io.IOException;
026import java.util.ArrayList;
027import java.util.Iterator;
028import java.util.List;
029import java.util.Set;
030import org.apache.hadoop.hbase.HBaseClassTestRule;
031import org.apache.hadoop.hbase.NamespaceDescriptor;
032import org.apache.hadoop.hbase.ServerName;
033import org.apache.hadoop.hbase.TableName;
034import org.apache.hadoop.hbase.Waiter;
035import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
036import org.apache.hadoop.hbase.client.TableDescriptor;
037import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
038import org.apache.hadoop.hbase.net.Address;
039import org.apache.hadoop.hbase.quotas.QuotaTableUtil;
040import org.apache.hadoop.hbase.quotas.QuotaUtil;
041import org.apache.hadoop.hbase.testclassification.MediumTests;
042import org.apache.hadoop.hbase.util.Bytes;
043import org.junit.After;
044import org.junit.AfterClass;
045import org.junit.Assert;
046import org.junit.Before;
047import org.junit.BeforeClass;
048import org.junit.ClassRule;
049import org.junit.Test;
050import org.junit.experimental.categories.Category;
051import org.slf4j.Logger;
052import org.slf4j.LoggerFactory;
053
054import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
055
056@Category({ MediumTests.class })
057public class TestRSGroupsBasics extends TestRSGroupsBase {
058
059  @ClassRule
060  public static final HBaseClassTestRule CLASS_RULE =
061    HBaseClassTestRule.forClass(TestRSGroupsBasics.class);
062
063  protected static final Logger LOG = LoggerFactory.getLogger(TestRSGroupsBasics.class);
064
065  @BeforeClass
066  public static void setUp() throws Exception {
067    setUpTestBeforeClass();
068  }
069
070  @AfterClass
071  public static void tearDown() throws Exception {
072    tearDownAfterClass();
073  }
074
075  @Before
076  public void beforeMethod() throws Exception {
077    setUpBeforeMethod();
078  }
079
080  @After
081  public void afterMethod() throws Exception {
082    tearDownAfterMethod();
083  }
084
085  @Test
086  public void testBasicStartUp() throws IOException {
087    RSGroupInfo defaultInfo = rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP);
088    assertEquals(NUM_SLAVES_BASE, defaultInfo.getServers().size());
089    // Assignment of meta, namespace and rsgroup regions.
090    int count = master.getAssignmentManager().getRegionStates().getRegionAssignments().size();
091    // 3 (meta, namespace and rsgroup)
092    assertEquals(3, count);
093  }
094
095  @Test
096  public void testCreateAndDrop() throws Exception {
097    TEST_UTIL.createTable(tableName, Bytes.toBytes("cf"));
098    // wait for created table to be assigned
099    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
100      @Override
101      public boolean evaluate() throws Exception {
102        return getTableRegionMap().get(tableName) != null;
103      }
104    });
105    TEST_UTIL.deleteTable(tableName);
106  }
107
108  @Test
109  public void testCreateMultiRegion() throws IOException {
110    byte[] end = { 1, 3, 5, 7, 9 };
111    byte[] start = { 0, 2, 4, 6, 8 };
112    byte[][] f = { Bytes.toBytes("f") };
113    TEST_UTIL.createTable(tableName, f, 1, start, end, 10);
114  }
115
116  @Test
117  public void testNamespaceCreateAndAssign() throws Exception {
118    LOG.info("testNamespaceCreateAndAssign");
119    String nsName = tablePrefix + "_foo";
120    final TableName tableName = TableName.valueOf(nsName, tablePrefix + "_testCreateAndAssign");
121    RSGroupInfo appInfo = addGroup("appInfo", 1);
122    admin.createNamespace(NamespaceDescriptor.create(nsName)
123      .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, "appInfo").build());
124    final TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName)
125      .setColumnFamily(ColumnFamilyDescriptorBuilder.of("f")).build();
126    admin.createTable(desc);
127    // wait for created table to be assigned
128    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
129      @Override
130      public boolean evaluate() throws Exception {
131        return getTableRegionMap().get(desc.getTableName()) != null;
132      }
133    });
134    ServerName targetServer = getServerName(appInfo.getServers().iterator().next());
135    // verify it was assigned to the right group
136    Assert.assertEquals(1, admin.getRegions(targetServer).size());
137  }
138
139  @Test
140  public void testCreateWhenRsgroupNoOnlineServers() throws Exception {
141    LOG.info("testCreateWhenRsgroupNoOnlineServers");
142
143    // set rsgroup has no online servers and test create table
144    final RSGroupInfo appInfo = addGroup("appInfo", 1);
145    Iterator<Address> iterator = appInfo.getServers().iterator();
146    List<ServerName> serversToDecommission = new ArrayList<>();
147    ServerName targetServer = getServerName(iterator.next());
148    assertTrue(master.getServerManager().getOnlineServers().containsKey(targetServer));
149    serversToDecommission.add(targetServer);
150    admin.decommissionRegionServers(serversToDecommission, true);
151    assertEquals(1, admin.listDecommissionedRegionServers().size());
152
153    final TableName tableName = TableName.valueOf(tablePrefix + "_ns", name.getMethodName());
154    admin.createNamespace(NamespaceDescriptor.create(tableName.getNamespaceAsString())
155      .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, appInfo.getName()).build());
156    final TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName)
157      .setColumnFamily(ColumnFamilyDescriptorBuilder.of("f")).build();
158    try {
159      admin.createTable(desc);
160      fail("Shouldn't create table successfully!");
161    } catch (Exception e) {
162      LOG.debug("create table error", e);
163    }
164
165    // recommission and test create table
166    admin.recommissionRegionServer(targetServer, null);
167    assertEquals(0, admin.listDecommissionedRegionServers().size());
168    admin.createTable(desc);
169    // wait for created table to be assigned
170    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
171      @Override
172      public boolean evaluate() throws Exception {
173        return getTableRegionMap().get(desc.getTableName()) != null;
174      }
175    });
176  }
177
178  @Test
179  public void testDefaultNamespaceCreateAndAssign() throws Exception {
180    LOG.info("testDefaultNamespaceCreateAndAssign");
181    String tableName = tablePrefix + "_testCreateAndAssign";
182    admin.modifyNamespace(NamespaceDescriptor.create("default")
183      .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, "default").build());
184    final TableDescriptor desc = TableDescriptorBuilder.newBuilder(TableName.valueOf(tableName))
185      .setColumnFamily(ColumnFamilyDescriptorBuilder.of("f")).build();
186    admin.createTable(desc);
187    // wait for created table to be assigned
188    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
189      @Override
190      public boolean evaluate() throws Exception {
191        return getTableRegionMap().get(desc.getTableName()) != null;
192      }
193    });
194  }
195
196  @Test
197  public void testCloneSnapshot() throws Exception {
198    byte[] FAMILY = Bytes.toBytes("test");
199    String snapshotName = tableName.getNameAsString() + "_snap";
200    TableName clonedTableName = TableName.valueOf(tableName.getNameAsString() + "_clone");
201
202    // create base table
203    TEST_UTIL.createTable(tableName, FAMILY);
204
205    // create snapshot
206    admin.snapshot(snapshotName, tableName);
207
208    // clone
209    admin.cloneSnapshot(snapshotName, clonedTableName);
210  }
211
212  @Test
213  public void testClearDeadServers() throws Exception {
214    LOG.info("testClearDeadServers");
215
216    // move region servers from default group to new group
217    final int serverCountToMoveToNewGroup = 3;
218    final RSGroupInfo newGroup =
219        addGroup(getGroupName(name.getMethodName()), serverCountToMoveToNewGroup);
220
221    // get the existing dead servers
222    NUM_DEAD_SERVERS = cluster.getClusterMetrics().getDeadServerNames().size();
223
224    // stop 1 region server in new group
225    ServerName serverToStop = getServerName(newGroup.getServers().iterator().next());
226    try {
227      // stopping may cause an exception
228      // due to the connection loss
229      admin.stopRegionServer(serverToStop.getAddress().toString());
230      NUM_DEAD_SERVERS++;
231    } catch (Exception e) {
232    }
233
234    // wait for the stopped region server to show up in the dead server list
235    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
236      @Override
237      public boolean evaluate() throws Exception {
238        return cluster.getClusterMetrics().getDeadServerNames().size() == NUM_DEAD_SERVERS &&
239          !master.getServerManager().areDeadServersInProgress();
240      }
241    });
242
243    // verify
244    assertFalse(cluster.getClusterMetrics().getLiveServerMetrics().containsKey(serverToStop));
245    assertTrue(cluster.getClusterMetrics().getDeadServerNames().contains(serverToStop));
246    assertTrue(newGroup.getServers().contains(serverToStop.getAddress()));
247
248    // clear dead servers list
249    List<ServerName> notClearedServers = admin.clearDeadServers(Lists.newArrayList(serverToStop));
250    assertEquals(0, notClearedServers.size());
251
252    // verify if the stopped region server gets cleared and removed from the group
253    Set<Address> newGroupServers = rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getServers();
254    assertFalse(newGroupServers.contains(serverToStop.getAddress()));
255    assertEquals(serverCountToMoveToNewGroup - 1 /* 1 stopped */, newGroupServers.size());
256  }
257
258  @Test
259  public void testClearNotProcessedDeadServer() throws Exception {
260    LOG.info("testClearNotProcessedDeadServer");
261
262    // get the existing dead servers
263    NUM_DEAD_SERVERS = cluster.getClusterMetrics().getDeadServerNames().size();
264
265    // move region servers from default group to "dead server" group
266    final int serverCountToMoveToDeadServerGroup = 1;
267    RSGroupInfo deadServerGroup =
268        addGroup("deadServerGroup", serverCountToMoveToDeadServerGroup);
269
270    // stop 1 region server in "dead server" group
271    ServerName serverToStop = getServerName(deadServerGroup.getServers().iterator().next());
272    try {
273      // stopping may cause an exception
274      // due to the connection loss
275      admin.stopRegionServer(serverToStop.getAddress().toString());
276      NUM_DEAD_SERVERS++;
277    } catch (Exception e) {
278    }
279    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
280      @Override
281      public boolean evaluate() throws Exception {
282        return cluster.getClusterMetrics().getDeadServerNames().size() == NUM_DEAD_SERVERS;
283      }
284    });
285
286    Set<Address> ServersInDeadServerGroup =
287        rsGroupAdmin.getRSGroupInfo(deadServerGroup.getName()).getServers();
288    assertEquals(serverCountToMoveToDeadServerGroup, ServersInDeadServerGroup.size());
289    assertTrue(ServersInDeadServerGroup.contains(serverToStop.getAddress()));
290  }
291
292  @Test
293  public void testRSGroupsWithHBaseQuota() throws Exception {
294    TEST_UTIL.getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
295    restartHBaseCluster();
296    try {
297      TEST_UTIL.waitFor(90000, new Waiter.Predicate<Exception>() {
298        @Override
299        public boolean evaluate() throws Exception {
300          return admin.isTableAvailable(QuotaTableUtil.QUOTA_TABLE_NAME);
301        }
302      });
303    } finally {
304      TEST_UTIL.getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, false);
305      restartHBaseCluster();
306    }
307  }
308
309  private void restartHBaseCluster() throws Exception {
310    LOG.info("\n\nShutting down cluster");
311    TEST_UTIL.shutdownMiniHBaseCluster();
312    LOG.info("\n\nSleeping a bit");
313    Thread.sleep(2000);
314    TEST_UTIL.restartHBaseCluster(NUM_SLAVES_BASE - 1);
315    initialize();
316  }
317}