001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.rsgroup; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertTrue; 022import static org.junit.Assert.fail; 023 024import java.io.IOException; 025import java.util.Iterator; 026import java.util.Optional; 027import java.util.Set; 028 029import org.apache.hadoop.hbase.HBaseClassTestRule; 030import org.apache.hadoop.hbase.HBaseTestingUtility; 031import org.apache.hadoop.hbase.HColumnDescriptor; 032import org.apache.hadoop.hbase.HConstants; 033import org.apache.hadoop.hbase.HTableDescriptor; 034import org.apache.hadoop.hbase.MiniHBaseCluster; 035import org.apache.hadoop.hbase.NamespaceDescriptor; 036import org.apache.hadoop.hbase.ServerName; 037import org.apache.hadoop.hbase.TableName; 038import org.apache.hadoop.hbase.Waiter; 039import org.apache.hadoop.hbase.Waiter.Predicate; 040import org.apache.hadoop.hbase.client.ClusterConnection; 041import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; 042import org.apache.hadoop.hbase.coprocessor.MasterCoprocessor; 043import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment; 044import org.apache.hadoop.hbase.coprocessor.MasterObserver; 045import org.apache.hadoop.hbase.coprocessor.ObserverContext; 046import org.apache.hadoop.hbase.master.MasterCoprocessorHost; 047import org.apache.hadoop.hbase.master.ServerManager; 048import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; 049import org.apache.hadoop.hbase.net.Address; 050import org.apache.hadoop.hbase.quotas.QuotaTableUtil; 051import org.apache.hadoop.hbase.quotas.QuotaUtil; 052import org.apache.hadoop.hbase.testclassification.MediumTests; 053import org.apache.hadoop.hbase.util.Bytes; 054import org.junit.After; 055import org.junit.AfterClass; 056import org.junit.Assert; 057import org.junit.Before; 058import org.junit.BeforeClass; 059import org.junit.ClassRule; 060import org.junit.Test; 061import org.junit.experimental.categories.Category; 062import org.slf4j.Logger; 063import org.slf4j.LoggerFactory; 064 065import org.apache.hbase.thirdparty.com.google.common.collect.Sets; 066 067import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; 068import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos; 069 070@Category({MediumTests.class}) 071public class TestRSGroups extends TestRSGroupsBase { 072 073 @ClassRule 074 public static final HBaseClassTestRule CLASS_RULE = 075 HBaseClassTestRule.forClass(TestRSGroups.class); 076 077 protected static final Logger LOG = LoggerFactory.getLogger(TestRSGroups.class); 078 private static boolean INIT = false; 079 private static RSGroupAdminEndpoint rsGroupAdminEndpoint; 080 private static CPMasterObserver observer; 081 082 @BeforeClass 083 public static void setUp() throws Exception { 084 TEST_UTIL = new HBaseTestingUtility(); 085 TEST_UTIL.getConfiguration().setFloat( 086 "hbase.master.balancer.stochastic.tableSkewCost", 6000); 087 TEST_UTIL.getConfiguration().set( 088 HConstants.HBASE_MASTER_LOADBALANCER_CLASS, 089 RSGroupBasedLoadBalancer.class.getName()); 090 TEST_UTIL.getConfiguration().set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, 091 RSGroupAdminEndpoint.class.getName() + "," + CPMasterObserver.class.getName()); 092 TEST_UTIL.startMiniCluster(NUM_SLAVES_BASE - 1); 093 TEST_UTIL.getConfiguration().setInt( 094 ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 095 NUM_SLAVES_BASE - 1); 096 TEST_UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); 097 098 initialize(); 099 } 100 101 private static void initialize() throws Exception { 102 admin = TEST_UTIL.getAdmin(); 103 cluster = TEST_UTIL.getHBaseCluster(); 104 master = ((MiniHBaseCluster)cluster).getMaster(); 105 106 //wait for balancer to come online 107 TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() { 108 @Override 109 public boolean evaluate() throws Exception { 110 return master.isInitialized() && 111 ((RSGroupBasedLoadBalancer) master.getLoadBalancer()).isOnline(); 112 } 113 }); 114 admin.setBalancerRunning(false,true); 115 rsGroupAdmin = new VerifyingRSGroupAdminClient( 116 new RSGroupAdminClient(TEST_UTIL.getConnection()), TEST_UTIL.getConfiguration()); 117 MasterCoprocessorHost host = master.getMasterCoprocessorHost(); 118 observer = (CPMasterObserver) host.findCoprocessor(CPMasterObserver.class.getName()); 119 rsGroupAdminEndpoint = (RSGroupAdminEndpoint) 120 host.findCoprocessor(RSGroupAdminEndpoint.class.getName()); 121 } 122 123 @AfterClass 124 public static void tearDown() throws Exception { 125 TEST_UTIL.shutdownMiniCluster(); 126 } 127 128 @Before 129 public void beforeMethod() throws Exception { 130 if (!INIT) { 131 INIT = true; 132 afterMethod(); 133 } 134 135 } 136 137 @After 138 public void afterMethod() throws Exception { 139 deleteTableIfNecessary(); 140 deleteNamespaceIfNecessary(); 141 deleteGroups(); 142 143 for(ServerName sn : admin.listDecommissionedRegionServers()){ 144 admin.recommissionRegionServer(sn, null); 145 } 146 assertTrue(admin.listDecommissionedRegionServers().isEmpty()); 147 148 int missing = NUM_SLAVES_BASE - getNumServers(); 149 LOG.info("Restoring servers: "+missing); 150 for(int i=0; i<missing; i++) { 151 ((MiniHBaseCluster)cluster).startRegionServer(); 152 } 153 154 rsGroupAdmin.addRSGroup("master"); 155 ServerName masterServerName = 156 ((MiniHBaseCluster)cluster).getMaster().getServerName(); 157 158 try { 159 rsGroupAdmin.moveServers(Sets.newHashSet(masterServerName.getAddress()), "master"); 160 } catch (Exception ex) { 161 LOG.warn("Got this on setup, FYI", ex); 162 } 163 assertTrue(observer.preMoveServersCalled); 164 TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() { 165 @Override 166 public boolean evaluate() throws Exception { 167 LOG.info("Waiting for cleanup to finish " + rsGroupAdmin.listRSGroups()); 168 //Might be greater since moving servers back to default 169 //is after starting a server 170 171 return rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size() 172 == NUM_SLAVES_BASE; 173 } 174 }); 175 } 176 177 @Test 178 public void testBasicStartUp() throws IOException { 179 RSGroupInfo defaultInfo = rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP); 180 assertEquals(4, defaultInfo.getServers().size()); 181 // Assignment of root and meta regions. 182 int count = master.getAssignmentManager().getRegionStates().getRegionAssignments().size(); 183 //3 meta,namespace, group 184 assertEquals(3, count); 185 } 186 187 @Test 188 public void testNamespaceCreateAndAssign() throws Exception { 189 LOG.info("testNamespaceCreateAndAssign"); 190 String nsName = tablePrefix+"_foo"; 191 final TableName tableName = TableName.valueOf(nsName, tablePrefix + "_testCreateAndAssign"); 192 RSGroupInfo appInfo = addGroup("appInfo", 1); 193 admin.createNamespace(NamespaceDescriptor.create(nsName) 194 .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, "appInfo").build()); 195 final HTableDescriptor desc = new HTableDescriptor(tableName); 196 desc.addFamily(new HColumnDescriptor("f")); 197 admin.createTable(desc); 198 //wait for created table to be assigned 199 TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() { 200 @Override 201 public boolean evaluate() throws Exception { 202 return getTableRegionMap().get(desc.getTableName()) != null; 203 } 204 }); 205 ServerName targetServer = 206 ServerName.parseServerName(appInfo.getServers().iterator().next().toString()); 207 AdminProtos.AdminService.BlockingInterface rs = 208 ((ClusterConnection) admin.getConnection()).getAdmin(targetServer); 209 //verify it was assigned to the right group 210 Assert.assertEquals(1, ProtobufUtil.getOnlineRegions(rs).size()); 211 } 212 213 @Test 214 public void testDefaultNamespaceCreateAndAssign() throws Exception { 215 LOG.info("testDefaultNamespaceCreateAndAssign"); 216 String tableName = tablePrefix + "_testCreateAndAssign"; 217 admin.modifyNamespace(NamespaceDescriptor.create("default") 218 .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, "default").build()); 219 final HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName)); 220 desc.addFamily(new HColumnDescriptor("f")); 221 admin.createTable(desc); 222 //wait for created table to be assigned 223 TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() { 224 @Override 225 public boolean evaluate() throws Exception { 226 return getTableRegionMap().get(desc.getTableName()) != null; 227 } 228 }); 229 } 230 231 @Test 232 public void testNamespaceConstraint() throws Exception { 233 String nsName = tablePrefix+"_foo"; 234 String groupName = tablePrefix+"_foo"; 235 LOG.info("testNamespaceConstraint"); 236 rsGroupAdmin.addRSGroup(groupName); 237 assertTrue(observer.preAddRSGroupCalled); 238 assertTrue(observer.postAddRSGroupCalled); 239 240 admin.createNamespace(NamespaceDescriptor.create(nsName) 241 .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, groupName) 242 .build()); 243 //test removing a referenced group 244 try { 245 rsGroupAdmin.removeRSGroup(groupName); 246 fail("Expected a constraint exception"); 247 } catch (IOException ex) { 248 } 249 //test modify group 250 //changing with the same name is fine 251 admin.modifyNamespace( 252 NamespaceDescriptor.create(nsName) 253 .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, groupName) 254 .build()); 255 String anotherGroup = tablePrefix+"_anotherGroup"; 256 rsGroupAdmin.addRSGroup(anotherGroup); 257 //test add non-existent group 258 admin.deleteNamespace(nsName); 259 rsGroupAdmin.removeRSGroup(groupName); 260 assertTrue(observer.preRemoveRSGroupCalled); 261 assertTrue(observer.postRemoveRSGroupCalled); 262 try { 263 admin.createNamespace(NamespaceDescriptor.create(nsName) 264 .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, "foo") 265 .build()); 266 fail("Expected a constraint exception"); 267 } catch (IOException ex) { 268 } 269 } 270 271 @Test 272 public void testGroupInfoMultiAccessing() throws Exception { 273 RSGroupInfoManager manager = rsGroupAdminEndpoint.getGroupInfoManager(); 274 RSGroupInfo defaultGroup = manager.getRSGroup("default"); 275 // getRSGroup updates default group's server list 276 // this process must not affect other threads iterating the list 277 Iterator<Address> it = defaultGroup.getServers().iterator(); 278 manager.getRSGroup("default"); 279 it.next(); 280 } 281 282 public static class CPMasterObserver implements MasterCoprocessor, MasterObserver { 283 boolean preBalanceRSGroupCalled = false; 284 boolean postBalanceRSGroupCalled = false; 285 boolean preMoveServersCalled = false; 286 boolean postMoveServersCalled = false; 287 boolean preMoveTablesCalled = false; 288 boolean postMoveTablesCalled = false; 289 boolean preAddRSGroupCalled = false; 290 boolean postAddRSGroupCalled = false; 291 boolean preRemoveRSGroupCalled = false; 292 boolean postRemoveRSGroupCalled = false; 293 boolean preRemoveServersCalled = false; 294 boolean postRemoveServersCalled = false; 295 boolean preMoveServersAndTables = false; 296 boolean postMoveServersAndTables = false; 297 298 @Override 299 public Optional<MasterObserver> getMasterObserver() { 300 return Optional.of(this); 301 } 302 @Override 303 public void preMoveServersAndTables(final ObserverContext<MasterCoprocessorEnvironment> ctx, 304 Set<Address> servers, Set<TableName> tables, String targetGroup) throws IOException { 305 preMoveServersAndTables = true; 306 } 307 @Override 308 public void postMoveServersAndTables(final ObserverContext<MasterCoprocessorEnvironment> ctx, 309 Set<Address> servers, Set<TableName> tables, String targetGroup) throws IOException { 310 postMoveServersAndTables = true; 311 } 312 @Override 313 public void preRemoveServers( 314 final ObserverContext<MasterCoprocessorEnvironment> ctx, 315 Set<Address> servers) throws IOException { 316 preRemoveServersCalled = true; 317 } 318 @Override 319 public void postRemoveServers( 320 final ObserverContext<MasterCoprocessorEnvironment> ctx, 321 Set<Address> servers) throws IOException { 322 postRemoveServersCalled = true; 323 } 324 @Override 325 public void preRemoveRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx, 326 String name) throws IOException { 327 preRemoveRSGroupCalled = true; 328 } 329 @Override 330 public void postRemoveRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx, 331 String name) throws IOException { 332 postRemoveRSGroupCalled = true; 333 } 334 @Override 335 public void preAddRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx, 336 String name) throws IOException { 337 preAddRSGroupCalled = true; 338 } 339 @Override 340 public void postAddRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx, 341 String name) throws IOException { 342 postAddRSGroupCalled = true; 343 } 344 @Override 345 public void preMoveTables(final ObserverContext<MasterCoprocessorEnvironment> ctx, 346 Set<TableName> tables, String targetGroup) throws IOException { 347 preMoveTablesCalled = true; 348 } 349 @Override 350 public void postMoveTables(final ObserverContext<MasterCoprocessorEnvironment> ctx, 351 Set<TableName> tables, String targetGroup) throws IOException { 352 postMoveTablesCalled = true; 353 } 354 @Override 355 public void preMoveServers(final ObserverContext<MasterCoprocessorEnvironment> ctx, 356 Set<Address> servers, String targetGroup) throws IOException { 357 preMoveServersCalled = true; 358 } 359 360 @Override 361 public void postMoveServers(final ObserverContext<MasterCoprocessorEnvironment> ctx, 362 Set<Address> servers, String targetGroup) throws IOException { 363 postMoveServersCalled = true; 364 } 365 @Override 366 public void preBalanceRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx, 367 String groupName) throws IOException { 368 preBalanceRSGroupCalled = true; 369 } 370 @Override 371 public void postBalanceRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx, 372 String groupName, boolean balancerRan) throws IOException { 373 postBalanceRSGroupCalled = true; 374 } 375 } 376 @Test 377 public void testMoveServersAndTables() throws Exception { 378 super.testMoveServersAndTables(); 379 assertTrue(observer.preMoveServersAndTables); 380 assertTrue(observer.postMoveServersAndTables); 381 } 382 @Test 383 public void testTableMoveTruncateAndDrop() throws Exception { 384 super.testTableMoveTruncateAndDrop(); 385 assertTrue(observer.preMoveTablesCalled); 386 assertTrue(observer.postMoveTablesCalled); 387 } 388 389 @Test 390 public void testRemoveServers() throws Exception { 391 super.testRemoveServers(); 392 assertTrue(observer.preRemoveServersCalled); 393 assertTrue(observer.postRemoveServersCalled); 394 } 395 396 @Test 397 public void testMisplacedRegions() throws Exception { 398 final TableName tableName = TableName.valueOf(tablePrefix+"_testMisplacedRegions"); 399 LOG.info("testMisplacedRegions"); 400 401 final RSGroupInfo RSGroupInfo = addGroup("testMisplacedRegions", 1); 402 403 TEST_UTIL.createMultiRegionTable(tableName, new byte[]{'f'}, 15); 404 TEST_UTIL.waitUntilAllRegionsAssigned(tableName); 405 406 rsGroupAdminEndpoint.getGroupInfoManager() 407 .moveTables(Sets.newHashSet(tableName), RSGroupInfo.getName()); 408 409 admin.setBalancerRunning(true,true); 410 assertTrue(rsGroupAdmin.balanceRSGroup(RSGroupInfo.getName())); 411 admin.setBalancerRunning(false,true); 412 assertTrue(observer.preBalanceRSGroupCalled); 413 assertTrue(observer.postBalanceRSGroupCalled); 414 415 TEST_UTIL.waitFor(60000, new Predicate<Exception>() { 416 @Override 417 public boolean evaluate() throws Exception { 418 ServerName serverName = 419 ServerName.valueOf(RSGroupInfo.getServers().iterator().next().toString(), 1); 420 return admin.getConnection().getAdmin() 421 .getOnlineRegions(serverName).size() == 15; 422 } 423 }); 424 } 425 426 @Test 427 public void testCloneSnapshot() throws Exception { 428 byte[] FAMILY = Bytes.toBytes("test"); 429 String snapshotName = tableName.getNameAsString() + "_snap"; 430 TableName clonedTableName = TableName.valueOf(tableName.getNameAsString() + "_clone"); 431 432 // create base table 433 TEST_UTIL.createTable(tableName, FAMILY); 434 435 // create snapshot 436 admin.snapshot(snapshotName, tableName); 437 438 // clone 439 admin.cloneSnapshot(snapshotName, clonedTableName); 440 } 441 442 @Test 443 public void testRSGroupsWithHBaseQuota() throws Exception { 444 TEST_UTIL.getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, true); 445 restartHBaseCluster(); 446 try { 447 TEST_UTIL.waitFor(90000, new Waiter.Predicate<Exception>() { 448 @Override 449 public boolean evaluate() throws Exception { 450 return admin.isTableAvailable(QuotaTableUtil.QUOTA_TABLE_NAME); 451 } 452 }); 453 } finally { 454 TEST_UTIL.getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, false); 455 restartHBaseCluster(); 456 } 457 } 458 459 private void restartHBaseCluster() throws Exception { 460 LOG.info("\n\nShutting down cluster"); 461 TEST_UTIL.shutdownMiniHBaseCluster(); 462 LOG.info("\n\nSleeping a bit"); 463 Thread.sleep(2000); 464 TEST_UTIL.restartHBaseCluster(NUM_SLAVES_BASE - 1); 465 initialize(); 466 } 467}