001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.regionserver; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertFalse; 022import static org.junit.Assert.assertNotNull; 023import static org.junit.Assert.assertNotSame; 024import static org.junit.Assert.assertNull; 025import static org.junit.Assert.assertTrue; 026import static org.junit.Assert.fail; 027 028import java.io.IOException; 029import java.util.Collection; 030import java.util.List; 031import java.util.Map; 032import java.util.Optional; 033import java.util.concurrent.CountDownLatch; 034import java.util.concurrent.ExecutionException; 035import java.util.concurrent.TimeUnit; 036import java.util.concurrent.TimeoutException; 037import java.util.concurrent.atomic.AtomicBoolean; 038import org.apache.hadoop.conf.Configuration; 039import org.apache.hadoop.fs.FileSystem; 040import org.apache.hadoop.fs.Path; 041import org.apache.hadoop.hbase.Coprocessor; 042import org.apache.hadoop.hbase.CoprocessorEnvironment; 043import org.apache.hadoop.hbase.DoNotRetryIOException; 044import org.apache.hadoop.hbase.HBaseClassTestRule; 045import org.apache.hadoop.hbase.HBaseTestingUtility; 046import org.apache.hadoop.hbase.HConstants; 047import org.apache.hadoop.hbase.HTableDescriptor; 048import org.apache.hadoop.hbase.MasterNotRunningException; 049import org.apache.hadoop.hbase.MetaTableAccessor; 050import org.apache.hadoop.hbase.MiniHBaseCluster; 051import org.apache.hadoop.hbase.ServerName; 052import org.apache.hadoop.hbase.StartMiniClusterOption; 053import org.apache.hadoop.hbase.TableName; 054import org.apache.hadoop.hbase.UnknownRegionException; 055import org.apache.hadoop.hbase.ZooKeeperConnectionException; 056import org.apache.hadoop.hbase.client.Admin; 057import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; 058import org.apache.hadoop.hbase.client.Consistency; 059import org.apache.hadoop.hbase.client.Delete; 060import org.apache.hadoop.hbase.client.DoNotRetryRegionException; 061import org.apache.hadoop.hbase.client.Get; 062import org.apache.hadoop.hbase.client.Mutation; 063import org.apache.hadoop.hbase.client.Put; 064import org.apache.hadoop.hbase.client.RegionInfo; 065import org.apache.hadoop.hbase.client.Result; 066import org.apache.hadoop.hbase.client.ResultScanner; 067import org.apache.hadoop.hbase.client.Scan; 068import org.apache.hadoop.hbase.client.Table; 069import org.apache.hadoop.hbase.client.TableDescriptor; 070import org.apache.hadoop.hbase.client.TableDescriptorBuilder; 071import org.apache.hadoop.hbase.client.TestReplicasClient.SlowMeCopro; 072import org.apache.hadoop.hbase.coprocessor.MasterCoprocessor; 073import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment; 074import org.apache.hadoop.hbase.coprocessor.MasterObserver; 075import org.apache.hadoop.hbase.coprocessor.ObserverContext; 076import org.apache.hadoop.hbase.master.HMaster; 077import org.apache.hadoop.hbase.master.LoadBalancer; 078import org.apache.hadoop.hbase.master.MasterRpcServices; 079import org.apache.hadoop.hbase.master.RegionState; 080import org.apache.hadoop.hbase.master.RegionState.State; 081import org.apache.hadoop.hbase.master.assignment.AssignmentManager; 082import org.apache.hadoop.hbase.master.assignment.AssignmentTestingUtil; 083import org.apache.hadoop.hbase.master.assignment.RegionStateNode; 084import org.apache.hadoop.hbase.master.assignment.RegionStates; 085import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; 086import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext; 087import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController; 088import org.apache.hadoop.hbase.testclassification.LargeTests; 089import org.apache.hadoop.hbase.testclassification.RegionServerTests; 090import org.apache.hadoop.hbase.util.Bytes; 091import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; 092import org.apache.hadoop.hbase.util.FSUtils; 093import org.apache.hadoop.hbase.util.HBaseFsck; 094import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread; 095import org.apache.hadoop.hbase.util.Threads; 096import org.apache.zookeeper.KeeperException; 097import org.apache.zookeeper.KeeperException.NodeExistsException; 098import org.junit.After; 099import org.junit.AfterClass; 100import org.junit.Assert; 101import org.junit.Before; 102import org.junit.BeforeClass; 103import org.junit.ClassRule; 104import org.junit.Rule; 105import org.junit.Test; 106import org.junit.experimental.categories.Category; 107import org.junit.rules.TestName; 108import org.slf4j.Logger; 109import org.slf4j.LoggerFactory; 110 111import org.apache.hbase.thirdparty.com.google.protobuf.RpcController; 112import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException; 113 114import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; 115import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode; 116import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionRequest; 117import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionResponse; 118 119/** 120 * The below tests are testing split region against a running cluster 121 */ 122@Category({RegionServerTests.class, LargeTests.class}) 123public class TestSplitTransactionOnCluster { 124 125 @ClassRule 126 public static final HBaseClassTestRule CLASS_RULE = 127 HBaseClassTestRule.forClass(TestSplitTransactionOnCluster.class); 128 129 private static final Logger LOG = LoggerFactory.getLogger(TestSplitTransactionOnCluster.class); 130 private Admin admin = null; 131 private MiniHBaseCluster cluster = null; 132 private static final int NB_SERVERS = 3; 133 134 static final HBaseTestingUtility TESTING_UTIL = 135 new HBaseTestingUtility(); 136 137 @Rule 138 public TestName name = new TestName(); 139 140 @BeforeClass public static void before() throws Exception { 141 TESTING_UTIL.getConfiguration().setInt(HConstants.HBASE_BALANCER_PERIOD, 60000); 142 StartMiniClusterOption option = StartMiniClusterOption.builder() 143 .masterClass(MyMaster.class).numRegionServers(NB_SERVERS).numDataNodes(NB_SERVERS).build(); 144 TESTING_UTIL.startMiniCluster(option); 145 } 146 147 @AfterClass public static void after() throws Exception { 148 TESTING_UTIL.shutdownMiniCluster(); 149 } 150 151 @Before public void setup() throws IOException { 152 TESTING_UTIL.ensureSomeNonStoppedRegionServersAvailable(NB_SERVERS); 153 this.admin = TESTING_UTIL.getAdmin(); 154 this.cluster = TESTING_UTIL.getMiniHBaseCluster(); 155 } 156 157 @After 158 public void tearDown() throws Exception { 159 this.admin.close(); 160 for (TableDescriptor htd: this.admin.listTableDescriptors()) { 161 LOG.info("Tear down, remove table=" + htd.getTableName()); 162 TESTING_UTIL.deleteTable(htd.getTableName()); 163 } 164 } 165 166 private RegionInfo getAndCheckSingleTableRegion(final List<HRegion> regions) 167 throws IOException, InterruptedException { 168 assertEquals(1, regions.size()); 169 RegionInfo hri = regions.get(0).getRegionInfo(); 170 AssignmentTestingUtil.waitForAssignment(cluster.getMaster().getAssignmentManager(), hri); 171 return hri; 172 } 173 174 private void requestSplitRegion( 175 final HRegionServer rsServer, 176 final Region region, 177 final byte[] midKey) throws IOException { 178 long procId = cluster.getMaster().splitRegion(region.getRegionInfo(), midKey, 0, 0); 179 // wait for the split to complete or get interrupted. If the split completes successfully, 180 // the procedure will return true; if the split fails, the procedure would throw exception. 181 ProcedureTestingUtility.waitProcedure(cluster.getMaster().getMasterProcedureExecutor(), procId); 182 } 183 184 @Test 185 public void testRITStateForRollback() throws Exception { 186 final TableName tableName = TableName.valueOf(name.getMethodName()); 187 final HMaster master = cluster.getMaster(); 188 try { 189 // Create table then get the single region for our new table. 190 Table t = createTableAndWait(tableName, Bytes.toBytes("cf")); 191 final List<HRegion> regions = cluster.getRegions(tableName); 192 final RegionInfo hri = getAndCheckSingleTableRegion(regions); 193 insertData(tableName, admin, t); 194 t.close(); 195 196 // Turn off balancer so it doesn't cut in and mess up our placements. 197 this.admin.balancerSwitch(false, true); 198 // Turn off the meta scanner so it don't remove parent on us. 199 master.setCatalogJanitorEnabled(false); 200 201 // find a splittable region 202 final HRegion region = findSplittableRegion(regions); 203 assertTrue("not able to find a splittable region", region != null); 204 205 // install master co-processor to fail splits 206 master.getMasterCoprocessorHost().load( 207 FailingSplitMasterObserver.class, 208 Coprocessor.PRIORITY_USER, 209 master.getConfiguration()); 210 211 // split async 212 this.admin.splitRegionAsync(region.getRegionInfo().getRegionName(), new byte[] { 42 }); 213 214 // we have to wait until the SPLITTING state is seen by the master 215 FailingSplitMasterObserver observer = 216 master.getMasterCoprocessorHost().findCoprocessor(FailingSplitMasterObserver.class); 217 assertNotNull(observer); 218 observer.latch.await(); 219 220 LOG.info("Waiting for region to come out of RIT"); 221 while (!cluster.getMaster().getAssignmentManager().getRegionStates().isRegionOnline(hri)) { 222 Threads.sleep(100); 223 } 224 assertTrue(cluster.getMaster().getAssignmentManager().getRegionStates().isRegionOnline(hri)); 225 } finally { 226 admin.balancerSwitch(true, false); 227 master.setCatalogJanitorEnabled(true); 228 abortAndWaitForMaster(); 229 TESTING_UTIL.deleteTable(tableName); 230 } 231 } 232 233 @Test 234 public void testSplitFailedCompactionAndSplit() throws Exception { 235 final TableName tableName = TableName.valueOf(name.getMethodName()); 236 // Create table then get the single region for our new table. 237 byte[] cf = Bytes.toBytes("cf"); 238 TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName) 239 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(cf)).build(); 240 admin.createTable(htd); 241 242 for (int i = 0; cluster.getRegions(tableName).isEmpty() && i < 100; i++) { 243 Thread.sleep(100); 244 } 245 assertEquals(1, cluster.getRegions(tableName).size()); 246 247 HRegion region = cluster.getRegions(tableName).get(0); 248 HStore store = region.getStore(cf); 249 int regionServerIndex = cluster.getServerWith(region.getRegionInfo().getRegionName()); 250 HRegionServer regionServer = cluster.getRegionServer(regionServerIndex); 251 252 Table t = TESTING_UTIL.getConnection().getTable(tableName); 253 // insert data 254 insertData(tableName, admin, t); 255 insertData(tableName, admin, t); 256 257 int fileNum = store.getStorefiles().size(); 258 // 0, Compaction Request 259 store.triggerMajorCompaction(); 260 Optional<CompactionContext> cc = store.requestCompaction(); 261 assertTrue(cc.isPresent()); 262 // 1, A timeout split 263 // 1.1 close region 264 assertEquals(2, region.close(false).get(cf).size()); 265 // 1.2 rollback and Region initialize again 266 region.initialize(); 267 268 // 2, Run Compaction cc 269 assertFalse(region.compact(cc.get(), store, NoLimitThroughputController.INSTANCE)); 270 assertTrue(fileNum > store.getStorefiles().size()); 271 272 // 3, Split 273 requestSplitRegion(regionServer, region, Bytes.toBytes("row3")); 274 assertEquals(2, cluster.getRegions(tableName).size()); 275 } 276 277 public static class FailingSplitMasterObserver implements MasterCoprocessor, MasterObserver { 278 volatile CountDownLatch latch; 279 280 @Override 281 public void start(CoprocessorEnvironment e) throws IOException { 282 latch = new CountDownLatch(1); 283 } 284 285 @Override 286 public Optional<MasterObserver> getMasterObserver() { 287 return Optional.of(this); 288 } 289 290 @Override 291 public void preSplitRegionBeforeMETAAction( 292 final ObserverContext<MasterCoprocessorEnvironment> ctx, 293 final byte[] splitKey, 294 final List<Mutation> metaEntries) throws IOException { 295 latch.countDown(); 296 throw new IOException("Causing rollback of region split"); 297 } 298 } 299 300 @Test 301 public void testSplitRollbackOnRegionClosing() throws Exception { 302 final TableName tableName = TableName.valueOf(name.getMethodName()); 303 304 // Create table then get the single region for our new table. 305 Table t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY); 306 List<HRegion> regions = cluster.getRegions(tableName); 307 RegionInfo hri = getAndCheckSingleTableRegion(regions); 308 309 int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri); 310 311 RegionStates regionStates = cluster.getMaster().getAssignmentManager().getRegionStates(); 312 313 // Turn off balancer so it doesn't cut in and mess up our placements. 314 this.admin.balancerSwitch(false, true); 315 // Turn off the meta scanner so it don't remove parent on us. 316 cluster.getMaster().setCatalogJanitorEnabled(false); 317 try { 318 // Add a bit of load up into the table so splittable. 319 TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY, false); 320 // Get region pre-split. 321 HRegionServer server = cluster.getRegionServer(tableRegionIndex); 322 printOutRegions(server, "Initial regions: "); 323 int regionCount = cluster.getRegions(hri.getTable()).size(); 324 regionStates.updateRegionState(hri, RegionState.State.CLOSING); 325 326 // Now try splitting.... should fail. And each should successfully 327 // rollback. 328 // We don't roll back here anymore. Instead we fail-fast on construction of the 329 // split transaction. Catch the exception instead. 330 try { 331 this.admin.splitRegionAsync(hri.getRegionName()); 332 fail(); 333 } catch (DoNotRetryRegionException e) { 334 // Expected 335 } 336 // Wait around a while and assert count of regions remains constant. 337 for (int i = 0; i < 10; i++) { 338 Thread.sleep(100); 339 assertEquals(regionCount, cluster.getRegions(hri.getTable()).size()); 340 } 341 regionStates.updateRegionState(hri, State.OPEN); 342 // Now try splitting and it should work. 343 admin.splitRegionAsync(hri.getRegionName()).get(2, TimeUnit.MINUTES); 344 // Get daughters 345 checkAndGetDaughters(tableName); 346 // OK, so split happened after we cleared the blocking node. 347 } finally { 348 admin.balancerSwitch(true, false); 349 cluster.getMaster().setCatalogJanitorEnabled(true); 350 t.close(); 351 } 352 } 353 354 /** 355 * Test that if daughter split on us, we won't do the shutdown handler fixup just because we can't 356 * find the immediate daughter of an offlined parent. 357 */ 358 @Test 359 public void testShutdownFixupWhenDaughterHasSplit() throws Exception { 360 final TableName tableName = TableName.valueOf(name.getMethodName()); 361 362 // Create table then get the single region for our new table. 363 Table t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY); 364 List<HRegion> regions = cluster.getRegions(tableName); 365 RegionInfo hri = getAndCheckSingleTableRegion(regions); 366 367 int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri); 368 369 // Turn off balancer so it doesn't cut in and mess up our placements. 370 this.admin.balancerSwitch(false, true); 371 // Turn off the meta scanner so it don't remove parent on us. 372 cluster.getMaster().setCatalogJanitorEnabled(false); 373 try { 374 // Add a bit of load up into the table so splittable. 375 TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY); 376 // Get region pre-split. 377 HRegionServer server = cluster.getRegionServer(tableRegionIndex); 378 printOutRegions(server, "Initial regions: "); 379 // Now split. 380 admin.splitRegionAsync(hri.getRegionName()).get(2, TimeUnit.MINUTES); 381 // Get daughters 382 List<HRegion> daughters = checkAndGetDaughters(tableName); 383 HRegion daughterRegion = daughters.get(0); 384 // Now split one of the daughters. 385 RegionInfo daughter = daughterRegion.getRegionInfo(); 386 LOG.info("Daughter we are going to split: " + daughter); 387 // Compact first to ensure we have cleaned up references -- else the split 388 // will fail. 389 daughterRegion.compact(true); 390 daughterRegion.getStores().get(0).closeAndArchiveCompactedFiles(); 391 for (int i = 0; i < 100; i++) { 392 if (!daughterRegion.hasReferences()) { 393 break; 394 } 395 Threads.sleep(100); 396 } 397 assertFalse("Waiting for reference to be compacted", daughterRegion.hasReferences()); 398 LOG.info("Daughter hri before split (has been compacted): " + daughter); 399 admin.splitRegionAsync(daughter.getRegionName()).get(2, TimeUnit.MINUTES); 400 // Get list of daughters 401 daughters = cluster.getRegions(tableName); 402 for (HRegion d: daughters) { 403 LOG.info("Regions before crash: " + d); 404 } 405 // Now crash the server 406 cluster.abortRegionServer(tableRegionIndex); 407 waitUntilRegionServerDead(); 408 awaitDaughters(tableName, daughters.size()); 409 // Assert daughters are online and ONLY the original daughters -- that 410 // fixup didn't insert one during server shutdown recover. 411 regions = cluster.getRegions(tableName); 412 for (HRegion d: daughters) { 413 LOG.info("Regions after crash: " + d); 414 } 415 if (daughters.size() != regions.size()) { 416 LOG.info("Daughters=" + daughters.size() + ", regions=" + regions.size()); 417 } 418 assertEquals(daughters.size(), regions.size()); 419 for (HRegion r: regions) { 420 LOG.info("Regions post crash " + r + ", contains=" + daughters.contains(r)); 421 assertTrue("Missing region post crash " + r, daughters.contains(r)); 422 } 423 } finally { 424 LOG.info("EXITING"); 425 admin.balancerSwitch(true, false); 426 cluster.getMaster().setCatalogJanitorEnabled(true); 427 t.close(); 428 } 429 } 430 431 @Test 432 public void testSplitShouldNotThrowNPEEvenARegionHasEmptySplitFiles() throws Exception { 433 TableName userTableName = TableName.valueOf(name.getMethodName()); 434 TableDescriptor htd = TableDescriptorBuilder.newBuilder(userTableName) 435 .setColumnFamily(ColumnFamilyDescriptorBuilder.of("col")).build(); 436 admin.createTable(htd); 437 Table table = TESTING_UTIL.getConnection().getTable(userTableName); 438 try { 439 for (int i = 0; i <= 5; i++) { 440 String row = "row" + i; 441 Put p = new Put(row.getBytes()); 442 String val = "Val" + i; 443 p.addColumn("col".getBytes(), "ql".getBytes(), val.getBytes()); 444 table.put(p); 445 admin.flush(userTableName); 446 Delete d = new Delete(row.getBytes()); 447 // Do a normal delete 448 table.delete(d); 449 admin.flush(userTableName); 450 } 451 admin.majorCompact(userTableName); 452 List<RegionInfo> regionsOfTable = 453 cluster.getMaster().getAssignmentManager().getRegionStates() 454 .getRegionsOfTable(userTableName); 455 assertEquals(1, regionsOfTable.size()); 456 RegionInfo hRegionInfo = regionsOfTable.get(0); 457 Put p = new Put("row6".getBytes()); 458 p.addColumn("col".getBytes(), "ql".getBytes(), "val".getBytes()); 459 table.put(p); 460 p = new Put("row7".getBytes()); 461 p.addColumn("col".getBytes(), "ql".getBytes(), "val".getBytes()); 462 table.put(p); 463 p = new Put("row8".getBytes()); 464 p.addColumn("col".getBytes(), "ql".getBytes(), "val".getBytes()); 465 table.put(p); 466 admin.flush(userTableName); 467 admin.splitRegionAsync(hRegionInfo.getRegionName(), "row7".getBytes()); 468 regionsOfTable = cluster.getMaster() 469 .getAssignmentManager().getRegionStates() 470 .getRegionsOfTable(userTableName); 471 472 while (regionsOfTable.size() != 2) { 473 Thread.sleep(1000); 474 regionsOfTable = cluster.getMaster() 475 .getAssignmentManager().getRegionStates() 476 .getRegionsOfTable(userTableName); 477 LOG.debug("waiting 2 regions to be available, got " + regionsOfTable.size() + 478 ": " + regionsOfTable); 479 480 } 481 Assert.assertEquals(2, regionsOfTable.size()); 482 483 Scan s = new Scan(); 484 ResultScanner scanner = table.getScanner(s); 485 int mainTableCount = 0; 486 for (Result rr = scanner.next(); rr != null; rr = scanner.next()) { 487 mainTableCount++; 488 } 489 Assert.assertEquals(3, mainTableCount); 490 } finally { 491 table.close(); 492 } 493 } 494 495 /** 496 * Verifies HBASE-5806. Here the case is that splitting is completed but before the CJ could 497 * remove the parent region the master is killed and restarted. 498 */ 499 @Test 500 public void testMasterRestartAtRegionSplitPendingCatalogJanitor() 501 throws IOException, InterruptedException, NodeExistsException, KeeperException, 502 ServiceException, ExecutionException, TimeoutException { 503 final TableName tableName = TableName.valueOf(name.getMethodName()); 504 // Create table then get the single region for our new table. 505 try (Table t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY)) { 506 List<HRegion> regions = cluster.getRegions(tableName); 507 RegionInfo hri = getAndCheckSingleTableRegion(regions); 508 509 int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri); 510 511 // Turn off balancer so it doesn't cut in and mess up our placements. 512 this.admin.balancerSwitch(false, true); 513 // Turn off the meta scanner so it don't remove parent on us. 514 cluster.getMaster().setCatalogJanitorEnabled(false); 515 // Add a bit of load up into the table so splittable. 516 TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY, false); 517 // Get region pre-split. 518 HRegionServer server = cluster.getRegionServer(tableRegionIndex); 519 printOutRegions(server, "Initial regions: "); 520 // Call split. 521 this.admin.splitRegionAsync(hri.getRegionName()).get(2, TimeUnit.MINUTES); 522 List<HRegion> daughters = checkAndGetDaughters(tableName); 523 524 // Before cleanup, get a new master. 525 HMaster master = abortAndWaitForMaster(); 526 // Now call compact on the daughters and clean up any references. 527 for (HRegion daughter : daughters) { 528 daughter.compact(true); 529 daughter.getStores().get(0).closeAndArchiveCompactedFiles(); 530 assertFalse(daughter.hasReferences()); 531 } 532 // BUT calling compact on the daughters is not enough. The CatalogJanitor looks 533 // in the filesystem, and the filesystem content is not same as what the Region 534 // is reading from. Compacted-away files are picked up later by the compacted 535 // file discharger process. It runs infrequently. Make it run so CatalogJanitor 536 // doens't find any references. 537 for (RegionServerThread rst : cluster.getRegionServerThreads()) { 538 boolean oldSetting = rst.getRegionServer().compactedFileDischarger.setUseExecutor(false); 539 rst.getRegionServer().compactedFileDischarger.run(); 540 rst.getRegionServer().compactedFileDischarger.setUseExecutor(oldSetting); 541 } 542 cluster.getMaster().setCatalogJanitorEnabled(true); 543 ProcedureTestingUtility.waitAllProcedures(cluster.getMaster().getMasterProcedureExecutor()); 544 LOG.info("Starting run of CatalogJanitor"); 545 cluster.getMaster().getCatalogJanitor().run(); 546 ProcedureTestingUtility.waitAllProcedures(cluster.getMaster().getMasterProcedureExecutor()); 547 RegionStates regionStates = master.getAssignmentManager().getRegionStates(); 548 ServerName regionServerOfRegion = regionStates.getRegionServerOfRegion(hri); 549 assertEquals(null, regionServerOfRegion); 550 } finally { 551 TESTING_UTIL.getAdmin().balancerSwitch(true, false); 552 cluster.getMaster().setCatalogJanitorEnabled(true); 553 } 554 } 555 556 @Test 557 public void testSplitWithRegionReplicas() throws Exception { 558 final TableName tableName = TableName.valueOf(name.getMethodName()); 559 HTableDescriptor htd = TESTING_UTIL.createTableDescriptor(name.getMethodName()); 560 htd.setRegionReplication(2); 561 htd.addCoprocessor(SlowMeCopro.class.getName()); 562 // Create table then get the single region for our new table. 563 Table t = TESTING_UTIL.createTable(htd, new byte[][]{Bytes.toBytes("cf")}, null); 564 List<HRegion> oldRegions; 565 do { 566 oldRegions = cluster.getRegions(tableName); 567 Thread.sleep(10); 568 } while (oldRegions.size() != 2); 569 for (HRegion h : oldRegions) LOG.debug("OLDREGION " + h.getRegionInfo()); 570 try { 571 int regionServerIndex = cluster.getServerWith(oldRegions.get(0).getRegionInfo() 572 .getRegionName()); 573 HRegionServer regionServer = cluster.getRegionServer(regionServerIndex); 574 insertData(tableName, admin, t); 575 // Turn off balancer so it doesn't cut in and mess up our placements. 576 admin.balancerSwitch(false, true); 577 // Turn off the meta scanner so it don't remove parent on us. 578 cluster.getMaster().setCatalogJanitorEnabled(false); 579 boolean tableExists = MetaTableAccessor.tableExists(regionServer.getConnection(), 580 tableName); 581 assertEquals("The specified table should be present.", true, tableExists); 582 final HRegion region = findSplittableRegion(oldRegions); 583 regionServerIndex = cluster.getServerWith(region.getRegionInfo().getRegionName()); 584 regionServer = cluster.getRegionServer(regionServerIndex); 585 assertTrue("not able to find a splittable region", region != null); 586 try { 587 requestSplitRegion(regionServer, region, Bytes.toBytes("row2")); 588 } catch (IOException e) { 589 e.printStackTrace(); 590 fail("Split execution should have succeeded with no exceptions thrown " + e); 591 } 592 //TESTING_UTIL.waitUntilAllRegionsAssigned(tableName); 593 List<HRegion> newRegions; 594 do { 595 newRegions = cluster.getRegions(tableName); 596 for (HRegion h : newRegions) LOG.debug("NEWREGION " + h.getRegionInfo()); 597 Thread.sleep(1000); 598 } while ((newRegions.contains(oldRegions.get(0)) || newRegions.contains(oldRegions.get(1))) 599 || newRegions.size() != 4); 600 tableExists = MetaTableAccessor.tableExists(regionServer.getConnection(), 601 tableName); 602 assertEquals("The specified table should be present.", true, tableExists); 603 // exists works on stale and we see the put after the flush 604 byte[] b1 = "row1".getBytes(); 605 Get g = new Get(b1); 606 g.setConsistency(Consistency.STRONG); 607 // The following GET will make a trip to the meta to get the new location of the 1st daughter 608 // In the process it will also get the location of the replica of the daughter (initially 609 // pointing to the parent's replica) 610 Result r = t.get(g); 611 Assert.assertFalse(r.isStale()); 612 LOG.info("exists stale after flush done"); 613 614 SlowMeCopro.getPrimaryCdl().set(new CountDownLatch(1)); 615 g = new Get(b1); 616 g.setConsistency(Consistency.TIMELINE); 617 // This will succeed because in the previous GET we get the location of the replica 618 r = t.get(g); 619 Assert.assertTrue(r.isStale()); 620 SlowMeCopro.getPrimaryCdl().get().countDown(); 621 } finally { 622 SlowMeCopro.getPrimaryCdl().get().countDown(); 623 admin.balancerSwitch(true, false); 624 cluster.getMaster().setCatalogJanitorEnabled(true); 625 t.close(); 626 } 627 } 628 629 private void insertData(final TableName tableName, Admin admin, Table t) throws IOException, 630 InterruptedException { 631 Put p = new Put(Bytes.toBytes("row1")); 632 p.addColumn(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("1")); 633 t.put(p); 634 p = new Put(Bytes.toBytes("row2")); 635 p.addColumn(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("2")); 636 t.put(p); 637 p = new Put(Bytes.toBytes("row3")); 638 p.addColumn(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("3")); 639 t.put(p); 640 p = new Put(Bytes.toBytes("row4")); 641 p.addColumn(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("4")); 642 t.put(p); 643 admin.flush(tableName); 644 } 645 646 /** 647 * If a table has regions that have no store files in a region, they should split successfully 648 * into two regions with no store files. 649 */ 650 @Test 651 public void testSplitRegionWithNoStoreFiles() throws Exception { 652 final TableName tableName = TableName.valueOf(name.getMethodName()); 653 // Create table then get the single region for our new table. 654 createTableAndWait(tableName, HConstants.CATALOG_FAMILY); 655 List<HRegion> regions = cluster.getRegions(tableName); 656 RegionInfo hri = getAndCheckSingleTableRegion(regions); 657 ensureTableRegionNotOnSameServerAsMeta(admin, hri); 658 int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionInfo() 659 .getRegionName()); 660 HRegionServer regionServer = cluster.getRegionServer(regionServerIndex); 661 // Turn off balancer so it doesn't cut in and mess up our placements. 662 this.admin.balancerSwitch(false, true); 663 // Turn off the meta scanner so it don't remove parent on us. 664 cluster.getMaster().setCatalogJanitorEnabled(false); 665 try { 666 // Precondition: we created a table with no data, no store files. 667 printOutRegions(regionServer, "Initial regions: "); 668 Configuration conf = cluster.getConfiguration(); 669 HBaseFsck.debugLsr(conf, new Path("/")); 670 Path rootDir = FSUtils.getRootDir(conf); 671 FileSystem fs = TESTING_UTIL.getDFSCluster().getFileSystem(); 672 Map<String, Path> storefiles = 673 FSUtils.getTableStoreFilePathMap(null, fs, rootDir, tableName); 674 assertEquals("Expected nothing but found " + storefiles.toString(), 0, storefiles.size()); 675 676 // find a splittable region. Refresh the regions list 677 regions = cluster.getRegions(tableName); 678 final HRegion region = findSplittableRegion(regions); 679 assertTrue("not able to find a splittable region", region != null); 680 681 // Now split. 682 try { 683 requestSplitRegion(regionServer, region, Bytes.toBytes("row2")); 684 } catch (IOException e) { 685 fail("Split execution should have succeeded with no exceptions thrown"); 686 } 687 688 // Postcondition: split the table with no store files into two regions, but still have no 689 // store files 690 List<HRegion> daughters = cluster.getRegions(tableName); 691 assertEquals(2, daughters.size()); 692 693 // check dirs 694 HBaseFsck.debugLsr(conf, new Path("/")); 695 Map<String, Path> storefilesAfter = 696 FSUtils.getTableStoreFilePathMap(null, fs, rootDir, tableName); 697 assertEquals("Expected nothing but found " + storefilesAfter.toString(), 0, 698 storefilesAfter.size()); 699 700 hri = region.getRegionInfo(); // split parent 701 AssignmentManager am = cluster.getMaster().getAssignmentManager(); 702 RegionStates regionStates = am.getRegionStates(); 703 long start = EnvironmentEdgeManager.currentTime(); 704 while (!regionStates.isRegionInState(hri, State.SPLIT)) { 705 LOG.debug("Waiting for SPLIT state on: " + hri); 706 assertFalse("Timed out in waiting split parent to be in state SPLIT", 707 EnvironmentEdgeManager.currentTime() - start > 60000); 708 Thread.sleep(500); 709 } 710 assertTrue(regionStates.isRegionInState(daughters.get(0).getRegionInfo(), State.OPEN)); 711 assertTrue(regionStates.isRegionInState(daughters.get(1).getRegionInfo(), State.OPEN)); 712 713 // We should not be able to assign it again 714 try { 715 am.assign(hri); 716 } catch (DoNotRetryIOException e) { 717 // Expected 718 } 719 assertFalse("Split region can't be assigned", regionStates.isRegionInTransition(hri)); 720 assertTrue(regionStates.isRegionInState(hri, State.SPLIT)); 721 722 // We should not be able to unassign it either 723 try { 724 am.unassign(hri); 725 fail("Should have thrown exception"); 726 } catch (DoNotRetryIOException e) { 727 // Expected 728 } 729 assertFalse("Split region can't be unassigned", regionStates.isRegionInTransition(hri)); 730 assertTrue(regionStates.isRegionInState(hri, State.SPLIT)); 731 } finally { 732 admin.balancerSwitch(true, false); 733 cluster.getMaster().setCatalogJanitorEnabled(true); 734 } 735 } 736 737 @Test 738 public void testStoreFileReferenceCreationWhenSplitPolicySaysToSkipRangeCheck() 739 throws Exception { 740 final TableName tableName = TableName.valueOf(name.getMethodName()); 741 try { 742 byte[] cf = Bytes.toBytes("f"); 743 byte[] cf1 = Bytes.toBytes("i_f"); 744 TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName) 745 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(cf)) 746 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(cf1)) 747 .setRegionSplitPolicyClassName(CustomSplitPolicy.class.getName()).build(); 748 admin.createTable(htd); 749 List<HRegion> regions = awaitTableRegions(tableName); 750 HRegion region = regions.get(0); 751 for(int i = 3;i<9;i++) { 752 Put p = new Put(Bytes.toBytes("row"+i)); 753 p.addColumn(cf, Bytes.toBytes("q"), Bytes.toBytes("value" + i)); 754 p.addColumn(cf1, Bytes.toBytes("q"), Bytes.toBytes("value" + i)); 755 region.put(p); 756 } 757 region.flush(true); 758 HStore store = region.getStore(cf); 759 Collection<HStoreFile> storefiles = store.getStorefiles(); 760 assertEquals(1, storefiles.size()); 761 assertFalse(region.hasReferences()); 762 Path referencePath = 763 region.getRegionFileSystem().splitStoreFile(region.getRegionInfo(), "f", 764 storefiles.iterator().next(), Bytes.toBytes("row1"), false, region.getSplitPolicy()); 765 assertNull(referencePath); 766 referencePath = 767 region.getRegionFileSystem().splitStoreFile(region.getRegionInfo(), "i_f", 768 storefiles.iterator().next(), Bytes.toBytes("row1"), false, region.getSplitPolicy()); 769 assertNotNull(referencePath); 770 } finally { 771 TESTING_UTIL.deleteTable(tableName); 772 } 773 } 774 775 private HRegion findSplittableRegion(final List<HRegion> regions) throws InterruptedException { 776 for (int i = 0; i < 5; ++i) { 777 for (HRegion r: regions) { 778 if (r.isSplittable() && r.getRegionInfo().getReplicaId() == 0) { 779 return(r); 780 } 781 } 782 Thread.sleep(100); 783 } 784 return null; 785 } 786 787 private List<HRegion> checkAndGetDaughters(TableName tableName) throws InterruptedException { 788 List<HRegion> daughters = null; 789 // try up to 10s 790 for (int i = 0; i < 100; i++) { 791 daughters = cluster.getRegions(tableName); 792 if (daughters.size() >= 2) { 793 break; 794 } 795 Thread.sleep(100); 796 } 797 assertTrue(daughters.size() >= 2); 798 return daughters; 799 } 800 801 private HMaster abortAndWaitForMaster() throws IOException, InterruptedException { 802 cluster.abortMaster(0); 803 cluster.waitOnMaster(0); 804 HMaster master = cluster.startMaster().getMaster(); 805 cluster.waitForActiveAndReadyMaster(); 806 return master; 807 } 808 809 /** 810 * Ensure single table region is not on same server as the single hbase:meta table 811 * region. 812 * @param admin 813 * @param hri 814 * @return Index of the server hosting the single table region 815 * @throws UnknownRegionException 816 * @throws MasterNotRunningException 817 * @throws org.apache.hadoop.hbase.ZooKeeperConnectionException 818 * @throws InterruptedException 819 */ 820 private int ensureTableRegionNotOnSameServerAsMeta(final Admin admin, 821 final RegionInfo hri) 822 throws IOException, MasterNotRunningException, 823 ZooKeeperConnectionException, InterruptedException { 824 // Now make sure that the table region is not on same server as that hosting 825 // hbase:meta We don't want hbase:meta replay polluting our test when we later crash 826 // the table region serving server. 827 int metaServerIndex = cluster.getServerWithMeta(); 828 boolean tablesOnMaster = LoadBalancer.isTablesOnMaster(TESTING_UTIL.getConfiguration()); 829 if (tablesOnMaster) { 830 // Need to check master is supposed to host meta... perhaps it is not. 831 throw new UnsupportedOperationException(); 832 // TODO: assertTrue(metaServerIndex == -1); // meta is on master now 833 } 834 HRegionServer metaRegionServer = tablesOnMaster? 835 cluster.getMaster(): cluster.getRegionServer(metaServerIndex); 836 int tableRegionIndex = cluster.getServerWith(hri.getRegionName()); 837 assertTrue(tableRegionIndex != -1); 838 HRegionServer tableRegionServer = cluster.getRegionServer(tableRegionIndex); 839 LOG.info("MetaRegionServer=" + metaRegionServer.getServerName() + 840 ", other=" + tableRegionServer.getServerName()); 841 if (metaRegionServer.getServerName().equals(tableRegionServer.getServerName())) { 842 HRegionServer hrs = getOtherRegionServer(cluster, metaRegionServer); 843 assertNotNull(hrs); 844 assertNotNull(hri); 845 LOG.info("Moving " + hri.getRegionNameAsString() + " from " + 846 metaRegionServer.getServerName() + " to " + 847 hrs.getServerName() + "; metaServerIndex=" + metaServerIndex); 848 admin.move(hri.getEncodedNameAsBytes(), hrs.getServerName()); 849 } 850 // Wait till table region is up on the server that is NOT carrying hbase:meta. 851 for (int i = 0; i < 100; i++) { 852 tableRegionIndex = cluster.getServerWith(hri.getRegionName()); 853 if (tableRegionIndex != -1 && tableRegionIndex != metaServerIndex) break; 854 LOG.debug("Waiting on region move off the hbase:meta server; current index " + 855 tableRegionIndex + " and metaServerIndex=" + metaServerIndex); 856 Thread.sleep(100); 857 } 858 assertTrue("Region not moved off hbase:meta server, tableRegionIndex=" + tableRegionIndex, 859 tableRegionIndex != -1 && tableRegionIndex != metaServerIndex); 860 // Verify for sure table region is not on same server as hbase:meta 861 tableRegionIndex = cluster.getServerWith(hri.getRegionName()); 862 assertTrue(tableRegionIndex != -1); 863 assertNotSame(metaServerIndex, tableRegionIndex); 864 return tableRegionIndex; 865 } 866 867 /** 868 * Find regionserver other than the one passed. 869 * Can't rely on indexes into list of regionservers since crashed servers 870 * occupy an index. 871 * @param cluster 872 * @param notThisOne 873 * @return A regionserver that is not <code>notThisOne</code> or null if none 874 * found 875 */ 876 private HRegionServer getOtherRegionServer(final MiniHBaseCluster cluster, 877 final HRegionServer notThisOne) { 878 for (RegionServerThread rst: cluster.getRegionServerThreads()) { 879 HRegionServer hrs = rst.getRegionServer(); 880 if (hrs.getServerName().equals(notThisOne.getServerName())) continue; 881 if (hrs.isStopping() || hrs.isStopped()) continue; 882 return hrs; 883 } 884 return null; 885 } 886 887 private void printOutRegions(final HRegionServer hrs, final String prefix) 888 throws IOException { 889 List<RegionInfo> regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices()); 890 for (RegionInfo region: regions) { 891 LOG.info(prefix + region.getRegionNameAsString()); 892 } 893 } 894 895 private void waitUntilRegionServerDead() throws InterruptedException, IOException { 896 // Wait until the master processes the RS shutdown 897 for (int i=0; (cluster.getMaster().getClusterMetrics() 898 .getLiveServerMetrics().size() > NB_SERVERS 899 || cluster.getLiveRegionServerThreads().size() > NB_SERVERS) && i<100; i++) { 900 LOG.info("Waiting on server to go down"); 901 Thread.sleep(100); 902 } 903 assertFalse("Waited too long for RS to die", 904 cluster.getMaster().getClusterMetrics(). getLiveServerMetrics().size() > NB_SERVERS 905 || cluster.getLiveRegionServerThreads().size() > NB_SERVERS); 906 } 907 908 private void awaitDaughters(TableName tableName, int numDaughters) throws InterruptedException { 909 // Wait till regions are back on line again. 910 for (int i = 0; cluster.getRegions(tableName).size() < numDaughters && i < 60; i++) { 911 LOG.info("Waiting for repair to happen"); 912 Thread.sleep(1000); 913 } 914 if (cluster.getRegions(tableName).size() < numDaughters) { 915 fail("Waiting too long for daughter regions"); 916 } 917 } 918 919 private List<HRegion> awaitTableRegions(final TableName tableName) throws InterruptedException { 920 List<HRegion> regions = null; 921 for (int i = 0; i < 100; i++) { 922 regions = cluster.getRegions(tableName); 923 if (regions.size() > 0) break; 924 Thread.sleep(100); 925 } 926 return regions; 927 } 928 929 private Table createTableAndWait(TableName tableName, byte[] cf) throws IOException, 930 InterruptedException { 931 Table t = TESTING_UTIL.createTable(tableName, cf); 932 awaitTableRegions(tableName); 933 assertTrue("Table not online: " + tableName, 934 cluster.getRegions(tableName).size() != 0); 935 return t; 936 } 937 938 // Make it public so that JVMClusterUtil can access it. 939 public static class MyMaster extends HMaster { 940 public MyMaster(Configuration conf) throws IOException, KeeperException, InterruptedException { 941 super(conf); 942 } 943 944 @Override 945 protected RSRpcServices createRpcServices() throws IOException { 946 return new MyMasterRpcServices(this); 947 } 948 } 949 950 static class MyMasterRpcServices extends MasterRpcServices { 951 static AtomicBoolean enabled = new AtomicBoolean(false); 952 953 private HMaster myMaster; 954 public MyMasterRpcServices(HMaster master) throws IOException { 955 super(master); 956 myMaster = master; 957 } 958 959 @Override 960 public ReportRegionStateTransitionResponse reportRegionStateTransition(RpcController c, 961 ReportRegionStateTransitionRequest req) throws ServiceException { 962 ReportRegionStateTransitionResponse resp = super.reportRegionStateTransition(c, req); 963 if (enabled.get() && req.getTransition(0).getTransitionCode().equals( 964 TransitionCode.READY_TO_SPLIT) && !resp.hasErrorMessage()) { 965 RegionStates regionStates = myMaster.getAssignmentManager().getRegionStates(); 966 for (RegionStateNode regionState: 967 regionStates.getRegionsInTransition()) { 968 /* TODO!!!! 969 // Find the merging_new region and remove it 970 if (regionState.isSplittingNew()) { 971 regionStates.deleteRegion(regionState.getRegion()); 972 } 973 */ 974 } 975 } 976 return resp; 977 } 978 } 979 980 static class CustomSplitPolicy extends IncreasingToUpperBoundRegionSplitPolicy { 981 982 @Override 983 protected boolean shouldSplit() { 984 return true; 985 } 986 987 @Override 988 public boolean skipStoreFileRangeCheck(String familyName) { 989 if(familyName.startsWith("i_")) { 990 return true; 991 } else { 992 return false; 993 } 994 } 995 } 996} 997