001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.master.assignment; 019 020import static org.apache.hadoop.hbase.master.assignment.AssignmentTestingUtil.insertData; 021import static org.junit.Assert.assertEquals; 022import static org.junit.Assert.assertFalse; 023import static org.junit.Assert.assertTrue; 024import static org.junit.Assert.fail; 025 026import java.io.IOException; 027import java.util.List; 028import java.util.Optional; 029import org.apache.hadoop.conf.Configuration; 030import org.apache.hadoop.hbase.Cell; 031import org.apache.hadoop.hbase.CellUtil; 032import org.apache.hadoop.hbase.DoNotRetryIOException; 033import org.apache.hadoop.hbase.HBaseClassTestRule; 034import org.apache.hadoop.hbase.HBaseTestingUtil; 035import org.apache.hadoop.hbase.HConstants; 036import org.apache.hadoop.hbase.TableName; 037import org.apache.hadoop.hbase.Waiter; 038import org.apache.hadoop.hbase.client.CompactionState; 039import org.apache.hadoop.hbase.client.Delete; 040import org.apache.hadoop.hbase.client.Get; 041import org.apache.hadoop.hbase.client.RegionInfo; 042import org.apache.hadoop.hbase.client.RegionReplicaUtil; 043import org.apache.hadoop.hbase.client.Result; 044import org.apache.hadoop.hbase.client.SnapshotDescription; 045import org.apache.hadoop.hbase.client.SnapshotType; 046import org.apache.hadoop.hbase.client.Table; 047import org.apache.hadoop.hbase.client.TableDescriptor; 048import org.apache.hadoop.hbase.coprocessor.ObserverContext; 049import org.apache.hadoop.hbase.coprocessor.RegionCoprocessor; 050import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; 051import org.apache.hadoop.hbase.coprocessor.RegionObserver; 052import org.apache.hadoop.hbase.master.RegionState; 053import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants; 054import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; 055import org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility; 056import org.apache.hadoop.hbase.master.procedure.TestSnapshotProcedure; 057import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; 058import org.apache.hadoop.hbase.procedure2.ProcedureMetrics; 059import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; 060import org.apache.hadoop.hbase.regionserver.HRegion; 061import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; 062import org.apache.hadoop.hbase.testclassification.MasterTests; 063import org.apache.hadoop.hbase.testclassification.MediumTests; 064import org.apache.hadoop.hbase.util.Bytes; 065import org.junit.After; 066import org.junit.AfterClass; 067import org.junit.Before; 068import org.junit.BeforeClass; 069import org.junit.ClassRule; 070import org.junit.Rule; 071import org.junit.Test; 072import org.junit.experimental.categories.Category; 073import org.junit.rules.TestName; 074import org.slf4j.Logger; 075import org.slf4j.LoggerFactory; 076 077import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; 078import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos; 079 080@Category({ MasterTests.class, MediumTests.class }) 081public class TestSplitTableRegionProcedure { 082 083 @ClassRule 084 public static final HBaseClassTestRule CLASS_RULE = 085 HBaseClassTestRule.forClass(TestSplitTableRegionProcedure.class); 086 087 private static final Logger LOG = LoggerFactory.getLogger(TestSplitTableRegionProcedure.class); 088 089 protected static final HBaseTestingUtil UTIL = new HBaseTestingUtil(); 090 091 private static String columnFamilyName1 = "cf1"; 092 private static String columnFamilyName2 = "cf2"; 093 094 private static final int startRowNum = 11; 095 private static final int rowCount = 60; 096 097 private AssignmentManager am; 098 099 private ProcedureMetrics splitProcMetrics; 100 private ProcedureMetrics assignProcMetrics; 101 private ProcedureMetrics unassignProcMetrics; 102 103 private long splitSubmittedCount = 0; 104 private long splitFailedCount = 0; 105 private long assignSubmittedCount = 0; 106 private long assignFailedCount = 0; 107 private long unassignSubmittedCount = 0; 108 private long unassignFailedCount = 0; 109 110 @Rule 111 public TestName name = new TestName(); 112 113 private static void setupConf(Configuration conf) { 114 conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1); 115 conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 0); 116 conf.set("hbase.coprocessor.region.classes", 117 RegionServerHostingReplicaSlowOpenCopro.class.getName()); 118 conf.setInt("hbase.client.sync.wait.timeout.msec", 1500); 119 } 120 121 /** 122 * This copro is used to slow down opening of the replica regions. 123 */ 124 public static class RegionServerHostingReplicaSlowOpenCopro 125 implements RegionCoprocessor, RegionObserver { 126 static int countForReplica = 0; 127 static boolean slowDownReplicaOpen = false; 128 129 @Override 130 public Optional<RegionObserver> getRegionObserver() { 131 return Optional.of(this); 132 } 133 134 @Override 135 public void preOpen(ObserverContext<RegionCoprocessorEnvironment> c) throws IOException { 136 int replicaId = c.getEnvironment().getRegion().getRegionInfo().getReplicaId(); 137 if ((replicaId != RegionInfo.DEFAULT_REPLICA_ID) && (countForReplica == 0)) { 138 countForReplica++; 139 while (slowDownReplicaOpen) { 140 LOG.info("Slow down replica region open a bit"); 141 try { 142 Thread.sleep(100); 143 } catch (InterruptedException ie) { 144 // Ingore 145 } 146 } 147 } 148 } 149 } 150 151 @BeforeClass 152 public static void setupCluster() throws Exception { 153 setupConf(UTIL.getConfiguration()); 154 UTIL.startMiniCluster(3); 155 } 156 157 @AfterClass 158 public static void cleanupTest() throws Exception { 159 try { 160 UTIL.shutdownMiniCluster(); 161 } catch (Exception e) { 162 LOG.warn("failure shutting down cluster", e); 163 } 164 } 165 166 @Before 167 public void setup() throws Exception { 168 ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(getMasterProcedureExecutor(), false); 169 170 // Turn off balancer so it doesn't cut in and mess up our placements. 171 UTIL.getAdmin().balancerSwitch(false, true); 172 // Turn off the meta scanner so it don't remove parent on us. 173 UTIL.getHBaseCluster().getMaster().setCatalogJanitorEnabled(false); 174 am = UTIL.getHBaseCluster().getMaster().getAssignmentManager(); 175 splitProcMetrics = am.getAssignmentManagerMetrics().getSplitProcMetrics(); 176 assignProcMetrics = am.getAssignmentManagerMetrics().getAssignProcMetrics(); 177 unassignProcMetrics = am.getAssignmentManagerMetrics().getUnassignProcMetrics(); 178 } 179 180 @After 181 public void tearDown() throws Exception { 182 ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(getMasterProcedureExecutor(), false); 183 for (TableDescriptor htd : UTIL.getAdmin().listTableDescriptors()) { 184 UTIL.deleteTable(htd.getTableName()); 185 } 186 } 187 188 @Test 189 public void testRollbackForSplitTableRegionWithReplica() throws Exception { 190 final TableName tableName = TableName.valueOf(name.getMethodName()); 191 final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); 192 193 RegionServerHostingReplicaSlowOpenCopro.slowDownReplicaOpen = true; 194 RegionInfo[] regions = 195 MasterProcedureTestingUtility.createTable(procExec, tableName, null, columnFamilyName1); 196 197 try { 198 HBaseTestingUtil.setReplicas(UTIL.getAdmin(), tableName, 2); 199 } catch (IOException ioe) { 200 201 } 202 203 // wait until the primary region is online. 204 HBaseTestingUtil.await(2000, () -> { 205 try { 206 AssignmentManager am = UTIL.getHBaseCluster().getMaster().getAssignmentManager(); 207 if (am == null) return false; 208 if (am.getRegionStates().getRegionState(regions[0]).isOpened()) { 209 return true; 210 } 211 return false; 212 } catch (Exception e) { 213 throw new RuntimeException(e); 214 } 215 }); 216 217 // Split region of the table, it will fail and rollback as replica parent region 218 // is still at OPENING state. 219 long procId = procExec.submitProcedure(new SplitTableRegionProcedure(procExec.getEnvironment(), 220 regions[0], HConstants.CATALOG_FAMILY)); 221 // Wait for the completion. 222 ProcedureTestingUtility.waitProcedure(procExec, procId); 223 224 // Let replica parent region open. 225 RegionServerHostingReplicaSlowOpenCopro.slowDownReplicaOpen = false; 226 227 // wait until the replica region is online. 228 HBaseTestingUtil.await(2000, () -> { 229 try { 230 AssignmentManager am = UTIL.getHBaseCluster().getMaster().getAssignmentManager(); 231 if (am == null) return false; 232 RegionInfo replicaRegion = RegionReplicaUtil.getRegionInfoForReplica(regions[0], 1); 233 if (am.getRegionStates().getRegionState(replicaRegion).isOpened()) { 234 return true; 235 } 236 return false; 237 } catch (Exception e) { 238 throw new RuntimeException(e); 239 } 240 }); 241 242 ProcedureTestingUtility.assertProcFailed(procExec, procId); 243 // There should not be any active OpenRegionProcedure 244 procExec.getActiveProceduresNoCopy() 245 .forEach(p -> assertFalse(p instanceof OpenRegionProcedure)); 246 247 // Check that procedure rollback reverted parent region state to OPEN 248 AssignmentManager am = UTIL.getHBaseCluster().getMaster().getAssignmentManager(); 249 RegionStateNode regionStateNode = am.getRegionStates().getRegionStateNode(regions[0]); 250 assertEquals(RegionState.State.OPEN, regionStateNode.getState()); 251 } 252 253 @Test 254 public void testSplitTableRegion() throws Exception { 255 final TableName tableName = TableName.valueOf(name.getMethodName()); 256 final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); 257 258 RegionInfo[] regions = MasterProcedureTestingUtility.createTable(procExec, tableName, null, 259 columnFamilyName1, columnFamilyName2); 260 insertData(UTIL, tableName, rowCount, startRowNum, columnFamilyName1, columnFamilyName2); 261 int splitRowNum = startRowNum + rowCount / 2; 262 byte[] splitKey = Bytes.toBytes("" + splitRowNum); 263 264 assertTrue("not able to find a splittable region", regions != null); 265 assertTrue("not able to find a splittable region", regions.length == 1); 266 267 // collect AM metrics before test 268 collectAssignmentManagerMetrics(); 269 270 // Split region of the table 271 long procId = procExec.submitProcedure( 272 new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey)); 273 // Wait the completion 274 ProcedureTestingUtility.waitProcedure(procExec, procId); 275 ProcedureTestingUtility.assertProcNotFailed(procExec, procId); 276 277 verify(tableName, splitRowNum); 278 279 assertEquals(splitSubmittedCount + 1, splitProcMetrics.getSubmittedCounter().getCount()); 280 assertEquals(splitFailedCount, splitProcMetrics.getFailedCounter().getCount()); 281 assertEquals(assignSubmittedCount + 2, assignProcMetrics.getSubmittedCounter().getCount()); 282 assertEquals(assignFailedCount, assignProcMetrics.getFailedCounter().getCount()); 283 assertEquals(unassignSubmittedCount + 1, unassignProcMetrics.getSubmittedCounter().getCount()); 284 assertEquals(unassignFailedCount, unassignProcMetrics.getFailedCounter().getCount()); 285 } 286 287 @Test 288 public void testSplitTableRegionNoStoreFile() throws Exception { 289 final TableName tableName = TableName.valueOf(name.getMethodName()); 290 final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); 291 292 RegionInfo[] regions = MasterProcedureTestingUtility.createTable(procExec, tableName, null, 293 columnFamilyName1, columnFamilyName2); 294 int splitRowNum = startRowNum + rowCount / 2; 295 byte[] splitKey = Bytes.toBytes("" + splitRowNum); 296 297 assertTrue("not able to find a splittable region", regions != null); 298 assertTrue("not able to find a splittable region", regions.length == 1); 299 300 // collect AM metrics before test 301 collectAssignmentManagerMetrics(); 302 303 // Split region of the table 304 long procId = procExec.submitProcedure( 305 new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey)); 306 // Wait the completion 307 ProcedureTestingUtility.waitProcedure(procExec, procId); 308 ProcedureTestingUtility.assertProcNotFailed(procExec, procId); 309 310 assertTrue(UTIL.getMiniHBaseCluster().getRegions(tableName).size() == 2); 311 assertTrue(UTIL.countRows(tableName) == 0); 312 313 assertEquals(splitSubmittedCount + 1, splitProcMetrics.getSubmittedCounter().getCount()); 314 assertEquals(splitFailedCount, splitProcMetrics.getFailedCounter().getCount()); 315 } 316 317 @Test 318 public void testSplitTableRegionUnevenDaughter() throws Exception { 319 final TableName tableName = TableName.valueOf(name.getMethodName()); 320 final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); 321 322 RegionInfo[] regions = MasterProcedureTestingUtility.createTable(procExec, tableName, null, 323 columnFamilyName1, columnFamilyName2); 324 insertData(UTIL, tableName, rowCount, startRowNum, columnFamilyName1, columnFamilyName2); 325 // Split to two daughters with one of them only has 1 row 326 int splitRowNum = startRowNum + rowCount / 4; 327 byte[] splitKey = Bytes.toBytes("" + splitRowNum); 328 329 assertTrue("not able to find a splittable region", regions != null); 330 assertTrue("not able to find a splittable region", regions.length == 1); 331 332 // collect AM metrics before test 333 collectAssignmentManagerMetrics(); 334 335 // Split region of the table 336 long procId = procExec.submitProcedure( 337 new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey)); 338 // Wait the completion 339 ProcedureTestingUtility.waitProcedure(procExec, procId); 340 ProcedureTestingUtility.assertProcNotFailed(procExec, procId); 341 342 verify(tableName, splitRowNum); 343 344 assertEquals(splitSubmittedCount + 1, splitProcMetrics.getSubmittedCounter().getCount()); 345 assertEquals(splitFailedCount, splitProcMetrics.getFailedCounter().getCount()); 346 } 347 348 @Test 349 public void testSplitTableRegionEmptyDaughter() throws Exception { 350 final TableName tableName = TableName.valueOf(name.getMethodName()); 351 final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); 352 353 RegionInfo[] regions = MasterProcedureTestingUtility.createTable(procExec, tableName, null, 354 columnFamilyName1, columnFamilyName2); 355 insertData(UTIL, tableName, rowCount, startRowNum, columnFamilyName1, columnFamilyName2); 356 // Split to two daughters with one of them only has 1 row 357 int splitRowNum = startRowNum + rowCount; 358 byte[] splitKey = Bytes.toBytes("" + splitRowNum); 359 360 assertTrue("not able to find a splittable region", regions != null); 361 assertTrue("not able to find a splittable region", regions.length == 1); 362 363 // collect AM metrics before test 364 collectAssignmentManagerMetrics(); 365 366 // Split region of the table 367 long procId = procExec.submitProcedure( 368 new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey)); 369 // Wait the completion 370 ProcedureTestingUtility.waitProcedure(procExec, procId); 371 ProcedureTestingUtility.assertProcNotFailed(procExec, procId); 372 373 // Make sure one daughter has 0 rows. 374 List<HRegion> daughters = UTIL.getMiniHBaseCluster().getRegions(tableName); 375 assertTrue(daughters.size() == 2); 376 assertTrue(UTIL.countRows(tableName) == rowCount); 377 assertTrue(UTIL.countRows(daughters.get(0)) == 0 || UTIL.countRows(daughters.get(1)) == 0); 378 379 assertEquals(splitSubmittedCount + 1, splitProcMetrics.getSubmittedCounter().getCount()); 380 assertEquals(splitFailedCount, splitProcMetrics.getFailedCounter().getCount()); 381 } 382 383 @Test 384 public void testSplitTableRegionDeletedRowsDaughter() throws Exception { 385 final TableName tableName = TableName.valueOf(name.getMethodName()); 386 final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); 387 388 RegionInfo[] regions = MasterProcedureTestingUtility.createTable(procExec, tableName, null, 389 columnFamilyName1, columnFamilyName2); 390 insertData(UTIL, tableName, rowCount, startRowNum, columnFamilyName1, columnFamilyName2); 391 // Split to two daughters with one of them only has 1 row 392 int splitRowNum = rowCount; 393 deleteData(tableName, splitRowNum); 394 byte[] splitKey = Bytes.toBytes("" + splitRowNum); 395 396 assertTrue("not able to find a splittable region", regions != null); 397 assertTrue("not able to find a splittable region", regions.length == 1); 398 399 // collect AM metrics before test 400 collectAssignmentManagerMetrics(); 401 402 // Split region of the table 403 long procId = procExec.submitProcedure( 404 new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey)); 405 // Wait the completion 406 ProcedureTestingUtility.waitProcedure(procExec, procId); 407 ProcedureTestingUtility.assertProcNotFailed(procExec, procId); 408 409 UTIL.getAdmin().majorCompact(tableName); 410 // waiting for the major compaction to complete 411 UTIL.waitFor(6000, new Waiter.Predicate<IOException>() { 412 @Override 413 public boolean evaluate() throws IOException { 414 return UTIL.getAdmin().getCompactionState(tableName) == CompactionState.NONE; 415 } 416 }); 417 418 // Make sure one daughter has 0 rows. 419 List<HRegion> daughters = UTIL.getMiniHBaseCluster().getRegions(tableName); 420 assertTrue(daughters.size() == 2); 421 final int currentRowCount = splitRowNum - startRowNum; 422 assertTrue(UTIL.countRows(tableName) == currentRowCount); 423 assertTrue(UTIL.countRows(daughters.get(0)) == 0 || UTIL.countRows(daughters.get(1)) == 0); 424 425 assertEquals(splitSubmittedCount + 1, splitProcMetrics.getSubmittedCounter().getCount()); 426 assertEquals(splitFailedCount, splitProcMetrics.getFailedCounter().getCount()); 427 } 428 429 @Test 430 public void testInvalidSplitKey() throws Exception { 431 final TableName tableName = TableName.valueOf(name.getMethodName()); 432 final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); 433 434 RegionInfo[] regions = MasterProcedureTestingUtility.createTable(procExec, tableName, null, 435 columnFamilyName1, columnFamilyName2); 436 insertData(UTIL, tableName, rowCount, startRowNum, columnFamilyName1, columnFamilyName2); 437 438 assertTrue("not able to find a splittable region", regions != null); 439 assertTrue("not able to find a splittable region", regions.length == 1); 440 441 // collect AM metrics before test 442 collectAssignmentManagerMetrics(); 443 444 // Split region of the table with null split key 445 try { 446 long procId1 = procExec.submitProcedure( 447 new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], null)); 448 ProcedureTestingUtility.waitProcedure(procExec, procId1); 449 fail("unexpected procedure start with invalid split-key"); 450 } catch (DoNotRetryIOException e) { 451 LOG.debug("Expected Split procedure construction failure: " + e.getMessage()); 452 } 453 454 assertEquals(splitSubmittedCount, splitProcMetrics.getSubmittedCounter().getCount()); 455 assertEquals(splitFailedCount, splitProcMetrics.getFailedCounter().getCount()); 456 } 457 458 @Test 459 public void testRollbackAndDoubleExecution() throws Exception { 460 final TableName tableName = TableName.valueOf(name.getMethodName()); 461 final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); 462 463 RegionInfo[] regions = MasterProcedureTestingUtility.createTable(procExec, tableName, null, 464 columnFamilyName1, columnFamilyName2); 465 insertData(UTIL, tableName, rowCount, startRowNum, columnFamilyName1, columnFamilyName2); 466 int splitRowNum = startRowNum + rowCount / 2; 467 byte[] splitKey = Bytes.toBytes("" + splitRowNum); 468 469 assertTrue("not able to find a splittable region", regions != null); 470 assertTrue("not able to find a splittable region", regions.length == 1); 471 ProcedureTestingUtility.waitNoProcedureRunning(procExec); 472 ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); 473 474 // collect AM metrics before test 475 collectAssignmentManagerMetrics(); 476 477 // Split region of the table 478 long procId = procExec.submitProcedure( 479 new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey)); 480 481 // Failing before SPLIT_TABLE_REGION_UPDATE_META we should trigger the 482 // rollback 483 // NOTE: the 7 (number of SPLIT_TABLE_REGION_UPDATE_META step) is 484 // hardcoded, so you have to look at this test at least once when you add a new step. 485 int lastStep = 7; 486 MasterProcedureTestingUtility.testRollbackAndDoubleExecution(procExec, procId, lastStep, true); 487 // check that we have only 1 region 488 assertEquals(1, UTIL.getAdmin().getRegions(tableName).size()); 489 UTIL.waitUntilAllRegionsAssigned(tableName); 490 List<HRegion> newRegions = UTIL.getMiniHBaseCluster().getRegions(tableName); 491 assertEquals(1, newRegions.size()); 492 verifyData(newRegions.get(0), startRowNum, rowCount, Bytes.toBytes(columnFamilyName1), 493 Bytes.toBytes(columnFamilyName2)); 494 495 assertEquals(splitSubmittedCount + 1, splitProcMetrics.getSubmittedCounter().getCount()); 496 assertEquals(splitFailedCount + 1, splitProcMetrics.getFailedCounter().getCount()); 497 } 498 499 @Test 500 public void testRecoveryAndDoubleExecution() throws Exception { 501 final TableName tableName = TableName.valueOf(name.getMethodName()); 502 final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); 503 504 RegionInfo[] regions = MasterProcedureTestingUtility.createTable(procExec, tableName, null, 505 columnFamilyName1, columnFamilyName2); 506 insertData(UTIL, tableName, rowCount, startRowNum, columnFamilyName1, columnFamilyName2); 507 int splitRowNum = startRowNum + rowCount / 2; 508 byte[] splitKey = Bytes.toBytes("" + splitRowNum); 509 510 assertTrue("not able to find a splittable region", regions != null); 511 assertTrue("not able to find a splittable region", regions.length == 1); 512 ProcedureTestingUtility.waitNoProcedureRunning(procExec); 513 ProcedureTestingUtility.setKillIfHasParent(procExec, false); 514 ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); 515 516 // collect AM metrics before test 517 collectAssignmentManagerMetrics(); 518 519 // Split region of the table 520 long procId = procExec.submitProcedure( 521 new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey)); 522 523 // Restart the executor and execute the step twice 524 MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId); 525 ProcedureTestingUtility.assertProcNotFailed(procExec, procId); 526 527 verify(tableName, splitRowNum); 528 529 assertEquals(splitSubmittedCount + 1, splitProcMetrics.getSubmittedCounter().getCount()); 530 assertEquals(splitFailedCount, splitProcMetrics.getFailedCounter().getCount()); 531 } 532 533 @Test 534 public void testSplitWithoutPONR() throws Exception { 535 final TableName tableName = TableName.valueOf(name.getMethodName()); 536 final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); 537 538 RegionInfo[] regions = MasterProcedureTestingUtility.createTable(procExec, tableName, null, 539 columnFamilyName1, columnFamilyName2); 540 insertData(UTIL, tableName, rowCount, startRowNum, columnFamilyName1, columnFamilyName2); 541 int splitRowNum = startRowNum + rowCount / 2; 542 byte[] splitKey = Bytes.toBytes("" + splitRowNum); 543 544 assertTrue("not able to find a splittable region", regions != null); 545 assertTrue("not able to find a splittable region", regions.length == 1); 546 ProcedureTestingUtility.waitNoProcedureRunning(procExec); 547 ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); 548 549 // Split region of the table 550 long procId = procExec.submitProcedure( 551 new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey)); 552 553 // Execute until step 7 of split procedure 554 // NOTE: the 7 (number after SPLIT_TABLE_REGION_UPDATE_META step) 555 MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId, 7, false); 556 557 // Unset Toggle Kill and make ProcExec work correctly 558 ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, false); 559 MasterProcedureTestingUtility.restartMasterProcedureExecutor(procExec); 560 ProcedureTestingUtility.waitProcedure(procExec, procId); 561 562 // Even split failed after step 4, it should still works fine 563 verify(tableName, splitRowNum); 564 } 565 566 @Test 567 public void testSplitRegionWhileTakingSnapshot() throws Exception { 568 final TableName tableName = TableName.valueOf(name.getMethodName()); 569 final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); 570 571 RegionInfo[] regions = MasterProcedureTestingUtility.createTable(procExec, tableName, null, 572 columnFamilyName1, columnFamilyName2); 573 int splitRowNum = startRowNum + rowCount / 2; 574 byte[] splitKey = Bytes.toBytes("" + splitRowNum); 575 576 assertTrue("not able to find a splittable region", regions != null); 577 assertTrue("not able to find a splittable region", regions.length == 1); 578 ProcedureTestingUtility.waitNoProcedureRunning(procExec); 579 580 // task snapshot 581 SnapshotDescription snapshot = 582 new SnapshotDescription("SnapshotProcedureTest", tableName, SnapshotType.FLUSH); 583 SnapshotProtos.SnapshotDescription snapshotProto = 584 ProtobufUtil.createHBaseProtosSnapshotDesc(snapshot); 585 snapshotProto = SnapshotDescriptionUtils.validate(snapshotProto, 586 UTIL.getHBaseCluster().getMaster().getConfiguration()); 587 long snapshotProcId = procExec.submitProcedure( 588 new TestSnapshotProcedure.DelaySnapshotProcedure(procExec.getEnvironment(), snapshotProto)); 589 UTIL.getHBaseCluster().getMaster().getSnapshotManager().registerSnapshotProcedure(snapshotProto, 590 snapshotProcId); 591 592 // collect AM metrics before test 593 collectAssignmentManagerMetrics(); 594 595 // Split region of the table 596 long procId = procExec.submitProcedure( 597 new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey)); 598 // Wait the completion 599 ProcedureTestingUtility.waitProcedure(procExec, procId); 600 ProcedureTestingUtility.waitProcedure(procExec, snapshotProcId); 601 602 ProcedureTestingUtility.assertProcFailed(procExec, procId); 603 ProcedureTestingUtility.assertProcNotFailed(procExec, snapshotProcId); 604 605 assertTrue(UTIL.getMiniHBaseCluster().getRegions(tableName).size() == 1); 606 assertTrue(UTIL.countRows(tableName) == 0); 607 608 assertEquals(splitSubmittedCount + 1, splitProcMetrics.getSubmittedCounter().getCount()); 609 assertEquals(splitFailedCount + 1, splitProcMetrics.getFailedCounter().getCount()); 610 } 611 612 private void deleteData(final TableName tableName, final int startDeleteRowNum) 613 throws IOException, InterruptedException { 614 Table t = UTIL.getConnection().getTable(tableName); 615 final int numRows = rowCount + startRowNum - startDeleteRowNum; 616 Delete d; 617 for (int i = startDeleteRowNum; i <= numRows + startDeleteRowNum; i++) { 618 d = new Delete(Bytes.toBytes("" + i)); 619 t.delete(d); 620 if (i % 5 == 0) { 621 UTIL.getAdmin().flush(tableName); 622 } 623 } 624 } 625 626 private void verify(final TableName tableName, final int splitRowNum) throws IOException { 627 List<HRegion> daughters = UTIL.getMiniHBaseCluster().getRegions(tableName); 628 assertTrue(daughters.size() == 2); 629 LOG.info("Row Count = " + UTIL.countRows(tableName)); 630 assertTrue(UTIL.countRows(tableName) == rowCount); 631 int startRow; 632 int numRows; 633 for (int i = 0; i < daughters.size(); i++) { 634 if ( 635 Bytes.compareTo(daughters.get(i).getRegionInfo().getStartKey(), HConstants.EMPTY_BYTE_ARRAY) 636 == 0 637 ) { 638 startRow = startRowNum; // first region 639 numRows = splitRowNum - startRowNum; 640 } else { 641 startRow = splitRowNum; 642 numRows = rowCount + startRowNum - splitRowNum; 643 } 644 verifyData(daughters.get(i), startRow, numRows, Bytes.toBytes(columnFamilyName1), 645 Bytes.toBytes(columnFamilyName2)); 646 } 647 } 648 649 private void verifyData(final HRegion newReg, final int startRow, final int numRows, 650 final byte[]... families) throws IOException { 651 for (int i = startRow; i < startRow + numRows; i++) { 652 byte[] row = Bytes.toBytes("" + i); 653 Get get = new Get(row); 654 Result result = newReg.get(get); 655 Cell[] raw = result.rawCells(); 656 assertEquals(families.length, result.size()); 657 for (int j = 0; j < families.length; j++) { 658 assertTrue(CellUtil.matchingRows(raw[j], row)); 659 assertTrue(CellUtil.matchingFamily(raw[j], families[j])); 660 } 661 } 662 } 663 664 private ProcedureExecutor<MasterProcedureEnv> getMasterProcedureExecutor() { 665 return UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor(); 666 } 667 668 private void collectAssignmentManagerMetrics() { 669 splitSubmittedCount = splitProcMetrics.getSubmittedCounter().getCount(); 670 splitFailedCount = splitProcMetrics.getFailedCounter().getCount(); 671 assignSubmittedCount = assignProcMetrics.getSubmittedCounter().getCount(); 672 assignFailedCount = assignProcMetrics.getFailedCounter().getCount(); 673 unassignSubmittedCount = unassignProcMetrics.getSubmittedCounter().getCount(); 674 unassignFailedCount = unassignProcMetrics.getFailedCounter().getCount(); 675 } 676}