001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.assignment;
019
020import static org.apache.hadoop.hbase.master.assignment.AssignmentTestingUtil.insertData;
021import static org.junit.Assert.assertEquals;
022import static org.junit.Assert.assertFalse;
023import static org.junit.Assert.assertTrue;
024import static org.junit.Assert.fail;
025
026import java.io.IOException;
027import java.util.List;
028import java.util.Optional;
029import org.apache.hadoop.conf.Configuration;
030import org.apache.hadoop.hbase.Cell;
031import org.apache.hadoop.hbase.CellUtil;
032import org.apache.hadoop.hbase.DoNotRetryIOException;
033import org.apache.hadoop.hbase.HBaseClassTestRule;
034import org.apache.hadoop.hbase.HBaseTestingUtil;
035import org.apache.hadoop.hbase.HConstants;
036import org.apache.hadoop.hbase.TableName;
037import org.apache.hadoop.hbase.Waiter;
038import org.apache.hadoop.hbase.client.CompactionState;
039import org.apache.hadoop.hbase.client.Delete;
040import org.apache.hadoop.hbase.client.Get;
041import org.apache.hadoop.hbase.client.RegionInfo;
042import org.apache.hadoop.hbase.client.RegionReplicaUtil;
043import org.apache.hadoop.hbase.client.Result;
044import org.apache.hadoop.hbase.client.Table;
045import org.apache.hadoop.hbase.client.TableDescriptor;
046import org.apache.hadoop.hbase.coprocessor.ObserverContext;
047import org.apache.hadoop.hbase.coprocessor.RegionCoprocessor;
048import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
049import org.apache.hadoop.hbase.coprocessor.RegionObserver;
050import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants;
051import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
052import org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility;
053import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
054import org.apache.hadoop.hbase.procedure2.ProcedureMetrics;
055import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
056import org.apache.hadoop.hbase.regionserver.HRegion;
057import org.apache.hadoop.hbase.testclassification.MasterTests;
058import org.apache.hadoop.hbase.testclassification.MediumTests;
059import org.apache.hadoop.hbase.util.Bytes;
060import org.junit.After;
061import org.junit.AfterClass;
062import org.junit.Before;
063import org.junit.BeforeClass;
064import org.junit.ClassRule;
065import org.junit.Rule;
066import org.junit.Test;
067import org.junit.experimental.categories.Category;
068import org.junit.rules.TestName;
069import org.slf4j.Logger;
070import org.slf4j.LoggerFactory;
071
072@Category({ MasterTests.class, MediumTests.class })
073public class TestSplitTableRegionProcedure {
074
075  @ClassRule
076  public static final HBaseClassTestRule CLASS_RULE =
077    HBaseClassTestRule.forClass(TestSplitTableRegionProcedure.class);
078
079  private static final Logger LOG = LoggerFactory.getLogger(TestSplitTableRegionProcedure.class);
080
081  protected static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
082
083  private static String columnFamilyName1 = "cf1";
084  private static String columnFamilyName2 = "cf2";
085
086  private static final int startRowNum = 11;
087  private static final int rowCount = 60;
088
089  private AssignmentManager am;
090
091  private ProcedureMetrics splitProcMetrics;
092  private ProcedureMetrics assignProcMetrics;
093  private ProcedureMetrics unassignProcMetrics;
094
095  private long splitSubmittedCount = 0;
096  private long splitFailedCount = 0;
097  private long assignSubmittedCount = 0;
098  private long assignFailedCount = 0;
099  private long unassignSubmittedCount = 0;
100  private long unassignFailedCount = 0;
101
102  @Rule
103  public TestName name = new TestName();
104
105  private static void setupConf(Configuration conf) {
106    conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1);
107    conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 0);
108    conf.set("hbase.coprocessor.region.classes",
109      RegionServerHostingReplicaSlowOpenCopro.class.getName());
110    conf.setInt("hbase.client.sync.wait.timeout.msec", 1500);
111  }
112
113  /**
114   * This copro is used to slow down opening of the replica regions.
115   */
116  public static class RegionServerHostingReplicaSlowOpenCopro
117    implements RegionCoprocessor, RegionObserver {
118    static int countForReplica = 0;
119    static boolean slowDownReplicaOpen = false;
120
121    @Override
122    public Optional<RegionObserver> getRegionObserver() {
123      return Optional.of(this);
124    }
125
126    @Override
127    public void preOpen(ObserverContext<RegionCoprocessorEnvironment> c) throws IOException {
128      int replicaId = c.getEnvironment().getRegion().getRegionInfo().getReplicaId();
129      if ((replicaId != RegionInfo.DEFAULT_REPLICA_ID) && (countForReplica == 0)) {
130        countForReplica++;
131        while (slowDownReplicaOpen) {
132          LOG.info("Slow down replica region open a bit");
133          try {
134            Thread.sleep(100);
135          } catch (InterruptedException ie) {
136            // Ingore
137          }
138        }
139      }
140    }
141  }
142
143  @BeforeClass
144  public static void setupCluster() throws Exception {
145    setupConf(UTIL.getConfiguration());
146    UTIL.startMiniCluster(3);
147  }
148
149  @AfterClass
150  public static void cleanupTest() throws Exception {
151    try {
152      UTIL.shutdownMiniCluster();
153    } catch (Exception e) {
154      LOG.warn("failure shutting down cluster", e);
155    }
156  }
157
158  @Before
159  public void setup() throws Exception {
160    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(getMasterProcedureExecutor(), false);
161
162    // Turn off balancer so it doesn't cut in and mess up our placements.
163    UTIL.getAdmin().balancerSwitch(false, true);
164    // Turn off the meta scanner so it don't remove parent on us.
165    UTIL.getHBaseCluster().getMaster().setCatalogJanitorEnabled(false);
166    am = UTIL.getHBaseCluster().getMaster().getAssignmentManager();
167    splitProcMetrics = am.getAssignmentManagerMetrics().getSplitProcMetrics();
168    assignProcMetrics = am.getAssignmentManagerMetrics().getAssignProcMetrics();
169    unassignProcMetrics = am.getAssignmentManagerMetrics().getUnassignProcMetrics();
170  }
171
172  @After
173  public void tearDown() throws Exception {
174    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(getMasterProcedureExecutor(), false);
175    for (TableDescriptor htd : UTIL.getAdmin().listTableDescriptors()) {
176      UTIL.deleteTable(htd.getTableName());
177    }
178  }
179
180  @Test
181  public void testRollbackForSplitTableRegionWithReplica() throws Exception {
182    final TableName tableName = TableName.valueOf(name.getMethodName());
183    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
184
185    RegionServerHostingReplicaSlowOpenCopro.slowDownReplicaOpen = true;
186    RegionInfo[] regions =
187      MasterProcedureTestingUtility.createTable(procExec, tableName, null, columnFamilyName1);
188
189    try {
190      HBaseTestingUtil.setReplicas(UTIL.getAdmin(), tableName, 2);
191    } catch (IOException ioe) {
192
193    }
194
195    // wait until the primary region is online.
196    HBaseTestingUtil.await(2000, () -> {
197      try {
198        AssignmentManager am = UTIL.getHBaseCluster().getMaster().getAssignmentManager();
199        if (am == null) return false;
200        if (am.getRegionStates().getRegionState(regions[0]).isOpened()) {
201          return true;
202        }
203        return false;
204      } catch (Exception e) {
205        throw new RuntimeException(e);
206      }
207    });
208
209    // Split region of the table, it will fail and rollback as replica parent region
210    // is still at OPENING state.
211    long procId = procExec.submitProcedure(new SplitTableRegionProcedure(procExec.getEnvironment(),
212      regions[0], HConstants.CATALOG_FAMILY));
213    // Wait for the completion.
214    ProcedureTestingUtility.waitProcedure(procExec, procId);
215
216    // Let replica parent region open.
217    RegionServerHostingReplicaSlowOpenCopro.slowDownReplicaOpen = false;
218
219    // wait until the replica region is online.
220    HBaseTestingUtil.await(2000, () -> {
221      try {
222        AssignmentManager am = UTIL.getHBaseCluster().getMaster().getAssignmentManager();
223        if (am == null) return false;
224        RegionInfo replicaRegion = RegionReplicaUtil.getRegionInfoForReplica(regions[0], 1);
225        if (am.getRegionStates().getRegionState(replicaRegion).isOpened()) {
226          return true;
227        }
228        return false;
229      } catch (Exception e) {
230        throw new RuntimeException(e);
231      }
232    });
233
234    ProcedureTestingUtility.assertProcFailed(procExec, procId);
235    // There should not be any active OpenRegionProcedure
236    procExec.getActiveProceduresNoCopy()
237      .forEach(p -> assertFalse(p instanceof OpenRegionProcedure));
238  }
239
240  @Test
241  public void testSplitTableRegion() throws Exception {
242    final TableName tableName = TableName.valueOf(name.getMethodName());
243    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
244
245    RegionInfo[] regions = MasterProcedureTestingUtility.createTable(procExec, tableName, null,
246      columnFamilyName1, columnFamilyName2);
247    insertData(UTIL, tableName, rowCount, startRowNum, columnFamilyName1, columnFamilyName2);
248    int splitRowNum = startRowNum + rowCount / 2;
249    byte[] splitKey = Bytes.toBytes("" + splitRowNum);
250
251    assertTrue("not able to find a splittable region", regions != null);
252    assertTrue("not able to find a splittable region", regions.length == 1);
253
254    // collect AM metrics before test
255    collectAssignmentManagerMetrics();
256
257    // Split region of the table
258    long procId = procExec.submitProcedure(
259      new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
260    // Wait the completion
261    ProcedureTestingUtility.waitProcedure(procExec, procId);
262    ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
263
264    verify(tableName, splitRowNum);
265
266    assertEquals(splitSubmittedCount + 1, splitProcMetrics.getSubmittedCounter().getCount());
267    assertEquals(splitFailedCount, splitProcMetrics.getFailedCounter().getCount());
268    assertEquals(assignSubmittedCount + 2, assignProcMetrics.getSubmittedCounter().getCount());
269    assertEquals(assignFailedCount, assignProcMetrics.getFailedCounter().getCount());
270    assertEquals(unassignSubmittedCount + 1, unassignProcMetrics.getSubmittedCounter().getCount());
271    assertEquals(unassignFailedCount, unassignProcMetrics.getFailedCounter().getCount());
272  }
273
274  @Test
275  public void testSplitTableRegionNoStoreFile() throws Exception {
276    final TableName tableName = TableName.valueOf(name.getMethodName());
277    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
278
279    RegionInfo[] regions = MasterProcedureTestingUtility.createTable(procExec, tableName, null,
280      columnFamilyName1, columnFamilyName2);
281    int splitRowNum = startRowNum + rowCount / 2;
282    byte[] splitKey = Bytes.toBytes("" + splitRowNum);
283
284    assertTrue("not able to find a splittable region", regions != null);
285    assertTrue("not able to find a splittable region", regions.length == 1);
286
287    // collect AM metrics before test
288    collectAssignmentManagerMetrics();
289
290    // Split region of the table
291    long procId = procExec.submitProcedure(
292      new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
293    // Wait the completion
294    ProcedureTestingUtility.waitProcedure(procExec, procId);
295    ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
296
297    assertTrue(UTIL.getMiniHBaseCluster().getRegions(tableName).size() == 2);
298    assertTrue(UTIL.countRows(tableName) == 0);
299
300    assertEquals(splitSubmittedCount + 1, splitProcMetrics.getSubmittedCounter().getCount());
301    assertEquals(splitFailedCount, splitProcMetrics.getFailedCounter().getCount());
302  }
303
304  @Test
305  public void testSplitTableRegionUnevenDaughter() throws Exception {
306    final TableName tableName = TableName.valueOf(name.getMethodName());
307    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
308
309    RegionInfo[] regions = MasterProcedureTestingUtility.createTable(procExec, tableName, null,
310      columnFamilyName1, columnFamilyName2);
311    insertData(UTIL, tableName, rowCount, startRowNum, columnFamilyName1, columnFamilyName2);
312    // Split to two daughters with one of them only has 1 row
313    int splitRowNum = startRowNum + rowCount / 4;
314    byte[] splitKey = Bytes.toBytes("" + splitRowNum);
315
316    assertTrue("not able to find a splittable region", regions != null);
317    assertTrue("not able to find a splittable region", regions.length == 1);
318
319    // collect AM metrics before test
320    collectAssignmentManagerMetrics();
321
322    // Split region of the table
323    long procId = procExec.submitProcedure(
324      new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
325    // Wait the completion
326    ProcedureTestingUtility.waitProcedure(procExec, procId);
327    ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
328
329    verify(tableName, splitRowNum);
330
331    assertEquals(splitSubmittedCount + 1, splitProcMetrics.getSubmittedCounter().getCount());
332    assertEquals(splitFailedCount, splitProcMetrics.getFailedCounter().getCount());
333  }
334
335  @Test
336  public void testSplitTableRegionEmptyDaughter() throws Exception {
337    final TableName tableName = TableName.valueOf(name.getMethodName());
338    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
339
340    RegionInfo[] regions = MasterProcedureTestingUtility.createTable(procExec, tableName, null,
341      columnFamilyName1, columnFamilyName2);
342    insertData(UTIL, tableName, rowCount, startRowNum, columnFamilyName1, columnFamilyName2);
343    // Split to two daughters with one of them only has 1 row
344    int splitRowNum = startRowNum + rowCount;
345    byte[] splitKey = Bytes.toBytes("" + splitRowNum);
346
347    assertTrue("not able to find a splittable region", regions != null);
348    assertTrue("not able to find a splittable region", regions.length == 1);
349
350    // collect AM metrics before test
351    collectAssignmentManagerMetrics();
352
353    // Split region of the table
354    long procId = procExec.submitProcedure(
355      new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
356    // Wait the completion
357    ProcedureTestingUtility.waitProcedure(procExec, procId);
358    ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
359
360    // Make sure one daughter has 0 rows.
361    List<HRegion> daughters = UTIL.getMiniHBaseCluster().getRegions(tableName);
362    assertTrue(daughters.size() == 2);
363    assertTrue(UTIL.countRows(tableName) == rowCount);
364    assertTrue(UTIL.countRows(daughters.get(0)) == 0 || UTIL.countRows(daughters.get(1)) == 0);
365
366    assertEquals(splitSubmittedCount + 1, splitProcMetrics.getSubmittedCounter().getCount());
367    assertEquals(splitFailedCount, splitProcMetrics.getFailedCounter().getCount());
368  }
369
370  @Test
371  public void testSplitTableRegionDeletedRowsDaughter() throws Exception {
372    final TableName tableName = TableName.valueOf(name.getMethodName());
373    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
374
375    RegionInfo[] regions = MasterProcedureTestingUtility.createTable(procExec, tableName, null,
376      columnFamilyName1, columnFamilyName2);
377    insertData(UTIL, tableName, rowCount, startRowNum, columnFamilyName1, columnFamilyName2);
378    // Split to two daughters with one of them only has 1 row
379    int splitRowNum = rowCount;
380    deleteData(tableName, splitRowNum);
381    byte[] splitKey = Bytes.toBytes("" + splitRowNum);
382
383    assertTrue("not able to find a splittable region", regions != null);
384    assertTrue("not able to find a splittable region", regions.length == 1);
385
386    // collect AM metrics before test
387    collectAssignmentManagerMetrics();
388
389    // Split region of the table
390    long procId = procExec.submitProcedure(
391      new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
392    // Wait the completion
393    ProcedureTestingUtility.waitProcedure(procExec, procId);
394    ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
395
396    UTIL.getAdmin().majorCompact(tableName);
397    // waiting for the major compaction to complete
398    UTIL.waitFor(6000, new Waiter.Predicate<IOException>() {
399      @Override
400      public boolean evaluate() throws IOException {
401        return UTIL.getAdmin().getCompactionState(tableName) == CompactionState.NONE;
402      }
403    });
404
405    // Make sure one daughter has 0 rows.
406    List<HRegion> daughters = UTIL.getMiniHBaseCluster().getRegions(tableName);
407    assertTrue(daughters.size() == 2);
408    final int currentRowCount = splitRowNum - startRowNum;
409    assertTrue(UTIL.countRows(tableName) == currentRowCount);
410    assertTrue(UTIL.countRows(daughters.get(0)) == 0 || UTIL.countRows(daughters.get(1)) == 0);
411
412    assertEquals(splitSubmittedCount + 1, splitProcMetrics.getSubmittedCounter().getCount());
413    assertEquals(splitFailedCount, splitProcMetrics.getFailedCounter().getCount());
414  }
415
416  @Test
417  public void testInvalidSplitKey() throws Exception {
418    final TableName tableName = TableName.valueOf(name.getMethodName());
419    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
420
421    RegionInfo[] regions = MasterProcedureTestingUtility.createTable(procExec, tableName, null,
422      columnFamilyName1, columnFamilyName2);
423    insertData(UTIL, tableName, rowCount, startRowNum, columnFamilyName1, columnFamilyName2);
424
425    assertTrue("not able to find a splittable region", regions != null);
426    assertTrue("not able to find a splittable region", regions.length == 1);
427
428    // collect AM metrics before test
429    collectAssignmentManagerMetrics();
430
431    // Split region of the table with null split key
432    try {
433      long procId1 = procExec.submitProcedure(
434        new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], null));
435      ProcedureTestingUtility.waitProcedure(procExec, procId1);
436      fail("unexpected procedure start with invalid split-key");
437    } catch (DoNotRetryIOException e) {
438      LOG.debug("Expected Split procedure construction failure: " + e.getMessage());
439    }
440
441    assertEquals(splitSubmittedCount, splitProcMetrics.getSubmittedCounter().getCount());
442    assertEquals(splitFailedCount, splitProcMetrics.getFailedCounter().getCount());
443  }
444
445  @Test
446  public void testRollbackAndDoubleExecution() throws Exception {
447    final TableName tableName = TableName.valueOf(name.getMethodName());
448    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
449
450    RegionInfo[] regions = MasterProcedureTestingUtility.createTable(procExec, tableName, null,
451      columnFamilyName1, columnFamilyName2);
452    insertData(UTIL, tableName, rowCount, startRowNum, columnFamilyName1, columnFamilyName2);
453    int splitRowNum = startRowNum + rowCount / 2;
454    byte[] splitKey = Bytes.toBytes("" + splitRowNum);
455
456    assertTrue("not able to find a splittable region", regions != null);
457    assertTrue("not able to find a splittable region", regions.length == 1);
458    ProcedureTestingUtility.waitNoProcedureRunning(procExec);
459    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
460
461    // collect AM metrics before test
462    collectAssignmentManagerMetrics();
463
464    // Split region of the table
465    long procId = procExec.submitProcedure(
466      new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
467
468    // Failing before SPLIT_TABLE_REGION_UPDATE_META we should trigger the
469    // rollback
470    // NOTE: the 7 (number of SPLIT_TABLE_REGION_UPDATE_META step) is
471    // hardcoded, so you have to look at this test at least once when you add a new step.
472    int lastStep = 7;
473    MasterProcedureTestingUtility.testRollbackAndDoubleExecution(procExec, procId, lastStep, true);
474    // check that we have only 1 region
475    assertEquals(1, UTIL.getAdmin().getRegions(tableName).size());
476    UTIL.waitUntilAllRegionsAssigned(tableName);
477    List<HRegion> newRegions = UTIL.getMiniHBaseCluster().getRegions(tableName);
478    assertEquals(1, newRegions.size());
479    verifyData(newRegions.get(0), startRowNum, rowCount, Bytes.toBytes(columnFamilyName1),
480      Bytes.toBytes(columnFamilyName2));
481
482    assertEquals(splitSubmittedCount + 1, splitProcMetrics.getSubmittedCounter().getCount());
483    assertEquals(splitFailedCount + 1, splitProcMetrics.getFailedCounter().getCount());
484  }
485
486  @Test
487  public void testRecoveryAndDoubleExecution() throws Exception {
488    final TableName tableName = TableName.valueOf(name.getMethodName());
489    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
490
491    RegionInfo[] regions = MasterProcedureTestingUtility.createTable(procExec, tableName, null,
492      columnFamilyName1, columnFamilyName2);
493    insertData(UTIL, tableName, rowCount, startRowNum, columnFamilyName1, columnFamilyName2);
494    int splitRowNum = startRowNum + rowCount / 2;
495    byte[] splitKey = Bytes.toBytes("" + splitRowNum);
496
497    assertTrue("not able to find a splittable region", regions != null);
498    assertTrue("not able to find a splittable region", regions.length == 1);
499    ProcedureTestingUtility.waitNoProcedureRunning(procExec);
500    ProcedureTestingUtility.setKillIfHasParent(procExec, false);
501    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
502
503    // collect AM metrics before test
504    collectAssignmentManagerMetrics();
505
506    // Split region of the table
507    long procId = procExec.submitProcedure(
508      new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
509
510    // Restart the executor and execute the step twice
511    MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId);
512    ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
513
514    verify(tableName, splitRowNum);
515
516    assertEquals(splitSubmittedCount + 1, splitProcMetrics.getSubmittedCounter().getCount());
517    assertEquals(splitFailedCount, splitProcMetrics.getFailedCounter().getCount());
518  }
519
520  @Test
521  public void testSplitWithoutPONR() throws Exception {
522    final TableName tableName = TableName.valueOf(name.getMethodName());
523    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
524
525    RegionInfo[] regions = MasterProcedureTestingUtility.createTable(procExec, tableName, null,
526      columnFamilyName1, columnFamilyName2);
527    insertData(UTIL, tableName, rowCount, startRowNum, columnFamilyName1, columnFamilyName2);
528    int splitRowNum = startRowNum + rowCount / 2;
529    byte[] splitKey = Bytes.toBytes("" + splitRowNum);
530
531    assertTrue("not able to find a splittable region", regions != null);
532    assertTrue("not able to find a splittable region", regions.length == 1);
533    ProcedureTestingUtility.waitNoProcedureRunning(procExec);
534    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
535
536    // Split region of the table
537    long procId = procExec.submitProcedure(
538      new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
539
540    // Execute until step 7 of split procedure
541    // NOTE: the 7 (number after SPLIT_TABLE_REGION_UPDATE_META step)
542    MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId, 7, false);
543
544    // Unset Toggle Kill and make ProcExec work correctly
545    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, false);
546    MasterProcedureTestingUtility.restartMasterProcedureExecutor(procExec);
547    ProcedureTestingUtility.waitProcedure(procExec, procId);
548
549    // Even split failed after step 4, it should still works fine
550    verify(tableName, splitRowNum);
551  }
552
553  private void deleteData(final TableName tableName, final int startDeleteRowNum)
554    throws IOException, InterruptedException {
555    Table t = UTIL.getConnection().getTable(tableName);
556    final int numRows = rowCount + startRowNum - startDeleteRowNum;
557    Delete d;
558    for (int i = startDeleteRowNum; i <= numRows + startDeleteRowNum; i++) {
559      d = new Delete(Bytes.toBytes("" + i));
560      t.delete(d);
561      if (i % 5 == 0) {
562        UTIL.getAdmin().flush(tableName);
563      }
564    }
565  }
566
567  private void verify(final TableName tableName, final int splitRowNum) throws IOException {
568    List<HRegion> daughters = UTIL.getMiniHBaseCluster().getRegions(tableName);
569    assertTrue(daughters.size() == 2);
570    LOG.info("Row Count = " + UTIL.countRows(tableName));
571    assertTrue(UTIL.countRows(tableName) == rowCount);
572    int startRow;
573    int numRows;
574    for (int i = 0; i < daughters.size(); i++) {
575      if (
576        Bytes.compareTo(daughters.get(i).getRegionInfo().getStartKey(), HConstants.EMPTY_BYTE_ARRAY)
577            == 0
578      ) {
579        startRow = startRowNum; // first region
580        numRows = splitRowNum - startRowNum;
581      } else {
582        startRow = splitRowNum;
583        numRows = rowCount + startRowNum - splitRowNum;
584      }
585      verifyData(daughters.get(i), startRow, numRows, Bytes.toBytes(columnFamilyName1),
586        Bytes.toBytes(columnFamilyName2));
587    }
588  }
589
590  private void verifyData(final HRegion newReg, final int startRow, final int numRows,
591    final byte[]... families) throws IOException {
592    for (int i = startRow; i < startRow + numRows; i++) {
593      byte[] row = Bytes.toBytes("" + i);
594      Get get = new Get(row);
595      Result result = newReg.get(get);
596      Cell[] raw = result.rawCells();
597      assertEquals(families.length, result.size());
598      for (int j = 0; j < families.length; j++) {
599        assertTrue(CellUtil.matchingRows(raw[j], row));
600        assertTrue(CellUtil.matchingFamily(raw[j], families[j]));
601      }
602    }
603  }
604
605  private ProcedureExecutor<MasterProcedureEnv> getMasterProcedureExecutor() {
606    return UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor();
607  }
608
609  private void collectAssignmentManagerMetrics() {
610    splitSubmittedCount = splitProcMetrics.getSubmittedCounter().getCount();
611    splitFailedCount = splitProcMetrics.getFailedCounter().getCount();
612    assignSubmittedCount = assignProcMetrics.getSubmittedCounter().getCount();
613    assignFailedCount = assignProcMetrics.getFailedCounter().getCount();
614    unassignSubmittedCount = unassignProcMetrics.getSubmittedCounter().getCount();
615    unassignFailedCount = unassignProcMetrics.getFailedCounter().getCount();
616  }
617}