001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.assignment;
019
020import static org.apache.hadoop.hbase.master.assignment.AssignmentTestingUtil.insertData;
021import static org.junit.Assert.assertEquals;
022import static org.junit.Assert.assertFalse;
023import static org.junit.Assert.assertTrue;
024import static org.junit.Assert.fail;
025
026import java.io.IOException;
027import java.util.List;
028import java.util.Optional;
029import org.apache.hadoop.conf.Configuration;
030import org.apache.hadoop.hbase.Cell;
031import org.apache.hadoop.hbase.CellUtil;
032import org.apache.hadoop.hbase.DoNotRetryIOException;
033import org.apache.hadoop.hbase.HBaseClassTestRule;
034import org.apache.hadoop.hbase.HBaseTestingUtil;
035import org.apache.hadoop.hbase.HConstants;
036import org.apache.hadoop.hbase.TableName;
037import org.apache.hadoop.hbase.Waiter;
038import org.apache.hadoop.hbase.client.CompactionState;
039import org.apache.hadoop.hbase.client.Delete;
040import org.apache.hadoop.hbase.client.Get;
041import org.apache.hadoop.hbase.client.RegionInfo;
042import org.apache.hadoop.hbase.client.RegionReplicaUtil;
043import org.apache.hadoop.hbase.client.Result;
044import org.apache.hadoop.hbase.client.SnapshotDescription;
045import org.apache.hadoop.hbase.client.SnapshotType;
046import org.apache.hadoop.hbase.client.Table;
047import org.apache.hadoop.hbase.client.TableDescriptor;
048import org.apache.hadoop.hbase.coprocessor.ObserverContext;
049import org.apache.hadoop.hbase.coprocessor.RegionCoprocessor;
050import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
051import org.apache.hadoop.hbase.coprocessor.RegionObserver;
052import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants;
053import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
054import org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility;
055import org.apache.hadoop.hbase.master.procedure.TestSnapshotProcedure;
056import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
057import org.apache.hadoop.hbase.procedure2.ProcedureMetrics;
058import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
059import org.apache.hadoop.hbase.regionserver.HRegion;
060import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
061import org.apache.hadoop.hbase.testclassification.MasterTests;
062import org.apache.hadoop.hbase.testclassification.MediumTests;
063import org.apache.hadoop.hbase.util.Bytes;
064import org.junit.After;
065import org.junit.AfterClass;
066import org.junit.Before;
067import org.junit.BeforeClass;
068import org.junit.ClassRule;
069import org.junit.Rule;
070import org.junit.Test;
071import org.junit.experimental.categories.Category;
072import org.junit.rules.TestName;
073import org.slf4j.Logger;
074import org.slf4j.LoggerFactory;
075
076import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
077import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos;
078
079@Category({ MasterTests.class, MediumTests.class })
080public class TestSplitTableRegionProcedure {
081
082  @ClassRule
083  public static final HBaseClassTestRule CLASS_RULE =
084    HBaseClassTestRule.forClass(TestSplitTableRegionProcedure.class);
085
086  private static final Logger LOG = LoggerFactory.getLogger(TestSplitTableRegionProcedure.class);
087
088  protected static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
089
090  private static String columnFamilyName1 = "cf1";
091  private static String columnFamilyName2 = "cf2";
092
093  private static final int startRowNum = 11;
094  private static final int rowCount = 60;
095
096  private AssignmentManager am;
097
098  private ProcedureMetrics splitProcMetrics;
099  private ProcedureMetrics assignProcMetrics;
100  private ProcedureMetrics unassignProcMetrics;
101
102  private long splitSubmittedCount = 0;
103  private long splitFailedCount = 0;
104  private long assignSubmittedCount = 0;
105  private long assignFailedCount = 0;
106  private long unassignSubmittedCount = 0;
107  private long unassignFailedCount = 0;
108
109  @Rule
110  public TestName name = new TestName();
111
112  private static void setupConf(Configuration conf) {
113    conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1);
114    conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 0);
115    conf.set("hbase.coprocessor.region.classes",
116      RegionServerHostingReplicaSlowOpenCopro.class.getName());
117    conf.setInt("hbase.client.sync.wait.timeout.msec", 1500);
118  }
119
120  /**
121   * This copro is used to slow down opening of the replica regions.
122   */
123  public static class RegionServerHostingReplicaSlowOpenCopro
124    implements RegionCoprocessor, RegionObserver {
125    static int countForReplica = 0;
126    static boolean slowDownReplicaOpen = false;
127
128    @Override
129    public Optional<RegionObserver> getRegionObserver() {
130      return Optional.of(this);
131    }
132
133    @Override
134    public void preOpen(ObserverContext<RegionCoprocessorEnvironment> c) throws IOException {
135      int replicaId = c.getEnvironment().getRegion().getRegionInfo().getReplicaId();
136      if ((replicaId != RegionInfo.DEFAULT_REPLICA_ID) && (countForReplica == 0)) {
137        countForReplica++;
138        while (slowDownReplicaOpen) {
139          LOG.info("Slow down replica region open a bit");
140          try {
141            Thread.sleep(100);
142          } catch (InterruptedException ie) {
143            // Ingore
144          }
145        }
146      }
147    }
148  }
149
150  @BeforeClass
151  public static void setupCluster() throws Exception {
152    setupConf(UTIL.getConfiguration());
153    UTIL.startMiniCluster(3);
154  }
155
156  @AfterClass
157  public static void cleanupTest() throws Exception {
158    try {
159      UTIL.shutdownMiniCluster();
160    } catch (Exception e) {
161      LOG.warn("failure shutting down cluster", e);
162    }
163  }
164
165  @Before
166  public void setup() throws Exception {
167    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(getMasterProcedureExecutor(), false);
168
169    // Turn off balancer so it doesn't cut in and mess up our placements.
170    UTIL.getAdmin().balancerSwitch(false, true);
171    // Turn off the meta scanner so it don't remove parent on us.
172    UTIL.getHBaseCluster().getMaster().setCatalogJanitorEnabled(false);
173    am = UTIL.getHBaseCluster().getMaster().getAssignmentManager();
174    splitProcMetrics = am.getAssignmentManagerMetrics().getSplitProcMetrics();
175    assignProcMetrics = am.getAssignmentManagerMetrics().getAssignProcMetrics();
176    unassignProcMetrics = am.getAssignmentManagerMetrics().getUnassignProcMetrics();
177  }
178
179  @After
180  public void tearDown() throws Exception {
181    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(getMasterProcedureExecutor(), false);
182    for (TableDescriptor htd : UTIL.getAdmin().listTableDescriptors()) {
183      UTIL.deleteTable(htd.getTableName());
184    }
185  }
186
187  @Test
188  public void testRollbackForSplitTableRegionWithReplica() throws Exception {
189    final TableName tableName = TableName.valueOf(name.getMethodName());
190    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
191
192    RegionServerHostingReplicaSlowOpenCopro.slowDownReplicaOpen = true;
193    RegionInfo[] regions =
194      MasterProcedureTestingUtility.createTable(procExec, tableName, null, columnFamilyName1);
195
196    try {
197      HBaseTestingUtil.setReplicas(UTIL.getAdmin(), tableName, 2);
198    } catch (IOException ioe) {
199
200    }
201
202    // wait until the primary region is online.
203    HBaseTestingUtil.await(2000, () -> {
204      try {
205        AssignmentManager am = UTIL.getHBaseCluster().getMaster().getAssignmentManager();
206        if (am == null) return false;
207        if (am.getRegionStates().getRegionState(regions[0]).isOpened()) {
208          return true;
209        }
210        return false;
211      } catch (Exception e) {
212        throw new RuntimeException(e);
213      }
214    });
215
216    // Split region of the table, it will fail and rollback as replica parent region
217    // is still at OPENING state.
218    long procId = procExec.submitProcedure(new SplitTableRegionProcedure(procExec.getEnvironment(),
219      regions[0], HConstants.CATALOG_FAMILY));
220    // Wait for the completion.
221    ProcedureTestingUtility.waitProcedure(procExec, procId);
222
223    // Let replica parent region open.
224    RegionServerHostingReplicaSlowOpenCopro.slowDownReplicaOpen = false;
225
226    // wait until the replica region is online.
227    HBaseTestingUtil.await(2000, () -> {
228      try {
229        AssignmentManager am = UTIL.getHBaseCluster().getMaster().getAssignmentManager();
230        if (am == null) return false;
231        RegionInfo replicaRegion = RegionReplicaUtil.getRegionInfoForReplica(regions[0], 1);
232        if (am.getRegionStates().getRegionState(replicaRegion).isOpened()) {
233          return true;
234        }
235        return false;
236      } catch (Exception e) {
237        throw new RuntimeException(e);
238      }
239    });
240
241    ProcedureTestingUtility.assertProcFailed(procExec, procId);
242    // There should not be any active OpenRegionProcedure
243    procExec.getActiveProceduresNoCopy()
244      .forEach(p -> assertFalse(p instanceof OpenRegionProcedure));
245  }
246
247  @Test
248  public void testSplitTableRegion() throws Exception {
249    final TableName tableName = TableName.valueOf(name.getMethodName());
250    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
251
252    RegionInfo[] regions = MasterProcedureTestingUtility.createTable(procExec, tableName, null,
253      columnFamilyName1, columnFamilyName2);
254    insertData(UTIL, tableName, rowCount, startRowNum, columnFamilyName1, columnFamilyName2);
255    int splitRowNum = startRowNum + rowCount / 2;
256    byte[] splitKey = Bytes.toBytes("" + splitRowNum);
257
258    assertTrue("not able to find a splittable region", regions != null);
259    assertTrue("not able to find a splittable region", regions.length == 1);
260
261    // collect AM metrics before test
262    collectAssignmentManagerMetrics();
263
264    // Split region of the table
265    long procId = procExec.submitProcedure(
266      new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
267    // Wait the completion
268    ProcedureTestingUtility.waitProcedure(procExec, procId);
269    ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
270
271    verify(tableName, splitRowNum);
272
273    assertEquals(splitSubmittedCount + 1, splitProcMetrics.getSubmittedCounter().getCount());
274    assertEquals(splitFailedCount, splitProcMetrics.getFailedCounter().getCount());
275    assertEquals(assignSubmittedCount + 2, assignProcMetrics.getSubmittedCounter().getCount());
276    assertEquals(assignFailedCount, assignProcMetrics.getFailedCounter().getCount());
277    assertEquals(unassignSubmittedCount + 1, unassignProcMetrics.getSubmittedCounter().getCount());
278    assertEquals(unassignFailedCount, unassignProcMetrics.getFailedCounter().getCount());
279  }
280
281  @Test
282  public void testSplitTableRegionNoStoreFile() throws Exception {
283    final TableName tableName = TableName.valueOf(name.getMethodName());
284    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
285
286    RegionInfo[] regions = MasterProcedureTestingUtility.createTable(procExec, tableName, null,
287      columnFamilyName1, columnFamilyName2);
288    int splitRowNum = startRowNum + rowCount / 2;
289    byte[] splitKey = Bytes.toBytes("" + splitRowNum);
290
291    assertTrue("not able to find a splittable region", regions != null);
292    assertTrue("not able to find a splittable region", regions.length == 1);
293
294    // collect AM metrics before test
295    collectAssignmentManagerMetrics();
296
297    // Split region of the table
298    long procId = procExec.submitProcedure(
299      new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
300    // Wait the completion
301    ProcedureTestingUtility.waitProcedure(procExec, procId);
302    ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
303
304    assertTrue(UTIL.getMiniHBaseCluster().getRegions(tableName).size() == 2);
305    assertTrue(UTIL.countRows(tableName) == 0);
306
307    assertEquals(splitSubmittedCount + 1, splitProcMetrics.getSubmittedCounter().getCount());
308    assertEquals(splitFailedCount, splitProcMetrics.getFailedCounter().getCount());
309  }
310
311  @Test
312  public void testSplitTableRegionUnevenDaughter() throws Exception {
313    final TableName tableName = TableName.valueOf(name.getMethodName());
314    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
315
316    RegionInfo[] regions = MasterProcedureTestingUtility.createTable(procExec, tableName, null,
317      columnFamilyName1, columnFamilyName2);
318    insertData(UTIL, tableName, rowCount, startRowNum, columnFamilyName1, columnFamilyName2);
319    // Split to two daughters with one of them only has 1 row
320    int splitRowNum = startRowNum + rowCount / 4;
321    byte[] splitKey = Bytes.toBytes("" + splitRowNum);
322
323    assertTrue("not able to find a splittable region", regions != null);
324    assertTrue("not able to find a splittable region", regions.length == 1);
325
326    // collect AM metrics before test
327    collectAssignmentManagerMetrics();
328
329    // Split region of the table
330    long procId = procExec.submitProcedure(
331      new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
332    // Wait the completion
333    ProcedureTestingUtility.waitProcedure(procExec, procId);
334    ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
335
336    verify(tableName, splitRowNum);
337
338    assertEquals(splitSubmittedCount + 1, splitProcMetrics.getSubmittedCounter().getCount());
339    assertEquals(splitFailedCount, splitProcMetrics.getFailedCounter().getCount());
340  }
341
342  @Test
343  public void testSplitTableRegionEmptyDaughter() throws Exception {
344    final TableName tableName = TableName.valueOf(name.getMethodName());
345    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
346
347    RegionInfo[] regions = MasterProcedureTestingUtility.createTable(procExec, tableName, null,
348      columnFamilyName1, columnFamilyName2);
349    insertData(UTIL, tableName, rowCount, startRowNum, columnFamilyName1, columnFamilyName2);
350    // Split to two daughters with one of them only has 1 row
351    int splitRowNum = startRowNum + rowCount;
352    byte[] splitKey = Bytes.toBytes("" + splitRowNum);
353
354    assertTrue("not able to find a splittable region", regions != null);
355    assertTrue("not able to find a splittable region", regions.length == 1);
356
357    // collect AM metrics before test
358    collectAssignmentManagerMetrics();
359
360    // Split region of the table
361    long procId = procExec.submitProcedure(
362      new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
363    // Wait the completion
364    ProcedureTestingUtility.waitProcedure(procExec, procId);
365    ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
366
367    // Make sure one daughter has 0 rows.
368    List<HRegion> daughters = UTIL.getMiniHBaseCluster().getRegions(tableName);
369    assertTrue(daughters.size() == 2);
370    assertTrue(UTIL.countRows(tableName) == rowCount);
371    assertTrue(UTIL.countRows(daughters.get(0)) == 0 || UTIL.countRows(daughters.get(1)) == 0);
372
373    assertEquals(splitSubmittedCount + 1, splitProcMetrics.getSubmittedCounter().getCount());
374    assertEquals(splitFailedCount, splitProcMetrics.getFailedCounter().getCount());
375  }
376
377  @Test
378  public void testSplitTableRegionDeletedRowsDaughter() throws Exception {
379    final TableName tableName = TableName.valueOf(name.getMethodName());
380    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
381
382    RegionInfo[] regions = MasterProcedureTestingUtility.createTable(procExec, tableName, null,
383      columnFamilyName1, columnFamilyName2);
384    insertData(UTIL, tableName, rowCount, startRowNum, columnFamilyName1, columnFamilyName2);
385    // Split to two daughters with one of them only has 1 row
386    int splitRowNum = rowCount;
387    deleteData(tableName, splitRowNum);
388    byte[] splitKey = Bytes.toBytes("" + splitRowNum);
389
390    assertTrue("not able to find a splittable region", regions != null);
391    assertTrue("not able to find a splittable region", regions.length == 1);
392
393    // collect AM metrics before test
394    collectAssignmentManagerMetrics();
395
396    // Split region of the table
397    long procId = procExec.submitProcedure(
398      new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
399    // Wait the completion
400    ProcedureTestingUtility.waitProcedure(procExec, procId);
401    ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
402
403    UTIL.getAdmin().majorCompact(tableName);
404    // waiting for the major compaction to complete
405    UTIL.waitFor(6000, new Waiter.Predicate<IOException>() {
406      @Override
407      public boolean evaluate() throws IOException {
408        return UTIL.getAdmin().getCompactionState(tableName) == CompactionState.NONE;
409      }
410    });
411
412    // Make sure one daughter has 0 rows.
413    List<HRegion> daughters = UTIL.getMiniHBaseCluster().getRegions(tableName);
414    assertTrue(daughters.size() == 2);
415    final int currentRowCount = splitRowNum - startRowNum;
416    assertTrue(UTIL.countRows(tableName) == currentRowCount);
417    assertTrue(UTIL.countRows(daughters.get(0)) == 0 || UTIL.countRows(daughters.get(1)) == 0);
418
419    assertEquals(splitSubmittedCount + 1, splitProcMetrics.getSubmittedCounter().getCount());
420    assertEquals(splitFailedCount, splitProcMetrics.getFailedCounter().getCount());
421  }
422
423  @Test
424  public void testInvalidSplitKey() throws Exception {
425    final TableName tableName = TableName.valueOf(name.getMethodName());
426    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
427
428    RegionInfo[] regions = MasterProcedureTestingUtility.createTable(procExec, tableName, null,
429      columnFamilyName1, columnFamilyName2);
430    insertData(UTIL, tableName, rowCount, startRowNum, columnFamilyName1, columnFamilyName2);
431
432    assertTrue("not able to find a splittable region", regions != null);
433    assertTrue("not able to find a splittable region", regions.length == 1);
434
435    // collect AM metrics before test
436    collectAssignmentManagerMetrics();
437
438    // Split region of the table with null split key
439    try {
440      long procId1 = procExec.submitProcedure(
441        new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], null));
442      ProcedureTestingUtility.waitProcedure(procExec, procId1);
443      fail("unexpected procedure start with invalid split-key");
444    } catch (DoNotRetryIOException e) {
445      LOG.debug("Expected Split procedure construction failure: " + e.getMessage());
446    }
447
448    assertEquals(splitSubmittedCount, splitProcMetrics.getSubmittedCounter().getCount());
449    assertEquals(splitFailedCount, splitProcMetrics.getFailedCounter().getCount());
450  }
451
452  @Test
453  public void testRollbackAndDoubleExecution() throws Exception {
454    final TableName tableName = TableName.valueOf(name.getMethodName());
455    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
456
457    RegionInfo[] regions = MasterProcedureTestingUtility.createTable(procExec, tableName, null,
458      columnFamilyName1, columnFamilyName2);
459    insertData(UTIL, tableName, rowCount, startRowNum, columnFamilyName1, columnFamilyName2);
460    int splitRowNum = startRowNum + rowCount / 2;
461    byte[] splitKey = Bytes.toBytes("" + splitRowNum);
462
463    assertTrue("not able to find a splittable region", regions != null);
464    assertTrue("not able to find a splittable region", regions.length == 1);
465    ProcedureTestingUtility.waitNoProcedureRunning(procExec);
466    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
467
468    // collect AM metrics before test
469    collectAssignmentManagerMetrics();
470
471    // Split region of the table
472    long procId = procExec.submitProcedure(
473      new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
474
475    // Failing before SPLIT_TABLE_REGION_UPDATE_META we should trigger the
476    // rollback
477    // NOTE: the 7 (number of SPLIT_TABLE_REGION_UPDATE_META step) is
478    // hardcoded, so you have to look at this test at least once when you add a new step.
479    int lastStep = 7;
480    MasterProcedureTestingUtility.testRollbackAndDoubleExecution(procExec, procId, lastStep, true);
481    // check that we have only 1 region
482    assertEquals(1, UTIL.getAdmin().getRegions(tableName).size());
483    UTIL.waitUntilAllRegionsAssigned(tableName);
484    List<HRegion> newRegions = UTIL.getMiniHBaseCluster().getRegions(tableName);
485    assertEquals(1, newRegions.size());
486    verifyData(newRegions.get(0), startRowNum, rowCount, Bytes.toBytes(columnFamilyName1),
487      Bytes.toBytes(columnFamilyName2));
488
489    assertEquals(splitSubmittedCount + 1, splitProcMetrics.getSubmittedCounter().getCount());
490    assertEquals(splitFailedCount + 1, splitProcMetrics.getFailedCounter().getCount());
491  }
492
493  @Test
494  public void testRecoveryAndDoubleExecution() throws Exception {
495    final TableName tableName = TableName.valueOf(name.getMethodName());
496    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
497
498    RegionInfo[] regions = MasterProcedureTestingUtility.createTable(procExec, tableName, null,
499      columnFamilyName1, columnFamilyName2);
500    insertData(UTIL, tableName, rowCount, startRowNum, columnFamilyName1, columnFamilyName2);
501    int splitRowNum = startRowNum + rowCount / 2;
502    byte[] splitKey = Bytes.toBytes("" + splitRowNum);
503
504    assertTrue("not able to find a splittable region", regions != null);
505    assertTrue("not able to find a splittable region", regions.length == 1);
506    ProcedureTestingUtility.waitNoProcedureRunning(procExec);
507    ProcedureTestingUtility.setKillIfHasParent(procExec, false);
508    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
509
510    // collect AM metrics before test
511    collectAssignmentManagerMetrics();
512
513    // Split region of the table
514    long procId = procExec.submitProcedure(
515      new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
516
517    // Restart the executor and execute the step twice
518    MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId);
519    ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
520
521    verify(tableName, splitRowNum);
522
523    assertEquals(splitSubmittedCount + 1, splitProcMetrics.getSubmittedCounter().getCount());
524    assertEquals(splitFailedCount, splitProcMetrics.getFailedCounter().getCount());
525  }
526
527  @Test
528  public void testSplitWithoutPONR() throws Exception {
529    final TableName tableName = TableName.valueOf(name.getMethodName());
530    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
531
532    RegionInfo[] regions = MasterProcedureTestingUtility.createTable(procExec, tableName, null,
533      columnFamilyName1, columnFamilyName2);
534    insertData(UTIL, tableName, rowCount, startRowNum, columnFamilyName1, columnFamilyName2);
535    int splitRowNum = startRowNum + rowCount / 2;
536    byte[] splitKey = Bytes.toBytes("" + splitRowNum);
537
538    assertTrue("not able to find a splittable region", regions != null);
539    assertTrue("not able to find a splittable region", regions.length == 1);
540    ProcedureTestingUtility.waitNoProcedureRunning(procExec);
541    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
542
543    // Split region of the table
544    long procId = procExec.submitProcedure(
545      new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
546
547    // Execute until step 7 of split procedure
548    // NOTE: the 7 (number after SPLIT_TABLE_REGION_UPDATE_META step)
549    MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId, 7, false);
550
551    // Unset Toggle Kill and make ProcExec work correctly
552    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, false);
553    MasterProcedureTestingUtility.restartMasterProcedureExecutor(procExec);
554    ProcedureTestingUtility.waitProcedure(procExec, procId);
555
556    // Even split failed after step 4, it should still works fine
557    verify(tableName, splitRowNum);
558  }
559
560  @Test
561  public void testSplitRegionWhileTakingSnapshot() throws Exception {
562    final TableName tableName = TableName.valueOf(name.getMethodName());
563    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
564
565    RegionInfo[] regions = MasterProcedureTestingUtility.createTable(procExec, tableName, null,
566      columnFamilyName1, columnFamilyName2);
567    int splitRowNum = startRowNum + rowCount / 2;
568    byte[] splitKey = Bytes.toBytes("" + splitRowNum);
569
570    assertTrue("not able to find a splittable region", regions != null);
571    assertTrue("not able to find a splittable region", regions.length == 1);
572    ProcedureTestingUtility.waitNoProcedureRunning(procExec);
573
574    // task snapshot
575    SnapshotDescription snapshot =
576      new SnapshotDescription("SnapshotProcedureTest", tableName, SnapshotType.FLUSH);
577    SnapshotProtos.SnapshotDescription snapshotProto =
578      ProtobufUtil.createHBaseProtosSnapshotDesc(snapshot);
579    snapshotProto = SnapshotDescriptionUtils.validate(snapshotProto,
580      UTIL.getHBaseCluster().getMaster().getConfiguration());
581    long snapshotProcId = procExec.submitProcedure(
582      new TestSnapshotProcedure.DelaySnapshotProcedure(procExec.getEnvironment(), snapshotProto));
583    UTIL.getHBaseCluster().getMaster().getSnapshotManager().registerSnapshotProcedure(snapshotProto,
584      snapshotProcId);
585
586    // collect AM metrics before test
587    collectAssignmentManagerMetrics();
588
589    // Split region of the table
590    long procId = procExec.submitProcedure(
591      new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
592    // Wait the completion
593    ProcedureTestingUtility.waitProcedure(procExec, procId);
594    ProcedureTestingUtility.waitProcedure(procExec, snapshotProcId);
595
596    ProcedureTestingUtility.assertProcFailed(procExec, procId);
597    ProcedureTestingUtility.assertProcNotFailed(procExec, snapshotProcId);
598
599    assertTrue(UTIL.getMiniHBaseCluster().getRegions(tableName).size() == 1);
600    assertTrue(UTIL.countRows(tableName) == 0);
601
602    assertEquals(splitSubmittedCount + 1, splitProcMetrics.getSubmittedCounter().getCount());
603    assertEquals(splitFailedCount + 1, splitProcMetrics.getFailedCounter().getCount());
604  }
605
606  private void deleteData(final TableName tableName, final int startDeleteRowNum)
607    throws IOException, InterruptedException {
608    Table t = UTIL.getConnection().getTable(tableName);
609    final int numRows = rowCount + startRowNum - startDeleteRowNum;
610    Delete d;
611    for (int i = startDeleteRowNum; i <= numRows + startDeleteRowNum; i++) {
612      d = new Delete(Bytes.toBytes("" + i));
613      t.delete(d);
614      if (i % 5 == 0) {
615        UTIL.getAdmin().flush(tableName);
616      }
617    }
618  }
619
620  private void verify(final TableName tableName, final int splitRowNum) throws IOException {
621    List<HRegion> daughters = UTIL.getMiniHBaseCluster().getRegions(tableName);
622    assertTrue(daughters.size() == 2);
623    LOG.info("Row Count = " + UTIL.countRows(tableName));
624    assertTrue(UTIL.countRows(tableName) == rowCount);
625    int startRow;
626    int numRows;
627    for (int i = 0; i < daughters.size(); i++) {
628      if (
629        Bytes.compareTo(daughters.get(i).getRegionInfo().getStartKey(), HConstants.EMPTY_BYTE_ARRAY)
630            == 0
631      ) {
632        startRow = startRowNum; // first region
633        numRows = splitRowNum - startRowNum;
634      } else {
635        startRow = splitRowNum;
636        numRows = rowCount + startRowNum - splitRowNum;
637      }
638      verifyData(daughters.get(i), startRow, numRows, Bytes.toBytes(columnFamilyName1),
639        Bytes.toBytes(columnFamilyName2));
640    }
641  }
642
643  private void verifyData(final HRegion newReg, final int startRow, final int numRows,
644    final byte[]... families) throws IOException {
645    for (int i = startRow; i < startRow + numRows; i++) {
646      byte[] row = Bytes.toBytes("" + i);
647      Get get = new Get(row);
648      Result result = newReg.get(get);
649      Cell[] raw = result.rawCells();
650      assertEquals(families.length, result.size());
651      for (int j = 0; j < families.length; j++) {
652        assertTrue(CellUtil.matchingRows(raw[j], row));
653        assertTrue(CellUtil.matchingFamily(raw[j], families[j]));
654      }
655    }
656  }
657
658  private ProcedureExecutor<MasterProcedureEnv> getMasterProcedureExecutor() {
659    return UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor();
660  }
661
662  private void collectAssignmentManagerMetrics() {
663    splitSubmittedCount = splitProcMetrics.getSubmittedCounter().getCount();
664    splitFailedCount = splitProcMetrics.getFailedCounter().getCount();
665    assignSubmittedCount = assignProcMetrics.getSubmittedCounter().getCount();
666    assignFailedCount = assignProcMetrics.getFailedCounter().getCount();
667    unassignSubmittedCount = unassignProcMetrics.getSubmittedCounter().getCount();
668    unassignFailedCount = unassignProcMetrics.getFailedCounter().getCount();
669  }
670}