Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.assignment;
019
020import static org.apache.hadoop.hbase.master.assignment.AssignmentTestingUtil.insertData;
021import static org.junit.jupiter.api.Assertions.assertEquals;
022import static org.junit.jupiter.api.Assertions.assertFalse;
023import static org.junit.jupiter.api.Assertions.assertTrue;
024import static org.junit.jupiter.api.Assertions.fail;
025
026import java.io.IOException;
027import java.util.List;
028import org.apache.hadoop.conf.Configuration;
029import org.apache.hadoop.hbase.Cell;
030import org.apache.hadoop.hbase.CellUtil;
031import org.apache.hadoop.hbase.DoNotRetryIOException;
032import org.apache.hadoop.hbase.HBaseTestingUtil;
033import org.apache.hadoop.hbase.HConstants;
034import org.apache.hadoop.hbase.TableName;
035import org.apache.hadoop.hbase.Waiter;
036import org.apache.hadoop.hbase.client.CompactionState;
037import org.apache.hadoop.hbase.client.Delete;
038import org.apache.hadoop.hbase.client.Get;
039import org.apache.hadoop.hbase.client.RegionInfo;
040import org.apache.hadoop.hbase.client.RegionReplicaUtil;
041import org.apache.hadoop.hbase.client.Result;
042import org.apache.hadoop.hbase.client.SnapshotDescription;
043import org.apache.hadoop.hbase.client.SnapshotType;
044import org.apache.hadoop.hbase.client.Table;
045import org.apache.hadoop.hbase.client.TableDescriptor;
046import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
047import org.apache.hadoop.hbase.master.RegionState;
048import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants;
049import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
050import org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility;
051import org.apache.hadoop.hbase.master.procedure.ModifyTableProcedure;
052import org.apache.hadoop.hbase.master.procedure.TestSnapshotProcedure;
053import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
054import org.apache.hadoop.hbase.procedure2.ProcedureMetrics;
055import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
056import org.apache.hadoop.hbase.regionserver.HRegion;
057import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
058import org.apache.hadoop.hbase.testclassification.LargeTests;
059import org.apache.hadoop.hbase.testclassification.MasterTests;
060import org.apache.hadoop.hbase.util.Bytes;
061import org.junit.jupiter.api.AfterAll;
062import org.junit.jupiter.api.AfterEach;
063import org.junit.jupiter.api.BeforeAll;
064import org.junit.jupiter.api.BeforeEach;
065import org.junit.jupiter.api.Tag;
066import org.junit.jupiter.api.Test;
067import org.junit.jupiter.api.TestInfo;
068import org.slf4j.Logger;
069import org.slf4j.LoggerFactory;
070
071import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
072import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos;
073
074@Tag(MasterTests.TAG)
075@Tag(LargeTests.TAG)
076public class TestSplitTableRegionProcedure {
077
078  private static final Logger LOG = LoggerFactory.getLogger(TestSplitTableRegionProcedure.class);
079
080  protected static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
081
082  private static String columnFamilyName1 = "cf1";
083  private static String columnFamilyName2 = "cf2";
084
085  private static final int startRowNum = 11;
086  private static final int rowCount = 60;
087
088  private AssignmentManager am;
089
090  private ProcedureMetrics splitProcMetrics;
091  private ProcedureMetrics assignProcMetrics;
092  private ProcedureMetrics unassignProcMetrics;
093
094  private long splitSubmittedCount = 0;
095  private long splitFailedCount = 0;
096  private long assignSubmittedCount = 0;
097  private long assignFailedCount = 0;
098  private long unassignSubmittedCount = 0;
099  private long unassignFailedCount = 0;
100  private String testMethodName;
101
102  @BeforeEach
103  public void setTestMethod(TestInfo testInfo) {
104    testMethodName = testInfo.getTestMethod().get().getName();
105  }
106
107  private static void setupConf(Configuration conf) {
108    conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1);
109    conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 0);
110    conf.set("hbase.coprocessor.region.classes",
111      RegionServerHostingReplicaSlowOpenCoprocessor.class.getName());
112  }
113
114  @BeforeAll
115  public static void setupCluster() throws Exception {
116    setupConf(UTIL.getConfiguration());
117    UTIL.startMiniCluster(3);
118  }
119
120  @AfterAll
121  public static void cleanupTest() throws Exception {
122    try {
123      UTIL.shutdownMiniCluster();
124    } catch (Exception e) {
125      LOG.warn("failure shutting down cluster", e);
126    }
127  }
128
129  @BeforeEach
130  public void setup() throws Exception {
131    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(getMasterProcedureExecutor(), false);
132
133    // Turn off balancer so it doesn't cut in and mess up our placements.
134    UTIL.getAdmin().balancerSwitch(false, true);
135    // Turn off the meta scanner so it don't remove parent on us.
136    UTIL.getHBaseCluster().getMaster().setCatalogJanitorEnabled(false);
137    am = UTIL.getHBaseCluster().getMaster().getAssignmentManager();
138    splitProcMetrics = am.getAssignmentManagerMetrics().getSplitProcMetrics();
139    assignProcMetrics = am.getAssignmentManagerMetrics().getAssignProcMetrics();
140    unassignProcMetrics = am.getAssignmentManagerMetrics().getUnassignProcMetrics();
141  }
142
143  @AfterEach
144  public void tearDown() throws Exception {
145    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(getMasterProcedureExecutor(), false);
146    for (TableDescriptor htd : UTIL.getAdmin().listTableDescriptors()) {
147      UTIL.deleteTable(htd.getTableName());
148    }
149  }
150
151  @Test
152  public void testRollbackForSplitTableRegionWithReplica() throws Exception {
153    final TableName tableName = TableName.valueOf(testMethodName);
154    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
155
156    RegionServerHostingReplicaSlowOpenCoprocessor.slowDownReplicaOpen = true;
157    RegionInfo[] regions =
158      MasterProcedureTestingUtility.createTable(procExec, tableName, null, columnFamilyName1);
159
160    TableDescriptor td = TableDescriptorBuilder.newBuilder(UTIL.getAdmin().getDescriptor(tableName))
161      .setRegionReplication(2).build();
162    procExec.submitProcedure(new ModifyTableProcedure(procExec.getEnvironment(), td));
163
164    // wait until the primary region is online.
165    HBaseTestingUtil.await(2000, () -> {
166      try {
167        AssignmentManager am = UTIL.getHBaseCluster().getMaster().getAssignmentManager();
168        if (am == null) return false;
169        if (am.getRegionStates().getRegionState(regions[0]).isOpened()) {
170          return true;
171        }
172        return false;
173      } catch (Exception e) {
174        throw new RuntimeException(e);
175      }
176    });
177
178    // Split region of the table, it will fail and rollback as replica parent region
179    // is still at OPENING state.
180    long procId = procExec.submitProcedure(new SplitTableRegionProcedure(procExec.getEnvironment(),
181      regions[0], HConstants.CATALOG_FAMILY));
182    // Wait for the completion.
183    ProcedureTestingUtility.waitProcedure(procExec, procId);
184
185    // Let replica parent region open.
186    RegionServerHostingReplicaSlowOpenCoprocessor.slowDownReplicaOpen = false;
187
188    // wait until the replica region is online.
189    HBaseTestingUtil.await(2000, () -> {
190      try {
191        AssignmentManager am = UTIL.getHBaseCluster().getMaster().getAssignmentManager();
192        if (am == null) return false;
193        RegionInfo replicaRegion = RegionReplicaUtil.getRegionInfoForReplica(regions[0], 1);
194        if (
195          am.getRegionStates().getRegionState(replicaRegion) != null
196            && am.getRegionStates().getRegionState(replicaRegion).isOpened()
197        ) {
198          return true;
199        }
200        return false;
201      } catch (Exception e) {
202        throw new RuntimeException(e);
203      }
204    });
205
206    ProcedureTestingUtility.assertProcFailed(procExec, procId);
207    // There should not be any active OpenRegionProcedure
208    procExec.getActiveProceduresNoCopy()
209      .forEach(p -> assertFalse(p instanceof OpenRegionProcedure));
210
211    // Check that procedure rollback reverted parent region state to OPEN
212    AssignmentManager am = UTIL.getHBaseCluster().getMaster().getAssignmentManager();
213    RegionStateNode regionStateNode = am.getRegionStates().getRegionStateNode(regions[0]);
214    assertEquals(RegionState.State.OPEN, regionStateNode.getState());
215  }
216
217  @Test
218  public void testSplitTableRegion() throws Exception {
219    final TableName tableName = TableName.valueOf(testMethodName);
220    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
221
222    RegionInfo[] regions = MasterProcedureTestingUtility.createTable(procExec, tableName, null,
223      columnFamilyName1, columnFamilyName2);
224    insertData(UTIL, tableName, rowCount, startRowNum, columnFamilyName1, columnFamilyName2);
225    int splitRowNum = startRowNum + rowCount / 2;
226    byte[] splitKey = Bytes.toBytes("" + splitRowNum);
227
228    assertTrue(regions != null, "not able to find a splittable region");
229    assertTrue(regions.length == 1, "not able to find a splittable region");
230
231    // collect AM metrics before test
232    collectAssignmentManagerMetrics();
233
234    // Split region of the table
235    long procId = procExec.submitProcedure(
236      new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
237    // Wait the completion
238    ProcedureTestingUtility.waitProcedure(procExec, procId);
239    ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
240
241    verify(tableName, splitRowNum);
242
243    assertEquals(splitSubmittedCount + 1, splitProcMetrics.getSubmittedCounter().getCount());
244    assertEquals(splitFailedCount, splitProcMetrics.getFailedCounter().getCount());
245    assertEquals(assignSubmittedCount + 2, assignProcMetrics.getSubmittedCounter().getCount());
246    assertEquals(assignFailedCount, assignProcMetrics.getFailedCounter().getCount());
247    assertEquals(unassignSubmittedCount + 1, unassignProcMetrics.getSubmittedCounter().getCount());
248    assertEquals(unassignFailedCount, unassignProcMetrics.getFailedCounter().getCount());
249  }
250
251  @Test
252  public void testSplitTableRegionNoStoreFile() throws Exception {
253    final TableName tableName = TableName.valueOf(testMethodName);
254    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
255
256    RegionInfo[] regions = MasterProcedureTestingUtility.createTable(procExec, tableName, null,
257      columnFamilyName1, columnFamilyName2);
258    int splitRowNum = startRowNum + rowCount / 2;
259    byte[] splitKey = Bytes.toBytes("" + splitRowNum);
260
261    assertTrue(regions != null, "not able to find a splittable region");
262    assertTrue(regions.length == 1, "not able to find a splittable region");
263
264    // collect AM metrics before test
265    collectAssignmentManagerMetrics();
266
267    // Split region of the table
268    long procId = procExec.submitProcedure(
269      new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
270    // Wait the completion
271    ProcedureTestingUtility.waitProcedure(procExec, procId);
272    ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
273
274    assertTrue(UTIL.getMiniHBaseCluster().getRegions(tableName).size() == 2);
275    assertTrue(UTIL.countRows(tableName) == 0);
276
277    assertEquals(splitSubmittedCount + 1, splitProcMetrics.getSubmittedCounter().getCount());
278    assertEquals(splitFailedCount, splitProcMetrics.getFailedCounter().getCount());
279  }
280
281  @Test
282  public void testSplitTableRegionUnevenDaughter() throws Exception {
283    final TableName tableName = TableName.valueOf(testMethodName);
284    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
285
286    RegionInfo[] regions = MasterProcedureTestingUtility.createTable(procExec, tableName, null,
287      columnFamilyName1, columnFamilyName2);
288    insertData(UTIL, tableName, rowCount, startRowNum, columnFamilyName1, columnFamilyName2);
289    // Split to two daughters with one of them only has 1 row
290    int splitRowNum = startRowNum + rowCount / 4;
291    byte[] splitKey = Bytes.toBytes("" + splitRowNum);
292
293    assertTrue(regions != null, "not able to find a splittable region");
294    assertTrue(regions.length == 1, "not able to find a splittable region");
295
296    // collect AM metrics before test
297    collectAssignmentManagerMetrics();
298
299    // Split region of the table
300    long procId = procExec.submitProcedure(
301      new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
302    // Wait the completion
303    ProcedureTestingUtility.waitProcedure(procExec, procId);
304    ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
305
306    verify(tableName, splitRowNum);
307
308    assertEquals(splitSubmittedCount + 1, splitProcMetrics.getSubmittedCounter().getCount());
309    assertEquals(splitFailedCount, splitProcMetrics.getFailedCounter().getCount());
310  }
311
312  @Test
313  public void testSplitTableRegionEmptyDaughter() throws Exception {
314    final TableName tableName = TableName.valueOf(testMethodName);
315    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
316
317    RegionInfo[] regions = MasterProcedureTestingUtility.createTable(procExec, tableName, null,
318      columnFamilyName1, columnFamilyName2);
319    insertData(UTIL, tableName, rowCount, startRowNum, columnFamilyName1, columnFamilyName2);
320    // Split to two daughters with one of them only has 1 row
321    int splitRowNum = startRowNum + rowCount;
322    byte[] splitKey = Bytes.toBytes("" + splitRowNum);
323
324    assertTrue(regions != null, "not able to find a splittable region");
325    assertTrue(regions.length == 1, "not able to find a splittable region");
326
327    // collect AM metrics before test
328    collectAssignmentManagerMetrics();
329
330    // Split region of the table
331    long procId = procExec.submitProcedure(
332      new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
333    // Wait the completion
334    ProcedureTestingUtility.waitProcedure(procExec, procId);
335    ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
336
337    // Make sure one daughter has 0 rows.
338    List<HRegion> daughters = UTIL.getMiniHBaseCluster().getRegions(tableName);
339    assertTrue(daughters.size() == 2);
340    assertTrue(UTIL.countRows(tableName) == rowCount);
341    assertTrue(UTIL.countRows(daughters.get(0)) == 0 || UTIL.countRows(daughters.get(1)) == 0);
342
343    assertEquals(splitSubmittedCount + 1, splitProcMetrics.getSubmittedCounter().getCount());
344    assertEquals(splitFailedCount, splitProcMetrics.getFailedCounter().getCount());
345  }
346
347  @Test
348  public void testSplitTableRegionDeletedRowsDaughter() throws Exception {
349    final TableName tableName = TableName.valueOf(testMethodName);
350    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
351
352    RegionInfo[] regions = MasterProcedureTestingUtility.createTable(procExec, tableName, null,
353      columnFamilyName1, columnFamilyName2);
354    insertData(UTIL, tableName, rowCount, startRowNum, columnFamilyName1, columnFamilyName2);
355    // Split to two daughters with one of them only has 1 row
356    int splitRowNum = rowCount;
357    deleteData(tableName, splitRowNum);
358    byte[] splitKey = Bytes.toBytes("" + splitRowNum);
359
360    assertTrue(regions != null, "not able to find a splittable region");
361    assertTrue(regions.length == 1, "not able to find a splittable region");
362
363    // collect AM metrics before test
364    collectAssignmentManagerMetrics();
365
366    // Split region of the table
367    long procId = procExec.submitProcedure(
368      new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
369    // Wait the completion
370    ProcedureTestingUtility.waitProcedure(procExec, procId);
371    ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
372
373    UTIL.getAdmin().majorCompact(tableName);
374    // waiting for the major compaction to complete
375    UTIL.waitFor(6000, new Waiter.Predicate<IOException>() {
376      @Override
377      public boolean evaluate() throws IOException {
378        return UTIL.getAdmin().getCompactionState(tableName) == CompactionState.NONE;
379      }
380    });
381
382    // Make sure one daughter has 0 rows.
383    List<HRegion> daughters = UTIL.getMiniHBaseCluster().getRegions(tableName);
384    assertTrue(daughters.size() == 2);
385    final int currentRowCount = splitRowNum - startRowNum;
386    assertTrue(UTIL.countRows(tableName) == currentRowCount);
387    assertTrue(UTIL.countRows(daughters.get(0)) == 0 || UTIL.countRows(daughters.get(1)) == 0);
388
389    assertEquals(splitSubmittedCount + 1, splitProcMetrics.getSubmittedCounter().getCount());
390    assertEquals(splitFailedCount, splitProcMetrics.getFailedCounter().getCount());
391  }
392
393  @Test
394  public void testInvalidSplitKey() throws Exception {
395    final TableName tableName = TableName.valueOf(testMethodName);
396    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
397
398    RegionInfo[] regions = MasterProcedureTestingUtility.createTable(procExec, tableName, null,
399      columnFamilyName1, columnFamilyName2);
400    insertData(UTIL, tableName, rowCount, startRowNum, columnFamilyName1, columnFamilyName2);
401
402    assertTrue(regions != null, "not able to find a splittable region");
403    assertTrue(regions.length == 1, "not able to find a splittable region");
404
405    // collect AM metrics before test
406    collectAssignmentManagerMetrics();
407
408    // Split region of the table with null split key
409    try {
410      long procId1 = procExec.submitProcedure(
411        new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], null));
412      ProcedureTestingUtility.waitProcedure(procExec, procId1);
413      fail("unexpected procedure start with invalid split-key");
414    } catch (DoNotRetryIOException e) {
415      LOG.debug("Expected Split procedure construction failure: " + e.getMessage());
416    }
417
418    assertEquals(splitSubmittedCount, splitProcMetrics.getSubmittedCounter().getCount());
419    assertEquals(splitFailedCount, splitProcMetrics.getFailedCounter().getCount());
420  }
421
422  @Test
423  public void testRollbackAndDoubleExecution() throws Exception {
424    final TableName tableName = TableName.valueOf(testMethodName);
425    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
426
427    RegionInfo[] regions = MasterProcedureTestingUtility.createTable(procExec, tableName, null,
428      columnFamilyName1, columnFamilyName2);
429    insertData(UTIL, tableName, rowCount, startRowNum, columnFamilyName1, columnFamilyName2);
430    int splitRowNum = startRowNum + rowCount / 2;
431    byte[] splitKey = Bytes.toBytes("" + splitRowNum);
432
433    assertTrue(regions != null, "not able to find a splittable region");
434    assertTrue(regions.length == 1, "not able to find a splittable region");
435    ProcedureTestingUtility.waitNoProcedureRunning(procExec);
436    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
437
438    // collect AM metrics before test
439    collectAssignmentManagerMetrics();
440
441    // Split region of the table
442    long procId = procExec.submitProcedure(
443      new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
444
445    // Failing before SPLIT_TABLE_REGION_UPDATE_META we should trigger the
446    // rollback
447    // NOTE: the 7 (number of SPLIT_TABLE_REGION_UPDATE_META step) is
448    // hardcoded, so you have to look at this test at least once when you add a new step.
449    int lastStep = 7;
450    MasterProcedureTestingUtility.testRollbackAndDoubleExecution(procExec, procId, lastStep, true);
451    // check that we have only 1 region
452    assertEquals(1, UTIL.getAdmin().getRegions(tableName).size());
453    UTIL.waitUntilAllRegionsAssigned(tableName);
454    List<HRegion> newRegions = UTIL.getMiniHBaseCluster().getRegions(tableName);
455    assertEquals(1, newRegions.size());
456    verifyData(newRegions.get(0), startRowNum, rowCount, Bytes.toBytes(columnFamilyName1),
457      Bytes.toBytes(columnFamilyName2));
458
459    assertEquals(splitSubmittedCount + 1, splitProcMetrics.getSubmittedCounter().getCount());
460    assertEquals(splitFailedCount + 1, splitProcMetrics.getFailedCounter().getCount());
461  }
462
463  @Test
464  public void testRecoveryAndDoubleExecution() throws Exception {
465    final TableName tableName = TableName.valueOf(testMethodName);
466    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
467
468    RegionInfo[] regions = MasterProcedureTestingUtility.createTable(procExec, tableName, null,
469      columnFamilyName1, columnFamilyName2);
470    insertData(UTIL, tableName, rowCount, startRowNum, columnFamilyName1, columnFamilyName2);
471    int splitRowNum = startRowNum + rowCount / 2;
472    byte[] splitKey = Bytes.toBytes("" + splitRowNum);
473
474    assertTrue(regions != null, "not able to find a splittable region");
475    assertTrue(regions.length == 1, "not able to find a splittable region");
476    ProcedureTestingUtility.waitNoProcedureRunning(procExec);
477    ProcedureTestingUtility.setKillIfHasParent(procExec, false);
478    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
479
480    // collect AM metrics before test
481    collectAssignmentManagerMetrics();
482
483    // Split region of the table
484    long procId = procExec.submitProcedure(
485      new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
486
487    // Restart the executor and execute the step twice
488    MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId);
489    ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
490
491    verify(tableName, splitRowNum);
492
493    assertEquals(splitSubmittedCount + 1, splitProcMetrics.getSubmittedCounter().getCount());
494    assertEquals(splitFailedCount, splitProcMetrics.getFailedCounter().getCount());
495  }
496
497  @Test
498  public void testSplitWithoutPONR() throws Exception {
499    final TableName tableName = TableName.valueOf(testMethodName);
500    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
501
502    RegionInfo[] regions = MasterProcedureTestingUtility.createTable(procExec, tableName, null,
503      columnFamilyName1, columnFamilyName2);
504    insertData(UTIL, tableName, rowCount, startRowNum, columnFamilyName1, columnFamilyName2);
505    int splitRowNum = startRowNum + rowCount / 2;
506    byte[] splitKey = Bytes.toBytes("" + splitRowNum);
507
508    assertTrue(regions != null, "not able to find a splittable region");
509    assertTrue(regions.length == 1, "not able to find a splittable region");
510    ProcedureTestingUtility.waitNoProcedureRunning(procExec);
511    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
512
513    // Split region of the table
514    long procId = procExec.submitProcedure(
515      new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
516
517    // Execute until step 7 of split procedure
518    // NOTE: the 7 (number after SPLIT_TABLE_REGION_UPDATE_META step)
519    MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId, 7, false);
520
521    // Unset Toggle Kill and make ProcExec work correctly
522    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, false);
523    MasterProcedureTestingUtility.restartMasterProcedureExecutor(procExec);
524    ProcedureTestingUtility.waitProcedure(procExec, procId);
525
526    // Even split failed after step 4, it should still works fine
527    verify(tableName, splitRowNum);
528  }
529
530  @Test
531  public void testSplitRegionWhileTakingSnapshot() throws Exception {
532    final TableName tableName = TableName.valueOf(testMethodName);
533    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
534
535    RegionInfo[] regions = MasterProcedureTestingUtility.createTable(procExec, tableName, null,
536      columnFamilyName1, columnFamilyName2);
537    int splitRowNum = startRowNum + rowCount / 2;
538    byte[] splitKey = Bytes.toBytes("" + splitRowNum);
539
540    assertTrue(regions != null, "not able to find a splittable region");
541    assertTrue(regions.length == 1, "not able to find a splittable region");
542    ProcedureTestingUtility.waitNoProcedureRunning(procExec);
543
544    // task snapshot
545    SnapshotDescription snapshot =
546      new SnapshotDescription("SnapshotProcedureTest", tableName, SnapshotType.FLUSH);
547    SnapshotProtos.SnapshotDescription snapshotProto =
548      ProtobufUtil.createHBaseProtosSnapshotDesc(snapshot);
549    snapshotProto = SnapshotDescriptionUtils.validate(snapshotProto,
550      UTIL.getHBaseCluster().getMaster().getConfiguration());
551    long snapshotProcId = procExec.submitProcedure(
552      new TestSnapshotProcedure.DelaySnapshotProcedure(procExec.getEnvironment(), snapshotProto));
553    UTIL.getHBaseCluster().getMaster().getSnapshotManager().registerSnapshotProcedure(snapshotProto,
554      snapshotProcId);
555
556    // collect AM metrics before test
557    collectAssignmentManagerMetrics();
558
559    // Split region of the table
560    long procId = procExec.submitProcedure(
561      new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
562    // Wait the completion
563    ProcedureTestingUtility.waitProcedure(procExec, procId);
564    ProcedureTestingUtility.waitProcedure(procExec, snapshotProcId);
565
566    ProcedureTestingUtility.assertProcFailed(procExec, procId);
567    ProcedureTestingUtility.assertProcNotFailed(procExec, snapshotProcId);
568
569    assertTrue(UTIL.getMiniHBaseCluster().getRegions(tableName).size() == 1);
570    assertTrue(UTIL.countRows(tableName) == 0);
571
572    assertEquals(splitSubmittedCount + 1, splitProcMetrics.getSubmittedCounter().getCount());
573    assertEquals(splitFailedCount + 1, splitProcMetrics.getFailedCounter().getCount());
574  }
575
576  @Test
577  public void testSplitDetectsModifyTableProcedure() throws Exception {
578    final TableName tableName = TableName.valueOf(testMethodName);
579    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
580
581    RegionInfo[] regions =
582      MasterProcedureTestingUtility.createTable(procExec, tableName, null, columnFamilyName1);
583    RegionServerHostingReplicaSlowOpenCoprocessor.slowDownReplicaOpen = true;
584    TableDescriptor td = TableDescriptorBuilder.newBuilder(UTIL.getAdmin().getDescriptor(tableName))
585      .setRegionReplication(2).build();
586    long modifyProcId =
587      procExec.submitProcedure(new ModifyTableProcedure(procExec.getEnvironment(), td));
588
589    // Split region of the table, the SplitTableRegionProcedure will fail because there is a
590    // ModifyTableProcedure in progress
591    SplitTableRegionProcedure splitProcedure = new SplitTableRegionProcedure(
592      procExec.getEnvironment(), regions[0], HConstants.CATALOG_FAMILY);
593    long splitProcId = procExec.submitProcedure(splitProcedure);
594    ProcedureTestingUtility.waitProcedure(procExec, splitProcId);
595    ProcedureTestingUtility.assertProcFailed(procExec, splitProcId);
596
597    RegionServerHostingReplicaSlowOpenCoprocessor.slowDownReplicaOpen = false;
598    ProcedureTestingUtility.waitProcedure(procExec, modifyProcId);
599    ProcedureTestingUtility.assertProcNotFailed(procExec, modifyProcId);
600  }
601
602  private void deleteData(final TableName tableName, final int startDeleteRowNum)
603    throws IOException, InterruptedException {
604    Table t = UTIL.getConnection().getTable(tableName);
605    final int numRows = rowCount + startRowNum - startDeleteRowNum;
606    Delete d;
607    for (int i = startDeleteRowNum; i <= numRows + startDeleteRowNum; i++) {
608      d = new Delete(Bytes.toBytes("" + i));
609      t.delete(d);
610      if (i % 5 == 0) {
611        UTIL.getAdmin().flush(tableName);
612      }
613    }
614  }
615
616  private void verify(final TableName tableName, final int splitRowNum) throws IOException {
617    List<HRegion> daughters = UTIL.getMiniHBaseCluster().getRegions(tableName);
618    assertTrue(daughters.size() == 2);
619    LOG.info("Row Count = " + UTIL.countRows(tableName));
620    assertTrue(UTIL.countRows(tableName) == rowCount);
621    int startRow;
622    int numRows;
623    for (int i = 0; i < daughters.size(); i++) {
624      if (
625        Bytes.compareTo(daughters.get(i).getRegionInfo().getStartKey(), HConstants.EMPTY_BYTE_ARRAY)
626            == 0
627      ) {
628        startRow = startRowNum; // first region
629        numRows = splitRowNum - startRowNum;
630      } else {
631        startRow = splitRowNum;
632        numRows = rowCount + startRowNum - splitRowNum;
633      }
634      verifyData(daughters.get(i), startRow, numRows, Bytes.toBytes(columnFamilyName1),
635        Bytes.toBytes(columnFamilyName2));
636    }
637  }
638
639  private void verifyData(final HRegion newReg, final int startRow, final int numRows,
640    final byte[]... families) throws IOException {
641    for (int i = startRow; i < startRow + numRows; i++) {
642      byte[] row = Bytes.toBytes("" + i);
643      Get get = new Get(row);
644      Result result = newReg.get(get);
645      Cell[] raw = result.rawCells();
646      assertEquals(families.length, result.size());
647      for (int j = 0; j < families.length; j++) {
648        assertTrue(CellUtil.matchingRows(raw[j], row));
649        assertTrue(CellUtil.matchingFamily(raw[j], families[j]));
650      }
651    }
652  }
653
654  private ProcedureExecutor<MasterProcedureEnv> getMasterProcedureExecutor() {
655    return UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor();
656  }
657
658  private void collectAssignmentManagerMetrics() {
659    splitSubmittedCount = splitProcMetrics.getSubmittedCounter().getCount();
660    splitFailedCount = splitProcMetrics.getFailedCounter().getCount();
661    assignSubmittedCount = assignProcMetrics.getSubmittedCounter().getCount();
662    assignFailedCount = assignProcMetrics.getFailedCounter().getCount();
663    unassignSubmittedCount = unassignProcMetrics.getSubmittedCounter().getCount();
664    unassignFailedCount = unassignProcMetrics.getFailedCounter().getCount();
665  }
666}