001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertNotEquals;
023import static org.junit.Assert.assertNotNull;
024import static org.junit.Assert.assertNotSame;
025import static org.junit.Assert.assertNull;
026import static org.junit.Assert.assertTrue;
027import static org.junit.Assert.fail;
028
029import java.io.IOException;
030import java.lang.reflect.Field;
031import java.util.ArrayList;
032import java.util.Collection;
033import java.util.List;
034import java.util.Map;
035import java.util.Optional;
036import java.util.concurrent.CountDownLatch;
037import java.util.concurrent.ExecutionException;
038import java.util.concurrent.TimeUnit;
039import java.util.concurrent.TimeoutException;
040import java.util.concurrent.atomic.AtomicBoolean;
041import org.apache.commons.io.IOUtils;
042import org.apache.hadoop.conf.Configuration;
043import org.apache.hadoop.fs.FileSystem;
044import org.apache.hadoop.fs.Path;
045import org.apache.hadoop.hbase.Coprocessor;
046import org.apache.hadoop.hbase.CoprocessorEnvironment;
047import org.apache.hadoop.hbase.DoNotRetryIOException;
048import org.apache.hadoop.hbase.HBaseClassTestRule;
049import org.apache.hadoop.hbase.HBaseTestingUtility;
050import org.apache.hadoop.hbase.HColumnDescriptor;
051import org.apache.hadoop.hbase.HConstants;
052import org.apache.hadoop.hbase.HTableDescriptor;
053import org.apache.hadoop.hbase.MasterNotRunningException;
054import org.apache.hadoop.hbase.MetaTableAccessor;
055import org.apache.hadoop.hbase.MiniHBaseCluster;
056import org.apache.hadoop.hbase.ServerName;
057import org.apache.hadoop.hbase.StartMiniClusterOption;
058import org.apache.hadoop.hbase.TableName;
059import org.apache.hadoop.hbase.UnknownRegionException;
060import org.apache.hadoop.hbase.ZooKeeperConnectionException;
061import org.apache.hadoop.hbase.client.Admin;
062import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
063import org.apache.hadoop.hbase.client.Consistency;
064import org.apache.hadoop.hbase.client.Delete;
065import org.apache.hadoop.hbase.client.DoNotRetryRegionException;
066import org.apache.hadoop.hbase.client.Get;
067import org.apache.hadoop.hbase.client.Mutation;
068import org.apache.hadoop.hbase.client.Put;
069import org.apache.hadoop.hbase.client.RegionInfo;
070import org.apache.hadoop.hbase.client.Result;
071import org.apache.hadoop.hbase.client.ResultScanner;
072import org.apache.hadoop.hbase.client.Scan;
073import org.apache.hadoop.hbase.client.Table;
074import org.apache.hadoop.hbase.client.TableDescriptor;
075import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
076import org.apache.hadoop.hbase.client.TestReplicasClient.SlowMeCopro;
077import org.apache.hadoop.hbase.coprocessor.MasterCoprocessor;
078import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
079import org.apache.hadoop.hbase.coprocessor.MasterObserver;
080import org.apache.hadoop.hbase.coprocessor.ObserverContext;
081import org.apache.hadoop.hbase.io.Reference;
082import org.apache.hadoop.hbase.master.HMaster;
083import org.apache.hadoop.hbase.master.LoadBalancer;
084import org.apache.hadoop.hbase.master.MasterRpcServices;
085import org.apache.hadoop.hbase.master.RegionState;
086import org.apache.hadoop.hbase.master.RegionState.State;
087import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
088import org.apache.hadoop.hbase.master.assignment.AssignmentTestingUtil;
089import org.apache.hadoop.hbase.master.assignment.RegionStateNode;
090import org.apache.hadoop.hbase.master.assignment.RegionStates;
091import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
092import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
093import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker;
094import org.apache.hadoop.hbase.regionserver.compactions.CompactionProgress;
095import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController;
096import org.apache.hadoop.hbase.testclassification.LargeTests;
097import org.apache.hadoop.hbase.testclassification.RegionServerTests;
098import org.apache.hadoop.hbase.util.Bytes;
099import org.apache.hadoop.hbase.util.CommonFSUtils;
100import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
101import org.apache.hadoop.hbase.util.FSUtils;
102import org.apache.hadoop.hbase.util.FutureUtils;
103import org.apache.hadoop.hbase.util.HBaseFsck;
104import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
105import org.apache.hadoop.hbase.util.Threads;
106import org.apache.zookeeper.KeeperException;
107import org.apache.zookeeper.KeeperException.NodeExistsException;
108import org.junit.After;
109import org.junit.AfterClass;
110import org.junit.Assert;
111import org.junit.Before;
112import org.junit.BeforeClass;
113import org.junit.ClassRule;
114import org.junit.Rule;
115import org.junit.Test;
116import org.junit.experimental.categories.Category;
117import org.junit.rules.TestName;
118import org.mockito.Mockito;
119import org.slf4j.Logger;
120import org.slf4j.LoggerFactory;
121
122import org.apache.hbase.thirdparty.com.google.protobuf.RpcController;
123import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException;
124
125import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
126import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode;
127import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionRequest;
128import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionResponse;
129
130/**
131 * The below tests are testing split region against a running cluster
132 */
133@Category({RegionServerTests.class, LargeTests.class})
134public class TestSplitTransactionOnCluster {
135
136  @ClassRule
137  public static final HBaseClassTestRule CLASS_RULE =
138      HBaseClassTestRule.forClass(TestSplitTransactionOnCluster.class);
139
140  private static final Logger LOG = LoggerFactory.getLogger(TestSplitTransactionOnCluster.class);
141  private Admin admin = null;
142  private MiniHBaseCluster cluster = null;
143  private static final int NB_SERVERS = 3;
144
145  static final HBaseTestingUtility TESTING_UTIL =
146    new HBaseTestingUtility();
147
148  @Rule
149  public TestName name = new TestName();
150
151  @BeforeClass public static void before() throws Exception {
152    TESTING_UTIL.getConfiguration().setInt(HConstants.HBASE_BALANCER_PERIOD, 60000);
153    StartMiniClusterOption option = StartMiniClusterOption.builder()
154        .masterClass(MyMaster.class).numRegionServers(NB_SERVERS).
155            numDataNodes(NB_SERVERS).build();
156    TESTING_UTIL.startMiniCluster(option);
157  }
158
159  @AfterClass public static void after() throws Exception {
160    TESTING_UTIL.shutdownMiniCluster();
161  }
162
163  @Before public void setup() throws IOException {
164    TESTING_UTIL.ensureSomeNonStoppedRegionServersAvailable(NB_SERVERS);
165    this.admin = TESTING_UTIL.getAdmin();
166    this.cluster = TESTING_UTIL.getMiniHBaseCluster();
167  }
168
169  @After
170  public void tearDown() throws Exception {
171    this.admin.close();
172    for (TableDescriptor htd: this.admin.listTableDescriptors()) {
173      LOG.info("Tear down, remove table=" + htd.getTableName());
174      TESTING_UTIL.deleteTable(htd.getTableName());
175    }
176  }
177
178  private RegionInfo getAndCheckSingleTableRegion(final List<HRegion> regions)
179      throws IOException, InterruptedException {
180    assertEquals(1, regions.size());
181    RegionInfo hri = regions.get(0).getRegionInfo();
182    AssignmentTestingUtil.waitForAssignment(cluster.getMaster().getAssignmentManager(), hri);
183    return hri;
184  }
185
186  private void requestSplitRegion(
187      final HRegionServer rsServer,
188      final Region region,
189      final byte[] midKey) throws IOException {
190    long procId = cluster.getMaster().splitRegion(region.getRegionInfo(), midKey, 0, 0);
191    // wait for the split to complete or get interrupted.  If the split completes successfully,
192    // the procedure will return true; if the split fails, the procedure would throw exception.
193    ProcedureTestingUtility.waitProcedure(cluster.getMaster().getMasterProcedureExecutor(), procId);
194  }
195
196  @Test
197  public void testRITStateForRollback() throws Exception {
198    final TableName tableName = TableName.valueOf(name.getMethodName());
199    final HMaster master = cluster.getMaster();
200    try {
201      // Create table then get the single region for our new table.
202      Table t = createTableAndWait(tableName, Bytes.toBytes("cf"));
203      final List<HRegion> regions = cluster.getRegions(tableName);
204      final RegionInfo hri = getAndCheckSingleTableRegion(regions);
205      insertData(tableName, admin, t);
206      t.close();
207
208      // Turn off balancer so it doesn't cut in and mess up our placements.
209      this.admin.balancerSwitch(false, true);
210      // Turn off the meta scanner so it don't remove parent on us.
211      master.setCatalogJanitorEnabled(false);
212
213      // find a splittable region
214      final HRegion region = findSplittableRegion(regions);
215      assertTrue("not able to find a splittable region", region != null);
216
217      // install master co-processor to fail splits
218      master.getMasterCoprocessorHost().load(
219        FailingSplitMasterObserver.class,
220        Coprocessor.PRIORITY_USER,
221        master.getConfiguration());
222
223      // split async
224      this.admin.splitRegionAsync(region.getRegionInfo().getRegionName(), new byte[] { 42 });
225
226      // we have to wait until the SPLITTING state is seen by the master
227      FailingSplitMasterObserver observer =
228          master.getMasterCoprocessorHost().findCoprocessor(FailingSplitMasterObserver.class);
229      assertNotNull(observer);
230      observer.latch.await();
231
232      LOG.info("Waiting for region to come out of RIT");
233      while (!cluster.getMaster().getAssignmentManager().getRegionStates().isRegionOnline(hri)) {
234        Threads.sleep(100);
235      }
236      assertTrue(cluster.getMaster().getAssignmentManager().getRegionStates().isRegionOnline(hri));
237    } finally {
238      admin.balancerSwitch(true, false);
239      master.setCatalogJanitorEnabled(true);
240      abortAndWaitForMaster();
241      TESTING_UTIL.deleteTable(tableName);
242    }
243  }
244
245  @Test
246  public void testSplitFailedCompactionAndSplit() throws Exception {
247    final TableName tableName = TableName.valueOf(name.getMethodName());
248    // Create table then get the single region for our new table.
249    byte[] cf = Bytes.toBytes("cf");
250    TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName)
251      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(cf)).build();
252    admin.createTable(htd);
253
254    for (int i = 0; cluster.getRegions(tableName).isEmpty() && i < 100; i++) {
255      Thread.sleep(100);
256    }
257    assertEquals(1, cluster.getRegions(tableName).size());
258
259    HRegion region = cluster.getRegions(tableName).get(0);
260    HStore store = region.getStore(cf);
261    int regionServerIndex = cluster.getServerWith(region.getRegionInfo().getRegionName());
262    HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
263
264    Table t = TESTING_UTIL.getConnection().getTable(tableName);
265    // insert data
266    insertData(tableName, admin, t);
267    insertData(tableName, admin, t);
268
269    int fileNum = store.getStorefiles().size();
270    // 0, Compaction Request
271    store.triggerMajorCompaction();
272    Optional<CompactionContext> cc = store.requestCompaction();
273    assertTrue(cc.isPresent());
274    // 1, A timeout split
275    // 1.1 close region
276    assertEquals(2, region.close(false).get(cf).size());
277    // 1.2 rollback and Region initialize again
278    region.initialize();
279
280    // 2, Run Compaction cc
281    assertFalse(region.compact(cc.get(), store, NoLimitThroughputController.INSTANCE));
282    assertTrue(fileNum > store.getStorefiles().size());
283
284    // 3, Split
285    requestSplitRegion(regionServer, region, Bytes.toBytes("row3"));
286    assertEquals(2, cluster.getRegions(tableName).size());
287  }
288
289  @Test
290  public void testSplitCompactWithPriority() throws Exception {
291    final TableName tableName = TableName.valueOf(name.getMethodName());
292    // Create table then get the single region for our new table.
293    byte[] cf = Bytes.toBytes("cf");
294    TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName)
295      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(cf)).build();
296    admin.createTable(htd);
297
298    assertNotEquals("Unable to retrieve regions of the table", -1,
299      TESTING_UTIL.waitFor(10000, () -> cluster.getRegions(tableName).size() == 1));
300
301    HRegion region = cluster.getRegions(tableName).get(0);
302    HStore store = region.getStore(cf);
303    int regionServerIndex = cluster.getServerWith(region.getRegionInfo().getRegionName());
304    HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
305
306    Table table = TESTING_UTIL.getConnection().getTable(tableName);
307    // insert data
308    insertData(tableName, admin, table);
309    insertData(tableName, admin, table, 20);
310    insertData(tableName, admin, table, 40);
311
312    // Compaction Request
313    store.triggerMajorCompaction();
314    Optional<CompactionContext> compactionContext = store.requestCompaction();
315    assertTrue(compactionContext.isPresent());
316    assertFalse(compactionContext.get().getRequest().isAfterSplit());
317    assertEquals(compactionContext.get().getRequest().getPriority(), 13);
318
319    // Split
320    long procId =
321      cluster.getMaster().splitRegion(region.getRegionInfo(), Bytes.toBytes("row4"), 0, 0);
322
323    // wait for the split to complete or get interrupted.  If the split completes successfully,
324    // the procedure will return true; if the split fails, the procedure would throw exception.
325    ProcedureTestingUtility.waitProcedure(cluster.getMaster().getMasterProcedureExecutor(),
326      procId);
327    Thread.sleep(3000);
328    assertNotEquals("Table is not split properly?", -1,
329      TESTING_UTIL.waitFor(3000,
330        () -> cluster.getRegions(tableName).size() == 2));
331    // we have 2 daughter regions
332    HRegion hRegion1 = cluster.getRegions(tableName).get(0);
333    HRegion hRegion2 = cluster.getRegions(tableName).get(1);
334    HStore hStore1 = hRegion1.getStore(cf);
335    HStore hStore2 = hRegion2.getStore(cf);
336
337    // For hStore1 && hStore2, set mock reference to one of the storeFiles
338    StoreFileInfo storeFileInfo1 = new ArrayList<>(hStore1.getStorefiles()).get(0).getFileInfo();
339    StoreFileInfo storeFileInfo2 = new ArrayList<>(hStore2.getStorefiles()).get(0).getFileInfo();
340    Field field = StoreFileInfo.class.getDeclaredField("reference");
341    field.setAccessible(true);
342    field.set(storeFileInfo1, Mockito.mock(Reference.class));
343    field.set(storeFileInfo2, Mockito.mock(Reference.class));
344    hStore1.triggerMajorCompaction();
345    hStore2.triggerMajorCompaction();
346
347    compactionContext = hStore1.requestCompaction();
348    assertTrue(compactionContext.isPresent());
349    // since we set mock reference to one of the storeFiles, we will get isAfterSplit=true &&
350    // highest priority for hStore1's compactionContext
351    assertTrue(compactionContext.get().getRequest().isAfterSplit());
352    assertEquals(compactionContext.get().getRequest().getPriority(), Integer.MIN_VALUE + 1000);
353
354    compactionContext =
355      hStore2.requestCompaction(Integer.MIN_VALUE + 10, CompactionLifeCycleTracker.DUMMY, null);
356    assertTrue(compactionContext.isPresent());
357    // compaction request contains higher priority than default priority of daughter region
358    // compaction (Integer.MIN_VALUE + 1000), hence we are expecting request priority to
359    // be accepted.
360    assertTrue(compactionContext.get().getRequest().isAfterSplit());
361    assertEquals(compactionContext.get().getRequest().getPriority(), Integer.MIN_VALUE + 10);
362    admin.disableTable(tableName);
363    admin.deleteTable(tableName);
364  }
365
366  public static class FailingSplitMasterObserver implements MasterCoprocessor, MasterObserver {
367    volatile CountDownLatch latch;
368
369    @Override
370    public void start(CoprocessorEnvironment e) throws IOException {
371      latch = new CountDownLatch(1);
372    }
373
374    @Override
375    public Optional<MasterObserver> getMasterObserver() {
376      return Optional.of(this);
377    }
378
379    @Override
380    public void preSplitRegionBeforeMETAAction(
381        final ObserverContext<MasterCoprocessorEnvironment> ctx,
382        final byte[] splitKey,
383        final List<Mutation> metaEntries) throws IOException {
384      latch.countDown();
385      throw new IOException("Causing rollback of region split");
386    }
387  }
388
389  @Test
390  public void testSplitRollbackOnRegionClosing() throws Exception {
391    final TableName tableName = TableName.valueOf(name.getMethodName());
392
393    // Create table then get the single region for our new table.
394    Table t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
395    List<HRegion> regions = cluster.getRegions(tableName);
396    RegionInfo hri = getAndCheckSingleTableRegion(regions);
397
398    int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
399
400    RegionStates regionStates = cluster.getMaster().getAssignmentManager().getRegionStates();
401
402    // Turn off balancer so it doesn't cut in and mess up our placements.
403    this.admin.balancerSwitch(false, true);
404    // Turn off the meta scanner so it don't remove parent on us.
405    cluster.getMaster().setCatalogJanitorEnabled(false);
406    try {
407      // Add a bit of load up into the table so splittable.
408      TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY, false);
409      // Get region pre-split.
410      HRegionServer server = cluster.getRegionServer(tableRegionIndex);
411      printOutRegions(server, "Initial regions: ");
412      int regionCount = cluster.getRegions(hri.getTable()).size();
413      regionStates.updateRegionState(hri, RegionState.State.CLOSING);
414
415      // Now try splitting.... should fail.  And each should successfully
416      // rollback.
417      // We don't roll back here anymore. Instead we fail-fast on construction of the
418      // split transaction. Catch the exception instead.
419      try {
420        FutureUtils.get(this.admin.splitRegionAsync(hri.getRegionName()));
421        fail();
422      } catch (DoNotRetryRegionException e) {
423        // Expected
424      }
425      // Wait around a while and assert count of regions remains constant.
426      for (int i = 0; i < 10; i++) {
427        Thread.sleep(100);
428        assertEquals(regionCount, cluster.getRegions(hri.getTable()).size());
429      }
430      regionStates.updateRegionState(hri, State.OPEN);
431      // Now try splitting and it should work.
432      admin.splitRegionAsync(hri.getRegionName()).get(2, TimeUnit.MINUTES);
433      // Get daughters
434      checkAndGetDaughters(tableName);
435      // OK, so split happened after we cleared the blocking node.
436    } finally {
437      admin.balancerSwitch(true, false);
438      cluster.getMaster().setCatalogJanitorEnabled(true);
439      t.close();
440    }
441  }
442
443  /**
444   * Test that if daughter split on us, we won't do the shutdown handler fixup just because we can't
445   * find the immediate daughter of an offlined parent.
446   */
447  @Test
448  public void testShutdownFixupWhenDaughterHasSplit() throws Exception {
449    final TableName tableName = TableName.valueOf(name.getMethodName());
450
451    // Create table then get the single region for our new table.
452    Table t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY); List<HRegion> regions =
453      cluster.getRegions(tableName); RegionInfo hri = getAndCheckSingleTableRegion(regions);
454    int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
455
456    // Turn off balancer so it doesn't cut in and mess up our placements.
457    this.admin.balancerSwitch(false, true);
458    // Turn off the meta scanner so it don't remove parent on us.
459    cluster.getMaster().setCatalogJanitorEnabled(false);
460    try {
461      // Add a bit of load up into the table so splittable.
462      TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY);
463      // Get region pre-split.
464      HRegionServer server = cluster.getRegionServer(tableRegionIndex);
465      printOutRegions(server, "Initial regions: ");
466      // Now split.
467      admin.splitRegionAsync(hri.getRegionName()).get(2, TimeUnit.MINUTES);
468      // Get daughters
469      List<HRegion> daughters = checkAndGetDaughters(tableName);
470      // Now split one of the daughters.
471      HRegion daughterRegion = daughters.get(0);
472      RegionInfo daughter = daughterRegion.getRegionInfo();
473      LOG.info("Daughter we are going to split: " + daughter);
474      clearReferences(daughterRegion);
475      LOG.info("Finished {} references={}", daughterRegion, daughterRegion.hasReferences());
476      admin.splitRegionAsync(daughter.getRegionName()).get(2, TimeUnit.MINUTES);
477      // Get list of daughters
478      daughters = cluster.getRegions(tableName);
479      for (HRegion d: daughters) {
480        LOG.info("Regions before crash: " + d);
481      }
482      // Now crash the server
483      cluster.abortRegionServer(tableRegionIndex);
484      waitUntilRegionServerDead();
485      awaitDaughters(tableName, daughters.size());
486      // Assert daughters are online and ONLY the original daughters -- that
487      // fixup didn't insert one during server shutdown recover.
488      regions = cluster.getRegions(tableName);
489      for (HRegion d: daughters) {
490        LOG.info("Regions after crash: " + d);
491      }
492      if (daughters.size() != regions.size()) {
493        LOG.info("Daughters=" + daughters.size() + ", regions=" + regions.size());
494      }
495      assertEquals(daughters.size(), regions.size());
496      for (HRegion r: regions) {
497        LOG.info("Regions post crash " + r + ", contains=" + daughters.contains(r));
498        assertTrue("Missing region post crash " + r, daughters.contains(r));
499      }
500    } finally {
501      LOG.info("EXITING");
502      admin.balancerSwitch(true, false);
503      cluster.getMaster().setCatalogJanitorEnabled(true);
504      t.close();
505    }
506  }
507
508  private void clearReferences(HRegion region) throws IOException {
509    // Presumption.
510    assertEquals(1, region.getStores().size());
511    HStore store = region.getStores().get(0);
512    while (store.hasReferences()) {
513      // Wait on any current compaction to complete first.
514      CompactionProgress progress = store.getCompactionProgress();
515      if (progress != null && progress.getProgressPct() < 1.0f) {
516        while (progress.getProgressPct() < 1.0f) {
517          LOG.info("Waiting, progress={}", progress.getProgressPct());
518          Threads.sleep(1000);
519        }
520      } else {
521        // Run new compaction. Shoudn't be any others running.
522        region.compact(true);
523      }
524      store.closeAndArchiveCompactedFiles();
525    }
526  }
527
528  @Test
529  public void testSplitShouldNotThrowNPEEvenARegionHasEmptySplitFiles() throws Exception {
530    TableName userTableName = TableName.valueOf(name.getMethodName());
531    TableDescriptor htd = TableDescriptorBuilder.newBuilder(userTableName)
532      .setColumnFamily(ColumnFamilyDescriptorBuilder.of("col")).build();
533    admin.createTable(htd);
534    Table table = TESTING_UTIL.getConnection().getTable(userTableName);
535    try {
536      for (int i = 0; i <= 5; i++) {
537        String row = "row" + i;
538        Put p = new Put(Bytes.toBytes(row));
539        String val = "Val" + i;
540        p.addColumn(Bytes.toBytes("col"), Bytes.toBytes("ql"), Bytes.toBytes(val));
541        table.put(p);
542        admin.flush(userTableName);
543        Delete d = new Delete(Bytes.toBytes(row));
544        // Do a normal delete
545        table.delete(d);
546        admin.flush(userTableName);
547      }
548      admin.majorCompact(userTableName);
549      List<RegionInfo> regionsOfTable =
550          cluster.getMaster().getAssignmentManager().getRegionStates()
551          .getRegionsOfTable(userTableName);
552      assertEquals(1, regionsOfTable.size());
553      RegionInfo hRegionInfo = regionsOfTable.get(0);
554      Put p = new Put(Bytes.toBytes("row6"));
555      p.addColumn(Bytes.toBytes("col"), Bytes.toBytes("ql"), Bytes.toBytes("val"));
556      table.put(p);
557      p = new Put(Bytes.toBytes("row7"));
558      p.addColumn(Bytes.toBytes("col"), Bytes.toBytes("ql"), Bytes.toBytes("val"));
559      table.put(p);
560      p = new Put(Bytes.toBytes("row8"));
561      p.addColumn(Bytes.toBytes("col"), Bytes.toBytes("ql"), Bytes.toBytes("val"));
562      table.put(p);
563      admin.flush(userTableName);
564      admin.splitRegionAsync(hRegionInfo.getRegionName(), Bytes.toBytes("row7"));
565      regionsOfTable = cluster.getMaster()
566          .getAssignmentManager().getRegionStates()
567          .getRegionsOfTable(userTableName);
568
569      while (regionsOfTable.size() != 2) {
570        Thread.sleep(1000);
571        regionsOfTable = cluster.getMaster()
572            .getAssignmentManager().getRegionStates()
573            .getRegionsOfTable(userTableName);
574        LOG.debug("waiting 2 regions to be available, got " + regionsOfTable.size() +
575          ": " + regionsOfTable);
576
577      }
578      Assert.assertEquals(2, regionsOfTable.size());
579
580      Scan s = new Scan();
581      ResultScanner scanner = table.getScanner(s);
582      int mainTableCount = 0;
583      for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
584        mainTableCount++;
585      }
586      Assert.assertEquals(3, mainTableCount);
587    } finally {
588      table.close();
589    }
590  }
591
592  /**
593   * Verifies HBASE-5806. Here the case is that splitting is completed but before the CJ could
594   * remove the parent region the master is killed and restarted.
595   */
596  @Test
597  public void testMasterRestartAtRegionSplitPendingCatalogJanitor()
598      throws IOException, InterruptedException, NodeExistsException, KeeperException,
599      ServiceException, ExecutionException, TimeoutException {
600    final TableName tableName = TableName.valueOf(name.getMethodName());
601    // Create table then get the single region for our new table.
602    try (Table t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY)) {
603      List<HRegion> regions = cluster.getRegions(tableName);
604      RegionInfo hri = getAndCheckSingleTableRegion(regions);
605
606      int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
607
608      // Turn off balancer so it doesn't cut in and mess up our placements.
609      this.admin.balancerSwitch(false, true);
610      // Turn off the meta scanner so it don't remove parent on us.
611      cluster.getMaster().setCatalogJanitorEnabled(false);
612      // Add a bit of load up into the table so splittable.
613      TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY, false);
614      // Get region pre-split.
615      HRegionServer server = cluster.getRegionServer(tableRegionIndex);
616      printOutRegions(server, "Initial regions: ");
617      // Call split.
618      this.admin.splitRegionAsync(hri.getRegionName()).get(2, TimeUnit.MINUTES);
619      List<HRegion> daughters = checkAndGetDaughters(tableName);
620
621      // Before cleanup, get a new master.
622      HMaster master = abortAndWaitForMaster();
623      // Now call compact on the daughters and clean up any references.
624      for (HRegion daughter : daughters) {
625        clearReferences(daughter);
626        assertFalse(daughter.hasReferences());
627      }
628      // BUT calling compact on the daughters is not enough. The CatalogJanitor looks
629      // in the filesystem, and the filesystem content is not same as what the Region
630      // is reading from. Compacted-away files are picked up later by the compacted
631      // file discharger process. It runs infrequently. Make it run so CatalogJanitor
632      // doens't find any references.
633      for (RegionServerThread rst : cluster.getRegionServerThreads()) {
634        boolean oldSetting = rst.getRegionServer().compactedFileDischarger.setUseExecutor(false);
635        rst.getRegionServer().compactedFileDischarger.run();
636        rst.getRegionServer().compactedFileDischarger.setUseExecutor(oldSetting);
637      }
638      cluster.getMaster().setCatalogJanitorEnabled(true);
639      ProcedureTestingUtility.waitAllProcedures(cluster.getMaster().getMasterProcedureExecutor());
640      LOG.info("Starting run of CatalogJanitor");
641      cluster.getMaster().getCatalogJanitor().run();
642      ProcedureTestingUtility.waitAllProcedures(cluster.getMaster().getMasterProcedureExecutor());
643      RegionStates regionStates = master.getAssignmentManager().getRegionStates();
644      ServerName regionServerOfRegion = regionStates.getRegionServerOfRegion(hri);
645      assertEquals(null, regionServerOfRegion);
646    } finally {
647      TESTING_UTIL.getAdmin().balancerSwitch(true, false);
648      cluster.getMaster().setCatalogJanitorEnabled(true);
649    }
650  }
651
652  @Test
653  public void testSplitWithRegionReplicas() throws Exception {
654    final TableName tableName = TableName.valueOf(name.getMethodName());
655    HTableDescriptor htd = TESTING_UTIL.createTableDescriptor(
656      TableName.valueOf(name.getMethodName()), HColumnDescriptor.DEFAULT_MIN_VERSIONS, 3,
657      HConstants.FOREVER, HColumnDescriptor.DEFAULT_KEEP_DELETED);
658    htd.setRegionReplication(2);
659    htd.addCoprocessor(SlowMeCopro.class.getName());
660    // Create table then get the single region for our new table.
661    Table t = TESTING_UTIL.createTable(htd, new byte[][]{Bytes.toBytes("cf")}, null);
662    List<HRegion> oldRegions;
663    do {
664      oldRegions = cluster.getRegions(tableName);
665      Thread.sleep(10);
666    } while (oldRegions.size() != 2);
667    for (HRegion h : oldRegions) LOG.debug("OLDREGION " + h.getRegionInfo());
668    try {
669      int regionServerIndex = cluster.getServerWith(oldRegions.get(0).getRegionInfo()
670        .getRegionName());
671      HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
672      insertData(tableName, admin, t);
673      // Turn off balancer so it doesn't cut in and mess up our placements.
674      admin.balancerSwitch(false, true);
675      // Turn off the meta scanner so it don't remove parent on us.
676      cluster.getMaster().setCatalogJanitorEnabled(false);
677      boolean tableExists = MetaTableAccessor.tableExists(regionServer.getConnection(),
678          tableName);
679      assertEquals("The specified table should be present.", true, tableExists);
680      final HRegion region = findSplittableRegion(oldRegions);
681      regionServerIndex = cluster.getServerWith(region.getRegionInfo().getRegionName());
682      regionServer = cluster.getRegionServer(regionServerIndex);
683      assertTrue("not able to find a splittable region", region != null);
684      try {
685        requestSplitRegion(regionServer, region, Bytes.toBytes("row2"));
686      } catch (IOException e) {
687        e.printStackTrace();
688        fail("Split execution should have succeeded with no exceptions thrown " + e);
689      }
690      //TESTING_UTIL.waitUntilAllRegionsAssigned(tableName);
691      List<HRegion> newRegions;
692      do {
693        newRegions = cluster.getRegions(tableName);
694        for (HRegion h : newRegions) LOG.debug("NEWREGION " + h.getRegionInfo());
695        Thread.sleep(1000);
696      } while ((newRegions.contains(oldRegions.get(0)) || newRegions.contains(oldRegions.get(1)))
697          || newRegions.size() != 4);
698      tableExists = MetaTableAccessor.tableExists(regionServer.getConnection(),
699          tableName);
700      assertEquals("The specified table should be present.", true, tableExists);
701      // exists works on stale and we see the put after the flush
702      byte[] b1 = Bytes.toBytes("row1");
703      Get g = new Get(b1);
704      g.setConsistency(Consistency.STRONG);
705      // The following GET will make a trip to the meta to get the new location of the 1st daughter
706      // In the process it will also get the location of the replica of the daughter (initially
707      // pointing to the parent's replica)
708      Result r = t.get(g);
709      Assert.assertFalse(r.isStale());
710      LOG.info("exists stale after flush done");
711
712      SlowMeCopro.getPrimaryCdl().set(new CountDownLatch(1));
713      g = new Get(b1);
714      g.setConsistency(Consistency.TIMELINE);
715      // This will succeed because in the previous GET we get the location of the replica
716      r = t.get(g);
717      Assert.assertTrue(r.isStale());
718      SlowMeCopro.getPrimaryCdl().get().countDown();
719    } finally {
720      SlowMeCopro.getPrimaryCdl().get().countDown();
721      admin.balancerSwitch(true, false);
722      cluster.getMaster().setCatalogJanitorEnabled(true);
723      t.close();
724    }
725  }
726
727  private void insertData(final TableName tableName, Admin admin, Table t) throws IOException {
728    insertData(tableName, admin, t, 1);
729  }
730
731  private void insertData(TableName tableName, Admin admin, Table t, int i) throws IOException {
732    Put p = new Put(Bytes.toBytes("row" + i));
733    p.addColumn(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("1"));
734    t.put(p);
735    p = new Put(Bytes.toBytes("row" + (i + 1)));
736    p.addColumn(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("2"));
737    t.put(p);
738    p = new Put(Bytes.toBytes("row" + (i + 2)));
739    p.addColumn(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("3"));
740    t.put(p);
741    p = new Put(Bytes.toBytes("row" + (i + 3)));
742    p.addColumn(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("4"));
743    t.put(p);
744    admin.flush(tableName);
745  }
746
747  /**
748   * If a table has regions that have no store files in a region, they should split successfully
749   * into two regions with no store files.
750   */
751  @Test
752  public void testSplitRegionWithNoStoreFiles() throws Exception {
753    final TableName tableName = TableName.valueOf(name.getMethodName());
754    // Create table then get the single region for our new table.
755    createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
756    List<HRegion> regions = cluster.getRegions(tableName);
757    RegionInfo hri = getAndCheckSingleTableRegion(regions);
758    ensureTableRegionNotOnSameServerAsMeta(admin, hri);
759    int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionInfo()
760      .getRegionName());
761    HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
762    // Turn off balancer so it doesn't cut in and mess up our placements.
763    this.admin.balancerSwitch(false, true);
764    // Turn off the meta scanner so it don't remove parent on us.
765    cluster.getMaster().setCatalogJanitorEnabled(false);
766    try {
767      // Precondition: we created a table with no data, no store files.
768      printOutRegions(regionServer, "Initial regions: ");
769      Configuration conf = cluster.getConfiguration();
770      HBaseFsck.debugLsr(conf, new Path("/"));
771      Path rootDir = CommonFSUtils.getRootDir(conf);
772      FileSystem fs = TESTING_UTIL.getDFSCluster().getFileSystem();
773      Map<String, Path> storefiles =
774          FSUtils.getTableStoreFilePathMap(null, fs, rootDir, tableName);
775      assertEquals("Expected nothing but found " + storefiles.toString(), 0, storefiles.size());
776
777      // find a splittable region.  Refresh the regions list
778      regions = cluster.getRegions(tableName);
779      final HRegion region = findSplittableRegion(regions);
780      assertTrue("not able to find a splittable region", region != null);
781
782      // Now split.
783      try {
784        requestSplitRegion(regionServer, region, Bytes.toBytes("row2"));
785      } catch (IOException e) {
786        fail("Split execution should have succeeded with no exceptions thrown");
787      }
788
789      // Postcondition: split the table with no store files into two regions, but still have no
790      // store files
791      List<HRegion> daughters = cluster.getRegions(tableName);
792      assertEquals(2, daughters.size());
793
794      // check dirs
795      HBaseFsck.debugLsr(conf, new Path("/"));
796      Map<String, Path> storefilesAfter =
797          FSUtils.getTableStoreFilePathMap(null, fs, rootDir, tableName);
798      assertEquals("Expected nothing but found " + storefilesAfter.toString(), 0,
799          storefilesAfter.size());
800
801      hri = region.getRegionInfo(); // split parent
802      AssignmentManager am = cluster.getMaster().getAssignmentManager();
803      RegionStates regionStates = am.getRegionStates();
804      long start = EnvironmentEdgeManager.currentTime();
805      while (!regionStates.isRegionInState(hri, State.SPLIT)) {
806        LOG.debug("Waiting for SPLIT state on: " + hri);
807        assertFalse("Timed out in waiting split parent to be in state SPLIT",
808          EnvironmentEdgeManager.currentTime() - start > 60000);
809        Thread.sleep(500);
810      }
811      assertTrue(regionStates.isRegionInState(daughters.get(0).getRegionInfo(), State.OPEN));
812      assertTrue(regionStates.isRegionInState(daughters.get(1).getRegionInfo(), State.OPEN));
813
814      // We should not be able to assign it again
815      try {
816        am.assign(hri);
817      } catch (DoNotRetryIOException e) {
818        // Expected
819      }
820      assertFalse("Split region can't be assigned", regionStates.isRegionInTransition(hri));
821      assertTrue(regionStates.isRegionInState(hri, State.SPLIT));
822
823      // We should not be able to unassign it either
824      try {
825        am.unassign(hri);
826        fail("Should have thrown exception");
827      } catch (DoNotRetryIOException e) {
828        // Expected
829      }
830      assertFalse("Split region can't be unassigned", regionStates.isRegionInTransition(hri));
831      assertTrue(regionStates.isRegionInState(hri, State.SPLIT));
832    } finally {
833      admin.balancerSwitch(true, false);
834      cluster.getMaster().setCatalogJanitorEnabled(true);
835    }
836  }
837
838  @Test
839  public void testStoreFileReferenceCreationWhenSplitPolicySaysToSkipRangeCheck()
840      throws Exception {
841    final TableName tableName = TableName.valueOf(name.getMethodName());
842    try {
843      byte[] cf = Bytes.toBytes("f");
844      byte[] cf1 = Bytes.toBytes("i_f");
845      TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName)
846        .setColumnFamily(ColumnFamilyDescriptorBuilder.of(cf))
847        .setColumnFamily(ColumnFamilyDescriptorBuilder.of(cf1))
848        .setRegionSplitPolicyClassName(CustomSplitPolicy.class.getName()).build();
849      admin.createTable(htd);
850      List<HRegion> regions = awaitTableRegions(tableName);
851      HRegion region = regions.get(0);
852      for(int i = 3;i<9;i++) {
853        Put p = new Put(Bytes.toBytes("row"+i));
854        p.addColumn(cf, Bytes.toBytes("q"), Bytes.toBytes("value" + i));
855        p.addColumn(cf1, Bytes.toBytes("q"), Bytes.toBytes("value" + i));
856        region.put(p);
857      }
858      region.flush(true);
859      HStore store = region.getStore(cf);
860      Collection<HStoreFile> storefiles = store.getStorefiles();
861      assertEquals(1, storefiles.size());
862      assertFalse(region.hasReferences());
863      Path referencePath =
864          region.getRegionFileSystem().splitStoreFile(region.getRegionInfo(), "f",
865            storefiles.iterator().next(), Bytes.toBytes("row1"), false, region.getSplitPolicy());
866      assertNull(referencePath);
867      referencePath =
868          region.getRegionFileSystem().splitStoreFile(region.getRegionInfo(), "i_f",
869            storefiles.iterator().next(), Bytes.toBytes("row1"), false, region.getSplitPolicy());
870      assertNotNull(referencePath);
871    } finally {
872      TESTING_UTIL.deleteTable(tableName);
873    }
874  }
875
876  private HRegion findSplittableRegion(final List<HRegion> regions) throws InterruptedException {
877    for (int i = 0; i < 5; ++i) {
878      for (HRegion r: regions) {
879        if (r.isSplittable() && r.getRegionInfo().getReplicaId() == 0) {
880          return(r);
881        }
882      }
883      Thread.sleep(100);
884    }
885    return null;
886  }
887
888  private List<HRegion> checkAndGetDaughters(TableName tableName) throws InterruptedException {
889    List<HRegion> daughters = null;
890    // try up to 10s
891    for (int i = 0; i < 100; i++) {
892      daughters = cluster.getRegions(tableName);
893      if (daughters.size() >= 2) {
894        break;
895      }
896      Thread.sleep(100);
897    }
898    assertTrue(daughters.size() >= 2);
899    return daughters;
900  }
901
902  private HMaster abortAndWaitForMaster() throws IOException, InterruptedException {
903    cluster.abortMaster(0);
904    cluster.waitOnMaster(0);
905    HMaster master = cluster.startMaster().getMaster();
906    cluster.waitForActiveAndReadyMaster();
907    // reset the connections
908    IOUtils.closeQuietly(admin);
909    TESTING_UTIL.invalidateConnection();
910    admin = TESTING_UTIL.getAdmin();
911    return master;
912  }
913
914  /**
915   * Ensure single table region is not on same server as the single hbase:meta table
916   * region.
917   * @return Index of the server hosting the single table region
918   * @throws UnknownRegionException
919   * @throws MasterNotRunningException
920   * @throws org.apache.hadoop.hbase.ZooKeeperConnectionException
921   * @throws InterruptedException
922   */
923  private int ensureTableRegionNotOnSameServerAsMeta(final Admin admin,
924      final RegionInfo hri)
925  throws IOException, MasterNotRunningException,
926  ZooKeeperConnectionException, InterruptedException {
927    // Now make sure that the table region is not on same server as that hosting
928    // hbase:meta  We don't want hbase:meta replay polluting our test when we later crash
929    // the table region serving server.
930    int metaServerIndex = cluster.getServerWithMeta();
931    boolean tablesOnMaster = LoadBalancer.isTablesOnMaster(TESTING_UTIL.getConfiguration());
932    if (tablesOnMaster) {
933      // Need to check master is supposed to host meta... perhaps it is not.
934      throw new UnsupportedOperationException();
935      // TODO: assertTrue(metaServerIndex == -1); // meta is on master now
936    }
937    HRegionServer metaRegionServer = tablesOnMaster?
938      cluster.getMaster(): cluster.getRegionServer(metaServerIndex);
939    int tableRegionIndex = cluster.getServerWith(hri.getRegionName());
940    assertTrue(tableRegionIndex != -1);
941    HRegionServer tableRegionServer = cluster.getRegionServer(tableRegionIndex);
942    LOG.info("MetaRegionServer=" + metaRegionServer.getServerName() +
943      ", other=" + tableRegionServer.getServerName());
944    if (metaRegionServer.getServerName().equals(tableRegionServer.getServerName())) {
945      HRegionServer hrs = getOtherRegionServer(cluster, metaRegionServer);
946      assertNotNull(hrs);
947      assertNotNull(hri);
948      LOG.info("Moving " + hri.getRegionNameAsString() + " from " +
949        metaRegionServer.getServerName() + " to " +
950        hrs.getServerName() + "; metaServerIndex=" + metaServerIndex);
951      admin.move(hri.getEncodedNameAsBytes(), hrs.getServerName());
952    }
953    // Wait till table region is up on the server that is NOT carrying hbase:meta.
954    for (int i = 0; i < 100; i++) {
955      tableRegionIndex = cluster.getServerWith(hri.getRegionName());
956      if (tableRegionIndex != -1 && tableRegionIndex != metaServerIndex) break;
957      LOG.debug("Waiting on region move off the hbase:meta server; current index " +
958        tableRegionIndex + " and metaServerIndex=" + metaServerIndex);
959      Thread.sleep(100);
960    }
961    assertTrue("Region not moved off hbase:meta server, tableRegionIndex=" + tableRegionIndex,
962      tableRegionIndex != -1 && tableRegionIndex != metaServerIndex);
963    // Verify for sure table region is not on same server as hbase:meta
964    tableRegionIndex = cluster.getServerWith(hri.getRegionName());
965    assertTrue(tableRegionIndex != -1);
966    assertNotSame(metaServerIndex, tableRegionIndex);
967    return tableRegionIndex;
968  }
969
970  /**
971   * Find regionserver other than the one passed.
972   * Can't rely on indexes into list of regionservers since crashed servers
973   * occupy an index.
974   * @param cluster
975   * @param notThisOne
976   * @return A regionserver that is not <code>notThisOne</code> or null if none
977   * found
978   */
979  private HRegionServer getOtherRegionServer(final MiniHBaseCluster cluster,
980      final HRegionServer notThisOne) {
981    for (RegionServerThread rst: cluster.getRegionServerThreads()) {
982      HRegionServer hrs = rst.getRegionServer();
983      if (hrs.getServerName().equals(notThisOne.getServerName())) continue;
984      if (hrs.isStopping() || hrs.isStopped()) continue;
985      return hrs;
986    }
987    return null;
988  }
989
990  private void printOutRegions(final HRegionServer hrs, final String prefix)
991      throws IOException {
992    List<RegionInfo> regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices());
993    for (RegionInfo region: regions) {
994      LOG.info(prefix + region.getRegionNameAsString());
995    }
996  }
997
998  private void waitUntilRegionServerDead() throws InterruptedException, IOException {
999    // Wait until the master processes the RS shutdown
1000    for (int i=0; (cluster.getMaster().getClusterMetrics()
1001        .getLiveServerMetrics().size() > NB_SERVERS
1002        || cluster.getLiveRegionServerThreads().size() > NB_SERVERS) && i<100; i++) {
1003      LOG.info("Waiting on server to go down");
1004      Thread.sleep(100);
1005    }
1006    assertFalse("Waited too long for RS to die",
1007      cluster.getMaster().getClusterMetrics(). getLiveServerMetrics().size() > NB_SERVERS
1008        || cluster.getLiveRegionServerThreads().size() > NB_SERVERS);
1009  }
1010
1011  private void awaitDaughters(TableName tableName, int numDaughters) throws InterruptedException {
1012    // Wait till regions are back on line again.
1013    for (int i = 0; cluster.getRegions(tableName).size() < numDaughters && i < 60; i++) {
1014      LOG.info("Waiting for repair to happen");
1015      Thread.sleep(1000);
1016    }
1017    if (cluster.getRegions(tableName).size() < numDaughters) {
1018      fail("Waiting too long for daughter regions");
1019    }
1020  }
1021
1022  private List<HRegion> awaitTableRegions(final TableName tableName) throws InterruptedException {
1023    List<HRegion> regions = null;
1024    for (int i = 0; i < 100; i++) {
1025      regions = cluster.getRegions(tableName);
1026      if (regions.size() > 0) break;
1027      Thread.sleep(100);
1028    }
1029    return regions;
1030  }
1031
1032  private Table createTableAndWait(TableName tableName, byte[] cf) throws IOException,
1033      InterruptedException {
1034    Table t = TESTING_UTIL.createTable(tableName, cf);
1035    awaitTableRegions(tableName);
1036    assertTrue("Table not online: " + tableName,
1037      cluster.getRegions(tableName).size() != 0);
1038    return t;
1039  }
1040
1041  // Make it public so that JVMClusterUtil can access it.
1042  public static class MyMaster extends HMaster {
1043    public MyMaster(Configuration conf) throws IOException, KeeperException, InterruptedException {
1044      super(conf);
1045    }
1046
1047    @Override
1048    protected RSRpcServices createRpcServices() throws IOException {
1049      return new MyMasterRpcServices(this);
1050    }
1051  }
1052
1053  static class MyMasterRpcServices extends MasterRpcServices {
1054    static AtomicBoolean enabled = new AtomicBoolean(false);
1055
1056    private HMaster myMaster;
1057    public MyMasterRpcServices(HMaster master) throws IOException {
1058      super(master);
1059      myMaster = master;
1060    }
1061
1062    @Override
1063    public ReportRegionStateTransitionResponse reportRegionStateTransition(RpcController c,
1064        ReportRegionStateTransitionRequest req) throws ServiceException {
1065      ReportRegionStateTransitionResponse resp = super.reportRegionStateTransition(c, req);
1066      if (enabled.get() && req.getTransition(0).getTransitionCode().equals(
1067          TransitionCode.READY_TO_SPLIT) && !resp.hasErrorMessage()) {
1068        RegionStates regionStates = myMaster.getAssignmentManager().getRegionStates();
1069        for (RegionStateNode regionState:
1070          regionStates.getRegionsInTransition()) {
1071          /* TODO!!!!
1072          // Find the merging_new region and remove it
1073          if (regionState.isSplittingNew()) {
1074            regionStates.deleteRegion(regionState.getRegion());
1075          }
1076          */
1077        }
1078      }
1079      return resp;
1080    }
1081  }
1082
1083  static class CustomSplitPolicy extends IncreasingToUpperBoundRegionSplitPolicy {
1084
1085    @Override
1086    protected boolean shouldSplit() {
1087      return true;
1088    }
1089
1090    @Override
1091    public boolean skipStoreFileRangeCheck(String familyName) {
1092      if(familyName.startsWith("i_")) {
1093        return true;
1094      } else {
1095        return false;
1096      }
1097    }
1098  }
1099}
1100