001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertNotEquals;
023import static org.junit.Assert.assertNotNull;
024import static org.junit.Assert.assertNotSame;
025import static org.junit.Assert.assertNull;
026import static org.junit.Assert.assertTrue;
027import static org.junit.Assert.fail;
028
029import java.io.IOException;
030import java.lang.reflect.Field;
031import java.util.ArrayList;
032import java.util.Collection;
033import java.util.List;
034import java.util.Map;
035import java.util.Optional;
036import java.util.concurrent.CountDownLatch;
037import java.util.concurrent.ExecutionException;
038import java.util.concurrent.TimeUnit;
039import java.util.concurrent.TimeoutException;
040import java.util.concurrent.atomic.AtomicBoolean;
041import org.apache.hadoop.conf.Configuration;
042import org.apache.hadoop.fs.FileSystem;
043import org.apache.hadoop.fs.Path;
044import org.apache.hadoop.hbase.Coprocessor;
045import org.apache.hadoop.hbase.CoprocessorEnvironment;
046import org.apache.hadoop.hbase.DoNotRetryIOException;
047import org.apache.hadoop.hbase.HBaseClassTestRule;
048import org.apache.hadoop.hbase.HBaseTestingUtil;
049import org.apache.hadoop.hbase.HConstants;
050import org.apache.hadoop.hbase.MasterNotRunningException;
051import org.apache.hadoop.hbase.ServerName;
052import org.apache.hadoop.hbase.SingleProcessHBaseCluster;
053import org.apache.hadoop.hbase.StartTestingClusterOption;
054import org.apache.hadoop.hbase.TableName;
055import org.apache.hadoop.hbase.UnknownRegionException;
056import org.apache.hadoop.hbase.ZooKeeperConnectionException;
057import org.apache.hadoop.hbase.client.Admin;
058import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
059import org.apache.hadoop.hbase.client.Consistency;
060import org.apache.hadoop.hbase.client.Delete;
061import org.apache.hadoop.hbase.client.DoNotRetryRegionException;
062import org.apache.hadoop.hbase.client.Get;
063import org.apache.hadoop.hbase.client.Mutation;
064import org.apache.hadoop.hbase.client.Put;
065import org.apache.hadoop.hbase.client.RegionInfo;
066import org.apache.hadoop.hbase.client.Result;
067import org.apache.hadoop.hbase.client.ResultScanner;
068import org.apache.hadoop.hbase.client.Scan;
069import org.apache.hadoop.hbase.client.Table;
070import org.apache.hadoop.hbase.client.TableDescriptor;
071import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
072import org.apache.hadoop.hbase.client.TestReplicasClient.SlowMeCopro;
073import org.apache.hadoop.hbase.coprocessor.MasterCoprocessor;
074import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
075import org.apache.hadoop.hbase.coprocessor.MasterObserver;
076import org.apache.hadoop.hbase.coprocessor.ObserverContext;
077import org.apache.hadoop.hbase.io.Reference;
078import org.apache.hadoop.hbase.master.HMaster;
079import org.apache.hadoop.hbase.master.MasterRpcServices;
080import org.apache.hadoop.hbase.master.RegionState;
081import org.apache.hadoop.hbase.master.RegionState.State;
082import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
083import org.apache.hadoop.hbase.master.assignment.AssignmentTestingUtil;
084import org.apache.hadoop.hbase.master.assignment.RegionStateNode;
085import org.apache.hadoop.hbase.master.assignment.RegionStates;
086import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
087import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
088import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker;
089import org.apache.hadoop.hbase.regionserver.compactions.CompactionProgress;
090import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController;
091import org.apache.hadoop.hbase.testclassification.LargeTests;
092import org.apache.hadoop.hbase.testclassification.RegionServerTests;
093import org.apache.hadoop.hbase.util.Bytes;
094import org.apache.hadoop.hbase.util.CommonFSUtils;
095import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
096import org.apache.hadoop.hbase.util.FSUtils;
097import org.apache.hadoop.hbase.util.FutureUtils;
098import org.apache.hadoop.hbase.util.HBaseFsck;
099import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
100import org.apache.hadoop.hbase.util.Threads;
101import org.apache.zookeeper.KeeperException;
102import org.apache.zookeeper.KeeperException.NodeExistsException;
103import org.junit.After;
104import org.junit.AfterClass;
105import org.junit.Assert;
106import org.junit.Before;
107import org.junit.BeforeClass;
108import org.junit.ClassRule;
109import org.junit.Rule;
110import org.junit.Test;
111import org.junit.experimental.categories.Category;
112import org.junit.rules.TestName;
113import org.mockito.Mockito;
114import org.slf4j.Logger;
115import org.slf4j.LoggerFactory;
116
117import org.apache.hbase.thirdparty.com.google.common.io.Closeables;
118import org.apache.hbase.thirdparty.com.google.protobuf.RpcController;
119import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException;
120
121import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
122import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode;
123import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionRequest;
124import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionResponse;
125
126/**
127 * The below tests are testing split region against a running cluster
128 */
129@Category({RegionServerTests.class, LargeTests.class})
130public class TestSplitTransactionOnCluster {
131
132  @ClassRule
133  public static final HBaseClassTestRule CLASS_RULE =
134      HBaseClassTestRule.forClass(TestSplitTransactionOnCluster.class);
135
136  private static final Logger LOG = LoggerFactory.getLogger(TestSplitTransactionOnCluster.class);
137  private Admin admin = null;
138  private SingleProcessHBaseCluster cluster = null;
139  private static final int NB_SERVERS = 3;
140
141  static final HBaseTestingUtil TESTING_UTIL =
142    new HBaseTestingUtil();
143
144  @Rule
145  public TestName name = new TestName();
146
147  @BeforeClass public static void before() throws Exception {
148    TESTING_UTIL.getConfiguration().setInt(HConstants.HBASE_BALANCER_PERIOD, 60000);
149    StartTestingClusterOption option = StartTestingClusterOption.builder()
150        .masterClass(MyMaster.class).numRegionServers(NB_SERVERS).
151            numDataNodes(NB_SERVERS).build();
152    TESTING_UTIL.startMiniCluster(option);
153  }
154
155  @AfterClass public static void after() throws Exception {
156    TESTING_UTIL.shutdownMiniCluster();
157  }
158
159  @Before public void setup() throws IOException {
160    TESTING_UTIL.ensureSomeNonStoppedRegionServersAvailable(NB_SERVERS);
161    this.admin = TESTING_UTIL.getAdmin();
162    this.cluster = TESTING_UTIL.getMiniHBaseCluster();
163  }
164
165  @After
166  public void tearDown() throws Exception {
167    this.admin.close();
168    for (TableDescriptor htd: this.admin.listTableDescriptors()) {
169      LOG.info("Tear down, remove table=" + htd.getTableName());
170      TESTING_UTIL.deleteTable(htd.getTableName());
171    }
172  }
173
174  private RegionInfo getAndCheckSingleTableRegion(final List<HRegion> regions)
175      throws IOException, InterruptedException {
176    assertEquals(1, regions.size());
177    RegionInfo hri = regions.get(0).getRegionInfo();
178    AssignmentTestingUtil.waitForAssignment(cluster.getMaster().getAssignmentManager(), hri);
179    return hri;
180  }
181
182  private void requestSplitRegion(
183      final HRegionServer rsServer,
184      final Region region,
185      final byte[] midKey) throws IOException {
186    long procId = cluster.getMaster().splitRegion(region.getRegionInfo(), midKey, 0, 0);
187    // wait for the split to complete or get interrupted.  If the split completes successfully,
188    // the procedure will return true; if the split fails, the procedure would throw exception.
189    ProcedureTestingUtility.waitProcedure(cluster.getMaster().getMasterProcedureExecutor(), procId);
190  }
191
192  @Test
193  public void testRITStateForRollback() throws Exception {
194    final TableName tableName = TableName.valueOf(name.getMethodName());
195    final HMaster master = cluster.getMaster();
196    try {
197      // Create table then get the single region for our new table.
198      Table t = createTableAndWait(tableName, Bytes.toBytes("cf"));
199      final List<HRegion> regions = cluster.getRegions(tableName);
200      final RegionInfo hri = getAndCheckSingleTableRegion(regions);
201      insertData(tableName, admin, t);
202      t.close();
203
204      // Turn off balancer so it doesn't cut in and mess up our placements.
205      this.admin.balancerSwitch(false, true);
206      // Turn off the meta scanner so it don't remove parent on us.
207      master.setCatalogJanitorEnabled(false);
208
209      // find a splittable region
210      final HRegion region = findSplittableRegion(regions);
211      assertTrue("not able to find a splittable region", region != null);
212
213      // install master co-processor to fail splits
214      master.getMasterCoprocessorHost().load(
215        FailingSplitMasterObserver.class,
216        Coprocessor.PRIORITY_USER,
217        master.getConfiguration());
218
219      // split async
220      this.admin.splitRegionAsync(region.getRegionInfo().getRegionName(), new byte[] { 42 });
221
222      // we have to wait until the SPLITTING state is seen by the master
223      FailingSplitMasterObserver observer =
224          master.getMasterCoprocessorHost().findCoprocessor(FailingSplitMasterObserver.class);
225      assertNotNull(observer);
226      observer.latch.await();
227
228      LOG.info("Waiting for region to come out of RIT");
229      while (!cluster.getMaster().getAssignmentManager().getRegionStates().isRegionOnline(hri)) {
230        Threads.sleep(100);
231      }
232      assertTrue(cluster.getMaster().getAssignmentManager().getRegionStates().isRegionOnline(hri));
233    } finally {
234      admin.balancerSwitch(true, false);
235      master.setCatalogJanitorEnabled(true);
236      abortAndWaitForMaster();
237      TESTING_UTIL.deleteTable(tableName);
238    }
239  }
240
241  @Test
242  public void testSplitFailedCompactionAndSplit() throws Exception {
243    final TableName tableName = TableName.valueOf(name.getMethodName());
244    // Create table then get the single region for our new table.
245    byte[] cf = Bytes.toBytes("cf");
246    TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName)
247      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(cf)).build();
248    admin.createTable(htd);
249
250    for (int i = 0; cluster.getRegions(tableName).isEmpty() && i < 100; i++) {
251      Thread.sleep(100);
252    }
253    assertEquals(1, cluster.getRegions(tableName).size());
254
255    HRegion region = cluster.getRegions(tableName).get(0);
256    HStore store = region.getStore(cf);
257    int regionServerIndex = cluster.getServerWith(region.getRegionInfo().getRegionName());
258    HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
259
260    Table t = TESTING_UTIL.getConnection().getTable(tableName);
261    // insert data
262    insertData(tableName, admin, t);
263    insertData(tableName, admin, t);
264
265    int fileNum = store.getStorefiles().size();
266    // 0, Compaction Request
267    store.triggerMajorCompaction();
268    Optional<CompactionContext> cc = store.requestCompaction();
269    assertTrue(cc.isPresent());
270    // 1, A timeout split
271    // 1.1 close region
272    assertEquals(2, region.close(false).get(cf).size());
273    // 1.2 rollback and Region initialize again
274    region.initialize();
275
276    // 2, Run Compaction cc
277    assertFalse(region.compact(cc.get(), store, NoLimitThroughputController.INSTANCE));
278    assertTrue(fileNum > store.getStorefiles().size());
279
280    // 3, Split
281    requestSplitRegion(regionServer, region, Bytes.toBytes("row3"));
282    assertEquals(2, cluster.getRegions(tableName).size());
283  }
284
285  @Test
286  public void testSplitCompactWithPriority() throws Exception {
287    final TableName tableName = TableName.valueOf(name.getMethodName());
288    // Create table then get the single region for our new table.
289    byte[] cf = Bytes.toBytes("cf");
290    TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName)
291      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(cf)).build();
292    admin.createTable(htd);
293
294    assertNotEquals("Unable to retrieve regions of the table", -1,
295      TESTING_UTIL.waitFor(10000, () -> cluster.getRegions(tableName).size() == 1));
296
297    HRegion region = cluster.getRegions(tableName).get(0);
298    HStore store = region.getStore(cf);
299    int regionServerIndex = cluster.getServerWith(region.getRegionInfo().getRegionName());
300    HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
301
302    Table table = TESTING_UTIL.getConnection().getTable(tableName);
303    // insert data
304    insertData(tableName, admin, table);
305    insertData(tableName, admin, table, 20);
306    insertData(tableName, admin, table, 40);
307
308    // Compaction Request
309    store.triggerMajorCompaction();
310    Optional<CompactionContext> compactionContext = store.requestCompaction();
311    assertTrue(compactionContext.isPresent());
312    assertFalse(compactionContext.get().getRequest().isAfterSplit());
313    assertEquals(compactionContext.get().getRequest().getPriority(), 13);
314
315    // Split
316    long procId =
317      cluster.getMaster().splitRegion(region.getRegionInfo(), Bytes.toBytes("row4"), 0, 0);
318
319    // wait for the split to complete or get interrupted.  If the split completes successfully,
320    // the procedure will return true; if the split fails, the procedure would throw exception.
321    ProcedureTestingUtility.waitProcedure(cluster.getMaster().getMasterProcedureExecutor(),
322      procId);
323    Thread.sleep(3000);
324    assertNotEquals("Table is not split properly?", -1,
325      TESTING_UTIL.waitFor(3000,
326        () -> cluster.getRegions(tableName).size() == 2));
327    // we have 2 daughter regions
328    HRegion hRegion1 = cluster.getRegions(tableName).get(0);
329    HRegion hRegion2 = cluster.getRegions(tableName).get(1);
330    HStore hStore1 = hRegion1.getStore(cf);
331    HStore hStore2 = hRegion2.getStore(cf);
332
333    // For hStore1 && hStore2, set mock reference to one of the storeFiles
334    StoreFileInfo storeFileInfo1 = new ArrayList<>(hStore1.getStorefiles()).get(0).getFileInfo();
335    StoreFileInfo storeFileInfo2 = new ArrayList<>(hStore2.getStorefiles()).get(0).getFileInfo();
336    Field field = StoreFileInfo.class.getDeclaredField("reference");
337    field.setAccessible(true);
338    field.set(storeFileInfo1, Mockito.mock(Reference.class));
339    field.set(storeFileInfo2, Mockito.mock(Reference.class));
340    hStore1.triggerMajorCompaction();
341    hStore2.triggerMajorCompaction();
342
343    compactionContext = hStore1.requestCompaction();
344    assertTrue(compactionContext.isPresent());
345    // since we set mock reference to one of the storeFiles, we will get isAfterSplit=true &&
346    // highest priority for hStore1's compactionContext
347    assertTrue(compactionContext.get().getRequest().isAfterSplit());
348    assertEquals(compactionContext.get().getRequest().getPriority(), Integer.MIN_VALUE + 1000);
349
350    compactionContext =
351      hStore2.requestCompaction(Integer.MIN_VALUE + 10, CompactionLifeCycleTracker.DUMMY, null);
352    assertTrue(compactionContext.isPresent());
353    // compaction request contains higher priority than default priority of daughter region
354    // compaction (Integer.MIN_VALUE + 1000), hence we are expecting request priority to
355    // be accepted.
356    assertTrue(compactionContext.get().getRequest().isAfterSplit());
357    assertEquals(compactionContext.get().getRequest().getPriority(), Integer.MIN_VALUE + 10);
358    admin.disableTable(tableName);
359    admin.deleteTable(tableName);
360  }
361
362  public static class FailingSplitMasterObserver implements MasterCoprocessor, MasterObserver {
363    volatile CountDownLatch latch;
364
365    @Override
366    public void start(CoprocessorEnvironment e) throws IOException {
367      latch = new CountDownLatch(1);
368    }
369
370    @Override
371    public Optional<MasterObserver> getMasterObserver() {
372      return Optional.of(this);
373    }
374
375    @Override
376    public void preSplitRegionBeforeMETAAction(
377        final ObserverContext<MasterCoprocessorEnvironment> ctx,
378        final byte[] splitKey,
379        final List<Mutation> metaEntries) throws IOException {
380      latch.countDown();
381      throw new IOException("Causing rollback of region split");
382    }
383  }
384
385  @Test
386  public void testSplitRollbackOnRegionClosing() throws Exception {
387    final TableName tableName = TableName.valueOf(name.getMethodName());
388
389    // Create table then get the single region for our new table.
390    Table t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
391    List<HRegion> regions = cluster.getRegions(tableName);
392    RegionInfo hri = getAndCheckSingleTableRegion(regions);
393
394    int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
395
396    RegionStates regionStates = cluster.getMaster().getAssignmentManager().getRegionStates();
397
398    // Turn off balancer so it doesn't cut in and mess up our placements.
399    this.admin.balancerSwitch(false, true);
400    // Turn off the meta scanner so it don't remove parent on us.
401    cluster.getMaster().setCatalogJanitorEnabled(false);
402    try {
403      // Add a bit of load up into the table so splittable.
404      TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY, false);
405      // Get region pre-split.
406      HRegionServer server = cluster.getRegionServer(tableRegionIndex);
407      printOutRegions(server, "Initial regions: ");
408      int regionCount = cluster.getRegions(hri.getTable()).size();
409      regionStates.updateRegionState(hri, RegionState.State.CLOSING);
410
411      // Now try splitting.... should fail.  And each should successfully
412      // rollback.
413      // We don't roll back here anymore. Instead we fail-fast on construction of the
414      // split transaction. Catch the exception instead.
415      try {
416        FutureUtils.get(this.admin.splitRegionAsync(hri.getRegionName()));
417        fail();
418      } catch (DoNotRetryRegionException e) {
419        // Expected
420      }
421      // Wait around a while and assert count of regions remains constant.
422      for (int i = 0; i < 10; i++) {
423        Thread.sleep(100);
424        assertEquals(regionCount, cluster.getRegions(hri.getTable()).size());
425      }
426      regionStates.updateRegionState(hri, State.OPEN);
427      // Now try splitting and it should work.
428      admin.splitRegionAsync(hri.getRegionName()).get(2, TimeUnit.MINUTES);
429      // Get daughters
430      checkAndGetDaughters(tableName);
431      // OK, so split happened after we cleared the blocking node.
432    } finally {
433      admin.balancerSwitch(true, false);
434      cluster.getMaster().setCatalogJanitorEnabled(true);
435      t.close();
436    }
437  }
438
439  /**
440   * Test that if daughter split on us, we won't do the shutdown handler fixup just because we can't
441   * find the immediate daughter of an offlined parent.
442   */
443  @Test
444  public void testShutdownFixupWhenDaughterHasSplit() throws Exception {
445    final TableName tableName = TableName.valueOf(name.getMethodName());
446
447    // Create table then get the single region for our new table.
448    Table t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY); List<HRegion> regions =
449      cluster.getRegions(tableName); RegionInfo hri = getAndCheckSingleTableRegion(regions);
450    int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
451
452    // Turn off balancer so it doesn't cut in and mess up our placements.
453    this.admin.balancerSwitch(false, true);
454    // Turn off the meta scanner so it don't remove parent on us.
455    cluster.getMaster().setCatalogJanitorEnabled(false);
456    try {
457      // Add a bit of load up into the table so splittable.
458      TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY);
459      // Get region pre-split.
460      HRegionServer server = cluster.getRegionServer(tableRegionIndex);
461      printOutRegions(server, "Initial regions: ");
462      // Now split.
463      admin.splitRegionAsync(hri.getRegionName()).get(2, TimeUnit.MINUTES);
464      // Get daughters
465      List<HRegion> daughters = checkAndGetDaughters(tableName);
466      // Now split one of the daughters.
467      HRegion daughterRegion = daughters.get(0);
468      RegionInfo daughter = daughterRegion.getRegionInfo();
469      LOG.info("Daughter we are going to split: " + daughter);
470      clearReferences(daughterRegion);
471      LOG.info("Finished {} references={}", daughterRegion, daughterRegion.hasReferences());
472      admin.splitRegionAsync(daughter.getRegionName()).get(2, TimeUnit.MINUTES);
473      // Get list of daughters
474      daughters = cluster.getRegions(tableName);
475      for (HRegion d: daughters) {
476        LOG.info("Regions before crash: " + d);
477      }
478      // Now crash the server
479      cluster.abortRegionServer(tableRegionIndex);
480      waitUntilRegionServerDead();
481      awaitDaughters(tableName, daughters.size());
482      // Assert daughters are online and ONLY the original daughters -- that
483      // fixup didn't insert one during server shutdown recover.
484      regions = cluster.getRegions(tableName);
485      for (HRegion d: daughters) {
486        LOG.info("Regions after crash: " + d);
487      }
488      if (daughters.size() != regions.size()) {
489        LOG.info("Daughters=" + daughters.size() + ", regions=" + regions.size());
490      }
491      assertEquals(daughters.size(), regions.size());
492      for (HRegion r: regions) {
493        LOG.info("Regions post crash " + r + ", contains=" + daughters.contains(r));
494        assertTrue("Missing region post crash " + r, daughters.contains(r));
495      }
496    } finally {
497      LOG.info("EXITING");
498      admin.balancerSwitch(true, false);
499      cluster.getMaster().setCatalogJanitorEnabled(true);
500      t.close();
501    }
502  }
503
504  private void clearReferences(HRegion region) throws IOException {
505    // Presumption.
506    assertEquals(1, region.getStores().size());
507    HStore store = region.getStores().get(0);
508    while (store.hasReferences()) {
509      // Wait on any current compaction to complete first.
510      CompactionProgress progress = store.getCompactionProgress();
511      if (progress != null && progress.getProgressPct() < 1.0f) {
512        while (progress.getProgressPct() < 1.0f) {
513          LOG.info("Waiting, progress={}", progress.getProgressPct());
514          Threads.sleep(1000);
515        }
516      } else {
517        // Run new compaction. Shoudn't be any others running.
518        region.compact(true);
519      }
520      store.closeAndArchiveCompactedFiles();
521    }
522  }
523
524  @Test
525  public void testSplitShouldNotThrowNPEEvenARegionHasEmptySplitFiles() throws Exception {
526    TableName userTableName = TableName.valueOf(name.getMethodName());
527    TableDescriptor htd = TableDescriptorBuilder.newBuilder(userTableName)
528      .setColumnFamily(ColumnFamilyDescriptorBuilder.of("col")).build();
529    admin.createTable(htd);
530    Table table = TESTING_UTIL.getConnection().getTable(userTableName);
531    try {
532      for (int i = 0; i <= 5; i++) {
533        String row = "row" + i;
534        Put p = new Put(Bytes.toBytes(row));
535        String val = "Val" + i;
536        p.addColumn(Bytes.toBytes("col"), Bytes.toBytes("ql"), Bytes.toBytes(val));
537        table.put(p);
538        admin.flush(userTableName);
539        Delete d = new Delete(Bytes.toBytes(row));
540        // Do a normal delete
541        table.delete(d);
542        admin.flush(userTableName);
543      }
544      admin.majorCompact(userTableName);
545      List<RegionInfo> regionsOfTable =
546          cluster.getMaster().getAssignmentManager().getRegionStates()
547          .getRegionsOfTable(userTableName);
548      assertEquals(1, regionsOfTable.size());
549      RegionInfo hRegionInfo = regionsOfTable.get(0);
550      Put p = new Put(Bytes.toBytes("row6"));
551      p.addColumn(Bytes.toBytes("col"), Bytes.toBytes("ql"), Bytes.toBytes("val"));
552      table.put(p);
553      p = new Put(Bytes.toBytes("row7"));
554      p.addColumn(Bytes.toBytes("col"), Bytes.toBytes("ql"), Bytes.toBytes("val"));
555      table.put(p);
556      p = new Put(Bytes.toBytes("row8"));
557      p.addColumn(Bytes.toBytes("col"), Bytes.toBytes("ql"), Bytes.toBytes("val"));
558      table.put(p);
559      admin.flush(userTableName);
560      admin.splitRegionAsync(hRegionInfo.getRegionName(), Bytes.toBytes("row7"));
561      regionsOfTable = cluster.getMaster()
562          .getAssignmentManager().getRegionStates()
563          .getRegionsOfTable(userTableName);
564
565      while (regionsOfTable.size() != 2) {
566        Thread.sleep(1000);
567        regionsOfTable = cluster.getMaster()
568            .getAssignmentManager().getRegionStates()
569            .getRegionsOfTable(userTableName);
570        LOG.debug("waiting 2 regions to be available, got " + regionsOfTable.size() +
571          ": " + regionsOfTable);
572
573      }
574      Assert.assertEquals(2, regionsOfTable.size());
575
576      Scan s = new Scan();
577      ResultScanner scanner = table.getScanner(s);
578      int mainTableCount = 0;
579      for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
580        mainTableCount++;
581      }
582      Assert.assertEquals(3, mainTableCount);
583    } finally {
584      table.close();
585    }
586  }
587
588  /**
589   * Verifies HBASE-5806. Here the case is that splitting is completed but before the CJ could
590   * remove the parent region the master is killed and restarted.
591   */
592  @Test
593  public void testMasterRestartAtRegionSplitPendingCatalogJanitor()
594      throws IOException, InterruptedException, NodeExistsException, KeeperException,
595      ServiceException, ExecutionException, TimeoutException {
596    final TableName tableName = TableName.valueOf(name.getMethodName());
597    // Create table then get the single region for our new table.
598    try (Table t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY)) {
599      List<HRegion> regions = cluster.getRegions(tableName);
600      RegionInfo hri = getAndCheckSingleTableRegion(regions);
601
602      int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
603
604      // Turn off balancer so it doesn't cut in and mess up our placements.
605      this.admin.balancerSwitch(false, true);
606      // Turn off the meta scanner so it don't remove parent on us.
607      cluster.getMaster().setCatalogJanitorEnabled(false);
608      // Add a bit of load up into the table so splittable.
609      TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY, false);
610      // Get region pre-split.
611      HRegionServer server = cluster.getRegionServer(tableRegionIndex);
612      printOutRegions(server, "Initial regions: ");
613      // Call split.
614      this.admin.splitRegionAsync(hri.getRegionName()).get(2, TimeUnit.MINUTES);
615      List<HRegion> daughters = checkAndGetDaughters(tableName);
616
617      // Before cleanup, get a new master.
618      HMaster master = abortAndWaitForMaster();
619      // Now call compact on the daughters and clean up any references.
620      for (HRegion daughter : daughters) {
621        clearReferences(daughter);
622        assertFalse(daughter.hasReferences());
623      }
624      // BUT calling compact on the daughters is not enough. The CatalogJanitor looks
625      // in the filesystem, and the filesystem content is not same as what the Region
626      // is reading from. Compacted-away files are picked up later by the compacted
627      // file discharger process. It runs infrequently. Make it run so CatalogJanitor
628      // doens't find any references.
629      for (RegionServerThread rst : cluster.getRegionServerThreads()) {
630        boolean oldSetting = rst.getRegionServer().compactedFileDischarger.setUseExecutor(false);
631        rst.getRegionServer().compactedFileDischarger.run();
632        rst.getRegionServer().compactedFileDischarger.setUseExecutor(oldSetting);
633      }
634      cluster.getMaster().setCatalogJanitorEnabled(true);
635      ProcedureTestingUtility.waitAllProcedures(cluster.getMaster().getMasterProcedureExecutor());
636      LOG.info("Starting run of CatalogJanitor");
637      cluster.getMaster().getCatalogJanitor().run();
638      ProcedureTestingUtility.waitAllProcedures(cluster.getMaster().getMasterProcedureExecutor());
639      RegionStates regionStates = master.getAssignmentManager().getRegionStates();
640      ServerName regionServerOfRegion = regionStates.getRegionServerOfRegion(hri);
641      assertEquals(null, regionServerOfRegion);
642    } finally {
643      TESTING_UTIL.getAdmin().balancerSwitch(true, false);
644      cluster.getMaster().setCatalogJanitorEnabled(true);
645    }
646  }
647
648  @Test
649  public void testSplitWithRegionReplicas() throws Exception {
650    final TableName tableName = TableName.valueOf(name.getMethodName());
651    TableDescriptor htd = TESTING_UTIL
652      .createModifyableTableDescriptor(TableName.valueOf(name.getMethodName()),
653        ColumnFamilyDescriptorBuilder.DEFAULT_MIN_VERSIONS, 3, HConstants.FOREVER,
654        ColumnFamilyDescriptorBuilder.DEFAULT_KEEP_DELETED)
655      .setRegionReplication(2).setCoprocessor(SlowMeCopro.class.getName()).build();
656    // Create table then get the single region for our new table.
657    Table t = TESTING_UTIL.createTable(htd, new byte[][]{Bytes.toBytes("cf")}, null);
658    List<HRegion> oldRegions;
659    do {
660      oldRegions = cluster.getRegions(tableName);
661      Thread.sleep(10);
662    } while (oldRegions.size() != 2);
663    for (HRegion h : oldRegions) LOG.debug("OLDREGION " + h.getRegionInfo());
664    try {
665      int regionServerIndex = cluster.getServerWith(oldRegions.get(0).getRegionInfo()
666        .getRegionName());
667      HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
668      insertData(tableName, admin, t);
669      // Turn off balancer so it doesn't cut in and mess up our placements.
670      admin.balancerSwitch(false, true);
671      // Turn off the meta scanner so it don't remove parent on us.
672      cluster.getMaster().setCatalogJanitorEnabled(false);
673      boolean tableExists = TESTING_UTIL.getAdmin().tableExists(tableName);
674      assertEquals("The specified table should be present.", true, tableExists);
675      final HRegion region = findSplittableRegion(oldRegions);
676      regionServerIndex = cluster.getServerWith(region.getRegionInfo().getRegionName());
677      regionServer = cluster.getRegionServer(regionServerIndex);
678      assertTrue("not able to find a splittable region", region != null);
679      try {
680        requestSplitRegion(regionServer, region, Bytes.toBytes("row2"));
681      } catch (IOException e) {
682        e.printStackTrace();
683        fail("Split execution should have succeeded with no exceptions thrown " + e);
684      }
685      //TESTING_UTIL.waitUntilAllRegionsAssigned(tableName);
686      List<HRegion> newRegions;
687      do {
688        newRegions = cluster.getRegions(tableName);
689        for (HRegion h : newRegions) LOG.debug("NEWREGION " + h.getRegionInfo());
690        Thread.sleep(1000);
691      } while ((newRegions.contains(oldRegions.get(0)) || newRegions.contains(oldRegions.get(1)))
692          || newRegions.size() != 4);
693      tableExists = TESTING_UTIL.getAdmin().tableExists(tableName);
694      assertEquals("The specified table should be present.", true, tableExists);
695      // exists works on stale and we see the put after the flush
696      byte[] b1 = Bytes.toBytes("row1");
697      Get g = new Get(b1);
698      g.setConsistency(Consistency.STRONG);
699      // The following GET will make a trip to the meta to get the new location of the 1st daughter
700      // In the process it will also get the location of the replica of the daughter (initially
701      // pointing to the parent's replica)
702      Result r = t.get(g);
703      Assert.assertFalse(r.isStale());
704      LOG.info("exists stale after flush done");
705
706      SlowMeCopro.getPrimaryCdl().set(new CountDownLatch(1));
707      g = new Get(b1);
708      g.setConsistency(Consistency.TIMELINE);
709      // This will succeed because in the previous GET we get the location of the replica
710      r = t.get(g);
711      Assert.assertTrue(r.isStale());
712      SlowMeCopro.getPrimaryCdl().get().countDown();
713    } finally {
714      SlowMeCopro.getPrimaryCdl().get().countDown();
715      admin.balancerSwitch(true, false);
716      cluster.getMaster().setCatalogJanitorEnabled(true);
717      t.close();
718    }
719  }
720
721  private void insertData(final TableName tableName, Admin admin, Table t) throws IOException {
722    insertData(tableName, admin, t, 1);
723  }
724
725  private void insertData(TableName tableName, Admin admin, Table t, int i) throws IOException {
726    Put p = new Put(Bytes.toBytes("row" + i));
727    p.addColumn(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("1"));
728    t.put(p);
729    p = new Put(Bytes.toBytes("row" + (i + 1)));
730    p.addColumn(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("2"));
731    t.put(p);
732    p = new Put(Bytes.toBytes("row" + (i + 2)));
733    p.addColumn(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("3"));
734    t.put(p);
735    p = new Put(Bytes.toBytes("row" + (i + 3)));
736    p.addColumn(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("4"));
737    t.put(p);
738    admin.flush(tableName);
739  }
740
741  /**
742   * If a table has regions that have no store files in a region, they should split successfully
743   * into two regions with no store files.
744   */
745  @Test
746  public void testSplitRegionWithNoStoreFiles() throws Exception {
747    final TableName tableName = TableName.valueOf(name.getMethodName());
748    // Create table then get the single region for our new table.
749    createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
750    List<HRegion> regions = cluster.getRegions(tableName);
751    RegionInfo hri = getAndCheckSingleTableRegion(regions);
752    ensureTableRegionNotOnSameServerAsMeta(admin, hri);
753    int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionInfo()
754      .getRegionName());
755    HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
756    // Turn off balancer so it doesn't cut in and mess up our placements.
757    this.admin.balancerSwitch(false, true);
758    // Turn off the meta scanner so it don't remove parent on us.
759    cluster.getMaster().setCatalogJanitorEnabled(false);
760    try {
761      // Precondition: we created a table with no data, no store files.
762      printOutRegions(regionServer, "Initial regions: ");
763      Configuration conf = cluster.getConfiguration();
764      HBaseFsck.debugLsr(conf, new Path("/"));
765      Path rootDir = CommonFSUtils.getRootDir(conf);
766      FileSystem fs = TESTING_UTIL.getDFSCluster().getFileSystem();
767      Map<String, Path> storefiles =
768          FSUtils.getTableStoreFilePathMap(null, fs, rootDir, tableName);
769      assertEquals("Expected nothing but found " + storefiles.toString(), 0, storefiles.size());
770
771      // find a splittable region.  Refresh the regions list
772      regions = cluster.getRegions(tableName);
773      final HRegion region = findSplittableRegion(regions);
774      assertTrue("not able to find a splittable region", region != null);
775
776      // Now split.
777      try {
778        requestSplitRegion(regionServer, region, Bytes.toBytes("row2"));
779      } catch (IOException e) {
780        fail("Split execution should have succeeded with no exceptions thrown");
781      }
782
783      // Postcondition: split the table with no store files into two regions, but still have no
784      // store files
785      List<HRegion> daughters = cluster.getRegions(tableName);
786      assertEquals(2, daughters.size());
787
788      // check dirs
789      HBaseFsck.debugLsr(conf, new Path("/"));
790      Map<String, Path> storefilesAfter =
791          FSUtils.getTableStoreFilePathMap(null, fs, rootDir, tableName);
792      assertEquals("Expected nothing but found " + storefilesAfter.toString(), 0,
793          storefilesAfter.size());
794
795      hri = region.getRegionInfo(); // split parent
796      AssignmentManager am = cluster.getMaster().getAssignmentManager();
797      RegionStates regionStates = am.getRegionStates();
798      long start = EnvironmentEdgeManager.currentTime();
799      while (!regionStates.isRegionInState(hri, State.SPLIT)) {
800        LOG.debug("Waiting for SPLIT state on: " + hri);
801        assertFalse("Timed out in waiting split parent to be in state SPLIT",
802          EnvironmentEdgeManager.currentTime() - start > 60000);
803        Thread.sleep(500);
804      }
805      assertTrue(regionStates.isRegionInState(daughters.get(0).getRegionInfo(), State.OPEN));
806      assertTrue(regionStates.isRegionInState(daughters.get(1).getRegionInfo(), State.OPEN));
807
808      // We should not be able to assign it again
809      try {
810        am.assign(hri);
811      } catch (DoNotRetryIOException e) {
812        // Expected
813      }
814      assertFalse("Split region can't be assigned", regionStates.isRegionInTransition(hri));
815      assertTrue(regionStates.isRegionInState(hri, State.SPLIT));
816
817      // We should not be able to unassign it either
818      try {
819        am.unassign(hri);
820        fail("Should have thrown exception");
821      } catch (DoNotRetryIOException e) {
822        // Expected
823      }
824      assertFalse("Split region can't be unassigned", regionStates.isRegionInTransition(hri));
825      assertTrue(regionStates.isRegionInState(hri, State.SPLIT));
826    } finally {
827      admin.balancerSwitch(true, false);
828      cluster.getMaster().setCatalogJanitorEnabled(true);
829    }
830  }
831
832  @Test
833  public void testStoreFileReferenceCreationWhenSplitPolicySaysToSkipRangeCheck()
834      throws Exception {
835    final TableName tableName = TableName.valueOf(name.getMethodName());
836    try {
837      byte[] cf = Bytes.toBytes("f");
838      byte[] cf1 = Bytes.toBytes("i_f");
839      TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName)
840        .setColumnFamily(ColumnFamilyDescriptorBuilder.of(cf))
841        .setColumnFamily(ColumnFamilyDescriptorBuilder.of(cf1))
842        .setRegionSplitPolicyClassName(CustomSplitPolicy.class.getName()).build();
843      admin.createTable(htd);
844      List<HRegion> regions = awaitTableRegions(tableName);
845      HRegion region = regions.get(0);
846      for(int i = 3;i<9;i++) {
847        Put p = new Put(Bytes.toBytes("row"+i));
848        p.addColumn(cf, Bytes.toBytes("q"), Bytes.toBytes("value" + i));
849        p.addColumn(cf1, Bytes.toBytes("q"), Bytes.toBytes("value" + i));
850        region.put(p);
851      }
852      region.flush(true);
853      HStore store = region.getStore(cf);
854      Collection<HStoreFile> storefiles = store.getStorefiles();
855      assertEquals(1, storefiles.size());
856      assertFalse(region.hasReferences());
857      Path referencePath =
858          region.getRegionFileSystem().splitStoreFile(region.getRegionInfo(), "f",
859            storefiles.iterator().next(), Bytes.toBytes("row1"), false, region.getSplitPolicy());
860      assertNull(referencePath);
861      referencePath =
862          region.getRegionFileSystem().splitStoreFile(region.getRegionInfo(), "i_f",
863            storefiles.iterator().next(), Bytes.toBytes("row1"), false, region.getSplitPolicy());
864      assertNotNull(referencePath);
865    } finally {
866      TESTING_UTIL.deleteTable(tableName);
867    }
868  }
869
870  private HRegion findSplittableRegion(final List<HRegion> regions) throws InterruptedException {
871    for (int i = 0; i < 5; ++i) {
872      for (HRegion r: regions) {
873        if (r.isSplittable() && r.getRegionInfo().getReplicaId() == 0) {
874          return(r);
875        }
876      }
877      Thread.sleep(100);
878    }
879    return null;
880  }
881
882  private List<HRegion> checkAndGetDaughters(TableName tableName) throws InterruptedException {
883    List<HRegion> daughters = null;
884    // try up to 10s
885    for (int i = 0; i < 100; i++) {
886      daughters = cluster.getRegions(tableName);
887      if (daughters.size() >= 2) {
888        break;
889      }
890      Thread.sleep(100);
891    }
892    assertTrue(daughters.size() >= 2);
893    return daughters;
894  }
895
896  private HMaster abortAndWaitForMaster() throws IOException, InterruptedException {
897    cluster.abortMaster(0);
898    cluster.waitOnMaster(0);
899    HMaster master = cluster.startMaster().getMaster();
900    cluster.waitForActiveAndReadyMaster();
901    // reset the connections
902    Closeables.close(admin, true);
903    TESTING_UTIL.invalidateConnection();
904    admin = TESTING_UTIL.getAdmin();
905    return master;
906  }
907
908  /**
909   * Ensure single table region is not on same server as the single hbase:meta table
910   * region.
911   * @return Index of the server hosting the single table region
912   * @throws UnknownRegionException
913   * @throws MasterNotRunningException
914   * @throws org.apache.hadoop.hbase.ZooKeeperConnectionException
915   * @throws InterruptedException
916   */
917  private int ensureTableRegionNotOnSameServerAsMeta(final Admin admin,
918      final RegionInfo hri)
919  throws IOException, MasterNotRunningException,
920  ZooKeeperConnectionException, InterruptedException {
921    // Now make sure that the table region is not on same server as that hosting
922    // hbase:meta  We don't want hbase:meta replay polluting our test when we later crash
923    // the table region serving server.
924    int metaServerIndex = cluster.getServerWithMeta();
925    HRegionServer metaRegionServer = cluster.getRegionServer(metaServerIndex);
926    int tableRegionIndex = cluster.getServerWith(hri.getRegionName());
927    assertTrue(tableRegionIndex != -1);
928    HRegionServer tableRegionServer = cluster.getRegionServer(tableRegionIndex);
929    LOG.info("MetaRegionServer=" + metaRegionServer.getServerName() +
930      ", other=" + tableRegionServer.getServerName());
931    if (metaRegionServer.getServerName().equals(tableRegionServer.getServerName())) {
932      HRegionServer hrs = getOtherRegionServer(cluster, metaRegionServer);
933      assertNotNull(hrs);
934      assertNotNull(hri);
935      LOG.info("Moving " + hri.getRegionNameAsString() + " from " +
936        metaRegionServer.getServerName() + " to " +
937        hrs.getServerName() + "; metaServerIndex=" + metaServerIndex);
938      admin.move(hri.getEncodedNameAsBytes(), hrs.getServerName());
939    }
940    // Wait till table region is up on the server that is NOT carrying hbase:meta.
941    for (int i = 0; i < 100; i++) {
942      tableRegionIndex = cluster.getServerWith(hri.getRegionName());
943      if (tableRegionIndex != -1 && tableRegionIndex != metaServerIndex) break;
944      LOG.debug("Waiting on region move off the hbase:meta server; current index " +
945        tableRegionIndex + " and metaServerIndex=" + metaServerIndex);
946      Thread.sleep(100);
947    }
948    assertTrue("Region not moved off hbase:meta server, tableRegionIndex=" + tableRegionIndex,
949      tableRegionIndex != -1 && tableRegionIndex != metaServerIndex);
950    // Verify for sure table region is not on same server as hbase:meta
951    tableRegionIndex = cluster.getServerWith(hri.getRegionName());
952    assertTrue(tableRegionIndex != -1);
953    assertNotSame(metaServerIndex, tableRegionIndex);
954    return tableRegionIndex;
955  }
956
957  /**
958   * Find regionserver other than the one passed.
959   * Can't rely on indexes into list of regionservers since crashed servers
960   * occupy an index.
961   * @param cluster
962   * @param notThisOne
963   * @return A regionserver that is not <code>notThisOne</code> or null if none
964   * found
965   */
966  private HRegionServer getOtherRegionServer(final SingleProcessHBaseCluster cluster,
967      final HRegionServer notThisOne) {
968    for (RegionServerThread rst: cluster.getRegionServerThreads()) {
969      HRegionServer hrs = rst.getRegionServer();
970      if (hrs.getServerName().equals(notThisOne.getServerName())) continue;
971      if (hrs.isStopping() || hrs.isStopped()) continue;
972      return hrs;
973    }
974    return null;
975  }
976
977  private void printOutRegions(final HRegionServer hrs, final String prefix)
978      throws IOException {
979    List<RegionInfo> regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices());
980    for (RegionInfo region: regions) {
981      LOG.info(prefix + region.getRegionNameAsString());
982    }
983  }
984
985  private void waitUntilRegionServerDead() throws InterruptedException, IOException {
986    // Wait until the master processes the RS shutdown
987    for (int i=0; (cluster.getMaster().getClusterMetrics()
988        .getLiveServerMetrics().size() > NB_SERVERS
989        || cluster.getLiveRegionServerThreads().size() > NB_SERVERS) && i<100; i++) {
990      LOG.info("Waiting on server to go down");
991      Thread.sleep(100);
992    }
993    assertFalse("Waited too long for RS to die",
994      cluster.getMaster().getClusterMetrics(). getLiveServerMetrics().size() > NB_SERVERS
995        || cluster.getLiveRegionServerThreads().size() > NB_SERVERS);
996  }
997
998  private void awaitDaughters(TableName tableName, int numDaughters) throws InterruptedException {
999    // Wait till regions are back on line again.
1000    for (int i = 0; cluster.getRegions(tableName).size() < numDaughters && i < 60; i++) {
1001      LOG.info("Waiting for repair to happen");
1002      Thread.sleep(1000);
1003    }
1004    if (cluster.getRegions(tableName).size() < numDaughters) {
1005      fail("Waiting too long for daughter regions");
1006    }
1007  }
1008
1009  private List<HRegion> awaitTableRegions(final TableName tableName) throws InterruptedException {
1010    List<HRegion> regions = null;
1011    for (int i = 0; i < 100; i++) {
1012      regions = cluster.getRegions(tableName);
1013      if (regions.size() > 0) break;
1014      Thread.sleep(100);
1015    }
1016    return regions;
1017  }
1018
1019  private Table createTableAndWait(TableName tableName, byte[] cf) throws IOException,
1020      InterruptedException {
1021    Table t = TESTING_UTIL.createTable(tableName, cf);
1022    awaitTableRegions(tableName);
1023    assertTrue("Table not online: " + tableName,
1024      cluster.getRegions(tableName).size() != 0);
1025    return t;
1026  }
1027
1028  // Make it public so that JVMClusterUtil can access it.
1029  public static class MyMaster extends HMaster {
1030    public MyMaster(Configuration conf) throws IOException, KeeperException, InterruptedException {
1031      super(conf);
1032    }
1033
1034    @Override
1035    protected RSRpcServices createRpcServices() throws IOException {
1036      return new MyMasterRpcServices(this);
1037    }
1038  }
1039
1040  static class MyMasterRpcServices extends MasterRpcServices {
1041    static AtomicBoolean enabled = new AtomicBoolean(false);
1042
1043    private HMaster myMaster;
1044    public MyMasterRpcServices(HMaster master) throws IOException {
1045      super(master);
1046      myMaster = master;
1047    }
1048
1049    @Override
1050    public ReportRegionStateTransitionResponse reportRegionStateTransition(RpcController c,
1051        ReportRegionStateTransitionRequest req) throws ServiceException {
1052      ReportRegionStateTransitionResponse resp = super.reportRegionStateTransition(c, req);
1053      if (enabled.get() && req.getTransition(0).getTransitionCode().equals(
1054          TransitionCode.READY_TO_SPLIT) && !resp.hasErrorMessage()) {
1055        RegionStates regionStates = myMaster.getAssignmentManager().getRegionStates();
1056        for (RegionStateNode regionState:
1057          regionStates.getRegionsInTransition()) {
1058          /* TODO!!!!
1059          // Find the merging_new region and remove it
1060          if (regionState.isSplittingNew()) {
1061            regionStates.deleteRegion(regionState.getRegion());
1062          }
1063          */
1064        }
1065      }
1066      return resp;
1067    }
1068  }
1069
1070  static class CustomSplitPolicy extends IncreasingToUpperBoundRegionSplitPolicy {
1071
1072    @Override
1073    protected boolean shouldSplit() {
1074      return true;
1075    }
1076
1077    @Override
1078    public boolean skipStoreFileRangeCheck(String familyName) {
1079      if(familyName.startsWith("i_")) {
1080        return true;
1081      } else {
1082        return false;
1083      }
1084    }
1085  }
1086}
1087