001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertNotNull;
023import static org.junit.Assert.assertNotSame;
024import static org.junit.Assert.assertNull;
025import static org.junit.Assert.assertTrue;
026import static org.junit.Assert.fail;
027
028import java.io.IOException;
029import java.util.Collection;
030import java.util.List;
031import java.util.Map;
032import java.util.Optional;
033import java.util.concurrent.CountDownLatch;
034import java.util.concurrent.ExecutionException;
035import java.util.concurrent.TimeUnit;
036import java.util.concurrent.TimeoutException;
037import java.util.concurrent.atomic.AtomicBoolean;
038import org.apache.hadoop.conf.Configuration;
039import org.apache.hadoop.fs.FileSystem;
040import org.apache.hadoop.fs.Path;
041import org.apache.hadoop.hbase.Coprocessor;
042import org.apache.hadoop.hbase.CoprocessorEnvironment;
043import org.apache.hadoop.hbase.DoNotRetryIOException;
044import org.apache.hadoop.hbase.HBaseClassTestRule;
045import org.apache.hadoop.hbase.HBaseTestingUtility;
046import org.apache.hadoop.hbase.HConstants;
047import org.apache.hadoop.hbase.HTableDescriptor;
048import org.apache.hadoop.hbase.MasterNotRunningException;
049import org.apache.hadoop.hbase.MetaTableAccessor;
050import org.apache.hadoop.hbase.MiniHBaseCluster;
051import org.apache.hadoop.hbase.ServerName;
052import org.apache.hadoop.hbase.StartMiniClusterOption;
053import org.apache.hadoop.hbase.TableName;
054import org.apache.hadoop.hbase.UnknownRegionException;
055import org.apache.hadoop.hbase.ZooKeeperConnectionException;
056import org.apache.hadoop.hbase.client.Admin;
057import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
058import org.apache.hadoop.hbase.client.Consistency;
059import org.apache.hadoop.hbase.client.Delete;
060import org.apache.hadoop.hbase.client.DoNotRetryRegionException;
061import org.apache.hadoop.hbase.client.Get;
062import org.apache.hadoop.hbase.client.Mutation;
063import org.apache.hadoop.hbase.client.Put;
064import org.apache.hadoop.hbase.client.RegionInfo;
065import org.apache.hadoop.hbase.client.Result;
066import org.apache.hadoop.hbase.client.ResultScanner;
067import org.apache.hadoop.hbase.client.Scan;
068import org.apache.hadoop.hbase.client.Table;
069import org.apache.hadoop.hbase.client.TableDescriptor;
070import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
071import org.apache.hadoop.hbase.client.TestReplicasClient.SlowMeCopro;
072import org.apache.hadoop.hbase.coprocessor.MasterCoprocessor;
073import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
074import org.apache.hadoop.hbase.coprocessor.MasterObserver;
075import org.apache.hadoop.hbase.coprocessor.ObserverContext;
076import org.apache.hadoop.hbase.master.HMaster;
077import org.apache.hadoop.hbase.master.LoadBalancer;
078import org.apache.hadoop.hbase.master.MasterRpcServices;
079import org.apache.hadoop.hbase.master.RegionState;
080import org.apache.hadoop.hbase.master.RegionState.State;
081import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
082import org.apache.hadoop.hbase.master.assignment.AssignmentTestingUtil;
083import org.apache.hadoop.hbase.master.assignment.RegionStateNode;
084import org.apache.hadoop.hbase.master.assignment.RegionStates;
085import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
086import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
087import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController;
088import org.apache.hadoop.hbase.testclassification.LargeTests;
089import org.apache.hadoop.hbase.testclassification.RegionServerTests;
090import org.apache.hadoop.hbase.util.Bytes;
091import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
092import org.apache.hadoop.hbase.util.FSUtils;
093import org.apache.hadoop.hbase.util.HBaseFsck;
094import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
095import org.apache.hadoop.hbase.util.Threads;
096import org.apache.zookeeper.KeeperException;
097import org.apache.zookeeper.KeeperException.NodeExistsException;
098import org.junit.After;
099import org.junit.AfterClass;
100import org.junit.Assert;
101import org.junit.Before;
102import org.junit.BeforeClass;
103import org.junit.ClassRule;
104import org.junit.Rule;
105import org.junit.Test;
106import org.junit.experimental.categories.Category;
107import org.junit.rules.TestName;
108import org.slf4j.Logger;
109import org.slf4j.LoggerFactory;
110
111import org.apache.hbase.thirdparty.com.google.protobuf.RpcController;
112import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException;
113
114import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
115import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode;
116import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionRequest;
117import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionResponse;
118
119/**
120 * The below tests are testing split region against a running cluster
121 */
122@Category({RegionServerTests.class, LargeTests.class})
123public class TestSplitTransactionOnCluster {
124
125  @ClassRule
126  public static final HBaseClassTestRule CLASS_RULE =
127      HBaseClassTestRule.forClass(TestSplitTransactionOnCluster.class);
128
129  private static final Logger LOG = LoggerFactory.getLogger(TestSplitTransactionOnCluster.class);
130  private Admin admin = null;
131  private MiniHBaseCluster cluster = null;
132  private static final int NB_SERVERS = 3;
133
134  static final HBaseTestingUtility TESTING_UTIL =
135    new HBaseTestingUtility();
136
137  @Rule
138  public TestName name = new TestName();
139
140  @BeforeClass public static void before() throws Exception {
141    TESTING_UTIL.getConfiguration().setInt(HConstants.HBASE_BALANCER_PERIOD, 60000);
142    StartMiniClusterOption option = StartMiniClusterOption.builder()
143        .masterClass(MyMaster.class).numRegionServers(NB_SERVERS).numDataNodes(NB_SERVERS).build();
144    TESTING_UTIL.startMiniCluster(option);
145  }
146
147  @AfterClass public static void after() throws Exception {
148    TESTING_UTIL.shutdownMiniCluster();
149  }
150
151  @Before public void setup() throws IOException {
152    TESTING_UTIL.ensureSomeNonStoppedRegionServersAvailable(NB_SERVERS);
153    this.admin = TESTING_UTIL.getAdmin();
154    this.cluster = TESTING_UTIL.getMiniHBaseCluster();
155  }
156
157  @After
158  public void tearDown() throws Exception {
159    this.admin.close();
160    for (TableDescriptor htd: this.admin.listTableDescriptors()) {
161      LOG.info("Tear down, remove table=" + htd.getTableName());
162      TESTING_UTIL.deleteTable(htd.getTableName());
163    }
164  }
165
166  private RegionInfo getAndCheckSingleTableRegion(final List<HRegion> regions)
167      throws IOException, InterruptedException {
168    assertEquals(1, regions.size());
169    RegionInfo hri = regions.get(0).getRegionInfo();
170    AssignmentTestingUtil.waitForAssignment(cluster.getMaster().getAssignmentManager(), hri);
171    return hri;
172  }
173
174  private void requestSplitRegion(
175      final HRegionServer rsServer,
176      final Region region,
177      final byte[] midKey) throws IOException {
178    long procId = cluster.getMaster().splitRegion(region.getRegionInfo(), midKey, 0, 0);
179    // wait for the split to complete or get interrupted.  If the split completes successfully,
180    // the procedure will return true; if the split fails, the procedure would throw exception.
181    ProcedureTestingUtility.waitProcedure(cluster.getMaster().getMasterProcedureExecutor(), procId);
182  }
183
184  @Test
185  public void testRITStateForRollback() throws Exception {
186    final TableName tableName = TableName.valueOf(name.getMethodName());
187    final HMaster master = cluster.getMaster();
188    try {
189      // Create table then get the single region for our new table.
190      Table t = createTableAndWait(tableName, Bytes.toBytes("cf"));
191      final List<HRegion> regions = cluster.getRegions(tableName);
192      final RegionInfo hri = getAndCheckSingleTableRegion(regions);
193      insertData(tableName, admin, t);
194      t.close();
195
196      // Turn off balancer so it doesn't cut in and mess up our placements.
197      this.admin.balancerSwitch(false, true);
198      // Turn off the meta scanner so it don't remove parent on us.
199      master.setCatalogJanitorEnabled(false);
200
201      // find a splittable region
202      final HRegion region = findSplittableRegion(regions);
203      assertTrue("not able to find a splittable region", region != null);
204
205      // install master co-processor to fail splits
206      master.getMasterCoprocessorHost().load(
207        FailingSplitMasterObserver.class,
208        Coprocessor.PRIORITY_USER,
209        master.getConfiguration());
210
211      // split async
212      this.admin.splitRegionAsync(region.getRegionInfo().getRegionName(), new byte[] { 42 });
213
214      // we have to wait until the SPLITTING state is seen by the master
215      FailingSplitMasterObserver observer =
216          master.getMasterCoprocessorHost().findCoprocessor(FailingSplitMasterObserver.class);
217      assertNotNull(observer);
218      observer.latch.await();
219
220      LOG.info("Waiting for region to come out of RIT");
221      while (!cluster.getMaster().getAssignmentManager().getRegionStates().isRegionOnline(hri)) {
222        Threads.sleep(100);
223      }
224      assertTrue(cluster.getMaster().getAssignmentManager().getRegionStates().isRegionOnline(hri));
225    } finally {
226      admin.balancerSwitch(true, false);
227      master.setCatalogJanitorEnabled(true);
228      abortAndWaitForMaster();
229      TESTING_UTIL.deleteTable(tableName);
230    }
231  }
232
233  @Test
234  public void testSplitFailedCompactionAndSplit() throws Exception {
235    final TableName tableName = TableName.valueOf(name.getMethodName());
236    // Create table then get the single region for our new table.
237    byte[] cf = Bytes.toBytes("cf");
238    TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName)
239      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(cf)).build();
240    admin.createTable(htd);
241
242    for (int i = 0; cluster.getRegions(tableName).isEmpty() && i < 100; i++) {
243      Thread.sleep(100);
244    }
245    assertEquals(1, cluster.getRegions(tableName).size());
246
247    HRegion region = cluster.getRegions(tableName).get(0);
248    HStore store = region.getStore(cf);
249    int regionServerIndex = cluster.getServerWith(region.getRegionInfo().getRegionName());
250    HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
251
252    Table t = TESTING_UTIL.getConnection().getTable(tableName);
253    // insert data
254    insertData(tableName, admin, t);
255    insertData(tableName, admin, t);
256
257    int fileNum = store.getStorefiles().size();
258    // 0, Compaction Request
259    store.triggerMajorCompaction();
260    Optional<CompactionContext> cc = store.requestCompaction();
261    assertTrue(cc.isPresent());
262    // 1, A timeout split
263    // 1.1 close region
264    assertEquals(2, region.close(false).get(cf).size());
265    // 1.2 rollback and Region initialize again
266    region.initialize();
267
268    // 2, Run Compaction cc
269    assertFalse(region.compact(cc.get(), store, NoLimitThroughputController.INSTANCE));
270    assertTrue(fileNum > store.getStorefiles().size());
271
272    // 3, Split
273    requestSplitRegion(regionServer, region, Bytes.toBytes("row3"));
274    assertEquals(2, cluster.getRegions(tableName).size());
275  }
276
277  public static class FailingSplitMasterObserver implements MasterCoprocessor, MasterObserver {
278    volatile CountDownLatch latch;
279
280    @Override
281    public void start(CoprocessorEnvironment e) throws IOException {
282      latch = new CountDownLatch(1);
283    }
284
285    @Override
286    public Optional<MasterObserver> getMasterObserver() {
287      return Optional.of(this);
288    }
289
290    @Override
291    public void preSplitRegionBeforeMETAAction(
292        final ObserverContext<MasterCoprocessorEnvironment> ctx,
293        final byte[] splitKey,
294        final List<Mutation> metaEntries) throws IOException {
295      latch.countDown();
296      throw new IOException("Causing rollback of region split");
297    }
298  }
299
300  @Test
301  public void testSplitRollbackOnRegionClosing() throws Exception {
302    final TableName tableName = TableName.valueOf(name.getMethodName());
303
304    // Create table then get the single region for our new table.
305    Table t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
306    List<HRegion> regions = cluster.getRegions(tableName);
307    RegionInfo hri = getAndCheckSingleTableRegion(regions);
308
309    int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
310
311    RegionStates regionStates = cluster.getMaster().getAssignmentManager().getRegionStates();
312
313    // Turn off balancer so it doesn't cut in and mess up our placements.
314    this.admin.balancerSwitch(false, true);
315    // Turn off the meta scanner so it don't remove parent on us.
316    cluster.getMaster().setCatalogJanitorEnabled(false);
317    try {
318      // Add a bit of load up into the table so splittable.
319      TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY, false);
320      // Get region pre-split.
321      HRegionServer server = cluster.getRegionServer(tableRegionIndex);
322      printOutRegions(server, "Initial regions: ");
323      int regionCount = cluster.getRegions(hri.getTable()).size();
324      regionStates.updateRegionState(hri, RegionState.State.CLOSING);
325
326      // Now try splitting.... should fail.  And each should successfully
327      // rollback.
328      // We don't roll back here anymore. Instead we fail-fast on construction of the
329      // split transaction. Catch the exception instead.
330      try {
331        this.admin.splitRegionAsync(hri.getRegionName());
332        fail();
333      } catch (DoNotRetryRegionException e) {
334        // Expected
335      }
336      // Wait around a while and assert count of regions remains constant.
337      for (int i = 0; i < 10; i++) {
338        Thread.sleep(100);
339        assertEquals(regionCount, cluster.getRegions(hri.getTable()).size());
340      }
341      regionStates.updateRegionState(hri, State.OPEN);
342      // Now try splitting and it should work.
343      admin.splitRegionAsync(hri.getRegionName()).get(2, TimeUnit.MINUTES);
344      // Get daughters
345      checkAndGetDaughters(tableName);
346      // OK, so split happened after we cleared the blocking node.
347    } finally {
348      admin.balancerSwitch(true, false);
349      cluster.getMaster().setCatalogJanitorEnabled(true);
350      t.close();
351    }
352  }
353
354  /**
355   * Test that if daughter split on us, we won't do the shutdown handler fixup just because we can't
356   * find the immediate daughter of an offlined parent.
357   */
358  @Test
359  public void testShutdownFixupWhenDaughterHasSplit() throws Exception {
360    final TableName tableName = TableName.valueOf(name.getMethodName());
361
362    // Create table then get the single region for our new table.
363    Table t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
364    List<HRegion> regions = cluster.getRegions(tableName);
365    RegionInfo hri = getAndCheckSingleTableRegion(regions);
366
367    int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
368
369    // Turn off balancer so it doesn't cut in and mess up our placements.
370    this.admin.balancerSwitch(false, true);
371    // Turn off the meta scanner so it don't remove parent on us.
372    cluster.getMaster().setCatalogJanitorEnabled(false);
373    try {
374      // Add a bit of load up into the table so splittable.
375      TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY);
376      // Get region pre-split.
377      HRegionServer server = cluster.getRegionServer(tableRegionIndex);
378      printOutRegions(server, "Initial regions: ");
379      // Now split.
380      admin.splitRegionAsync(hri.getRegionName()).get(2, TimeUnit.MINUTES);
381      // Get daughters
382      List<HRegion> daughters = checkAndGetDaughters(tableName);
383      HRegion daughterRegion = daughters.get(0);
384      // Now split one of the daughters.
385      RegionInfo daughter = daughterRegion.getRegionInfo();
386      LOG.info("Daughter we are going to split: " + daughter);
387      // Compact first to ensure we have cleaned up references -- else the split
388      // will fail.
389      daughterRegion.compact(true);
390      daughterRegion.getStores().get(0).closeAndArchiveCompactedFiles();
391      for (int i = 0; i < 100; i++) {
392        if (!daughterRegion.hasReferences()) {
393          break;
394        }
395        Threads.sleep(100);
396      }
397      assertFalse("Waiting for reference to be compacted", daughterRegion.hasReferences());
398      LOG.info("Daughter hri before split (has been compacted): " + daughter);
399      admin.splitRegionAsync(daughter.getRegionName()).get(2, TimeUnit.MINUTES);
400      // Get list of daughters
401      daughters = cluster.getRegions(tableName);
402      for (HRegion d: daughters) {
403        LOG.info("Regions before crash: " + d);
404      }
405      // Now crash the server
406      cluster.abortRegionServer(tableRegionIndex);
407      waitUntilRegionServerDead();
408      awaitDaughters(tableName, daughters.size());
409      // Assert daughters are online and ONLY the original daughters -- that
410      // fixup didn't insert one during server shutdown recover.
411      regions = cluster.getRegions(tableName);
412      for (HRegion d: daughters) {
413        LOG.info("Regions after crash: " + d);
414      }
415      if (daughters.size() != regions.size()) {
416        LOG.info("Daughters=" + daughters.size() + ", regions=" + regions.size());
417      }
418      assertEquals(daughters.size(), regions.size());
419      for (HRegion r: regions) {
420        LOG.info("Regions post crash " + r + ", contains=" + daughters.contains(r));
421        assertTrue("Missing region post crash " + r, daughters.contains(r));
422      }
423    } finally {
424      LOG.info("EXITING");
425      admin.balancerSwitch(true, false);
426      cluster.getMaster().setCatalogJanitorEnabled(true);
427      t.close();
428    }
429  }
430
431  @Test
432  public void testSplitShouldNotThrowNPEEvenARegionHasEmptySplitFiles() throws Exception {
433    TableName userTableName = TableName.valueOf(name.getMethodName());
434    TableDescriptor htd = TableDescriptorBuilder.newBuilder(userTableName)
435      .setColumnFamily(ColumnFamilyDescriptorBuilder.of("col")).build();
436    admin.createTable(htd);
437    Table table = TESTING_UTIL.getConnection().getTable(userTableName);
438    try {
439      for (int i = 0; i <= 5; i++) {
440        String row = "row" + i;
441        Put p = new Put(row.getBytes());
442        String val = "Val" + i;
443        p.addColumn("col".getBytes(), "ql".getBytes(), val.getBytes());
444        table.put(p);
445        admin.flush(userTableName);
446        Delete d = new Delete(row.getBytes());
447        // Do a normal delete
448        table.delete(d);
449        admin.flush(userTableName);
450      }
451      admin.majorCompact(userTableName);
452      List<RegionInfo> regionsOfTable =
453          cluster.getMaster().getAssignmentManager().getRegionStates()
454          .getRegionsOfTable(userTableName);
455      assertEquals(1, regionsOfTable.size());
456      RegionInfo hRegionInfo = regionsOfTable.get(0);
457      Put p = new Put("row6".getBytes());
458      p.addColumn("col".getBytes(), "ql".getBytes(), "val".getBytes());
459      table.put(p);
460      p = new Put("row7".getBytes());
461      p.addColumn("col".getBytes(), "ql".getBytes(), "val".getBytes());
462      table.put(p);
463      p = new Put("row8".getBytes());
464      p.addColumn("col".getBytes(), "ql".getBytes(), "val".getBytes());
465      table.put(p);
466      admin.flush(userTableName);
467      admin.splitRegionAsync(hRegionInfo.getRegionName(), "row7".getBytes());
468      regionsOfTable = cluster.getMaster()
469          .getAssignmentManager().getRegionStates()
470          .getRegionsOfTable(userTableName);
471
472      while (regionsOfTable.size() != 2) {
473        Thread.sleep(1000);
474        regionsOfTable = cluster.getMaster()
475            .getAssignmentManager().getRegionStates()
476            .getRegionsOfTable(userTableName);
477        LOG.debug("waiting 2 regions to be available, got " + regionsOfTable.size() +
478          ": " + regionsOfTable);
479
480      }
481      Assert.assertEquals(2, regionsOfTable.size());
482
483      Scan s = new Scan();
484      ResultScanner scanner = table.getScanner(s);
485      int mainTableCount = 0;
486      for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
487        mainTableCount++;
488      }
489      Assert.assertEquals(3, mainTableCount);
490    } finally {
491      table.close();
492    }
493  }
494
495  /**
496   * Verifies HBASE-5806. Here the case is that splitting is completed but before the CJ could
497   * remove the parent region the master is killed and restarted.
498   */
499  @Test
500  public void testMasterRestartAtRegionSplitPendingCatalogJanitor()
501      throws IOException, InterruptedException, NodeExistsException, KeeperException,
502      ServiceException, ExecutionException, TimeoutException {
503    final TableName tableName = TableName.valueOf(name.getMethodName());
504    // Create table then get the single region for our new table.
505    try (Table t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY)) {
506      List<HRegion> regions = cluster.getRegions(tableName);
507      RegionInfo hri = getAndCheckSingleTableRegion(regions);
508
509      int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
510
511      // Turn off balancer so it doesn't cut in and mess up our placements.
512      this.admin.balancerSwitch(false, true);
513      // Turn off the meta scanner so it don't remove parent on us.
514      cluster.getMaster().setCatalogJanitorEnabled(false);
515      // Add a bit of load up into the table so splittable.
516      TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY, false);
517      // Get region pre-split.
518      HRegionServer server = cluster.getRegionServer(tableRegionIndex);
519      printOutRegions(server, "Initial regions: ");
520      // Call split.
521      this.admin.splitRegionAsync(hri.getRegionName()).get(2, TimeUnit.MINUTES);
522      List<HRegion> daughters = checkAndGetDaughters(tableName);
523
524      // Before cleanup, get a new master.
525      HMaster master = abortAndWaitForMaster();
526      // Now call compact on the daughters and clean up any references.
527      for (HRegion daughter : daughters) {
528        daughter.compact(true);
529        daughter.getStores().get(0).closeAndArchiveCompactedFiles();
530        assertFalse(daughter.hasReferences());
531      }
532      // BUT calling compact on the daughters is not enough. The CatalogJanitor looks
533      // in the filesystem, and the filesystem content is not same as what the Region
534      // is reading from. Compacted-away files are picked up later by the compacted
535      // file discharger process. It runs infrequently. Make it run so CatalogJanitor
536      // doens't find any references.
537      for (RegionServerThread rst : cluster.getRegionServerThreads()) {
538        boolean oldSetting = rst.getRegionServer().compactedFileDischarger.setUseExecutor(false);
539        rst.getRegionServer().compactedFileDischarger.run();
540        rst.getRegionServer().compactedFileDischarger.setUseExecutor(oldSetting);
541      }
542      cluster.getMaster().setCatalogJanitorEnabled(true);
543      ProcedureTestingUtility.waitAllProcedures(cluster.getMaster().getMasterProcedureExecutor());
544      LOG.info("Starting run of CatalogJanitor");
545      cluster.getMaster().getCatalogJanitor().run();
546      ProcedureTestingUtility.waitAllProcedures(cluster.getMaster().getMasterProcedureExecutor());
547      RegionStates regionStates = master.getAssignmentManager().getRegionStates();
548      ServerName regionServerOfRegion = regionStates.getRegionServerOfRegion(hri);
549      assertEquals(null, regionServerOfRegion);
550    } finally {
551      TESTING_UTIL.getAdmin().balancerSwitch(true, false);
552      cluster.getMaster().setCatalogJanitorEnabled(true);
553    }
554  }
555
556  @Test
557  public void testSplitWithRegionReplicas() throws Exception {
558    final TableName tableName = TableName.valueOf(name.getMethodName());
559    HTableDescriptor htd = TESTING_UTIL.createTableDescriptor(name.getMethodName());
560    htd.setRegionReplication(2);
561    htd.addCoprocessor(SlowMeCopro.class.getName());
562    // Create table then get the single region for our new table.
563    Table t = TESTING_UTIL.createTable(htd, new byte[][]{Bytes.toBytes("cf")}, null);
564    List<HRegion> oldRegions;
565    do {
566      oldRegions = cluster.getRegions(tableName);
567      Thread.sleep(10);
568    } while (oldRegions.size() != 2);
569    for (HRegion h : oldRegions) LOG.debug("OLDREGION " + h.getRegionInfo());
570    try {
571      int regionServerIndex = cluster.getServerWith(oldRegions.get(0).getRegionInfo()
572        .getRegionName());
573      HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
574      insertData(tableName, admin, t);
575      // Turn off balancer so it doesn't cut in and mess up our placements.
576      admin.balancerSwitch(false, true);
577      // Turn off the meta scanner so it don't remove parent on us.
578      cluster.getMaster().setCatalogJanitorEnabled(false);
579      boolean tableExists = MetaTableAccessor.tableExists(regionServer.getConnection(),
580          tableName);
581      assertEquals("The specified table should be present.", true, tableExists);
582      final HRegion region = findSplittableRegion(oldRegions);
583      regionServerIndex = cluster.getServerWith(region.getRegionInfo().getRegionName());
584      regionServer = cluster.getRegionServer(regionServerIndex);
585      assertTrue("not able to find a splittable region", region != null);
586      try {
587        requestSplitRegion(regionServer, region, Bytes.toBytes("row2"));
588      } catch (IOException e) {
589        e.printStackTrace();
590        fail("Split execution should have succeeded with no exceptions thrown " + e);
591      }
592      //TESTING_UTIL.waitUntilAllRegionsAssigned(tableName);
593      List<HRegion> newRegions;
594      do {
595        newRegions = cluster.getRegions(tableName);
596        for (HRegion h : newRegions) LOG.debug("NEWREGION " + h.getRegionInfo());
597        Thread.sleep(1000);
598      } while ((newRegions.contains(oldRegions.get(0)) || newRegions.contains(oldRegions.get(1)))
599          || newRegions.size() != 4);
600      tableExists = MetaTableAccessor.tableExists(regionServer.getConnection(),
601          tableName);
602      assertEquals("The specified table should be present.", true, tableExists);
603      // exists works on stale and we see the put after the flush
604      byte[] b1 = "row1".getBytes();
605      Get g = new Get(b1);
606      g.setConsistency(Consistency.STRONG);
607      // The following GET will make a trip to the meta to get the new location of the 1st daughter
608      // In the process it will also get the location of the replica of the daughter (initially
609      // pointing to the parent's replica)
610      Result r = t.get(g);
611      Assert.assertFalse(r.isStale());
612      LOG.info("exists stale after flush done");
613
614      SlowMeCopro.getPrimaryCdl().set(new CountDownLatch(1));
615      g = new Get(b1);
616      g.setConsistency(Consistency.TIMELINE);
617      // This will succeed because in the previous GET we get the location of the replica
618      r = t.get(g);
619      Assert.assertTrue(r.isStale());
620      SlowMeCopro.getPrimaryCdl().get().countDown();
621    } finally {
622      SlowMeCopro.getPrimaryCdl().get().countDown();
623      admin.balancerSwitch(true, false);
624      cluster.getMaster().setCatalogJanitorEnabled(true);
625      t.close();
626    }
627  }
628
629  private void insertData(final TableName tableName, Admin admin, Table t) throws IOException,
630      InterruptedException {
631    Put p = new Put(Bytes.toBytes("row1"));
632    p.addColumn(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("1"));
633    t.put(p);
634    p = new Put(Bytes.toBytes("row2"));
635    p.addColumn(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("2"));
636    t.put(p);
637    p = new Put(Bytes.toBytes("row3"));
638    p.addColumn(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("3"));
639    t.put(p);
640    p = new Put(Bytes.toBytes("row4"));
641    p.addColumn(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("4"));
642    t.put(p);
643    admin.flush(tableName);
644  }
645
646  /**
647   * If a table has regions that have no store files in a region, they should split successfully
648   * into two regions with no store files.
649   */
650  @Test
651  public void testSplitRegionWithNoStoreFiles() throws Exception {
652    final TableName tableName = TableName.valueOf(name.getMethodName());
653    // Create table then get the single region for our new table.
654    createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
655    List<HRegion> regions = cluster.getRegions(tableName);
656    RegionInfo hri = getAndCheckSingleTableRegion(regions);
657    ensureTableRegionNotOnSameServerAsMeta(admin, hri);
658    int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionInfo()
659      .getRegionName());
660    HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
661    // Turn off balancer so it doesn't cut in and mess up our placements.
662    this.admin.balancerSwitch(false, true);
663    // Turn off the meta scanner so it don't remove parent on us.
664    cluster.getMaster().setCatalogJanitorEnabled(false);
665    try {
666      // Precondition: we created a table with no data, no store files.
667      printOutRegions(regionServer, "Initial regions: ");
668      Configuration conf = cluster.getConfiguration();
669      HBaseFsck.debugLsr(conf, new Path("/"));
670      Path rootDir = FSUtils.getRootDir(conf);
671      FileSystem fs = TESTING_UTIL.getDFSCluster().getFileSystem();
672      Map<String, Path> storefiles =
673          FSUtils.getTableStoreFilePathMap(null, fs, rootDir, tableName);
674      assertEquals("Expected nothing but found " + storefiles.toString(), 0, storefiles.size());
675
676      // find a splittable region.  Refresh the regions list
677      regions = cluster.getRegions(tableName);
678      final HRegion region = findSplittableRegion(regions);
679      assertTrue("not able to find a splittable region", region != null);
680
681      // Now split.
682      try {
683        requestSplitRegion(regionServer, region, Bytes.toBytes("row2"));
684      } catch (IOException e) {
685        fail("Split execution should have succeeded with no exceptions thrown");
686      }
687
688      // Postcondition: split the table with no store files into two regions, but still have no
689      // store files
690      List<HRegion> daughters = cluster.getRegions(tableName);
691      assertEquals(2, daughters.size());
692
693      // check dirs
694      HBaseFsck.debugLsr(conf, new Path("/"));
695      Map<String, Path> storefilesAfter =
696          FSUtils.getTableStoreFilePathMap(null, fs, rootDir, tableName);
697      assertEquals("Expected nothing but found " + storefilesAfter.toString(), 0,
698          storefilesAfter.size());
699
700      hri = region.getRegionInfo(); // split parent
701      AssignmentManager am = cluster.getMaster().getAssignmentManager();
702      RegionStates regionStates = am.getRegionStates();
703      long start = EnvironmentEdgeManager.currentTime();
704      while (!regionStates.isRegionInState(hri, State.SPLIT)) {
705        LOG.debug("Waiting for SPLIT state on: " + hri);
706        assertFalse("Timed out in waiting split parent to be in state SPLIT",
707          EnvironmentEdgeManager.currentTime() - start > 60000);
708        Thread.sleep(500);
709      }
710      assertTrue(regionStates.isRegionInState(daughters.get(0).getRegionInfo(), State.OPEN));
711      assertTrue(regionStates.isRegionInState(daughters.get(1).getRegionInfo(), State.OPEN));
712
713      // We should not be able to assign it again
714      try {
715        am.assign(hri);
716      } catch (DoNotRetryIOException e) {
717        // Expected
718      }
719      assertFalse("Split region can't be assigned", regionStates.isRegionInTransition(hri));
720      assertTrue(regionStates.isRegionInState(hri, State.SPLIT));
721
722      // We should not be able to unassign it either
723      try {
724        am.unassign(hri);
725        fail("Should have thrown exception");
726      } catch (DoNotRetryIOException e) {
727        // Expected
728      }
729      assertFalse("Split region can't be unassigned", regionStates.isRegionInTransition(hri));
730      assertTrue(regionStates.isRegionInState(hri, State.SPLIT));
731    } finally {
732      admin.balancerSwitch(true, false);
733      cluster.getMaster().setCatalogJanitorEnabled(true);
734    }
735  }
736
737  @Test
738  public void testStoreFileReferenceCreationWhenSplitPolicySaysToSkipRangeCheck()
739      throws Exception {
740    final TableName tableName = TableName.valueOf(name.getMethodName());
741    try {
742      byte[] cf = Bytes.toBytes("f");
743      byte[] cf1 = Bytes.toBytes("i_f");
744      TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName)
745        .setColumnFamily(ColumnFamilyDescriptorBuilder.of(cf))
746        .setColumnFamily(ColumnFamilyDescriptorBuilder.of(cf1))
747        .setRegionSplitPolicyClassName(CustomSplitPolicy.class.getName()).build();
748      admin.createTable(htd);
749      List<HRegion> regions = awaitTableRegions(tableName);
750      HRegion region = regions.get(0);
751      for(int i = 3;i<9;i++) {
752        Put p = new Put(Bytes.toBytes("row"+i));
753        p.addColumn(cf, Bytes.toBytes("q"), Bytes.toBytes("value" + i));
754        p.addColumn(cf1, Bytes.toBytes("q"), Bytes.toBytes("value" + i));
755        region.put(p);
756      }
757      region.flush(true);
758      HStore store = region.getStore(cf);
759      Collection<HStoreFile> storefiles = store.getStorefiles();
760      assertEquals(1, storefiles.size());
761      assertFalse(region.hasReferences());
762      Path referencePath =
763          region.getRegionFileSystem().splitStoreFile(region.getRegionInfo(), "f",
764            storefiles.iterator().next(), Bytes.toBytes("row1"), false, region.getSplitPolicy());
765      assertNull(referencePath);
766      referencePath =
767          region.getRegionFileSystem().splitStoreFile(region.getRegionInfo(), "i_f",
768            storefiles.iterator().next(), Bytes.toBytes("row1"), false, region.getSplitPolicy());
769      assertNotNull(referencePath);
770    } finally {
771      TESTING_UTIL.deleteTable(tableName);
772    }
773  }
774
775  private HRegion findSplittableRegion(final List<HRegion> regions) throws InterruptedException {
776    for (int i = 0; i < 5; ++i) {
777      for (HRegion r: regions) {
778        if (r.isSplittable() && r.getRegionInfo().getReplicaId() == 0) {
779          return(r);
780        }
781      }
782      Thread.sleep(100);
783    }
784    return null;
785  }
786
787  private List<HRegion> checkAndGetDaughters(TableName tableName) throws InterruptedException {
788    List<HRegion> daughters = null;
789    // try up to 10s
790    for (int i = 0; i < 100; i++) {
791      daughters = cluster.getRegions(tableName);
792      if (daughters.size() >= 2) {
793        break;
794      }
795      Thread.sleep(100);
796    }
797    assertTrue(daughters.size() >= 2);
798    return daughters;
799  }
800
801  private HMaster abortAndWaitForMaster() throws IOException, InterruptedException {
802    cluster.abortMaster(0);
803    cluster.waitOnMaster(0);
804    HMaster master = cluster.startMaster().getMaster();
805    cluster.waitForActiveAndReadyMaster();
806    return master;
807  }
808
809  /**
810   * Ensure single table region is not on same server as the single hbase:meta table
811   * region.
812   * @param admin
813   * @param hri
814   * @return Index of the server hosting the single table region
815   * @throws UnknownRegionException
816   * @throws MasterNotRunningException
817   * @throws org.apache.hadoop.hbase.ZooKeeperConnectionException
818   * @throws InterruptedException
819   */
820  private int ensureTableRegionNotOnSameServerAsMeta(final Admin admin,
821      final RegionInfo hri)
822  throws IOException, MasterNotRunningException,
823  ZooKeeperConnectionException, InterruptedException {
824    // Now make sure that the table region is not on same server as that hosting
825    // hbase:meta  We don't want hbase:meta replay polluting our test when we later crash
826    // the table region serving server.
827    int metaServerIndex = cluster.getServerWithMeta();
828    boolean tablesOnMaster = LoadBalancer.isTablesOnMaster(TESTING_UTIL.getConfiguration());
829    if (tablesOnMaster) {
830      // Need to check master is supposed to host meta... perhaps it is not.
831      throw new UnsupportedOperationException();
832      // TODO: assertTrue(metaServerIndex == -1); // meta is on master now
833    }
834    HRegionServer metaRegionServer = tablesOnMaster?
835      cluster.getMaster(): cluster.getRegionServer(metaServerIndex);
836    int tableRegionIndex = cluster.getServerWith(hri.getRegionName());
837    assertTrue(tableRegionIndex != -1);
838    HRegionServer tableRegionServer = cluster.getRegionServer(tableRegionIndex);
839    LOG.info("MetaRegionServer=" + metaRegionServer.getServerName() +
840      ", other=" + tableRegionServer.getServerName());
841    if (metaRegionServer.getServerName().equals(tableRegionServer.getServerName())) {
842      HRegionServer hrs = getOtherRegionServer(cluster, metaRegionServer);
843      assertNotNull(hrs);
844      assertNotNull(hri);
845      LOG.info("Moving " + hri.getRegionNameAsString() + " from " +
846        metaRegionServer.getServerName() + " to " +
847        hrs.getServerName() + "; metaServerIndex=" + metaServerIndex);
848      admin.move(hri.getEncodedNameAsBytes(), hrs.getServerName());
849    }
850    // Wait till table region is up on the server that is NOT carrying hbase:meta.
851    for (int i = 0; i < 100; i++) {
852      tableRegionIndex = cluster.getServerWith(hri.getRegionName());
853      if (tableRegionIndex != -1 && tableRegionIndex != metaServerIndex) break;
854      LOG.debug("Waiting on region move off the hbase:meta server; current index " +
855        tableRegionIndex + " and metaServerIndex=" + metaServerIndex);
856      Thread.sleep(100);
857    }
858    assertTrue("Region not moved off hbase:meta server, tableRegionIndex=" + tableRegionIndex,
859      tableRegionIndex != -1 && tableRegionIndex != metaServerIndex);
860    // Verify for sure table region is not on same server as hbase:meta
861    tableRegionIndex = cluster.getServerWith(hri.getRegionName());
862    assertTrue(tableRegionIndex != -1);
863    assertNotSame(metaServerIndex, tableRegionIndex);
864    return tableRegionIndex;
865  }
866
867  /**
868   * Find regionserver other than the one passed.
869   * Can't rely on indexes into list of regionservers since crashed servers
870   * occupy an index.
871   * @param cluster
872   * @param notThisOne
873   * @return A regionserver that is not <code>notThisOne</code> or null if none
874   * found
875   */
876  private HRegionServer getOtherRegionServer(final MiniHBaseCluster cluster,
877      final HRegionServer notThisOne) {
878    for (RegionServerThread rst: cluster.getRegionServerThreads()) {
879      HRegionServer hrs = rst.getRegionServer();
880      if (hrs.getServerName().equals(notThisOne.getServerName())) continue;
881      if (hrs.isStopping() || hrs.isStopped()) continue;
882      return hrs;
883    }
884    return null;
885  }
886
887  private void printOutRegions(final HRegionServer hrs, final String prefix)
888      throws IOException {
889    List<RegionInfo> regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices());
890    for (RegionInfo region: regions) {
891      LOG.info(prefix + region.getRegionNameAsString());
892    }
893  }
894
895  private void waitUntilRegionServerDead() throws InterruptedException, IOException {
896    // Wait until the master processes the RS shutdown
897    for (int i=0; (cluster.getMaster().getClusterMetrics()
898        .getLiveServerMetrics().size() > NB_SERVERS
899        || cluster.getLiveRegionServerThreads().size() > NB_SERVERS) && i<100; i++) {
900      LOG.info("Waiting on server to go down");
901      Thread.sleep(100);
902    }
903    assertFalse("Waited too long for RS to die",
904      cluster.getMaster().getClusterMetrics(). getLiveServerMetrics().size() > NB_SERVERS
905        || cluster.getLiveRegionServerThreads().size() > NB_SERVERS);
906  }
907
908  private void awaitDaughters(TableName tableName, int numDaughters) throws InterruptedException {
909    // Wait till regions are back on line again.
910    for (int i = 0; cluster.getRegions(tableName).size() < numDaughters && i < 60; i++) {
911      LOG.info("Waiting for repair to happen");
912      Thread.sleep(1000);
913    }
914    if (cluster.getRegions(tableName).size() < numDaughters) {
915      fail("Waiting too long for daughter regions");
916    }
917  }
918
919  private List<HRegion> awaitTableRegions(final TableName tableName) throws InterruptedException {
920    List<HRegion> regions = null;
921    for (int i = 0; i < 100; i++) {
922      regions = cluster.getRegions(tableName);
923      if (regions.size() > 0) break;
924      Thread.sleep(100);
925    }
926    return regions;
927  }
928
929  private Table createTableAndWait(TableName tableName, byte[] cf) throws IOException,
930      InterruptedException {
931    Table t = TESTING_UTIL.createTable(tableName, cf);
932    awaitTableRegions(tableName);
933    assertTrue("Table not online: " + tableName,
934      cluster.getRegions(tableName).size() != 0);
935    return t;
936  }
937
938  // Make it public so that JVMClusterUtil can access it.
939  public static class MyMaster extends HMaster {
940    public MyMaster(Configuration conf) throws IOException, KeeperException, InterruptedException {
941      super(conf);
942    }
943
944    @Override
945    protected RSRpcServices createRpcServices() throws IOException {
946      return new MyMasterRpcServices(this);
947    }
948  }
949
950  static class MyMasterRpcServices extends MasterRpcServices {
951    static AtomicBoolean enabled = new AtomicBoolean(false);
952
953    private HMaster myMaster;
954    public MyMasterRpcServices(HMaster master) throws IOException {
955      super(master);
956      myMaster = master;
957    }
958
959    @Override
960    public ReportRegionStateTransitionResponse reportRegionStateTransition(RpcController c,
961        ReportRegionStateTransitionRequest req) throws ServiceException {
962      ReportRegionStateTransitionResponse resp = super.reportRegionStateTransition(c, req);
963      if (enabled.get() && req.getTransition(0).getTransitionCode().equals(
964          TransitionCode.READY_TO_SPLIT) && !resp.hasErrorMessage()) {
965        RegionStates regionStates = myMaster.getAssignmentManager().getRegionStates();
966        for (RegionStateNode regionState:
967          regionStates.getRegionsInTransition()) {
968          /* TODO!!!!
969          // Find the merging_new region and remove it
970          if (regionState.isSplittingNew()) {
971            regionStates.deleteRegion(regionState.getRegion());
972          }
973          */
974        }
975      }
976      return resp;
977    }
978  }
979
980  static class CustomSplitPolicy extends IncreasingToUpperBoundRegionSplitPolicy {
981
982    @Override
983    protected boolean shouldSplit() {
984      return true;
985    }
986
987    @Override
988    public boolean skipStoreFileRangeCheck(String familyName) {
989      if(familyName.startsWith("i_")) {
990        return true;
991      } else {
992        return false;
993      }
994    }
995  }
996}
997