001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.tool;
019
020import static org.junit.jupiter.api.Assertions.assertArrayEquals;
021import static org.junit.jupiter.api.Assertions.assertEquals;
022import static org.junit.jupiter.api.Assertions.assertNotNull;
023import static org.junit.jupiter.api.Assertions.assertNull;
024import static org.junit.jupiter.api.Assertions.assertThrows;
025import static org.junit.jupiter.api.Assertions.assertTrue;
026import static org.junit.jupiter.api.Assertions.fail;
027import static org.mockito.ArgumentMatchers.any;
028import static org.mockito.ArgumentMatchers.anyBoolean;
029import static org.mockito.ArgumentMatchers.anyList;
030import static org.mockito.Mockito.doReturn;
031import static org.mockito.Mockito.spy;
032
033import java.io.IOException;
034import java.nio.ByteBuffer;
035import java.util.Deque;
036import java.util.List;
037import java.util.Map;
038import java.util.concurrent.CompletableFuture;
039import java.util.concurrent.atomic.AtomicInteger;
040import java.util.stream.IntStream;
041import org.apache.hadoop.conf.Configuration;
042import org.apache.hadoop.fs.FileStatus;
043import org.apache.hadoop.fs.FileSystem;
044import org.apache.hadoop.fs.Path;
045import org.apache.hadoop.hbase.HBaseTestingUtil;
046import org.apache.hadoop.hbase.HConstants;
047import org.apache.hadoop.hbase.TableExistsException;
048import org.apache.hadoop.hbase.TableName;
049import org.apache.hadoop.hbase.client.Admin;
050import org.apache.hadoop.hbase.client.AsyncClusterConnection;
051import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
052import org.apache.hadoop.hbase.client.Connection;
053import org.apache.hadoop.hbase.client.RegionInfo;
054import org.apache.hadoop.hbase.client.Result;
055import org.apache.hadoop.hbase.client.ResultScanner;
056import org.apache.hadoop.hbase.client.Scan;
057import org.apache.hadoop.hbase.client.Table;
058import org.apache.hadoop.hbase.client.TableDescriptor;
059import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
060import org.apache.hadoop.hbase.regionserver.HRegionServer;
061import org.apache.hadoop.hbase.regionserver.TestHRegionServerBulkLoad;
062import org.apache.hadoop.hbase.util.Bytes;
063import org.apache.hadoop.hbase.util.CommonFSUtils;
064import org.apache.hadoop.hbase.util.Pair;
065import org.junit.jupiter.api.AfterAll;
066import org.junit.jupiter.api.Test;
067import org.junit.jupiter.api.TestInfo;
068import org.slf4j.Logger;
069import org.slf4j.LoggerFactory;
070
071import org.apache.hbase.thirdparty.com.google.common.collect.Multimap;
072
073import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
074
075/**
076 * Test cases for the atomic load error handling of the bulk load functionality.
077 */
078public class BulkLoadHFilesSplitRecoveryTestBase {
079
080  private static final Logger LOG =
081    LoggerFactory.getLogger(BulkLoadHFilesSplitRecoveryTestBase.class);
082
083  static HBaseTestingUtil util;
084  // used by secure subclass
085  static boolean useSecure = false;
086
087  final static int NUM_CFS = 10;
088  final static byte[] QUAL = Bytes.toBytes("qual");
089  final static int ROWCOUNT = 100;
090
091  private final static byte[][] families = new byte[NUM_CFS][];
092
093  static {
094    for (int i = 0; i < NUM_CFS; i++) {
095      families[i] = Bytes.toBytes(family(i));
096    }
097  }
098
099  static byte[] rowkey(int i) {
100    return Bytes.toBytes(String.format("row_%08d", i));
101  }
102
103  static String family(int i) {
104    return String.format("family_%04d", i);
105  }
106
107  static byte[] value(int i) {
108    return Bytes.toBytes(String.format("%010d", i));
109  }
110
111  public static void buildHFiles(FileSystem fs, Path dir, int value) throws IOException {
112    byte[] val = value(value);
113    for (int i = 0; i < NUM_CFS; i++) {
114      Path testIn = new Path(dir, family(i));
115
116      TestHRegionServerBulkLoad.createHFile(fs, new Path(testIn, "hfile_" + i),
117        Bytes.toBytes(family(i)), QUAL, val, ROWCOUNT);
118    }
119  }
120
121  private TableDescriptor createTableDesc(TableName name, int cfs) {
122    TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(name);
123    IntStream.range(0, cfs).mapToObj(i -> ColumnFamilyDescriptorBuilder.of(family(i)))
124      .forEachOrdered(builder::setColumnFamily);
125    return builder.build();
126  }
127
128  /**
129   * Creates a table with given table name and specified number of column families if the table does
130   * not already exist.
131   */
132  private void setupTable(final Connection connection, TableName table, int cfs)
133    throws IOException {
134    try {
135      LOG.info("Creating table " + table);
136      try (Admin admin = connection.getAdmin()) {
137        admin.createTable(createTableDesc(table, cfs));
138      }
139    } catch (TableExistsException tee) {
140      LOG.info("Table " + table + " already exists");
141    }
142  }
143
144  /**
145   * Creates a table with given table name,specified number of column families<br>
146   * and splitkeys if the table does not already exist.
147   */
148  private void setupTableWithSplitkeys(TableName table, int cfs, byte[][] SPLIT_KEYS)
149    throws IOException {
150    try {
151      LOG.info("Creating table " + table);
152      util.createTable(createTableDesc(table, cfs), SPLIT_KEYS);
153    } catch (TableExistsException tee) {
154      LOG.info("Table " + table + " already exists");
155    }
156  }
157
158  private Path buildBulkFiles(TableName table, int value) throws Exception {
159    Path dir = util.getDataTestDirOnTestFS(table.getNameAsString());
160    Path bulk1 = new Path(dir, table.getNameAsString() + value);
161    FileSystem fs = util.getTestFileSystem();
162    buildHFiles(fs, bulk1, value);
163    return bulk1;
164  }
165
166  /**
167   * Populate table with known values.
168   */
169  private void populateTable(final Connection connection, TableName table, int value)
170    throws Exception {
171    // create HFiles for different column families
172    Path dir = buildBulkFiles(table, value);
173    BulkLoadHFiles.create(util.getConfiguration()).bulkLoad(table, dir);
174  }
175
176  /**
177   * Split the known table in half. (this is hard coded for this test suite)
178   */
179  private void forceSplit(TableName table) {
180    try {
181      // need to call regions server to by synchronous but isn't visible.
182      HRegionServer hrs = util.getRSForFirstRegionInTable(table);
183
184      for (RegionInfo hri : ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices())) {
185        if (hri.getTable().equals(table)) {
186          util.getAdmin().splitRegionAsync(hri.getRegionName(), rowkey(ROWCOUNT / 2));
187          // ProtobufUtil.split(null, hrs.getRSRpcServices(), hri, rowkey(ROWCOUNT / 2));
188        }
189      }
190
191      // verify that split completed.
192      int regions;
193      do {
194        regions = 0;
195        for (RegionInfo hri : ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices())) {
196          if (hri.getTable().equals(table)) {
197            regions++;
198          }
199        }
200        if (regions != 2) {
201          LOG.info("Taking some time to complete split...");
202          Thread.sleep(250);
203        }
204      } while (regions != 2);
205    } catch (IOException e) {
206      e.printStackTrace();
207    } catch (InterruptedException e) {
208      e.printStackTrace();
209    }
210  }
211
212  @AfterAll
213  public static void teardownCluster() throws Exception {
214    util.shutdownMiniCluster();
215  }
216
217  /**
218   * Checks that all columns have the expected value and that there is the expected number of rows.
219   */
220  void assertExpectedTable(TableName table, int count, int value) throws IOException {
221    TableDescriptor htd = util.getAdmin().getDescriptor(table);
222    assertNotNull(htd);
223    try (Table t = util.getConnection().getTable(table);
224      ResultScanner sr = t.getScanner(new Scan())) {
225      int i = 0;
226      for (Result r; (r = sr.next()) != null;) {
227        r.getNoVersionMap().values().stream().flatMap(m -> m.values().stream())
228          .forEach(v -> assertArrayEquals(value(value), v));
229        i++;
230      }
231      assertEquals(count, i);
232    } catch (IOException e) {
233      fail("Failed due to exception");
234    }
235  }
236
237  private static <T> CompletableFuture<T> failedFuture(Throwable error) {
238    CompletableFuture<T> future = new CompletableFuture<>();
239    future.completeExceptionally(error);
240    return future;
241  }
242
243  private static AsyncClusterConnection mockAndInjectError(AsyncClusterConnection conn) {
244    AsyncClusterConnection errConn = spy(conn);
245    doReturn(failedFuture(new IOException("injecting bulk load error"))).when(errConn).bulkLoad(
246      any(), anyList(), any(), anyBoolean(), any(), any(), anyBoolean(), anyList(), anyBoolean());
247    return errConn;
248  }
249
250  /**
251   * Test that shows that exception thrown from the RS side will result in an exception on the
252   * LIHFile client.
253   */
254  @Test
255  public void testBulkLoadPhaseFailure(TestInfo testInfo) throws Exception {
256    final TableName table = TableName.valueOf(testInfo.getTestMethod().get().getName());
257    final AtomicInteger attemptedCalls = new AtomicInteger();
258    Configuration conf = new Configuration(util.getConfiguration());
259    conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2);
260    BulkLoadHFilesTool loader = new BulkLoadHFilesTool(conf) {
261
262      @Override
263      protected void bulkLoadPhase(AsyncClusterConnection conn, TableName tableName,
264        Deque<LoadQueueItem> queue, Multimap<ByteBuffer, LoadQueueItem> regionGroups,
265        boolean copyFiles, Map<LoadQueueItem, ByteBuffer> item2RegionMap) throws IOException {
266        AsyncClusterConnection c =
267          attemptedCalls.incrementAndGet() == 1 ? mockAndInjectError(conn) : conn;
268        super.bulkLoadPhase(c, tableName, queue, regionGroups, copyFiles, item2RegionMap);
269      }
270    };
271    Path dir = buildBulkFiles(table, 1);
272    assertThrows(IOException.class, () -> loader.bulkLoad(table, dir));
273  }
274
275  /**
276   * Test that shows that exception thrown from the RS side will result in the expected number of
277   * retries set by ${@link HConstants#HBASE_CLIENT_RETRIES_NUMBER} when
278   * ${@link BulkLoadHFiles#RETRY_ON_IO_EXCEPTION} is set
279   */
280  @Test
281  public void testRetryOnIOException(TestInfo testInfo) throws Exception {
282    TableName table = TableName.valueOf(testInfo.getTestMethod().get().getName());
283    AtomicInteger calls = new AtomicInteger(0);
284    setupTable(util.getConnection(), table, 10);
285    Configuration conf = new Configuration(util.getConfiguration());
286    conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2);
287    conf.setBoolean(BulkLoadHFiles.RETRY_ON_IO_EXCEPTION, true);
288    BulkLoadHFilesTool loader = new BulkLoadHFilesTool(conf) {
289
290      @Override
291      protected void bulkLoadPhase(AsyncClusterConnection conn, TableName tableName,
292        Deque<LoadQueueItem> queue, Multimap<ByteBuffer, LoadQueueItem> regionGroups,
293        boolean copyFiles, Map<LoadQueueItem, ByteBuffer> item2RegionMap) throws IOException {
294        if (
295          calls.get() < conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
296            HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER)
297        ) {
298          calls.incrementAndGet();
299          super.bulkLoadPhase(mockAndInjectError(conn), tableName, queue, regionGroups, copyFiles,
300            item2RegionMap);
301        } else {
302          super.bulkLoadPhase(conn, tableName, queue, regionGroups, copyFiles, item2RegionMap);
303        }
304      }
305    };
306    Path dir = buildBulkFiles(table, 1);
307    loader.bulkLoad(table, dir);
308    assertEquals(calls.get(), 2);
309  }
310
311  /**
312   * This test exercises the path where there is a split after initial validation but before the
313   * atomic bulk load call. We cannot use presplitting to test this path, so we actually inject a
314   * split just before the atomic region load.
315   */
316  @Test
317  public void testSplitWhileBulkLoadPhase(TestInfo testInfo) throws Exception {
318    final TableName table = TableName.valueOf(testInfo.getTestMethod().get().getName());
319    setupTable(util.getConnection(), table, 10);
320    populateTable(util.getConnection(), table, 1);
321    assertExpectedTable(table, ROWCOUNT, 1);
322
323    // Now let's cause trouble. This will occur after checks and cause bulk
324    // files to fail when attempt to atomically import. This is recoverable.
325    final AtomicInteger attemptedCalls = new AtomicInteger();
326    BulkLoadHFilesTool loader = new BulkLoadHFilesTool(util.getConfiguration()) {
327
328      @Override
329      protected void bulkLoadPhase(AsyncClusterConnection conn, TableName tableName,
330        Deque<LoadQueueItem> queue, Multimap<ByteBuffer, LoadQueueItem> regionGroups,
331        boolean copyFiles, Map<LoadQueueItem, ByteBuffer> item2RegionMap) throws IOException {
332        int i = attemptedCalls.incrementAndGet();
333        if (i == 1) {
334          // On first attempt force a split.
335          forceSplit(table);
336        }
337        super.bulkLoadPhase(conn, tableName, queue, regionGroups, copyFiles, item2RegionMap);
338      }
339    };
340
341    // create HFiles for different column families
342    Path dir = buildBulkFiles(table, 2);
343    loader.bulkLoad(table, dir);
344
345    // check that data was loaded
346    // The three expected attempts are 1) failure because need to split, 2)
347    // load of split top 3) load of split bottom
348    assertEquals(3, attemptedCalls.get());
349    assertExpectedTable(table, ROWCOUNT, 2);
350  }
351
352  /**
353   * This test splits a table and attempts to bulk load. The bulk import files should be split
354   * before atomically importing.
355   */
356  @Test
357  public void testGroupOrSplitPresplit(TestInfo testInfo) throws Exception {
358    final TableName table = TableName.valueOf(testInfo.getTestMethod().get().getName());
359    setupTable(util.getConnection(), table, 10);
360    populateTable(util.getConnection(), table, 1);
361    assertExpectedTable(util.getConnection(), table, ROWCOUNT, 1);
362    forceSplit(table);
363
364    final AtomicInteger countedLqis = new AtomicInteger();
365    BulkLoadHFilesTool loader = new BulkLoadHFilesTool(util.getConfiguration()) {
366
367      @Override
368      protected Pair<List<LoadQueueItem>, String> groupOrSplit(AsyncClusterConnection conn,
369        TableName tableName, Multimap<ByteBuffer, LoadQueueItem> regionGroups, LoadQueueItem item,
370        List<Pair<byte[], byte[]>> startEndKeys) throws IOException {
371        Pair<List<LoadQueueItem>, String> lqis =
372          super.groupOrSplit(conn, tableName, regionGroups, item, startEndKeys);
373        if (lqis != null && lqis.getFirst() != null) {
374          countedLqis.addAndGet(lqis.getFirst().size());
375        }
376        return lqis;
377      }
378    };
379
380    // create HFiles for different column families
381    Path dir = buildBulkFiles(table, 2);
382    loader.bulkLoad(table, dir);
383    assertExpectedTable(util.getConnection(), table, ROWCOUNT, 2);
384    assertEquals(20, countedLqis.get());
385  }
386
387  @Test
388  public void testCorrectSplitPoint(TestInfo testInfo) throws Exception {
389    final TableName table = TableName.valueOf(testInfo.getTestMethod().get().getName());
390    byte[][] SPLIT_KEYS = new byte[][] { Bytes.toBytes("row_00000010"),
391      Bytes.toBytes("row_00000020"), Bytes.toBytes("row_00000030"), Bytes.toBytes("row_00000040"),
392      Bytes.toBytes("row_00000050"), Bytes.toBytes("row_00000060"), Bytes.toBytes("row_00000070") };
393    setupTableWithSplitkeys(table, NUM_CFS, SPLIT_KEYS);
394
395    final AtomicInteger bulkloadRpcTimes = new AtomicInteger();
396    BulkLoadHFilesTool loader = new BulkLoadHFilesTool(util.getConfiguration()) {
397
398      @Override
399      protected void bulkLoadPhase(AsyncClusterConnection conn, TableName tableName,
400        Deque<LoadQueueItem> queue, Multimap<ByteBuffer, LoadQueueItem> regionGroups,
401        boolean copyFiles, Map<LoadQueueItem, ByteBuffer> item2RegionMap) throws IOException {
402        bulkloadRpcTimes.addAndGet(1);
403        super.bulkLoadPhase(conn, tableName, queue, regionGroups, copyFiles, item2RegionMap);
404      }
405    };
406
407    Path dir = buildBulkFiles(table, 1);
408    loader.bulkLoad(table, dir);
409    // before HBASE-25281 we need invoke bulkload rpc 8 times
410    assertEquals(4, bulkloadRpcTimes.get());
411  }
412
413  @Test
414  public void testFreshRetryAfterSplitFailure(TestInfo testInfo) throws Exception {
415    TableName table = TableName.valueOf(testInfo.getTestMethod().get().getName());
416    setupTableWithSplitkeys(table, NUM_CFS, new byte[][] { rowkey(ROWCOUNT / 2) });
417
418    Path dir = buildBulkFiles(table, 1);
419    FileSystem fs = dir.getFileSystem(util.getConfiguration());
420    Path familyDir = new Path(dir, family(0));
421    Path originalHFile = new Path(familyDir, "hfile_0");
422    Path tmpDir = new Path(familyDir, BulkLoadHFilesTool.TMP_DIR);
423    assertTrue(fs.exists(originalHFile));
424
425    BulkLoadHFilesTool failingLoader = new BulkLoadHFilesTool(util.getConfiguration()) {
426      @Override
427      protected void bulkLoadPhase(AsyncClusterConnection conn, TableName tableName,
428        Deque<LoadQueueItem> queue, Multimap<ByteBuffer, LoadQueueItem> regionGroups,
429        boolean copyFiles, Map<LoadQueueItem, ByteBuffer> item2RegionMap) throws IOException {
430        throw new IOException("failure after splitting");
431      }
432    };
433
434    assertThrows(IOException.class, () -> failingLoader.bulkLoad(table, dir));
435
436    assertTrue(fs.exists(originalHFile));
437    FileStatus[] abandonedChildren = CommonFSUtils.listStatus(fs, tmpDir);
438    assertNotNull(abandonedChildren);
439    assertEquals(2, abandonedChildren.length);
440    Path top = null;
441    Path bottom = null;
442    for (FileStatus child : abandonedChildren) {
443      if (child.getPath().getName().endsWith(".top")) {
444        top = child.getPath();
445      } else if (child.getPath().getName().endsWith(".bottom")) {
446        bottom = child.getPath();
447      }
448    }
449    assertNotNull(top);
450    assertNotNull(bottom);
451
452    // Use a new loader to simulate retrying in another process. The original HFiles must remain in
453    // the family directory so the retry does not depend on in-memory references to split outputs.
454    BulkLoadHFiles.create(util.getConfiguration()).bulkLoad(table, dir);
455
456    // The retry consumes the preserved original HFiles. Split outputs abandoned by the failed
457    // invocation remain temporary and are deliberately not treated as input because they may be
458    // incomplete.
459    assertTrue(fs.exists(originalHFile));
460    assertTrue(fs.exists(top));
461    assertTrue(fs.exists(bottom));
462    assertEquals(2, CommonFSUtils.listStatus(fs, tmpDir).length);
463    assertExpectedTable(table, ROWCOUNT, 1);
464  }
465
466  /**
467   * This test creates a table with many small regions. The bulk load files would be split multiple
468   * times before all of them can be loaded successfully.
469   */
470  @Test
471  public void testSplitTmpFileCleanUp(TestInfo testInfo) throws Exception {
472    final TableName table = TableName.valueOf(testInfo.getTestMethod().get().getName());
473    byte[][] SPLIT_KEYS = new byte[][] { Bytes.toBytes("row_00000010"),
474      Bytes.toBytes("row_00000020"), Bytes.toBytes("row_00000030"), Bytes.toBytes("row_00000040"),
475      Bytes.toBytes("row_00000050") };
476    setupTableWithSplitkeys(table, 10, SPLIT_KEYS);
477
478    BulkLoadHFiles loader = BulkLoadHFiles.create(util.getConfiguration());
479
480    // create HFiles
481    Path dir = buildBulkFiles(table, 2);
482    loader.bulkLoad(table, dir);
483    // family path
484    Path tmpPath = new Path(dir, family(0));
485    // TMP_DIR under family path
486    tmpPath = new Path(tmpPath, BulkLoadHFilesTool.TMP_DIR);
487    FileSystem fs = dir.getFileSystem(util.getConfiguration());
488    // HFiles have been splitted, there is TMP_DIR
489    assertTrue(fs.exists(tmpPath));
490    // TMP_DIR should have been cleaned-up
491    assertNull(CommonFSUtils.listStatus(fs, tmpPath),
492      BulkLoadHFilesTool.TMP_DIR + " should be empty.");
493    assertExpectedTable(util.getConnection(), table, ROWCOUNT, 2);
494  }
495
496  /**
497   * This simulates an remote exception which should cause LIHF to exit with an exception.
498   */
499  @Test
500  public void testGroupOrSplitFailure(TestInfo testInfo) throws Exception {
501    final TableName tableName = TableName.valueOf(testInfo.getTestMethod().get().getName());
502    setupTable(util.getConnection(), tableName, 10);
503    BulkLoadHFilesTool loader = new BulkLoadHFilesTool(util.getConfiguration()) {
504
505      private int i = 0;
506
507      @Override
508      protected Pair<List<LoadQueueItem>, String> groupOrSplit(AsyncClusterConnection conn,
509        TableName tableName, Multimap<ByteBuffer, LoadQueueItem> regionGroups, LoadQueueItem item,
510        List<Pair<byte[], byte[]>> startEndKeys) throws IOException {
511        i++;
512
513        if (i == 5) {
514          throw new IOException("failure");
515        }
516        return super.groupOrSplit(conn, tableName, regionGroups, item, startEndKeys);
517      }
518    };
519
520    // create HFiles for different column families
521    Path dir = buildBulkFiles(tableName, 1);
522    assertThrows(IOException.class, () -> loader.bulkLoad(tableName, dir));
523  }
524
525  /**
526   * We are testing a split after initial validation but before the atomic bulk load call. We cannot
527   * use presplitting to test this path, so we actually inject a split just before the atomic region
528   * load. However, we will pass null item2RegionMap and that should not affect the bulk load
529   * behavior.
530   */
531  @Test
532  public void testSplitWhileBulkLoadPhaseWithoutItemMap(TestInfo testInfo) throws Exception {
533    final TableName table = TableName.valueOf(testInfo.getTestMethod().get().getName());
534    setupTable(util.getConnection(), table, 10);
535    populateTable(util.getConnection(), table, 1);
536    assertExpectedTable(table, ROWCOUNT, 1);
537
538    // Now let's cause trouble. This will occur after checks and cause bulk
539    // files to fail when attempt to atomically import. This is recoverable.
540    final AtomicInteger attemptedCalls = new AtomicInteger();
541    BulkLoadHFilesTool loader = new BulkLoadHFilesTool(util.getConfiguration()) {
542
543      @Override
544      protected void bulkLoadPhase(final AsyncClusterConnection conn, final TableName tableName,
545        final Deque<LoadQueueItem> queue, final Multimap<ByteBuffer, LoadQueueItem> regionGroups,
546        final boolean copyFiles, final Map<LoadQueueItem, ByteBuffer> item2RegionMap)
547        throws IOException {
548
549        int i = attemptedCalls.incrementAndGet();
550        if (i == 1) {
551          // On first attempt force a split.
552          forceSplit(table);
553        }
554
555        // Passing item2RegionMap null
556        // In the absence of LoadQueueItem, bulk load should work as expected
557        super.bulkLoadPhase(conn, tableName, queue, regionGroups, copyFiles, null);
558      }
559
560    };
561
562    // create HFiles for different column families
563    Path dir = buildBulkFiles(table, 2);
564    loader.bulkLoad(table, dir);
565
566    // check that data was loaded
567    // The three expected attempts are 1) failure because need to split, 2)
568    // load of split top 3) load of split bottom
569    assertEquals(3, attemptedCalls.get());
570    assertExpectedTable(table, ROWCOUNT, 2);
571  }
572
573  /**
574   * Checks that all columns have the expected value and that there is the expected number of rows.
575   */
576  void assertExpectedTable(final Connection connection, TableName table, int count, int value)
577    throws IOException {
578    TableDescriptor htd = util.getAdmin().getDescriptor(table);
579    assertNotNull(htd);
580    try (Table t = connection.getTable(table); ResultScanner sr = t.getScanner(new Scan())) {
581      int i = 0;
582      for (Result r; (r = sr.next()) != null;) {
583        r.getNoVersionMap().values().stream().flatMap(m -> m.values().stream())
584          .forEach(v -> assertArrayEquals(value(value), v));
585        i++;
586      }
587      assertEquals(count, i);
588    } catch (IOException e) {
589      fail("Failed due to exception");
590    }
591  }
592}