001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.tool; 019 020import static org.junit.jupiter.api.Assertions.assertArrayEquals; 021import static org.junit.jupiter.api.Assertions.assertEquals; 022import static org.junit.jupiter.api.Assertions.assertNotNull; 023import static org.junit.jupiter.api.Assertions.assertNull; 024import static org.junit.jupiter.api.Assertions.assertThrows; 025import static org.junit.jupiter.api.Assertions.assertTrue; 026import static org.junit.jupiter.api.Assertions.fail; 027import static org.mockito.ArgumentMatchers.any; 028import static org.mockito.ArgumentMatchers.anyBoolean; 029import static org.mockito.ArgumentMatchers.anyList; 030import static org.mockito.Mockito.doReturn; 031import static org.mockito.Mockito.spy; 032 033import java.io.IOException; 034import java.nio.ByteBuffer; 035import java.util.Deque; 036import java.util.List; 037import java.util.Map; 038import java.util.concurrent.CompletableFuture; 039import java.util.concurrent.atomic.AtomicInteger; 040import java.util.stream.IntStream; 041import org.apache.hadoop.conf.Configuration; 042import org.apache.hadoop.fs.FileStatus; 043import org.apache.hadoop.fs.FileSystem; 044import org.apache.hadoop.fs.Path; 045import org.apache.hadoop.hbase.HBaseTestingUtil; 046import org.apache.hadoop.hbase.HConstants; 047import org.apache.hadoop.hbase.TableExistsException; 048import org.apache.hadoop.hbase.TableName; 049import org.apache.hadoop.hbase.client.Admin; 050import org.apache.hadoop.hbase.client.AsyncClusterConnection; 051import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; 052import org.apache.hadoop.hbase.client.Connection; 053import org.apache.hadoop.hbase.client.RegionInfo; 054import org.apache.hadoop.hbase.client.Result; 055import org.apache.hadoop.hbase.client.ResultScanner; 056import org.apache.hadoop.hbase.client.Scan; 057import org.apache.hadoop.hbase.client.Table; 058import org.apache.hadoop.hbase.client.TableDescriptor; 059import org.apache.hadoop.hbase.client.TableDescriptorBuilder; 060import org.apache.hadoop.hbase.regionserver.HRegionServer; 061import org.apache.hadoop.hbase.regionserver.TestHRegionServerBulkLoad; 062import org.apache.hadoop.hbase.util.Bytes; 063import org.apache.hadoop.hbase.util.CommonFSUtils; 064import org.apache.hadoop.hbase.util.Pair; 065import org.junit.jupiter.api.AfterAll; 066import org.junit.jupiter.api.Test; 067import org.junit.jupiter.api.TestInfo; 068import org.slf4j.Logger; 069import org.slf4j.LoggerFactory; 070 071import org.apache.hbase.thirdparty.com.google.common.collect.Multimap; 072 073import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; 074 075/** 076 * Test cases for the atomic load error handling of the bulk load functionality. 077 */ 078public class BulkLoadHFilesSplitRecoveryTestBase { 079 080 private static final Logger LOG = 081 LoggerFactory.getLogger(BulkLoadHFilesSplitRecoveryTestBase.class); 082 083 static HBaseTestingUtil util; 084 // used by secure subclass 085 static boolean useSecure = false; 086 087 final static int NUM_CFS = 10; 088 final static byte[] QUAL = Bytes.toBytes("qual"); 089 final static int ROWCOUNT = 100; 090 091 private final static byte[][] families = new byte[NUM_CFS][]; 092 093 static { 094 for (int i = 0; i < NUM_CFS; i++) { 095 families[i] = Bytes.toBytes(family(i)); 096 } 097 } 098 099 static byte[] rowkey(int i) { 100 return Bytes.toBytes(String.format("row_%08d", i)); 101 } 102 103 static String family(int i) { 104 return String.format("family_%04d", i); 105 } 106 107 static byte[] value(int i) { 108 return Bytes.toBytes(String.format("%010d", i)); 109 } 110 111 public static void buildHFiles(FileSystem fs, Path dir, int value) throws IOException { 112 byte[] val = value(value); 113 for (int i = 0; i < NUM_CFS; i++) { 114 Path testIn = new Path(dir, family(i)); 115 116 TestHRegionServerBulkLoad.createHFile(fs, new Path(testIn, "hfile_" + i), 117 Bytes.toBytes(family(i)), QUAL, val, ROWCOUNT); 118 } 119 } 120 121 private TableDescriptor createTableDesc(TableName name, int cfs) { 122 TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(name); 123 IntStream.range(0, cfs).mapToObj(i -> ColumnFamilyDescriptorBuilder.of(family(i))) 124 .forEachOrdered(builder::setColumnFamily); 125 return builder.build(); 126 } 127 128 /** 129 * Creates a table with given table name and specified number of column families if the table does 130 * not already exist. 131 */ 132 private void setupTable(final Connection connection, TableName table, int cfs) 133 throws IOException { 134 try { 135 LOG.info("Creating table " + table); 136 try (Admin admin = connection.getAdmin()) { 137 admin.createTable(createTableDesc(table, cfs)); 138 } 139 } catch (TableExistsException tee) { 140 LOG.info("Table " + table + " already exists"); 141 } 142 } 143 144 /** 145 * Creates a table with given table name,specified number of column families<br> 146 * and splitkeys if the table does not already exist. 147 */ 148 private void setupTableWithSplitkeys(TableName table, int cfs, byte[][] SPLIT_KEYS) 149 throws IOException { 150 try { 151 LOG.info("Creating table " + table); 152 util.createTable(createTableDesc(table, cfs), SPLIT_KEYS); 153 } catch (TableExistsException tee) { 154 LOG.info("Table " + table + " already exists"); 155 } 156 } 157 158 private Path buildBulkFiles(TableName table, int value) throws Exception { 159 Path dir = util.getDataTestDirOnTestFS(table.getNameAsString()); 160 Path bulk1 = new Path(dir, table.getNameAsString() + value); 161 FileSystem fs = util.getTestFileSystem(); 162 buildHFiles(fs, bulk1, value); 163 return bulk1; 164 } 165 166 /** 167 * Populate table with known values. 168 */ 169 private void populateTable(final Connection connection, TableName table, int value) 170 throws Exception { 171 // create HFiles for different column families 172 Path dir = buildBulkFiles(table, value); 173 BulkLoadHFiles.create(util.getConfiguration()).bulkLoad(table, dir); 174 } 175 176 /** 177 * Split the known table in half. (this is hard coded for this test suite) 178 */ 179 private void forceSplit(TableName table) { 180 try { 181 // need to call regions server to by synchronous but isn't visible. 182 HRegionServer hrs = util.getRSForFirstRegionInTable(table); 183 184 for (RegionInfo hri : ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices())) { 185 if (hri.getTable().equals(table)) { 186 util.getAdmin().splitRegionAsync(hri.getRegionName(), rowkey(ROWCOUNT / 2)); 187 // ProtobufUtil.split(null, hrs.getRSRpcServices(), hri, rowkey(ROWCOUNT / 2)); 188 } 189 } 190 191 // verify that split completed. 192 int regions; 193 do { 194 regions = 0; 195 for (RegionInfo hri : ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices())) { 196 if (hri.getTable().equals(table)) { 197 regions++; 198 } 199 } 200 if (regions != 2) { 201 LOG.info("Taking some time to complete split..."); 202 Thread.sleep(250); 203 } 204 } while (regions != 2); 205 } catch (IOException e) { 206 e.printStackTrace(); 207 } catch (InterruptedException e) { 208 e.printStackTrace(); 209 } 210 } 211 212 @AfterAll 213 public static void teardownCluster() throws Exception { 214 util.shutdownMiniCluster(); 215 } 216 217 /** 218 * Checks that all columns have the expected value and that there is the expected number of rows. 219 */ 220 void assertExpectedTable(TableName table, int count, int value) throws IOException { 221 TableDescriptor htd = util.getAdmin().getDescriptor(table); 222 assertNotNull(htd); 223 try (Table t = util.getConnection().getTable(table); 224 ResultScanner sr = t.getScanner(new Scan())) { 225 int i = 0; 226 for (Result r; (r = sr.next()) != null;) { 227 r.getNoVersionMap().values().stream().flatMap(m -> m.values().stream()) 228 .forEach(v -> assertArrayEquals(value(value), v)); 229 i++; 230 } 231 assertEquals(count, i); 232 } catch (IOException e) { 233 fail("Failed due to exception"); 234 } 235 } 236 237 private static <T> CompletableFuture<T> failedFuture(Throwable error) { 238 CompletableFuture<T> future = new CompletableFuture<>(); 239 future.completeExceptionally(error); 240 return future; 241 } 242 243 private static AsyncClusterConnection mockAndInjectError(AsyncClusterConnection conn) { 244 AsyncClusterConnection errConn = spy(conn); 245 doReturn(failedFuture(new IOException("injecting bulk load error"))).when(errConn).bulkLoad( 246 any(), anyList(), any(), anyBoolean(), any(), any(), anyBoolean(), anyList(), anyBoolean()); 247 return errConn; 248 } 249 250 /** 251 * Test that shows that exception thrown from the RS side will result in an exception on the 252 * LIHFile client. 253 */ 254 @Test 255 public void testBulkLoadPhaseFailure(TestInfo testInfo) throws Exception { 256 final TableName table = TableName.valueOf(testInfo.getTestMethod().get().getName()); 257 final AtomicInteger attemptedCalls = new AtomicInteger(); 258 Configuration conf = new Configuration(util.getConfiguration()); 259 conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2); 260 BulkLoadHFilesTool loader = new BulkLoadHFilesTool(conf) { 261 262 @Override 263 protected void bulkLoadPhase(AsyncClusterConnection conn, TableName tableName, 264 Deque<LoadQueueItem> queue, Multimap<ByteBuffer, LoadQueueItem> regionGroups, 265 boolean copyFiles, Map<LoadQueueItem, ByteBuffer> item2RegionMap) throws IOException { 266 AsyncClusterConnection c = 267 attemptedCalls.incrementAndGet() == 1 ? mockAndInjectError(conn) : conn; 268 super.bulkLoadPhase(c, tableName, queue, regionGroups, copyFiles, item2RegionMap); 269 } 270 }; 271 Path dir = buildBulkFiles(table, 1); 272 assertThrows(IOException.class, () -> loader.bulkLoad(table, dir)); 273 } 274 275 /** 276 * Test that shows that exception thrown from the RS side will result in the expected number of 277 * retries set by ${@link HConstants#HBASE_CLIENT_RETRIES_NUMBER} when 278 * ${@link BulkLoadHFiles#RETRY_ON_IO_EXCEPTION} is set 279 */ 280 @Test 281 public void testRetryOnIOException(TestInfo testInfo) throws Exception { 282 TableName table = TableName.valueOf(testInfo.getTestMethod().get().getName()); 283 AtomicInteger calls = new AtomicInteger(0); 284 setupTable(util.getConnection(), table, 10); 285 Configuration conf = new Configuration(util.getConfiguration()); 286 conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2); 287 conf.setBoolean(BulkLoadHFiles.RETRY_ON_IO_EXCEPTION, true); 288 BulkLoadHFilesTool loader = new BulkLoadHFilesTool(conf) { 289 290 @Override 291 protected void bulkLoadPhase(AsyncClusterConnection conn, TableName tableName, 292 Deque<LoadQueueItem> queue, Multimap<ByteBuffer, LoadQueueItem> regionGroups, 293 boolean copyFiles, Map<LoadQueueItem, ByteBuffer> item2RegionMap) throws IOException { 294 if ( 295 calls.get() < conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 296 HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER) 297 ) { 298 calls.incrementAndGet(); 299 super.bulkLoadPhase(mockAndInjectError(conn), tableName, queue, regionGroups, copyFiles, 300 item2RegionMap); 301 } else { 302 super.bulkLoadPhase(conn, tableName, queue, regionGroups, copyFiles, item2RegionMap); 303 } 304 } 305 }; 306 Path dir = buildBulkFiles(table, 1); 307 loader.bulkLoad(table, dir); 308 assertEquals(calls.get(), 2); 309 } 310 311 /** 312 * This test exercises the path where there is a split after initial validation but before the 313 * atomic bulk load call. We cannot use presplitting to test this path, so we actually inject a 314 * split just before the atomic region load. 315 */ 316 @Test 317 public void testSplitWhileBulkLoadPhase(TestInfo testInfo) throws Exception { 318 final TableName table = TableName.valueOf(testInfo.getTestMethod().get().getName()); 319 setupTable(util.getConnection(), table, 10); 320 populateTable(util.getConnection(), table, 1); 321 assertExpectedTable(table, ROWCOUNT, 1); 322 323 // Now let's cause trouble. This will occur after checks and cause bulk 324 // files to fail when attempt to atomically import. This is recoverable. 325 final AtomicInteger attemptedCalls = new AtomicInteger(); 326 BulkLoadHFilesTool loader = new BulkLoadHFilesTool(util.getConfiguration()) { 327 328 @Override 329 protected void bulkLoadPhase(AsyncClusterConnection conn, TableName tableName, 330 Deque<LoadQueueItem> queue, Multimap<ByteBuffer, LoadQueueItem> regionGroups, 331 boolean copyFiles, Map<LoadQueueItem, ByteBuffer> item2RegionMap) throws IOException { 332 int i = attemptedCalls.incrementAndGet(); 333 if (i == 1) { 334 // On first attempt force a split. 335 forceSplit(table); 336 } 337 super.bulkLoadPhase(conn, tableName, queue, regionGroups, copyFiles, item2RegionMap); 338 } 339 }; 340 341 // create HFiles for different column families 342 Path dir = buildBulkFiles(table, 2); 343 loader.bulkLoad(table, dir); 344 345 // check that data was loaded 346 // The three expected attempts are 1) failure because need to split, 2) 347 // load of split top 3) load of split bottom 348 assertEquals(3, attemptedCalls.get()); 349 assertExpectedTable(table, ROWCOUNT, 2); 350 } 351 352 /** 353 * This test splits a table and attempts to bulk load. The bulk import files should be split 354 * before atomically importing. 355 */ 356 @Test 357 public void testGroupOrSplitPresplit(TestInfo testInfo) throws Exception { 358 final TableName table = TableName.valueOf(testInfo.getTestMethod().get().getName()); 359 setupTable(util.getConnection(), table, 10); 360 populateTable(util.getConnection(), table, 1); 361 assertExpectedTable(util.getConnection(), table, ROWCOUNT, 1); 362 forceSplit(table); 363 364 final AtomicInteger countedLqis = new AtomicInteger(); 365 BulkLoadHFilesTool loader = new BulkLoadHFilesTool(util.getConfiguration()) { 366 367 @Override 368 protected Pair<List<LoadQueueItem>, String> groupOrSplit(AsyncClusterConnection conn, 369 TableName tableName, Multimap<ByteBuffer, LoadQueueItem> regionGroups, LoadQueueItem item, 370 List<Pair<byte[], byte[]>> startEndKeys) throws IOException { 371 Pair<List<LoadQueueItem>, String> lqis = 372 super.groupOrSplit(conn, tableName, regionGroups, item, startEndKeys); 373 if (lqis != null && lqis.getFirst() != null) { 374 countedLqis.addAndGet(lqis.getFirst().size()); 375 } 376 return lqis; 377 } 378 }; 379 380 // create HFiles for different column families 381 Path dir = buildBulkFiles(table, 2); 382 loader.bulkLoad(table, dir); 383 assertExpectedTable(util.getConnection(), table, ROWCOUNT, 2); 384 assertEquals(20, countedLqis.get()); 385 } 386 387 @Test 388 public void testCorrectSplitPoint(TestInfo testInfo) throws Exception { 389 final TableName table = TableName.valueOf(testInfo.getTestMethod().get().getName()); 390 byte[][] SPLIT_KEYS = new byte[][] { Bytes.toBytes("row_00000010"), 391 Bytes.toBytes("row_00000020"), Bytes.toBytes("row_00000030"), Bytes.toBytes("row_00000040"), 392 Bytes.toBytes("row_00000050"), Bytes.toBytes("row_00000060"), Bytes.toBytes("row_00000070") }; 393 setupTableWithSplitkeys(table, NUM_CFS, SPLIT_KEYS); 394 395 final AtomicInteger bulkloadRpcTimes = new AtomicInteger(); 396 BulkLoadHFilesTool loader = new BulkLoadHFilesTool(util.getConfiguration()) { 397 398 @Override 399 protected void bulkLoadPhase(AsyncClusterConnection conn, TableName tableName, 400 Deque<LoadQueueItem> queue, Multimap<ByteBuffer, LoadQueueItem> regionGroups, 401 boolean copyFiles, Map<LoadQueueItem, ByteBuffer> item2RegionMap) throws IOException { 402 bulkloadRpcTimes.addAndGet(1); 403 super.bulkLoadPhase(conn, tableName, queue, regionGroups, copyFiles, item2RegionMap); 404 } 405 }; 406 407 Path dir = buildBulkFiles(table, 1); 408 loader.bulkLoad(table, dir); 409 // before HBASE-25281 we need invoke bulkload rpc 8 times 410 assertEquals(4, bulkloadRpcTimes.get()); 411 } 412 413 @Test 414 public void testFreshRetryAfterSplitFailure(TestInfo testInfo) throws Exception { 415 TableName table = TableName.valueOf(testInfo.getTestMethod().get().getName()); 416 setupTableWithSplitkeys(table, NUM_CFS, new byte[][] { rowkey(ROWCOUNT / 2) }); 417 418 Path dir = buildBulkFiles(table, 1); 419 FileSystem fs = dir.getFileSystem(util.getConfiguration()); 420 Path familyDir = new Path(dir, family(0)); 421 Path originalHFile = new Path(familyDir, "hfile_0"); 422 Path tmpDir = new Path(familyDir, BulkLoadHFilesTool.TMP_DIR); 423 assertTrue(fs.exists(originalHFile)); 424 425 BulkLoadHFilesTool failingLoader = new BulkLoadHFilesTool(util.getConfiguration()) { 426 @Override 427 protected void bulkLoadPhase(AsyncClusterConnection conn, TableName tableName, 428 Deque<LoadQueueItem> queue, Multimap<ByteBuffer, LoadQueueItem> regionGroups, 429 boolean copyFiles, Map<LoadQueueItem, ByteBuffer> item2RegionMap) throws IOException { 430 throw new IOException("failure after splitting"); 431 } 432 }; 433 434 assertThrows(IOException.class, () -> failingLoader.bulkLoad(table, dir)); 435 436 assertTrue(fs.exists(originalHFile)); 437 FileStatus[] abandonedChildren = CommonFSUtils.listStatus(fs, tmpDir); 438 assertNotNull(abandonedChildren); 439 assertEquals(2, abandonedChildren.length); 440 Path top = null; 441 Path bottom = null; 442 for (FileStatus child : abandonedChildren) { 443 if (child.getPath().getName().endsWith(".top")) { 444 top = child.getPath(); 445 } else if (child.getPath().getName().endsWith(".bottom")) { 446 bottom = child.getPath(); 447 } 448 } 449 assertNotNull(top); 450 assertNotNull(bottom); 451 452 // Use a new loader to simulate retrying in another process. The original HFiles must remain in 453 // the family directory so the retry does not depend on in-memory references to split outputs. 454 BulkLoadHFiles.create(util.getConfiguration()).bulkLoad(table, dir); 455 456 // The retry consumes the preserved original HFiles. Split outputs abandoned by the failed 457 // invocation remain temporary and are deliberately not treated as input because they may be 458 // incomplete. 459 assertTrue(fs.exists(originalHFile)); 460 assertTrue(fs.exists(top)); 461 assertTrue(fs.exists(bottom)); 462 assertEquals(2, CommonFSUtils.listStatus(fs, tmpDir).length); 463 assertExpectedTable(table, ROWCOUNT, 1); 464 } 465 466 /** 467 * This test creates a table with many small regions. The bulk load files would be split multiple 468 * times before all of them can be loaded successfully. 469 */ 470 @Test 471 public void testSplitTmpFileCleanUp(TestInfo testInfo) throws Exception { 472 final TableName table = TableName.valueOf(testInfo.getTestMethod().get().getName()); 473 byte[][] SPLIT_KEYS = new byte[][] { Bytes.toBytes("row_00000010"), 474 Bytes.toBytes("row_00000020"), Bytes.toBytes("row_00000030"), Bytes.toBytes("row_00000040"), 475 Bytes.toBytes("row_00000050") }; 476 setupTableWithSplitkeys(table, 10, SPLIT_KEYS); 477 478 BulkLoadHFiles loader = BulkLoadHFiles.create(util.getConfiguration()); 479 480 // create HFiles 481 Path dir = buildBulkFiles(table, 2); 482 loader.bulkLoad(table, dir); 483 // family path 484 Path tmpPath = new Path(dir, family(0)); 485 // TMP_DIR under family path 486 tmpPath = new Path(tmpPath, BulkLoadHFilesTool.TMP_DIR); 487 FileSystem fs = dir.getFileSystem(util.getConfiguration()); 488 // HFiles have been splitted, there is TMP_DIR 489 assertTrue(fs.exists(tmpPath)); 490 // TMP_DIR should have been cleaned-up 491 assertNull(CommonFSUtils.listStatus(fs, tmpPath), 492 BulkLoadHFilesTool.TMP_DIR + " should be empty."); 493 assertExpectedTable(util.getConnection(), table, ROWCOUNT, 2); 494 } 495 496 /** 497 * This simulates an remote exception which should cause LIHF to exit with an exception. 498 */ 499 @Test 500 public void testGroupOrSplitFailure(TestInfo testInfo) throws Exception { 501 final TableName tableName = TableName.valueOf(testInfo.getTestMethod().get().getName()); 502 setupTable(util.getConnection(), tableName, 10); 503 BulkLoadHFilesTool loader = new BulkLoadHFilesTool(util.getConfiguration()) { 504 505 private int i = 0; 506 507 @Override 508 protected Pair<List<LoadQueueItem>, String> groupOrSplit(AsyncClusterConnection conn, 509 TableName tableName, Multimap<ByteBuffer, LoadQueueItem> regionGroups, LoadQueueItem item, 510 List<Pair<byte[], byte[]>> startEndKeys) throws IOException { 511 i++; 512 513 if (i == 5) { 514 throw new IOException("failure"); 515 } 516 return super.groupOrSplit(conn, tableName, regionGroups, item, startEndKeys); 517 } 518 }; 519 520 // create HFiles for different column families 521 Path dir = buildBulkFiles(tableName, 1); 522 assertThrows(IOException.class, () -> loader.bulkLoad(tableName, dir)); 523 } 524 525 /** 526 * We are testing a split after initial validation but before the atomic bulk load call. We cannot 527 * use presplitting to test this path, so we actually inject a split just before the atomic region 528 * load. However, we will pass null item2RegionMap and that should not affect the bulk load 529 * behavior. 530 */ 531 @Test 532 public void testSplitWhileBulkLoadPhaseWithoutItemMap(TestInfo testInfo) throws Exception { 533 final TableName table = TableName.valueOf(testInfo.getTestMethod().get().getName()); 534 setupTable(util.getConnection(), table, 10); 535 populateTable(util.getConnection(), table, 1); 536 assertExpectedTable(table, ROWCOUNT, 1); 537 538 // Now let's cause trouble. This will occur after checks and cause bulk 539 // files to fail when attempt to atomically import. This is recoverable. 540 final AtomicInteger attemptedCalls = new AtomicInteger(); 541 BulkLoadHFilesTool loader = new BulkLoadHFilesTool(util.getConfiguration()) { 542 543 @Override 544 protected void bulkLoadPhase(final AsyncClusterConnection conn, final TableName tableName, 545 final Deque<LoadQueueItem> queue, final Multimap<ByteBuffer, LoadQueueItem> regionGroups, 546 final boolean copyFiles, final Map<LoadQueueItem, ByteBuffer> item2RegionMap) 547 throws IOException { 548 549 int i = attemptedCalls.incrementAndGet(); 550 if (i == 1) { 551 // On first attempt force a split. 552 forceSplit(table); 553 } 554 555 // Passing item2RegionMap null 556 // In the absence of LoadQueueItem, bulk load should work as expected 557 super.bulkLoadPhase(conn, tableName, queue, regionGroups, copyFiles, null); 558 } 559 560 }; 561 562 // create HFiles for different column families 563 Path dir = buildBulkFiles(table, 2); 564 loader.bulkLoad(table, dir); 565 566 // check that data was loaded 567 // The three expected attempts are 1) failure because need to split, 2) 568 // load of split top 3) load of split bottom 569 assertEquals(3, attemptedCalls.get()); 570 assertExpectedTable(table, ROWCOUNT, 2); 571 } 572 573 /** 574 * Checks that all columns have the expected value and that there is the expected number of rows. 575 */ 576 void assertExpectedTable(final Connection connection, TableName table, int count, int value) 577 throws IOException { 578 TableDescriptor htd = util.getAdmin().getDescriptor(table); 579 assertNotNull(htd); 580 try (Table t = connection.getTable(table); ResultScanner sr = t.getScanner(new Scan())) { 581 int i = 0; 582 for (Result r; (r = sr.next()) != null;) { 583 r.getNoVersionMap().values().stream().flatMap(m -> m.values().stream()) 584 .forEach(v -> assertArrayEquals(value(value), v)); 585 i++; 586 } 587 assertEquals(count, i); 588 } catch (IOException e) { 589 fail("Failed due to exception"); 590 } 591 } 592}