001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.regionserver; 019 020import static org.apache.hadoop.hbase.HBaseTestingUtil.fam1; 021import static org.apache.hadoop.hbase.HBaseTestingUtil.fam2; 022import static org.apache.hadoop.hbase.HBaseTestingUtil.fam3; 023import static org.junit.jupiter.api.Assertions.assertArrayEquals; 024import static org.junit.jupiter.api.Assertions.assertEquals; 025import static org.junit.jupiter.api.Assertions.assertFalse; 026import static org.junit.jupiter.api.Assertions.assertInstanceOf; 027import static org.junit.jupiter.api.Assertions.assertNull; 028import static org.junit.jupiter.api.Assertions.assertThrows; 029import static org.junit.jupiter.api.Assertions.assertTrue; 030import static org.junit.jupiter.api.Assertions.fail; 031import static org.mockito.Mockito.doThrow; 032import static org.mockito.Mockito.mock; 033 034import java.io.IOException; 035import java.io.InterruptedIOException; 036import java.util.ArrayList; 037import java.util.Arrays; 038import java.util.List; 039import java.util.concurrent.CountDownLatch; 040import java.util.concurrent.TimeUnit; 041import java.util.concurrent.atomic.AtomicBoolean; 042import java.util.concurrent.atomic.AtomicLong; 043import java.util.concurrent.atomic.AtomicReference; 044import org.apache.hadoop.conf.Configuration; 045import org.apache.hadoop.hbase.Cell; 046import org.apache.hadoop.hbase.CellUtil; 047import org.apache.hadoop.hbase.CompareOperator; 048import org.apache.hadoop.hbase.DroppedSnapshotException; 049import org.apache.hadoop.hbase.ExtendedCell; 050import org.apache.hadoop.hbase.HBaseConfiguration; 051import org.apache.hadoop.hbase.HBaseTestingUtil; 052import org.apache.hadoop.hbase.HConstants; 053import org.apache.hadoop.hbase.MultithreadedTestUtil; 054import org.apache.hadoop.hbase.MultithreadedTestUtil.RepeatingTestThread; 055import org.apache.hadoop.hbase.NotServingRegionException; 056import org.apache.hadoop.hbase.TableName; 057import org.apache.hadoop.hbase.client.Append; 058import org.apache.hadoop.hbase.client.ClientInternalHelper; 059import org.apache.hadoop.hbase.client.Delete; 060import org.apache.hadoop.hbase.client.Durability; 061import org.apache.hadoop.hbase.client.Get; 062import org.apache.hadoop.hbase.client.Increment; 063import org.apache.hadoop.hbase.client.Mutation; 064import org.apache.hadoop.hbase.client.Put; 065import org.apache.hadoop.hbase.client.RegionInfo; 066import org.apache.hadoop.hbase.client.RegionInfoBuilder; 067import org.apache.hadoop.hbase.client.Result; 068import org.apache.hadoop.hbase.client.RowMutations; 069import org.apache.hadoop.hbase.client.Scan; 070import org.apache.hadoop.hbase.filter.BinaryComparator; 071import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; 072import org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL; 073import org.apache.hadoop.hbase.testclassification.LargeTests; 074import org.apache.hadoop.hbase.testclassification.VerySlowRegionServerTests; 075import org.apache.hadoop.hbase.util.Bytes; 076import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; 077import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper; 078import org.apache.hadoop.hbase.wal.NettyAsyncFSWALConfigHelper; 079import org.apache.hadoop.hbase.wal.WAL; 080import org.junit.jupiter.api.AfterEach; 081import org.junit.jupiter.api.BeforeEach; 082import org.junit.jupiter.api.Tag; 083import org.junit.jupiter.api.Test; 084import org.junit.jupiter.api.TestInfo; 085import org.slf4j.Logger; 086import org.slf4j.LoggerFactory; 087 088import org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup; 089import org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoopGroup; 090import org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel; 091 092/** 093 * Concurrent region operations with regular and in-memory-compacting MemStores. 094 */ 095@Tag(VerySlowRegionServerTests.TAG) 096@Tag(LargeTests.TAG) 097public class TestHRegionConcurrency { 098 099 private static final Logger LOG = LoggerFactory.getLogger(TestHRegionConcurrency.class); 100 private static final EventLoopGroup GROUP = new NioEventLoopGroup(); 101 private static final long THREAD_JOIN_TIMEOUT_NS = TimeUnit.SECONDS.toNanos(30); 102 private static final int MAX_STORE_FILES = 10; 103 104 private HBaseTestingUtil testUtil; 105 private Configuration conf; 106 private HRegion region; 107 private String method; 108 private TableName tableName; 109 private final byte[] qual1 = Bytes.toBytes("qual1"); 110 private final byte[] qual2 = Bytes.toBytes("qual2"); 111 private final byte[] qual3 = Bytes.toBytes("qual3"); 112 113 protected boolean isInMemory() { 114 return false; 115 } 116 117 @BeforeEach 118 public void setup(TestInfo testInfo) { 119 testUtil = new HBaseTestingUtil(); 120 conf = testUtil.getConfiguration(); 121 NettyAsyncFSWALConfigHelper.setEventLoopConfig(conf, GROUP, NioSocketChannel.class); 122 method = testInfo.getTestMethod().get().getName(); 123 tableName = TableName.valueOf(method); 124 conf.set(CompactingMemStore.IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, String.valueOf(0.09)); 125 conf.setLong(AbstractFSWAL.WAL_SYNC_TIMEOUT_MS, 10000); 126 } 127 128 @AfterEach 129 public void tearDown() throws IOException { 130 try { 131 HBaseTestingUtil.closeRegionAndWAL(region); 132 } finally { 133 EnvironmentEdgeManagerTestHelper.reset(); 134 testUtil.cleanupTestDir(); 135 } 136 } 137 138 private HRegion initHRegion(Configuration configuration, byte[]... families) throws IOException { 139 RegionInfo hri = RegionInfoBuilder.newBuilder(tableName).build(); 140 WAL wal = HBaseTestingUtil.createWal(configuration, 141 testUtil.getDataTestDirOnTestFS(method + ".log"), hri); 142 ChunkCreator.initialize(MemStoreLAB.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null, 143 MemStoreLAB.INDEX_CHUNK_SIZE_PERCENTAGE_DEFAULT); 144 if (isInMemory()) { 145 boolean[] flags = new boolean[families.length]; 146 Arrays.fill(flags, true); 147 return testUtil.createLocalHRegionWithInMemoryFlags(tableName, null, null, configuration, 148 false, Durability.SYNC_WAL, wal, flags, families); 149 } 150 return testUtil.createLocalHRegion(tableName, null, null, configuration, false, 151 Durability.SYNC_WAL, wal, families); 152 } 153 154 private Configuration initSplit() { 155 conf.setInt("hbase.hstore.compactionThreshold", 2); 156 conf.setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, 10 * 1000); 157 conf.setLong("hbase.client.pause", 15 * 1000); 158 conf.setLong(HConstants.HREGION_MAX_FILESIZE, 1024 * 128); 159 return conf; 160 } 161 162 private static void interruptAndJoinThreads(Thread... threads) { 163 for (Thread thread : threads) { 164 if (thread != null) { 165 thread.interrupt(); 166 } 167 } 168 joinThreads(threads); 169 } 170 171 private static void joinThreads(Thread... threads) { 172 long deadline = System.nanoTime() + THREAD_JOIN_TIMEOUT_NS; 173 boolean interrupted = false; 174 List<String> aliveThreads = new ArrayList<>(); 175 for (Thread thread : threads) { 176 if (thread == null) { 177 continue; 178 } 179 while (thread.isAlive()) { 180 long remaining = deadline - System.nanoTime(); 181 if (remaining <= 0) { 182 break; 183 } 184 try { 185 TimeUnit.NANOSECONDS.timedJoin(thread, remaining); 186 } catch (InterruptedException e) { 187 interrupted = true; 188 } 189 } 190 if (thread.isAlive()) { 191 aliveThreads.add(thread.getName()); 192 } 193 } 194 if (interrupted) { 195 Thread.currentThread().interrupt(); 196 } 197 assertTrue(aliveThreads.isEmpty(), "Threads did not stop: " + aliveThreads); 198 } 199 200 @Test 201 public void testGetWhileRegionClose() throws Exception { 202 Configuration hc = initSplit(); 203 byte[][] families = { fam1, fam2, fam3 }; 204 region = initHRegion(hc, families); 205 final int startRow = 100; 206 for (byte[] qualifier : new byte[][] { qual1, qual2, qual3 }) { 207 for (int i = startRow; i < startRow + 100; i++) { 208 Put put = new Put(Bytes.toBytes("" + i)); 209 put.setDurability(Durability.SKIP_WAL); 210 for (byte[] family : families) { 211 put.addColumn(family, qualifier, null); 212 } 213 region.put(put); 214 } 215 } 216 217 AtomicBoolean done = new AtomicBoolean(false); 218 CountDownLatch initialGets = new CountDownLatch(5); 219 GetTillDoneOrException[] threads = new GetTillDoneOrException[10]; 220 try { 221 for (int i = 0; i < threads.length / 2; i++) { 222 threads[i] = 223 new GetTillDoneOrException(region, i, Bytes.toBytes("" + startRow), done, initialGets); 224 threads[i].start(); 225 } 226 assertTrue(initialGets.await(30, TimeUnit.SECONDS), "Initial getters did not read"); 227 // Force the closing state while readers are still active. 228 region.closing.set(true); 229 for (int i = threads.length / 2; i < threads.length; i++) { 230 threads[i] = 231 new GetTillDoneOrException(region, i, Bytes.toBytes("" + startRow), done, initialGets); 232 threads[i].start(); 233 } 234 joinThreads(threads); 235 for (GetTillDoneOrException thread : threads) { 236 assertInstanceOf(NotServingRegionException.class, thread.error, thread.getName()); 237 } 238 } finally { 239 done.set(true); 240 interruptAndJoinThreads(threads); 241 } 242 } 243 244 private class GetTillDoneOrException extends Thread { 245 private final HRegion testRegion; 246 private final Get get; 247 private final AtomicBoolean done; 248 private final CountDownLatch initialGets; 249 private Throwable error; 250 251 GetTillDoneOrException(HRegion testRegion, int i, byte[] row, AtomicBoolean done, 252 CountDownLatch initialGets) { 253 super("Getter-" + i + "-" + method); 254 this.testRegion = testRegion; 255 this.get = new Get(row); 256 this.done = done; 257 this.initialGets = initialGets; 258 } 259 260 @Override 261 public void run() { 262 boolean firstGet = true; 263 while (!done.get() && !isInterrupted()) { 264 try { 265 assertTrue(testRegion.get(get).size() > 0); 266 if (firstGet) { 267 initialGets.countDown(); 268 firstGet = false; 269 } 270 } catch (Throwable t) { 271 error = t; 272 return; 273 } 274 } 275 } 276 } 277 278 /** 279 * Flushes the cache in a thread while scanning. The tests verify that the scan is coherent - e.g. 280 * the returned results are always of the same or later update as the previous results. scan / 281 * compact thread join 282 */ 283 @Test 284 public void testFlushCacheWhileScanning() throws IOException, InterruptedException { 285 byte[] family = Bytes.toBytes("family"); 286 int numRows = 1000; 287 int flushAndScanInterval = 10; 288 int compactInterval = 10 * flushAndScanInterval; 289 290 this.region = initHRegion(conf, family); 291 FlushThread flushThread = new FlushThread(); 292 try { 293 flushThread.start(); 294 295 Scan scan = new Scan(); 296 scan.addFamily(family); 297 scan.setFilter(new SingleColumnValueFilter(family, qual1, CompareOperator.EQUAL, 298 new BinaryComparator(Bytes.toBytes(5L)))); 299 300 int expectedCount = 0; 301 List<Cell> res = new ArrayList<>(); 302 303 boolean toggle = true; 304 for (long i = 0; i < numRows; i++) { 305 Put put = new Put(Bytes.toBytes(i)); 306 put.setDurability(Durability.SKIP_WAL); 307 put.addColumn(family, qual1, Bytes.toBytes(i % 10)); 308 region.put(put); 309 310 if (i != 0 && i % compactInterval == 0) { 311 LOG.debug("iteration = " + i + " ts=" + EnvironmentEdgeManager.currentTime()); 312 region.compact(true); 313 } 314 315 if (i % 10 == 5L) { 316 expectedCount++; 317 } 318 319 if (i != 0 && i % flushAndScanInterval == 0) { 320 res.clear(); 321 try (InternalScanner scanner = region.getScanner(scan)) { 322 if (toggle) { 323 flushThread.flush(); 324 } 325 while (scanner.next(res)) { 326 // ignore 327 } 328 } 329 if (!toggle) { 330 flushThread.flush(); 331 } 332 assertEquals(expectedCount, res.size(), 333 "toggle=" + toggle + "i=" + i + " ts=" + EnvironmentEdgeManager.currentTime()); 334 toggle = !toggle; 335 } 336 } 337 338 } finally { 339 flushThread.done(); 340 try { 341 joinThreads(flushThread); 342 } finally { 343 interruptAndJoinThreads(flushThread); 344 } 345 flushThread.checkNoError(); 346 HBaseTestingUtil.closeRegionAndWAL(this.region); 347 this.region = null; 348 } 349 } 350 351 private class FlushThread extends Thread { 352 private volatile boolean done; 353 private Throwable error = null; 354 355 FlushThread() { 356 super("FlushThread-" + method); 357 } 358 359 public void done() { 360 synchronized (this) { 361 done = true; 362 notifyAll(); 363 } 364 } 365 366 public void checkNoError() { 367 if (error != null) { 368 assertNull(error); 369 } 370 } 371 372 @Override 373 public void run() { 374 while (!done && !Thread.currentThread().isInterrupted()) { 375 synchronized (this) { 376 if (done) { 377 break; 378 } 379 try { 380 wait(); 381 } catch (InterruptedException e) { 382 Thread.currentThread().interrupt(); 383 break; 384 } 385 if (done) { 386 break; 387 } 388 } 389 try { 390 region.flush(true); 391 } catch (IOException e) { 392 if (!done) { 393 LOG.error("Error while flushing cache", e); 394 error = e; 395 } 396 break; 397 } catch (Throwable t) { 398 LOG.error("Uncaught exception", t); 399 throw t; 400 } 401 } 402 } 403 404 public void flush() { 405 synchronized (this) { 406 notify(); 407 } 408 } 409 } 410 411 /** 412 * So can be overridden in subclasses. 413 */ 414 protected int getNumQualifiersForTestWritesWhileScanning() { 415 return 100; 416 } 417 418 /** 419 * So can be overridden in subclasses. 420 */ 421 protected int getTestCountForTestWritesWhileScanning() { 422 return 100; 423 } 424 425 /** 426 * Writes very wide records and scans for the latest every time.. Flushes and compacts the region 427 * every now and then to keep things realistic. by flush / scan / compaction when joining threads 428 */ 429 @Test 430 public void testWritesWhileScanning() throws IOException, InterruptedException { 431 int testCount = getTestCountForTestWritesWhileScanning(); 432 int numRows = 1; 433 int numFamilies = 10; 434 int numQualifiers = getNumQualifiersForTestWritesWhileScanning(); 435 int flushInterval = 7; 436 int compactInterval = 5 * flushInterval; 437 byte[][] families = new byte[numFamilies][]; 438 for (int i = 0; i < numFamilies; i++) { 439 families[i] = Bytes.toBytes("family" + i); 440 } 441 byte[][] qualifiers = new byte[numQualifiers][]; 442 for (int i = 0; i < numQualifiers; i++) { 443 qualifiers[i] = Bytes.toBytes("qual" + i); 444 } 445 446 this.region = initHRegion(conf, families); 447 FlushThread flushThread = new FlushThread(); 448 PutThread putThread = new PutThread(numRows, families, qualifiers); 449 try { 450 putThread.start(); 451 putThread.waitForFirstPut(); 452 453 flushThread.start(); 454 455 Scan scan = new Scan().withStartRow(Bytes.toBytes("row0")).withStopRow(Bytes.toBytes("row1")); 456 457 int expectedCount = numFamilies * numQualifiers; 458 List<Cell> res = new ArrayList<>(); 459 460 long prevTimestamp = 0L; 461 for (int i = 0; i < testCount; i++) { 462 463 if (i != 0 && i % compactInterval == 0) { 464 region.compact(true); 465 for (HStore store : region.getStores()) { 466 store.closeAndArchiveCompactedFiles(); 467 } 468 } 469 470 if (i != 0 && i % flushInterval == 0) { 471 flushThread.flush(); 472 } 473 474 boolean previousEmpty = res.isEmpty(); 475 res.clear(); 476 try (InternalScanner scanner = region.getScanner(scan)) { 477 boolean moreRows; 478 do { 479 moreRows = scanner.next(res); 480 } while (moreRows); 481 } 482 if (!res.isEmpty() || !previousEmpty || i > compactInterval) { 483 assertEquals(expectedCount, res.size(), "i=" + i); 484 long timestamp = res.get(0).getTimestamp(); 485 assertTrue(timestamp >= prevTimestamp, 486 "Timestamps were broke: " + timestamp + " prev: " + prevTimestamp); 487 prevTimestamp = timestamp; 488 } 489 } 490 491 putThread.done(); 492 493 region.flush(true); 494 495 } finally { 496 putThread.done(); 497 flushThread.done(); 498 try { 499 joinThreads(flushThread, putThread); 500 } finally { 501 interruptAndJoinThreads(flushThread, putThread); 502 } 503 flushThread.checkNoError(); 504 putThread.checkNoError(); 505 506 try { 507 HBaseTestingUtil.closeRegionAndWAL(this.region); 508 } catch (DroppedSnapshotException dse) { 509 // We could get this on way out because we interrupt the background flusher and it could 510 // fail anywhere causing a DSE over in the background flusher... only it is not properly 511 // dealt with so could still be memory hanging out when we get to here -- memory we can't 512 // flush because the accounting is 'off' since original DSE. 513 } 514 this.region = null; 515 } 516 } 517 518 private final class PutThread extends Thread { 519 private volatile boolean done; 520 private volatile int numPutsFinished = 0; 521 522 private Throwable error = null; 523 private int numRows; 524 private byte[][] families; 525 private byte[][] qualifiers; 526 527 private PutThread(int numRows, byte[][] families, byte[][] qualifiers) { 528 super("PutThread-" + method); 529 this.numRows = numRows; 530 this.families = families; 531 this.qualifiers = qualifiers; 532 } 533 534 /** 535 * Block calling thread until this instance of PutThread has put at least one row. 536 */ 537 public void waitForFirstPut() throws InterruptedException { 538 // wait until put thread actually puts some data 539 while (isAlive() && numPutsFinished == 0) { 540 checkNoError(); 541 Thread.sleep(50); 542 } 543 } 544 545 public void done() { 546 done = true; 547 interrupt(); 548 } 549 550 public void checkNoError() { 551 if (error != null) { 552 assertNull(error); 553 } 554 } 555 556 @Override 557 public void run() { 558 while (!done && !Thread.currentThread().isInterrupted()) { 559 try { 560 for (int r = 0; r < numRows; r++) { 561 byte[] row = Bytes.toBytes("row" + r); 562 Put put = new Put(row); 563 put.setDurability(Durability.SKIP_WAL); 564 byte[] value = Bytes.toBytes(String.valueOf(numPutsFinished)); 565 for (byte[] family : families) { 566 for (byte[] qualifier : qualifiers) { 567 put.addColumn(family, qualifier, numPutsFinished, value); 568 } 569 } 570 region.put(put); 571 numPutsFinished++; 572 if (numPutsFinished > 0 && numPutsFinished % 47 == 0) { 573 LOG.debug("put iteration = {}", numPutsFinished); 574 Delete delete = new Delete(row, (long) numPutsFinished - 30); 575 region.delete(delete); 576 } 577 numPutsFinished++; 578 } 579 } catch (InterruptedIOException e) { 580 // This is fine. It means we are done, or didn't get the lock on time 581 LOG.info("Interrupted", e); 582 } catch (IOException e) { 583 LOG.error("Error while putting records", e); 584 error = e; 585 break; 586 } 587 } 588 589 } 590 591 } 592 593 /** 594 * Writes very wide records and gets the latest row every time.. Flushes and compacts the region 595 * aggressivly to catch issues. by flush / scan / compaction when joining threads 596 */ 597 @Test 598 public void testWritesWhileGetting() throws Exception { 599 int testCount = 50; 600 int numRows = 1; 601 int numFamilies = 10; 602 int numQualifiers = 100; 603 int compactInterval = 100; 604 byte[][] families = new byte[numFamilies][]; 605 for (int i = 0; i < numFamilies; i++) { 606 families[i] = Bytes.toBytes("family" + i); 607 } 608 byte[][] qualifiers = new byte[numQualifiers][]; 609 for (int i = 0; i < numQualifiers; i++) { 610 qualifiers[i] = Bytes.toBytes("qual" + i); 611 } 612 613 // This test flushes constantly and can cause many files to be created, 614 // possibly 615 // extending over the ulimit. Make sure compactions are aggressive in 616 // reducing 617 // the number of HFiles created. 618 Configuration configuration = HBaseConfiguration.create(conf); 619 configuration.setInt("hbase.hstore.compaction.min", 1); 620 configuration.setInt("hbase.hstore.compaction.max", 1000); 621 this.region = initHRegion(configuration, families); 622 PutThread putThread = new PutThread(numRows, families, qualifiers); 623 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(configuration); 624 RepeatingTestThread flushThread = createCompactingFlushThread(ctx); 625 try { 626 putThread.start(); 627 putThread.waitForFirstPut(); 628 ctx.addThread(flushThread); 629 ctx.startThreads(); 630 631 Get get = new Get(Bytes.toBytes("row0")); 632 Result result = null; 633 634 int expectedCount = numFamilies * numQualifiers; 635 636 long prevTimestamp = 0L; 637 for (int i = 0; i < testCount; i++) { 638 LOG.info("testWritesWhileGetting verify turn " + i); 639 boolean previousEmpty = result == null || result.isEmpty(); 640 result = region.get(get); 641 if (!result.isEmpty() || !previousEmpty || i > compactInterval) { 642 assertEquals(expectedCount, result.size(), "i=" + i); 643 // TODO this was removed, now what dangit?! 644 // search looking for the qualifier in question? 645 long timestamp = 0; 646 for (Cell kv : result.rawCells()) { 647 if ( 648 CellUtil.matchingFamily(kv, families[0]) 649 && CellUtil.matchingQualifier(kv, qualifiers[0]) 650 ) { 651 timestamp = kv.getTimestamp(); 652 } 653 } 654 assertTrue(timestamp >= prevTimestamp); 655 prevTimestamp = timestamp; 656 ExtendedCell previousKV = null; 657 658 for (ExtendedCell kv : ClientInternalHelper.getExtendedRawCells(result)) { 659 byte[] thisValue = CellUtil.cloneValue(kv); 660 if (previousKV != null) { 661 if (Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue) != 0) { 662 LOG.warn("These two KV should have the same value." + " Previous KV:" + previousKV 663 + "(memStoreTS:" + previousKV.getSequenceId() + ")" + ", New KV: " + kv 664 + "(memStoreTS:" + kv.getSequenceId() + ")"); 665 assertEquals(0, Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue)); 666 } 667 } 668 previousKV = kv; 669 } 670 } 671 } 672 } finally { 673 stopWriteThreads(putThread, ctx, flushThread); 674 } 675 region.flush(true); 676 } 677 678 /** 679 * TestCase for increment 680 */ 681 private static class Incrementer implements Runnable { 682 private HRegion region; 683 private final static byte[] incRow = Bytes.toBytes("incRow"); 684 private final static byte[] family = Bytes.toBytes("family"); 685 private final static byte[] qualifier = Bytes.toBytes("qualifier"); 686 private final static long ONE = 1L; 687 private int incCounter; 688 689 public Incrementer(HRegion region, int incCounter) { 690 this.region = region; 691 this.incCounter = incCounter; 692 } 693 694 @Override 695 public void run() { 696 int count = 0; 697 while (count < incCounter && !Thread.currentThread().isInterrupted()) { 698 Increment inc = new Increment(incRow); 699 inc.addColumn(family, qualifier, ONE); 700 count++; 701 try { 702 region.increment(inc); 703 } catch (IOException e) { 704 LOG.info("Count=" + count + ", " + e); 705 break; 706 } 707 } 708 } 709 } 710 711 /** 712 * Test case to check increment function with memstore flushing 713 */ 714 @Test 715 public void testParallelIncrementWithMemStoreFlush() throws Exception { 716 byte[] family = Incrementer.family; 717 this.region = initHRegion(conf, family); 718 final HRegion region = this.region; 719 final AtomicBoolean incrementDone = new AtomicBoolean(false); 720 Runnable flusher = new Runnable() { 721 @Override 722 public void run() { 723 while (!incrementDone.get() && !Thread.currentThread().isInterrupted()) { 724 try { 725 flushAndCompact(region, family); 726 } catch (Exception e) { 727 if (!incrementDone.get()) { 728 LOG.warn("Failed to flush or compact region", e); 729 } 730 } 731 } 732 } 733 }; 734 735 // after all increment finished, the row will increment to 20*100 = 2000 736 int threadNum = 20; 737 int incCounter = 100; 738 long expected = (long) threadNum * incCounter; 739 List<Thread> incrementers = new ArrayList<>(threadNum); 740 Thread flushThread = new Thread(flusher, "FlushThread-" + method); 741 try { 742 for (int i = 0; i < threadNum; i++) { 743 Thread incrementer = 744 new Thread(new Incrementer(this.region, incCounter), "Incrementer-" + i + "-" + method); 745 incrementers.add(incrementer); 746 incrementer.start(); 747 } 748 flushThread.start(); 749 for (Thread incrementer : incrementers) { 750 incrementer.join(); 751 } 752 753 incrementDone.set(true); 754 joinThreads(flushThread); 755 assertTrue(region.getStore(family).getStorefilesCount() < MAX_STORE_FILES); 756 757 Get get = new Get(Incrementer.incRow); 758 get.addColumn(Incrementer.family, Incrementer.qualifier); 759 get.readVersions(1); 760 Result res = this.region.get(get); 761 List<Cell> kvs = res.getColumnCells(Incrementer.family, Incrementer.qualifier); 762 763 // we just got the latest version 764 assertEquals(1, kvs.size()); 765 Cell kv = kvs.get(0); 766 assertEquals(expected, Bytes.toLong(kv.getValueArray(), kv.getValueOffset())); 767 } finally { 768 incrementDone.set(true); 769 incrementers.add(flushThread); 770 interruptAndJoinThreads(incrementers.toArray(new Thread[0])); 771 } 772 } 773 774 /** 775 * TestCase for append 776 */ 777 private static class Appender implements Runnable { 778 private HRegion region; 779 private final static byte[] appendRow = Bytes.toBytes("appendRow"); 780 private final static byte[] family = Bytes.toBytes("family"); 781 private final static byte[] qualifier = Bytes.toBytes("qualifier"); 782 private final static byte[] CHAR = Bytes.toBytes("a"); 783 private int appendCounter; 784 785 public Appender(HRegion region, int appendCounter) { 786 this.region = region; 787 this.appendCounter = appendCounter; 788 } 789 790 @Override 791 public void run() { 792 int count = 0; 793 while (count < appendCounter && !Thread.currentThread().isInterrupted()) { 794 Append app = new Append(appendRow); 795 app.addColumn(family, qualifier, CHAR); 796 count++; 797 try { 798 region.append(app); 799 } catch (IOException e) { 800 LOG.info("Count=" + count + ", max=" + appendCounter + ", " + e); 801 break; 802 } 803 } 804 } 805 } 806 807 /** 808 * Test case to check append function with memstore flushing 809 */ 810 @Test 811 public void testParallelAppendWithMemStoreFlush() throws Exception { 812 byte[] family = Appender.family; 813 this.region = initHRegion(conf, family); 814 final HRegion region = this.region; 815 final AtomicBoolean appendDone = new AtomicBoolean(false); 816 Runnable flusher = new Runnable() { 817 @Override 818 public void run() { 819 while (!appendDone.get() && !Thread.currentThread().isInterrupted()) { 820 try { 821 flushAndCompact(region, family); 822 } catch (Exception e) { 823 if (!appendDone.get()) { 824 LOG.warn("Failed to flush or compact region", e); 825 } 826 } 827 } 828 } 829 }; 830 831 // After all append finished, the value will append to threadNum * 832 // appendCounter Appender.CHAR 833 int threadNum = 20; 834 int appendCounter = 100; 835 byte[] expected = new byte[threadNum * appendCounter]; 836 for (int i = 0; i < threadNum * appendCounter; i++) { 837 System.arraycopy(Appender.CHAR, 0, expected, i, 1); 838 } 839 List<Thread> appenders = new ArrayList<>(threadNum); 840 Thread flushThread = new Thread(flusher, "FlushThread-" + method); 841 try { 842 for (int i = 0; i < threadNum; i++) { 843 Thread appender = 844 new Thread(new Appender(this.region, appendCounter), "Appender-" + i + "-" + method); 845 appenders.add(appender); 846 appender.start(); 847 } 848 flushThread.start(); 849 for (Thread appender : appenders) { 850 appender.join(); 851 } 852 853 appendDone.set(true); 854 joinThreads(flushThread); 855 assertTrue(region.getStore(family).getStorefilesCount() < MAX_STORE_FILES); 856 857 Get get = new Get(Appender.appendRow); 858 get.addColumn(Appender.family, Appender.qualifier); 859 get.readVersions(1); 860 Result res = this.region.get(get); 861 List<Cell> kvs = res.getColumnCells(Appender.family, Appender.qualifier); 862 863 // we just got the latest version 864 assertEquals(1, kvs.size()); 865 Cell kv = kvs.get(0); 866 byte[] appendResult = new byte[kv.getValueLength()]; 867 System.arraycopy(kv.getValueArray(), kv.getValueOffset(), appendResult, 0, 868 kv.getValueLength()); 869 assertArrayEquals(expected, appendResult); 870 } finally { 871 appendDone.set(true); 872 appenders.add(flushThread); 873 interruptAndJoinThreads(appenders.toArray(new Thread[0])); 874 } 875 } 876 877 private static void flushAndCompact(HRegion region, byte[] family) throws IOException { 878 region.flush(true); 879 HStore store = region.getStore(family); 880 // Standalone regions have no background compaction or compacted-file discharger. 881 if (store.getStorefilesCount() >= MAX_STORE_FILES) { 882 region.compact(true); 883 store.closeAndArchiveCompactedFiles(); 884 } 885 } 886 887 @Test 888 public void testFlusherCompactsStoreFiles() throws IOException { 889 region = initHRegion(conf, fam1); 890 HStore store = region.getStore(fam1); 891 for (int i = 0; i < MAX_STORE_FILES * 2; i++) { 892 region.put(new Put(Bytes.toBytes(i)).addColumn(fam1, qual1, Bytes.toBytes(i))); 893 flushAndCompact(region, fam1); 894 int storeFiles = store.getStorefilesCount(); 895 assertTrue(storeFiles > 0 && storeFiles < MAX_STORE_FILES, 896 "Unexpected store file count after flush " + i + ": " + storeFiles); 897 } 898 assertEquals(0, store.getCompactedFilesCount()); 899 for (int i = 0; i < MAX_STORE_FILES * 2; i++) { 900 Result result = region.get(new Get(Bytes.toBytes(i))); 901 assertEquals(i, Bytes.toInt(result.getValue(fam1, qual1))); 902 } 903 } 904 905 @Test 906 public void testMutateRowInParallel() throws Exception { 907 final int numReaderThreads = 100; 908 final CountDownLatch latch = new CountDownLatch(numReaderThreads); 909 910 final byte[] row = Bytes.toBytes("row"); 911 final byte[] q1 = Bytes.toBytes("q1"); 912 final byte[] q2 = Bytes.toBytes("q2"); 913 final byte[] q3 = Bytes.toBytes("q3"); 914 final byte[] q4 = Bytes.toBytes("q4"); 915 final String v1 = "v1"; 916 final String v2 = "v2"; 917 918 // We need to ensure the timestamp of the delete operation is more than the previous one 919 final AtomicLong deleteTimestamp = new AtomicLong(); 920 921 region = initHRegion(conf, fam1); 922 923 // Initial values 924 region.batchMutate(new Mutation[] { new Put(row).addColumn(fam1, q1, Bytes.toBytes(v1)) 925 .addColumn(fam1, q2, deleteTimestamp.getAndIncrement(), Bytes.toBytes(v2)) 926 .addColumn(fam1, q3, Bytes.toBytes(1L)).addColumn(fam1, q4, Bytes.toBytes("a")) }); 927 928 final AtomicReference<AssertionError> assertionError = new AtomicReference<>(); 929 List<Thread> threads = new ArrayList<>(numReaderThreads + 1); 930 931 // Writer thread 932 Thread writerThread = new Thread(() -> { 933 try { 934 while (!Thread.currentThread().isInterrupted()) { 935 // If all the reader threads finish, then stop the writer thread 936 if (latch.await(0, TimeUnit.MILLISECONDS)) { 937 return; 938 } 939 940 // Execute the mutations. This should be done atomically 941 region.mutateRow(new RowMutations(row) 942 .add(Arrays.asList(new Put(row).addColumn(fam1, q1, Bytes.toBytes(v2)), 943 new Delete(row).addColumns(fam1, q2, deleteTimestamp.getAndIncrement()), 944 new Increment(row).addColumn(fam1, q3, 1L), 945 new Append(row).addColumn(fam1, q4, Bytes.toBytes("b"))))); 946 947 // We need to ensure the timestamps of the Increment/Append operations are more than the 948 // previous ones 949 Result result = region.get(new Get(row).addColumn(fam1, q3).addColumn(fam1, q4)); 950 long tsIncrement = result.getColumnLatestCell(fam1, q3).getTimestamp(); 951 long tsAppend = result.getColumnLatestCell(fam1, q4).getTimestamp(); 952 953 // Put the initial values 954 region.batchMutate(new Mutation[] { new Put(row).addColumn(fam1, q1, Bytes.toBytes(v1)) 955 .addColumn(fam1, q2, deleteTimestamp.getAndIncrement(), Bytes.toBytes(v2)) 956 .addColumn(fam1, q3, tsIncrement + 1, Bytes.toBytes(1L)) 957 .addColumn(fam1, q4, tsAppend + 1, Bytes.toBytes("a")) }); 958 } 959 } catch (InterruptedException e) { 960 Thread.currentThread().interrupt(); 961 } catch (Exception e) { 962 assertionError.compareAndSet(null, new AssertionError(e)); 963 } 964 }, "WriterThread-" + method); 965 threads.add(writerThread); 966 967 try { 968 writerThread.start(); 969 970 // Reader threads 971 for (int i = 0; i < numReaderThreads; i++) { 972 Thread readerThread = new Thread(() -> { 973 try { 974 for (int j = 0; j < 10000 && assertionError.get() == null 975 && !Thread.currentThread().isInterrupted(); j++) { 976 // Verify the values 977 Result result = region.get(new Get(row)); 978 979 // The values should be equals to either the initial values or the values after 980 // executing the mutations 981 String q1Value = Bytes.toString(result.getValue(fam1, q1)); 982 if (v1.equals(q1Value)) { 983 assertEquals(v2, Bytes.toString(result.getValue(fam1, q2))); 984 assertEquals(1L, Bytes.toLong(result.getValue(fam1, q3))); 985 assertEquals("a", Bytes.toString(result.getValue(fam1, q4))); 986 } else if (v2.equals(q1Value)) { 987 assertNull(Bytes.toString(result.getValue(fam1, q2))); 988 assertEquals(2L, Bytes.toLong(result.getValue(fam1, q3))); 989 assertEquals("ab", Bytes.toString(result.getValue(fam1, q4))); 990 } else { 991 fail("the qualifier " + Bytes.toString(q1) + " should be " + v1 + " or " + v2 992 + ", but " + q1Value); 993 } 994 } 995 } catch (Exception e) { 996 assertionError.compareAndSet(null, new AssertionError(e)); 997 } catch (AssertionError e) { 998 assertionError.compareAndSet(null, e); 999 } finally { 1000 latch.countDown(); 1001 } 1002 }, "ReaderThread-" + i + "-" + method); 1003 threads.add(readerThread); 1004 readerThread.start(); 1005 } 1006 1007 writerThread.join(); 1008 1009 if (assertionError.get() != null) { 1010 throw assertionError.get(); 1011 } 1012 } finally { 1013 interruptAndJoinThreads(threads.toArray(new Thread[0])); 1014 } 1015 } 1016 1017 private RepeatingTestThread createCompactingFlushThread(MultithreadedTestUtil.TestContext ctx) { 1018 RepeatingTestThread thread = new RepeatingTestThread(ctx) { 1019 @Override 1020 public void doAnAction() throws Exception { 1021 region.flush(true); 1022 region.compact(false); 1023 for (HStore store : region.getStores()) { 1024 store.closeAndArchiveCompactedFiles(); 1025 } 1026 } 1027 }; 1028 thread.setName("CompactingFlushThread-" + method); 1029 return thread; 1030 } 1031 1032 private void stopWriteThreads(PutThread putThread, MultithreadedTestUtil.TestContext ctx, 1033 Thread flushThread) throws Exception { 1034 putThread.done(); 1035 ctx.setStopFlag(true); 1036 try { 1037 joinThreads(putThread, flushThread); 1038 } finally { 1039 interruptAndJoinThreads(putThread, flushThread); 1040 } 1041 ctx.stop(); 1042 putThread.checkNoError(); 1043 } 1044 1045 @Test 1046 public void testWriteThreadsCleanupAfterFlushFailure() throws Exception { 1047 region = initHRegion(conf, fam1); 1048 PutThread putThread = new PutThread(1, new byte[][] { fam1 }, new byte[][] { qual1 }); 1049 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(conf); 1050 RepeatingTestThread flushThread = createCompactingFlushThread(ctx); 1051 HRegion failingRegion = mock(HRegion.class); 1052 doThrow(new IOException("Injected flush failure")).when(failingRegion).flush(true); 1053 try { 1054 putThread.start(); 1055 putThread.waitForFirstPut(); 1056 ctx.addThread(flushThread); 1057 ctx.startThreads(); 1058 IOException error = assertThrows(IOException.class, () -> { 1059 try { 1060 failingRegion.flush(true); 1061 } finally { 1062 stopWriteThreads(putThread, ctx, flushThread); 1063 } 1064 }); 1065 assertEquals("Injected flush failure", error.getMessage()); 1066 assertFalse(putThread.isAlive()); 1067 assertFalse(flushThread.isAlive()); 1068 } finally { 1069 stopWriteThreads(putThread, ctx, flushThread); 1070 } 1071 } 1072 1073 @Test 1074 public void testStopBeforeThreadStart() throws Exception { 1075 region = initHRegion(conf, fam1); 1076 PutThread putThread = new PutThread(1, new byte[][] { fam1 }, new byte[][] { qual1 }); 1077 FlushThread flushThread = new FlushThread(); 1078 putThread.done(); 1079 flushThread.done(); 1080 try { 1081 putThread.start(); 1082 flushThread.start(); 1083 joinThreads(putThread, flushThread); 1084 putThread.checkNoError(); 1085 flushThread.checkNoError(); 1086 assertEquals(0, putThread.numPutsFinished); 1087 } finally { 1088 interruptAndJoinThreads(putThread, flushThread); 1089 } 1090 } 1091 1092 @Test 1093 public void testThreadCleanupPreservesInterrupt() { 1094 CountDownLatch release = new CountDownLatch(1); 1095 Thread worker = new Thread(() -> { 1096 try { 1097 release.await(); 1098 } catch (InterruptedException e) { 1099 Thread.currentThread().interrupt(); 1100 } 1101 }, "CleanupWorker-" + method); 1102 try { 1103 worker.start(); 1104 Thread.currentThread().interrupt(); 1105 interruptAndJoinThreads(worker); 1106 assertTrue(Thread.currentThread().isInterrupted()); 1107 assertFalse(worker.isAlive()); 1108 } finally { 1109 Thread.interrupted(); 1110 release.countDown(); 1111 interruptAndJoinThreads(worker); 1112 } 1113 } 1114}