001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.apache.hadoop.hbase.HBaseTestingUtil.fam1;
021import static org.apache.hadoop.hbase.HBaseTestingUtil.fam2;
022import static org.apache.hadoop.hbase.HBaseTestingUtil.fam3;
023import static org.junit.jupiter.api.Assertions.assertArrayEquals;
024import static org.junit.jupiter.api.Assertions.assertEquals;
025import static org.junit.jupiter.api.Assertions.assertFalse;
026import static org.junit.jupiter.api.Assertions.assertInstanceOf;
027import static org.junit.jupiter.api.Assertions.assertNull;
028import static org.junit.jupiter.api.Assertions.assertThrows;
029import static org.junit.jupiter.api.Assertions.assertTrue;
030import static org.junit.jupiter.api.Assertions.fail;
031import static org.mockito.Mockito.doThrow;
032import static org.mockito.Mockito.mock;
033
034import java.io.IOException;
035import java.io.InterruptedIOException;
036import java.util.ArrayList;
037import java.util.Arrays;
038import java.util.List;
039import java.util.concurrent.CountDownLatch;
040import java.util.concurrent.TimeUnit;
041import java.util.concurrent.atomic.AtomicBoolean;
042import java.util.concurrent.atomic.AtomicLong;
043import java.util.concurrent.atomic.AtomicReference;
044import org.apache.hadoop.conf.Configuration;
045import org.apache.hadoop.hbase.Cell;
046import org.apache.hadoop.hbase.CellUtil;
047import org.apache.hadoop.hbase.CompareOperator;
048import org.apache.hadoop.hbase.DroppedSnapshotException;
049import org.apache.hadoop.hbase.ExtendedCell;
050import org.apache.hadoop.hbase.HBaseConfiguration;
051import org.apache.hadoop.hbase.HBaseTestingUtil;
052import org.apache.hadoop.hbase.HConstants;
053import org.apache.hadoop.hbase.MultithreadedTestUtil;
054import org.apache.hadoop.hbase.MultithreadedTestUtil.RepeatingTestThread;
055import org.apache.hadoop.hbase.NotServingRegionException;
056import org.apache.hadoop.hbase.TableName;
057import org.apache.hadoop.hbase.client.Append;
058import org.apache.hadoop.hbase.client.ClientInternalHelper;
059import org.apache.hadoop.hbase.client.Delete;
060import org.apache.hadoop.hbase.client.Durability;
061import org.apache.hadoop.hbase.client.Get;
062import org.apache.hadoop.hbase.client.Increment;
063import org.apache.hadoop.hbase.client.Mutation;
064import org.apache.hadoop.hbase.client.Put;
065import org.apache.hadoop.hbase.client.RegionInfo;
066import org.apache.hadoop.hbase.client.RegionInfoBuilder;
067import org.apache.hadoop.hbase.client.Result;
068import org.apache.hadoop.hbase.client.RowMutations;
069import org.apache.hadoop.hbase.client.Scan;
070import org.apache.hadoop.hbase.filter.BinaryComparator;
071import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
072import org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL;
073import org.apache.hadoop.hbase.testclassification.LargeTests;
074import org.apache.hadoop.hbase.testclassification.VerySlowRegionServerTests;
075import org.apache.hadoop.hbase.util.Bytes;
076import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
077import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
078import org.apache.hadoop.hbase.wal.NettyAsyncFSWALConfigHelper;
079import org.apache.hadoop.hbase.wal.WAL;
080import org.junit.jupiter.api.AfterEach;
081import org.junit.jupiter.api.BeforeEach;
082import org.junit.jupiter.api.Tag;
083import org.junit.jupiter.api.Test;
084import org.junit.jupiter.api.TestInfo;
085import org.slf4j.Logger;
086import org.slf4j.LoggerFactory;
087
088import org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup;
089import org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoopGroup;
090import org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel;
091
092/**
093 * Concurrent region operations with regular and in-memory-compacting MemStores.
094 */
095@Tag(VerySlowRegionServerTests.TAG)
096@Tag(LargeTests.TAG)
097public class TestHRegionConcurrency {
098
099  private static final Logger LOG = LoggerFactory.getLogger(TestHRegionConcurrency.class);
100  private static final EventLoopGroup GROUP = new NioEventLoopGroup();
101  private static final long THREAD_JOIN_TIMEOUT_NS = TimeUnit.SECONDS.toNanos(30);
102  private static final int MAX_STORE_FILES = 10;
103
104  private HBaseTestingUtil testUtil;
105  private Configuration conf;
106  private HRegion region;
107  private String method;
108  private TableName tableName;
109  private final byte[] qual1 = Bytes.toBytes("qual1");
110  private final byte[] qual2 = Bytes.toBytes("qual2");
111  private final byte[] qual3 = Bytes.toBytes("qual3");
112
113  protected boolean isInMemory() {
114    return false;
115  }
116
117  @BeforeEach
118  public void setup(TestInfo testInfo) {
119    testUtil = new HBaseTestingUtil();
120    conf = testUtil.getConfiguration();
121    NettyAsyncFSWALConfigHelper.setEventLoopConfig(conf, GROUP, NioSocketChannel.class);
122    method = testInfo.getTestMethod().get().getName();
123    tableName = TableName.valueOf(method);
124    conf.set(CompactingMemStore.IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, String.valueOf(0.09));
125    conf.setLong(AbstractFSWAL.WAL_SYNC_TIMEOUT_MS, 10000);
126  }
127
128  @AfterEach
129  public void tearDown() throws IOException {
130    try {
131      HBaseTestingUtil.closeRegionAndWAL(region);
132    } finally {
133      EnvironmentEdgeManagerTestHelper.reset();
134      testUtil.cleanupTestDir();
135    }
136  }
137
138  private HRegion initHRegion(Configuration configuration, byte[]... families) throws IOException {
139    RegionInfo hri = RegionInfoBuilder.newBuilder(tableName).build();
140    WAL wal = HBaseTestingUtil.createWal(configuration,
141      testUtil.getDataTestDirOnTestFS(method + ".log"), hri);
142    ChunkCreator.initialize(MemStoreLAB.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null,
143      MemStoreLAB.INDEX_CHUNK_SIZE_PERCENTAGE_DEFAULT);
144    if (isInMemory()) {
145      boolean[] flags = new boolean[families.length];
146      Arrays.fill(flags, true);
147      return testUtil.createLocalHRegionWithInMemoryFlags(tableName, null, null, configuration,
148        false, Durability.SYNC_WAL, wal, flags, families);
149    }
150    return testUtil.createLocalHRegion(tableName, null, null, configuration, false,
151      Durability.SYNC_WAL, wal, families);
152  }
153
154  private Configuration initSplit() {
155    conf.setInt("hbase.hstore.compactionThreshold", 2);
156    conf.setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, 10 * 1000);
157    conf.setLong("hbase.client.pause", 15 * 1000);
158    conf.setLong(HConstants.HREGION_MAX_FILESIZE, 1024 * 128);
159    return conf;
160  }
161
162  private static void interruptAndJoinThreads(Thread... threads) {
163    for (Thread thread : threads) {
164      if (thread != null) {
165        thread.interrupt();
166      }
167    }
168    joinThreads(threads);
169  }
170
171  private static void joinThreads(Thread... threads) {
172    long deadline = System.nanoTime() + THREAD_JOIN_TIMEOUT_NS;
173    boolean interrupted = false;
174    List<String> aliveThreads = new ArrayList<>();
175    for (Thread thread : threads) {
176      if (thread == null) {
177        continue;
178      }
179      while (thread.isAlive()) {
180        long remaining = deadline - System.nanoTime();
181        if (remaining <= 0) {
182          break;
183        }
184        try {
185          TimeUnit.NANOSECONDS.timedJoin(thread, remaining);
186        } catch (InterruptedException e) {
187          interrupted = true;
188        }
189      }
190      if (thread.isAlive()) {
191        aliveThreads.add(thread.getName());
192      }
193    }
194    if (interrupted) {
195      Thread.currentThread().interrupt();
196    }
197    assertTrue(aliveThreads.isEmpty(), "Threads did not stop: " + aliveThreads);
198  }
199
200  @Test
201  public void testGetWhileRegionClose() throws Exception {
202    Configuration hc = initSplit();
203    byte[][] families = { fam1, fam2, fam3 };
204    region = initHRegion(hc, families);
205    final int startRow = 100;
206    for (byte[] qualifier : new byte[][] { qual1, qual2, qual3 }) {
207      for (int i = startRow; i < startRow + 100; i++) {
208        Put put = new Put(Bytes.toBytes("" + i));
209        put.setDurability(Durability.SKIP_WAL);
210        for (byte[] family : families) {
211          put.addColumn(family, qualifier, null);
212        }
213        region.put(put);
214      }
215    }
216
217    AtomicBoolean done = new AtomicBoolean(false);
218    CountDownLatch initialGets = new CountDownLatch(5);
219    GetTillDoneOrException[] threads = new GetTillDoneOrException[10];
220    try {
221      for (int i = 0; i < threads.length / 2; i++) {
222        threads[i] =
223          new GetTillDoneOrException(region, i, Bytes.toBytes("" + startRow), done, initialGets);
224        threads[i].start();
225      }
226      assertTrue(initialGets.await(30, TimeUnit.SECONDS), "Initial getters did not read");
227      // Force the closing state while readers are still active.
228      region.closing.set(true);
229      for (int i = threads.length / 2; i < threads.length; i++) {
230        threads[i] =
231          new GetTillDoneOrException(region, i, Bytes.toBytes("" + startRow), done, initialGets);
232        threads[i].start();
233      }
234      joinThreads(threads);
235      for (GetTillDoneOrException thread : threads) {
236        assertInstanceOf(NotServingRegionException.class, thread.error, thread.getName());
237      }
238    } finally {
239      done.set(true);
240      interruptAndJoinThreads(threads);
241    }
242  }
243
244  private class GetTillDoneOrException extends Thread {
245    private final HRegion testRegion;
246    private final Get get;
247    private final AtomicBoolean done;
248    private final CountDownLatch initialGets;
249    private Throwable error;
250
251    GetTillDoneOrException(HRegion testRegion, int i, byte[] row, AtomicBoolean done,
252      CountDownLatch initialGets) {
253      super("Getter-" + i + "-" + method);
254      this.testRegion = testRegion;
255      this.get = new Get(row);
256      this.done = done;
257      this.initialGets = initialGets;
258    }
259
260    @Override
261    public void run() {
262      boolean firstGet = true;
263      while (!done.get() && !isInterrupted()) {
264        try {
265          assertTrue(testRegion.get(get).size() > 0);
266          if (firstGet) {
267            initialGets.countDown();
268            firstGet = false;
269          }
270        } catch (Throwable t) {
271          error = t;
272          return;
273        }
274      }
275    }
276  }
277
278  /**
279   * Flushes the cache in a thread while scanning. The tests verify that the scan is coherent - e.g.
280   * the returned results are always of the same or later update as the previous results. scan /
281   * compact thread join
282   */
283  @Test
284  public void testFlushCacheWhileScanning() throws IOException, InterruptedException {
285    byte[] family = Bytes.toBytes("family");
286    int numRows = 1000;
287    int flushAndScanInterval = 10;
288    int compactInterval = 10 * flushAndScanInterval;
289
290    this.region = initHRegion(conf, family);
291    FlushThread flushThread = new FlushThread();
292    try {
293      flushThread.start();
294
295      Scan scan = new Scan();
296      scan.addFamily(family);
297      scan.setFilter(new SingleColumnValueFilter(family, qual1, CompareOperator.EQUAL,
298        new BinaryComparator(Bytes.toBytes(5L))));
299
300      int expectedCount = 0;
301      List<Cell> res = new ArrayList<>();
302
303      boolean toggle = true;
304      for (long i = 0; i < numRows; i++) {
305        Put put = new Put(Bytes.toBytes(i));
306        put.setDurability(Durability.SKIP_WAL);
307        put.addColumn(family, qual1, Bytes.toBytes(i % 10));
308        region.put(put);
309
310        if (i != 0 && i % compactInterval == 0) {
311          LOG.debug("iteration = " + i + " ts=" + EnvironmentEdgeManager.currentTime());
312          region.compact(true);
313        }
314
315        if (i % 10 == 5L) {
316          expectedCount++;
317        }
318
319        if (i != 0 && i % flushAndScanInterval == 0) {
320          res.clear();
321          try (InternalScanner scanner = region.getScanner(scan)) {
322            if (toggle) {
323              flushThread.flush();
324            }
325            while (scanner.next(res)) {
326              // ignore
327            }
328          }
329          if (!toggle) {
330            flushThread.flush();
331          }
332          assertEquals(expectedCount, res.size(),
333            "toggle=" + toggle + "i=" + i + " ts=" + EnvironmentEdgeManager.currentTime());
334          toggle = !toggle;
335        }
336      }
337
338    } finally {
339      flushThread.done();
340      try {
341        joinThreads(flushThread);
342      } finally {
343        interruptAndJoinThreads(flushThread);
344      }
345      flushThread.checkNoError();
346      HBaseTestingUtil.closeRegionAndWAL(this.region);
347      this.region = null;
348    }
349  }
350
351  private class FlushThread extends Thread {
352    private volatile boolean done;
353    private Throwable error = null;
354
355    FlushThread() {
356      super("FlushThread-" + method);
357    }
358
359    public void done() {
360      synchronized (this) {
361        done = true;
362        notifyAll();
363      }
364    }
365
366    public void checkNoError() {
367      if (error != null) {
368        assertNull(error);
369      }
370    }
371
372    @Override
373    public void run() {
374      while (!done && !Thread.currentThread().isInterrupted()) {
375        synchronized (this) {
376          if (done) {
377            break;
378          }
379          try {
380            wait();
381          } catch (InterruptedException e) {
382            Thread.currentThread().interrupt();
383            break;
384          }
385          if (done) {
386            break;
387          }
388        }
389        try {
390          region.flush(true);
391        } catch (IOException e) {
392          if (!done) {
393            LOG.error("Error while flushing cache", e);
394            error = e;
395          }
396          break;
397        } catch (Throwable t) {
398          LOG.error("Uncaught exception", t);
399          throw t;
400        }
401      }
402    }
403
404    public void flush() {
405      synchronized (this) {
406        notify();
407      }
408    }
409  }
410
411  /**
412   * So can be overridden in subclasses.
413   */
414  protected int getNumQualifiersForTestWritesWhileScanning() {
415    return 100;
416  }
417
418  /**
419   * So can be overridden in subclasses.
420   */
421  protected int getTestCountForTestWritesWhileScanning() {
422    return 100;
423  }
424
425  /**
426   * Writes very wide records and scans for the latest every time.. Flushes and compacts the region
427   * every now and then to keep things realistic. by flush / scan / compaction when joining threads
428   */
429  @Test
430  public void testWritesWhileScanning() throws IOException, InterruptedException {
431    int testCount = getTestCountForTestWritesWhileScanning();
432    int numRows = 1;
433    int numFamilies = 10;
434    int numQualifiers = getNumQualifiersForTestWritesWhileScanning();
435    int flushInterval = 7;
436    int compactInterval = 5 * flushInterval;
437    byte[][] families = new byte[numFamilies][];
438    for (int i = 0; i < numFamilies; i++) {
439      families[i] = Bytes.toBytes("family" + i);
440    }
441    byte[][] qualifiers = new byte[numQualifiers][];
442    for (int i = 0; i < numQualifiers; i++) {
443      qualifiers[i] = Bytes.toBytes("qual" + i);
444    }
445
446    this.region = initHRegion(conf, families);
447    FlushThread flushThread = new FlushThread();
448    PutThread putThread = new PutThread(numRows, families, qualifiers);
449    try {
450      putThread.start();
451      putThread.waitForFirstPut();
452
453      flushThread.start();
454
455      Scan scan = new Scan().withStartRow(Bytes.toBytes("row0")).withStopRow(Bytes.toBytes("row1"));
456
457      int expectedCount = numFamilies * numQualifiers;
458      List<Cell> res = new ArrayList<>();
459
460      long prevTimestamp = 0L;
461      for (int i = 0; i < testCount; i++) {
462
463        if (i != 0 && i % compactInterval == 0) {
464          region.compact(true);
465          for (HStore store : region.getStores()) {
466            store.closeAndArchiveCompactedFiles();
467          }
468        }
469
470        if (i != 0 && i % flushInterval == 0) {
471          flushThread.flush();
472        }
473
474        boolean previousEmpty = res.isEmpty();
475        res.clear();
476        try (InternalScanner scanner = region.getScanner(scan)) {
477          boolean moreRows;
478          do {
479            moreRows = scanner.next(res);
480          } while (moreRows);
481        }
482        if (!res.isEmpty() || !previousEmpty || i > compactInterval) {
483          assertEquals(expectedCount, res.size(), "i=" + i);
484          long timestamp = res.get(0).getTimestamp();
485          assertTrue(timestamp >= prevTimestamp,
486            "Timestamps were broke: " + timestamp + " prev: " + prevTimestamp);
487          prevTimestamp = timestamp;
488        }
489      }
490
491      putThread.done();
492
493      region.flush(true);
494
495    } finally {
496      putThread.done();
497      flushThread.done();
498      try {
499        joinThreads(flushThread, putThread);
500      } finally {
501        interruptAndJoinThreads(flushThread, putThread);
502      }
503      flushThread.checkNoError();
504      putThread.checkNoError();
505
506      try {
507        HBaseTestingUtil.closeRegionAndWAL(this.region);
508      } catch (DroppedSnapshotException dse) {
509        // We could get this on way out because we interrupt the background flusher and it could
510        // fail anywhere causing a DSE over in the background flusher... only it is not properly
511        // dealt with so could still be memory hanging out when we get to here -- memory we can't
512        // flush because the accounting is 'off' since original DSE.
513      }
514      this.region = null;
515    }
516  }
517
518  private final class PutThread extends Thread {
519    private volatile boolean done;
520    private volatile int numPutsFinished = 0;
521
522    private Throwable error = null;
523    private int numRows;
524    private byte[][] families;
525    private byte[][] qualifiers;
526
527    private PutThread(int numRows, byte[][] families, byte[][] qualifiers) {
528      super("PutThread-" + method);
529      this.numRows = numRows;
530      this.families = families;
531      this.qualifiers = qualifiers;
532    }
533
534    /**
535     * Block calling thread until this instance of PutThread has put at least one row.
536     */
537    public void waitForFirstPut() throws InterruptedException {
538      // wait until put thread actually puts some data
539      while (isAlive() && numPutsFinished == 0) {
540        checkNoError();
541        Thread.sleep(50);
542      }
543    }
544
545    public void done() {
546      done = true;
547      interrupt();
548    }
549
550    public void checkNoError() {
551      if (error != null) {
552        assertNull(error);
553      }
554    }
555
556    @Override
557    public void run() {
558      while (!done && !Thread.currentThread().isInterrupted()) {
559        try {
560          for (int r = 0; r < numRows; r++) {
561            byte[] row = Bytes.toBytes("row" + r);
562            Put put = new Put(row);
563            put.setDurability(Durability.SKIP_WAL);
564            byte[] value = Bytes.toBytes(String.valueOf(numPutsFinished));
565            for (byte[] family : families) {
566              for (byte[] qualifier : qualifiers) {
567                put.addColumn(family, qualifier, numPutsFinished, value);
568              }
569            }
570            region.put(put);
571            numPutsFinished++;
572            if (numPutsFinished > 0 && numPutsFinished % 47 == 0) {
573              LOG.debug("put iteration = {}", numPutsFinished);
574              Delete delete = new Delete(row, (long) numPutsFinished - 30);
575              region.delete(delete);
576            }
577            numPutsFinished++;
578          }
579        } catch (InterruptedIOException e) {
580          // This is fine. It means we are done, or didn't get the lock on time
581          LOG.info("Interrupted", e);
582        } catch (IOException e) {
583          LOG.error("Error while putting records", e);
584          error = e;
585          break;
586        }
587      }
588
589    }
590
591  }
592
593  /**
594   * Writes very wide records and gets the latest row every time.. Flushes and compacts the region
595   * aggressivly to catch issues. by flush / scan / compaction when joining threads
596   */
597  @Test
598  public void testWritesWhileGetting() throws Exception {
599    int testCount = 50;
600    int numRows = 1;
601    int numFamilies = 10;
602    int numQualifiers = 100;
603    int compactInterval = 100;
604    byte[][] families = new byte[numFamilies][];
605    for (int i = 0; i < numFamilies; i++) {
606      families[i] = Bytes.toBytes("family" + i);
607    }
608    byte[][] qualifiers = new byte[numQualifiers][];
609    for (int i = 0; i < numQualifiers; i++) {
610      qualifiers[i] = Bytes.toBytes("qual" + i);
611    }
612
613    // This test flushes constantly and can cause many files to be created,
614    // possibly
615    // extending over the ulimit. Make sure compactions are aggressive in
616    // reducing
617    // the number of HFiles created.
618    Configuration configuration = HBaseConfiguration.create(conf);
619    configuration.setInt("hbase.hstore.compaction.min", 1);
620    configuration.setInt("hbase.hstore.compaction.max", 1000);
621    this.region = initHRegion(configuration, families);
622    PutThread putThread = new PutThread(numRows, families, qualifiers);
623    MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(configuration);
624    RepeatingTestThread flushThread = createCompactingFlushThread(ctx);
625    try {
626      putThread.start();
627      putThread.waitForFirstPut();
628      ctx.addThread(flushThread);
629      ctx.startThreads();
630
631      Get get = new Get(Bytes.toBytes("row0"));
632      Result result = null;
633
634      int expectedCount = numFamilies * numQualifiers;
635
636      long prevTimestamp = 0L;
637      for (int i = 0; i < testCount; i++) {
638        LOG.info("testWritesWhileGetting verify turn " + i);
639        boolean previousEmpty = result == null || result.isEmpty();
640        result = region.get(get);
641        if (!result.isEmpty() || !previousEmpty || i > compactInterval) {
642          assertEquals(expectedCount, result.size(), "i=" + i);
643          // TODO this was removed, now what dangit?!
644          // search looking for the qualifier in question?
645          long timestamp = 0;
646          for (Cell kv : result.rawCells()) {
647            if (
648              CellUtil.matchingFamily(kv, families[0])
649                && CellUtil.matchingQualifier(kv, qualifiers[0])
650            ) {
651              timestamp = kv.getTimestamp();
652            }
653          }
654          assertTrue(timestamp >= prevTimestamp);
655          prevTimestamp = timestamp;
656          ExtendedCell previousKV = null;
657
658          for (ExtendedCell kv : ClientInternalHelper.getExtendedRawCells(result)) {
659            byte[] thisValue = CellUtil.cloneValue(kv);
660            if (previousKV != null) {
661              if (Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue) != 0) {
662                LOG.warn("These two KV should have the same value." + " Previous KV:" + previousKV
663                  + "(memStoreTS:" + previousKV.getSequenceId() + ")" + ", New KV: " + kv
664                  + "(memStoreTS:" + kv.getSequenceId() + ")");
665                assertEquals(0, Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue));
666              }
667            }
668            previousKV = kv;
669          }
670        }
671      }
672    } finally {
673      stopWriteThreads(putThread, ctx, flushThread);
674    }
675    region.flush(true);
676  }
677
678  /**
679   * TestCase for increment
680   */
681  private static class Incrementer implements Runnable {
682    private HRegion region;
683    private final static byte[] incRow = Bytes.toBytes("incRow");
684    private final static byte[] family = Bytes.toBytes("family");
685    private final static byte[] qualifier = Bytes.toBytes("qualifier");
686    private final static long ONE = 1L;
687    private int incCounter;
688
689    public Incrementer(HRegion region, int incCounter) {
690      this.region = region;
691      this.incCounter = incCounter;
692    }
693
694    @Override
695    public void run() {
696      int count = 0;
697      while (count < incCounter && !Thread.currentThread().isInterrupted()) {
698        Increment inc = new Increment(incRow);
699        inc.addColumn(family, qualifier, ONE);
700        count++;
701        try {
702          region.increment(inc);
703        } catch (IOException e) {
704          LOG.info("Count=" + count + ", " + e);
705          break;
706        }
707      }
708    }
709  }
710
711  /**
712   * Test case to check increment function with memstore flushing
713   */
714  @Test
715  public void testParallelIncrementWithMemStoreFlush() throws Exception {
716    byte[] family = Incrementer.family;
717    this.region = initHRegion(conf, family);
718    final HRegion region = this.region;
719    final AtomicBoolean incrementDone = new AtomicBoolean(false);
720    Runnable flusher = new Runnable() {
721      @Override
722      public void run() {
723        while (!incrementDone.get() && !Thread.currentThread().isInterrupted()) {
724          try {
725            flushAndCompact(region, family);
726          } catch (Exception e) {
727            if (!incrementDone.get()) {
728              LOG.warn("Failed to flush or compact region", e);
729            }
730          }
731        }
732      }
733    };
734
735    // after all increment finished, the row will increment to 20*100 = 2000
736    int threadNum = 20;
737    int incCounter = 100;
738    long expected = (long) threadNum * incCounter;
739    List<Thread> incrementers = new ArrayList<>(threadNum);
740    Thread flushThread = new Thread(flusher, "FlushThread-" + method);
741    try {
742      for (int i = 0; i < threadNum; i++) {
743        Thread incrementer =
744          new Thread(new Incrementer(this.region, incCounter), "Incrementer-" + i + "-" + method);
745        incrementers.add(incrementer);
746        incrementer.start();
747      }
748      flushThread.start();
749      for (Thread incrementer : incrementers) {
750        incrementer.join();
751      }
752
753      incrementDone.set(true);
754      joinThreads(flushThread);
755      assertTrue(region.getStore(family).getStorefilesCount() < MAX_STORE_FILES);
756
757      Get get = new Get(Incrementer.incRow);
758      get.addColumn(Incrementer.family, Incrementer.qualifier);
759      get.readVersions(1);
760      Result res = this.region.get(get);
761      List<Cell> kvs = res.getColumnCells(Incrementer.family, Incrementer.qualifier);
762
763      // we just got the latest version
764      assertEquals(1, kvs.size());
765      Cell kv = kvs.get(0);
766      assertEquals(expected, Bytes.toLong(kv.getValueArray(), kv.getValueOffset()));
767    } finally {
768      incrementDone.set(true);
769      incrementers.add(flushThread);
770      interruptAndJoinThreads(incrementers.toArray(new Thread[0]));
771    }
772  }
773
774  /**
775   * TestCase for append
776   */
777  private static class Appender implements Runnable {
778    private HRegion region;
779    private final static byte[] appendRow = Bytes.toBytes("appendRow");
780    private final static byte[] family = Bytes.toBytes("family");
781    private final static byte[] qualifier = Bytes.toBytes("qualifier");
782    private final static byte[] CHAR = Bytes.toBytes("a");
783    private int appendCounter;
784
785    public Appender(HRegion region, int appendCounter) {
786      this.region = region;
787      this.appendCounter = appendCounter;
788    }
789
790    @Override
791    public void run() {
792      int count = 0;
793      while (count < appendCounter && !Thread.currentThread().isInterrupted()) {
794        Append app = new Append(appendRow);
795        app.addColumn(family, qualifier, CHAR);
796        count++;
797        try {
798          region.append(app);
799        } catch (IOException e) {
800          LOG.info("Count=" + count + ", max=" + appendCounter + ", " + e);
801          break;
802        }
803      }
804    }
805  }
806
807  /**
808   * Test case to check append function with memstore flushing
809   */
810  @Test
811  public void testParallelAppendWithMemStoreFlush() throws Exception {
812    byte[] family = Appender.family;
813    this.region = initHRegion(conf, family);
814    final HRegion region = this.region;
815    final AtomicBoolean appendDone = new AtomicBoolean(false);
816    Runnable flusher = new Runnable() {
817      @Override
818      public void run() {
819        while (!appendDone.get() && !Thread.currentThread().isInterrupted()) {
820          try {
821            flushAndCompact(region, family);
822          } catch (Exception e) {
823            if (!appendDone.get()) {
824              LOG.warn("Failed to flush or compact region", e);
825            }
826          }
827        }
828      }
829    };
830
831    // After all append finished, the value will append to threadNum *
832    // appendCounter Appender.CHAR
833    int threadNum = 20;
834    int appendCounter = 100;
835    byte[] expected = new byte[threadNum * appendCounter];
836    for (int i = 0; i < threadNum * appendCounter; i++) {
837      System.arraycopy(Appender.CHAR, 0, expected, i, 1);
838    }
839    List<Thread> appenders = new ArrayList<>(threadNum);
840    Thread flushThread = new Thread(flusher, "FlushThread-" + method);
841    try {
842      for (int i = 0; i < threadNum; i++) {
843        Thread appender =
844          new Thread(new Appender(this.region, appendCounter), "Appender-" + i + "-" + method);
845        appenders.add(appender);
846        appender.start();
847      }
848      flushThread.start();
849      for (Thread appender : appenders) {
850        appender.join();
851      }
852
853      appendDone.set(true);
854      joinThreads(flushThread);
855      assertTrue(region.getStore(family).getStorefilesCount() < MAX_STORE_FILES);
856
857      Get get = new Get(Appender.appendRow);
858      get.addColumn(Appender.family, Appender.qualifier);
859      get.readVersions(1);
860      Result res = this.region.get(get);
861      List<Cell> kvs = res.getColumnCells(Appender.family, Appender.qualifier);
862
863      // we just got the latest version
864      assertEquals(1, kvs.size());
865      Cell kv = kvs.get(0);
866      byte[] appendResult = new byte[kv.getValueLength()];
867      System.arraycopy(kv.getValueArray(), kv.getValueOffset(), appendResult, 0,
868        kv.getValueLength());
869      assertArrayEquals(expected, appendResult);
870    } finally {
871      appendDone.set(true);
872      appenders.add(flushThread);
873      interruptAndJoinThreads(appenders.toArray(new Thread[0]));
874    }
875  }
876
877  private static void flushAndCompact(HRegion region, byte[] family) throws IOException {
878    region.flush(true);
879    HStore store = region.getStore(family);
880    // Standalone regions have no background compaction or compacted-file discharger.
881    if (store.getStorefilesCount() >= MAX_STORE_FILES) {
882      region.compact(true);
883      store.closeAndArchiveCompactedFiles();
884    }
885  }
886
887  @Test
888  public void testFlusherCompactsStoreFiles() throws IOException {
889    region = initHRegion(conf, fam1);
890    HStore store = region.getStore(fam1);
891    for (int i = 0; i < MAX_STORE_FILES * 2; i++) {
892      region.put(new Put(Bytes.toBytes(i)).addColumn(fam1, qual1, Bytes.toBytes(i)));
893      flushAndCompact(region, fam1);
894      int storeFiles = store.getStorefilesCount();
895      assertTrue(storeFiles > 0 && storeFiles < MAX_STORE_FILES,
896        "Unexpected store file count after flush " + i + ": " + storeFiles);
897    }
898    assertEquals(0, store.getCompactedFilesCount());
899    for (int i = 0; i < MAX_STORE_FILES * 2; i++) {
900      Result result = region.get(new Get(Bytes.toBytes(i)));
901      assertEquals(i, Bytes.toInt(result.getValue(fam1, qual1)));
902    }
903  }
904
905  @Test
906  public void testMutateRowInParallel() throws Exception {
907    final int numReaderThreads = 100;
908    final CountDownLatch latch = new CountDownLatch(numReaderThreads);
909
910    final byte[] row = Bytes.toBytes("row");
911    final byte[] q1 = Bytes.toBytes("q1");
912    final byte[] q2 = Bytes.toBytes("q2");
913    final byte[] q3 = Bytes.toBytes("q3");
914    final byte[] q4 = Bytes.toBytes("q4");
915    final String v1 = "v1";
916    final String v2 = "v2";
917
918    // We need to ensure the timestamp of the delete operation is more than the previous one
919    final AtomicLong deleteTimestamp = new AtomicLong();
920
921    region = initHRegion(conf, fam1);
922
923    // Initial values
924    region.batchMutate(new Mutation[] { new Put(row).addColumn(fam1, q1, Bytes.toBytes(v1))
925      .addColumn(fam1, q2, deleteTimestamp.getAndIncrement(), Bytes.toBytes(v2))
926      .addColumn(fam1, q3, Bytes.toBytes(1L)).addColumn(fam1, q4, Bytes.toBytes("a")) });
927
928    final AtomicReference<AssertionError> assertionError = new AtomicReference<>();
929    List<Thread> threads = new ArrayList<>(numReaderThreads + 1);
930
931    // Writer thread
932    Thread writerThread = new Thread(() -> {
933      try {
934        while (!Thread.currentThread().isInterrupted()) {
935          // If all the reader threads finish, then stop the writer thread
936          if (latch.await(0, TimeUnit.MILLISECONDS)) {
937            return;
938          }
939
940          // Execute the mutations. This should be done atomically
941          region.mutateRow(new RowMutations(row)
942            .add(Arrays.asList(new Put(row).addColumn(fam1, q1, Bytes.toBytes(v2)),
943              new Delete(row).addColumns(fam1, q2, deleteTimestamp.getAndIncrement()),
944              new Increment(row).addColumn(fam1, q3, 1L),
945              new Append(row).addColumn(fam1, q4, Bytes.toBytes("b")))));
946
947          // We need to ensure the timestamps of the Increment/Append operations are more than the
948          // previous ones
949          Result result = region.get(new Get(row).addColumn(fam1, q3).addColumn(fam1, q4));
950          long tsIncrement = result.getColumnLatestCell(fam1, q3).getTimestamp();
951          long tsAppend = result.getColumnLatestCell(fam1, q4).getTimestamp();
952
953          // Put the initial values
954          region.batchMutate(new Mutation[] { new Put(row).addColumn(fam1, q1, Bytes.toBytes(v1))
955            .addColumn(fam1, q2, deleteTimestamp.getAndIncrement(), Bytes.toBytes(v2))
956            .addColumn(fam1, q3, tsIncrement + 1, Bytes.toBytes(1L))
957            .addColumn(fam1, q4, tsAppend + 1, Bytes.toBytes("a")) });
958        }
959      } catch (InterruptedException e) {
960        Thread.currentThread().interrupt();
961      } catch (Exception e) {
962        assertionError.compareAndSet(null, new AssertionError(e));
963      }
964    }, "WriterThread-" + method);
965    threads.add(writerThread);
966
967    try {
968      writerThread.start();
969
970      // Reader threads
971      for (int i = 0; i < numReaderThreads; i++) {
972        Thread readerThread = new Thread(() -> {
973          try {
974            for (int j = 0; j < 10000 && assertionError.get() == null
975              && !Thread.currentThread().isInterrupted(); j++) {
976              // Verify the values
977              Result result = region.get(new Get(row));
978
979              // The values should be equals to either the initial values or the values after
980              // executing the mutations
981              String q1Value = Bytes.toString(result.getValue(fam1, q1));
982              if (v1.equals(q1Value)) {
983                assertEquals(v2, Bytes.toString(result.getValue(fam1, q2)));
984                assertEquals(1L, Bytes.toLong(result.getValue(fam1, q3)));
985                assertEquals("a", Bytes.toString(result.getValue(fam1, q4)));
986              } else if (v2.equals(q1Value)) {
987                assertNull(Bytes.toString(result.getValue(fam1, q2)));
988                assertEquals(2L, Bytes.toLong(result.getValue(fam1, q3)));
989                assertEquals("ab", Bytes.toString(result.getValue(fam1, q4)));
990              } else {
991                fail("the qualifier " + Bytes.toString(q1) + " should be " + v1 + " or " + v2
992                  + ", but " + q1Value);
993              }
994            }
995          } catch (Exception e) {
996            assertionError.compareAndSet(null, new AssertionError(e));
997          } catch (AssertionError e) {
998            assertionError.compareAndSet(null, e);
999          } finally {
1000            latch.countDown();
1001          }
1002        }, "ReaderThread-" + i + "-" + method);
1003        threads.add(readerThread);
1004        readerThread.start();
1005      }
1006
1007      writerThread.join();
1008
1009      if (assertionError.get() != null) {
1010        throw assertionError.get();
1011      }
1012    } finally {
1013      interruptAndJoinThreads(threads.toArray(new Thread[0]));
1014    }
1015  }
1016
1017  private RepeatingTestThread createCompactingFlushThread(MultithreadedTestUtil.TestContext ctx) {
1018    RepeatingTestThread thread = new RepeatingTestThread(ctx) {
1019      @Override
1020      public void doAnAction() throws Exception {
1021        region.flush(true);
1022        region.compact(false);
1023        for (HStore store : region.getStores()) {
1024          store.closeAndArchiveCompactedFiles();
1025        }
1026      }
1027    };
1028    thread.setName("CompactingFlushThread-" + method);
1029    return thread;
1030  }
1031
1032  private void stopWriteThreads(PutThread putThread, MultithreadedTestUtil.TestContext ctx,
1033    Thread flushThread) throws Exception {
1034    putThread.done();
1035    ctx.setStopFlag(true);
1036    try {
1037      joinThreads(putThread, flushThread);
1038    } finally {
1039      interruptAndJoinThreads(putThread, flushThread);
1040    }
1041    ctx.stop();
1042    putThread.checkNoError();
1043  }
1044
1045  @Test
1046  public void testWriteThreadsCleanupAfterFlushFailure() throws Exception {
1047    region = initHRegion(conf, fam1);
1048    PutThread putThread = new PutThread(1, new byte[][] { fam1 }, new byte[][] { qual1 });
1049    MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(conf);
1050    RepeatingTestThread flushThread = createCompactingFlushThread(ctx);
1051    HRegion failingRegion = mock(HRegion.class);
1052    doThrow(new IOException("Injected flush failure")).when(failingRegion).flush(true);
1053    try {
1054      putThread.start();
1055      putThread.waitForFirstPut();
1056      ctx.addThread(flushThread);
1057      ctx.startThreads();
1058      IOException error = assertThrows(IOException.class, () -> {
1059        try {
1060          failingRegion.flush(true);
1061        } finally {
1062          stopWriteThreads(putThread, ctx, flushThread);
1063        }
1064      });
1065      assertEquals("Injected flush failure", error.getMessage());
1066      assertFalse(putThread.isAlive());
1067      assertFalse(flushThread.isAlive());
1068    } finally {
1069      stopWriteThreads(putThread, ctx, flushThread);
1070    }
1071  }
1072
1073  @Test
1074  public void testStopBeforeThreadStart() throws Exception {
1075    region = initHRegion(conf, fam1);
1076    PutThread putThread = new PutThread(1, new byte[][] { fam1 }, new byte[][] { qual1 });
1077    FlushThread flushThread = new FlushThread();
1078    putThread.done();
1079    flushThread.done();
1080    try {
1081      putThread.start();
1082      flushThread.start();
1083      joinThreads(putThread, flushThread);
1084      putThread.checkNoError();
1085      flushThread.checkNoError();
1086      assertEquals(0, putThread.numPutsFinished);
1087    } finally {
1088      interruptAndJoinThreads(putThread, flushThread);
1089    }
1090  }
1091
1092  @Test
1093  public void testThreadCleanupPreservesInterrupt() {
1094    CountDownLatch release = new CountDownLatch(1);
1095    Thread worker = new Thread(() -> {
1096      try {
1097        release.await();
1098      } catch (InterruptedException e) {
1099        Thread.currentThread().interrupt();
1100      }
1101    }, "CleanupWorker-" + method);
1102    try {
1103      worker.start();
1104      Thread.currentThread().interrupt();
1105      interruptAndJoinThreads(worker);
1106      assertTrue(Thread.currentThread().isInterrupted());
1107      assertFalse(worker.isAlive());
1108    } finally {
1109      Thread.interrupted();
1110      release.countDown();
1111      interruptAndJoinThreads(worker);
1112    }
1113  }
1114}