001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.apache.hadoop.hbase.io.hfile.CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY;
021import static org.apache.hadoop.hbase.io.hfile.CacheConfig.CACHE_DATA_ON_READ_KEY;
022import static org.apache.hadoop.hbase.io.hfile.CacheConfig.DEFAULT_CACHE_DATA_ON_READ;
023import static org.apache.hadoop.hbase.io.hfile.CacheConfig.DEFAULT_CACHE_DATA_ON_WRITE;
024import static org.apache.hadoop.hbase.io.hfile.CacheConfig.DEFAULT_EVICT_ON_CLOSE;
025import static org.apache.hadoop.hbase.io.hfile.CacheConfig.EVICT_BLOCKS_ON_CLOSE_KEY;
026import static org.apache.hadoop.hbase.regionserver.DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY;
027import static org.junit.jupiter.api.Assertions.assertArrayEquals;
028import static org.junit.jupiter.api.Assertions.assertEquals;
029import static org.junit.jupiter.api.Assertions.assertFalse;
030import static org.junit.jupiter.api.Assertions.assertNull;
031import static org.junit.jupiter.api.Assertions.assertTrue;
032import static org.junit.jupiter.api.Assertions.fail;
033import static org.mockito.ArgumentMatchers.any;
034import static org.mockito.Mockito.mock;
035import static org.mockito.Mockito.spy;
036import static org.mockito.Mockito.times;
037import static org.mockito.Mockito.verify;
038import static org.mockito.Mockito.when;
039
040import java.io.FileNotFoundException;
041import java.io.IOException;
042import java.lang.ref.SoftReference;
043import java.security.PrivilegedExceptionAction;
044import java.util.ArrayList;
045import java.util.Arrays;
046import java.util.Collection;
047import java.util.Collections;
048import java.util.Iterator;
049import java.util.List;
050import java.util.ListIterator;
051import java.util.NavigableSet;
052import java.util.Optional;
053import java.util.TreeSet;
054import java.util.concurrent.BrokenBarrierException;
055import java.util.concurrent.ConcurrentSkipListSet;
056import java.util.concurrent.CountDownLatch;
057import java.util.concurrent.CyclicBarrier;
058import java.util.concurrent.ExecutorService;
059import java.util.concurrent.Executors;
060import java.util.concurrent.Future;
061import java.util.concurrent.ThreadPoolExecutor;
062import java.util.concurrent.TimeUnit;
063import java.util.concurrent.atomic.AtomicBoolean;
064import java.util.concurrent.atomic.AtomicInteger;
065import java.util.concurrent.atomic.AtomicLong;
066import java.util.concurrent.atomic.AtomicReference;
067import java.util.concurrent.locks.ReentrantReadWriteLock;
068import java.util.function.Consumer;
069import java.util.function.IntBinaryOperator;
070import java.util.function.IntConsumer;
071import org.apache.hadoop.conf.Configuration;
072import org.apache.hadoop.fs.FSDataOutputStream;
073import org.apache.hadoop.fs.FileStatus;
074import org.apache.hadoop.fs.FileSystem;
075import org.apache.hadoop.fs.FilterFileSystem;
076import org.apache.hadoop.fs.LocalFileSystem;
077import org.apache.hadoop.fs.Path;
078import org.apache.hadoop.fs.permission.FsPermission;
079import org.apache.hadoop.hbase.Cell;
080import org.apache.hadoop.hbase.CellBuilderType;
081import org.apache.hadoop.hbase.CellComparator;
082import org.apache.hadoop.hbase.CellComparatorImpl;
083import org.apache.hadoop.hbase.CellUtil;
084import org.apache.hadoop.hbase.ExtendedCell;
085import org.apache.hadoop.hbase.ExtendedCellBuilderFactory;
086import org.apache.hadoop.hbase.HBaseConfiguration;
087import org.apache.hadoop.hbase.HBaseTestingUtil;
088import org.apache.hadoop.hbase.HConstants;
089import org.apache.hadoop.hbase.KeyValue;
090import org.apache.hadoop.hbase.MemoryCompactionPolicy;
091import org.apache.hadoop.hbase.NamespaceDescriptor;
092import org.apache.hadoop.hbase.PrivateCellUtil;
093import org.apache.hadoop.hbase.TableName;
094import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
095import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
096import org.apache.hadoop.hbase.client.Get;
097import org.apache.hadoop.hbase.client.RegionInfo;
098import org.apache.hadoop.hbase.client.RegionInfoBuilder;
099import org.apache.hadoop.hbase.client.Scan;
100import org.apache.hadoop.hbase.client.Scan.ReadType;
101import org.apache.hadoop.hbase.client.TableDescriptor;
102import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
103import org.apache.hadoop.hbase.conf.ConfigurationManager;
104import org.apache.hadoop.hbase.exceptions.IllegalArgumentIOException;
105import org.apache.hadoop.hbase.filter.Filter;
106import org.apache.hadoop.hbase.filter.FilterBase;
107import org.apache.hadoop.hbase.io.compress.Compression;
108import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
109import org.apache.hadoop.hbase.io.hfile.BlockCache;
110import org.apache.hadoop.hbase.io.hfile.CacheConfig;
111import org.apache.hadoop.hbase.io.hfile.HFile;
112import org.apache.hadoop.hbase.io.hfile.HFileContext;
113import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
114import org.apache.hadoop.hbase.monitoring.MonitoredTask;
115import org.apache.hadoop.hbase.nio.RefCnt;
116import org.apache.hadoop.hbase.quotas.RegionSizeStoreImpl;
117import org.apache.hadoop.hbase.regionserver.ChunkCreator.ChunkType;
118import org.apache.hadoop.hbase.regionserver.MemStoreCompactionStrategy.Action;
119import org.apache.hadoop.hbase.regionserver.compactions.CompactionConfiguration;
120import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
121import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor;
122import org.apache.hadoop.hbase.regionserver.compactions.EverythingPolicy;
123import org.apache.hadoop.hbase.regionserver.querymatcher.ScanQueryMatcher;
124import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker;
125import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
126import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController;
127import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController;
128import org.apache.hadoop.hbase.security.User;
129import org.apache.hadoop.hbase.testclassification.MediumTests;
130import org.apache.hadoop.hbase.testclassification.RegionServerTests;
131import org.apache.hadoop.hbase.util.BloomFilterUtil;
132import org.apache.hadoop.hbase.util.Bytes;
133import org.apache.hadoop.hbase.util.CommonFSUtils;
134import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
135import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
136import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge;
137import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
138import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
139import org.apache.hadoop.hbase.wal.WALFactory;
140import org.apache.hadoop.util.Progressable;
141import org.junit.jupiter.api.AfterAll;
142import org.junit.jupiter.api.AfterEach;
143import org.junit.jupiter.api.BeforeEach;
144import org.junit.jupiter.api.Tag;
145import org.junit.jupiter.api.Test;
146import org.junit.jupiter.api.TestInfo;
147import org.junit.jupiter.api.Timeout;
148import org.mockito.Mockito;
149import org.slf4j.Logger;
150import org.slf4j.LoggerFactory;
151
152import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
153
154/**
155 * Test class for the HStore
156 */
157@Tag(RegionServerTests.TAG)
158@Tag(MediumTests.TAG)
159public class TestHStore {
160
161  private static final Logger LOG = LoggerFactory.getLogger(TestHStore.class);
162  private String name;
163
164  HRegion region;
165  HStore store;
166  byte[] table = Bytes.toBytes("table");
167  byte[] family = Bytes.toBytes("family");
168
169  byte[] row = Bytes.toBytes("row");
170  byte[] row2 = Bytes.toBytes("row2");
171  byte[] qf1 = Bytes.toBytes("qf1");
172  byte[] qf2 = Bytes.toBytes("qf2");
173  byte[] qf3 = Bytes.toBytes("qf3");
174  byte[] qf4 = Bytes.toBytes("qf4");
175  byte[] qf5 = Bytes.toBytes("qf5");
176  byte[] qf6 = Bytes.toBytes("qf6");
177
178  NavigableSet<byte[]> qualifiers = new ConcurrentSkipListSet<>(Bytes.BYTES_COMPARATOR);
179
180  List<Cell> expected = new ArrayList<>();
181  List<Cell> result = new ArrayList<>();
182
183  long id = EnvironmentEdgeManager.currentTime();
184  Get get = new Get(row);
185
186  private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
187  private static final String DIR = TEST_UTIL.getDataTestDir("TestStore").toString();
188
189  @BeforeEach
190  public void setUp(TestInfo testInfo) throws IOException {
191    this.name = testInfo.getTestMethod().get().getName();
192    qualifiers.clear();
193    qualifiers.add(qf1);
194    qualifiers.add(qf3);
195    qualifiers.add(qf5);
196
197    Iterator<byte[]> iter = qualifiers.iterator();
198    while (iter.hasNext()) {
199      byte[] next = iter.next();
200      expected.add(new KeyValue(row, family, next, 1, (byte[]) null));
201      get.addColumn(family, next);
202    }
203  }
204
205  private void init(String methodName) throws IOException {
206    init(methodName, TEST_UTIL.getConfiguration());
207  }
208
209  private HStore init(String methodName, Configuration conf) throws IOException {
210    // some of the tests write 4 versions and then flush
211    // (with HBASE-4241, lower versions are collected on flush)
212    return init(methodName, conf,
213      ColumnFamilyDescriptorBuilder.newBuilder(family).setMaxVersions(4).build());
214  }
215
216  private HStore init(String methodName, Configuration conf, ColumnFamilyDescriptor hcd)
217    throws IOException {
218    return init(methodName, conf, TableDescriptorBuilder.newBuilder(TableName.valueOf(table)), hcd);
219  }
220
221  private HStore init(String methodName, Configuration conf, TableDescriptorBuilder builder,
222    ColumnFamilyDescriptor hcd) throws IOException {
223    return init(methodName, conf, builder, hcd, null);
224  }
225
226  private HStore init(String methodName, Configuration conf, TableDescriptorBuilder builder,
227    ColumnFamilyDescriptor hcd, MyStoreHook hook) throws IOException {
228    return init(methodName, conf, builder, hcd, hook, false);
229  }
230
231  private void initHRegion(String methodName, Configuration conf, TableDescriptorBuilder builder,
232    ColumnFamilyDescriptor hcd, MyStoreHook hook, boolean switchToPread) throws IOException {
233    TableDescriptor htd = builder.setColumnFamily(hcd).build();
234    Path basedir = new Path(DIR + methodName);
235    Path tableDir = CommonFSUtils.getTableDir(basedir, htd.getTableName());
236    final Path logdir = new Path(basedir, AbstractFSWALProvider.getWALDirectoryName(methodName));
237
238    FileSystem fs = FileSystem.get(conf);
239
240    fs.delete(logdir, true);
241    ChunkCreator.initialize(MemStoreLAB.CHUNK_SIZE_DEFAULT, false,
242      MemStoreLABImpl.CHUNK_SIZE_DEFAULT, 1, 0, null,
243      MemStoreLAB.INDEX_CHUNK_SIZE_PERCENTAGE_DEFAULT);
244    RegionInfo info = RegionInfoBuilder.newBuilder(htd.getTableName()).build();
245    Configuration walConf = new Configuration(conf);
246    CommonFSUtils.setRootDir(walConf, basedir);
247    WALFactory wals = new WALFactory(walConf, methodName);
248    region = new HRegion(new HRegionFileSystem(conf, fs, tableDir, info), wals.getWAL(info), conf,
249      htd, null);
250    region.regionServicesForStores = Mockito.spy(region.regionServicesForStores);
251    ThreadPoolExecutor pool = (ThreadPoolExecutor) Executors.newFixedThreadPool(1);
252    Mockito.when(region.regionServicesForStores.getInMemoryCompactionPool()).thenReturn(pool);
253  }
254
255  private HStore init(String methodName, Configuration conf, TableDescriptorBuilder builder,
256    ColumnFamilyDescriptor hcd, MyStoreHook hook, boolean switchToPread) throws IOException {
257    initHRegion(methodName, conf, builder, hcd, hook, switchToPread);
258    if (hook == null) {
259      store = new HStore(region, hcd, conf, false);
260    } else {
261      store = new MyStore(region, hcd, conf, hook, switchToPread);
262    }
263    region.stores.put(store.getColumnFamilyDescriptor().getName(), store);
264    return store;
265  }
266
267  /**
268   * Test we do not lose data if we fail a flush and then close. Part of HBase-10466
269   */
270
271  @Test
272  public void testFlushSizeSizing() throws Exception {
273    LOG.info("Setting up a faulty file system that cannot write in " + name);
274    final Configuration conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration());
275    // Only retry once.
276    conf.setInt("hbase.hstore.flush.retries.number", 1);
277    User user = User.createUserForTesting(conf, name, new String[] { "foo" });
278    // Inject our faulty LocalFileSystem
279    conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class);
280    user.runAs(new PrivilegedExceptionAction<Object>() {
281      @Override
282      public Object run() throws Exception {
283        // Make sure it worked (above is sensitive to caching details in hadoop core)
284        FileSystem fs = FileSystem.get(conf);
285        assertEquals(FaultyFileSystem.class, fs.getClass());
286        FaultyFileSystem ffs = (FaultyFileSystem) fs;
287
288        // Initialize region
289        init(name, conf);
290
291        MemStoreSize mss = store.memstore.getFlushableSize();
292        assertEquals(0, mss.getDataSize());
293        LOG.info("Adding some data");
294        MemStoreSizing kvSize = new NonThreadSafeMemStoreSizing();
295        store.add(new KeyValue(row, family, qf1, 1, (byte[]) null), kvSize);
296        // add the heap size of active (mutable) segment
297        kvSize.incMemStoreSize(0, MutableSegment.DEEP_OVERHEAD, 0, 0);
298        mss = store.memstore.getFlushableSize();
299        assertEquals(kvSize.getMemStoreSize(), mss);
300        // Flush. Bug #1 from HBASE-10466. Make sure size calculation on failed flush is right.
301        try {
302          LOG.info("Flushing");
303          flushStore(store, id++);
304          fail("Didn't bubble up IOE!");
305        } catch (IOException ioe) {
306          assertTrue(ioe.getMessage().contains("Fault injected"));
307        }
308        // due to snapshot, change mutable to immutable segment
309        kvSize.incMemStoreSize(0,
310          CSLMImmutableSegment.DEEP_OVERHEAD_CSLM - MutableSegment.DEEP_OVERHEAD, 0, 0);
311        mss = store.memstore.getFlushableSize();
312        assertEquals(kvSize.getMemStoreSize(), mss);
313        MemStoreSizing kvSize2 = new NonThreadSafeMemStoreSizing();
314        store.add(new KeyValue(row, family, qf2, 2, (byte[]) null), kvSize2);
315        kvSize2.incMemStoreSize(0, MutableSegment.DEEP_OVERHEAD, 0, 0);
316        // Even though we add a new kv, we expect the flushable size to be 'same' since we have
317        // not yet cleared the snapshot -- the above flush failed.
318        assertEquals(kvSize.getMemStoreSize(), mss);
319        ffs.fault.set(false);
320        flushStore(store, id++);
321        mss = store.memstore.getFlushableSize();
322        // Size should be the foreground kv size.
323        assertEquals(kvSize2.getMemStoreSize(), mss);
324        flushStore(store, id++);
325        mss = store.memstore.getFlushableSize();
326        assertEquals(0, mss.getDataSize());
327        assertEquals(MutableSegment.DEEP_OVERHEAD, mss.getHeapSize());
328        return null;
329      }
330    });
331  }
332
333  @Test
334  public void testStoreBloomFilterMetricsWithBloomRowCol() throws IOException {
335    int numStoreFiles = 5;
336    writeAndRead(BloomType.ROWCOL, numStoreFiles);
337
338    assertEquals(0, store.getBloomFilterEligibleRequestsCount());
339    // hard to know exactly the numbers here, we are just trying to
340    // prove that they are incrementing
341    assertTrue(store.getBloomFilterRequestsCount() >= numStoreFiles);
342    assertTrue(store.getBloomFilterNegativeResultsCount() > 0);
343  }
344
345  @Test
346  public void testStoreBloomFilterMetricsWithBloomRow() throws IOException {
347    int numStoreFiles = 5;
348    writeAndRead(BloomType.ROWCOL, numStoreFiles);
349
350    assertEquals(0, store.getBloomFilterEligibleRequestsCount());
351    // hard to know exactly the numbers here, we are just trying to
352    // prove that they are incrementing
353    assertTrue(store.getBloomFilterRequestsCount() >= numStoreFiles);
354    assertTrue(store.getBloomFilterNegativeResultsCount() > 0);
355  }
356
357  @Test
358  public void testStoreBloomFilterMetricsWithBloomRowPrefix() throws IOException {
359    int numStoreFiles = 5;
360    writeAndRead(BloomType.ROWPREFIX_FIXED_LENGTH, numStoreFiles);
361
362    assertEquals(0, store.getBloomFilterEligibleRequestsCount());
363    // hard to know exactly the numbers here, we are just trying to
364    // prove that they are incrementing
365    assertTrue(store.getBloomFilterRequestsCount() >= numStoreFiles);
366  }
367
368  @Test
369  public void testStoreBloomFilterMetricsWithBloomNone() throws IOException {
370    int numStoreFiles = 5;
371    writeAndRead(BloomType.NONE, numStoreFiles);
372
373    assertEquals(0, store.getBloomFilterRequestsCount());
374    assertEquals(0, store.getBloomFilterNegativeResultsCount());
375
376    // hard to know exactly the numbers here, we are just trying to
377    // prove that they are incrementing
378    assertTrue(store.getBloomFilterEligibleRequestsCount() >= numStoreFiles);
379  }
380
381  private void writeAndRead(BloomType bloomType, int numStoreFiles) throws IOException {
382    Configuration conf = HBaseConfiguration.create();
383    FileSystem fs = FileSystem.get(conf);
384
385    ColumnFamilyDescriptor hcd = ColumnFamilyDescriptorBuilder.newBuilder(family)
386      .setCompressionType(Compression.Algorithm.GZ).setBloomFilterType(bloomType)
387      .setConfiguration(BloomFilterUtil.PREFIX_LENGTH_KEY, "3").build();
388    init(name, conf, hcd);
389
390    for (int i = 1; i <= numStoreFiles; i++) {
391      byte[] row = Bytes.toBytes("row" + i);
392      LOG.info("Adding some data for the store file #" + i);
393      long timeStamp = EnvironmentEdgeManager.currentTime();
394      this.store.add(new KeyValue(row, family, qf1, timeStamp, (byte[]) null), null);
395      this.store.add(new KeyValue(row, family, qf2, timeStamp, (byte[]) null), null);
396      this.store.add(new KeyValue(row, family, qf3, timeStamp, (byte[]) null), null);
397      flush(i);
398    }
399
400    // Verify the total number of store files
401    assertEquals(numStoreFiles, this.store.getStorefiles().size());
402
403    TreeSet<byte[]> columns = new TreeSet<>(Bytes.BYTES_COMPARATOR);
404    columns.add(qf1);
405
406    for (int i = 1; i <= numStoreFiles; i++) {
407      KeyValueScanner scanner =
408        store.getScanner(new Scan(new Get(Bytes.toBytes("row" + i))), columns, 0);
409      scanner.peek();
410    }
411  }
412
413  /**
414   * Verify that compression and data block encoding are respected by the createWriter method, used
415   * on store flush.
416   */
417  @Test
418  public void testCreateWriter() throws Exception {
419    Configuration conf = HBaseConfiguration.create();
420    FileSystem fs = FileSystem.get(conf);
421
422    ColumnFamilyDescriptor hcd =
423      ColumnFamilyDescriptorBuilder.newBuilder(family).setCompressionType(Compression.Algorithm.GZ)
424        .setDataBlockEncoding(DataBlockEncoding.DIFF).build();
425    init(name, conf, hcd);
426
427    // Test createWriter
428    StoreFileWriter writer = store.getStoreEngine()
429      .createWriter(CreateStoreFileWriterParams.create().maxKeyCount(4)
430        .compression(hcd.getCompressionType()).isCompaction(false).includeMVCCReadpoint(true)
431        .includesTag(false).shouldDropBehind(false));
432    Path path = writer.getPath();
433    writer.append(new KeyValue(row, family, qf1, Bytes.toBytes(1)));
434    writer.append(new KeyValue(row, family, qf2, Bytes.toBytes(2)));
435    writer.append(new KeyValue(row2, family, qf1, Bytes.toBytes(3)));
436    writer.append(new KeyValue(row2, family, qf2, Bytes.toBytes(4)));
437    writer.close();
438
439    // Verify that compression and encoding settings are respected
440    HFile.Reader reader = HFile.createReader(fs, path, new CacheConfig(conf), true, conf);
441    assertEquals(hcd.getCompressionType(), reader.getTrailer().getCompressionCodec());
442    assertEquals(hcd.getDataBlockEncoding(), reader.getDataBlockEncoding());
443    reader.close();
444  }
445
446  @Test
447  public void testDeleteExpiredStoreFiles() throws Exception {
448    testDeleteExpiredStoreFiles(0);
449    testDeleteExpiredStoreFiles(1);
450  }
451
452  /**
453   * @param minVersions the MIN_VERSIONS for the column family
454   */
455  public void testDeleteExpiredStoreFiles(int minVersions) throws Exception {
456    int storeFileNum = 4;
457    int ttl = 4;
458    IncrementingEnvironmentEdge edge = new IncrementingEnvironmentEdge();
459    EnvironmentEdgeManagerTestHelper.injectEdge(edge);
460
461    Configuration conf = HBaseConfiguration.create();
462    // Enable the expired store file deletion
463    conf.setBoolean("hbase.store.delete.expired.storefile", true);
464    // Set the compaction threshold higher to avoid normal compactions.
465    conf.setInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MIN_KEY, 5);
466
467    init(name + "-" + minVersions, conf, ColumnFamilyDescriptorBuilder.newBuilder(family)
468      .setMinVersions(minVersions).setTimeToLive(ttl).build());
469
470    long storeTtl = this.store.getScanInfo().getTtl();
471    long sleepTime = storeTtl / storeFileNum;
472    long timeStamp;
473    // There are 4 store files and the max time stamp difference among these
474    // store files will be (this.store.ttl / storeFileNum)
475    for (int i = 1; i <= storeFileNum; i++) {
476      LOG.info("Adding some data for the store file #" + i);
477      timeStamp = EnvironmentEdgeManager.currentTime();
478      this.store.add(new KeyValue(row, family, qf1, timeStamp, (byte[]) null), null);
479      this.store.add(new KeyValue(row, family, qf2, timeStamp, (byte[]) null), null);
480      this.store.add(new KeyValue(row, family, qf3, timeStamp, (byte[]) null), null);
481      flush(i);
482      edge.incrementTime(sleepTime);
483    }
484
485    // Verify the total number of store files
486    assertEquals(storeFileNum, this.store.getStorefiles().size());
487
488    // Each call will find one expired store file and delete it before compaction happens.
489    // There will be no compaction due to threshold above. Last file will not be replaced.
490    for (int i = 1; i <= storeFileNum - 1; i++) {
491      // verify the expired store file.
492      assertFalse(this.store.requestCompaction().isPresent());
493      Collection<HStoreFile> sfs = this.store.getStorefiles();
494      // Ensure i files are gone.
495      if (minVersions == 0) {
496        assertEquals(storeFileNum - i, sfs.size());
497        // Ensure only non-expired files remain.
498        for (HStoreFile sf : sfs) {
499          assertTrue(sf.getReader().getMaxTimestamp() >= (edge.currentTime() - storeTtl));
500        }
501      } else {
502        assertEquals(storeFileNum, sfs.size());
503      }
504      // Let the next store file expired.
505      edge.incrementTime(sleepTime);
506    }
507    assertFalse(this.store.requestCompaction().isPresent());
508
509    Collection<HStoreFile> sfs = this.store.getStorefiles();
510    // Assert the last expired file is not removed.
511    if (minVersions == 0) {
512      assertEquals(1, sfs.size());
513    }
514    long ts = sfs.iterator().next().getReader().getMaxTimestamp();
515    assertTrue(ts < (edge.currentTime() - storeTtl));
516
517    for (HStoreFile sf : sfs) {
518      sf.closeStoreFile(true);
519    }
520  }
521
522  @Test
523  public void testLowestModificationTime() throws Exception {
524    Configuration conf = HBaseConfiguration.create();
525    FileSystem fs = FileSystem.get(conf);
526    // Initialize region
527    init(name, conf);
528
529    int storeFileNum = 4;
530    for (int i = 1; i <= storeFileNum; i++) {
531      LOG.info("Adding some data for the store file #" + i);
532      this.store.add(new KeyValue(row, family, qf1, i, (byte[]) null), null);
533      this.store.add(new KeyValue(row, family, qf2, i, (byte[]) null), null);
534      this.store.add(new KeyValue(row, family, qf3, i, (byte[]) null), null);
535      flush(i);
536    }
537    // after flush; check the lowest time stamp
538    long lowestTimeStampFromManager = StoreUtils.getLowestTimestamp(store.getStorefiles());
539    long lowestTimeStampFromFS = getLowestTimeStampFromFS(fs, store.getStorefiles());
540    assertEquals(lowestTimeStampFromManager, lowestTimeStampFromFS);
541
542    // after compact; check the lowest time stamp
543    store.compact(store.requestCompaction().get(), NoLimitThroughputController.INSTANCE, null);
544    lowestTimeStampFromManager = StoreUtils.getLowestTimestamp(store.getStorefiles());
545    lowestTimeStampFromFS = getLowestTimeStampFromFS(fs, store.getStorefiles());
546    assertEquals(lowestTimeStampFromManager, lowestTimeStampFromFS);
547  }
548
549  private static long getLowestTimeStampFromFS(FileSystem fs,
550    final Collection<HStoreFile> candidates) throws IOException {
551    long minTs = Long.MAX_VALUE;
552    if (candidates.isEmpty()) {
553      return minTs;
554    }
555    Path[] p = new Path[candidates.size()];
556    int i = 0;
557    for (HStoreFile sf : candidates) {
558      p[i] = sf.getPath();
559      ++i;
560    }
561
562    FileStatus[] stats = fs.listStatus(p);
563    if (stats == null || stats.length == 0) {
564      return minTs;
565    }
566    for (FileStatus s : stats) {
567      minTs = Math.min(minTs, s.getModificationTime());
568    }
569    return minTs;
570  }
571
572  //////////////////////////////////////////////////////////////////////////////
573  // Get tests
574  //////////////////////////////////////////////////////////////////////////////
575
576  private static final int BLOCKSIZE_SMALL = 8192;
577
578  /**
579   * Test for hbase-1686.
580   */
581  @Test
582  public void testEmptyStoreFile() throws IOException {
583    init(name);
584    // Write a store file.
585    this.store.add(new KeyValue(row, family, qf1, 1, (byte[]) null), null);
586    this.store.add(new KeyValue(row, family, qf2, 1, (byte[]) null), null);
587    flush(1);
588    // Now put in place an empty store file. Its a little tricky. Have to
589    // do manually with hacked in sequence id.
590    HStoreFile f = this.store.getStorefiles().iterator().next();
591    Path storedir = f.getPath().getParent();
592    long seqid = f.getMaxSequenceId();
593    Configuration c = HBaseConfiguration.create();
594    FileSystem fs = FileSystem.get(c);
595    HFileContext meta = new HFileContextBuilder().withBlockSize(BLOCKSIZE_SMALL).build();
596    StoreFileWriter w = new StoreFileWriter.Builder(c, new CacheConfig(c), fs)
597      .withOutputDir(storedir).withFileContext(meta).build();
598    w.appendMetadata(seqid + 1, false);
599    w.close();
600    this.store.close();
601    // Reopen it... should pick up two files
602    this.store =
603      new HStore(this.store.getHRegion(), this.store.getColumnFamilyDescriptor(), c, false);
604    assertEquals(2, this.store.getStorefilesCount());
605
606    result = HBaseTestingUtil.getFromStoreFile(store, get.getRow(), qualifiers);
607    assertEquals(1, result.size());
608  }
609
610  /**
611   * Getting data from memstore only
612   */
613  @Test
614  public void testGet_FromMemStoreOnly() throws IOException {
615    init(name);
616
617    // Put data in memstore
618    this.store.add(new KeyValue(row, family, qf1, 1, (byte[]) null), null);
619    this.store.add(new KeyValue(row, family, qf2, 1, (byte[]) null), null);
620    this.store.add(new KeyValue(row, family, qf3, 1, (byte[]) null), null);
621    this.store.add(new KeyValue(row, family, qf4, 1, (byte[]) null), null);
622    this.store.add(new KeyValue(row, family, qf5, 1, (byte[]) null), null);
623    this.store.add(new KeyValue(row, family, qf6, 1, (byte[]) null), null);
624
625    // Get
626    result = HBaseTestingUtil.getFromStoreFile(store, get.getRow(), qualifiers);
627
628    // Compare
629    assertCheck();
630  }
631
632  @Test
633  public void testTimeRangeIfSomeCellsAreDroppedInFlush() throws IOException {
634    testTimeRangeIfSomeCellsAreDroppedInFlush(1);
635    testTimeRangeIfSomeCellsAreDroppedInFlush(3);
636    testTimeRangeIfSomeCellsAreDroppedInFlush(5);
637  }
638
639  private void testTimeRangeIfSomeCellsAreDroppedInFlush(int maxVersion) throws IOException {
640    init(name, TEST_UTIL.getConfiguration(),
641      ColumnFamilyDescriptorBuilder.newBuilder(family).setMaxVersions(maxVersion).build());
642    long currentTs = 100;
643    long minTs = currentTs;
644    // the extra cell won't be flushed to disk,
645    // so the min of timerange will be different between memStore and hfile.
646    for (int i = 0; i != (maxVersion + 1); ++i) {
647      this.store.add(new KeyValue(row, family, qf1, ++currentTs, (byte[]) null), null);
648      if (i == 1) {
649        minTs = currentTs;
650      }
651    }
652    flushStore(store, id++);
653
654    Collection<HStoreFile> files = store.getStorefiles();
655    assertEquals(1, files.size());
656    HStoreFile f = files.iterator().next();
657    f.initReader();
658    StoreFileReader reader = f.getReader();
659    assertEquals(minTs, reader.timeRange.getMin());
660    assertEquals(currentTs, reader.timeRange.getMax());
661  }
662
663  /**
664   * Getting data from files only
665   */
666  @Test
667  public void testGet_FromFilesOnly() throws IOException {
668    init(name);
669
670    // Put data in memstore
671    this.store.add(new KeyValue(row, family, qf1, 1, (byte[]) null), null);
672    this.store.add(new KeyValue(row, family, qf2, 1, (byte[]) null), null);
673    // flush
674    flush(1);
675
676    // Add more data
677    this.store.add(new KeyValue(row, family, qf3, 1, (byte[]) null), null);
678    this.store.add(new KeyValue(row, family, qf4, 1, (byte[]) null), null);
679    // flush
680    flush(2);
681
682    // Add more data
683    this.store.add(new KeyValue(row, family, qf5, 1, (byte[]) null), null);
684    this.store.add(new KeyValue(row, family, qf6, 1, (byte[]) null), null);
685    // flush
686    flush(3);
687
688    // Get
689    result = HBaseTestingUtil.getFromStoreFile(store, get.getRow(), qualifiers);
690    // this.store.get(get, qualifiers, result);
691
692    // Need to sort the result since multiple files
693    Collections.sort(result, CellComparatorImpl.COMPARATOR);
694
695    // Compare
696    assertCheck();
697  }
698
699  /**
700   * Getting data from memstore and files
701   */
702  @Test
703  public void testGet_FromMemStoreAndFiles() throws IOException {
704    init(name);
705
706    // Put data in memstore
707    this.store.add(new KeyValue(row, family, qf1, 1, (byte[]) null), null);
708    this.store.add(new KeyValue(row, family, qf2, 1, (byte[]) null), null);
709    // flush
710    flush(1);
711
712    // Add more data
713    this.store.add(new KeyValue(row, family, qf3, 1, (byte[]) null), null);
714    this.store.add(new KeyValue(row, family, qf4, 1, (byte[]) null), null);
715    // flush
716    flush(2);
717
718    // Add more data
719    this.store.add(new KeyValue(row, family, qf5, 1, (byte[]) null), null);
720    this.store.add(new KeyValue(row, family, qf6, 1, (byte[]) null), null);
721
722    // Get
723    result = HBaseTestingUtil.getFromStoreFile(store, get.getRow(), qualifiers);
724
725    // Need to sort the result since multiple files
726    Collections.sort(result, CellComparatorImpl.COMPARATOR);
727
728    // Compare
729    assertCheck();
730  }
731
732  private void flush(int storeFilessize) throws IOException {
733    flushStore(store, id++);
734    assertEquals(storeFilessize, this.store.getStorefiles().size());
735    assertEquals(0, ((AbstractMemStore) this.store.memstore).getActive().getCellsCount());
736  }
737
738  private void assertCheck() {
739    assertEquals(expected.size(), result.size());
740    for (int i = 0; i < expected.size(); i++) {
741      assertEquals(expected.get(i), result.get(i));
742    }
743  }
744
745  @AfterEach
746  public void tearDown() throws Exception {
747    EnvironmentEdgeManagerTestHelper.reset();
748    if (store != null) {
749      try {
750        store.close();
751      } catch (IOException e) {
752      }
753      store = null;
754    }
755    if (region != null) {
756      region.close();
757      region = null;
758    }
759  }
760
761  @AfterAll
762  public static void tearDownAfterClass() throws IOException {
763    TEST_UTIL.cleanupTestDir();
764  }
765
766  @Test
767  public void testHandleErrorsInFlush() throws Exception {
768    LOG.info("Setting up a faulty file system that cannot write");
769
770    final Configuration conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration());
771    User user = User.createUserForTesting(conf, "testhandleerrorsinflush", new String[] { "foo" });
772    // Inject our faulty LocalFileSystem
773    conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class);
774    user.runAs(new PrivilegedExceptionAction<Object>() {
775      @Override
776      public Object run() throws Exception {
777        // Make sure it worked (above is sensitive to caching details in hadoop core)
778        FileSystem fs = FileSystem.get(conf);
779        assertEquals(FaultyFileSystem.class, fs.getClass());
780
781        // Initialize region
782        init(name, conf);
783
784        LOG.info("Adding some data");
785        store.add(new KeyValue(row, family, qf1, 1, (byte[]) null), null);
786        store.add(new KeyValue(row, family, qf2, 1, (byte[]) null), null);
787        store.add(new KeyValue(row, family, qf3, 1, (byte[]) null), null);
788
789        LOG.info("Before flush, we should have no files");
790
791        StoreFileTracker sft = StoreFileTrackerFactory.create(conf, false, store.getStoreContext());
792        Collection<StoreFileInfo> files = sft.load();
793        assertEquals(0, files != null ? files.size() : 0);
794
795        // flush
796        try {
797          LOG.info("Flushing");
798          flush(1);
799          fail("Didn't bubble up IOE!");
800        } catch (IOException ioe) {
801          assertTrue(ioe.getMessage().contains("Fault injected"));
802        }
803
804        LOG.info("After failed flush, we should still have no files!");
805        files = sft.load();
806        assertEquals(0, files != null ? files.size() : 0);
807        store.getHRegion().getWAL().close();
808        return null;
809      }
810    });
811    FileSystem.closeAllForUGI(user.getUGI());
812  }
813
814  /**
815   * Faulty file system that will fail if you write past its fault position the FIRST TIME only;
816   * thereafter it will succeed. Used by {@link TestHRegion} too.
817   */
818  static class FaultyFileSystem extends FilterFileSystem {
819    List<SoftReference<FaultyOutputStream>> outStreams = new ArrayList<>();
820    private long faultPos = 200;
821    AtomicBoolean fault = new AtomicBoolean(true);
822
823    public FaultyFileSystem() {
824      super(new LocalFileSystem());
825      LOG.info("Creating faulty!");
826    }
827
828    @Override
829    public FSDataOutputStream create(Path p) throws IOException {
830      return new FaultyOutputStream(super.create(p), faultPos, fault);
831    }
832
833    @Override
834    public FSDataOutputStream create(Path f, FsPermission permission, boolean overwrite,
835      int bufferSize, short replication, long blockSize, Progressable progress) throws IOException {
836      return new FaultyOutputStream(
837        super.create(f, permission, overwrite, bufferSize, replication, blockSize, progress),
838        faultPos, fault);
839    }
840
841    @Override
842    public FSDataOutputStream createNonRecursive(Path f, boolean overwrite, int bufferSize,
843      short replication, long blockSize, Progressable progress) throws IOException {
844      // Fake it. Call create instead. The default implementation throws an IOE
845      // that this is not supported.
846      return create(f, overwrite, bufferSize, replication, blockSize, progress);
847    }
848  }
849
850  static class FaultyOutputStream extends FSDataOutputStream {
851    volatile long faultPos = Long.MAX_VALUE;
852    private final AtomicBoolean fault;
853
854    public FaultyOutputStream(FSDataOutputStream out, long faultPos, final AtomicBoolean fault)
855      throws IOException {
856      super(out, null);
857      this.faultPos = faultPos;
858      this.fault = fault;
859    }
860
861    @Override
862    public synchronized void write(byte[] buf, int offset, int length) throws IOException {
863      LOG.info("faulty stream write at pos " + getPos());
864      injectFault();
865      super.write(buf, offset, length);
866    }
867
868    private void injectFault() throws IOException {
869      if (this.fault.get() && getPos() >= faultPos) {
870        throw new IOException("Fault injected");
871      }
872    }
873  }
874
875  private static StoreFlushContext flushStore(HStore store, long id) throws IOException {
876    StoreFlushContext storeFlushCtx = store.createFlushContext(id, FlushLifeCycleTracker.DUMMY);
877    storeFlushCtx.prepare();
878    storeFlushCtx.flushCache(Mockito.mock(MonitoredTask.class));
879    storeFlushCtx.commit(Mockito.mock(MonitoredTask.class));
880    return storeFlushCtx;
881  }
882
883  /**
884   * Generate a list of KeyValues for testing based on given parameters
885   * @return the rows key-value list
886   */
887  private List<ExtendedCell> getKeyValueSet(long[] timestamps, int numRows, byte[] qualifier,
888    byte[] family) {
889    List<ExtendedCell> kvList = new ArrayList<>();
890    for (int i = 1; i <= numRows; i++) {
891      byte[] b = Bytes.toBytes(i);
892      for (long timestamp : timestamps) {
893        kvList.add(new KeyValue(b, family, qualifier, timestamp, b));
894      }
895    }
896    return kvList;
897  }
898
899  /**
900   * Test to ensure correctness when using Stores with multiple timestamps
901   */
902  @Test
903  public void testMultipleTimestamps() throws IOException {
904    int numRows = 1;
905    long[] timestamps1 = new long[] { 1, 5, 10, 20 };
906    long[] timestamps2 = new long[] { 30, 80 };
907
908    init(name);
909
910    List<ExtendedCell> kvList1 = getKeyValueSet(timestamps1, numRows, qf1, family);
911    for (ExtendedCell kv : kvList1) {
912      this.store.add(kv, null);
913    }
914
915    flushStore(store, id++);
916
917    List<ExtendedCell> kvList2 = getKeyValueSet(timestamps2, numRows, qf1, family);
918    for (ExtendedCell kv : kvList2) {
919      this.store.add(kv, null);
920    }
921
922    List<Cell> result;
923    Get get = new Get(Bytes.toBytes(1));
924    get.addColumn(family, qf1);
925
926    get.setTimeRange(0, 15);
927    result = HBaseTestingUtil.getFromStoreFile(store, get);
928    assertTrue(result.size() > 0);
929
930    get.setTimeRange(40, 90);
931    result = HBaseTestingUtil.getFromStoreFile(store, get);
932    assertTrue(result.size() > 0);
933
934    get.setTimeRange(10, 45);
935    result = HBaseTestingUtil.getFromStoreFile(store, get);
936    assertTrue(result.size() > 0);
937
938    get.setTimeRange(80, 145);
939    result = HBaseTestingUtil.getFromStoreFile(store, get);
940    assertTrue(result.size() > 0);
941
942    get.setTimeRange(1, 2);
943    result = HBaseTestingUtil.getFromStoreFile(store, get);
944    assertTrue(result.size() > 0);
945
946    get.setTimeRange(90, 200);
947    result = HBaseTestingUtil.getFromStoreFile(store, get);
948    assertTrue(result.size() == 0);
949  }
950
951  /**
952   * Test for HBASE-3492 - Test split on empty colfam (no store files).
953   * @throws IOException When the IO operations fail.
954   */
955  @Test
956  public void testSplitWithEmptyColFam() throws IOException {
957    init(name);
958    assertFalse(store.getSplitPoint().isPresent());
959  }
960
961  @Test
962  public void testStoreUsesConfigurationFromHcdAndHtd() throws Exception {
963    final String CONFIG_KEY = "hbase.regionserver.thread.compaction.throttle";
964    long anyValue = 10;
965
966    // We'll check that it uses correct config and propagates it appropriately by going thru
967    // the simplest "real" path I can find - "throttleCompaction", which just checks whether
968    // a number we pass in is higher than some config value, inside compactionPolicy.
969    Configuration conf = HBaseConfiguration.create();
970    conf.setLong(CONFIG_KEY, anyValue);
971    init(name + "-xml", conf);
972    assertTrue(store.throttleCompaction(anyValue + 1));
973    assertFalse(store.throttleCompaction(anyValue));
974
975    // HTD overrides XML.
976    --anyValue;
977    init(name + "-htd", conf, TableDescriptorBuilder.newBuilder(TableName.valueOf(table))
978      .setValue(CONFIG_KEY, Long.toString(anyValue)), ColumnFamilyDescriptorBuilder.of(family));
979    assertTrue(store.throttleCompaction(anyValue + 1));
980    assertFalse(store.throttleCompaction(anyValue));
981
982    // HCD overrides them both.
983    --anyValue;
984    init(name + "-hcd", conf,
985      TableDescriptorBuilder.newBuilder(TableName.valueOf(table)).setValue(CONFIG_KEY,
986        Long.toString(anyValue)),
987      ColumnFamilyDescriptorBuilder.newBuilder(family).setValue(CONFIG_KEY, Long.toString(anyValue))
988        .build());
989    assertTrue(store.throttleCompaction(anyValue + 1));
990    assertFalse(store.throttleCompaction(anyValue));
991  }
992
993  public static class DummyStoreEngine extends DefaultStoreEngine {
994    public static DefaultCompactor lastCreatedCompactor = null;
995
996    @Override
997    protected void createComponents(Configuration conf, HStore store, CellComparator comparator)
998      throws IOException {
999      super.createComponents(conf, store, comparator);
1000      lastCreatedCompactor = this.compactor;
1001    }
1002  }
1003
1004  @Test
1005  public void testStoreUsesSearchEngineOverride() throws Exception {
1006    Configuration conf = HBaseConfiguration.create();
1007    conf.set(StoreEngine.STORE_ENGINE_CLASS_KEY, DummyStoreEngine.class.getName());
1008    init(name, conf);
1009    assertEquals(DummyStoreEngine.lastCreatedCompactor, this.store.storeEngine.getCompactor());
1010  }
1011
1012  private void addStoreFile() throws IOException {
1013    HStoreFile f = this.store.getStorefiles().iterator().next();
1014    Path storedir = f.getPath().getParent();
1015    long seqid = this.store.getMaxSequenceId().orElse(0L);
1016    Configuration c = TEST_UTIL.getConfiguration();
1017    FileSystem fs = FileSystem.get(c);
1018    HFileContext fileContext = new HFileContextBuilder().withBlockSize(BLOCKSIZE_SMALL).build();
1019    StoreFileWriter w = new StoreFileWriter.Builder(c, new CacheConfig(c), fs)
1020      .withOutputDir(storedir).withFileContext(fileContext).build();
1021    w.appendMetadata(seqid + 1, false);
1022    w.close();
1023    LOG.info("Added store file:" + w.getPath());
1024  }
1025
1026  private void archiveStoreFile(int index) throws IOException {
1027    Collection<HStoreFile> files = this.store.getStorefiles();
1028    HStoreFile sf = null;
1029    Iterator<HStoreFile> it = files.iterator();
1030    for (int i = 0; i <= index; i++) {
1031      sf = it.next();
1032    }
1033    StoreFileTracker sft =
1034      StoreFileTrackerFactory.create(store.getRegionFileSystem().getFileSystem().getConf(),
1035        store.isPrimaryReplicaStore(), store.getStoreContext());
1036    sft.removeStoreFiles(Lists.newArrayList(sf));
1037  }
1038
1039  private void closeCompactedFile(int index) throws IOException {
1040    Collection<HStoreFile> files =
1041      this.store.getStoreEngine().getStoreFileManager().getCompactedfiles();
1042    if (files.size() > 0) {
1043      HStoreFile sf = null;
1044      Iterator<HStoreFile> it = files.iterator();
1045      for (int i = 0; i <= index; i++) {
1046        sf = it.next();
1047      }
1048      sf.closeStoreFile(true);
1049      store.getStoreEngine().getStoreFileManager()
1050        .removeCompactedFiles(Collections.singletonList(sf));
1051    }
1052  }
1053
1054  @Test
1055  public void testRefreshStoreFiles() throws Exception {
1056    init(name);
1057
1058    assertEquals(0, this.store.getStorefilesCount());
1059
1060    // Test refreshing store files when no store files are there
1061    store.refreshStoreFiles();
1062    assertEquals(0, this.store.getStorefilesCount());
1063
1064    // add some data, flush
1065    this.store.add(new KeyValue(row, family, qf1, 1, (byte[]) null), null);
1066    flush(1);
1067    assertEquals(1, this.store.getStorefilesCount());
1068
1069    // add one more file
1070    addStoreFile();
1071
1072    assertEquals(1, this.store.getStorefilesCount());
1073    store.refreshStoreFiles();
1074    assertEquals(2, this.store.getStorefilesCount());
1075
1076    // add three more files
1077    addStoreFile();
1078    addStoreFile();
1079    addStoreFile();
1080
1081    assertEquals(2, this.store.getStorefilesCount());
1082    store.refreshStoreFiles();
1083    assertEquals(5, this.store.getStorefilesCount());
1084
1085    closeCompactedFile(0);
1086    archiveStoreFile(0);
1087
1088    assertEquals(5, this.store.getStorefilesCount());
1089    store.refreshStoreFiles();
1090    assertEquals(4, this.store.getStorefilesCount());
1091
1092    archiveStoreFile(0);
1093    archiveStoreFile(1);
1094    archiveStoreFile(2);
1095
1096    assertEquals(4, this.store.getStorefilesCount());
1097    store.refreshStoreFiles();
1098    assertEquals(1, this.store.getStorefilesCount());
1099
1100    archiveStoreFile(0);
1101    store.refreshStoreFiles();
1102    assertEquals(0, this.store.getStorefilesCount());
1103  }
1104
1105  @Test
1106  public void testRefreshStoreFilesNotChanged() throws IOException {
1107    init(name);
1108
1109    assertEquals(0, this.store.getStorefilesCount());
1110
1111    // add some data, flush
1112    this.store.add(new KeyValue(row, family, qf1, 1, (byte[]) null), null);
1113    flush(1);
1114    // add one more file
1115    addStoreFile();
1116
1117    StoreEngine<?, ?, ?, ?> spiedStoreEngine = spy(store.getStoreEngine());
1118
1119    // call first time after files changed
1120    spiedStoreEngine.refreshStoreFiles();
1121    assertEquals(2, this.store.getStorefilesCount());
1122    verify(spiedStoreEngine, times(1)).replaceStoreFiles(any(), any(), any(), any());
1123
1124    // call second time
1125    spiedStoreEngine.refreshStoreFiles();
1126
1127    // ensure that replaceStoreFiles is not called, i.e, the times does not change, if files are not
1128    // refreshed,
1129    verify(spiedStoreEngine, times(1)).replaceStoreFiles(any(), any(), any(), any());
1130  }
1131
1132  @Test
1133  public void testScanWithCompactionAfterFlush() throws Exception {
1134    TEST_UTIL.getConfiguration().set(DEFAULT_COMPACTION_POLICY_CLASS_KEY,
1135      EverythingPolicy.class.getName());
1136    init(name);
1137
1138    assertEquals(0, this.store.getStorefilesCount());
1139
1140    KeyValue kv = new KeyValue(row, family, qf1, 1, (byte[]) null);
1141    // add some data, flush
1142    this.store.add(kv, null);
1143    flush(1);
1144    kv = new KeyValue(row, family, qf2, 1, (byte[]) null);
1145    // add some data, flush
1146    this.store.add(kv, null);
1147    flush(2);
1148    kv = new KeyValue(row, family, qf3, 1, (byte[]) null);
1149    // add some data, flush
1150    this.store.add(kv, null);
1151    flush(3);
1152
1153    ExecutorService service = Executors.newFixedThreadPool(2);
1154
1155    Scan scan = new Scan(new Get(row));
1156    Future<KeyValueScanner> scanFuture = service.submit(() -> {
1157      try {
1158        LOG.info(">>>> creating scanner");
1159        return this.store.createScanner(scan,
1160          new ScanInfo(HBaseConfiguration.create(),
1161            ColumnFamilyDescriptorBuilder.newBuilder(family).setMaxVersions(4).build(),
1162            Long.MAX_VALUE, 0, CellComparator.getInstance()),
1163          scan.getFamilyMap().get(store.getColumnFamilyDescriptor().getName()), 0);
1164      } catch (IOException e) {
1165        e.printStackTrace();
1166        return null;
1167      }
1168    });
1169    Future compactFuture = service.submit(() -> {
1170      try {
1171        LOG.info(">>>>>> starting compaction");
1172        Optional<CompactionContext> opCompaction = this.store.requestCompaction();
1173        assertTrue(opCompaction.isPresent());
1174        store.compact(opCompaction.get(), new NoLimitThroughputController(), User.getCurrent());
1175        LOG.info(">>>>>> Compaction is finished");
1176        this.store.closeAndArchiveCompactedFiles();
1177        LOG.info(">>>>>> Compacted files deleted");
1178      } catch (IOException e) {
1179        e.printStackTrace();
1180      }
1181    });
1182
1183    KeyValueScanner kvs = scanFuture.get();
1184    compactFuture.get();
1185    ((StoreScanner) kvs).currentScanners.forEach(s -> {
1186      if (s instanceof StoreFileScanner) {
1187        assertEquals(1, ((StoreFileScanner) s).getReader().getRefCount());
1188      }
1189    });
1190    kvs.seek(kv);
1191    service.shutdownNow();
1192  }
1193
1194  private long countMemStoreScanner(StoreScanner scanner) {
1195    if (scanner.currentScanners == null) {
1196      return 0;
1197    }
1198    return scanner.currentScanners.stream().filter(s -> !s.isFileScanner()).count();
1199  }
1200
1201  @Test
1202  public void testNumberOfMemStoreScannersAfterFlush() throws IOException {
1203    long seqId = 100;
1204    long timestamp = EnvironmentEdgeManager.currentTime();
1205    ExtendedCell cell0 =
1206      ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(row).setFamily(family)
1207        .setQualifier(qf1).setTimestamp(timestamp).setType(Cell.Type.Put).setValue(qf1).build();
1208    PrivateCellUtil.setSequenceId(cell0, seqId);
1209    testNumberOfMemStoreScannersAfterFlush(Arrays.asList(cell0), Collections.emptyList());
1210
1211    ExtendedCell cell1 =
1212      ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(row).setFamily(family)
1213        .setQualifier(qf2).setTimestamp(timestamp).setType(Cell.Type.Put).setValue(qf1).build();
1214    PrivateCellUtil.setSequenceId(cell1, seqId);
1215    testNumberOfMemStoreScannersAfterFlush(Arrays.asList(cell0), Arrays.asList(cell1));
1216
1217    seqId = 101;
1218    timestamp = EnvironmentEdgeManager.currentTime();
1219    ExtendedCell cell2 =
1220      ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(row2).setFamily(family)
1221        .setQualifier(qf2).setTimestamp(timestamp).setType(Cell.Type.Put).setValue(qf1).build();
1222    PrivateCellUtil.setSequenceId(cell2, seqId);
1223    testNumberOfMemStoreScannersAfterFlush(Arrays.asList(cell0), Arrays.asList(cell1, cell2));
1224  }
1225
1226  private void testNumberOfMemStoreScannersAfterFlush(List<ExtendedCell> inputCellsBeforeSnapshot,
1227    List<ExtendedCell> inputCellsAfterSnapshot) throws IOException {
1228    init(name + "-" + inputCellsBeforeSnapshot.size());
1229    TreeSet<byte[]> quals = new TreeSet<>(Bytes.BYTES_COMPARATOR);
1230    long seqId = Long.MIN_VALUE;
1231    for (ExtendedCell c : inputCellsBeforeSnapshot) {
1232      quals.add(CellUtil.cloneQualifier(c));
1233      seqId = Math.max(seqId, c.getSequenceId());
1234    }
1235    for (ExtendedCell c : inputCellsAfterSnapshot) {
1236      quals.add(CellUtil.cloneQualifier(c));
1237      seqId = Math.max(seqId, c.getSequenceId());
1238    }
1239    inputCellsBeforeSnapshot.forEach(c -> store.add(c, null));
1240    StoreFlushContext storeFlushCtx = store.createFlushContext(id++, FlushLifeCycleTracker.DUMMY);
1241    storeFlushCtx.prepare();
1242    inputCellsAfterSnapshot.forEach(c -> store.add(c, null));
1243    int numberOfMemScannersBeforeFlush = inputCellsAfterSnapshot.isEmpty() ? 1 : 2;
1244    try (StoreScanner s = (StoreScanner) store.getScanner(new Scan(), quals, seqId)) {
1245      // snapshot + active (if inputCellsAfterSnapshot isn't empty)
1246      assertEquals(numberOfMemScannersBeforeFlush, countMemStoreScanner(s));
1247      storeFlushCtx.flushCache(Mockito.mock(MonitoredTask.class));
1248      storeFlushCtx.commit(Mockito.mock(MonitoredTask.class));
1249      // snapshot has no data after flush
1250      int numberOfMemScannersAfterFlush = inputCellsAfterSnapshot.isEmpty() ? 0 : 1;
1251      boolean more;
1252      int cellCount = 0;
1253      do {
1254        List<Cell> cells = new ArrayList<>();
1255        more = s.next(cells);
1256        cellCount += cells.size();
1257        assertEquals(more ? numberOfMemScannersAfterFlush : 0, countMemStoreScanner(s));
1258      } while (more);
1259      assertEquals(inputCellsBeforeSnapshot.size() + inputCellsAfterSnapshot.size(), cellCount,
1260        "The number of cells added before snapshot is " + inputCellsBeforeSnapshot.size()
1261          + ", The number of cells added after snapshot is " + inputCellsAfterSnapshot.size());
1262      // the current scanners is cleared
1263      assertEquals(0, countMemStoreScanner(s));
1264    }
1265  }
1266
1267  private ExtendedCell createCell(byte[] qualifier, long ts, long sequenceId, byte[] value)
1268    throws IOException {
1269    return createCell(row, qualifier, ts, sequenceId, value);
1270  }
1271
1272  private ExtendedCell createCell(byte[] row, byte[] qualifier, long ts, long sequenceId,
1273    byte[] value) throws IOException {
1274    return ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(row)
1275      .setFamily(family).setQualifier(qualifier).setTimestamp(ts).setType(Cell.Type.Put)
1276      .setValue(value).setSequenceId(sequenceId).build();
1277  }
1278
1279  private ExtendedCell createDeleteCell(byte[] row, byte[] qualifier, long ts, long sequenceId) {
1280    return ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(row)
1281      .setFamily(family).setQualifier(qualifier).setTimestamp(ts).setType(Cell.Type.DeleteColumn)
1282      .setSequenceId(sequenceId).build();
1283  }
1284
1285  @Test
1286  public void testFlushBeforeCompletingScanWoFilter() throws IOException, InterruptedException {
1287    final AtomicBoolean timeToGoNextRow = new AtomicBoolean(false);
1288    final int expectedSize = 3;
1289    testFlushBeforeCompletingScan(new MyListHook() {
1290      @Override
1291      public void hook(int currentSize) {
1292        if (currentSize == expectedSize - 1) {
1293          try {
1294            flushStore(store, id++);
1295            timeToGoNextRow.set(true);
1296          } catch (IOException e) {
1297            throw new RuntimeException(e);
1298          }
1299        }
1300      }
1301    }, new FilterBase() {
1302      @Override
1303      public Filter.ReturnCode filterCell(final Cell c) throws IOException {
1304        return ReturnCode.INCLUDE;
1305      }
1306    }, expectedSize);
1307  }
1308
1309  @Test
1310  public void testFlushBeforeCompletingScanWithFilter() throws IOException, InterruptedException {
1311    final AtomicBoolean timeToGoNextRow = new AtomicBoolean(false);
1312    final int expectedSize = 2;
1313    testFlushBeforeCompletingScan(new MyListHook() {
1314      @Override
1315      public void hook(int currentSize) {
1316        if (currentSize == expectedSize - 1) {
1317          try {
1318            flushStore(store, id++);
1319            timeToGoNextRow.set(true);
1320          } catch (IOException e) {
1321            throw new RuntimeException(e);
1322          }
1323        }
1324      }
1325    }, new FilterBase() {
1326      @Override
1327      public Filter.ReturnCode filterCell(final Cell c) throws IOException {
1328        if (timeToGoNextRow.get()) {
1329          timeToGoNextRow.set(false);
1330          return ReturnCode.NEXT_ROW;
1331        } else {
1332          return ReturnCode.INCLUDE;
1333        }
1334      }
1335    }, expectedSize);
1336  }
1337
1338  @Test
1339  public void testFlushBeforeCompletingScanWithFilterHint()
1340    throws IOException, InterruptedException {
1341    final AtomicBoolean timeToGetHint = new AtomicBoolean(false);
1342    final int expectedSize = 2;
1343    testFlushBeforeCompletingScan(new MyListHook() {
1344      @Override
1345      public void hook(int currentSize) {
1346        if (currentSize == expectedSize - 1) {
1347          try {
1348            flushStore(store, id++);
1349            timeToGetHint.set(true);
1350          } catch (IOException e) {
1351            throw new RuntimeException(e);
1352          }
1353        }
1354      }
1355    }, new FilterBase() {
1356      @Override
1357      public Filter.ReturnCode filterCell(final Cell c) throws IOException {
1358        if (timeToGetHint.get()) {
1359          timeToGetHint.set(false);
1360          return Filter.ReturnCode.SEEK_NEXT_USING_HINT;
1361        } else {
1362          return Filter.ReturnCode.INCLUDE;
1363        }
1364      }
1365
1366      @Override
1367      public Cell getNextCellHint(Cell currentCell) throws IOException {
1368        return currentCell;
1369      }
1370    }, expectedSize);
1371  }
1372
1373  private void testFlushBeforeCompletingScan(MyListHook hook, Filter filter, int expectedSize)
1374    throws IOException, InterruptedException {
1375    Configuration conf = HBaseConfiguration.create();
1376    byte[] r0 = Bytes.toBytes("row0");
1377    byte[] r1 = Bytes.toBytes("row1");
1378    byte[] r2 = Bytes.toBytes("row2");
1379    byte[] value0 = Bytes.toBytes("value0");
1380    byte[] value1 = Bytes.toBytes("value1");
1381    byte[] value2 = Bytes.toBytes("value2");
1382    MemStoreSizing memStoreSizing = new NonThreadSafeMemStoreSizing();
1383    long ts = EnvironmentEdgeManager.currentTime();
1384    long seqId = 100;
1385    init(name, conf, TableDescriptorBuilder.newBuilder(TableName.valueOf(table)),
1386      ColumnFamilyDescriptorBuilder.newBuilder(family).setMaxVersions(1).build(),
1387      new MyStoreHook() {
1388        @Override
1389        public long getSmallestReadPoint(HStore store) {
1390          return seqId + 3;
1391        }
1392      });
1393    // The cells having the value0 won't be flushed to disk because the value of max version is 1
1394    store.add(createCell(r0, qf1, ts, seqId, value0), memStoreSizing);
1395    store.add(createCell(r0, qf2, ts, seqId, value0), memStoreSizing);
1396    store.add(createCell(r0, qf3, ts, seqId, value0), memStoreSizing);
1397    store.add(createCell(r1, qf1, ts + 1, seqId + 1, value1), memStoreSizing);
1398    store.add(createCell(r1, qf2, ts + 1, seqId + 1, value1), memStoreSizing);
1399    store.add(createCell(r1, qf3, ts + 1, seqId + 1, value1), memStoreSizing);
1400    store.add(createCell(r2, qf1, ts + 2, seqId + 2, value2), memStoreSizing);
1401    store.add(createCell(r2, qf2, ts + 2, seqId + 2, value2), memStoreSizing);
1402    store.add(createCell(r2, qf3, ts + 2, seqId + 2, value2), memStoreSizing);
1403    store.add(createCell(r1, qf1, ts + 3, seqId + 3, value1), memStoreSizing);
1404    store.add(createCell(r1, qf2, ts + 3, seqId + 3, value1), memStoreSizing);
1405    store.add(createCell(r1, qf3, ts + 3, seqId + 3, value1), memStoreSizing);
1406    List<Cell> myList = new MyList<>(hook);
1407    Scan scan = new Scan().withStartRow(r1).setFilter(filter);
1408    try (InternalScanner scanner = (InternalScanner) store.getScanner(scan, null, seqId + 3)) {
1409      // r1
1410      scanner.next(myList);
1411      assertEquals(expectedSize, myList.size());
1412      for (Cell c : myList) {
1413        byte[] actualValue = CellUtil.cloneValue(c);
1414        assertTrue(Bytes.equals(actualValue, value1), "expected:" + Bytes.toStringBinary(value1)
1415          + ", actual:" + Bytes.toStringBinary(actualValue));
1416      }
1417      List<Cell> normalList = new ArrayList<>(3);
1418      // r2
1419      scanner.next(normalList);
1420      assertEquals(3, normalList.size());
1421      for (Cell c : normalList) {
1422        byte[] actualValue = CellUtil.cloneValue(c);
1423        assertTrue(Bytes.equals(actualValue, value2), "expected:" + Bytes.toStringBinary(value2)
1424          + ", actual:" + Bytes.toStringBinary(actualValue));
1425      }
1426    }
1427  }
1428
1429  @Test
1430  public void testFlushBeforeCompletingScanWithDeleteCell() throws IOException {
1431    final Configuration conf = HBaseConfiguration.create();
1432
1433    byte[] r1 = Bytes.toBytes("row1");
1434    byte[] r2 = Bytes.toBytes("row2");
1435
1436    byte[] value1 = Bytes.toBytes("value1");
1437    byte[] value2 = Bytes.toBytes("value2");
1438
1439    final MemStoreSizing memStoreSizing = new NonThreadSafeMemStoreSizing();
1440    final long ts = EnvironmentEdgeManager.currentTime();
1441    final long seqId = 100;
1442
1443    init(name, conf, TableDescriptorBuilder.newBuilder(TableName.valueOf(table)),
1444      ColumnFamilyDescriptorBuilder.newBuilder(family).setMaxVersions(1).build(),
1445      new MyStoreHook() {
1446        @Override
1447        long getSmallestReadPoint(HStore store) {
1448          return seqId + 3;
1449        }
1450      });
1451
1452    store.add(createCell(r1, qf1, ts + 1, seqId + 1, value2), memStoreSizing);
1453    store.add(createCell(r1, qf2, ts + 1, seqId + 1, value2), memStoreSizing);
1454    store.add(createCell(r1, qf3, ts + 1, seqId + 1, value2), memStoreSizing);
1455
1456    store.add(createDeleteCell(r1, qf1, ts + 2, seqId + 2), memStoreSizing);
1457    store.add(createDeleteCell(r1, qf2, ts + 2, seqId + 2), memStoreSizing);
1458    store.add(createDeleteCell(r1, qf3, ts + 2, seqId + 2), memStoreSizing);
1459
1460    store.add(createCell(r2, qf1, ts + 3, seqId + 3, value1), memStoreSizing);
1461    store.add(createCell(r2, qf2, ts + 3, seqId + 3, value1), memStoreSizing);
1462    store.add(createCell(r2, qf3, ts + 3, seqId + 3, value1), memStoreSizing);
1463
1464    Scan scan = new Scan().withStartRow(r1);
1465
1466    try (final InternalScanner scanner =
1467      new StoreScanner(store, store.getScanInfo(), scan, null, seqId + 3) {
1468        @Override
1469        protected KeyValueHeap newKVHeap(List<? extends KeyValueScanner> scanners,
1470          CellComparator comparator) throws IOException {
1471          return new MyKeyValueHeap(scanners, comparator, recordBlockSizeCallCount -> {
1472            if (recordBlockSizeCallCount == 6) {
1473              try {
1474                flushStore(store, id++);
1475              } catch (IOException e) {
1476                throw new RuntimeException(e);
1477              }
1478            }
1479          });
1480        }
1481      }) {
1482      List<Cell> cellResult = new ArrayList<>();
1483
1484      scanner.next(cellResult);
1485      assertEquals(0, cellResult.size());
1486
1487      cellResult.clear();
1488
1489      scanner.next(cellResult);
1490      assertEquals(3, cellResult.size());
1491      for (Cell cell : cellResult) {
1492        assertArrayEquals(r2, CellUtil.cloneRow(cell));
1493      }
1494    }
1495  }
1496
1497  @Test
1498  public void testCreateScannerAndSnapshotConcurrently() throws IOException, InterruptedException {
1499    Configuration conf = HBaseConfiguration.create();
1500    conf.set(HStore.MEMSTORE_CLASS_NAME, MyCompactingMemStore.class.getName());
1501    init(name, conf, ColumnFamilyDescriptorBuilder.newBuilder(family)
1502      .setInMemoryCompaction(MemoryCompactionPolicy.BASIC).build());
1503    byte[] value = Bytes.toBytes("value");
1504    MemStoreSizing memStoreSizing = new NonThreadSafeMemStoreSizing();
1505    long ts = EnvironmentEdgeManager.currentTime();
1506    long seqId = 100;
1507    // older data whihc shouldn't be "seen" by client
1508    store.add(createCell(qf1, ts, seqId, value), memStoreSizing);
1509    store.add(createCell(qf2, ts, seqId, value), memStoreSizing);
1510    store.add(createCell(qf3, ts, seqId, value), memStoreSizing);
1511    TreeSet<byte[]> quals = new TreeSet<>(Bytes.BYTES_COMPARATOR);
1512    quals.add(qf1);
1513    quals.add(qf2);
1514    quals.add(qf3);
1515    StoreFlushContext storeFlushCtx = store.createFlushContext(id++, FlushLifeCycleTracker.DUMMY);
1516    MyCompactingMemStore.START_TEST.set(true);
1517    Runnable flush = () -> {
1518      // this is blocked until we create first scanner from pipeline and snapshot -- phase (1/5)
1519      // recreate the active memstore -- phase (4/5)
1520      storeFlushCtx.prepare();
1521    };
1522    ExecutorService service = Executors.newSingleThreadExecutor();
1523    service.execute(flush);
1524    // we get scanner from pipeline and snapshot but they are empty. -- phase (2/5)
1525    // this is blocked until we recreate the active memstore -- phase (3/5)
1526    // we get scanner from active memstore but it is empty -- phase (5/5)
1527    InternalScanner scanner =
1528      (InternalScanner) store.getScanner(new Scan(new Get(row)), quals, seqId + 1);
1529    service.shutdown();
1530    service.awaitTermination(20, TimeUnit.SECONDS);
1531    try {
1532      try {
1533        List<Cell> results = new ArrayList<>();
1534        scanner.next(results);
1535        assertEquals(3, results.size());
1536        for (Cell c : results) {
1537          byte[] actualValue = CellUtil.cloneValue(c);
1538          assertTrue(Bytes.equals(actualValue, value), "expected:" + Bytes.toStringBinary(value)
1539            + ", actual:" + Bytes.toStringBinary(actualValue));
1540        }
1541      } finally {
1542        scanner.close();
1543      }
1544    } finally {
1545      MyCompactingMemStore.START_TEST.set(false);
1546      storeFlushCtx.flushCache(Mockito.mock(MonitoredTask.class));
1547      storeFlushCtx.commit(Mockito.mock(MonitoredTask.class));
1548    }
1549  }
1550
1551  @Test
1552  public void testScanWithDoubleFlush() throws IOException {
1553    Configuration conf = HBaseConfiguration.create();
1554    // Initialize region
1555    MyStore myStore = initMyStore(name, conf, new MyStoreHook() {
1556      @Override
1557      public void getScanners(MyStore store) throws IOException {
1558        final long tmpId = id++;
1559        ExecutorService s = Executors.newSingleThreadExecutor();
1560        s.execute(() -> {
1561          try {
1562            // flush the store before storescanner updates the scanners from store.
1563            // The current data will be flushed into files, and the memstore will
1564            // be clear.
1565            // -- phase (4/4)
1566            flushStore(store, tmpId);
1567          } catch (IOException ex) {
1568            throw new RuntimeException(ex);
1569          }
1570        });
1571        s.shutdown();
1572        try {
1573          // wait for the flush, the thread will be blocked in HStore#notifyChangedReadersObservers.
1574          s.awaitTermination(3, TimeUnit.SECONDS);
1575        } catch (InterruptedException ex) {
1576        }
1577      }
1578    });
1579    byte[] oldValue = Bytes.toBytes("oldValue");
1580    byte[] currentValue = Bytes.toBytes("currentValue");
1581    MemStoreSizing memStoreSizing = new NonThreadSafeMemStoreSizing();
1582    long ts = EnvironmentEdgeManager.currentTime();
1583    long seqId = 100;
1584    // older data whihc shouldn't be "seen" by client
1585    myStore.add(createCell(qf1, ts, seqId, oldValue), memStoreSizing);
1586    myStore.add(createCell(qf2, ts, seqId, oldValue), memStoreSizing);
1587    myStore.add(createCell(qf3, ts, seqId, oldValue), memStoreSizing);
1588    long snapshotId = id++;
1589    // push older data into snapshot -- phase (1/4)
1590    StoreFlushContext storeFlushCtx =
1591      store.createFlushContext(snapshotId, FlushLifeCycleTracker.DUMMY);
1592    storeFlushCtx.prepare();
1593
1594    // insert current data into active -- phase (2/4)
1595    myStore.add(createCell(qf1, ts + 1, seqId + 1, currentValue), memStoreSizing);
1596    myStore.add(createCell(qf2, ts + 1, seqId + 1, currentValue), memStoreSizing);
1597    myStore.add(createCell(qf3, ts + 1, seqId + 1, currentValue), memStoreSizing);
1598    TreeSet<byte[]> quals = new TreeSet<>(Bytes.BYTES_COMPARATOR);
1599    quals.add(qf1);
1600    quals.add(qf2);
1601    quals.add(qf3);
1602    try (InternalScanner scanner =
1603      (InternalScanner) myStore.getScanner(new Scan(new Get(row)), quals, seqId + 1)) {
1604      // complete the flush -- phase (3/4)
1605      storeFlushCtx.flushCache(Mockito.mock(MonitoredTask.class));
1606      storeFlushCtx.commit(Mockito.mock(MonitoredTask.class));
1607
1608      List<Cell> results = new ArrayList<>();
1609      scanner.next(results);
1610      assertEquals(3, results.size());
1611      for (Cell c : results) {
1612        byte[] actualValue = CellUtil.cloneValue(c);
1613        assertTrue(Bytes.equals(actualValue, currentValue), "expected:"
1614          + Bytes.toStringBinary(currentValue) + ", actual:" + Bytes.toStringBinary(actualValue));
1615      }
1616    }
1617  }
1618
1619  /**
1620   * This test is for HBASE-27519, when the {@link StoreScanner} is scanning,the Flush and the
1621   * Compaction execute concurrently and theCcompaction compact and archive the flushed
1622   * {@link HStoreFile} which is used by {@link StoreScanner#updateReaders}.Before
1623   * HBASE-27519,{@link StoreScanner.updateReaders} would throw {@link FileNotFoundException}.
1624   */
1625  @Test
1626  public void testStoreScannerUpdateReadersWhenFlushAndCompactConcurrently() throws IOException {
1627    Configuration conf = HBaseConfiguration.create();
1628    conf.setBoolean(WALFactory.WAL_ENABLED, false);
1629    conf.set(DEFAULT_COMPACTION_POLICY_CLASS_KEY, EverythingPolicy.class.getName());
1630    byte[] r0 = Bytes.toBytes("row0");
1631    byte[] r1 = Bytes.toBytes("row1");
1632    final CyclicBarrier cyclicBarrier = new CyclicBarrier(2);
1633    final AtomicBoolean shouldWaitRef = new AtomicBoolean(false);
1634    // Initialize region
1635    final MyStore myStore = initMyStore(name, conf, new MyStoreHook() {
1636      @Override
1637      public void getScanners(MyStore store) throws IOException {
1638        try {
1639          // Here this method is called by StoreScanner.updateReaders which is invoked by the
1640          // following TestHStore.flushStore
1641          if (shouldWaitRef.get()) {
1642            // wait the following compaction Task start
1643            cyclicBarrier.await();
1644            // wait the following HStore.closeAndArchiveCompactedFiles end.
1645            cyclicBarrier.await();
1646          }
1647        } catch (BrokenBarrierException | InterruptedException e) {
1648          throw new RuntimeException(e);
1649        }
1650      }
1651    });
1652
1653    final AtomicReference<Throwable> compactionExceptionRef = new AtomicReference<Throwable>(null);
1654    Runnable compactionTask = () -> {
1655      try {
1656        // Only when the StoreScanner.updateReaders invoked by TestHStore.flushStore prepares for
1657        // entering the MyStore.getScanners, compactionTask could start.
1658        cyclicBarrier.await();
1659        region.compactStore(family, new NoLimitThroughputController());
1660        myStore.closeAndArchiveCompactedFiles();
1661        // Notify StoreScanner.updateReaders could enter MyStore.getScanners.
1662        cyclicBarrier.await();
1663      } catch (Throwable e) {
1664        compactionExceptionRef.set(e);
1665      }
1666    };
1667
1668    long ts = EnvironmentEdgeManager.currentTime();
1669    long seqId = 100;
1670    byte[] value = Bytes.toBytes("value");
1671    // older data whihc shouldn't be "seen" by client
1672    myStore.add(createCell(r0, qf1, ts, seqId, value), null);
1673    flushStore(myStore, id++);
1674    myStore.add(createCell(r0, qf2, ts, seqId, value), null);
1675    flushStore(myStore, id++);
1676    myStore.add(createCell(r0, qf3, ts, seqId, value), null);
1677    TreeSet<byte[]> quals = new TreeSet<>(Bytes.BYTES_COMPARATOR);
1678    quals.add(qf1);
1679    quals.add(qf2);
1680    quals.add(qf3);
1681
1682    myStore.add(createCell(r1, qf1, ts, seqId, value), null);
1683    myStore.add(createCell(r1, qf2, ts, seqId, value), null);
1684    myStore.add(createCell(r1, qf3, ts, seqId, value), null);
1685
1686    Thread.currentThread()
1687      .setName("testStoreScannerUpdateReadersWhenFlushAndCompactConcurrently thread");
1688    Scan scan = new Scan();
1689    scan.withStartRow(r0, true);
1690    try (InternalScanner scanner = (InternalScanner) myStore.getScanner(scan, quals, seqId)) {
1691      List<Cell> results = new MyList<>(size -> {
1692        switch (size) {
1693          case 1:
1694            shouldWaitRef.set(true);
1695            Thread thread = new Thread(compactionTask);
1696            thread.setName("MyCompacting Thread.");
1697            thread.start();
1698            try {
1699              flushStore(myStore, id++);
1700              thread.join();
1701            } catch (IOException | InterruptedException e) {
1702              throw new RuntimeException(e);
1703            }
1704            shouldWaitRef.set(false);
1705            break;
1706          default:
1707            break;
1708        }
1709      });
1710      // Before HBASE-27519, here would throw java.io.FileNotFoundException because the storeFile
1711      // which used by StoreScanner.updateReaders is deleted by compactionTask.
1712      scanner.next(results);
1713      // The results is r0 row cells.
1714      assertEquals(3, results.size());
1715      assertTrue(compactionExceptionRef.get() == null);
1716    }
1717  }
1718
1719  @Test
1720  public void testReclaimChunkWhenScaning() throws IOException {
1721    init("testReclaimChunkWhenScaning");
1722    long ts = EnvironmentEdgeManager.currentTime();
1723    long seqId = 100;
1724    byte[] value = Bytes.toBytes("value");
1725    // older data whihc shouldn't be "seen" by client
1726    store.add(createCell(qf1, ts, seqId, value), null);
1727    store.add(createCell(qf2, ts, seqId, value), null);
1728    store.add(createCell(qf3, ts, seqId, value), null);
1729    TreeSet<byte[]> quals = new TreeSet<>(Bytes.BYTES_COMPARATOR);
1730    quals.add(qf1);
1731    quals.add(qf2);
1732    quals.add(qf3);
1733    try (InternalScanner scanner =
1734      (InternalScanner) store.getScanner(new Scan(new Get(row)), quals, seqId)) {
1735      List<Cell> results = new MyList<>(size -> {
1736        switch (size) {
1737          // 1) we get the first cell (qf1)
1738          // 2) flush the data to have StoreScanner update inner scanners
1739          // 3) the chunk will be reclaimed after updaing
1740          case 1:
1741            try {
1742              flushStore(store, id++);
1743            } catch (IOException e) {
1744              throw new RuntimeException(e);
1745            }
1746            break;
1747          // 1) we get the second cell (qf2)
1748          // 2) add some cell to fill some byte into the chunk (we have only one chunk)
1749          case 2:
1750            try {
1751              byte[] newValue = Bytes.toBytes("newValue");
1752              // older data whihc shouldn't be "seen" by client
1753              store.add(createCell(qf1, ts + 1, seqId + 1, newValue), null);
1754              store.add(createCell(qf2, ts + 1, seqId + 1, newValue), null);
1755              store.add(createCell(qf3, ts + 1, seqId + 1, newValue), null);
1756            } catch (IOException e) {
1757              throw new RuntimeException(e);
1758            }
1759            break;
1760          default:
1761            break;
1762        }
1763      });
1764      scanner.next(results);
1765      assertEquals(3, results.size());
1766      for (Cell c : results) {
1767        byte[] actualValue = CellUtil.cloneValue(c);
1768        assertTrue(Bytes.equals(actualValue, value), "expected:" + Bytes.toStringBinary(value)
1769          + ", actual:" + Bytes.toStringBinary(actualValue));
1770      }
1771    }
1772  }
1773
1774  /**
1775   * If there are two running InMemoryFlushRunnable, the later InMemoryFlushRunnable may change the
1776   * versionedList. And the first InMemoryFlushRunnable will use the chagned versionedList to remove
1777   * the corresponding segments. In short, there will be some segements which isn't in merge are
1778   * removed.
1779   */
1780  @Test
1781  public void testRunDoubleMemStoreCompactors() throws IOException, InterruptedException {
1782    int flushSize = 500;
1783    Configuration conf = HBaseConfiguration.create();
1784    conf.set(HStore.MEMSTORE_CLASS_NAME, MyCompactingMemStoreWithCustomCompactor.class.getName());
1785    conf.setDouble(CompactingMemStore.IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, 0.25);
1786    MyCompactingMemStoreWithCustomCompactor.RUNNER_COUNT.set(0);
1787    conf.set(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, String.valueOf(flushSize));
1788    // Set the lower threshold to invoke the "MERGE" policy
1789    conf.set(MemStoreCompactionStrategy.COMPACTING_MEMSTORE_THRESHOLD_KEY, String.valueOf(0));
1790    init(name, conf, ColumnFamilyDescriptorBuilder.newBuilder(family)
1791      .setInMemoryCompaction(MemoryCompactionPolicy.BASIC).build());
1792    byte[] value = Bytes.toBytes("thisisavarylargevalue");
1793    MemStoreSizing memStoreSizing = new NonThreadSafeMemStoreSizing();
1794    long ts = EnvironmentEdgeManager.currentTime();
1795    long seqId = 100;
1796    // older data whihc shouldn't be "seen" by client
1797    store.add(createCell(qf1, ts, seqId, value), memStoreSizing);
1798    store.add(createCell(qf2, ts, seqId, value), memStoreSizing);
1799    store.add(createCell(qf3, ts, seqId, value), memStoreSizing);
1800    assertEquals(1, MyCompactingMemStoreWithCustomCompactor.RUNNER_COUNT.get());
1801    StoreFlushContext storeFlushCtx = store.createFlushContext(id++, FlushLifeCycleTracker.DUMMY);
1802    storeFlushCtx.prepare();
1803    // This shouldn't invoke another in-memory flush because the first compactor thread
1804    // hasn't accomplished the in-memory compaction.
1805    store.add(createCell(qf1, ts + 1, seqId + 1, value), memStoreSizing);
1806    store.add(createCell(qf1, ts + 1, seqId + 1, value), memStoreSizing);
1807    store.add(createCell(qf1, ts + 1, seqId + 1, value), memStoreSizing);
1808    assertEquals(1, MyCompactingMemStoreWithCustomCompactor.RUNNER_COUNT.get());
1809    // okay. Let the compaction be completed
1810    MyMemStoreCompactor.START_COMPACTOR_LATCH.countDown();
1811    CompactingMemStore mem = (CompactingMemStore) ((HStore) store).memstore;
1812    while (mem.isMemStoreFlushingInMemory()) {
1813      TimeUnit.SECONDS.sleep(1);
1814    }
1815    // This should invoke another in-memory flush.
1816    store.add(createCell(qf1, ts + 2, seqId + 2, value), memStoreSizing);
1817    store.add(createCell(qf1, ts + 2, seqId + 2, value), memStoreSizing);
1818    store.add(createCell(qf1, ts + 2, seqId + 2, value), memStoreSizing);
1819    assertEquals(2, MyCompactingMemStoreWithCustomCompactor.RUNNER_COUNT.get());
1820    conf.set(HConstants.HREGION_MEMSTORE_FLUSH_SIZE,
1821      String.valueOf(TableDescriptorBuilder.DEFAULT_MEMSTORE_FLUSH_SIZE));
1822    storeFlushCtx.flushCache(Mockito.mock(MonitoredTask.class));
1823    storeFlushCtx.commit(Mockito.mock(MonitoredTask.class));
1824  }
1825
1826  @Test
1827  public void testAge() throws IOException {
1828    long currentTime = EnvironmentEdgeManager.currentTime();
1829    ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
1830    edge.setValue(currentTime);
1831    EnvironmentEdgeManager.injectEdge(edge);
1832    Configuration conf = TEST_UTIL.getConfiguration();
1833    ColumnFamilyDescriptor hcd = ColumnFamilyDescriptorBuilder.of(family);
1834    initHRegion(name, conf, TableDescriptorBuilder.newBuilder(TableName.valueOf(table)), hcd, null,
1835      false);
1836    HStore store = new HStore(region, hcd, conf, false) {
1837
1838      @Override
1839      protected StoreEngine<?, ?, ?, ?> createStoreEngine(HStore store, Configuration conf,
1840        CellComparator kvComparator) throws IOException {
1841        List<HStoreFile> storefiles =
1842          Arrays.asList(mockStoreFile(currentTime - 10), mockStoreFile(currentTime - 100),
1843            mockStoreFile(currentTime - 1000), mockStoreFile(currentTime - 10000));
1844        StoreFileManager sfm = mock(StoreFileManager.class);
1845        when(sfm.getStoreFiles()).thenReturn(storefiles);
1846        StoreEngine<?, ?, ?, ?> storeEngine = mock(StoreEngine.class);
1847        when(storeEngine.getStoreFileManager()).thenReturn(sfm);
1848        return storeEngine;
1849      }
1850    };
1851    assertEquals(10L, store.getMinStoreFileAge().getAsLong());
1852    assertEquals(10000L, store.getMaxStoreFileAge().getAsLong());
1853    assertEquals((10 + 100 + 1000 + 10000) / 4.0, store.getAvgStoreFileAge().getAsDouble(), 1E-4);
1854  }
1855
1856  private HStoreFile mockStoreFile(long createdTime) {
1857    StoreFileInfo info = mock(StoreFileInfo.class);
1858    when(info.getCreatedTimestamp()).thenReturn(createdTime);
1859    HStoreFile sf = mock(HStoreFile.class);
1860    when(sf.getReader()).thenReturn(mock(StoreFileReader.class));
1861    when(sf.isHFile()).thenReturn(true);
1862    when(sf.getFileInfo()).thenReturn(info);
1863    return sf;
1864  }
1865
1866  private MyStore initMyStore(String methodName, Configuration conf, MyStoreHook hook)
1867    throws IOException {
1868    return (MyStore) init(methodName, conf,
1869      TableDescriptorBuilder.newBuilder(TableName.valueOf(table)),
1870      ColumnFamilyDescriptorBuilder.newBuilder(family).setMaxVersions(5).build(), hook);
1871  }
1872
1873  private static class MyStore extends HStore {
1874    private final MyStoreHook hook;
1875
1876    MyStore(final HRegion region, final ColumnFamilyDescriptor family,
1877      final Configuration confParam, MyStoreHook hook, boolean switchToPread) throws IOException {
1878      super(region, family, confParam, false);
1879      this.hook = hook;
1880    }
1881
1882    @Override
1883    public List<KeyValueScanner> getScanners(List<HStoreFile> files, boolean cacheBlocks,
1884      boolean usePread, boolean isCompaction, ScanQueryMatcher matcher, byte[] startRow,
1885      boolean includeStartRow, byte[] stopRow, boolean includeStopRow, long readPt,
1886      boolean includeMemstoreScanner, boolean onlyLatestVersion) throws IOException {
1887      hook.getScanners(this);
1888      return super.getScanners(files, cacheBlocks, usePread, isCompaction, matcher, startRow, true,
1889        stopRow, false, readPt, includeMemstoreScanner, onlyLatestVersion);
1890    }
1891
1892    @Override
1893    public long getSmallestReadPoint() {
1894      return hook.getSmallestReadPoint(this);
1895    }
1896  }
1897
1898  private abstract static class MyStoreHook {
1899
1900    void getScanners(MyStore store) throws IOException {
1901    }
1902
1903    long getSmallestReadPoint(HStore store) {
1904      return store.getHRegion().getSmallestReadPoint();
1905    }
1906  }
1907
1908  @Test
1909  public void testSwitchingPreadtoStreamParallelyWithCompactionDischarger() throws Exception {
1910    Configuration conf = HBaseConfiguration.create();
1911    conf.set("hbase.hstore.engine.class", DummyStoreEngine.class.getName());
1912    conf.setLong(StoreScanner.STORESCANNER_PREAD_MAX_BYTES, 0);
1913    // Set the lower threshold to invoke the "MERGE" policy
1914    MyStore store = initMyStore(name, conf, new MyStoreHook() {
1915    });
1916    MemStoreSizing memStoreSizing = new NonThreadSafeMemStoreSizing();
1917    long ts = EnvironmentEdgeManager.currentTime();
1918    long seqID = 1L;
1919    // Add some data to the region and do some flushes
1920    for (int i = 1; i < 10; i++) {
1921      store.add(createCell(Bytes.toBytes("row" + i), qf1, ts, seqID++, Bytes.toBytes("")),
1922        memStoreSizing);
1923    }
1924    // flush them
1925    flushStore(store, seqID);
1926    for (int i = 11; i < 20; i++) {
1927      store.add(createCell(Bytes.toBytes("row" + i), qf1, ts, seqID++, Bytes.toBytes("")),
1928        memStoreSizing);
1929    }
1930    // flush them
1931    flushStore(store, seqID);
1932    for (int i = 21; i < 30; i++) {
1933      store.add(createCell(Bytes.toBytes("row" + i), qf1, ts, seqID++, Bytes.toBytes("")),
1934        memStoreSizing);
1935    }
1936    // flush them
1937    flushStore(store, seqID);
1938
1939    assertEquals(3, store.getStorefilesCount());
1940    Scan scan = new Scan();
1941    scan.addFamily(family);
1942    Collection<HStoreFile> storefiles2 = store.getStorefiles();
1943    ArrayList<HStoreFile> actualStorefiles = Lists.newArrayList(storefiles2);
1944    StoreScanner storeScanner =
1945      (StoreScanner) store.getScanner(scan, scan.getFamilyMap().get(family), Long.MAX_VALUE);
1946    // get the current heap
1947    KeyValueHeap heap = storeScanner.heap;
1948    // create more store files
1949    for (int i = 31; i < 40; i++) {
1950      store.add(createCell(Bytes.toBytes("row" + i), qf1, ts, seqID++, Bytes.toBytes("")),
1951        memStoreSizing);
1952    }
1953    // flush them
1954    flushStore(store, seqID);
1955
1956    for (int i = 41; i < 50; i++) {
1957      store.add(createCell(Bytes.toBytes("row" + i), qf1, ts, seqID++, Bytes.toBytes("")),
1958        memStoreSizing);
1959    }
1960    // flush them
1961    flushStore(store, seqID);
1962    storefiles2 = store.getStorefiles();
1963    ArrayList<HStoreFile> actualStorefiles1 = Lists.newArrayList(storefiles2);
1964    actualStorefiles1.removeAll(actualStorefiles);
1965    // Do compaction
1966    MyThread thread = new MyThread(storeScanner);
1967    thread.start();
1968    store.replaceStoreFiles(actualStorefiles, actualStorefiles1, false);
1969    thread.join();
1970    KeyValueHeap heap2 = thread.getHeap();
1971    assertFalse(heap.equals(heap2));
1972  }
1973
1974  @Test
1975  public void testMaxPreadBytesConfiguredToBeLessThanZero() throws Exception {
1976    Configuration conf = HBaseConfiguration.create();
1977    conf.set("hbase.hstore.engine.class", DummyStoreEngine.class.getName());
1978    // Set 'hbase.storescanner.pread.max.bytes' < 0, so that StoreScanner will be a STREAM type.
1979    conf.setLong(StoreScanner.STORESCANNER_PREAD_MAX_BYTES, -1);
1980    MyStore store = initMyStore(name, conf, new MyStoreHook() {
1981    });
1982    Scan scan = new Scan();
1983    scan.addFamily(family);
1984    // ReadType on Scan is still DEFAULT only.
1985    assertEquals(ReadType.DEFAULT, scan.getReadType());
1986    StoreScanner storeScanner =
1987      (StoreScanner) store.getScanner(scan, scan.getFamilyMap().get(family), Long.MAX_VALUE);
1988    assertFalse(storeScanner.isScanUsePread());
1989  }
1990
1991  @Test
1992  public void testInMemoryCompactionTypeWithLowerCase() throws IOException, InterruptedException {
1993    Configuration conf = HBaseConfiguration.create();
1994    conf.set("hbase.systemtables.compacting.memstore.type", "eager");
1995    init(name, conf,
1996      TableDescriptorBuilder.newBuilder(
1997        TableName.valueOf(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME, "meta".getBytes())),
1998      ColumnFamilyDescriptorBuilder.newBuilder(family)
1999        .setInMemoryCompaction(MemoryCompactionPolicy.NONE).build());
2000    assertTrue(((MemStoreCompactor) ((CompactingMemStore) store.memstore).compactor).toString()
2001      .startsWith("eager".toUpperCase()));
2002  }
2003
2004  @Test
2005  public void testSpaceQuotaChangeAfterReplacement() throws IOException {
2006    final TableName tn = TableName.valueOf(name);
2007    init(name);
2008
2009    RegionSizeStoreImpl sizeStore = new RegionSizeStoreImpl();
2010
2011    HStoreFile sf1 = mockStoreFileWithLength(1024L);
2012    HStoreFile sf2 = mockStoreFileWithLength(2048L);
2013    HStoreFile sf3 = mockStoreFileWithLength(4096L);
2014    HStoreFile sf4 = mockStoreFileWithLength(8192L);
2015
2016    RegionInfo regionInfo = RegionInfoBuilder.newBuilder(tn).setStartKey(Bytes.toBytes("a"))
2017      .setEndKey(Bytes.toBytes("b")).build();
2018
2019    // Compacting two files down to one, reducing size
2020    sizeStore.put(regionInfo, 1024L + 4096L);
2021    store.updateSpaceQuotaAfterFileReplacement(sizeStore, regionInfo, Arrays.asList(sf1, sf3),
2022      Arrays.asList(sf2));
2023
2024    assertEquals(2048L, sizeStore.getRegionSize(regionInfo).getSize());
2025
2026    // The same file length in and out should have no change
2027    store.updateSpaceQuotaAfterFileReplacement(sizeStore, regionInfo, Arrays.asList(sf2),
2028      Arrays.asList(sf2));
2029
2030    assertEquals(2048L, sizeStore.getRegionSize(regionInfo).getSize());
2031
2032    // Increase the total size used
2033    store.updateSpaceQuotaAfterFileReplacement(sizeStore, regionInfo, Arrays.asList(sf2),
2034      Arrays.asList(sf3));
2035
2036    assertEquals(4096L, sizeStore.getRegionSize(regionInfo).getSize());
2037
2038    RegionInfo regionInfo2 = RegionInfoBuilder.newBuilder(tn).setStartKey(Bytes.toBytes("b"))
2039      .setEndKey(Bytes.toBytes("c")).build();
2040    store.updateSpaceQuotaAfterFileReplacement(sizeStore, regionInfo2, null, Arrays.asList(sf4));
2041
2042    assertEquals(8192L, sizeStore.getRegionSize(regionInfo2).getSize());
2043  }
2044
2045  @Test
2046  public void testHFileContextSetWithCFAndTable() throws Exception {
2047    init(name);
2048    StoreFileWriter writer = store.getStoreEngine()
2049      .createWriter(CreateStoreFileWriterParams.create().maxKeyCount(10000L)
2050        .compression(Compression.Algorithm.NONE).isCompaction(true).includeMVCCReadpoint(true)
2051        .includesTag(false).shouldDropBehind(true));
2052    HFileContext hFileContext = writer.getLiveFileWriter().getFileContext();
2053    assertArrayEquals(family, hFileContext.getColumnFamily());
2054    assertArrayEquals(table, hFileContext.getTableName());
2055  }
2056
2057  // This test is for HBASE-26026, HBase Write be stuck when active segment has no cell
2058  // but its dataSize exceeds inmemoryFlushSize
2059  @Test
2060  public void testCompactingMemStoreNoCellButDataSizeExceedsInmemoryFlushSize()
2061    throws IOException, InterruptedException {
2062    Configuration conf = HBaseConfiguration.create();
2063
2064    byte[] smallValue = new byte[3];
2065    byte[] largeValue = new byte[9];
2066    final long timestamp = EnvironmentEdgeManager.currentTime();
2067    final long seqId = 100;
2068    final ExtendedCell smallCell = createCell(qf1, timestamp, seqId, smallValue);
2069    final ExtendedCell largeCell = createCell(qf2, timestamp, seqId, largeValue);
2070    int smallCellByteSize = MutableSegment.getCellLength(smallCell);
2071    int largeCellByteSize = MutableSegment.getCellLength(largeCell);
2072    int flushByteSize = smallCellByteSize + largeCellByteSize - 2;
2073
2074    // set CompactingMemStore.inmemoryFlushSize to flushByteSize.
2075    conf.set(HStore.MEMSTORE_CLASS_NAME, MyCompactingMemStore2.class.getName());
2076    conf.setDouble(CompactingMemStore.IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, 0.005);
2077    conf.set(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, String.valueOf(flushByteSize * 200));
2078
2079    init(name, conf, ColumnFamilyDescriptorBuilder.newBuilder(family)
2080      .setInMemoryCompaction(MemoryCompactionPolicy.BASIC).build());
2081
2082    MyCompactingMemStore2 myCompactingMemStore = ((MyCompactingMemStore2) store.memstore);
2083    assertTrue((int) (myCompactingMemStore.getInmemoryFlushSize()) == flushByteSize);
2084    myCompactingMemStore.smallCellPreUpdateCounter.set(0);
2085    myCompactingMemStore.largeCellPreUpdateCounter.set(0);
2086
2087    final AtomicReference<Throwable> exceptionRef = new AtomicReference<Throwable>();
2088    Thread smallCellThread = new Thread(() -> {
2089      try {
2090        store.add(smallCell, new NonThreadSafeMemStoreSizing());
2091      } catch (Throwable exception) {
2092        exceptionRef.set(exception);
2093      }
2094    });
2095    smallCellThread.setName(MyCompactingMemStore2.SMALL_CELL_THREAD_NAME);
2096    smallCellThread.start();
2097
2098    String oldThreadName = Thread.currentThread().getName();
2099    try {
2100      /**
2101       * 1.smallCellThread enters CompactingMemStore.checkAndAddToActiveSize first, then
2102       * largeCellThread enters CompactingMemStore.checkAndAddToActiveSize, and then largeCellThread
2103       * invokes flushInMemory.
2104       * <p/>
2105       * 2. After largeCellThread finished CompactingMemStore.flushInMemory method, smallCellThread
2106       * can add cell to currentActive . That is to say when largeCellThread called flushInMemory
2107       * method, CompactingMemStore.active has no cell.
2108       */
2109      Thread.currentThread().setName(MyCompactingMemStore2.LARGE_CELL_THREAD_NAME);
2110      store.add(largeCell, new NonThreadSafeMemStoreSizing());
2111      smallCellThread.join();
2112
2113      for (int i = 0; i < 100; i++) {
2114        long currentTimestamp = timestamp + 100 + i;
2115        ExtendedCell cell = createCell(qf2, currentTimestamp, seqId, largeValue);
2116        store.add(cell, new NonThreadSafeMemStoreSizing());
2117      }
2118    } finally {
2119      Thread.currentThread().setName(oldThreadName);
2120    }
2121
2122    assertTrue(exceptionRef.get() == null);
2123
2124  }
2125
2126  // This test is for HBASE-26210, HBase Write be stuck when there is cell which size exceeds
2127  // InmemoryFlushSize
2128  @Test
2129  @Timeout(value = 60000, unit = TimeUnit.MILLISECONDS)
2130  public void testCompactingMemStoreCellExceedInmemoryFlushSize() throws Exception {
2131    Configuration conf = HBaseConfiguration.create();
2132    conf.set(HStore.MEMSTORE_CLASS_NAME, MyCompactingMemStore6.class.getName());
2133
2134    init(name, conf, ColumnFamilyDescriptorBuilder.newBuilder(family)
2135      .setInMemoryCompaction(MemoryCompactionPolicy.BASIC).build());
2136
2137    MyCompactingMemStore6 myCompactingMemStore = ((MyCompactingMemStore6) store.memstore);
2138
2139    int size = (int) (myCompactingMemStore.getInmemoryFlushSize());
2140    byte[] value = new byte[size + 1];
2141
2142    MemStoreSizing memStoreSizing = new NonThreadSafeMemStoreSizing();
2143    long timestamp = EnvironmentEdgeManager.currentTime();
2144    long seqId = 100;
2145    ExtendedCell cell = createCell(qf1, timestamp, seqId, value);
2146    int cellByteSize = MutableSegment.getCellLength(cell);
2147    store.add(cell, memStoreSizing);
2148    assertTrue(memStoreSizing.getCellsCount() == 1);
2149    assertTrue(memStoreSizing.getDataSize() == cellByteSize);
2150    // Waiting the in memory compaction completed, see HBASE-26438
2151    myCompactingMemStore.inMemoryCompactionEndCyclicBarrier.await();
2152  }
2153
2154  /**
2155   * This test is for HBASE-27464, before this JIRA,when init {@link CellChunkImmutableSegment} for
2156   * 'COMPACT' action, we not force copy to current MSLab. When cell size bigger than
2157   * {@link MemStoreLABImpl#maxAlloc}, cell will stay in previous chunk which will recycle after
2158   * segment replace, and we may read wrong data when these chunk reused by others.
2159   */
2160  @Test
2161  public void testForceCloneOfBigCellForCellChunkImmutableSegment() throws Exception {
2162    Configuration conf = HBaseConfiguration.create();
2163    int maxAllocByteSize = conf.getInt(MemStoreLAB.MAX_ALLOC_KEY, MemStoreLAB.MAX_ALLOC_DEFAULT);
2164
2165    // Construct big cell,which is large than {@link MemStoreLABImpl#maxAlloc}.
2166    byte[] cellValue = new byte[maxAllocByteSize + 1];
2167    final long timestamp = EnvironmentEdgeManager.currentTime();
2168    final long seqId = 100;
2169    final byte[] rowKey1 = Bytes.toBytes("rowKey1");
2170    final ExtendedCell originalCell1 = createCell(rowKey1, qf1, timestamp, seqId, cellValue);
2171    final byte[] rowKey2 = Bytes.toBytes("rowKey2");
2172    final ExtendedCell originalCell2 = createCell(rowKey2, qf1, timestamp, seqId, cellValue);
2173    TreeSet<byte[]> quals = new TreeSet<>(Bytes.BYTES_COMPARATOR);
2174    quals.add(qf1);
2175
2176    int cellByteSize = MutableSegment.getCellLength(originalCell1);
2177    int inMemoryFlushByteSize = cellByteSize - 1;
2178
2179    // set CompactingMemStore.inmemoryFlushSize to flushByteSize.
2180    conf.set(HStore.MEMSTORE_CLASS_NAME, MyCompactingMemStore6.class.getName());
2181    conf.setDouble(CompactingMemStore.IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, 0.005);
2182    conf.set(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, String.valueOf(inMemoryFlushByteSize * 200));
2183    conf.setBoolean(WALFactory.WAL_ENABLED, false);
2184
2185    // Use {@link MemoryCompactionPolicy#EAGER} for always compacting.
2186    init(name, conf, ColumnFamilyDescriptorBuilder.newBuilder(family)
2187      .setInMemoryCompaction(MemoryCompactionPolicy.EAGER).build());
2188
2189    MyCompactingMemStore6 myCompactingMemStore = ((MyCompactingMemStore6) store.memstore);
2190    assertTrue((int) (myCompactingMemStore.getInmemoryFlushSize()) == inMemoryFlushByteSize);
2191
2192    // Data chunk Pool is disabled.
2193    assertTrue(ChunkCreator.getInstance().getMaxCount(ChunkType.DATA_CHUNK) == 0);
2194
2195    MemStoreSizing memStoreSizing = new NonThreadSafeMemStoreSizing();
2196
2197    // First compact
2198    store.add(originalCell1, memStoreSizing);
2199    // Waiting for the first in-memory compaction finished
2200    myCompactingMemStore.inMemoryCompactionEndCyclicBarrier.await();
2201
2202    StoreScanner storeScanner =
2203      (StoreScanner) store.getScanner(new Scan(new Get(rowKey1)), quals, seqId + 1);
2204    SegmentScanner segmentScanner = getTypeKeyValueScanner(storeScanner, SegmentScanner.class);
2205    ExtendedCell resultCell1 = segmentScanner.next();
2206    assertTrue(PrivateCellUtil.equals(resultCell1, originalCell1));
2207    int cell1ChunkId = resultCell1.getChunkId();
2208    assertTrue(cell1ChunkId != ExtendedCell.CELL_NOT_BASED_ON_CHUNK);
2209    assertNull(segmentScanner.next());
2210    segmentScanner.close();
2211    storeScanner.close();
2212    Segment segment = segmentScanner.segment;
2213    assertTrue(segment instanceof CellChunkImmutableSegment);
2214    MemStoreLABImpl memStoreLAB1 = (MemStoreLABImpl) (segmentScanner.segment.getMemStoreLAB());
2215    assertTrue(!memStoreLAB1.isClosed());
2216    assertTrue(!memStoreLAB1.chunks.isEmpty());
2217    assertTrue(!memStoreLAB1.isReclaimed());
2218
2219    // Second compact
2220    store.add(originalCell2, memStoreSizing);
2221    // Waiting for the second in-memory compaction finished
2222    myCompactingMemStore.inMemoryCompactionEndCyclicBarrier.await();
2223
2224    // Before HBASE-27464, here may throw java.lang.IllegalArgumentException: In CellChunkMap, cell
2225    // must be associated with chunk.. We were looking for a cell at index 0.
2226    // The cause for this exception is because the data chunk Pool is disabled,when the data chunks
2227    // are recycled after the second in-memory compaction finished,the
2228    // {@link ChunkCreator.putbackChunks} method does not put the chunks back to the data chunk
2229    // pool,it just removes them from {@link ChunkCreator#chunkIdMap},so in
2230    // {@link CellChunkMap#getCell} we could not get the data chunk by chunkId.
2231    storeScanner = (StoreScanner) store.getScanner(new Scan(new Get(rowKey1)), quals, seqId + 1);
2232    segmentScanner = getTypeKeyValueScanner(storeScanner, SegmentScanner.class);
2233    ExtendedCell newResultCell1 = segmentScanner.next();
2234    assertTrue(newResultCell1 != resultCell1);
2235    assertTrue(PrivateCellUtil.equals(newResultCell1, originalCell1));
2236
2237    ExtendedCell resultCell2 = segmentScanner.next();
2238    assertTrue(PrivateCellUtil.equals(resultCell2, originalCell2));
2239    assertNull(segmentScanner.next());
2240    segmentScanner.close();
2241    storeScanner.close();
2242
2243    segment = segmentScanner.segment;
2244    assertTrue(segment instanceof CellChunkImmutableSegment);
2245    MemStoreLABImpl memStoreLAB2 = (MemStoreLABImpl) (segmentScanner.segment.getMemStoreLAB());
2246    assertTrue(!memStoreLAB2.isClosed());
2247    assertTrue(!memStoreLAB2.chunks.isEmpty());
2248    assertTrue(!memStoreLAB2.isReclaimed());
2249    assertTrue(memStoreLAB1.isClosed());
2250    assertTrue(memStoreLAB1.chunks.isEmpty());
2251    assertTrue(memStoreLAB1.isReclaimed());
2252  }
2253
2254  // This test is for HBASE-26210 also, test write large cell and small cell concurrently when
2255  // InmemoryFlushSize is smaller,equal with and larger than cell size.
2256  @Test
2257  public void testCompactingMemStoreWriteLargeCellAndSmallCellConcurrently()
2258    throws IOException, InterruptedException {
2259    doWriteTestLargeCellAndSmallCellConcurrently(
2260      (smallCellByteSize, largeCellByteSize) -> largeCellByteSize - 1);
2261    doWriteTestLargeCellAndSmallCellConcurrently(
2262      (smallCellByteSize, largeCellByteSize) -> largeCellByteSize);
2263    doWriteTestLargeCellAndSmallCellConcurrently(
2264      (smallCellByteSize, largeCellByteSize) -> smallCellByteSize + largeCellByteSize - 1);
2265    doWriteTestLargeCellAndSmallCellConcurrently(
2266      (smallCellByteSize, largeCellByteSize) -> smallCellByteSize + largeCellByteSize);
2267    doWriteTestLargeCellAndSmallCellConcurrently(
2268      (smallCellByteSize, largeCellByteSize) -> smallCellByteSize + largeCellByteSize + 1);
2269  }
2270
2271  private void doWriteTestLargeCellAndSmallCellConcurrently(IntBinaryOperator getFlushByteSize)
2272    throws IOException, InterruptedException {
2273
2274    Configuration conf = HBaseConfiguration.create();
2275
2276    byte[] smallValue = new byte[3];
2277    byte[] largeValue = new byte[100];
2278    final long timestamp = EnvironmentEdgeManager.currentTime();
2279    final long seqId = 100;
2280    final Cell smallCell = createCell(qf1, timestamp, seqId, smallValue);
2281    final Cell largeCell = createCell(qf2, timestamp, seqId, largeValue);
2282    int smallCellByteSize = MutableSegment.getCellLength(smallCell);
2283    int largeCellByteSize = MutableSegment.getCellLength(largeCell);
2284    int flushByteSize = getFlushByteSize.applyAsInt(smallCellByteSize, largeCellByteSize);
2285    boolean flushByteSizeLessThanSmallAndLargeCellSize =
2286      flushByteSize < (smallCellByteSize + largeCellByteSize);
2287
2288    conf.set(HStore.MEMSTORE_CLASS_NAME, MyCompactingMemStore3.class.getName());
2289    conf.setDouble(CompactingMemStore.IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, 0.005);
2290    conf.set(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, String.valueOf(flushByteSize * 200));
2291
2292    init(name, conf, ColumnFamilyDescriptorBuilder.newBuilder(family)
2293      .setInMemoryCompaction(MemoryCompactionPolicy.BASIC).build());
2294
2295    MyCompactingMemStore3 myCompactingMemStore = ((MyCompactingMemStore3) store.memstore);
2296    assertTrue((int) (myCompactingMemStore.getInmemoryFlushSize()) == flushByteSize);
2297    myCompactingMemStore.disableCompaction();
2298    if (flushByteSizeLessThanSmallAndLargeCellSize) {
2299      myCompactingMemStore.flushByteSizeLessThanSmallAndLargeCellSize = true;
2300    } else {
2301      myCompactingMemStore.flushByteSizeLessThanSmallAndLargeCellSize = false;
2302    }
2303
2304    final ThreadSafeMemStoreSizing memStoreSizing = new ThreadSafeMemStoreSizing();
2305    final AtomicLong totalCellByteSize = new AtomicLong(0);
2306    final AtomicReference<Throwable> exceptionRef = new AtomicReference<Throwable>();
2307    Thread smallCellThread = new Thread(() -> {
2308      try {
2309        for (int i = 1; i <= MyCompactingMemStore3.CELL_COUNT; i++) {
2310          long currentTimestamp = timestamp + i;
2311          ExtendedCell cell = createCell(qf1, currentTimestamp, seqId, smallValue);
2312          totalCellByteSize.addAndGet(MutableSegment.getCellLength(cell));
2313          store.add(cell, memStoreSizing);
2314        }
2315      } catch (Throwable exception) {
2316        exceptionRef.set(exception);
2317
2318      }
2319    });
2320    smallCellThread.setName(MyCompactingMemStore3.SMALL_CELL_THREAD_NAME);
2321    smallCellThread.start();
2322
2323    String oldThreadName = Thread.currentThread().getName();
2324    try {
2325      /**
2326       * When flushByteSizeLessThanSmallAndLargeCellSize is true:
2327       * </p>
2328       * 1.smallCellThread enters MyCompactingMemStore3.checkAndAddToActiveSize first, then
2329       * largeCellThread enters MyCompactingMemStore3.checkAndAddToActiveSize, and then
2330       * largeCellThread invokes flushInMemory.
2331       * <p/>
2332       * 2. After largeCellThread finished CompactingMemStore.flushInMemory method, smallCellThread
2333       * can run into MyCompactingMemStore3.checkAndAddToActiveSize again.
2334       * <p/>
2335       * When flushByteSizeLessThanSmallAndLargeCellSize is false: smallCellThread and
2336       * largeCellThread concurrently write one cell and wait each other, and then write another
2337       * cell etc.
2338       */
2339      Thread.currentThread().setName(MyCompactingMemStore3.LARGE_CELL_THREAD_NAME);
2340      for (int i = 1; i <= MyCompactingMemStore3.CELL_COUNT; i++) {
2341        long currentTimestamp = timestamp + i;
2342        ExtendedCell cell = createCell(qf2, currentTimestamp, seqId, largeValue);
2343        totalCellByteSize.addAndGet(MutableSegment.getCellLength(cell));
2344        store.add(cell, memStoreSizing);
2345      }
2346      smallCellThread.join();
2347
2348      assertTrue(exceptionRef.get() == null);
2349      assertTrue(memStoreSizing.getCellsCount() == (MyCompactingMemStore3.CELL_COUNT * 2));
2350      assertTrue(memStoreSizing.getDataSize() == totalCellByteSize.get());
2351      if (flushByteSizeLessThanSmallAndLargeCellSize) {
2352        assertTrue(myCompactingMemStore.flushCounter.get() == MyCompactingMemStore3.CELL_COUNT);
2353      } else {
2354        assertTrue(
2355          myCompactingMemStore.flushCounter.get() <= (MyCompactingMemStore3.CELL_COUNT - 1));
2356      }
2357    } finally {
2358      Thread.currentThread().setName(oldThreadName);
2359    }
2360  }
2361
2362  /**
2363   * <pre>
2364   * This test is for HBASE-26384,
2365   * test {@link CompactingMemStore#flattenOneSegment} and {@link CompactingMemStore#snapshot()}
2366   * execute concurrently.
2367   * The threads sequence before HBASE-26384 is(The bug only exists for branch-2,and I add UTs
2368   * for both branch-2 and master):
2369   * 1. The {@link CompactingMemStore} size exceeds
2370   *    {@link CompactingMemStore#getInmemoryFlushSize()},the write thread adds a new
2371   *    {@link ImmutableSegment}  to the head of {@link CompactingMemStore#pipeline},and start a
2372   *    in memory compact thread to execute {@link CompactingMemStore#inMemoryCompaction}.
2373   * 2. The in memory compact thread starts and then stopping before
2374   *    {@link CompactingMemStore#flattenOneSegment}.
2375   * 3. The snapshot thread starts {@link CompactingMemStore#snapshot} concurrently,after the
2376   *    snapshot thread executing {@link CompactingMemStore#getImmutableSegments},the in memory
2377   *    compact thread continues.
2378   *    Assuming {@link VersionedSegmentsList#version} returned from
2379   *    {@link CompactingMemStore#getImmutableSegments} is v.
2380   * 4. The snapshot thread stopping before {@link CompactingMemStore#swapPipelineWithNull}.
2381   * 5. The in memory compact thread completes {@link CompactingMemStore#flattenOneSegment},
2382   *    {@link CompactionPipeline#version} is still v.
2383   * 6. The snapshot thread continues {@link CompactingMemStore#swapPipelineWithNull}, and because
2384   *    {@link CompactionPipeline#version} is v, {@link CompactingMemStore#swapPipelineWithNull}
2385   *    thinks it is successful and continue flushing,but the {@link ImmutableSegment} in
2386   *    {@link CompactionPipeline} has changed because
2387   *    {@link CompactingMemStore#flattenOneSegment},so the {@link ImmutableSegment} is not
2388   *    removed in fact and still remaining in {@link CompactionPipeline}.
2389   *
2390   * After HBASE-26384, the 5-6 step is changed to following, which is expected behavior:
2391   * 5. The in memory compact thread completes {@link CompactingMemStore#flattenOneSegment},
2392   *    {@link CompactingMemStore#flattenOneSegment} change {@link CompactionPipeline#version} to
2393   *    v+1.
2394   * 6. The snapshot thread continues {@link CompactingMemStore#swapPipelineWithNull}, and because
2395   *    {@link CompactionPipeline#version} is v+1, {@link CompactingMemStore#swapPipelineWithNull}
2396   *    failed and retry the while loop in {@link CompactingMemStore#pushPipelineToSnapshot} once
2397   *    again, because there is no concurrent {@link CompactingMemStore#inMemoryCompaction} now,
2398   *    {@link CompactingMemStore#swapPipelineWithNull} succeeds.
2399   * </pre>
2400   */
2401  @Test
2402  public void testFlattenAndSnapshotCompactingMemStoreConcurrently() throws Exception {
2403    Configuration conf = HBaseConfiguration.create();
2404
2405    byte[] smallValue = new byte[3];
2406    byte[] largeValue = new byte[9];
2407    final long timestamp = EnvironmentEdgeManager.currentTime();
2408    final long seqId = 100;
2409    final ExtendedCell smallCell = createCell(qf1, timestamp, seqId, smallValue);
2410    final ExtendedCell largeCell = createCell(qf2, timestamp, seqId, largeValue);
2411    int smallCellByteSize = MutableSegment.getCellLength(smallCell);
2412    int largeCellByteSize = MutableSegment.getCellLength(largeCell);
2413    int totalCellByteSize = (smallCellByteSize + largeCellByteSize);
2414    int flushByteSize = totalCellByteSize - 2;
2415
2416    // set CompactingMemStore.inmemoryFlushSize to flushByteSize.
2417    conf.set(HStore.MEMSTORE_CLASS_NAME, MyCompactingMemStore4.class.getName());
2418    conf.setDouble(CompactingMemStore.IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, 0.005);
2419    conf.set(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, String.valueOf(flushByteSize * 200));
2420
2421    init(name, conf, ColumnFamilyDescriptorBuilder.newBuilder(family)
2422      .setInMemoryCompaction(MemoryCompactionPolicy.BASIC).build());
2423
2424    MyCompactingMemStore4 myCompactingMemStore = ((MyCompactingMemStore4) store.memstore);
2425    assertTrue((int) (myCompactingMemStore.getInmemoryFlushSize()) == flushByteSize);
2426
2427    store.add(smallCell, new NonThreadSafeMemStoreSizing());
2428    store.add(largeCell, new NonThreadSafeMemStoreSizing());
2429
2430    String oldThreadName = Thread.currentThread().getName();
2431    try {
2432      Thread.currentThread().setName(MyCompactingMemStore4.TAKE_SNAPSHOT_THREAD_NAME);
2433      /**
2434       * {@link CompactingMemStore#snapshot} must wait the in memory compact thread enters
2435       * {@link CompactingMemStore#flattenOneSegment},because {@link CompactingMemStore#snapshot}
2436       * would invoke {@link CompactingMemStore#stopCompaction}.
2437       */
2438      myCompactingMemStore.snapShotStartCyclicCyclicBarrier.await();
2439
2440      MemStoreSnapshot memStoreSnapshot = myCompactingMemStore.snapshot();
2441      myCompactingMemStore.inMemoryCompactionEndCyclicBarrier.await();
2442
2443      assertTrue(memStoreSnapshot.getCellsCount() == 2);
2444      assertTrue(((int) (memStoreSnapshot.getDataSize())) == totalCellByteSize);
2445      VersionedSegmentsList segments = myCompactingMemStore.getImmutableSegments();
2446      assertTrue(segments.getNumOfSegments() == 0);
2447      assertTrue(segments.getNumOfCells() == 0);
2448      assertTrue(myCompactingMemStore.setInMemoryCompactionFlagCounter.get() == 1);
2449      assertTrue(myCompactingMemStore.swapPipelineWithNullCounter.get() == 2);
2450    } finally {
2451      Thread.currentThread().setName(oldThreadName);
2452    }
2453  }
2454
2455  /**
2456   * <pre>
2457   * This test is for HBASE-26384,
2458   * test {@link CompactingMemStore#flattenOneSegment}{@link CompactingMemStore#snapshot()}
2459   * and writeMemStore execute concurrently.
2460   * The threads sequence before HBASE-26384 is(The bug only exists for branch-2,and I add UTs
2461   * for both branch-2 and master):
2462   * 1. The {@link CompactingMemStore} size exceeds
2463   *    {@link CompactingMemStore#getInmemoryFlushSize()},the write thread adds a new
2464   *    {@link ImmutableSegment}  to the head of {@link CompactingMemStore#pipeline},and start a
2465   *    in memory compact thread to execute {@link CompactingMemStore#inMemoryCompaction}.
2466   * 2. The in memory compact thread starts and then stopping before
2467   *    {@link CompactingMemStore#flattenOneSegment}.
2468   * 3. The snapshot thread starts {@link CompactingMemStore#snapshot} concurrently,after the
2469   *    snapshot thread executing {@link CompactingMemStore#getImmutableSegments},the in memory
2470   *    compact thread continues.
2471   *    Assuming {@link VersionedSegmentsList#version} returned from
2472   *    {@link CompactingMemStore#getImmutableSegments} is v.
2473   * 4. The snapshot thread stopping before {@link CompactingMemStore#swapPipelineWithNull}.
2474   * 5. The in memory compact thread completes {@link CompactingMemStore#flattenOneSegment},
2475   *    {@link CompactionPipeline#version} is still v.
2476   * 6. The snapshot thread continues {@link CompactingMemStore#swapPipelineWithNull}, and because
2477   *    {@link CompactionPipeline#version} is v, {@link CompactingMemStore#swapPipelineWithNull}
2478   *    thinks it is successful and continue flushing,but the {@link ImmutableSegment} in
2479   *    {@link CompactionPipeline} has changed because
2480   *    {@link CompactingMemStore#flattenOneSegment},so the {@link ImmutableSegment} is not
2481   *    removed in fact and still remaining in {@link CompactionPipeline}.
2482   *
2483   * After HBASE-26384, the 5-6 step is changed to following, which is expected behavior,
2484   * and I add step 7-8 to test there is new segment added before retry.
2485   * 5. The in memory compact thread completes {@link CompactingMemStore#flattenOneSegment},
2486   *    {@link CompactingMemStore#flattenOneSegment} change {@link CompactionPipeline#version} to
2487   *     v+1.
2488   * 6. The snapshot thread continues {@link CompactingMemStore#swapPipelineWithNull}, and because
2489   *    {@link CompactionPipeline#version} is v+1, {@link CompactingMemStore#swapPipelineWithNull}
2490   *    failed and retry,{@link VersionedSegmentsList#version} returned from
2491   *    {@link CompactingMemStore#getImmutableSegments} is v+1.
2492   * 7. The write thread continues writing to {@link CompactingMemStore} and
2493   *    {@link CompactingMemStore} size exceeds {@link CompactingMemStore#getInmemoryFlushSize()},
2494   *    {@link CompactingMemStore#flushInMemory(MutableSegment)} is called and a new
2495   *    {@link ImmutableSegment} is added to the head of {@link CompactingMemStore#pipeline},
2496   *    {@link CompactionPipeline#version} is still v+1.
2497   * 8. The snapshot thread continues {@link CompactingMemStore#swapPipelineWithNull}, and because
2498   *    {@link CompactionPipeline#version} is still v+1,
2499   *    {@link CompactingMemStore#swapPipelineWithNull} succeeds.The new {@link ImmutableSegment}
2500   *    remained at the head of {@link CompactingMemStore#pipeline},the old is removed by
2501   *    {@link CompactingMemStore#swapPipelineWithNull}.
2502   * </pre>
2503   */
2504  @Test
2505  public void testFlattenSnapshotWriteCompactingMemeStoreConcurrently() throws Exception {
2506    Configuration conf = HBaseConfiguration.create();
2507
2508    byte[] smallValue = new byte[3];
2509    byte[] largeValue = new byte[9];
2510    final long timestamp = EnvironmentEdgeManager.currentTime();
2511    final long seqId = 100;
2512    final ExtendedCell smallCell = createCell(qf1, timestamp, seqId, smallValue);
2513    final ExtendedCell largeCell = createCell(qf2, timestamp, seqId, largeValue);
2514    int smallCellByteSize = MutableSegment.getCellLength(smallCell);
2515    int largeCellByteSize = MutableSegment.getCellLength(largeCell);
2516    int firstWriteCellByteSize = (smallCellByteSize + largeCellByteSize);
2517    int flushByteSize = firstWriteCellByteSize - 2;
2518
2519    // set CompactingMemStore.inmemoryFlushSize to flushByteSize.
2520    conf.set(HStore.MEMSTORE_CLASS_NAME, MyCompactingMemStore5.class.getName());
2521    conf.setDouble(CompactingMemStore.IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, 0.005);
2522    conf.set(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, String.valueOf(flushByteSize * 200));
2523
2524    init(name, conf, ColumnFamilyDescriptorBuilder.newBuilder(family)
2525      .setInMemoryCompaction(MemoryCompactionPolicy.BASIC).build());
2526
2527    final MyCompactingMemStore5 myCompactingMemStore = ((MyCompactingMemStore5) store.memstore);
2528    assertTrue((int) (myCompactingMemStore.getInmemoryFlushSize()) == flushByteSize);
2529
2530    store.add(smallCell, new NonThreadSafeMemStoreSizing());
2531    store.add(largeCell, new NonThreadSafeMemStoreSizing());
2532
2533    final AtomicReference<Throwable> exceptionRef = new AtomicReference<Throwable>();
2534    final ExtendedCell writeAgainCell1 = createCell(qf3, timestamp, seqId + 1, largeValue);
2535    final ExtendedCell writeAgainCell2 = createCell(qf4, timestamp, seqId + 1, largeValue);
2536    final int writeAgainCellByteSize =
2537      MutableSegment.getCellLength(writeAgainCell1) + MutableSegment.getCellLength(writeAgainCell2);
2538    final Thread writeAgainThread = new Thread(() -> {
2539      try {
2540        myCompactingMemStore.writeMemStoreAgainStartCyclicBarrier.await();
2541
2542        store.add(writeAgainCell1, new NonThreadSafeMemStoreSizing());
2543        store.add(writeAgainCell2, new NonThreadSafeMemStoreSizing());
2544
2545        myCompactingMemStore.writeMemStoreAgainEndCyclicBarrier.await();
2546      } catch (Throwable exception) {
2547        exceptionRef.set(exception);
2548      }
2549    });
2550    writeAgainThread.setName(MyCompactingMemStore5.WRITE_AGAIN_THREAD_NAME);
2551    writeAgainThread.start();
2552
2553    String oldThreadName = Thread.currentThread().getName();
2554    try {
2555      Thread.currentThread().setName(MyCompactingMemStore5.TAKE_SNAPSHOT_THREAD_NAME);
2556      /**
2557       * {@link CompactingMemStore#snapshot} must wait the in memory compact thread enters
2558       * {@link CompactingMemStore#flattenOneSegment},because {@link CompactingMemStore#snapshot}
2559       * would invoke {@link CompactingMemStore#stopCompaction}.
2560       */
2561      myCompactingMemStore.snapShotStartCyclicCyclicBarrier.await();
2562      MemStoreSnapshot memStoreSnapshot = myCompactingMemStore.snapshot();
2563      myCompactingMemStore.inMemoryCompactionEndCyclicBarrier.await();
2564      writeAgainThread.join();
2565
2566      assertTrue(memStoreSnapshot.getCellsCount() == 2);
2567      assertTrue(((int) (memStoreSnapshot.getDataSize())) == firstWriteCellByteSize);
2568      VersionedSegmentsList segments = myCompactingMemStore.getImmutableSegments();
2569      assertTrue(segments.getNumOfSegments() == 1);
2570      assertTrue(
2571        ((int) (segments.getStoreSegments().get(0).getDataSize())) == writeAgainCellByteSize);
2572      assertTrue(segments.getNumOfCells() == 2);
2573      assertTrue(myCompactingMemStore.setInMemoryCompactionFlagCounter.get() == 2);
2574      assertTrue(exceptionRef.get() == null);
2575      assertTrue(myCompactingMemStore.swapPipelineWithNullCounter.get() == 2);
2576    } finally {
2577      Thread.currentThread().setName(oldThreadName);
2578    }
2579  }
2580
2581  /**
2582   * <pre>
2583   * This test is for HBASE-26465,
2584   * test {@link DefaultMemStore#clearSnapshot} and {@link DefaultMemStore#getScanners} execute
2585   * concurrently. The threads sequence before HBASE-26465 is:
2586   * 1.The flush thread starts {@link DefaultMemStore} flushing after some cells have be added to
2587   *  {@link DefaultMemStore}.
2588   * 2.The flush thread stopping before {@link DefaultMemStore#clearSnapshot} in
2589   *   {@link HStore#updateStorefiles} after completed flushing memStore to hfile.
2590   * 3.The scan thread starts and stopping after {@link DefaultMemStore#getSnapshotSegments} in
2591   *   {@link DefaultMemStore#getScanners},here the scan thread gets the
2592   *   {@link DefaultMemStore#snapshot} which is created by the flush thread.
2593   * 4.The flush thread continues {@link DefaultMemStore#clearSnapshot} and close
2594   *   {@link DefaultMemStore#snapshot},because the reference count of the corresponding
2595   *   {@link MemStoreLABImpl} is 0, the {@link Chunk}s in corresponding {@link MemStoreLABImpl}
2596   *   are recycled.
2597   * 5.The scan thread continues {@link DefaultMemStore#getScanners},and create a
2598   *   {@link SegmentScanner} for this {@link DefaultMemStore#snapshot}, and increase the
2599   *   reference count of the corresponding {@link MemStoreLABImpl}, but {@link Chunk}s in
2600   *   corresponding {@link MemStoreLABImpl} are recycled by step 4, and these {@link Chunk}s may
2601   *   be overwritten by other write threads,which may cause serious problem.
2602   * After HBASE-26465,{@link DefaultMemStore#getScanners} and
2603   * {@link DefaultMemStore#clearSnapshot} could not execute concurrently.
2604   * </pre>
2605   */
2606  @Test
2607  public void testClearSnapshotGetScannerConcurrently() throws Exception {
2608    Configuration conf = HBaseConfiguration.create();
2609
2610    byte[] smallValue = new byte[3];
2611    byte[] largeValue = new byte[9];
2612    final long timestamp = EnvironmentEdgeManager.currentTime();
2613    final long seqId = 100;
2614    final ExtendedCell smallCell = createCell(qf1, timestamp, seqId, smallValue);
2615    final ExtendedCell largeCell = createCell(qf2, timestamp, seqId, largeValue);
2616    TreeSet<byte[]> quals = new TreeSet<>(Bytes.BYTES_COMPARATOR);
2617    quals.add(qf1);
2618    quals.add(qf2);
2619
2620    conf.set(HStore.MEMSTORE_CLASS_NAME, MyDefaultMemStore.class.getName());
2621    conf.setBoolean(WALFactory.WAL_ENABLED, false);
2622
2623    init(name, conf, ColumnFamilyDescriptorBuilder.newBuilder(family).build());
2624    MyDefaultMemStore myDefaultMemStore = (MyDefaultMemStore) (store.memstore);
2625    myDefaultMemStore.store = store;
2626
2627    MemStoreSizing memStoreSizing = new NonThreadSafeMemStoreSizing();
2628    store.add(smallCell, memStoreSizing);
2629    store.add(largeCell, memStoreSizing);
2630
2631    final AtomicReference<Throwable> exceptionRef = new AtomicReference<Throwable>();
2632    final Thread flushThread = new Thread(() -> {
2633      try {
2634        flushStore(store, id++);
2635      } catch (Throwable exception) {
2636        exceptionRef.set(exception);
2637      }
2638    });
2639    flushThread.setName(MyDefaultMemStore.FLUSH_THREAD_NAME);
2640    flushThread.start();
2641
2642    String oldThreadName = Thread.currentThread().getName();
2643    StoreScanner storeScanner = null;
2644    try {
2645      Thread.currentThread().setName(MyDefaultMemStore.GET_SCANNER_THREAD_NAME);
2646
2647      /**
2648       * Wait flush thread stopping before {@link DefaultMemStore#doClearSnapshot}
2649       */
2650      myDefaultMemStore.getScannerCyclicBarrier.await();
2651
2652      storeScanner = (StoreScanner) store.getScanner(new Scan(new Get(row)), quals, seqId + 1);
2653      flushThread.join();
2654
2655      if (myDefaultMemStore.shouldWait) {
2656        SegmentScanner segmentScanner = getTypeKeyValueScanner(storeScanner, SegmentScanner.class);
2657        MemStoreLABImpl memStoreLAB = (MemStoreLABImpl) (segmentScanner.segment.getMemStoreLAB());
2658        assertTrue(memStoreLAB.isClosed());
2659        assertTrue(!memStoreLAB.chunks.isEmpty());
2660        assertTrue(!memStoreLAB.isReclaimed());
2661
2662        ExtendedCell cell1 = segmentScanner.next();
2663        PrivateCellUtil.equals(smallCell, cell1);
2664        ExtendedCell cell2 = segmentScanner.next();
2665        PrivateCellUtil.equals(largeCell, cell2);
2666        assertNull(segmentScanner.next());
2667      } else {
2668        List<ExtendedCell> results = new ArrayList<>();
2669        storeScanner.next(results);
2670        assertEquals(2, results.size());
2671        PrivateCellUtil.equals(smallCell, results.get(0));
2672        PrivateCellUtil.equals(largeCell, results.get(1));
2673      }
2674      assertTrue(exceptionRef.get() == null);
2675    } finally {
2676      if (storeScanner != null) {
2677        storeScanner.close();
2678      }
2679      Thread.currentThread().setName(oldThreadName);
2680    }
2681  }
2682
2683  @SuppressWarnings("unchecked")
2684  private <T> T getTypeKeyValueScanner(StoreScanner storeScanner, Class<T> keyValueScannerClass) {
2685    List<T> resultScanners = new ArrayList<T>();
2686    for (KeyValueScanner keyValueScanner : storeScanner.currentScanners) {
2687      if (keyValueScannerClass.isInstance(keyValueScanner)) {
2688        resultScanners.add((T) keyValueScanner);
2689      }
2690    }
2691    assertTrue(resultScanners.size() == 1);
2692    return resultScanners.get(0);
2693  }
2694
2695  @Test
2696  public void testOnConfigurationChange() throws IOException {
2697    final int COMMON_MAX_FILES_TO_COMPACT = 10;
2698    final int NEW_COMMON_MAX_FILES_TO_COMPACT = 8;
2699    final int STORE_MAX_FILES_TO_COMPACT = 6;
2700
2701    // Build a table that its maxFileToCompact different from common configuration.
2702    Configuration conf = HBaseConfiguration.create();
2703    conf.setInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MAX_KEY,
2704      COMMON_MAX_FILES_TO_COMPACT);
2705    conf.setBoolean(CACHE_DATA_ON_READ_KEY, false);
2706    conf.setBoolean(CACHE_BLOCKS_ON_WRITE_KEY, true);
2707    conf.setBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, true);
2708    ColumnFamilyDescriptor hcd = ColumnFamilyDescriptorBuilder.newBuilder(family)
2709      .setConfiguration(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MAX_KEY,
2710        String.valueOf(STORE_MAX_FILES_TO_COMPACT))
2711      .build();
2712    init(name, conf, hcd);
2713
2714    // After updating common configuration, the conf in HStore itself must not be changed.
2715    conf.setInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MAX_KEY,
2716      NEW_COMMON_MAX_FILES_TO_COMPACT);
2717    this.store.onConfigurationChange(conf);
2718
2719    assertEquals(STORE_MAX_FILES_TO_COMPACT,
2720      store.getStoreEngine().getCompactionPolicy().getConf().getMaxFilesToCompact());
2721
2722    assertEquals(conf.getBoolean(CACHE_DATA_ON_READ_KEY, DEFAULT_CACHE_DATA_ON_READ), false);
2723    assertEquals(conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_DATA_ON_WRITE), true);
2724    assertEquals(conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, DEFAULT_EVICT_ON_CLOSE), true);
2725
2726    // reset to default values
2727    conf.getBoolean(CACHE_DATA_ON_READ_KEY, DEFAULT_CACHE_DATA_ON_READ);
2728    conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_DATA_ON_WRITE);
2729    conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, DEFAULT_EVICT_ON_CLOSE);
2730    this.store.onConfigurationChange(conf);
2731  }
2732
2733  @Test
2734  public void testCacheConfigKeepsStoreConfigurationOnConfigurationChange() throws IOException {
2735    Configuration conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration());
2736    conf.setBoolean(CACHE_DATA_ON_READ_KEY, true);
2737    conf.setBoolean(CACHE_BLOCKS_ON_WRITE_KEY, false);
2738    conf.setBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, true);
2739    // Override the global cache settings in the table and column family configuration.
2740    TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(TableName.valueOf(table))
2741      .setValue(EVICT_BLOCKS_ON_CLOSE_KEY, "false");
2742    ColumnFamilyDescriptor hcd = ColumnFamilyDescriptorBuilder.newBuilder(family)
2743      .setConfiguration(CACHE_BLOCKS_ON_WRITE_KEY, "true").build();
2744    initHRegion(name, conf, builder, hcd, null, false);
2745    BlockCache blockCache = mock(BlockCache.class);
2746    region.setBlockCache(blockCache);
2747    store = new HStore(region, hcd, conf, false);
2748    CacheConfig cacheConfig = store.getCacheConfig();
2749    assertTrue(cacheConfig.shouldCacheDataOnRead());
2750    assertTrue(cacheConfig.shouldCacheDataOnWrite());
2751    assertFalse(cacheConfig.shouldEvictOnClose());
2752
2753    ConfigurationManager manager = new ConfigurationManager();
2754    manager.registerObserver(store);
2755    // The cache config is refreshed through the store, while the shared block cache still listens
2756    // to the global configuration.
2757    assertFalse(manager.containsObserver(cacheConfig));
2758    assertTrue(manager.containsObserver(blockCache));
2759
2760    Configuration newConf = new Configuration(conf);
2761    newConf.setBoolean(CACHE_DATA_ON_READ_KEY, false);
2762    manager.notifyAllObservers(newConf);
2763    assertFalse(cacheConfig.shouldCacheDataOnRead());
2764    assertTrue(cacheConfig.shouldCacheDataOnWrite());
2765    assertFalse(cacheConfig.shouldEvictOnClose());
2766    verify(blockCache).onConfigurationChange(newConf);
2767  }
2768
2769  /**
2770   * This test is for HBASE-26476
2771   */
2772  @Test
2773  public void testExtendsDefaultMemStore() throws Exception {
2774    Configuration conf = HBaseConfiguration.create();
2775    conf.setBoolean(WALFactory.WAL_ENABLED, false);
2776
2777    init(name, conf, ColumnFamilyDescriptorBuilder.newBuilder(family).build());
2778    assertTrue(this.store.memstore.getClass() == DefaultMemStore.class);
2779    tearDown();
2780
2781    conf.set(HStore.MEMSTORE_CLASS_NAME, CustomDefaultMemStore.class.getName());
2782    init(name, conf, ColumnFamilyDescriptorBuilder.newBuilder(family).build());
2783    assertTrue(this.store.memstore.getClass() == CustomDefaultMemStore.class);
2784  }
2785
2786  static class CustomDefaultMemStore extends DefaultMemStore {
2787
2788    public CustomDefaultMemStore(Configuration conf, CellComparator c,
2789      RegionServicesForStores regionServices) {
2790      super(conf, c, regionServices);
2791    }
2792
2793  }
2794
2795  /**
2796   * This test is for HBASE-26488
2797   */
2798  @Test
2799  public void testMemoryLeakWhenFlushMemStoreRetrying() throws Exception {
2800    Configuration conf = HBaseConfiguration.create();
2801
2802    byte[] smallValue = new byte[3];
2803    byte[] largeValue = new byte[9];
2804    final long timestamp = EnvironmentEdgeManager.currentTime();
2805    final long seqId = 100;
2806    final ExtendedCell smallCell = createCell(qf1, timestamp, seqId, smallValue);
2807    final ExtendedCell largeCell = createCell(qf2, timestamp, seqId, largeValue);
2808    TreeSet<byte[]> quals = new TreeSet<>(Bytes.BYTES_COMPARATOR);
2809    quals.add(qf1);
2810    quals.add(qf2);
2811
2812    conf.set(HStore.MEMSTORE_CLASS_NAME, MyDefaultMemStore1.class.getName());
2813    conf.setBoolean(WALFactory.WAL_ENABLED, false);
2814    conf.set(DefaultStoreEngine.DEFAULT_STORE_FLUSHER_CLASS_KEY,
2815      MyDefaultStoreFlusher.class.getName());
2816
2817    init(name, conf, ColumnFamilyDescriptorBuilder.newBuilder(family).build());
2818    MyDefaultMemStore1 myDefaultMemStore = (MyDefaultMemStore1) (store.memstore);
2819    assertTrue((store.storeEngine.getStoreFlusher()) instanceof MyDefaultStoreFlusher);
2820
2821    MemStoreSizing memStoreSizing = new NonThreadSafeMemStoreSizing();
2822    store.add(smallCell, memStoreSizing);
2823    store.add(largeCell, memStoreSizing);
2824    flushStore(store, id++);
2825
2826    MemStoreLABImpl memStoreLAB =
2827      (MemStoreLABImpl) (myDefaultMemStore.snapshotImmutableSegment.getMemStoreLAB());
2828    assertTrue(memStoreLAB.isClosed());
2829    assertTrue(memStoreLAB.getRefCntValue() == 0);
2830    assertTrue(memStoreLAB.isReclaimed());
2831    assertTrue(memStoreLAB.chunks.isEmpty());
2832    StoreScanner storeScanner = null;
2833    try {
2834      storeScanner = (StoreScanner) store.getScanner(new Scan(new Get(row)), quals, seqId + 1);
2835      assertTrue(store.storeEngine.getStoreFileManager().getStorefileCount() == 1);
2836      assertTrue(store.memstore.size().getCellsCount() == 0);
2837      assertTrue(store.memstore.getSnapshotSize().getCellsCount() == 0);
2838      assertTrue(storeScanner.currentScanners.size() == 1);
2839      assertTrue(storeScanner.currentScanners.get(0) instanceof StoreFileScanner);
2840
2841      List<ExtendedCell> results = new ArrayList<>();
2842      storeScanner.next(results);
2843      assertEquals(2, results.size());
2844      PrivateCellUtil.equals(smallCell, results.get(0));
2845      PrivateCellUtil.equals(largeCell, results.get(1));
2846    } finally {
2847      if (storeScanner != null) {
2848        storeScanner.close();
2849      }
2850    }
2851  }
2852
2853  static class MyDefaultMemStore1 extends DefaultMemStore {
2854
2855    private ImmutableSegment snapshotImmutableSegment;
2856
2857    public MyDefaultMemStore1(Configuration conf, CellComparator c,
2858      RegionServicesForStores regionServices) {
2859      super(conf, c, regionServices);
2860    }
2861
2862    @Override
2863    public MemStoreSnapshot snapshot() {
2864      MemStoreSnapshot result = super.snapshot();
2865      this.snapshotImmutableSegment = snapshot;
2866      return result;
2867    }
2868
2869  }
2870
2871  public static class MyDefaultStoreFlusher extends DefaultStoreFlusher {
2872    private static final AtomicInteger failCounter = new AtomicInteger(1);
2873    private static final AtomicInteger counter = new AtomicInteger(0);
2874
2875    public MyDefaultStoreFlusher(Configuration conf, HStore store) {
2876      super(conf, store);
2877    }
2878
2879    @Override
2880    public List<Path> flushSnapshot(MemStoreSnapshot snapshot, long cacheFlushId,
2881      MonitoredTask status, ThroughputController throughputController,
2882      FlushLifeCycleTracker tracker, Consumer<Path> writerCreationTracker) throws IOException {
2883      counter.incrementAndGet();
2884      return super.flushSnapshot(snapshot, cacheFlushId, status, throughputController, tracker,
2885        writerCreationTracker);
2886    }
2887
2888    @Override
2889    protected void performFlush(InternalScanner scanner, final CellSink sink,
2890      ThroughputController throughputController) throws IOException {
2891
2892      final int currentCount = counter.get();
2893      CellSink newCellSink = (cell) -> {
2894        if (currentCount <= failCounter.get()) {
2895          throw new IOException("Simulated exception by tests");
2896        }
2897        sink.append(cell);
2898      };
2899      super.performFlush(scanner, newCellSink, throughputController);
2900    }
2901  }
2902
2903  /**
2904   * This test is for HBASE-26494, test the {@link RefCnt} behaviors in {@link ImmutableMemStoreLAB}
2905   */
2906  @Test
2907  public void testImmutableMemStoreLABRefCnt() throws Exception {
2908    Configuration conf = HBaseConfiguration.create();
2909
2910    byte[] smallValue = new byte[3];
2911    byte[] largeValue = new byte[9];
2912    final long timestamp = EnvironmentEdgeManager.currentTime();
2913    final long seqId = 100;
2914    final ExtendedCell smallCell1 = createCell(qf1, timestamp, seqId, smallValue);
2915    final ExtendedCell largeCell1 = createCell(qf2, timestamp, seqId, largeValue);
2916    final ExtendedCell smallCell2 = createCell(qf3, timestamp, seqId + 1, smallValue);
2917    final ExtendedCell largeCell2 = createCell(qf4, timestamp, seqId + 1, largeValue);
2918    final ExtendedCell smallCell3 = createCell(qf5, timestamp, seqId + 2, smallValue);
2919    final ExtendedCell largeCell3 = createCell(qf6, timestamp, seqId + 2, largeValue);
2920
2921    int smallCellByteSize = MutableSegment.getCellLength(smallCell1);
2922    int largeCellByteSize = MutableSegment.getCellLength(largeCell1);
2923    int firstWriteCellByteSize = (smallCellByteSize + largeCellByteSize);
2924    int flushByteSize = firstWriteCellByteSize - 2;
2925
2926    // set CompactingMemStore.inmemoryFlushSize to flushByteSize.
2927    conf.set(HStore.MEMSTORE_CLASS_NAME, CompactingMemStore.class.getName());
2928    conf.setDouble(CompactingMemStore.IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, 0.005);
2929    conf.set(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, String.valueOf(flushByteSize * 200));
2930    conf.setBoolean(WALFactory.WAL_ENABLED, false);
2931
2932    init(name, conf, ColumnFamilyDescriptorBuilder.newBuilder(family)
2933      .setInMemoryCompaction(MemoryCompactionPolicy.BASIC).build());
2934
2935    final CompactingMemStore myCompactingMemStore = ((CompactingMemStore) store.memstore);
2936    assertTrue((int) (myCompactingMemStore.getInmemoryFlushSize()) == flushByteSize);
2937    myCompactingMemStore.allowCompaction.set(false);
2938
2939    NonThreadSafeMemStoreSizing memStoreSizing = new NonThreadSafeMemStoreSizing();
2940    store.add(smallCell1, memStoreSizing);
2941    store.add(largeCell1, memStoreSizing);
2942    store.add(smallCell2, memStoreSizing);
2943    store.add(largeCell2, memStoreSizing);
2944    store.add(smallCell3, memStoreSizing);
2945    store.add(largeCell3, memStoreSizing);
2946    VersionedSegmentsList versionedSegmentsList = myCompactingMemStore.getImmutableSegments();
2947    assertTrue(versionedSegmentsList.getNumOfSegments() == 3);
2948    List<ImmutableSegment> segments = versionedSegmentsList.getStoreSegments();
2949    List<MemStoreLABImpl> memStoreLABs = new ArrayList<MemStoreLABImpl>(segments.size());
2950    for (ImmutableSegment segment : segments) {
2951      memStoreLABs.add((MemStoreLABImpl) segment.getMemStoreLAB());
2952    }
2953    List<KeyValueScanner> scanners1 = myCompactingMemStore.getScanners(Long.MAX_VALUE);
2954    for (MemStoreLABImpl memStoreLAB : memStoreLABs) {
2955      assertTrue(memStoreLAB.getRefCntValue() == 2);
2956    }
2957
2958    myCompactingMemStore.allowCompaction.set(true);
2959    myCompactingMemStore.flushInMemory();
2960
2961    versionedSegmentsList = myCompactingMemStore.getImmutableSegments();
2962    assertTrue(versionedSegmentsList.getNumOfSegments() == 1);
2963    ImmutableMemStoreLAB immutableMemStoreLAB =
2964      (ImmutableMemStoreLAB) (versionedSegmentsList.getStoreSegments().get(0).getMemStoreLAB());
2965    for (MemStoreLABImpl memStoreLAB : memStoreLABs) {
2966      assertTrue(memStoreLAB.getRefCntValue() == 2);
2967    }
2968
2969    List<KeyValueScanner> scanners2 = myCompactingMemStore.getScanners(Long.MAX_VALUE);
2970    for (MemStoreLABImpl memStoreLAB : memStoreLABs) {
2971      assertTrue(memStoreLAB.getRefCntValue() == 2);
2972    }
2973    assertTrue(immutableMemStoreLAB.getRefCntValue() == 2);
2974    for (KeyValueScanner scanner : scanners1) {
2975      scanner.close();
2976    }
2977    for (MemStoreLABImpl memStoreLAB : memStoreLABs) {
2978      assertTrue(memStoreLAB.getRefCntValue() == 1);
2979    }
2980    for (KeyValueScanner scanner : scanners2) {
2981      scanner.close();
2982    }
2983    for (MemStoreLABImpl memStoreLAB : memStoreLABs) {
2984      assertTrue(memStoreLAB.getRefCntValue() == 1);
2985    }
2986    assertTrue(immutableMemStoreLAB.getRefCntValue() == 1);
2987    flushStore(store, id++);
2988    for (MemStoreLABImpl memStoreLAB : memStoreLABs) {
2989      assertTrue(memStoreLAB.getRefCntValue() == 0);
2990    }
2991    assertTrue(immutableMemStoreLAB.getRefCntValue() == 0);
2992    assertTrue(immutableMemStoreLAB.isClosed());
2993    for (MemStoreLABImpl memStoreLAB : memStoreLABs) {
2994      assertTrue(memStoreLAB.isClosed());
2995      assertTrue(memStoreLAB.isReclaimed());
2996      assertTrue(memStoreLAB.chunks.isEmpty());
2997    }
2998  }
2999
3000  private HStoreFile mockStoreFileWithLength(long length) {
3001    HStoreFile sf = mock(HStoreFile.class);
3002    StoreFileReader sfr = mock(StoreFileReader.class);
3003    when(sf.isHFile()).thenReturn(true);
3004    when(sf.getReader()).thenReturn(sfr);
3005    when(sfr.length()).thenReturn(length);
3006    return sf;
3007  }
3008
3009  private static class MyThread extends Thread {
3010    private StoreScanner scanner;
3011    private KeyValueHeap heap;
3012
3013    public MyThread(StoreScanner scanner) {
3014      this.scanner = scanner;
3015    }
3016
3017    public KeyValueHeap getHeap() {
3018      return this.heap;
3019    }
3020
3021    @Override
3022    public void run() {
3023      scanner.trySwitchToStreamRead();
3024      heap = scanner.heap;
3025    }
3026  }
3027
3028  private static class MyMemStoreCompactor extends MemStoreCompactor {
3029    private static final AtomicInteger RUNNER_COUNT = new AtomicInteger(0);
3030    private static final CountDownLatch START_COMPACTOR_LATCH = new CountDownLatch(1);
3031
3032    public MyMemStoreCompactor(CompactingMemStore compactingMemStore,
3033      MemoryCompactionPolicy compactionPolicy) throws IllegalArgumentIOException {
3034      super(compactingMemStore, compactionPolicy);
3035    }
3036
3037    @Override
3038    public boolean start() throws IOException {
3039      boolean isFirst = RUNNER_COUNT.getAndIncrement() == 0;
3040      if (isFirst) {
3041        try {
3042          START_COMPACTOR_LATCH.await();
3043          return super.start();
3044        } catch (InterruptedException ex) {
3045          throw new RuntimeException(ex);
3046        }
3047      }
3048      return super.start();
3049    }
3050  }
3051
3052  public static class MyCompactingMemStoreWithCustomCompactor extends CompactingMemStore {
3053    private static final AtomicInteger RUNNER_COUNT = new AtomicInteger(0);
3054
3055    public MyCompactingMemStoreWithCustomCompactor(Configuration conf, CellComparatorImpl c,
3056      HStore store, RegionServicesForStores regionServices, MemoryCompactionPolicy compactionPolicy)
3057      throws IOException {
3058      super(conf, c, store, regionServices, compactionPolicy);
3059    }
3060
3061    @Override
3062    protected MemStoreCompactor createMemStoreCompactor(MemoryCompactionPolicy compactionPolicy)
3063      throws IllegalArgumentIOException {
3064      return new MyMemStoreCompactor(this, compactionPolicy);
3065    }
3066
3067    @Override
3068    protected boolean setInMemoryCompactionFlag() {
3069      boolean rval = super.setInMemoryCompactionFlag();
3070      if (rval) {
3071        RUNNER_COUNT.incrementAndGet();
3072        if (LOG.isDebugEnabled()) {
3073          LOG.debug("runner count: " + RUNNER_COUNT.get());
3074        }
3075      }
3076      return rval;
3077    }
3078  }
3079
3080  public static class MyCompactingMemStore extends CompactingMemStore {
3081    private static final AtomicBoolean START_TEST = new AtomicBoolean(false);
3082    private final CountDownLatch getScannerLatch = new CountDownLatch(1);
3083    private final CountDownLatch snapshotLatch = new CountDownLatch(1);
3084
3085    public MyCompactingMemStore(Configuration conf, CellComparatorImpl c, HStore store,
3086      RegionServicesForStores regionServices, MemoryCompactionPolicy compactionPolicy)
3087      throws IOException {
3088      super(conf, c, store, regionServices, compactionPolicy);
3089    }
3090
3091    @Override
3092    protected List<KeyValueScanner> createList(int capacity) {
3093      if (START_TEST.get()) {
3094        try {
3095          getScannerLatch.countDown();
3096          snapshotLatch.await();
3097        } catch (InterruptedException e) {
3098          throw new RuntimeException(e);
3099        }
3100      }
3101      return new ArrayList<>(capacity);
3102    }
3103
3104    @Override
3105    protected void pushActiveToPipeline(MutableSegment active, boolean checkEmpty) {
3106      if (START_TEST.get()) {
3107        try {
3108          getScannerLatch.await();
3109        } catch (InterruptedException e) {
3110          throw new RuntimeException(e);
3111        }
3112      }
3113
3114      super.pushActiveToPipeline(active, checkEmpty);
3115      if (START_TEST.get()) {
3116        snapshotLatch.countDown();
3117      }
3118    }
3119  }
3120
3121  interface MyListHook {
3122    void hook(int currentSize);
3123  }
3124
3125  private static class MyList<T> implements List<T> {
3126    private final List<T> delegatee = new ArrayList<>();
3127    private final MyListHook hookAtAdd;
3128
3129    MyList(final MyListHook hookAtAdd) {
3130      this.hookAtAdd = hookAtAdd;
3131    }
3132
3133    @Override
3134    public int size() {
3135      return delegatee.size();
3136    }
3137
3138    @Override
3139    public boolean isEmpty() {
3140      return delegatee.isEmpty();
3141    }
3142
3143    @Override
3144    public boolean contains(Object o) {
3145      return delegatee.contains(o);
3146    }
3147
3148    @Override
3149    public Iterator<T> iterator() {
3150      return delegatee.iterator();
3151    }
3152
3153    @Override
3154    public Object[] toArray() {
3155      return delegatee.toArray();
3156    }
3157
3158    @Override
3159    public <R> R[] toArray(R[] a) {
3160      return delegatee.toArray(a);
3161    }
3162
3163    @Override
3164    public boolean add(T e) {
3165      hookAtAdd.hook(size());
3166      return delegatee.add(e);
3167    }
3168
3169    @Override
3170    public boolean remove(Object o) {
3171      return delegatee.remove(o);
3172    }
3173
3174    @Override
3175    public boolean containsAll(Collection<?> c) {
3176      return delegatee.containsAll(c);
3177    }
3178
3179    @Override
3180    public boolean addAll(Collection<? extends T> c) {
3181      return delegatee.addAll(c);
3182    }
3183
3184    @Override
3185    public boolean addAll(int index, Collection<? extends T> c) {
3186      return delegatee.addAll(index, c);
3187    }
3188
3189    @Override
3190    public boolean removeAll(Collection<?> c) {
3191      return delegatee.removeAll(c);
3192    }
3193
3194    @Override
3195    public boolean retainAll(Collection<?> c) {
3196      return delegatee.retainAll(c);
3197    }
3198
3199    @Override
3200    public void clear() {
3201      delegatee.clear();
3202    }
3203
3204    @Override
3205    public T get(int index) {
3206      return delegatee.get(index);
3207    }
3208
3209    @Override
3210    public T set(int index, T element) {
3211      return delegatee.set(index, element);
3212    }
3213
3214    @Override
3215    public void add(int index, T element) {
3216      delegatee.add(index, element);
3217    }
3218
3219    @Override
3220    public T remove(int index) {
3221      return delegatee.remove(index);
3222    }
3223
3224    @Override
3225    public int indexOf(Object o) {
3226      return delegatee.indexOf(o);
3227    }
3228
3229    @Override
3230    public int lastIndexOf(Object o) {
3231      return delegatee.lastIndexOf(o);
3232    }
3233
3234    @Override
3235    public ListIterator<T> listIterator() {
3236      return delegatee.listIterator();
3237    }
3238
3239    @Override
3240    public ListIterator<T> listIterator(int index) {
3241      return delegatee.listIterator(index);
3242    }
3243
3244    @Override
3245    public List<T> subList(int fromIndex, int toIndex) {
3246      return delegatee.subList(fromIndex, toIndex);
3247    }
3248  }
3249
3250  private interface MyKeyValueHeapHook {
3251    void onRecordBlockSize(int recordBlockSizeCallCount);
3252  }
3253
3254  private static class MyKeyValueHeap extends KeyValueHeap {
3255    private final MyKeyValueHeapHook hook;
3256    private int recordBlockSizeCallCount;
3257
3258    public MyKeyValueHeap(List<? extends KeyValueScanner> scanners, CellComparator comparator,
3259      MyKeyValueHeapHook hook) throws IOException {
3260      super(scanners, comparator);
3261      this.hook = hook;
3262    }
3263
3264    @Override
3265    public void recordBlockSize(IntConsumer blockSizeConsumer) {
3266      recordBlockSizeCallCount++;
3267      hook.onRecordBlockSize(recordBlockSizeCallCount);
3268      super.recordBlockSize(blockSizeConsumer);
3269    }
3270  }
3271
3272  public static class MyCompactingMemStore2 extends CompactingMemStore {
3273    private static final String LARGE_CELL_THREAD_NAME = "largeCellThread";
3274    private static final String SMALL_CELL_THREAD_NAME = "smallCellThread";
3275    private final CyclicBarrier preCyclicBarrier = new CyclicBarrier(2);
3276    private final CyclicBarrier postCyclicBarrier = new CyclicBarrier(2);
3277    private final AtomicInteger largeCellPreUpdateCounter = new AtomicInteger(0);
3278    private final AtomicInteger smallCellPreUpdateCounter = new AtomicInteger(0);
3279
3280    public MyCompactingMemStore2(Configuration conf, CellComparatorImpl cellComparator,
3281      HStore store, RegionServicesForStores regionServices, MemoryCompactionPolicy compactionPolicy)
3282      throws IOException {
3283      super(conf, cellComparator, store, regionServices, compactionPolicy);
3284    }
3285
3286    @Override
3287    protected boolean checkAndAddToActiveSize(MutableSegment currActive, Cell cellToAdd,
3288      MemStoreSizing memstoreSizing) {
3289      if (Thread.currentThread().getName().equals(LARGE_CELL_THREAD_NAME)) {
3290        int currentCount = largeCellPreUpdateCounter.incrementAndGet();
3291        if (currentCount <= 1) {
3292          try {
3293            /**
3294             * smallCellThread enters CompactingMemStore.checkAndAddToActiveSize first, then
3295             * largeCellThread enters CompactingMemStore.checkAndAddToActiveSize, and then
3296             * largeCellThread invokes flushInMemory.
3297             */
3298            preCyclicBarrier.await();
3299          } catch (Throwable e) {
3300            throw new RuntimeException(e);
3301          }
3302        }
3303      }
3304
3305      boolean returnValue = super.checkAndAddToActiveSize(currActive, cellToAdd, memstoreSizing);
3306      if (Thread.currentThread().getName().equals(SMALL_CELL_THREAD_NAME)) {
3307        try {
3308          preCyclicBarrier.await();
3309        } catch (Throwable e) {
3310          throw new RuntimeException(e);
3311        }
3312      }
3313      return returnValue;
3314    }
3315
3316    @Override
3317    protected void doAdd(MutableSegment currentActive, ExtendedCell cell,
3318      MemStoreSizing memstoreSizing) {
3319      if (Thread.currentThread().getName().equals(SMALL_CELL_THREAD_NAME)) {
3320        try {
3321          /**
3322           * After largeCellThread finished flushInMemory method, smallCellThread can add cell to
3323           * currentActive . That is to say when largeCellThread called flushInMemory method,
3324           * currentActive has no cell.
3325           */
3326          postCyclicBarrier.await();
3327        } catch (Throwable e) {
3328          throw new RuntimeException(e);
3329        }
3330      }
3331      super.doAdd(currentActive, cell, memstoreSizing);
3332    }
3333
3334    @Override
3335    protected void flushInMemory(MutableSegment currentActiveMutableSegment) {
3336      super.flushInMemory(currentActiveMutableSegment);
3337      if (Thread.currentThread().getName().equals(LARGE_CELL_THREAD_NAME)) {
3338        if (largeCellPreUpdateCounter.get() <= 1) {
3339          try {
3340            postCyclicBarrier.await();
3341          } catch (Throwable e) {
3342            throw new RuntimeException(e);
3343          }
3344        }
3345      }
3346    }
3347
3348  }
3349
3350  public static class MyCompactingMemStore3 extends CompactingMemStore {
3351    private static final String LARGE_CELL_THREAD_NAME = "largeCellThread";
3352    private static final String SMALL_CELL_THREAD_NAME = "smallCellThread";
3353
3354    private final CyclicBarrier preCyclicBarrier = new CyclicBarrier(2);
3355    private final CyclicBarrier postCyclicBarrier = new CyclicBarrier(2);
3356    private final AtomicInteger flushCounter = new AtomicInteger(0);
3357    private static final int CELL_COUNT = 5;
3358    private boolean flushByteSizeLessThanSmallAndLargeCellSize = true;
3359
3360    public MyCompactingMemStore3(Configuration conf, CellComparatorImpl cellComparator,
3361      HStore store, RegionServicesForStores regionServices, MemoryCompactionPolicy compactionPolicy)
3362      throws IOException {
3363      super(conf, cellComparator, store, regionServices, compactionPolicy);
3364    }
3365
3366    @Override
3367    protected boolean checkAndAddToActiveSize(MutableSegment currActive, Cell cellToAdd,
3368      MemStoreSizing memstoreSizing) {
3369      if (!flushByteSizeLessThanSmallAndLargeCellSize) {
3370        return super.checkAndAddToActiveSize(currActive, cellToAdd, memstoreSizing);
3371      }
3372      if (Thread.currentThread().getName().equals(LARGE_CELL_THREAD_NAME)) {
3373        try {
3374          preCyclicBarrier.await();
3375        } catch (Throwable e) {
3376          throw new RuntimeException(e);
3377        }
3378      }
3379
3380      boolean returnValue = super.checkAndAddToActiveSize(currActive, cellToAdd, memstoreSizing);
3381      if (Thread.currentThread().getName().equals(SMALL_CELL_THREAD_NAME)) {
3382        try {
3383          preCyclicBarrier.await();
3384        } catch (Throwable e) {
3385          throw new RuntimeException(e);
3386        }
3387      }
3388      return returnValue;
3389    }
3390
3391    @Override
3392    protected void postUpdate(MutableSegment currentActiveMutableSegment) {
3393      super.postUpdate(currentActiveMutableSegment);
3394      if (!flushByteSizeLessThanSmallAndLargeCellSize) {
3395        try {
3396          postCyclicBarrier.await();
3397        } catch (Throwable e) {
3398          throw new RuntimeException(e);
3399        }
3400        return;
3401      }
3402
3403      if (Thread.currentThread().getName().equals(SMALL_CELL_THREAD_NAME)) {
3404        try {
3405          postCyclicBarrier.await();
3406        } catch (Throwable e) {
3407          throw new RuntimeException(e);
3408        }
3409      }
3410    }
3411
3412    @Override
3413    protected void flushInMemory(MutableSegment currentActiveMutableSegment) {
3414      super.flushInMemory(currentActiveMutableSegment);
3415      flushCounter.incrementAndGet();
3416      if (!flushByteSizeLessThanSmallAndLargeCellSize) {
3417        return;
3418      }
3419
3420      assertTrue(Thread.currentThread().getName().equals(LARGE_CELL_THREAD_NAME));
3421      try {
3422        postCyclicBarrier.await();
3423      } catch (Throwable e) {
3424        throw new RuntimeException(e);
3425      }
3426
3427    }
3428
3429    void disableCompaction() {
3430      allowCompaction.set(false);
3431    }
3432
3433    void enableCompaction() {
3434      allowCompaction.set(true);
3435    }
3436
3437  }
3438
3439  public static class MyCompactingMemStore4 extends CompactingMemStore {
3440    private static final String TAKE_SNAPSHOT_THREAD_NAME = "takeSnapShotThread";
3441    /**
3442     * {@link CompactingMemStore#flattenOneSegment} must execute after
3443     * {@link CompactingMemStore#getImmutableSegments}
3444     */
3445    private final CyclicBarrier flattenOneSegmentPreCyclicBarrier = new CyclicBarrier(2);
3446    /**
3447     * Only after {@link CompactingMemStore#flattenOneSegment} completed,
3448     * {@link CompactingMemStore#swapPipelineWithNull} could execute.
3449     */
3450    private final CyclicBarrier flattenOneSegmentPostCyclicBarrier = new CyclicBarrier(2);
3451    /**
3452     * Only the in memory compact thread enters {@link CompactingMemStore#flattenOneSegment},the
3453     * snapshot thread starts {@link CompactingMemStore#snapshot},because
3454     * {@link CompactingMemStore#snapshot} would invoke {@link CompactingMemStore#stopCompaction}.
3455     */
3456    private final CyclicBarrier snapShotStartCyclicCyclicBarrier = new CyclicBarrier(2);
3457    /**
3458     * To wait for {@link CompactingMemStore.InMemoryCompactionRunnable} stopping.
3459     */
3460    private final CyclicBarrier inMemoryCompactionEndCyclicBarrier = new CyclicBarrier(2);
3461    private final AtomicInteger getImmutableSegmentsListCounter = new AtomicInteger(0);
3462    private final AtomicInteger swapPipelineWithNullCounter = new AtomicInteger(0);
3463    private final AtomicInteger flattenOneSegmentCounter = new AtomicInteger(0);
3464    private final AtomicInteger setInMemoryCompactionFlagCounter = new AtomicInteger(0);
3465
3466    public MyCompactingMemStore4(Configuration conf, CellComparatorImpl cellComparator,
3467      HStore store, RegionServicesForStores regionServices, MemoryCompactionPolicy compactionPolicy)
3468      throws IOException {
3469      super(conf, cellComparator, store, regionServices, compactionPolicy);
3470    }
3471
3472    @Override
3473    public VersionedSegmentsList getImmutableSegments() {
3474      VersionedSegmentsList result = super.getImmutableSegments();
3475      if (Thread.currentThread().getName().equals(TAKE_SNAPSHOT_THREAD_NAME)) {
3476        int currentCount = getImmutableSegmentsListCounter.incrementAndGet();
3477        if (currentCount <= 1) {
3478          try {
3479            flattenOneSegmentPreCyclicBarrier.await();
3480          } catch (Throwable e) {
3481            throw new RuntimeException(e);
3482          }
3483        }
3484      }
3485      return result;
3486    }
3487
3488    @Override
3489    protected boolean swapPipelineWithNull(VersionedSegmentsList segments) {
3490      if (Thread.currentThread().getName().equals(TAKE_SNAPSHOT_THREAD_NAME)) {
3491        int currentCount = swapPipelineWithNullCounter.incrementAndGet();
3492        if (currentCount <= 1) {
3493          try {
3494            flattenOneSegmentPostCyclicBarrier.await();
3495          } catch (Throwable e) {
3496            throw new RuntimeException(e);
3497          }
3498        }
3499      }
3500      boolean result = super.swapPipelineWithNull(segments);
3501      if (Thread.currentThread().getName().equals(TAKE_SNAPSHOT_THREAD_NAME)) {
3502        int currentCount = swapPipelineWithNullCounter.get();
3503        if (currentCount <= 1) {
3504          assertTrue(!result);
3505        }
3506        if (currentCount == 2) {
3507          assertTrue(result);
3508        }
3509      }
3510      return result;
3511
3512    }
3513
3514    @Override
3515    public void flattenOneSegment(long requesterVersion, Action action) {
3516      int currentCount = flattenOneSegmentCounter.incrementAndGet();
3517      if (currentCount <= 1) {
3518        try {
3519          /**
3520           * {@link CompactingMemStore#snapshot} could start.
3521           */
3522          snapShotStartCyclicCyclicBarrier.await();
3523          flattenOneSegmentPreCyclicBarrier.await();
3524        } catch (Throwable e) {
3525          throw new RuntimeException(e);
3526        }
3527      }
3528      super.flattenOneSegment(requesterVersion, action);
3529      if (currentCount <= 1) {
3530        try {
3531          flattenOneSegmentPostCyclicBarrier.await();
3532        } catch (Throwable e) {
3533          throw new RuntimeException(e);
3534        }
3535      }
3536    }
3537
3538    @Override
3539    protected boolean setInMemoryCompactionFlag() {
3540      boolean result = super.setInMemoryCompactionFlag();
3541      assertTrue(result);
3542      setInMemoryCompactionFlagCounter.incrementAndGet();
3543      return result;
3544    }
3545
3546    @Override
3547    void inMemoryCompaction() {
3548      try {
3549        super.inMemoryCompaction();
3550      } finally {
3551        try {
3552          inMemoryCompactionEndCyclicBarrier.await();
3553        } catch (Throwable e) {
3554          throw new RuntimeException(e);
3555        }
3556
3557      }
3558    }
3559
3560  }
3561
3562  public static class MyCompactingMemStore5 extends CompactingMemStore {
3563    private static final String TAKE_SNAPSHOT_THREAD_NAME = "takeSnapShotThread";
3564    private static final String WRITE_AGAIN_THREAD_NAME = "writeAgainThread";
3565    /**
3566     * {@link CompactingMemStore#flattenOneSegment} must execute after
3567     * {@link CompactingMemStore#getImmutableSegments}
3568     */
3569    private final CyclicBarrier flattenOneSegmentPreCyclicBarrier = new CyclicBarrier(2);
3570    /**
3571     * Only after {@link CompactingMemStore#flattenOneSegment} completed,
3572     * {@link CompactingMemStore#swapPipelineWithNull} could execute.
3573     */
3574    private final CyclicBarrier flattenOneSegmentPostCyclicBarrier = new CyclicBarrier(2);
3575    /**
3576     * Only the in memory compact thread enters {@link CompactingMemStore#flattenOneSegment},the
3577     * snapshot thread starts {@link CompactingMemStore#snapshot},because
3578     * {@link CompactingMemStore#snapshot} would invoke {@link CompactingMemStore#stopCompaction}.
3579     */
3580    private final CyclicBarrier snapShotStartCyclicCyclicBarrier = new CyclicBarrier(2);
3581    /**
3582     * To wait for {@link CompactingMemStore.InMemoryCompactionRunnable} stopping.
3583     */
3584    private final CyclicBarrier inMemoryCompactionEndCyclicBarrier = new CyclicBarrier(2);
3585    private final AtomicInteger getImmutableSegmentsListCounter = new AtomicInteger(0);
3586    private final AtomicInteger swapPipelineWithNullCounter = new AtomicInteger(0);
3587    private final AtomicInteger flattenOneSegmentCounter = new AtomicInteger(0);
3588    private final AtomicInteger setInMemoryCompactionFlagCounter = new AtomicInteger(0);
3589    /**
3590     * Only the snapshot thread retry {@link CompactingMemStore#swapPipelineWithNull}, writeAgain
3591     * thread could start.
3592     */
3593    private final CyclicBarrier writeMemStoreAgainStartCyclicBarrier = new CyclicBarrier(2);
3594    /**
3595     * This is used for snapshot thread,writeAgain thread and in memory compact thread. Only the
3596     * writeAgain thread completes, {@link CompactingMemStore#swapPipelineWithNull} would
3597     * execute,and in memory compact thread would exit,because we expect that in memory compact
3598     * executing only once.
3599     */
3600    private final CyclicBarrier writeMemStoreAgainEndCyclicBarrier = new CyclicBarrier(3);
3601
3602    public MyCompactingMemStore5(Configuration conf, CellComparatorImpl cellComparator,
3603      HStore store, RegionServicesForStores regionServices, MemoryCompactionPolicy compactionPolicy)
3604      throws IOException {
3605      super(conf, cellComparator, store, regionServices, compactionPolicy);
3606    }
3607
3608    @Override
3609    public VersionedSegmentsList getImmutableSegments() {
3610      VersionedSegmentsList result = super.getImmutableSegments();
3611      if (Thread.currentThread().getName().equals(TAKE_SNAPSHOT_THREAD_NAME)) {
3612        int currentCount = getImmutableSegmentsListCounter.incrementAndGet();
3613        if (currentCount <= 1) {
3614          try {
3615            flattenOneSegmentPreCyclicBarrier.await();
3616          } catch (Throwable e) {
3617            throw new RuntimeException(e);
3618          }
3619        }
3620
3621      }
3622
3623      return result;
3624    }
3625
3626    @Override
3627    protected boolean swapPipelineWithNull(VersionedSegmentsList segments) {
3628      if (Thread.currentThread().getName().equals(TAKE_SNAPSHOT_THREAD_NAME)) {
3629        int currentCount = swapPipelineWithNullCounter.incrementAndGet();
3630        if (currentCount <= 1) {
3631          try {
3632            flattenOneSegmentPostCyclicBarrier.await();
3633          } catch (Throwable e) {
3634            throw new RuntimeException(e);
3635          }
3636        }
3637
3638        if (currentCount == 2) {
3639          try {
3640            /**
3641             * Only the snapshot thread retry {@link CompactingMemStore#swapPipelineWithNull},
3642             * writeAgain thread could start.
3643             */
3644            writeMemStoreAgainStartCyclicBarrier.await();
3645            /**
3646             * Only the writeAgain thread completes, retry
3647             * {@link CompactingMemStore#swapPipelineWithNull} would execute.
3648             */
3649            writeMemStoreAgainEndCyclicBarrier.await();
3650          } catch (Throwable e) {
3651            throw new RuntimeException(e);
3652          }
3653        }
3654
3655      }
3656      boolean result = super.swapPipelineWithNull(segments);
3657      if (Thread.currentThread().getName().equals(TAKE_SNAPSHOT_THREAD_NAME)) {
3658        int currentCount = swapPipelineWithNullCounter.get();
3659        if (currentCount <= 1) {
3660          assertTrue(!result);
3661        }
3662        if (currentCount == 2) {
3663          assertTrue(result);
3664        }
3665      }
3666      return result;
3667
3668    }
3669
3670    @Override
3671    public void flattenOneSegment(long requesterVersion, Action action) {
3672      int currentCount = flattenOneSegmentCounter.incrementAndGet();
3673      if (currentCount <= 1) {
3674        try {
3675          /**
3676           * {@link CompactingMemStore#snapshot} could start.
3677           */
3678          snapShotStartCyclicCyclicBarrier.await();
3679          flattenOneSegmentPreCyclicBarrier.await();
3680        } catch (Throwable e) {
3681          throw new RuntimeException(e);
3682        }
3683      }
3684      super.flattenOneSegment(requesterVersion, action);
3685      if (currentCount <= 1) {
3686        try {
3687          flattenOneSegmentPostCyclicBarrier.await();
3688          /**
3689           * Only the writeAgain thread completes, in memory compact thread would exit,because we
3690           * expect that in memory compact executing only once.
3691           */
3692          writeMemStoreAgainEndCyclicBarrier.await();
3693        } catch (Throwable e) {
3694          throw new RuntimeException(e);
3695        }
3696
3697      }
3698    }
3699
3700    @Override
3701    protected boolean setInMemoryCompactionFlag() {
3702      boolean result = super.setInMemoryCompactionFlag();
3703      int count = setInMemoryCompactionFlagCounter.incrementAndGet();
3704      if (count <= 1) {
3705        assertTrue(result);
3706      }
3707      if (count == 2) {
3708        assertTrue(!result);
3709      }
3710      return result;
3711    }
3712
3713    @Override
3714    void inMemoryCompaction() {
3715      try {
3716        super.inMemoryCompaction();
3717      } finally {
3718        try {
3719          inMemoryCompactionEndCyclicBarrier.await();
3720        } catch (Throwable e) {
3721          throw new RuntimeException(e);
3722        }
3723
3724      }
3725    }
3726  }
3727
3728  public static class MyCompactingMemStore6 extends CompactingMemStore {
3729    private final CyclicBarrier inMemoryCompactionEndCyclicBarrier = new CyclicBarrier(2);
3730
3731    public MyCompactingMemStore6(Configuration conf, CellComparatorImpl cellComparator,
3732      HStore store, RegionServicesForStores regionServices, MemoryCompactionPolicy compactionPolicy)
3733      throws IOException {
3734      super(conf, cellComparator, store, regionServices, compactionPolicy);
3735    }
3736
3737    @Override
3738    void inMemoryCompaction() {
3739      try {
3740        super.inMemoryCompaction();
3741      } finally {
3742        try {
3743          inMemoryCompactionEndCyclicBarrier.await();
3744        } catch (Throwable e) {
3745          throw new RuntimeException(e);
3746        }
3747
3748      }
3749    }
3750  }
3751
3752  public static class MyDefaultMemStore extends DefaultMemStore {
3753    private static final String GET_SCANNER_THREAD_NAME = "getScannerMyThread";
3754    private static final String FLUSH_THREAD_NAME = "flushMyThread";
3755    /**
3756     * Only when flush thread enters {@link DefaultMemStore#doClearSnapShot}, getScanner thread
3757     * could start.
3758     */
3759    private final CyclicBarrier getScannerCyclicBarrier = new CyclicBarrier(2);
3760    /**
3761     * Used by getScanner thread notifies flush thread {@link DefaultMemStore#getSnapshotSegments}
3762     * completed, {@link DefaultMemStore#doClearSnapShot} could continue.
3763     */
3764    private final CyclicBarrier preClearSnapShotCyclicBarrier = new CyclicBarrier(2);
3765    /**
3766     * Used by flush thread notifies getScanner thread {@link DefaultMemStore#doClearSnapShot}
3767     * completed, {@link DefaultMemStore#getScanners} could continue.
3768     */
3769    private final CyclicBarrier postClearSnapShotCyclicBarrier = new CyclicBarrier(2);
3770    private final AtomicInteger getSnapshotSegmentsCounter = new AtomicInteger(0);
3771    private final AtomicInteger clearSnapshotCounter = new AtomicInteger(0);
3772    private volatile boolean shouldWait = true;
3773    private volatile HStore store = null;
3774
3775    public MyDefaultMemStore(Configuration conf, CellComparator cellComparator,
3776      RegionServicesForStores regionServices) throws IOException {
3777      super(conf, cellComparator, regionServices);
3778    }
3779
3780    @Override
3781    protected List<Segment> getSnapshotSegments() {
3782
3783      List<Segment> result = super.getSnapshotSegments();
3784
3785      if (Thread.currentThread().getName().equals(GET_SCANNER_THREAD_NAME)) {
3786        int currentCount = getSnapshotSegmentsCounter.incrementAndGet();
3787        if (currentCount == 1) {
3788          if (this.shouldWait) {
3789            try {
3790              /**
3791               * Notify flush thread {@link DefaultMemStore#getSnapshotSegments} completed,
3792               * {@link DefaultMemStore#doClearSnapShot} could continue.
3793               */
3794              preClearSnapShotCyclicBarrier.await();
3795              /**
3796               * Wait for {@link DefaultMemStore#doClearSnapShot} completed.
3797               */
3798              postClearSnapShotCyclicBarrier.await();
3799
3800            } catch (Throwable e) {
3801              throw new RuntimeException(e);
3802            }
3803          }
3804        }
3805      }
3806      return result;
3807    }
3808
3809    @Override
3810    protected void doClearSnapShot() {
3811      if (Thread.currentThread().getName().equals(FLUSH_THREAD_NAME)) {
3812        int currentCount = clearSnapshotCounter.incrementAndGet();
3813        if (currentCount == 1) {
3814          try {
3815            if (
3816              ((ReentrantReadWriteLock) store.getStoreEngine().getLock())
3817                .isWriteLockedByCurrentThread()
3818            ) {
3819              shouldWait = false;
3820            }
3821            /**
3822             * Only when flush thread enters {@link DefaultMemStore#doClearSnapShot}, getScanner
3823             * thread could start.
3824             */
3825            getScannerCyclicBarrier.await();
3826
3827            if (shouldWait) {
3828              /**
3829               * Wait for {@link DefaultMemStore#getSnapshotSegments} completed.
3830               */
3831              preClearSnapShotCyclicBarrier.await();
3832            }
3833          } catch (Throwable e) {
3834            throw new RuntimeException(e);
3835          }
3836        }
3837      }
3838      super.doClearSnapShot();
3839
3840      if (Thread.currentThread().getName().equals(FLUSH_THREAD_NAME)) {
3841        int currentCount = clearSnapshotCounter.get();
3842        if (currentCount == 1) {
3843          if (shouldWait) {
3844            try {
3845              /**
3846               * Notify getScanner thread {@link DefaultMemStore#doClearSnapShot} completed,
3847               * {@link DefaultMemStore#getScanners} could continue.
3848               */
3849              postClearSnapShotCyclicBarrier.await();
3850            } catch (Throwable e) {
3851              throw new RuntimeException(e);
3852            }
3853          }
3854        }
3855      }
3856    }
3857  }
3858}