Package org.apache.hadoop.hbase.regionserver

Class TestHStore

java.lang.Object

org.apache.hadoop.hbase.regionserver.TestHStore

public class TestHStore
extends Object

Test class for the HStore

Nested Class Summary

Nested Classes

Modifier and Type	Class	Description
(package private) static class	TestHStore.CustomDefaultMemStore
static class	TestHStore.DummyStoreEngine
(package private) static class	TestHStore.FaultyFileSystem	Faulty file system that will fail if you write past its fault position the FIRST TIME only; thereafter it will succeed.
(package private) static class	TestHStore.FaultyOutputStream
static class	TestHStore.MyCompactingMemStore
static class	TestHStore.MyCompactingMemStore2
static class	TestHStore.MyCompactingMemStore3
static class	TestHStore.MyCompactingMemStore4
static class	TestHStore.MyCompactingMemStore5
static class	TestHStore.MyCompactingMemStore6
static class	TestHStore.MyCompactingMemStoreWithCustomCompactor
static class	TestHStore.MyDefaultMemStore
(package private) static class	TestHStore.MyDefaultMemStore1
static class	TestHStore.MyDefaultStoreFlusher
private static class	TestHStore.MyList<T>
(package private) static interface	TestHStore.MyListHook
private static class	TestHStore.MyMemStoreCompactor
private static class	TestHStore.MyStore
private static class	TestHStore.MyStoreHook
private static class	TestHStore.MyThread

Field Summary

Fields

Modifier and Type	Field	Description
private static final int	BLOCKSIZE_SMALL
static final HBaseClassTestRule	CLASS_RULE
private static final String	DIR
(package private) List<org.apache.hadoop.hbase.Cell>	expected
(package private) byte[]	family
(package private) org.apache.hadoop.hbase.client.Get	get
(package private) long	id
private static final org.slf4j.Logger	LOG
org.junit.rules.TestName	name
(package private) byte[]	qf1
(package private) byte[]	qf2
(package private) byte[]	qf3
(package private) byte[]	qf4
(package private) byte[]	qf5
(package private) byte[]	qf6
(package private) NavigableSet<byte[]>	qualifiers
(package private) org.apache.hadoop.hbase.regionserver.HRegion	region
(package private) List<org.apache.hadoop.hbase.Cell>	result
(package private) byte[]	row
(package private) byte[]	row2
(package private) org.apache.hadoop.hbase.regionserver.HStore	store
(package private) byte[]	table
private static final HBaseTestingUtility	TEST_UTIL

Constructor Summary

Constructors

Constructor	Description
TestHStore()

Method Summary

Modifier and Type	Method	Description
private void	addStoreFile()
private void	archiveStoreFile(int index)
private void	assertCheck()
private void	closeCompactedFile(int index)
private long	countMemStoreScanner(org.apache.hadoop.hbase.regionserver.StoreScanner scanner)
private org.apache.hadoop.hbase.Cell	createCell(byte[] row, byte[] qualifier, long ts, long sequenceId, byte[] value)
private org.apache.hadoop.hbase.Cell	createCell(byte[] qualifier, long ts, long sequenceId, byte[] value)
private void	doWriteTestLargeCellAndSmallCellConcurrently(IntBinaryOperator getFlushByteSize)
private void	flush(int storeFilessize)
private static org.apache.hadoop.hbase.regionserver.StoreFlushContext	flushStore(org.apache.hadoop.hbase.regionserver.HStore store, long id)
private List<org.apache.hadoop.hbase.Cell>	getKeyValueSet(long[] timestamps, int numRows, byte[] qualifier, byte[] family)	Generate a list of KeyValues for testing based on given parameters
private static long	getLowestTimeStampFromFS(org.apache.hadoop.fs.FileSystem fs, Collection<org.apache.hadoop.hbase.regionserver.HStoreFile> candidates)
private <T> T	getTypeKeyValueScanner(org.apache.hadoop.hbase.regionserver.StoreScanner storeScanner, Class<T> keyValueScannerClass)
private void	init(String methodName)
private org.apache.hadoop.hbase.regionserver.HStore	init(String methodName, org.apache.hadoop.conf.Configuration conf)
private org.apache.hadoop.hbase.regionserver.HStore	init(String methodName, org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.hbase.client.ColumnFamilyDescriptor hcd)
private org.apache.hadoop.hbase.regionserver.HStore	init(String methodName, org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.hbase.client.TableDescriptorBuilder builder, org.apache.hadoop.hbase.client.ColumnFamilyDescriptor hcd)
private org.apache.hadoop.hbase.regionserver.HStore	init(String methodName, org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.hbase.client.TableDescriptorBuilder builder, org.apache.hadoop.hbase.client.ColumnFamilyDescriptor hcd, TestHStore.MyStoreHook hook)
private org.apache.hadoop.hbase.regionserver.HStore	init(String methodName, org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.hbase.client.TableDescriptorBuilder builder, org.apache.hadoop.hbase.client.ColumnFamilyDescriptor hcd, TestHStore.MyStoreHook hook, boolean switchToPread)
private void	initHRegion(String methodName, org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.hbase.client.TableDescriptorBuilder builder, org.apache.hadoop.hbase.client.ColumnFamilyDescriptor hcd, TestHStore.MyStoreHook hook, boolean switchToPread)
private TestHStore.MyStore	initMyStore(String methodName, org.apache.hadoop.conf.Configuration conf, TestHStore.MyStoreHook hook)
private org.apache.hadoop.hbase.regionserver.HStoreFile	mockStoreFile(long createdTime)
private org.apache.hadoop.hbase.regionserver.HStoreFile	mockStoreFileWithLength(long length)
void	setUp()
void	tearDown()
static void	tearDownAfterClass()
void	testAge()
void	testClearSnapshotGetScannerConcurrently()	This test is for HBASE-26465, test AbstractMemStore.clearSnapshot(long) and DefaultMemStore.getScanners(long) execute concurrently.
void	testCompactingMemStoreCellExceedInmemoryFlushSize()
void	testCompactingMemStoreNoCellButDataSizeExceedsInmemoryFlushSize()
void	testCompactingMemStoreWriteLargeCellAndSmallCellConcurrently()
void	testCreateScannerAndSnapshotConcurrently()
void	testCreateWriter()	Verify that compression and data block encoding are respected by the createWriter method, used on store flush.
void	testDeleteExpiredStoreFiles()
void	testDeleteExpiredStoreFiles(int minVersions)
void	testEmptyStoreFile()	Test for hbase-1686.
void	testExtendsDefaultMemStore()	This test is for HBASE-26476
void	testFlattenAndSnapshotCompactingMemStoreConcurrently()	This test is for HBASE-26384, test CompactingMemStore.flattenOneSegment(long, org.apache.hadoop.hbase.regionserver.MemStoreCompactionStrategy.Action) and CompactingMemStore.snapshot() execute concurrently.
void	testFlattenSnapshotWriteCompactingMemeStoreConcurrently()	This test is for HBASE-26384, test CompactingMemStore.flattenOneSegment(long, org.apache.hadoop.hbase.regionserver.MemStoreCompactionStrategy.Action)CompactingMemStore.snapshot() and writeMemStore execute concurrently.
private void	testFlushBeforeCompletingScan(TestHStore.MyListHook hook, org.apache.hadoop.hbase.filter.Filter filter, int expectedSize)
void	testFlushBeforeCompletingScanWithFilter()
void	testFlushBeforeCompletingScanWithFilterHint()
void	testFlushBeforeCompletingScanWoFilter()
void	testFlushSizeSizing()	Test we do not lose data if we fail a flush and then close.
void	testForceCloneOfBigCellForCellChunkImmutableSegment()	This test is for HBASE-27464, before this JIRA,when init CellChunkImmutableSegment for 'COMPACT' action, we not force copy to current MSLab.
void	testGet_FromFilesOnly()	Getting data from files only
void	testGet_FromMemStoreAndFiles()	Getting data from memstore and files
void	testGet_FromMemStoreOnly()	Getting data from memstore only
void	testHandleErrorsInFlush()
void	testHFileContextSetWithCFAndTable()
void	testImmutableMemStoreLABRefCnt()	This test is for HBASE-26494, test the RefCnt behaviors in ImmutableMemStoreLAB
void	testInMemoryCompactionTypeWithLowerCase()
void	testLowestModificationTime()
void	testMaxPreadBytesConfiguredToBeLessThanZero()
void	testMemoryLeakWhenFlushMemStoreRetrying()	This test is for HBASE-26488
void	testMultipleTimestamps()	Test to ensure correctness when using Stores with multiple timestamps
void	testNumberOfMemStoreScannersAfterFlush()
private void	testNumberOfMemStoreScannersAfterFlush(List<org.apache.hadoop.hbase.Cell> inputCellsBeforeSnapshot, List<org.apache.hadoop.hbase.Cell> inputCellsAfterSnapshot)
void	testOnConfigurationChange()
void	testReclaimChunkWhenScaning()
void	testRefreshStoreFiles()
void	testRefreshStoreFilesNotChanged()
void	testRunDoubleMemStoreCompactors()	If there are two running InMemoryFlushRunnable, the later InMemoryFlushRunnable may change the versionedList.
void	testScanWithCompactionAfterFlush()
void	testScanWithDoubleFlush()
void	testSpaceQuotaChangeAfterReplacement()
void	testSplitWithEmptyColFam()	Test for HBASE-3492 - Test split on empty colfam (no store files).
void	testStoreBloomFilterMetricsWithBloomNone()
void	testStoreBloomFilterMetricsWithBloomRow()
void	testStoreBloomFilterMetricsWithBloomRowCol()
void	testStoreBloomFilterMetricsWithBloomRowPrefix()
void	testStoreScannerUpdateReadersWhenFlushAndCompactConcurrently()	This test is for HBASE-27519, when the StoreScanner is scanning,the Flush and the Compaction execute concurrently and theCcompaction compact and archive the flushed HStoreFile which is used by StoreScanner.updateReaders(java.util.List<org.apache.hadoop.hbase.regionserver.HStoreFile>, java.util.List<org.apache.hadoop.hbase.regionserver.KeyValueScanner>).Before HBASE-27519,StoreScanner.updateReaders would throw FileNotFoundException.
void	testStoreUsesConfigurationFromHcdAndHtd()
void	testStoreUsesSearchEngineOverride()
void	testSwitchingPreadtoStreamParallelyWithCompactionDischarger()
void	testTimeRangeIfSomeCellsAreDroppedInFlush()
private void	testTimeRangeIfSomeCellsAreDroppedInFlush(int maxVersion)
private void	writeAndRead(org.apache.hadoop.hbase.regionserver.BloomType bloomType, int numStoreFiles)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Details

CLASS_RULE

public static final HBaseClassTestRule CLASS_RULE

LOG

private static final org.slf4j.Logger LOG

name

public org.junit.rules.TestName name

region

org.apache.hadoop.hbase.regionserver.HRegion region

store

org.apache.hadoop.hbase.regionserver.HStore store

table

byte[] table

family

byte[] family

row

byte[] row

row2

byte[] row2

qf1

byte[] qf1

qf2

byte[] qf2

qf3

byte[] qf3

qf4

byte[] qf4

qf5

byte[] qf5

qf6

byte[] qf6

qualifiers

NavigableSet<byte[]> qualifiers

expected

List<org.apache.hadoop.hbase.Cell> expected

result

List<org.apache.hadoop.hbase.Cell> result

id

long id

get

org.apache.hadoop.hbase.client.Get get

TEST_UTIL

private static final HBaseTestingUtility TEST_UTIL

DIR

private static final String DIR

BLOCKSIZE_SMALL

private static final int BLOCKSIZE_SMALL

See Also:

• Constant Field Values

Constructor Details

TestHStore

public TestHStore()

Method Details

setUp

public void setUp()
           throws IOException

Throws:

IOException

init

private void init(String methodName)
           throws IOException

Throws:

IOException

init

private org.apache.hadoop.hbase.regionserver.HStore init(String methodName,
 org.apache.hadoop.conf.Configuration conf)
                                                  throws IOException

Throws:

IOException

init

private org.apache.hadoop.hbase.regionserver.HStore init(String methodName,
 org.apache.hadoop.conf.Configuration conf,
 org.apache.hadoop.hbase.client.ColumnFamilyDescriptor hcd)
                                                  throws IOException

Throws:

IOException

init

private org.apache.hadoop.hbase.regionserver.HStore init(String methodName,
 org.apache.hadoop.conf.Configuration conf,
 org.apache.hadoop.hbase.client.TableDescriptorBuilder builder,
 org.apache.hadoop.hbase.client.ColumnFamilyDescriptor hcd)
                                                  throws IOException

Throws:

IOException

init

private org.apache.hadoop.hbase.regionserver.HStore init(String methodName,
 org.apache.hadoop.conf.Configuration conf,
 org.apache.hadoop.hbase.client.TableDescriptorBuilder builder,
 org.apache.hadoop.hbase.client.ColumnFamilyDescriptor hcd,
 TestHStore.MyStoreHook hook)
                                                  throws IOException

Throws:

IOException

initHRegion

private void initHRegion(String methodName,
 org.apache.hadoop.conf.Configuration conf,
 org.apache.hadoop.hbase.client.TableDescriptorBuilder builder,
 org.apache.hadoop.hbase.client.ColumnFamilyDescriptor hcd,
 TestHStore.MyStoreHook hook,
 boolean switchToPread)
                  throws IOException

Throws:

IOException

init

private org.apache.hadoop.hbase.regionserver.HStore init(String methodName,
 org.apache.hadoop.conf.Configuration conf,
 org.apache.hadoop.hbase.client.TableDescriptorBuilder builder,
 org.apache.hadoop.hbase.client.ColumnFamilyDescriptor hcd,
 TestHStore.MyStoreHook hook,
 boolean switchToPread)
                                                  throws IOException

Throws:

IOException

testFlushSizeSizing

public void testFlushSizeSizing()
                         throws Exception

Test we do not lose data if we fail a flush and then close. Part of HBase-10466

Throws:

Exception

testStoreBloomFilterMetricsWithBloomRowCol

public void testStoreBloomFilterMetricsWithBloomRowCol()
                                                throws IOException

Throws:

IOException

testStoreBloomFilterMetricsWithBloomRow

public void testStoreBloomFilterMetricsWithBloomRow()
                                             throws IOException

Throws:

IOException

testStoreBloomFilterMetricsWithBloomRowPrefix

public void testStoreBloomFilterMetricsWithBloomRowPrefix()
                                                   throws IOException

Throws:

IOException

testStoreBloomFilterMetricsWithBloomNone

public void testStoreBloomFilterMetricsWithBloomNone()
                                              throws IOException

Throws:

IOException

writeAndRead

private void writeAndRead(org.apache.hadoop.hbase.regionserver.BloomType bloomType,
 int numStoreFiles)
                   throws IOException

Throws:

IOException

testCreateWriter

public void testCreateWriter()
                      throws Exception

Verify that compression and data block encoding are respected by the createWriter method, used on store flush.

Throws:

Exception

testDeleteExpiredStoreFiles

public void testDeleteExpiredStoreFiles()
                                 throws Exception

Throws:

Exception

testDeleteExpiredStoreFiles

public void testDeleteExpiredStoreFiles(int minVersions)
                                 throws Exception

Parameters:

minVersions - the MIN_VERSIONS for the column family

Throws:

Exception

testLowestModificationTime

public void testLowestModificationTime()
                                throws Exception

Throws:

Exception

getLowestTimeStampFromFS

private static long getLowestTimeStampFromFS(org.apache.hadoop.fs.FileSystem fs,
 Collection<org.apache.hadoop.hbase.regionserver.HStoreFile> candidates)
                                      throws IOException

Throws:

IOException

testEmptyStoreFile

public void testEmptyStoreFile()
                        throws IOException

Test for hbase-1686.

Throws:

IOException

testGet_FromMemStoreOnly

public void testGet_FromMemStoreOnly()
                              throws IOException

Getting data from memstore only

Throws:

IOException

testTimeRangeIfSomeCellsAreDroppedInFlush

public void testTimeRangeIfSomeCellsAreDroppedInFlush()
                                               throws IOException

Throws:

IOException

testTimeRangeIfSomeCellsAreDroppedInFlush

private void testTimeRangeIfSomeCellsAreDroppedInFlush(int maxVersion)
                                                throws IOException

Throws:

IOException

testGet_FromFilesOnly

public void testGet_FromFilesOnly()
                           throws IOException

Getting data from files only

Throws:

IOException

testGet_FromMemStoreAndFiles

public void testGet_FromMemStoreAndFiles()
                                  throws IOException

Getting data from memstore and files

Throws:

IOException

flush

private void flush(int storeFilessize)
            throws IOException

Throws:

IOException

assertCheck

private void assertCheck()

tearDown

public void tearDown()
              throws Exception

Throws:

Exception

tearDownAfterClass

public static void tearDownAfterClass()
                               throws IOException

Throws:

IOException

testHandleErrorsInFlush

public void testHandleErrorsInFlush()
                             throws Exception

Throws:

Exception

flushStore

private static org.apache.hadoop.hbase.regionserver.StoreFlushContext flushStore(org.apache.hadoop.hbase.regionserver.HStore store,
 long id)
                                                                          throws IOException

Throws:

IOException

getKeyValueSet

private List<org.apache.hadoop.hbase.Cell> getKeyValueSet(long[] timestamps,
 int numRows,
 byte[] qualifier,
 byte[] family)

Generate a list of KeyValues for testing based on given parameters

Returns:

the rows key-value list

testMultipleTimestamps

public void testMultipleTimestamps()
                            throws IOException

Test to ensure correctness when using Stores with multiple timestamps

Throws:

IOException

testSplitWithEmptyColFam

public void testSplitWithEmptyColFam()
                              throws IOException

Test for HBASE-3492 - Test split on empty colfam (no store files).

Throws:

IOException - When the IO operations fail.

testStoreUsesConfigurationFromHcdAndHtd

public void testStoreUsesConfigurationFromHcdAndHtd()
                                             throws Exception

Throws:

Exception

testStoreUsesSearchEngineOverride

public void testStoreUsesSearchEngineOverride()
                                       throws Exception

Throws:

Exception

addStoreFile

private void addStoreFile()
                   throws IOException

Throws:

IOException

archiveStoreFile

private void archiveStoreFile(int index)
                       throws IOException

Throws:

IOException

closeCompactedFile

private void closeCompactedFile(int index)
                         throws IOException

Throws:

IOException

testRefreshStoreFiles

public void testRefreshStoreFiles()
                           throws Exception

Throws:

Exception

testRefreshStoreFilesNotChanged

public void testRefreshStoreFilesNotChanged()
                                     throws IOException

Throws:

IOException

testScanWithCompactionAfterFlush

public void testScanWithCompactionAfterFlush()
                                      throws Exception

Throws:

Exception

countMemStoreScanner

private long countMemStoreScanner(org.apache.hadoop.hbase.regionserver.StoreScanner scanner)

testNumberOfMemStoreScannersAfterFlush

public void testNumberOfMemStoreScannersAfterFlush()
                                            throws IOException

Throws:

IOException

testNumberOfMemStoreScannersAfterFlush

private void testNumberOfMemStoreScannersAfterFlush(List<org.apache.hadoop.hbase.Cell> inputCellsBeforeSnapshot,
 List<org.apache.hadoop.hbase.Cell> inputCellsAfterSnapshot)
                                             throws IOException

Throws:

IOException

createCell

private org.apache.hadoop.hbase.Cell createCell(byte[] qualifier,
 long ts,
 long sequenceId,
 byte[] value)
                                         throws IOException

Throws:

IOException

createCell

private org.apache.hadoop.hbase.Cell createCell(byte[] row,
 byte[] qualifier,
 long ts,
 long sequenceId,
 byte[] value)
                                         throws IOException

Throws:

IOException

testFlushBeforeCompletingScanWoFilter

public void testFlushBeforeCompletingScanWoFilter()
                                           throws IOException,
InterruptedException

Throws:

IOException

InterruptedException

testFlushBeforeCompletingScanWithFilter

public void testFlushBeforeCompletingScanWithFilter()
                                             throws IOException,
InterruptedException

Throws:

IOException

InterruptedException

testFlushBeforeCompletingScanWithFilterHint

public void testFlushBeforeCompletingScanWithFilterHint()
                                                 throws IOException,
InterruptedException

Throws:

IOException

InterruptedException

testFlushBeforeCompletingScan

private void testFlushBeforeCompletingScan(TestHStore.MyListHook hook,
 org.apache.hadoop.hbase.filter.Filter filter,
 int expectedSize)
                                    throws IOException,
InterruptedException

Throws:

IOException

InterruptedException

testCreateScannerAndSnapshotConcurrently

public void testCreateScannerAndSnapshotConcurrently()
                                              throws IOException,
InterruptedException

Throws:

IOException

InterruptedException

testScanWithDoubleFlush

public void testScanWithDoubleFlush()
                             throws IOException

Throws:

IOException

testStoreScannerUpdateReadersWhenFlushAndCompactConcurrently

public void testStoreScannerUpdateReadersWhenFlushAndCompactConcurrently()
                                                                  throws IOException

This test is for HBASE-27519, when the StoreScanner is scanning,the Flush and the Compaction execute concurrently and theCcompaction compact and archive the flushed HStoreFile which is used by StoreScanner.updateReaders(java.util.List<org.apache.hadoop.hbase.regionserver.HStoreFile>, java.util.List<org.apache.hadoop.hbase.regionserver.KeyValueScanner>).Before HBASE-27519,StoreScanner.updateReaders would throw FileNotFoundException.

Throws:

IOException

testReclaimChunkWhenScaning

public void testReclaimChunkWhenScaning()
                                 throws IOException

Throws:

IOException

testRunDoubleMemStoreCompactors

public void testRunDoubleMemStoreCompactors()
                                     throws IOException,
InterruptedException

If there are two running InMemoryFlushRunnable, the later InMemoryFlushRunnable may change the versionedList. And the first InMemoryFlushRunnable will use the chagned versionedList to remove the corresponding segments. In short, there will be some segements which isn't in merge are removed.

Throws:

IOException

InterruptedException

testAge

public void testAge()
             throws IOException

Throws:

IOException

mockStoreFile

private org.apache.hadoop.hbase.regionserver.HStoreFile mockStoreFile(long createdTime)

initMyStore

private TestHStore.MyStore initMyStore(String methodName,
 org.apache.hadoop.conf.Configuration conf,
 TestHStore.MyStoreHook hook)
                                throws IOException

Throws:

IOException

testSwitchingPreadtoStreamParallelyWithCompactionDischarger

public void testSwitchingPreadtoStreamParallelyWithCompactionDischarger()
                                                                 throws Exception

Throws:

Exception

testMaxPreadBytesConfiguredToBeLessThanZero

public void testMaxPreadBytesConfiguredToBeLessThanZero()
                                                 throws Exception

Throws:

Exception

testInMemoryCompactionTypeWithLowerCase

public void testInMemoryCompactionTypeWithLowerCase()
                                             throws IOException,
InterruptedException

Throws:

IOException

InterruptedException

testSpaceQuotaChangeAfterReplacement

public void testSpaceQuotaChangeAfterReplacement()
                                          throws IOException

Throws:

IOException

testHFileContextSetWithCFAndTable

public void testHFileContextSetWithCFAndTable()
                                       throws Exception

Throws:

Exception

testCompactingMemStoreNoCellButDataSizeExceedsInmemoryFlushSize

public void testCompactingMemStoreNoCellButDataSizeExceedsInmemoryFlushSize()
                                                                     throws IOException,
InterruptedException

Throws:

IOException

InterruptedException

testCompactingMemStoreCellExceedInmemoryFlushSize

public void testCompactingMemStoreCellExceedInmemoryFlushSize()
                                                       throws Exception

Throws:

Exception

testForceCloneOfBigCellForCellChunkImmutableSegment

public void testForceCloneOfBigCellForCellChunkImmutableSegment()
                                                         throws Exception

This test is for HBASE-27464, before this JIRA,when init CellChunkImmutableSegment for 'COMPACT' action, we not force copy to current MSLab. When cell size bigger than MemStoreLABImpl.maxAlloc, cell will stay in previous chunk which will recycle after segment replace, and we may read wrong data when these chunk reused by others.

Throws:

Exception

testCompactingMemStoreWriteLargeCellAndSmallCellConcurrently

public void testCompactingMemStoreWriteLargeCellAndSmallCellConcurrently()
                                                                  throws IOException,
InterruptedException

Throws:

IOException

InterruptedException

doWriteTestLargeCellAndSmallCellConcurrently

private void doWriteTestLargeCellAndSmallCellConcurrently(IntBinaryOperator getFlushByteSize)
                                                   throws IOException,
InterruptedException

Throws:

IOException

InterruptedException

testFlattenAndSnapshotCompactingMemStoreConcurrently

public void testFlattenAndSnapshotCompactingMemStoreConcurrently()
                                                          throws Exception

 This test is for HBASE-26384,
 test CompactingMemStore.flattenOneSegment(long, org.apache.hadoop.hbase.regionserver.MemStoreCompactionStrategy.Action) and CompactingMemStore.snapshot()
 execute concurrently.
 The threads sequence before HBASE-26384 is(The bug only exists for branch-2,and I add UTs
 for both branch-2 and master):
 1. The CompactingMemStore size exceeds
    CompactingMemStore.getInmemoryFlushSize(),the write thread adds a new
    ImmutableSegment  to the head of CompactingMemStore.pipeline,and start a
    in memory compact thread to execute CompactingMemStore.inMemoryCompaction().
 2. The in memory compact thread starts and then stopping before
    CompactingMemStore.flattenOneSegment(long, org.apache.hadoop.hbase.regionserver.MemStoreCompactionStrategy.Action).
 3. The snapshot thread starts CompactingMemStore.snapshot() concurrently,after the
    snapshot thread executing CompactingMemStore.getImmutableSegments(),the in memory
    compact thread continues.
    Assuming VersionedSegmentsList.version returned from
    CompactingMemStore.getImmutableSegments() is v.
 4. The snapshot thread stopping before CompactingMemStore.swapPipelineWithNull(org.apache.hadoop.hbase.regionserver.VersionedSegmentsList).
 5. The in memory compact thread completes CompactingMemStore.flattenOneSegment(long, org.apache.hadoop.hbase.regionserver.MemStoreCompactionStrategy.Action),
    CompactionPipeline.version is still v.
 6. The snapshot thread continues CompactingMemStore.swapPipelineWithNull(org.apache.hadoop.hbase.regionserver.VersionedSegmentsList), and because
    CompactionPipeline.version is v, CompactingMemStore.swapPipelineWithNull(org.apache.hadoop.hbase.regionserver.VersionedSegmentsList)
    thinks it is successful and continue flushing,but the ImmutableSegment in
    CompactionPipeline has changed because
    CompactingMemStore.flattenOneSegment(long, org.apache.hadoop.hbase.regionserver.MemStoreCompactionStrategy.Action),so the ImmutableSegment is not
    removed in fact and still remaining in CompactionPipeline.

 After HBASE-26384, the 5-6 step is changed to following, which is expected behavior:
 5. The in memory compact thread completes CompactingMemStore.flattenOneSegment(long, org.apache.hadoop.hbase.regionserver.MemStoreCompactionStrategy.Action),
    CompactingMemStore.flattenOneSegment(long, org.apache.hadoop.hbase.regionserver.MemStoreCompactionStrategy.Action) change CompactionPipeline.version to
    v+1.
 6. The snapshot thread continues CompactingMemStore.swapPipelineWithNull(org.apache.hadoop.hbase.regionserver.VersionedSegmentsList), and because
    CompactionPipeline.version is v+1, CompactingMemStore.swapPipelineWithNull(org.apache.hadoop.hbase.regionserver.VersionedSegmentsList)
    failed and retry the while loop in CompactingMemStore.pushPipelineToSnapshot() once
    again, because there is no concurrent CompactingMemStore.inMemoryCompaction() now,
    CompactingMemStore.swapPipelineWithNull(org.apache.hadoop.hbase.regionserver.VersionedSegmentsList) succeeds.
 

Throws:

Exception

testFlattenSnapshotWriteCompactingMemeStoreConcurrently

public void testFlattenSnapshotWriteCompactingMemeStoreConcurrently()
                                                             throws Exception

 This test is for HBASE-26384,
 test CompactingMemStore.flattenOneSegment(long, org.apache.hadoop.hbase.regionserver.MemStoreCompactionStrategy.Action)CompactingMemStore.snapshot()
 and writeMemStore execute concurrently.
 The threads sequence before HBASE-26384 is(The bug only exists for branch-2,and I add UTs
 for both branch-2 and master):
 1. The CompactingMemStore size exceeds
    CompactingMemStore.getInmemoryFlushSize(),the write thread adds a new
    ImmutableSegment  to the head of CompactingMemStore.pipeline,and start a
    in memory compact thread to execute CompactingMemStore.inMemoryCompaction().
 2. The in memory compact thread starts and then stopping before
    CompactingMemStore.flattenOneSegment(long, org.apache.hadoop.hbase.regionserver.MemStoreCompactionStrategy.Action).
 3. The snapshot thread starts CompactingMemStore.snapshot() concurrently,after the
    snapshot thread executing CompactingMemStore.getImmutableSegments(),the in memory
    compact thread continues.
    Assuming VersionedSegmentsList.version returned from
    CompactingMemStore.getImmutableSegments() is v.
 4. The snapshot thread stopping before CompactingMemStore.swapPipelineWithNull(org.apache.hadoop.hbase.regionserver.VersionedSegmentsList).
 5. The in memory compact thread completes CompactingMemStore.flattenOneSegment(long, org.apache.hadoop.hbase.regionserver.MemStoreCompactionStrategy.Action),
    CompactionPipeline.version is still v.
 6. The snapshot thread continues CompactingMemStore.swapPipelineWithNull(org.apache.hadoop.hbase.regionserver.VersionedSegmentsList), and because
    CompactionPipeline.version is v, CompactingMemStore.swapPipelineWithNull(org.apache.hadoop.hbase.regionserver.VersionedSegmentsList)
    thinks it is successful and continue flushing,but the ImmutableSegment in
    CompactionPipeline has changed because
    CompactingMemStore.flattenOneSegment(long, org.apache.hadoop.hbase.regionserver.MemStoreCompactionStrategy.Action),so the ImmutableSegment is not
    removed in fact and still remaining in CompactionPipeline.

 After HBASE-26384, the 5-6 step is changed to following, which is expected behavior,
 and I add step 7-8 to test there is new segment added before retry.
 5. The in memory compact thread completes CompactingMemStore.flattenOneSegment(long, org.apache.hadoop.hbase.regionserver.MemStoreCompactionStrategy.Action),
    CompactingMemStore.flattenOneSegment(long, org.apache.hadoop.hbase.regionserver.MemStoreCompactionStrategy.Action) change CompactionPipeline.version to
     v+1.
 6. The snapshot thread continues CompactingMemStore.swapPipelineWithNull(org.apache.hadoop.hbase.regionserver.VersionedSegmentsList), and because
    CompactionPipeline.version is v+1, CompactingMemStore.swapPipelineWithNull(org.apache.hadoop.hbase.regionserver.VersionedSegmentsList)
    failed and retry,VersionedSegmentsList.version returned from
    CompactingMemStore.getImmutableSegments() is v+1.
 7. The write thread continues writing to CompactingMemStore and
    CompactingMemStore size exceeds CompactingMemStore.getInmemoryFlushSize(),
    CompactingMemStore.flushInMemory(MutableSegment) is called and a new
    ImmutableSegment is added to the head of CompactingMemStore.pipeline,
    CompactionPipeline.version is still v+1.
 8. The snapshot thread continues CompactingMemStore.swapPipelineWithNull(org.apache.hadoop.hbase.regionserver.VersionedSegmentsList), and because
    CompactionPipeline.version is still v+1,
    CompactingMemStore.swapPipelineWithNull(org.apache.hadoop.hbase.regionserver.VersionedSegmentsList) succeeds.The new ImmutableSegment
    remained at the head of CompactingMemStore.pipeline,the old is removed by
    CompactingMemStore.swapPipelineWithNull(org.apache.hadoop.hbase.regionserver.VersionedSegmentsList).
 

Throws:

Exception

testClearSnapshotGetScannerConcurrently

public void testClearSnapshotGetScannerConcurrently()
                                             throws Exception

 This test is for HBASE-26465,
 test AbstractMemStore.clearSnapshot(long) and DefaultMemStore.getScanners(long) execute
 concurrently. The threads sequence before HBASE-26465 is:
 1.The flush thread starts DefaultMemStore flushing after some cells have be added to
  DefaultMemStore.
 2.The flush thread stopping before AbstractMemStore.clearSnapshot(long) in
   HStore#updateStorefiles after completed flushing memStore to hfile.
 3.The scan thread starts and stopping after DefaultMemStore.getSnapshotSegments() in
   DefaultMemStore.getScanners(long),here the scan thread gets the
   DefaultMemStore.snapshot() which is created by the flush thread.
 4.The flush thread continues AbstractMemStore.clearSnapshot(long) and close
   DefaultMemStore.snapshot(),because the reference count of the corresponding
   MemStoreLABImpl is 0, the Chunks in corresponding MemStoreLABImpl
   are recycled.
 5.The scan thread continues DefaultMemStore.getScanners(long),and create a
   SegmentScanner for this DefaultMemStore.snapshot(), and increase the
   reference count of the corresponding MemStoreLABImpl, but Chunks in
   corresponding MemStoreLABImpl are recycled by step 4, and these Chunks may
   be overwritten by other write threads,which may cause serious problem.
 After HBASE-26465,DefaultMemStore.getScanners(long) and
 AbstractMemStore.clearSnapshot(long) could not execute concurrently.
 

Throws:

Exception

getTypeKeyValueScanner

private <T> T getTypeKeyValueScanner(org.apache.hadoop.hbase.regionserver.StoreScanner storeScanner,
 Class<T> keyValueScannerClass)

testOnConfigurationChange

public void testOnConfigurationChange()
                               throws IOException

Throws:

IOException

testExtendsDefaultMemStore

public void testExtendsDefaultMemStore()
                                throws Exception

This test is for HBASE-26476

Throws:

Exception

testMemoryLeakWhenFlushMemStoreRetrying

public void testMemoryLeakWhenFlushMemStoreRetrying()
                                             throws Exception

This test is for HBASE-26488

Throws:

Exception

testImmutableMemStoreLABRefCnt

public void testImmutableMemStoreLABRefCnt()
                                    throws Exception

This test is for HBASE-26494, test the RefCnt behaviors in ImmutableMemStoreLAB

Throws:

Exception

mockStoreFileWithLength

private org.apache.hadoop.hbase.regionserver.HStoreFile mockStoreFileWithLength(long length)