org.apache.hadoop.hbase.regionserver

Class TestHStore

java.lang.Object

org.apache.hadoop.hbase.regionserver.TestHStore

public class TestHStore
extends Object

Test class for the HStore

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
static class	TestHStore.DummyStoreEngine
(package private) static class	TestHStore.FaultyFileSystem Faulty file system that will fail if you write past its fault position the FIRST TIME only; thereafter it will succeed.
(package private) static class	TestHStore.FaultyOutputStream
static class	TestHStore.MyCompactingMemStore
static class	TestHStore.MyCompactingMemStoreWithCustomCompactor
private static class	TestHStore.MyList<T>
(package private) static interface	TestHStore.MyListHook
private static class	TestHStore.MyMemStoreCompactor
private static class	TestHStore.MyStore
private static class	TestHStore.MyStoreHook
private static class	TestHStore.MyThread

Field Summary

Fields

Modifier and Type	Field and Description
private static int	BLOCKSIZE_SMALL
static HBaseClassTestRule	CLASS_RULE
private static String	DIR
(package private) List<org.apache.hadoop.hbase.Cell>	expected
(package private) byte[]	family
(package private) org.apache.hadoop.hbase.client.Get	get
(package private) long	id
private static org.slf4j.Logger	LOG
org.junit.rules.TestName	name
(package private) byte[]	qf1
(package private) byte[]	qf2
(package private) byte[]	qf3
(package private) byte[]	qf4
(package private) byte[]	qf5
(package private) byte[]	qf6
(package private) NavigableSet<byte[]>	qualifiers
(package private) org.apache.hadoop.hbase.regionserver.HRegion	region
(package private) List<org.apache.hadoop.hbase.Cell>	result
(package private) byte[]	row
(package private) byte[]	row2
(package private) org.apache.hadoop.hbase.regionserver.HStore	store
(package private) byte[]	table
private static HBaseTestingUtility	TEST_UTIL

Constructor Summary

Constructors

Constructor and Description
TestHStore()

Method Summary

Modifier and Type	Method and Description
private void	addStoreFile()
private void	archiveStoreFile(int index)
private void	assertCheck()
private void	closeCompactedFile(int index)
private long	countMemStoreScanner(org.apache.hadoop.hbase.regionserver.StoreScanner scanner)
private org.apache.hadoop.hbase.Cell	createCell(byte[] row, byte[] qualifier, long ts, long sequenceId, byte[] value)
private org.apache.hadoop.hbase.Cell	createCell(byte[] qualifier, long ts, long sequenceId, byte[] value)
private void	flush(int storeFilessize)
private static void	flushStore(org.apache.hadoop.hbase.regionserver.HStore store, long id)
(package private) List<org.apache.hadoop.hbase.Cell>	getKeyValueSet(long[] timestamps, int numRows, byte[] qualifier, byte[] family) Generate a list of KeyValues for testing based on given parameters
private static long	getLowestTimeStampFromFS(org.apache.hadoop.fs.FileSystem fs, Collection<org.apache.hadoop.hbase.regionserver.HStoreFile> candidates)
private void	init(String methodName)
private org.apache.hadoop.hbase.regionserver.HStore	init(String methodName, org.apache.hadoop.conf.Configuration conf)
private org.apache.hadoop.hbase.regionserver.HStore	init(String methodName, org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.hbase.client.ColumnFamilyDescriptor hcd)
private org.apache.hadoop.hbase.regionserver.HStore	init(String methodName, org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.hbase.client.TableDescriptorBuilder builder, org.apache.hadoop.hbase.client.ColumnFamilyDescriptor hcd)
private org.apache.hadoop.hbase.regionserver.HStore	init(String methodName, org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.hbase.client.TableDescriptorBuilder builder, org.apache.hadoop.hbase.client.ColumnFamilyDescriptor hcd, TestHStore.MyStoreHook hook)
private org.apache.hadoop.hbase.regionserver.HStore	init(String methodName, org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.hbase.client.TableDescriptorBuilder builder, org.apache.hadoop.hbase.client.ColumnFamilyDescriptor hcd, TestHStore.MyStoreHook hook, boolean switchToPread)
private void	initHRegion(String methodName, org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.hbase.client.TableDescriptorBuilder builder, org.apache.hadoop.hbase.client.ColumnFamilyDescriptor hcd, TestHStore.MyStoreHook hook, boolean switchToPread)
private TestHStore.MyStore	initMyStore(String methodName, org.apache.hadoop.conf.Configuration conf, TestHStore.MyStoreHook hook)
private org.apache.hadoop.hbase.regionserver.HStoreFile	mockStoreFile(long createdTime)
private org.apache.hadoop.hbase.regionserver.HStoreFile	mockStoreFileWithLength(long length)
void	setUp() Setup
void	tearDown()
static void	tearDownAfterClass()
void	testAge()
void	testCreateScannerAndSnapshotConcurrently()
void	testCreateWriter() Verify that compression and data block encoding are respected by the Store.createWriterInTmp() method, used on store flush.
void	testDeleteExpiredStoreFiles()
void	testDeleteExpiredStoreFiles(int minVersions)
void	testEmptyStoreFile() Test for hbase-1686.
private void	testFlushBeforeCompletingScan(TestHStore.MyListHook hook, org.apache.hadoop.hbase.filter.Filter filter, int expectedSize)
void	testFlushBeforeCompletingScanWithFilter()
void	testFlushBeforeCompletingScanWithFilterHint()
void	testFlushBeforeCompletingScanWoFilter()
void	testFlushSizeSizing() Test we do not lose data if we fail a flush and then close.
void	testGet_FromFilesOnly() Getting data from files only
void	testGet_FromMemStoreAndFiles() Getting data from memstore and files
void	testGet_FromMemStoreOnly() Getting data from memstore only
void	testHandleErrorsInFlush()
void	testInMemoryCompactionTypeWithLowerCase()
void	testLowestModificationTime()
void	testMultipleTimestamps() Test to ensure correctness when using Stores with multiple timestamps
void	testNumberOfMemStoreScannersAfterFlush()
private void	testNumberOfMemStoreScannersAfterFlush(List<org.apache.hadoop.hbase.Cell> inputCellsBeforeSnapshot, List<org.apache.hadoop.hbase.Cell> inputCellsAfterSnapshot)
void	testReclaimChunkWhenScaning()
void	testRefreshStoreFiles()
void	testRefreshStoreFilesNotChanged()
void	testRunDoubleMemStoreCompactors() If there are two running InMemoryFlushRunnable, the later InMemoryFlushRunnable may change the versionedList.
void	testScanWithDoubleFlush()
void	testSpaceQuotaChangeAfterReplacement()
void	testSplitWithEmptyColFam() Test for HBASE-3492 - Test split on empty colfam (no store files).
void	testStoreUsesConfigurationFromHcdAndHtd()
void	testStoreUsesSearchEngineOverride()
void	testSwitchingPreadtoStreamParallelyWithCompactionDischarger()
void	testTimeRangeIfSomeCellsAreDroppedInFlush()
private void	testTimeRangeIfSomeCellsAreDroppedInFlush(int maxVersion)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

CLASS_RULE

public static final HBaseClassTestRule CLASS_RULE

LOG

private static final org.slf4j.Logger LOG

name

public org.junit.rules.TestName name

region

org.apache.hadoop.hbase.regionserver.HRegion region

store

org.apache.hadoop.hbase.regionserver.HStore store

table

byte[] table

family

byte[] family

row

byte[] row

row2

byte[] row2

qf1

byte[] qf1

qf2

byte[] qf2

qf3

byte[] qf3

qf4

byte[] qf4

qf5

byte[] qf5

qf6

byte[] qf6

qualifiers

NavigableSet<byte[]> qualifiers

expected

List<org.apache.hadoop.hbase.Cell> expected

result

List<org.apache.hadoop.hbase.Cell> result

id

long id

get

org.apache.hadoop.hbase.client.Get get

TEST_UTIL

private static final HBaseTestingUtility TEST_UTIL

DIR

private static final String DIR

BLOCKSIZE_SMALL

private static final int BLOCKSIZE_SMALL

See Also:

Constant Field Values

Constructor Detail

TestHStore

public TestHStore()

Method Detail

setUp

public void setUp()
           throws IOException

Setup

Throws:

IOException

init

private void init(String methodName)
           throws IOException

Throws:

IOException

init

private org.apache.hadoop.hbase.regionserver.HStore init(String methodName,
                                                         org.apache.hadoop.conf.Configuration conf)
                                                  throws IOException

Throws:

IOException

init

private org.apache.hadoop.hbase.regionserver.HStore init(String methodName,
                                                         org.apache.hadoop.conf.Configuration conf,
                                                         org.apache.hadoop.hbase.client.ColumnFamilyDescriptor hcd)
                                                  throws IOException

Throws:

IOException

init

private org.apache.hadoop.hbase.regionserver.HStore init(String methodName,
                                                         org.apache.hadoop.conf.Configuration conf,
                                                         org.apache.hadoop.hbase.client.TableDescriptorBuilder builder,
                                                         org.apache.hadoop.hbase.client.ColumnFamilyDescriptor hcd)
                                                  throws IOException

Throws:

IOException

init

private org.apache.hadoop.hbase.regionserver.HStore init(String methodName,
                                                         org.apache.hadoop.conf.Configuration conf,
                                                         org.apache.hadoop.hbase.client.TableDescriptorBuilder builder,
                                                         org.apache.hadoop.hbase.client.ColumnFamilyDescriptor hcd,
                                                         TestHStore.MyStoreHook hook)
                                                  throws IOException

Throws:

IOException

initHRegion

private void initHRegion(String methodName,
                         org.apache.hadoop.conf.Configuration conf,
                         org.apache.hadoop.hbase.client.TableDescriptorBuilder builder,
                         org.apache.hadoop.hbase.client.ColumnFamilyDescriptor hcd,
                         TestHStore.MyStoreHook hook,
                         boolean switchToPread)
                  throws IOException

Throws:

IOException

init

private org.apache.hadoop.hbase.regionserver.HStore init(String methodName,
                                                         org.apache.hadoop.conf.Configuration conf,
                                                         org.apache.hadoop.hbase.client.TableDescriptorBuilder builder,
                                                         org.apache.hadoop.hbase.client.ColumnFamilyDescriptor hcd,
                                                         TestHStore.MyStoreHook hook,
                                                         boolean switchToPread)
                                                  throws IOException

Throws:

IOException

testFlushSizeSizing

public void testFlushSizeSizing()
                         throws Exception

Test we do not lose data if we fail a flush and then close. Part of HBase-10466

Throws:

Exception

testCreateWriter

public void testCreateWriter()
                      throws Exception

Verify that compression and data block encoding are respected by the Store.createWriterInTmp() method, used on store flush.

Throws:

Exception

testDeleteExpiredStoreFiles

public void testDeleteExpiredStoreFiles()
                                 throws Exception

Throws:

Exception

testDeleteExpiredStoreFiles

public void testDeleteExpiredStoreFiles(int minVersions)
                                 throws Exception

Throws:

Exception

testLowestModificationTime

public void testLowestModificationTime()
                                throws Exception

Throws:

Exception

getLowestTimeStampFromFS

private static long getLowestTimeStampFromFS(org.apache.hadoop.fs.FileSystem fs,
                                             Collection<org.apache.hadoop.hbase.regionserver.HStoreFile> candidates)
                                      throws IOException

Throws:

IOException

testEmptyStoreFile

public void testEmptyStoreFile()
                        throws IOException

Test for hbase-1686.

Throws:

IOException

testGet_FromMemStoreOnly

public void testGet_FromMemStoreOnly()
                              throws IOException

Getting data from memstore only

Throws:

IOException

testTimeRangeIfSomeCellsAreDroppedInFlush

public void testTimeRangeIfSomeCellsAreDroppedInFlush()
                                               throws IOException

Throws:

IOException

testTimeRangeIfSomeCellsAreDroppedInFlush

private void testTimeRangeIfSomeCellsAreDroppedInFlush(int maxVersion)
                                                throws IOException

Throws:

IOException

testGet_FromFilesOnly

public void testGet_FromFilesOnly()
                           throws IOException

Getting data from files only

Throws:

IOException

testGet_FromMemStoreAndFiles

public void testGet_FromMemStoreAndFiles()
                                  throws IOException

Getting data from memstore and files

Throws:

IOException

flush

private void flush(int storeFilessize)
            throws IOException

Throws:

IOException

assertCheck

private void assertCheck()

tearDown

public void tearDown()
              throws Exception

Throws:

Exception

tearDownAfterClass

public static void tearDownAfterClass()
                               throws IOException

Throws:

IOException

testHandleErrorsInFlush

public void testHandleErrorsInFlush()
                             throws Exception

Throws:

Exception

flushStore

private static void flushStore(org.apache.hadoop.hbase.regionserver.HStore store,
                               long id)
                        throws IOException

Throws:

IOException

getKeyValueSet

List<org.apache.hadoop.hbase.Cell> getKeyValueSet(long[] timestamps,
                                                  int numRows,
                                                  byte[] qualifier,
                                                  byte[] family)

Generate a list of KeyValues for testing based on given parameters

Parameters:

timestamps -

numRows -

qualifier -

family -

Returns:

the rows key-value list

testMultipleTimestamps

public void testMultipleTimestamps()
                            throws IOException

Test to ensure correctness when using Stores with multiple timestamps

Throws:

IOException

testSplitWithEmptyColFam

public void testSplitWithEmptyColFam()
                              throws IOException

Test for HBASE-3492 - Test split on empty colfam (no store files).

Throws:

IOException - When the IO operations fail.

testStoreUsesConfigurationFromHcdAndHtd

public void testStoreUsesConfigurationFromHcdAndHtd()
                                             throws Exception

Throws:

Exception

testStoreUsesSearchEngineOverride

public void testStoreUsesSearchEngineOverride()
                                       throws Exception

Throws:

Exception

addStoreFile

private void addStoreFile()
                   throws IOException

Throws:

IOException

archiveStoreFile

private void archiveStoreFile(int index)
                       throws IOException

Throws:

IOException

closeCompactedFile

private void closeCompactedFile(int index)
                         throws IOException

Throws:

IOException

testRefreshStoreFiles

public void testRefreshStoreFiles()
                           throws Exception

Throws:

Exception

testRefreshStoreFilesNotChanged

public void testRefreshStoreFilesNotChanged()
                                     throws IOException

Throws:

IOException

countMemStoreScanner

private long countMemStoreScanner(org.apache.hadoop.hbase.regionserver.StoreScanner scanner)

testNumberOfMemStoreScannersAfterFlush

public void testNumberOfMemStoreScannersAfterFlush()
                                            throws IOException

Throws:

IOException

testNumberOfMemStoreScannersAfterFlush

private void testNumberOfMemStoreScannersAfterFlush(List<org.apache.hadoop.hbase.Cell> inputCellsBeforeSnapshot,
                                                    List<org.apache.hadoop.hbase.Cell> inputCellsAfterSnapshot)
                                             throws IOException

Throws:

IOException

createCell

private org.apache.hadoop.hbase.Cell createCell(byte[] qualifier,
                                                long ts,
                                                long sequenceId,
                                                byte[] value)
                                         throws IOException

Throws:

IOException

createCell

private org.apache.hadoop.hbase.Cell createCell(byte[] row,
                                                byte[] qualifier,
                                                long ts,
                                                long sequenceId,
                                                byte[] value)
                                         throws IOException

Throws:

IOException

testFlushBeforeCompletingScanWoFilter

public void testFlushBeforeCompletingScanWoFilter()
                                           throws IOException,
                                                  InterruptedException

Throws:

IOException

InterruptedException

testFlushBeforeCompletingScanWithFilter

public void testFlushBeforeCompletingScanWithFilter()
                                             throws IOException,
                                                    InterruptedException

Throws:

IOException

InterruptedException

testFlushBeforeCompletingScanWithFilterHint

public void testFlushBeforeCompletingScanWithFilterHint()
                                                 throws IOException,
                                                        InterruptedException

Throws:

IOException

InterruptedException

testFlushBeforeCompletingScan

private void testFlushBeforeCompletingScan(TestHStore.MyListHook hook,
                                           org.apache.hadoop.hbase.filter.Filter filter,
                                           int expectedSize)
                                    throws IOException,
                                           InterruptedException

Throws:

IOException

InterruptedException

testCreateScannerAndSnapshotConcurrently

public void testCreateScannerAndSnapshotConcurrently()
                                              throws IOException,
                                                     InterruptedException

Throws:

IOException

InterruptedException

testScanWithDoubleFlush

public void testScanWithDoubleFlush()
                             throws IOException

Throws:

IOException

testReclaimChunkWhenScaning

public void testReclaimChunkWhenScaning()
                                 throws IOException

Throws:

IOException

testRunDoubleMemStoreCompactors

public void testRunDoubleMemStoreCompactors()
                                     throws IOException,
                                            InterruptedException

If there are two running InMemoryFlushRunnable, the later InMemoryFlushRunnable may change the versionedList. And the first InMemoryFlushRunnable will use the chagned versionedList to remove the corresponding segments. In short, there will be some segements which isn't in merge are removed.

Throws:

IOException

InterruptedException

testAge

public void testAge()
             throws IOException

Throws:

IOException

mockStoreFile

private org.apache.hadoop.hbase.regionserver.HStoreFile mockStoreFile(long createdTime)

initMyStore

private TestHStore.MyStore initMyStore(String methodName,
                                       org.apache.hadoop.conf.Configuration conf,
                                       TestHStore.MyStoreHook hook)
                                throws IOException

Throws:

IOException

testSwitchingPreadtoStreamParallelyWithCompactionDischarger

public void testSwitchingPreadtoStreamParallelyWithCompactionDischarger()
                                                                 throws Exception

Throws:

Exception

testInMemoryCompactionTypeWithLowerCase

public void testInMemoryCompactionTypeWithLowerCase()
                                             throws IOException,
                                                    InterruptedException

Throws:

IOException

InterruptedException

testSpaceQuotaChangeAfterReplacement

public void testSpaceQuotaChangeAfterReplacement()
                                          throws IOException

Throws:

IOException

mockStoreFileWithLength

private org.apache.hadoop.hbase.regionserver.HStoreFile mockStoreFileWithLength(long length)