Package org.apache.hadoop.hbase.regionserver

Class TestDataTieringManager

java.lang.Object

org.apache.hadoop.hbase.regionserver.TestDataTieringManager

public class TestDataTieringManager
extends Object

This class is used to test the functionality of the DataTieringManager. The mock online regions are stored in testOnlineRegions. For all tests, the setup of testOnlineRegions occurs only once. Please refer to setupOnlineRegions() for the structure. Additionally, a list of all store files is maintained in hStoreFiles. The characteristics of these store files are listed below:

Nested Class Summary

Nested Classes

Modifier and Type	Class	Description
(package private) static interface	TestDataTieringManager.DataTieringMethodCallerWithKey

Field Summary

Fields

Modifier and Type	Field	Description
private static org.apache.hadoop.hbase.io.hfile.BlockCache	blockCache
private static org.apache.hadoop.hbase.io.hfile.CacheConfig	cacheConf
static final HBaseClassTestRule	CLASS_RULE
private static org.apache.hadoop.hbase.regionserver.DataTieringManager	dataTieringManager
private static final long	DAY
private static org.apache.hadoop.conf.Configuration	defaultConf
private static org.apache.hadoop.fs.FileSystem	fs
private static final List<org.apache.hadoop.hbase.regionserver.HStoreFile>	hStoreFiles
private static final org.slf4j.Logger	LOG
private static String	rowKeyString	Represents the current lexicographically increasing string used as a row key when writing HFiles.
private static final HBaseTestingUtil	TEST_UTIL
private static org.apache.hadoop.fs.Path	testDir
private static final Map<String,org.apache.hadoop.hbase.regionserver.HRegion>	testOnlineRegions

Constructor Summary

Constructors

Constructor	Description
TestDataTieringManager()

Method Summary

Modifier and Type	Method	Description
private static org.apache.hadoop.hbase.regionserver.HRegion	createHRegion(String table)
private static org.apache.hadoop.hbase.regionserver.HRegion	createHRegion(String table, org.apache.hadoop.conf.Configuration conf)
private static org.apache.hadoop.hbase.regionserver.HStore	createHStore(org.apache.hadoop.hbase.regionserver.HRegion region, String columnFamily)
private static org.apache.hadoop.hbase.regionserver.HStore	createHStore(org.apache.hadoop.hbase.regionserver.HRegion region, String columnFamily, org.apache.hadoop.conf.Configuration conf)
(package private) static org.apache.hadoop.hbase.regionserver.HStoreFile	createHStoreFile(org.apache.hadoop.fs.Path storeDir, org.apache.hadoop.conf.Configuration conf, long timestamp, org.apache.hadoop.hbase.regionserver.HRegionFileSystem regionFs)
private void	createTestFilesForCompaction(org.apache.hadoop.hbase.regionserver.HStore hStore)
private static byte[]	generateRandomBytes(int sizeInBytes)
private static org.apache.hadoop.conf.Configuration	getConfWithGracePeriod(long hotDataAge, long gracePeriod)
private static org.apache.hadoop.conf.Configuration	getConfWithTimeRangeDataTieringEnabled(long hotDataAge)
private static void	initializeTestEnvironment()
private static String	nextString()	Returns the lexicographically larger string every time it's called.
private void	setCacheCompactBlocksOnWrite()
private void	setPrefetchBlocksOnOpen()
static void	setupBeforeClass()
private static void	setupFileSystemAndCache()
private static void	setupOnlineRegions()
void	testBlockEvictions()
void	testBlockEvictionsAllColdBlocks()
void	testBlockEvictionsHotBlocks()
private void	testCacheCompactedBlocksOnWrite(org.apache.hadoop.hbase.regionserver.HRegion region, boolean expectDataBlocksCached)
void	testCacheCompactedBlocksOnWriteDataTieringDisabled()
void	testCacheCompactedBlocksOnWriteWithColdData()
void	testCacheCompactedBlocksOnWriteWithHotData()
void	testCacheConfigShouldCacheFile()
private void	testCacheOnRead(org.apache.hadoop.hbase.regionserver.HStoreFile hStoreFile, org.apache.hadoop.hbase.io.hfile.BlockCacheKey key, long onDiskBlockSize, boolean expectedCached)
void	testCacheOnReadColdFile()
void	testCacheOnReadHotFile()
void	testColdDataFiles()
void	testDataTieringEnabledWithKey()
private void	testDataTieringMethodWithKey(TestDataTieringManager.DataTieringMethodCallerWithKey caller, org.apache.hadoop.hbase.io.hfile.BlockCacheKey key, boolean expectedResult, org.apache.hadoop.hbase.regionserver.DataTieringException exception)
private void	testDataTieringMethodWithKeyExpectingException(TestDataTieringManager.DataTieringMethodCallerWithKey caller, org.apache.hadoop.hbase.io.hfile.BlockCacheKey key, org.apache.hadoop.hbase.regionserver.DataTieringException exception)
private void	testDataTieringMethodWithKeyNoException(TestDataTieringManager.DataTieringMethodCallerWithKey caller, org.apache.hadoop.hbase.io.hfile.BlockCacheKey key, boolean expectedResult)
void	testFeatureKeyDisabled()
void	testFileIsColdWithoutGracePeriod()
void	testGracePeriodMakesColdFileHot()
void	testHotDataWithKey()
void	testPickColdDataFiles()
void	testPrefetchWhenDataTieringEnabled()
private static void	updateCommonConfigurations()
private void	validateBlocks(Set<org.apache.hadoop.hbase.io.hfile.BlockCacheKey> keys, int expectedTotalKeys, int expectedHotBlocks, int expectedColdBlocks)
private static void	writeStoreFileRandomData(org.apache.hadoop.hbase.regionserver.StoreFileWriter writer, byte[] columnFamily, long timestamp)	Writes random data to a store file with rows arranged in lexicographically increasing order.

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Details

CLASS_RULE

public static final HBaseClassTestRule CLASS_RULE

LOG

private static final org.slf4j.Logger LOG

TEST_UTIL

private static final HBaseTestingUtil TEST_UTIL

DAY

private static final long DAY

See Also:

• Constant Field Values

defaultConf

private static org.apache.hadoop.conf.Configuration defaultConf

fs

private static org.apache.hadoop.fs.FileSystem fs

blockCache

private static org.apache.hadoop.hbase.io.hfile.BlockCache blockCache

cacheConf

private static org.apache.hadoop.hbase.io.hfile.CacheConfig cacheConf

testDir

private static org.apache.hadoop.fs.Path testDir

testOnlineRegions

private static final Map<String,org.apache.hadoop.hbase.regionserver.HRegion> testOnlineRegions

dataTieringManager

private static org.apache.hadoop.hbase.regionserver.DataTieringManager dataTieringManager

hStoreFiles

private static final List<org.apache.hadoop.hbase.regionserver.HStoreFile> hStoreFiles

rowKeyString

private static String rowKeyString

Represents the current lexicographically increasing string used as a row key when writing HFiles. It is incremented each time nextString() is called to generate unique row keys.

Constructor Details

TestDataTieringManager

public TestDataTieringManager()

Method Details

setupBeforeClass

public static void setupBeforeClass()
                             throws Exception

Throws:

Exception

updateCommonConfigurations

private static void updateCommonConfigurations()

testDataTieringEnabledWithKey

public void testDataTieringEnabledWithKey()
                                   throws IOException

Throws:

IOException

testHotDataWithKey

public void testHotDataWithKey()
                        throws IOException

Throws:

IOException

testGracePeriodMakesColdFileHot

public void testGracePeriodMakesColdFileHot()
                                     throws IOException,
org.apache.hadoop.hbase.regionserver.DataTieringException

Throws:

IOException

org.apache.hadoop.hbase.regionserver.DataTieringException

testFileIsColdWithoutGracePeriod

public void testFileIsColdWithoutGracePeriod()
                                      throws IOException,
org.apache.hadoop.hbase.regionserver.DataTieringException

Throws:

IOException

org.apache.hadoop.hbase.regionserver.DataTieringException

testPrefetchWhenDataTieringEnabled

public void testPrefetchWhenDataTieringEnabled()
                                        throws IOException

Throws:

IOException

setPrefetchBlocksOnOpen

private void setPrefetchBlocksOnOpen()

testColdDataFiles

public void testColdDataFiles()
                       throws IOException

Throws:

IOException

testCacheCompactedBlocksOnWriteDataTieringDisabled

public void testCacheCompactedBlocksOnWriteDataTieringDisabled()
                                                        throws IOException

Throws:

IOException

testCacheCompactedBlocksOnWriteWithHotData

public void testCacheCompactedBlocksOnWriteWithHotData()
                                                throws IOException

Throws:

IOException

testCacheCompactedBlocksOnWriteWithColdData

public void testCacheCompactedBlocksOnWriteWithColdData()
                                                 throws IOException

Throws:

IOException

setCacheCompactBlocksOnWrite

private void setCacheCompactBlocksOnWrite()

testCacheCompactedBlocksOnWrite

private void testCacheCompactedBlocksOnWrite(org.apache.hadoop.hbase.regionserver.HRegion region,
 boolean expectDataBlocksCached)
                                      throws IOException

Throws:

IOException

createTestFilesForCompaction

private void createTestFilesForCompaction(org.apache.hadoop.hbase.regionserver.HStore hStore)
                                   throws IOException

Throws:

IOException

testPickColdDataFiles

public void testPickColdDataFiles()
                           throws IOException

Throws:

IOException

testBlockEvictions

public void testBlockEvictions()
                        throws Exception

Throws:

Exception

testBlockEvictionsAllColdBlocks

public void testBlockEvictionsAllColdBlocks()
                                     throws Exception

Throws:

Exception

testBlockEvictionsHotBlocks

public void testBlockEvictionsHotBlocks()
                                 throws Exception

Throws:

Exception

testFeatureKeyDisabled

public void testFeatureKeyDisabled()
                            throws Exception

Throws:

Exception

testCacheConfigShouldCacheFile

public void testCacheConfigShouldCacheFile()
                                    throws Exception

Throws:

Exception

testCacheOnReadColdFile

public void testCacheOnReadColdFile()
                             throws Exception

Throws:

Exception

testCacheOnReadHotFile

public void testCacheOnReadHotFile()
                            throws Exception

Throws:

Exception

testCacheOnRead

private void testCacheOnRead(org.apache.hadoop.hbase.regionserver.HStoreFile hStoreFile,
 org.apache.hadoop.hbase.io.hfile.BlockCacheKey key,
 long onDiskBlockSize,
 boolean expectedCached)
                      throws Exception

Throws:

Exception

validateBlocks

private void validateBlocks(Set<org.apache.hadoop.hbase.io.hfile.BlockCacheKey> keys,
 int expectedTotalKeys,
 int expectedHotBlocks,
 int expectedColdBlocks)

testDataTieringMethodWithKey

private void testDataTieringMethodWithKey(TestDataTieringManager.DataTieringMethodCallerWithKey caller,
 org.apache.hadoop.hbase.io.hfile.BlockCacheKey key,
 boolean expectedResult,
 org.apache.hadoop.hbase.regionserver.DataTieringException exception)

testDataTieringMethodWithKeyExpectingException

private void testDataTieringMethodWithKeyExpectingException(TestDataTieringManager.DataTieringMethodCallerWithKey caller,
 org.apache.hadoop.hbase.io.hfile.BlockCacheKey key,
 org.apache.hadoop.hbase.regionserver.DataTieringException exception)

testDataTieringMethodWithKeyNoException

private void testDataTieringMethodWithKeyNoException(TestDataTieringManager.DataTieringMethodCallerWithKey caller,
 org.apache.hadoop.hbase.io.hfile.BlockCacheKey key,
 boolean expectedResult)

initializeTestEnvironment

private static void initializeTestEnvironment()
                                       throws IOException

Throws:

IOException

setupFileSystemAndCache

private static void setupFileSystemAndCache()
                                     throws IOException

Throws:

IOException

setupOnlineRegions

private static void setupOnlineRegions()
                                throws IOException

Throws:

IOException

createHRegion

private static org.apache.hadoop.hbase.regionserver.HRegion createHRegion(String table)
                                                                   throws IOException

Throws:

IOException

createHRegion

private static org.apache.hadoop.hbase.regionserver.HRegion createHRegion(String table,
 org.apache.hadoop.conf.Configuration conf)
                                                                   throws IOException

Throws:

IOException

createHStore

private static org.apache.hadoop.hbase.regionserver.HStore createHStore(org.apache.hadoop.hbase.regionserver.HRegion region,
 String columnFamily)
                                                                 throws IOException

Throws:

IOException

createHStore

private static org.apache.hadoop.hbase.regionserver.HStore createHStore(org.apache.hadoop.hbase.regionserver.HRegion region,
 String columnFamily,
 org.apache.hadoop.conf.Configuration conf)
                                                                 throws IOException

Throws:

IOException

getConfWithTimeRangeDataTieringEnabled

private static org.apache.hadoop.conf.Configuration getConfWithTimeRangeDataTieringEnabled(long hotDataAge)

getConfWithGracePeriod

private static org.apache.hadoop.conf.Configuration getConfWithGracePeriod(long hotDataAge,
 long gracePeriod)

createHStoreFile

static org.apache.hadoop.hbase.regionserver.HStoreFile createHStoreFile(org.apache.hadoop.fs.Path storeDir,
 org.apache.hadoop.conf.Configuration conf,
 long timestamp,
 org.apache.hadoop.hbase.regionserver.HRegionFileSystem regionFs)
                                                                 throws IOException

Throws:

IOException

writeStoreFileRandomData

private static void writeStoreFileRandomData(org.apache.hadoop.hbase.regionserver.StoreFileWriter writer,
 byte[] columnFamily,
 long timestamp)
                                      throws IOException

Writes random data to a store file with rows arranged in lexicographically increasing order. Each row is generated using the nextString() method, ensuring that each subsequent row is lexicographically larger than the previous one.

Throws:

IOException

generateRandomBytes

private static byte[] generateRandomBytes(int sizeInBytes)

nextString

private static String nextString()

Returns the lexicographically larger string every time it's called.