Package org.apache.hadoop.hbase.regionserver

Class TestCustomCellDataTieringManager

java.lang.Object

org.apache.hadoop.hbase.regionserver.TestCustomCellDataTieringManager

public class TestCustomCellDataTieringManager
extends Object

This class is used to test the functionality of the DataTieringManager. The mock online regions are stored in testOnlineRegions. For all tests, the setup of testOnlineRegions occurs only once. Please refer to TestCustomCellDataTieringManager#setupOnlineRegions() for the structure. Additionally, a list of all store files is maintained in hStoreFiles. The characteristics of these store files are listed below:

Nested Class Summary

Nested Classes

Modifier and Type	Class	Description
(package private) static interface	TestCustomCellDataTieringManager.DataTieringMethodCallerWithKey
(package private) static interface	TestCustomCellDataTieringManager.DataTieringMethodCallerWithPath

Field Summary

Fields

Modifier and Type	Field	Description
private org.apache.hadoop.hbase.io.hfile.BlockCache	blockCache
private static org.apache.hadoop.hbase.io.hfile.CacheConfig	cacheConf
static final HBaseClassTestRule	CLASS_RULE
private static org.apache.hadoop.hbase.regionserver.DataTieringManager	dataTieringManager
private static final long	DAY
private static org.apache.hadoop.conf.Configuration	defaultConf
private static org.apache.hadoop.fs.FileSystem	fs
private static final List<org.apache.hadoop.hbase.regionserver.HStoreFile>	hStoreFiles
private static final org.slf4j.Logger	LOG
private static String	rowKeyString	Represents the current lexicographically increasing string used as a row key when writing HFiles.
private static final HBaseTestingUtil	TEST_UTIL
private static org.apache.hadoop.fs.Path	testDir
private static final Map<String,org.apache.hadoop.hbase.regionserver.HRegion>	testOnlineRegions

Constructor Summary

Constructors

Constructor	Description
TestCustomCellDataTieringManager()

Method Summary

Modifier and Type	Method	Description
private static org.apache.hadoop.hbase.regionserver.HRegion	createHRegion(String table, org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.hbase.io.hfile.BlockCache blockCache)
private static org.apache.hadoop.hbase.regionserver.HRegion	createHRegion(String table, org.apache.hadoop.hbase.io.hfile.BlockCache blockCache)
private static org.apache.hadoop.hbase.regionserver.HStore	createHStore(org.apache.hadoop.hbase.regionserver.HRegion region, String columnFamily)
private static org.apache.hadoop.hbase.regionserver.HStore	createHStore(org.apache.hadoop.hbase.regionserver.HRegion region, String columnFamily, org.apache.hadoop.conf.Configuration conf)
private static org.apache.hadoop.hbase.regionserver.HStoreFile	createHStoreFile(org.apache.hadoop.fs.Path storeDir, org.apache.hadoop.conf.Configuration conf, long timestamp, org.apache.hadoop.hbase.regionserver.HRegionFileSystem regionFs)
private void	createTestFilesForCompaction(org.apache.hadoop.hbase.regionserver.HStore hStore)
private static byte[]	generateRandomBytes(int sizeInBytes)
private static org.apache.hadoop.conf.Configuration	getConfWithCustomCellDataTieringEnabled(long hotDataAge)
private static org.apache.hadoop.hbase.io.hfile.BlockCache	initializeTestEnvironment()
private static String	nextString()	Returns the lexicographically larger string every time it's called.
private void	setCacheCompactBlocksOnWrite()
private void	setPrefetchBlocksOnOpen()
static void	setupBeforeClass()
private static org.apache.hadoop.hbase.io.hfile.BlockCache	setupFileSystemAndCache()
private static void	setupOnlineRegions(org.apache.hadoop.hbase.io.hfile.BlockCache blockCache)
void	testBlockEvictions()
void	testBlockEvictionsAllColdBlocks()
void	testBlockEvictionsHotBlocks()
private void	testCacheCompactedBlocksOnWrite(org.apache.hadoop.hbase.regionserver.HRegion region, boolean expectDataBlocksCached)
void	testCacheCompactedBlocksOnWriteDataTieringDisabled()
void	testCacheCompactedBlocksOnWriteWithColdData()
void	testCacheCompactedBlocksOnWriteWithHotData()
void	testCacheConfigShouldCacheFile()
private void	testCacheOnRead(org.apache.hadoop.hbase.regionserver.HStoreFile hStoreFile, org.apache.hadoop.hbase.io.hfile.BlockCacheKey key, long onDiskBlockSize, boolean expectedCached)
void	testCacheOnReadColdFile()
void	testCacheOnReadHotFile()
void	testColdDataFiles()
void	testDataTieringEnabledWithKey()
void	testDataTieringEnabledWithPath()
private void	testDataTieringMethodWithKey(TestCustomCellDataTieringManager.DataTieringMethodCallerWithKey caller, org.apache.hadoop.hbase.io.hfile.BlockCacheKey key, boolean expectedResult, org.apache.hadoop.hbase.regionserver.DataTieringException exception)
private void	testDataTieringMethodWithKeyExpectingException(TestCustomCellDataTieringManager.DataTieringMethodCallerWithKey caller, org.apache.hadoop.hbase.io.hfile.BlockCacheKey key, org.apache.hadoop.hbase.regionserver.DataTieringException exception)
private void	testDataTieringMethodWithKeyNoException(TestCustomCellDataTieringManager.DataTieringMethodCallerWithKey caller, org.apache.hadoop.hbase.io.hfile.BlockCacheKey key, boolean expectedResult)
private void	testDataTieringMethodWithPath(TestCustomCellDataTieringManager.DataTieringMethodCallerWithPath caller, org.apache.hadoop.fs.Path path, boolean expectedResult, org.apache.hadoop.hbase.regionserver.DataTieringException exception)
private void	testDataTieringMethodWithPathExpectingException(TestCustomCellDataTieringManager.DataTieringMethodCallerWithPath caller, org.apache.hadoop.fs.Path path, org.apache.hadoop.hbase.regionserver.DataTieringException exception)
private void	testDataTieringMethodWithPathNoException(TestCustomCellDataTieringManager.DataTieringMethodCallerWithPath caller, org.apache.hadoop.fs.Path path, boolean expectedResult)
void	testFeatureKeyDisabled()
void	testHotDataWithKey()
void	testHotDataWithPath()
void	testPickColdDataFiles()
void	testPrefetchWhenDataTieringEnabled()
private static void	updateCommonConfigurations()
private void	validateBlocks(Set<org.apache.hadoop.hbase.io.hfile.BlockCacheKey> keys, int expectedTotalKeys, int expectedHotBlocks, int expectedColdBlocks)
private static void	writeStoreFileRandomData(org.apache.hadoop.hbase.regionserver.StoreFileWriter writer, byte[] columnFamily, long timestamp)	Writes random data to a store file with rows arranged in lexicographically increasing order.

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Details

CLASS_RULE

public static final HBaseClassTestRule CLASS_RULE

LOG

private static final org.slf4j.Logger LOG

TEST_UTIL

private static final HBaseTestingUtil TEST_UTIL

DAY

private static final long DAY

See Also:

• Constant Field Values

defaultConf

private static org.apache.hadoop.conf.Configuration defaultConf

fs

private static org.apache.hadoop.fs.FileSystem fs

blockCache

private org.apache.hadoop.hbase.io.hfile.BlockCache blockCache

cacheConf

private static org.apache.hadoop.hbase.io.hfile.CacheConfig cacheConf

testDir

private static org.apache.hadoop.fs.Path testDir

testOnlineRegions

private static final Map<String,org.apache.hadoop.hbase.regionserver.HRegion> testOnlineRegions

dataTieringManager

private static org.apache.hadoop.hbase.regionserver.DataTieringManager dataTieringManager

hStoreFiles

private static final List<org.apache.hadoop.hbase.regionserver.HStoreFile> hStoreFiles

rowKeyString

private static String rowKeyString

Represents the current lexicographically increasing string used as a row key when writing HFiles. It is incremented each time nextString() is called to generate unique row keys.

Constructor Details

TestCustomCellDataTieringManager

public TestCustomCellDataTieringManager()

Method Details

setupBeforeClass

public static void setupBeforeClass()
                             throws Exception

Throws:

Exception

updateCommonConfigurations

private static void updateCommonConfigurations()

testDataTieringEnabledWithKey

public void testDataTieringEnabledWithKey()
                                   throws IOException

Throws:

IOException

testDataTieringEnabledWithPath

public void testDataTieringEnabledWithPath()
                                    throws IOException

Throws:

IOException

testHotDataWithKey

public void testHotDataWithKey()
                        throws IOException

Throws:

IOException

testHotDataWithPath

public void testHotDataWithPath()
                         throws IOException

Throws:

IOException

testPrefetchWhenDataTieringEnabled

public void testPrefetchWhenDataTieringEnabled()
                                        throws IOException

Throws:

IOException

setPrefetchBlocksOnOpen

private void setPrefetchBlocksOnOpen()

testColdDataFiles

public void testColdDataFiles()
                       throws IOException

Throws:

IOException

testCacheCompactedBlocksOnWriteDataTieringDisabled

public void testCacheCompactedBlocksOnWriteDataTieringDisabled()
                                                        throws IOException

Throws:

IOException

testCacheCompactedBlocksOnWriteWithHotData

public void testCacheCompactedBlocksOnWriteWithHotData()
                                                throws IOException

Throws:

IOException

testCacheCompactedBlocksOnWriteWithColdData

public void testCacheCompactedBlocksOnWriteWithColdData()
                                                 throws IOException

Throws:

IOException

setCacheCompactBlocksOnWrite

private void setCacheCompactBlocksOnWrite()

testCacheCompactedBlocksOnWrite

private void testCacheCompactedBlocksOnWrite(org.apache.hadoop.hbase.regionserver.HRegion region,
 boolean expectDataBlocksCached)
                                      throws IOException

Throws:

IOException

createTestFilesForCompaction

private void createTestFilesForCompaction(org.apache.hadoop.hbase.regionserver.HStore hStore)
                                   throws IOException

Throws:

IOException

testPickColdDataFiles

public void testPickColdDataFiles()
                           throws IOException

Throws:

IOException

testBlockEvictions

public void testBlockEvictions()
                        throws Exception

Throws:

Exception

testBlockEvictionsAllColdBlocks

public void testBlockEvictionsAllColdBlocks()
                                     throws Exception

Throws:

Exception

testBlockEvictionsHotBlocks

public void testBlockEvictionsHotBlocks()
                                 throws Exception

Throws:

Exception

testFeatureKeyDisabled

public void testFeatureKeyDisabled()
                            throws Exception

Throws:

Exception

testCacheConfigShouldCacheFile

public void testCacheConfigShouldCacheFile()
                                    throws Exception

Throws:

Exception

testCacheOnReadColdFile

public void testCacheOnReadColdFile()
                             throws Exception

Throws:

Exception

testCacheOnReadHotFile

public void testCacheOnReadHotFile()
                            throws Exception

Throws:

Exception

testCacheOnRead

private void testCacheOnRead(org.apache.hadoop.hbase.regionserver.HStoreFile hStoreFile,
 org.apache.hadoop.hbase.io.hfile.BlockCacheKey key,
 long onDiskBlockSize,
 boolean expectedCached)
                      throws Exception

Throws:

Exception

validateBlocks

private void validateBlocks(Set<org.apache.hadoop.hbase.io.hfile.BlockCacheKey> keys,
 int expectedTotalKeys,
 int expectedHotBlocks,
 int expectedColdBlocks)

testDataTieringMethodWithPath

private void testDataTieringMethodWithPath(TestCustomCellDataTieringManager.DataTieringMethodCallerWithPath caller,
 org.apache.hadoop.fs.Path path,
 boolean expectedResult,
 org.apache.hadoop.hbase.regionserver.DataTieringException exception)

testDataTieringMethodWithKey

private void testDataTieringMethodWithKey(TestCustomCellDataTieringManager.DataTieringMethodCallerWithKey caller,
 org.apache.hadoop.hbase.io.hfile.BlockCacheKey key,
 boolean expectedResult,
 org.apache.hadoop.hbase.regionserver.DataTieringException exception)

testDataTieringMethodWithPathExpectingException

private void testDataTieringMethodWithPathExpectingException(TestCustomCellDataTieringManager.DataTieringMethodCallerWithPath caller,
 org.apache.hadoop.fs.Path path,
 org.apache.hadoop.hbase.regionserver.DataTieringException exception)

testDataTieringMethodWithPathNoException

private void testDataTieringMethodWithPathNoException(TestCustomCellDataTieringManager.DataTieringMethodCallerWithPath caller,
 org.apache.hadoop.fs.Path path,
 boolean expectedResult)

testDataTieringMethodWithKeyExpectingException

private void testDataTieringMethodWithKeyExpectingException(TestCustomCellDataTieringManager.DataTieringMethodCallerWithKey caller,
 org.apache.hadoop.hbase.io.hfile.BlockCacheKey key,
 org.apache.hadoop.hbase.regionserver.DataTieringException exception)

testDataTieringMethodWithKeyNoException

private void testDataTieringMethodWithKeyNoException(TestCustomCellDataTieringManager.DataTieringMethodCallerWithKey caller,
 org.apache.hadoop.hbase.io.hfile.BlockCacheKey key,
 boolean expectedResult)

initializeTestEnvironment

private static org.apache.hadoop.hbase.io.hfile.BlockCache initializeTestEnvironment()
                                                                              throws IOException

Throws:

IOException

setupFileSystemAndCache

private static org.apache.hadoop.hbase.io.hfile.BlockCache setupFileSystemAndCache()
                                                                            throws IOException

Throws:

IOException

setupOnlineRegions

private static void setupOnlineRegions(org.apache.hadoop.hbase.io.hfile.BlockCache blockCache)
                                throws IOException

Throws:

IOException

createHRegion

private static org.apache.hadoop.hbase.regionserver.HRegion createHRegion(String table,
 org.apache.hadoop.hbase.io.hfile.BlockCache blockCache)
                                                                   throws IOException

Throws:

IOException

createHRegion

private static org.apache.hadoop.hbase.regionserver.HRegion createHRegion(String table,
 org.apache.hadoop.conf.Configuration conf,
 org.apache.hadoop.hbase.io.hfile.BlockCache blockCache)
                                                                   throws IOException

Throws:

IOException

createHStore

private static org.apache.hadoop.hbase.regionserver.HStore createHStore(org.apache.hadoop.hbase.regionserver.HRegion region,
 String columnFamily)
                                                                 throws IOException

Throws:

IOException

createHStore

private static org.apache.hadoop.hbase.regionserver.HStore createHStore(org.apache.hadoop.hbase.regionserver.HRegion region,
 String columnFamily,
 org.apache.hadoop.conf.Configuration conf)
                                                                 throws IOException

Throws:

IOException

createHStoreFile

private static org.apache.hadoop.hbase.regionserver.HStoreFile createHStoreFile(org.apache.hadoop.fs.Path storeDir,
 org.apache.hadoop.conf.Configuration conf,
 long timestamp,
 org.apache.hadoop.hbase.regionserver.HRegionFileSystem regionFs)
                                                                         throws IOException

Throws:

IOException

getConfWithCustomCellDataTieringEnabled

private static org.apache.hadoop.conf.Configuration getConfWithCustomCellDataTieringEnabled(long hotDataAge)

writeStoreFileRandomData

private static void writeStoreFileRandomData(org.apache.hadoop.hbase.regionserver.StoreFileWriter writer,
 byte[] columnFamily,
 long timestamp)
                                      throws IOException

Writes random data to a store file with rows arranged in lexicographically increasing order. Each row is generated using the nextString() method, ensuring that each subsequent row is lexicographically larger than the previous one.

Throws:

IOException

generateRandomBytes

private static byte[] generateRandomBytes(int sizeInBytes)

nextString

private static String nextString()

Returns the lexicographically larger string every time it's called.