Package org.apache.hadoop.hbase.io.hfile

Class TestHFile

java.lang.Object

org.apache.hadoop.hbase.io.hfile.TestHFile

public class TestHFile
extends Object

test hfile features.

Field Summary

Fields

Modifier and Type	Field	Description
private static org.apache.hadoop.hbase.io.hfile.CacheConfig	cacheConf
static final HBaseClassTestRule	CLASS_RULE
private static org.apache.hadoop.conf.Configuration	conf
private static org.apache.hadoop.fs.FileSystem	fs
private static String	localFormatter
private static final org.slf4j.Logger	LOG
private final int	minBlockSize
private static final int	NUM_VALID_KEY_TYPES
private static String	ROOT_DIR
private static final HBaseTestingUtility	TEST_UTIL
org.junit.rules.TestName	testName

Constructor Summary

Constructors

Constructor	Description
TestHFile()

Method Summary

Modifier and Type	Method	Description
(package private) void	basicWithSomeCodec(String codec, boolean useTags)	test none codecs
private org.apache.hadoop.fs.FSDataOutputStream	createFSOutput(org.apache.hadoop.fs.Path name)
static org.apache.hadoop.hbase.io.hfile.HFile.Reader	createReaderFromStream(org.apache.hadoop.hbase.io.hfile.ReaderContext context, org.apache.hadoop.hbase.io.hfile.CacheConfig cacheConf, org.apache.hadoop.conf.Configuration conf)
private void	fillByteBuffAllocator(org.apache.hadoop.hbase.io.ByteBuffAllocator alloc, int bufCount)
static org.apache.hadoop.hbase.KeyValue.Type	generateKeyType(Random rand)
private org.apache.hadoop.hbase.Cell	getCell(byte[] row, byte[] family, byte[] qualifier)
private byte[]	getSomeKey(int rowId)
private org.apache.hadoop.hbase.io.ByteBuffAllocator	initAllocator(boolean reservoirEnabled, int bufSize, int bufCount, int minAllocSize)
private org.apache.hadoop.hbase.io.hfile.BlockCache	initCombinedBlockCache(String l1CachePolicy)
private void	metablocks(String compress)
private void	readAllRecords(org.apache.hadoop.hbase.io.hfile.HFileScanner scanner)
private int	readAndCheckbytes(org.apache.hadoop.hbase.io.hfile.HFileScanner scanner, int start, int n)
private long	readAtOffsetWithAllocationAsserts(org.apache.hadoop.hbase.io.ByteBuffAllocator alloc, org.apache.hadoop.hbase.io.hfile.HFile.Reader reader, long offset, boolean cacheBlock, org.apache.hadoop.hbase.io.hfile.BlockType blockType, boolean expectSharedMem)
private void	readNumMetablocks(org.apache.hadoop.hbase.io.hfile.HFile.Reader reader, int n)
private void	readStoreFile(org.apache.hadoop.fs.Path storeFilePath, org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.hbase.io.ByteBuffAllocator alloc)
static void	setUp()
private void	someReadingWithMetaBlock(org.apache.hadoop.hbase.io.hfile.HFile.Reader reader)
private void	someTestingWithMetaBlock(org.apache.hadoop.hbase.io.hfile.HFile.Writer writer)
void	testCompressionOrdinance()	Make sure the ordinals for our compression algorithms do not change on us.
void	testCorrupt0LengthHFile()	Create 0-length hfile and show that it fails
void	testCorruptOutOfOrderHFileWrite()
void	testCorruptTruncatedHFile()	Create a truncated hfile and verify that exception thrown.
void	testDBEShipped()
void	testEmptyHFile()	Test empty HFile.
void	testGetShortMidpoint()
void	testHFileContextBuilderWithIndexEncoding()
protected void	testHFilefeaturesInternals(boolean useTags)
void	testMetaBlocks()
void	testNullMetaBlocks()
void	testReaderBlockAllocationWithCombinedCache()	Tests that we properly allocate from the off-heap or on-heap when CombinedCache is configured.
private void	testReaderBlockAllocationWithCombinedCache(boolean cacheConfigCacheBlockOnRead, boolean cacheBlock, org.apache.hadoop.hbase.io.hfile.BlockType blockType, boolean expectSharedMem)
void	testReaderBlockAllocationWithLRUCache()	Tests that we properly allocate from the off-heap or on-heap when LRUCache is configured.
private void	testReaderBlockAllocationWithLRUCache(boolean cacheConfigCacheBlockOnRead, boolean cacheBlock, org.apache.hadoop.hbase.io.hfile.BlockType blockType, boolean expectSharedMem)
private void	testReaderCombinedCache(String l1CachePolicy)
void	testReaderWithAdaptiveLruCombinedBlockCache()	Test case for CombinedBlockCache with AdaptiveLRU as L1 cache
void	testReaderWithCombinedBlockCache()	Test case for HBASE-22127 in CombinedBlockCache
void	testReaderWithLRUBlockCache()	Test case for HBASE-22127 in LruBlockCache.
void	testReaderWithLruCombinedBlockCache()	Test case for CombinedBlockCache with AdaptiveLRU as L1 cache
void	testReaderWithoutBlockCache()
void	testReaderWithTinyLfuCombinedBlockCache()	Test case for CombinedBlockCache with TinyLfu as L1 cache
void	testShortMidpointSameQual()
void	testTFileFeatures()
static void	truncateFile(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path src, org.apache.hadoop.fs.Path dst)
private void	writeNumMetablocks(org.apache.hadoop.hbase.io.hfile.HFile.Writer writer, int n)
private void	writeRecords(org.apache.hadoop.hbase.io.hfile.HFile.Writer writer, boolean useTags)
private int	writeSomeRecords(org.apache.hadoop.hbase.io.hfile.HFile.Writer writer, int start, int n, boolean useTags)
private org.apache.hadoop.fs.Path	writeStoreFile()

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Details

CLASS_RULE

public static final HBaseClassTestRule CLASS_RULE

testName

public org.junit.rules.TestName testName

LOG

private static final org.slf4j.Logger LOG

NUM_VALID_KEY_TYPES

private static final int NUM_VALID_KEY_TYPES

TEST_UTIL

private static final HBaseTestingUtility TEST_UTIL

ROOT_DIR

private static String ROOT_DIR

minBlockSize

private final int minBlockSize

See Also:

• Constant Field Values

localFormatter

private static String localFormatter

cacheConf

private static org.apache.hadoop.hbase.io.hfile.CacheConfig cacheConf

conf

private static org.apache.hadoop.conf.Configuration conf

fs

private static org.apache.hadoop.fs.FileSystem fs

Constructor Details

TestHFile

public TestHFile()

Method Details

setUp

public static void setUp()
                  throws Exception

Throws:

Exception

createReaderFromStream

public static org.apache.hadoop.hbase.io.hfile.HFile.Reader createReaderFromStream(org.apache.hadoop.hbase.io.hfile.ReaderContext context,
 org.apache.hadoop.hbase.io.hfile.CacheConfig cacheConf,
 org.apache.hadoop.conf.Configuration conf)
                                                                            throws IOException

Throws:

IOException

initAllocator

private org.apache.hadoop.hbase.io.ByteBuffAllocator initAllocator(boolean reservoirEnabled,
 int bufSize,
 int bufCount,
 int minAllocSize)

fillByteBuffAllocator

private void fillByteBuffAllocator(org.apache.hadoop.hbase.io.ByteBuffAllocator alloc,
 int bufCount)

testReaderWithoutBlockCache

public void testReaderWithoutBlockCache()
                                 throws Exception

Throws:

Exception

testReaderWithLRUBlockCache

public void testReaderWithLRUBlockCache()
                                 throws Exception

Test case for HBASE-22127 in LruBlockCache.

Throws:

Exception

initCombinedBlockCache

private org.apache.hadoop.hbase.io.hfile.BlockCache initCombinedBlockCache(String l1CachePolicy)

testReaderWithCombinedBlockCache

public void testReaderWithCombinedBlockCache()
                                      throws Exception

Test case for HBASE-22127 in CombinedBlockCache

Throws:

Exception

testReaderBlockAllocationWithLRUCache

public void testReaderBlockAllocationWithLRUCache()
                                           throws IOException

Tests that we properly allocate from the off-heap or on-heap when LRUCache is configured. In this case, the determining factor is whether we end up caching the block or not. So the below test cases try different permutations of enabling/disabling via CacheConfig and via user request (cacheblocks), along with different expected block types.

Throws:

IOException

testReaderBlockAllocationWithLRUCache

private void testReaderBlockAllocationWithLRUCache(boolean cacheConfigCacheBlockOnRead,
 boolean cacheBlock,
 org.apache.hadoop.hbase.io.hfile.BlockType blockType,
 boolean expectSharedMem)
                                            throws IOException

Throws:

IOException

testReaderBlockAllocationWithCombinedCache

public void testReaderBlockAllocationWithCombinedCache()
                                                throws IOException

Tests that we properly allocate from the off-heap or on-heap when CombinedCache is configured. In this case, we should always use off-heap unless the block is an INDEX (which always goes to L1 cache which is on-heap)

Throws:

IOException

testReaderBlockAllocationWithCombinedCache

private void testReaderBlockAllocationWithCombinedCache(boolean cacheConfigCacheBlockOnRead,
 boolean cacheBlock,
 org.apache.hadoop.hbase.io.hfile.BlockType blockType,
 boolean expectSharedMem)
                                                 throws IOException

Throws:

IOException

readAtOffsetWithAllocationAsserts

private long readAtOffsetWithAllocationAsserts(org.apache.hadoop.hbase.io.ByteBuffAllocator alloc,
 org.apache.hadoop.hbase.io.hfile.HFile.Reader reader,
 long offset,
 boolean cacheBlock,
 org.apache.hadoop.hbase.io.hfile.BlockType blockType,
 boolean expectSharedMem)
                                        throws IOException

Throws:

IOException

readStoreFile

private void readStoreFile(org.apache.hadoop.fs.Path storeFilePath,
 org.apache.hadoop.conf.Configuration conf,
 org.apache.hadoop.hbase.io.ByteBuffAllocator alloc)
                    throws Exception

Throws:

Exception

writeStoreFile

private org.apache.hadoop.fs.Path writeStoreFile()
                                          throws IOException

Throws:

IOException

generateKeyType

public static org.apache.hadoop.hbase.KeyValue.Type generateKeyType(Random rand)

testEmptyHFile

public void testEmptyHFile()
                    throws IOException

Test empty HFile. Test all features work reasonably when hfile is empty of entries.

Throws:

IOException

testCorrupt0LengthHFile

public void testCorrupt0LengthHFile()
                             throws IOException

Create 0-length hfile and show that it fails

Throws:

IOException

testCorruptOutOfOrderHFileWrite

public void testCorruptOutOfOrderHFileWrite()
                                     throws IOException

Throws:

IOException

truncateFile

public static void truncateFile(org.apache.hadoop.fs.FileSystem fs,
 org.apache.hadoop.fs.Path src,
 org.apache.hadoop.fs.Path dst)
                         throws IOException

Throws:

IOException

testCorruptTruncatedHFile

public void testCorruptTruncatedHFile()
                               throws IOException

Create a truncated hfile and verify that exception thrown.

Throws:

IOException

writeSomeRecords

private int writeSomeRecords(org.apache.hadoop.hbase.io.hfile.HFile.Writer writer,
 int start,
 int n,
 boolean useTags)
                      throws IOException

Throws:

IOException

readAllRecords

private void readAllRecords(org.apache.hadoop.hbase.io.hfile.HFileScanner scanner)
                     throws IOException

Throws:

IOException

readAndCheckbytes

private int readAndCheckbytes(org.apache.hadoop.hbase.io.hfile.HFileScanner scanner,
 int start,
 int n)
                       throws IOException

Throws:

IOException

getSomeKey

private byte[] getSomeKey(int rowId)

writeRecords

private void writeRecords(org.apache.hadoop.hbase.io.hfile.HFile.Writer writer,
 boolean useTags)
                   throws IOException

Throws:

IOException

createFSOutput

private org.apache.hadoop.fs.FSDataOutputStream createFSOutput(org.apache.hadoop.fs.Path name)
                                                        throws IOException

Throws:

IOException

basicWithSomeCodec

void basicWithSomeCodec(String codec,
 boolean useTags)
                 throws IOException

test none codecs

Throws:

IOException

testTFileFeatures

public void testTFileFeatures()
                       throws IOException

Throws:

IOException

testHFilefeaturesInternals

protected void testHFilefeaturesInternals(boolean useTags)
                                   throws IOException

Throws:

IOException

writeNumMetablocks

private void writeNumMetablocks(org.apache.hadoop.hbase.io.hfile.HFile.Writer writer,
 int n)

someTestingWithMetaBlock

private void someTestingWithMetaBlock(org.apache.hadoop.hbase.io.hfile.HFile.Writer writer)

readNumMetablocks

private void readNumMetablocks(org.apache.hadoop.hbase.io.hfile.HFile.Reader reader,
 int n)
                        throws IOException

Throws:

IOException

someReadingWithMetaBlock

private void someReadingWithMetaBlock(org.apache.hadoop.hbase.io.hfile.HFile.Reader reader)
                               throws IOException

Throws:

IOException

metablocks

private void metablocks(String compress)
                 throws Exception

Throws:

Exception

testMetaBlocks

public void testMetaBlocks()
                    throws Exception

Throws:

Exception

testNullMetaBlocks

public void testNullMetaBlocks()
                        throws Exception

Throws:

Exception

testCompressionOrdinance

public void testCompressionOrdinance()

Make sure the ordinals for our compression algorithms do not change on us.

testShortMidpointSameQual

public void testShortMidpointSameQual()

getCell

private org.apache.hadoop.hbase.Cell getCell(byte[] row,
 byte[] family,
 byte[] qualifier)

testGetShortMidpoint

public void testGetShortMidpoint()

testDBEShipped

public void testDBEShipped()
                    throws IOException

Throws:

IOException

testReaderWithTinyLfuCombinedBlockCache

public void testReaderWithTinyLfuCombinedBlockCache()
                                             throws Exception

Test case for CombinedBlockCache with TinyLfu as L1 cache

Throws:

Exception

testReaderWithAdaptiveLruCombinedBlockCache

public void testReaderWithAdaptiveLruCombinedBlockCache()
                                                 throws Exception

Test case for CombinedBlockCache with AdaptiveLRU as L1 cache

Throws:

Exception

testReaderWithLruCombinedBlockCache

public void testReaderWithLruCombinedBlockCache()
                                         throws Exception

Test case for CombinedBlockCache with AdaptiveLRU as L1 cache

Throws:

Exception

testReaderCombinedCache

private void testReaderCombinedCache(String l1CachePolicy)
                              throws Exception

Throws:

Exception

testHFileContextBuilderWithIndexEncoding

public void testHFileContextBuilderWithIndexEncoding()
                                              throws IOException

Throws:

IOException