Package org.apache.hadoop.hbase.io.hfile

Class TestHFileBlockIndex

java.lang.Object

org.apache.hadoop.hbase.io.hfile.TestHFileBlockIndex

public class TestHFileBlockIndex
extends Object

Nested Class Summary

Nested Classes

Modifier and Type	Class	Description
private static class	TestHFileBlockIndex.BlockReaderWrapper	A wrapper around a block reader which only caches the results of the last operation.
(package private) static class	TestHFileBlockIndex.MyEncoder

Field Summary

Fields

Modifier and Type	Field	Description
static final HBaseClassTestRule	CLASS_RULE
private final org.apache.hadoop.hbase.io.compress.Compression.Algorithm	compr
private org.apache.hadoop.conf.Configuration	conf
private static final int[]	EXPECTED_NUM_LEVELS
private byte[]	firstKeyInFile
private static org.apache.hadoop.fs.FileSystem	fs
private static final boolean	includesMemstoreTS
private static final int[]	INDEX_CHUNK_SIZES
private static final List<byte[]>	keys
private static final org.slf4j.Logger	LOG
private static final int	NUM_DATA_BLOCKS
private static final int	NUM_KV
private int	numLevels
private int	numRootEntries
private org.apache.hadoop.fs.Path	path
private static final Random	RNG
private long	rootIndexOffset
private static final int	SMALL_BLOCK_SIZE
private static final HBaseTestingUtil	TEST_UTIL
private static final int[]	UNCOMPRESSED_INDEX_SIZES

Constructor Summary

Constructors

Constructor	Description
TestHFileBlockIndex(org.apache.hadoop.hbase.io.compress.Compression.Algorithm compr)

Method Summary

Modifier and Type	Method	Description
private void	assertArrayEqualsBuffer(String msgPrefix, byte[] arr, ByteBuffer buf)
private void	checkKeyValue(String msgPrefix, byte[] expectedKey, byte[] expectedValue, ByteBuffer keyRead, ByteBuffer valueRead)	Check a key/value pair after it was read by the reader
private void	checkSeekTo(byte[][] keys, org.apache.hadoop.hbase.io.hfile.HFileScanner scanner, int i)
private void	clear()
static Collection<Object[]>	compressionAlgorithms()
private static final long	getDummyFileOffset(int i)
private static final int	getDummyOnDiskSize(int i)
private void	readIndex(boolean useTags)
void	setUp()
void	testBlockIndex()
void	testBlockIndexChunk()
private void	testBlockIndexInternals(boolean useTags)
void	testBlockIndexWithOffHeapBuffer()
void	testHeapSizeForBlockIndex()	Checks if the HeapSize calculator is within reason
void	testHFileWriterAndReader()	Testing block index through the HFile writer/reader APIs.
void	testIntermediateLevelIndicesWithLargeKeys()
void	testIntermediateLevelIndicesWithLargeKeys(int minNumEntries)
void	testIntermediateLevelIndicesWithLargeKeysWithMinNumEntries()
void	testMidKeyOnLeafIndexBlockBoundary()	to check if looks good when midKey on a leaf index block boundary
void	testMidKeyReadSuccessfullyFromRootIndexBlock()	This test is for HBASE-27940, which midkey metadata in root index block would always be ignored by HFileBlockIndex.BlockIndexReader.readMultiLevelIndexRoot(org.apache.hadoop.hbase.io.hfile.HFileBlock, int).
void	testSecondaryIndexBinarySearch()
private void	writeDataBlocksAndCreateIndex(org.apache.hadoop.hbase.io.hfile.HFileBlock.Writer hbw, org.apache.hadoop.fs.FSDataOutputStream outputStream, org.apache.hadoop.hbase.io.hfile.HFileBlockIndex.BlockIndexWriter biw)
private void	writeInlineBlocks(org.apache.hadoop.hbase.io.hfile.HFileBlock.Writer hbw, org.apache.hadoop.fs.FSDataOutputStream outputStream, org.apache.hadoop.hbase.io.hfile.HFileBlockIndex.BlockIndexWriter biw, boolean isClosing)
private void	writeWholeIndex(boolean useTags)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Details

CLASS_RULE

public static final HBaseClassTestRule CLASS_RULE

LOG

private static final org.slf4j.Logger LOG

RNG

private static final Random RNG

NUM_DATA_BLOCKS

private static final int NUM_DATA_BLOCKS

See Also:

• Constant Field Values

TEST_UTIL

private static final HBaseTestingUtil TEST_UTIL

SMALL_BLOCK_SIZE

private static final int SMALL_BLOCK_SIZE

See Also:

• Constant Field Values

NUM_KV

private static final int NUM_KV

See Also:

• Constant Field Values

fs

private static org.apache.hadoop.fs.FileSystem fs

path

private org.apache.hadoop.fs.Path path

rootIndexOffset

private long rootIndexOffset

numRootEntries

private int numRootEntries

numLevels

private int numLevels

keys

private static final List<byte[]> keys

compr

private final org.apache.hadoop.hbase.io.compress.Compression.Algorithm compr

firstKeyInFile

private byte[] firstKeyInFile

conf

private org.apache.hadoop.conf.Configuration conf

INDEX_CHUNK_SIZES

private static final int[] INDEX_CHUNK_SIZES

EXPECTED_NUM_LEVELS

private static final int[] EXPECTED_NUM_LEVELS

UNCOMPRESSED_INDEX_SIZES

private static final int[] UNCOMPRESSED_INDEX_SIZES

includesMemstoreTS

private static final boolean includesMemstoreTS

See Also:

• Constant Field Values

Constructor Details

TestHFileBlockIndex

public TestHFileBlockIndex(org.apache.hadoop.hbase.io.compress.Compression.Algorithm compr)

Method Details

compressionAlgorithms

public static Collection<Object[]> compressionAlgorithms()

setUp

public void setUp()
           throws IOException

Throws:

IOException

testBlockIndex

public void testBlockIndex()
                    throws IOException

Throws:

IOException

writeDataBlocksAndCreateIndex

private void writeDataBlocksAndCreateIndex(org.apache.hadoop.hbase.io.hfile.HFileBlock.Writer hbw,
 org.apache.hadoop.fs.FSDataOutputStream outputStream,
 org.apache.hadoop.hbase.io.hfile.HFileBlockIndex.BlockIndexWriter biw)
                                    throws IOException

Throws:

IOException

testBlockIndexWithOffHeapBuffer

public void testBlockIndexWithOffHeapBuffer()
                                     throws Exception

Throws:

Exception

clear

private void clear()
            throws IOException

Throws:

IOException

testBlockIndexInternals

private void testBlockIndexInternals(boolean useTags)
                              throws IOException

Throws:

IOException

readIndex

private void readIndex(boolean useTags)
                throws IOException

Throws:

IOException

writeWholeIndex

private void writeWholeIndex(boolean useTags)
                      throws IOException

Throws:

IOException

writeInlineBlocks

private void writeInlineBlocks(org.apache.hadoop.hbase.io.hfile.HFileBlock.Writer hbw,
 org.apache.hadoop.fs.FSDataOutputStream outputStream,
 org.apache.hadoop.hbase.io.hfile.HFileBlockIndex.BlockIndexWriter biw,
 boolean isClosing)
                        throws IOException

Throws:

IOException

getDummyFileOffset

private static final long getDummyFileOffset(int i)

getDummyOnDiskSize

private static final int getDummyOnDiskSize(int i)

testSecondaryIndexBinarySearch

public void testSecondaryIndexBinarySearch()
                                    throws IOException

Throws:

IOException

testBlockIndexChunk

public void testBlockIndexChunk()
                         throws IOException

Throws:

IOException

testHeapSizeForBlockIndex

public void testHeapSizeForBlockIndex()
                               throws IOException

Checks if the HeapSize calculator is within reason

Throws:

IOException

testMidKeyOnLeafIndexBlockBoundary

public void testMidKeyOnLeafIndexBlockBoundary()
                                        throws IOException

to check if looks good when midKey on a leaf index block boundary

Throws:

IOException

testHFileWriterAndReader

public void testHFileWriterAndReader()
                              throws IOException

Testing block index through the HFile writer/reader APIs. Allows to test setting index block size through configuration, intermediate-level index blocks, and caching index blocks on write.

Throws:

IOException

checkSeekTo

private void checkSeekTo(byte[][] keys,
 org.apache.hadoop.hbase.io.hfile.HFileScanner scanner,
 int i)
                  throws IOException

Throws:

IOException

assertArrayEqualsBuffer

private void assertArrayEqualsBuffer(String msgPrefix,
 byte[] arr,
 ByteBuffer buf)

checkKeyValue

private void checkKeyValue(String msgPrefix,
 byte[] expectedKey,
 byte[] expectedValue,
 ByteBuffer keyRead,
 ByteBuffer valueRead)

Check a key/value pair after it was read by the reader

testIntermediateLevelIndicesWithLargeKeys

public void testIntermediateLevelIndicesWithLargeKeys()
                                               throws IOException

Throws:

IOException

testIntermediateLevelIndicesWithLargeKeysWithMinNumEntries

public void testIntermediateLevelIndicesWithLargeKeysWithMinNumEntries()
                                                                throws IOException

Throws:

IOException

testIntermediateLevelIndicesWithLargeKeys

public void testIntermediateLevelIndicesWithLargeKeys(int minNumEntries)
                                               throws IOException

Throws:

IOException

testMidKeyReadSuccessfullyFromRootIndexBlock

public void testMidKeyReadSuccessfullyFromRootIndexBlock()
                                                  throws IOException

This test is for HBASE-27940, which midkey metadata in root index block would always be ignored by HFileBlockIndex.BlockIndexReader.readMultiLevelIndexRoot(org.apache.hadoop.hbase.io.hfile.HFileBlock, int).

Throws:

IOException