Package org.apache.hadoop.hbase.tool

Class BulkLoadHFilesTestBase

java.lang.Object

org.apache.hadoop.hbase.tool.BulkLoadHFilesTestBase

Direct Known Subclasses:

TestBulkLoadHFiles, TestBulkLoadHFilesSFT, TestSecureBulkLoadHFiles

public class BulkLoadHFilesTestBase
extends Object

Test cases for the "load" half of the HFileOutputFormat bulk load functionality. These tests run faster than the full MR cluster tests in TestHFileOutputFormat

Field Summary

Fields

Modifier and Type	Field	Description
(package private) static final String	EXPECTED_MSG_FOR_NON_EXISTING_FAMILY
private static final byte[]	FAMILY
(package private) static final int	MAX_FILES_PER_REGION_PER_FAMILY
private static final String	NAMESPACE
private static final byte[]	QUALIFIER
private static final byte[][]	SPLIT_KEYS
(package private) static HBaseTestingUtil	util

Constructor Summary

Constructors

Constructor	Description
BulkLoadHFilesTestBase()

Method Summary

Modifier and Type	Method	Description
private void	addStartEndKeysForTest(TreeMap<byte[],Integer> map, byte[] first, byte[] last)
private org.apache.hadoop.hbase.client.TableDescriptor	buildHTD(org.apache.hadoop.hbase.TableName tableName, org.apache.hadoop.hbase.regionserver.BloomType bloomType)
private static void	createRandomDataFile(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path path, int size)
static int	loadHFiles(String testName, org.apache.hadoop.hbase.client.TableDescriptor htd, HBaseTestingUtil util, byte[] fam, byte[] qual, boolean preCreateTable, byte[][] tableSplitKeys, byte[][][] hfileRanges, boolean useMap, boolean deleteFile, boolean copyFiles, int initRowCount, int factor, int depth)
private void	runTest(String testName, org.apache.hadoop.hbase.client.TableDescriptor htd, boolean preCreateTable, byte[][] tableSplitKeys, byte[][][] hfileRanges, boolean useMap, boolean copyFiles, int depth)
private void	runTest(String testName, org.apache.hadoop.hbase.regionserver.BloomType bloomType, byte[][][] hfileRanges)
private void	runTest(String testName, org.apache.hadoop.hbase.regionserver.BloomType bloomType, byte[][][] hfileRanges, boolean useMap)
private void	runTest(String testName, org.apache.hadoop.hbase.regionserver.BloomType bloomType, byte[][] tableSplitKeys, byte[][][] hfileRanges)
private void	runTest(String testName, org.apache.hadoop.hbase.regionserver.BloomType bloomType, byte[][] tableSplitKeys, byte[][][] hfileRanges, boolean useMap)
private void	runTest(String testName, org.apache.hadoop.hbase.TableName tableName, org.apache.hadoop.hbase.regionserver.BloomType bloomType, boolean preCreateTable, byte[][] tableSplitKeys, byte[][][] hfileRanges, boolean useMap, int depth)
protected static void	setupNamespace()
static void	tearDownAfterClass()
void	testBulkLoadByFamily(org.junit.jupiter.api.TestInfo testInfo)
void	testFailIfNeedSplitHFile(org.junit.jupiter.api.TestInfo testInfo)
void	testInferBoundaries()
void	testLoadTooMayHFiles()
void	testNonexistentColumnFamilyLoad(org.junit.jupiter.api.TestInfo testInfo)	Test loading into a column family that does not exist.
void	testNonHfileFolder()
private void	testNonHfileFolder(String tableName, boolean preCreateTable)	Write a random data file and a non-file in a dir with a valid family name but not part of the table families.
void	testNonHfileFolderWithUnmatchedFamilyName()
void	testRegionCrossingHFileSplit()	Test case that creates some regions and loads HFiles that cross the boundaries and have different region boundaries than the table pre-split.
private void	testRegionCrossingHFileSplit(org.apache.hadoop.hbase.regionserver.BloomType bloomType)
void	testRegionCrossingHFileSplitRowBloom()	Test case that creates some regions and loads HFiles that cross the boundaries have a ROW bloom filter and a different region boundaries than the table pre-split.
void	testRegionCrossingHFileSplitRowColBloom()	Test case that creates some regions and loads HFiles that cross the boundaries have a ROWCOL bloom filter and a different region boundaries than the table pre-split.
void	testRegionCrossingLoad()	Test case that creates some regions and loads HFiles that cross the boundaries of those regions
void	testRegionCrossingRowBloom()	Test loading into a column family that has a ROW bloom filter.
void	testRegionCrossingRowColBloom()	Test loading into a column family that has a ROWCOL bloom filter.
void	testSimpleHFileSplit()	Test case that creates some regions and loads HFiles that have different region boundaries than the table pre-split.
void	testSimpleLoad()	Test case that creates some regions and loads HFiles that fit snugly inside those regions
void	testSimpleLoadWithFileCopy(org.junit.jupiter.api.TestInfo testInfo)
void	testSimpleLoadWithMap()
void	testSplitALot()
void	testSplitStoreFile(org.junit.jupiter.api.TestInfo testInfo)
void	testSplitStoreFileWithCreateTimeTS(org.junit.jupiter.api.TestInfo testInfo)
private void	testSplitStoreFileWithDifferentEncoding(org.apache.hadoop.hbase.io.encoding.DataBlockEncoding bulkloadEncoding, org.apache.hadoop.hbase.io.encoding.DataBlockEncoding cfEncoding, org.junit.jupiter.api.TestInfo testInfo)
void	testSplitStoreFileWithEncodedToEncoded(org.junit.jupiter.api.TestInfo testInfo)
void	testSplitStoreFileWithEncodedToNone(org.junit.jupiter.api.TestInfo testInfo)
void	testSplitStoreFileWithFavoriteNodes(org.junit.jupiter.api.TestInfo testInfo)	Test hfile splits with the favored nodes
void	testSplitStoreFileWithNoneToEncoded(org.junit.jupiter.api.TestInfo testInfo)
void	testSplitStoreFileWithNoneToNone(org.junit.jupiter.api.TestInfo testInfo)
void	testTableWithCFNameStartWithUnderScore(org.junit.jupiter.api.TestInfo testInfo)
void	testTagsSurviveBulkLoadSplit(org.junit.jupiter.api.TestInfo testInfo)	Test that tags survive through a bulk load that needs to split hfiles.
void	testWithoutAnExistingTableAndCreateTableSetToNo()
private int	verifyHFile(org.apache.hadoop.fs.Path p)
private void	verifyHFileCreateTimeTS(org.apache.hadoop.fs.Path p)
private void	verifyHFileFavoriteNode(org.apache.hadoop.fs.Path p, org.apache.hadoop.hbase.client.AsyncTableRegionLocator regionLocator, org.apache.hadoop.fs.FileSystem fs)	test split storefile with favorite node information

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Details

QUALIFIER

private static final byte[] QUALIFIER

FAMILY

private static final byte[] FAMILY

NAMESPACE

private static final String NAMESPACE

See Also:

• Constant Field Values

EXPECTED_MSG_FOR_NON_EXISTING_FAMILY

static final String EXPECTED_MSG_FOR_NON_EXISTING_FAMILY

See Also:

• Constant Field Values

MAX_FILES_PER_REGION_PER_FAMILY

static final int MAX_FILES_PER_REGION_PER_FAMILY

See Also:

• Constant Field Values

SPLIT_KEYS

private static final byte[][] SPLIT_KEYS

util

static HBaseTestingUtil util

Constructor Details

BulkLoadHFilesTestBase

public BulkLoadHFilesTestBase()

Method Details

setupNamespace

protected static void setupNamespace()
                              throws Exception

Throws:

Exception

tearDownAfterClass

@AfterAll
public static void tearDownAfterClass()
                               throws Exception

Throws:

Exception

testSimpleLoadWithMap

@Test
public void testSimpleLoadWithMap()
                           throws Exception

Throws:

Exception

testSimpleLoad

@Test
public void testSimpleLoad()
                    throws Exception

Test case that creates some regions and loads HFiles that fit snugly inside those regions

Throws:

Exception

testSimpleLoadWithFileCopy

@Test
public void testSimpleLoadWithFileCopy(org.junit.jupiter.api.TestInfo testInfo)
                                throws Exception

Throws:

Exception

testRegionCrossingLoad

@Test
public void testRegionCrossingLoad()
                            throws Exception

Test case that creates some regions and loads HFiles that cross the boundaries of those regions

Throws:

Exception

testRegionCrossingRowBloom

@Test
public void testRegionCrossingRowBloom()
                                throws Exception

Test loading into a column family that has a ROW bloom filter.

Throws:

Exception

testRegionCrossingRowColBloom

@Test
public void testRegionCrossingRowColBloom()
                                   throws Exception

Test loading into a column family that has a ROWCOL bloom filter.

Throws:

Exception

testSimpleHFileSplit

@Test
public void testSimpleHFileSplit()
                          throws Exception

Test case that creates some regions and loads HFiles that have different region boundaries than the table pre-split.

Throws:

Exception

testRegionCrossingHFileSplit

@Test
public void testRegionCrossingHFileSplit()
                                  throws Exception

Test case that creates some regions and loads HFiles that cross the boundaries and have different region boundaries than the table pre-split.

Throws:

Exception

testRegionCrossingHFileSplitRowBloom

@Test
public void testRegionCrossingHFileSplitRowBloom()
                                          throws Exception

Test case that creates some regions and loads HFiles that cross the boundaries have a ROW bloom filter and a different region boundaries than the table pre-split.

Throws:

Exception

testRegionCrossingHFileSplitRowColBloom

@Test
public void testRegionCrossingHFileSplitRowColBloom()
                                             throws Exception

Test case that creates some regions and loads HFiles that cross the boundaries have a ROWCOL bloom filter and a different region boundaries than the table pre-split.

Throws:

Exception

testSplitALot

@Test
public void testSplitALot()
                   throws Exception

Throws:

Exception

testRegionCrossingHFileSplit

private void testRegionCrossingHFileSplit(org.apache.hadoop.hbase.regionserver.BloomType bloomType)
                                   throws Exception

Throws:

Exception

buildHTD

private org.apache.hadoop.hbase.client.TableDescriptor buildHTD(org.apache.hadoop.hbase.TableName tableName,
 org.apache.hadoop.hbase.regionserver.BloomType bloomType)

runTest

private void runTest(String testName,
 org.apache.hadoop.hbase.regionserver.BloomType bloomType,
 byte[][][] hfileRanges)
              throws Exception

Throws:

Exception

runTest

private void runTest(String testName,
 org.apache.hadoop.hbase.regionserver.BloomType bloomType,
 byte[][][] hfileRanges,
 boolean useMap)
              throws Exception

Throws:

Exception

runTest

private void runTest(String testName,
 org.apache.hadoop.hbase.regionserver.BloomType bloomType,
 byte[][] tableSplitKeys,
 byte[][][] hfileRanges)
              throws Exception

Throws:

Exception

runTest

private void runTest(String testName,
 org.apache.hadoop.hbase.regionserver.BloomType bloomType,
 byte[][] tableSplitKeys,
 byte[][][] hfileRanges,
 boolean useMap)
              throws Exception

Throws:

Exception

runTest

private void runTest(String testName,
 org.apache.hadoop.hbase.TableName tableName,
 org.apache.hadoop.hbase.regionserver.BloomType bloomType,
 boolean preCreateTable,
 byte[][] tableSplitKeys,
 byte[][][] hfileRanges,
 boolean useMap,
 int depth)
              throws Exception

Throws:

Exception

loadHFiles

public static int loadHFiles(String testName,
 org.apache.hadoop.hbase.client.TableDescriptor htd,
 HBaseTestingUtil util,
 byte[] fam,
 byte[] qual,
 boolean preCreateTable,
 byte[][] tableSplitKeys,
 byte[][][] hfileRanges,
 boolean useMap,
 boolean deleteFile,
 boolean copyFiles,
 int initRowCount,
 int factor,
 int depth)
                      throws Exception

Throws:

Exception

runTest

private void runTest(String testName,
 org.apache.hadoop.hbase.client.TableDescriptor htd,
 boolean preCreateTable,
 byte[][] tableSplitKeys,
 byte[][][] hfileRanges,
 boolean useMap,
 boolean copyFiles,
 int depth)
              throws Exception

Throws:

Exception

testTagsSurviveBulkLoadSplit

@Test
public void testTagsSurviveBulkLoadSplit(org.junit.jupiter.api.TestInfo testInfo)
                                  throws Exception

Test that tags survive through a bulk load that needs to split hfiles. This test depends on the "hbase.client.rpc.codec" = KeyValueCodecWithTags so that the client can get tags in the responses.

Throws:

Exception

testNonexistentColumnFamilyLoad

@Test
public void testNonexistentColumnFamilyLoad(org.junit.jupiter.api.TestInfo testInfo)
                                     throws Exception

Test loading into a column family that does not exist.

Throws:

Exception

testNonHfileFolderWithUnmatchedFamilyName

@Test
public void testNonHfileFolderWithUnmatchedFamilyName()
                                               throws Exception

Throws:

Exception

testNonHfileFolder

@Test
public void testNonHfileFolder()
                        throws Exception

Throws:

Exception

testNonHfileFolder

private void testNonHfileFolder(String tableName,
 boolean preCreateTable)
                         throws Exception

Write a random data file and a non-file in a dir with a valid family name but not part of the table families. we should we able to bulkload without getting the unmatched family exception. HBASE-13037/HBASE-13227

Throws:

Exception

createRandomDataFile

private static void createRandomDataFile(org.apache.hadoop.fs.FileSystem fs,
 org.apache.hadoop.fs.Path path,
 int size)
                                  throws IOException

Throws:

IOException

testSplitStoreFile

@Test
public void testSplitStoreFile(org.junit.jupiter.api.TestInfo testInfo)
                        throws IOException

Throws:

IOException

testSplitStoreFileWithFavoriteNodes

@Test
public void testSplitStoreFileWithFavoriteNodes(org.junit.jupiter.api.TestInfo testInfo)
                                         throws IOException

Test hfile splits with the favored nodes

Throws:

IOException

testSplitStoreFileWithCreateTimeTS

@Test
public void testSplitStoreFileWithCreateTimeTS(org.junit.jupiter.api.TestInfo testInfo)
                                        throws IOException

Throws:

IOException

testSplitStoreFileWithNoneToNone

@Test
public void testSplitStoreFileWithNoneToNone(org.junit.jupiter.api.TestInfo testInfo)
                                      throws IOException

Throws:

IOException

testSplitStoreFileWithEncodedToEncoded

@Test
public void testSplitStoreFileWithEncodedToEncoded(org.junit.jupiter.api.TestInfo testInfo)
                                            throws IOException

Throws:

IOException

testSplitStoreFileWithEncodedToNone

@Test
public void testSplitStoreFileWithEncodedToNone(org.junit.jupiter.api.TestInfo testInfo)
                                         throws IOException

Throws:

IOException

testSplitStoreFileWithNoneToEncoded

@Test
public void testSplitStoreFileWithNoneToEncoded(org.junit.jupiter.api.TestInfo testInfo)
                                         throws IOException

Throws:

IOException

testSplitStoreFileWithDifferentEncoding

private void testSplitStoreFileWithDifferentEncoding(org.apache.hadoop.hbase.io.encoding.DataBlockEncoding bulkloadEncoding,
 org.apache.hadoop.hbase.io.encoding.DataBlockEncoding cfEncoding,
 org.junit.jupiter.api.TestInfo testInfo)
                                              throws IOException

Throws:

IOException

verifyHFile

private int verifyHFile(org.apache.hadoop.fs.Path p)
                 throws IOException

Throws:

IOException

verifyHFileCreateTimeTS

private void verifyHFileCreateTimeTS(org.apache.hadoop.fs.Path p)
                              throws IOException

Throws:

IOException

verifyHFileFavoriteNode

private void verifyHFileFavoriteNode(org.apache.hadoop.fs.Path p,
 org.apache.hadoop.hbase.client.AsyncTableRegionLocator regionLocator,
 org.apache.hadoop.fs.FileSystem fs)
                              throws IOException

test split storefile with favorite node information

Throws:

IOException

addStartEndKeysForTest

private void addStartEndKeysForTest(TreeMap<byte[],Integer> map,
 byte[] first,
 byte[] last)

testInferBoundaries

@Test
public void testInferBoundaries()

testLoadTooMayHFiles

@Test
public void testLoadTooMayHFiles()
                          throws Exception

Throws:

Exception

testWithoutAnExistingTableAndCreateTableSetToNo

@Test
public void testWithoutAnExistingTableAndCreateTableSetToNo()
                                                     throws Exception

Throws:

Exception

testTableWithCFNameStartWithUnderScore

@Test
public void testTableWithCFNameStartWithUnderScore(org.junit.jupiter.api.TestInfo testInfo)
                                            throws Exception

Throws:

Exception

testBulkLoadByFamily

@Test
public void testBulkLoadByFamily(org.junit.jupiter.api.TestInfo testInfo)
                          throws Exception

Throws:

Exception

testFailIfNeedSplitHFile

@Test
public void testFailIfNeedSplitHFile(org.junit.jupiter.api.TestInfo testInfo)
                              throws IOException

Throws:

IOException