org.apache.hadoop.hbase.mob.compactions

Class TestMobCompactor

java.lang.Object

org.apache.hadoop.hbase.mob.compactions.TestMobCompactor

public class TestMobCompactor
extends Object

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
static class	TestMobCompactor.CompactTwoLatestHfilesCopro This copro overwrites the default compaction policy.

Field Summary

Fields

Modifier and Type	Field and Description
private static org.apache.hadoop.hbase.client.Admin	admin
private org.apache.hadoop.hbase.client.BufferedMutator	bufMut
private static int	cellNumPerRow
static HBaseClassTestRule	CLASS_RULE
private static org.apache.hadoop.conf.Configuration	conf
private static org.apache.hadoop.hbase.client.Connection	conn
private static int	delCellNum
private static int	delRowNum
private org.apache.hadoop.hbase.HTableDescriptor	desc
private static String	family1
private static String	family2
private static org.apache.hadoop.fs.FileSystem	fs
private org.apache.hadoop.hbase.HColumnDescriptor	hcd1
private org.apache.hadoop.hbase.HColumnDescriptor	hcd2
private static byte[]	KEYS
private static org.slf4j.Logger	LOG
private static byte[]	mobKey01
private static byte[]	mobKey02
private static byte[]	mobKey03
private static byte[]	mobKey04
private static byte[]	mobKey05
private static byte[]	mobKey06
private static byte[]	mobKey1
private static byte[]	mobKey2
private static byte[]	mobKey3
private static byte[]	mobKey4
private static byte[]	mobKey5
private static byte[]	mobKey6
private static byte[]	mobKey7
private static byte[]	mobKey8
private static String	mobValue0
private static String	mobValue1
private static String	mobValue2
private static String	mobValue3
private static String	mobValue4
private static String	mobValue5
private static String	mobValue6
private static String	mobValue7
private static String	mobValue8
org.junit.rules.TestName	name
private static ExecutorService	pool
private static String	qf1
private static String	qf2
private static int	regionNum
private static int	rowNumPerFile
private org.apache.hadoop.hbase.client.Table	table
private org.apache.hadoop.hbase.TableName	tableName
private static HBaseTestingUtility	TEST_UTIL
private static long	tsFor20150907Monday
private static long	tsFor20151120Sunday
private static long	tsFor20151128Saturday
private static long	tsFor20151130Monday
private static long	tsFor20151201Tuesday
private static long	tsFor20151205Saturday
private static long	tsFor20151228Monday
private static long	tsFor20151231Thursday
private static long	tsFor20160101Friday
private static long	tsFor20160103Sunday

Constructor Summary

Constructors

Constructor and Description
TestMobCompactor()

Method Summary

Modifier and Type	Method and Description
private void	alterForPolicyTest(org.apache.hadoop.hbase.client.MobCompactPartitionPolicy type)
private void	assertRefFileNameEqual(String familyName)
private void	commonPolicyTestLogic(String tableNameAsString, org.apache.hadoop.hbase.client.MobCompactPartitionPolicy pType, boolean majorCompact, int expectedFileNumbers, String[] expectedFileNames, boolean setupAndLoadData)
private int	countFiles(org.apache.hadoop.hbase.TableName tableName, boolean isMobFile, String familyName) Gets the number of files in the mob path.
private int	countHFileLinks(String familyName) Gets the number of HFileLink in the mob path.
private int	countLargeFiles(int size, org.apache.hadoop.hbase.TableName tableName, String familyName) Gets the number of files.
private int	countMobCells(org.apache.hadoop.hbase.client.Table table) Gets the number of cells in the given table.
private int	countMobRows(org.apache.hadoop.hbase.client.Table table) Gets the number of rows in the given table.
private void	createDelFile(org.apache.hadoop.hbase.client.Table table, org.apache.hadoop.hbase.TableName tableName, byte[] family, byte[] qf) delete the row, family and cell to create the del file
private static ExecutorService	createThreadPool(org.apache.hadoop.conf.Configuration conf)
private byte[][]	getSplitKeys() Gets the split keys
private void	loadData(org.apache.hadoop.hbase.client.Admin admin, org.apache.hadoop.hbase.client.BufferedMutator table, org.apache.hadoop.hbase.TableName tableName, int fileNum, int rowNumPerFile) loads some data to the table.
private void	loadData(org.apache.hadoop.hbase.client.Admin admin, org.apache.hadoop.hbase.client.BufferedMutator table, org.apache.hadoop.hbase.TableName tableName, org.apache.hadoop.hbase.client.Put[] puts)
private void	loadDataForPartitionPolicy(org.apache.hadoop.hbase.client.Admin admin, org.apache.hadoop.hbase.client.BufferedMutator table, org.apache.hadoop.hbase.TableName tableName)
private byte[]	makeDummyData(int size) Creates the dummy data with a specific size.
private void	resetConf() Resets the configuration.
void	setUp(String tableNameAsString)
static void	setUpBeforeClass()
private void	setUpForPolicyTest(String tableNameAsString, org.apache.hadoop.hbase.client.MobCompactPartitionPolicy type)
static void	tearDownAfterClass()
void	testCompactionWithHFileLink()
void	testGetAfterCompaction() This case tests the following mob compaction and normal compaction scenario, after mob compaction, the mob reference in new bulkloaded hfile will win even after it is compacted with some other normal hfiles.
void	testMajorCompactionFromAdmin()
void	testMajorCompactionWithMonthlyPolicy()
void	testMajorCompactionWithWeeklyFollowedByMonthly()
void	testMajorCompactionWithWeeklyFollowedByMonthlyFollowedByDaily()
void	testMajorCompactionWithWeeklyFollowedByMonthlyFollowedByWeekly()
void	testMajorCompactionWithWeeklyPolicy()
void	testMinorCompaction()
void	testMinorCompactionWithMonthlyPolicy()
void	testMinorCompactionWithWeeklyPolicy()
void	testScannerOnBulkLoadRefHFiles()
private boolean	verifyEncryption(org.apache.hadoop.hbase.TableName tableName, String familyName)
private void	verifyPolicyValues() Verify mob partition policy compaction values.
private void	waitUntilFilesShowup(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path path, int num)
private void	waitUntilMobCompactionFinished(org.apache.hadoop.hbase.TableName tableName)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

CLASS_RULE

public static final HBaseClassTestRule CLASS_RULE

LOG

private static final org.slf4j.Logger LOG

TEST_UTIL

private static final HBaseTestingUtility TEST_UTIL

conf

private static org.apache.hadoop.conf.Configuration conf

tableName

private org.apache.hadoop.hbase.TableName tableName

conn

private static org.apache.hadoop.hbase.client.Connection conn

bufMut

private org.apache.hadoop.hbase.client.BufferedMutator bufMut

table

private org.apache.hadoop.hbase.client.Table table

admin

private static org.apache.hadoop.hbase.client.Admin admin

desc

private org.apache.hadoop.hbase.HTableDescriptor desc

hcd1

private org.apache.hadoop.hbase.HColumnDescriptor hcd1

hcd2

private org.apache.hadoop.hbase.HColumnDescriptor hcd2

fs

private static org.apache.hadoop.fs.FileSystem fs

family1

private static final String family1

See Also:

Constant Field Values

family2

private static final String family2

See Also:

Constant Field Values

qf1

private static final String qf1

See Also:

Constant Field Values

qf2

private static final String qf2

See Also:

Constant Field Values

tsFor20150907Monday

private static long tsFor20150907Monday

tsFor20151120Sunday

private static long tsFor20151120Sunday

tsFor20151128Saturday

private static long tsFor20151128Saturday

tsFor20151130Monday

private static long tsFor20151130Monday

tsFor20151201Tuesday

private static long tsFor20151201Tuesday

tsFor20151205Saturday

private static long tsFor20151205Saturday

tsFor20151228Monday

private static long tsFor20151228Monday

tsFor20151231Thursday

private static long tsFor20151231Thursday

tsFor20160101Friday

private static long tsFor20160101Friday

tsFor20160103Sunday

private static long tsFor20160103Sunday

mobKey01

private static final byte[] mobKey01

mobKey02

private static final byte[] mobKey02

mobKey03

private static final byte[] mobKey03

mobKey04

private static final byte[] mobKey04

mobKey05

private static final byte[] mobKey05

mobKey06

private static final byte[] mobKey06

mobKey1

private static final byte[] mobKey1

mobKey2

private static final byte[] mobKey2

mobKey3

private static final byte[] mobKey3

mobKey4

private static final byte[] mobKey4

mobKey5

private static final byte[] mobKey5

mobKey6

private static final byte[] mobKey6

mobKey7

private static final byte[] mobKey7

mobKey8

private static final byte[] mobKey8

mobValue0

private static final String mobValue0

See Also:

Constant Field Values

mobValue1

private static final String mobValue1

See Also:

Constant Field Values

mobValue2

private static final String mobValue2

See Also:

Constant Field Values

mobValue3

private static final String mobValue3

See Also:

Constant Field Values

mobValue4

private static final String mobValue4

See Also:

Constant Field Values

mobValue5

private static final String mobValue5

See Also:

Constant Field Values

mobValue6

private static final String mobValue6

See Also:

Constant Field Values

mobValue7

private static final String mobValue7

See Also:

Constant Field Values

mobValue8

private static final String mobValue8

See Also:

Constant Field Values

KEYS

private static byte[] KEYS

regionNum

private static int regionNum

delRowNum

private static int delRowNum

delCellNum

private static int delCellNum

cellNumPerRow

private static int cellNumPerRow

rowNumPerFile

private static int rowNumPerFile

pool

private static ExecutorService pool

name

public org.junit.rules.TestName name

Constructor Detail

TestMobCompactor

public TestMobCompactor()

Method Detail

setUpBeforeClass

public static void setUpBeforeClass()
                             throws Exception

Throws:

Exception

tearDownAfterClass

public static void tearDownAfterClass()
                               throws Exception

Throws:

Exception

setUp

public void setUp(String tableNameAsString)
           throws IOException

Throws:

IOException

setUpForPolicyTest

private void setUpForPolicyTest(String tableNameAsString,
                                org.apache.hadoop.hbase.client.MobCompactPartitionPolicy type)
                         throws IOException

Throws:

IOException

alterForPolicyTest

private void alterForPolicyTest(org.apache.hadoop.hbase.client.MobCompactPartitionPolicy type)
                         throws Exception

Throws:

Exception

testMinorCompaction

public void testMinorCompaction()
                         throws Exception

Throws:

Exception

testMinorCompactionWithWeeklyPolicy

public void testMinorCompactionWithWeeklyPolicy()
                                         throws Exception

Throws:

Exception

testMajorCompactionWithWeeklyPolicy

public void testMajorCompactionWithWeeklyPolicy()
                                         throws Exception

Throws:

Exception

testMinorCompactionWithMonthlyPolicy

public void testMinorCompactionWithMonthlyPolicy()
                                          throws Exception

Throws:

Exception

testMajorCompactionWithMonthlyPolicy

public void testMajorCompactionWithMonthlyPolicy()
                                          throws Exception

Throws:

Exception

testMajorCompactionWithWeeklyFollowedByMonthly

public void testMajorCompactionWithWeeklyFollowedByMonthly()
                                                    throws Exception

Throws:

Exception

testMajorCompactionWithWeeklyFollowedByMonthlyFollowedByWeekly

public void testMajorCompactionWithWeeklyFollowedByMonthlyFollowedByWeekly()
                                                                    throws Exception

Throws:

Exception

testMajorCompactionWithWeeklyFollowedByMonthlyFollowedByDaily

public void testMajorCompactionWithWeeklyFollowedByMonthlyFollowedByDaily()
                                                                   throws Exception

Throws:

Exception

testCompactionWithHFileLink

public void testCompactionWithHFileLink()
                                 throws IOException,
                                        InterruptedException

Throws:

IOException

InterruptedException

testMajorCompactionFromAdmin

public void testMajorCompactionFromAdmin()
                                  throws Exception

Throws:

Exception

testScannerOnBulkLoadRefHFiles

public void testScannerOnBulkLoadRefHFiles()
                                    throws Exception

Throws:

Exception

testGetAfterCompaction

public void testGetAfterCompaction()
                            throws Exception

This case tests the following mob compaction and normal compaction scenario, after mob compaction, the mob reference in new bulkloaded hfile will win even after it is compacted with some other normal hfiles. This is to make sure the mvcc is included after compaction for mob enabled store files.

Throws:

Exception

waitUntilFilesShowup

private void waitUntilFilesShowup(org.apache.hadoop.fs.FileSystem fs,
                                  org.apache.hadoop.fs.Path path,
                                  int num)
                           throws InterruptedException,
                                  IOException

Throws:

InterruptedException

IOException

waitUntilMobCompactionFinished

private void waitUntilMobCompactionFinished(org.apache.hadoop.hbase.TableName tableName)
                                     throws IOException,
                                            InterruptedException

Throws:

IOException

InterruptedException

countMobRows

private int countMobRows(org.apache.hadoop.hbase.client.Table table)
                  throws IOException

Gets the number of rows in the given table.

Parameters:

table - to get the scanner

Returns:

the number of rows

Throws:

IOException

countMobCells

private int countMobCells(org.apache.hadoop.hbase.client.Table table)
                   throws IOException

Gets the number of cells in the given table.

Parameters:

table - to get the scanner

Returns:

the number of cells

Throws:

IOException

countFiles

private int countFiles(org.apache.hadoop.hbase.TableName tableName,
                       boolean isMobFile,
                       String familyName)
                throws IOException

Gets the number of files in the mob path.

Parameters:

isMobFile - gets number of the mob files or del files

familyName - the family name

Returns:

the number of the files

Throws:

IOException

verifyEncryption

private boolean verifyEncryption(org.apache.hadoop.hbase.TableName tableName,
                                 String familyName)
                          throws IOException

Throws:

IOException

countHFileLinks

private int countHFileLinks(String familyName)
                     throws IOException

Gets the number of HFileLink in the mob path.

Parameters:

familyName - the family name

Returns:

the number of the HFileLink

Throws:

IOException

countLargeFiles

private int countLargeFiles(int size,
                            org.apache.hadoop.hbase.TableName tableName,
                            String familyName)
                     throws IOException

Gets the number of files.

Parameters:

size - the size of the file

tableName - the current table name

familyName - the family name

Returns:

the number of files large than the size

Throws:

IOException

loadData

private void loadData(org.apache.hadoop.hbase.client.Admin admin,
                      org.apache.hadoop.hbase.client.BufferedMutator table,
                      org.apache.hadoop.hbase.TableName tableName,
                      int fileNum,
                      int rowNumPerFile)
               throws IOException,
                      InterruptedException

loads some data to the table.

Throws:

IOException

InterruptedException

loadData

private void loadData(org.apache.hadoop.hbase.client.Admin admin,
                      org.apache.hadoop.hbase.client.BufferedMutator table,
                      org.apache.hadoop.hbase.TableName tableName,
                      org.apache.hadoop.hbase.client.Put[] puts)
               throws IOException

Throws:

IOException

loadDataForPartitionPolicy

private void loadDataForPartitionPolicy(org.apache.hadoop.hbase.client.Admin admin,
                                        org.apache.hadoop.hbase.client.BufferedMutator table,
                                        org.apache.hadoop.hbase.TableName tableName)
                                 throws IOException

Throws:

IOException

createDelFile

private void createDelFile(org.apache.hadoop.hbase.client.Table table,
                           org.apache.hadoop.hbase.TableName tableName,
                           byte[] family,
                           byte[] qf)
                    throws IOException,
                           InterruptedException

delete the row, family and cell to create the del file

Throws:

IOException

InterruptedException

makeDummyData

private byte[] makeDummyData(int size)

Creates the dummy data with a specific size.

Parameters:

size - the size of value

Returns:

the dummy data

getSplitKeys

private byte[][] getSplitKeys()

Gets the split keys

createThreadPool

private static ExecutorService createThreadPool(org.apache.hadoop.conf.Configuration conf)

assertRefFileNameEqual

private void assertRefFileNameEqual(String familyName)
                             throws IOException

Throws:

IOException

resetConf

private void resetConf()

Resets the configuration.

verifyPolicyValues

private void verifyPolicyValues()
                         throws Exception

Verify mob partition policy compaction values.

Throws:

Exception

commonPolicyTestLogic

private void commonPolicyTestLogic(String tableNameAsString,
                                   org.apache.hadoop.hbase.client.MobCompactPartitionPolicy pType,
                                   boolean majorCompact,
                                   int expectedFileNumbers,
                                   String[] expectedFileNames,
                                   boolean setupAndLoadData)
                            throws Exception

Throws:

Exception