org.apache.hadoop.hbase.mob.compactions

Class TestPartitionedMobCompactor

java.lang.Object

org.apache.hadoop.hbase.mob.compactions.TestPartitionedMobCompactor

public class TestPartitionedMobCompactor
extends Object

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
(package private) static class	TestPartitionedMobCompactor.FaultyDistributedFileSystem The customized Distributed File System Implementation
(package private) static class	TestPartitionedMobCompactor.MyPartitionedMobCompactor

Field Summary

Fields

Modifier and Type	Field and Description
private List<org.apache.hadoop.fs.FileStatus>	allFiles
private org.apache.hadoop.fs.Path	basePath
private org.apache.hadoop.hbase.io.hfile.CacheConfig	cacheConf
static HBaseClassTestRule	CLASS_RULE
private org.apache.hadoop.conf.Configuration	conf
private long	DAY_IN_MS
private List<org.apache.hadoop.fs.Path>	delFiles
private String	delSuffix
private static String	family
private org.apache.hadoop.fs.FileSystem	fs
private org.apache.hadoop.hbase.HColumnDescriptor	hcd
private static byte[]	KEYS
private static org.slf4j.Logger	LOG
private List<org.apache.hadoop.fs.FileStatus>	mobFiles
private String	mobSuffix
org.junit.rules.TestName	name
private static ExecutorService	pool
private static String	qf
private static HBaseTestingUtility	TEST_UTIL

Constructor Summary

Constructors

Constructor and Description
TestPartitionedMobCompactor()

Method Summary

Modifier and Type	Method and Description
private void	compareCompactedPartitions(List<String> expected, Collection<org.apache.hadoop.hbase.mob.compactions.PartitionedMobCompactionRequest.CompactionPartition> partitions) Compares the compacted partitions.
private void	compareDelFiles(List<org.apache.hadoop.hbase.mob.compactions.PartitionedMobCompactionRequest.CompactionDelPartition> delPartitions) Compares the del files.
private int	countDelCellsInDelFiles(List<org.apache.hadoop.fs.Path> paths) Gets the number of del cell in the del files
private void	createMobDelFile(org.apache.hadoop.fs.Path basePath, int startKey) Create mulitple partition delete files
private void	createMobFile(org.apache.hadoop.fs.Path basePath) Create mulitple partition files
private void	createStoreFiles(org.apache.hadoop.fs.Path basePath, String family, String qualifier, int count, org.apache.hadoop.hbase.KeyValue.Type type, boolean sameStartKey, Date date)
private void	createStoreFiles(org.apache.hadoop.fs.Path basePath, String family, String qualifier, int count, org.apache.hadoop.hbase.KeyValue.Type type, Date date) Creates store files.
private static ExecutorService	createThreadPool()
private void	init(String tableName)
private void	listFiles() Lists the files in the path
private void	resetConf() Resets the configuration.
static void	setUpBeforeClass()
static void	tearDownAfterClass()
private void	testCompactDelFiles(String tableName, int expectedFileCount, int expectedCellCount, boolean isForceAllFiles) Tests the compacteDelFile
private void	testCompactDelFilesAtBatchSize(String tableName, int batchSize, int delfileMaxCount)
void	testCompactDelFilesChangeMaxDelFileCount()
void	testCompactDelFilesWithDefaultBatchSize()
void	testCompactDelFilesWithSmallBatchSize()
void	testCompactFilesWithDstDirFull()
void	testCompactFilesWithMultiDelFiles()
void	testCompactFilesWithOneDelFile()
void	testCompactFilesWithoutDelFile()
private void	testCompactionAtMergeSize(String tableName, long mergeSize, org.apache.hadoop.hbase.mob.compactions.MobCompactionRequest.CompactionType type, boolean isForceAllFiles)
private void	testCompactionAtMergeSize(String tableName, long mergeSize, org.apache.hadoop.hbase.mob.compactions.MobCompactionRequest.CompactionType type, boolean isForceAllFiles, boolean createDelFiles)
private void	testCompactionAtMergeSize(String tableName, long mergeSize, org.apache.hadoop.hbase.mob.compactions.MobCompactionRequest.CompactionType type, boolean isForceAllFiles, boolean createDelFiles, Date date)
private void	testCompactionAtMergeSize(String tableName, long mergeSize, org.apache.hadoop.hbase.mob.compactions.MobCompactionRequest.CompactionType type, boolean isForceAllFiles, boolean createDelFiles, Date date, org.apache.hadoop.hbase.client.MobCompactPartitionPolicy policy, long mergeSizeMultiFactor)
void	testCompactionSelectAllFilesMonthlyPolicy()
void	testCompactionSelectAllFilesMonthlyPolicyWithPastMonth()
void	testCompactionSelectAllFilesMonthlyPolicyWithPastWeek()
void	testCompactionSelectAllFilesWeeklyPolicy()
void	testCompactionSelectAllFilesWeeklyPolicyWithPastWeek()
void	testCompactionSelectNoFilesWithinCurrentWeekMonthlyPolicy()
void	testCompactionSelectPartFilesMonthlyPolicy()
void	testCompactionSelectPartFilesMonthlyPolicyWithPastMonth()
void	testCompactionSelectPartFilesMonthlyPolicyWithPastWeek()
void	testCompactionSelectPartFilesWeeklyPolicy()
void	testCompactionSelectPartFilesWeeklyPolicyWithPastWeek()
void	testCompactionSelectWithAllFiles()
void	testCompactionSelectWithForceAllFiles()
void	testCompactionSelectWithPartFiles()
private void	testSelectFiles(String tableName, org.apache.hadoop.hbase.mob.compactions.MobCompactionRequest.CompactionType type, boolean isForceAllFiles, List<String> expected) Tests the selectFiles
private static void	writeStoreFile(org.apache.hadoop.hbase.regionserver.StoreFileWriter writer, byte[] row, byte[] family, byte[] qualifier, org.apache.hadoop.hbase.KeyValue.Type type, int size) Writes data to store file.

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

CLASS_RULE

public static final HBaseClassTestRule CLASS_RULE

LOG

private static final org.slf4j.Logger LOG

TEST_UTIL

private static final HBaseTestingUtility TEST_UTIL

family

private static final String family

See Also:

Constant Field Values

qf

private static final String qf

See Also:

Constant Field Values

DAY_IN_MS

private final long DAY_IN_MS

See Also:

Constant Field Values

KEYS

private static byte[] KEYS

hcd

private org.apache.hadoop.hbase.HColumnDescriptor hcd

conf

private org.apache.hadoop.conf.Configuration conf

cacheConf

private org.apache.hadoop.hbase.io.hfile.CacheConfig cacheConf

fs

private org.apache.hadoop.fs.FileSystem fs

mobFiles

private List<org.apache.hadoop.fs.FileStatus> mobFiles

delFiles

private List<org.apache.hadoop.fs.Path> delFiles

allFiles

private List<org.apache.hadoop.fs.FileStatus> allFiles

basePath

private org.apache.hadoop.fs.Path basePath

mobSuffix

private String mobSuffix

delSuffix

private String delSuffix

pool

private static ExecutorService pool

name

public org.junit.rules.TestName name

Constructor Detail

TestPartitionedMobCompactor

public TestPartitionedMobCompactor()

Method Detail

setUpBeforeClass

public static void setUpBeforeClass()
                             throws Exception

Throws:

Exception

tearDownAfterClass

public static void tearDownAfterClass()
                               throws Exception

Throws:

Exception

init

private void init(String tableName)
           throws Exception

Throws:

Exception

testCompactionSelectAllFilesWeeklyPolicy

public void testCompactionSelectAllFilesWeeklyPolicy()
                                              throws Exception

Throws:

Exception

testCompactionSelectPartFilesWeeklyPolicy

public void testCompactionSelectPartFilesWeeklyPolicy()
                                               throws Exception

Throws:

Exception

testCompactionSelectPartFilesWeeklyPolicyWithPastWeek

public void testCompactionSelectPartFilesWeeklyPolicyWithPastWeek()
                                                           throws Exception

Throws:

Exception

testCompactionSelectAllFilesWeeklyPolicyWithPastWeek

public void testCompactionSelectAllFilesWeeklyPolicyWithPastWeek()
                                                          throws Exception

Throws:

Exception

testCompactionSelectAllFilesMonthlyPolicy

public void testCompactionSelectAllFilesMonthlyPolicy()
                                               throws Exception

Throws:

Exception

testCompactionSelectNoFilesWithinCurrentWeekMonthlyPolicy

public void testCompactionSelectNoFilesWithinCurrentWeekMonthlyPolicy()
                                                               throws Exception

Throws:

Exception

testCompactionSelectPartFilesMonthlyPolicy

public void testCompactionSelectPartFilesMonthlyPolicy()
                                                throws Exception

Throws:

Exception

testCompactionSelectPartFilesMonthlyPolicyWithPastWeek

public void testCompactionSelectPartFilesMonthlyPolicyWithPastWeek()
                                                            throws Exception

Throws:

Exception

testCompactionSelectAllFilesMonthlyPolicyWithPastWeek

public void testCompactionSelectAllFilesMonthlyPolicyWithPastWeek()
                                                           throws Exception

Throws:

Exception

testCompactionSelectPartFilesMonthlyPolicyWithPastMonth

public void testCompactionSelectPartFilesMonthlyPolicyWithPastMonth()
                                                             throws Exception

Throws:

Exception

testCompactionSelectAllFilesMonthlyPolicyWithPastMonth

public void testCompactionSelectAllFilesMonthlyPolicyWithPastMonth()
                                                            throws Exception

Throws:

Exception

testCompactionSelectWithAllFiles

public void testCompactionSelectWithAllFiles()
                                      throws Exception

Throws:

Exception

testCompactionSelectWithPartFiles

public void testCompactionSelectWithPartFiles()
                                       throws Exception

Throws:

Exception

testCompactionSelectWithForceAllFiles

public void testCompactionSelectWithForceAllFiles()
                                           throws Exception

Throws:

Exception

testCompactionAtMergeSize

private void testCompactionAtMergeSize(String tableName,
                                       long mergeSize,
                                       org.apache.hadoop.hbase.mob.compactions.MobCompactionRequest.CompactionType type,
                                       boolean isForceAllFiles)
                                throws Exception

Throws:

Exception

testCompactionAtMergeSize

private void testCompactionAtMergeSize(String tableName,
                                       long mergeSize,
                                       org.apache.hadoop.hbase.mob.compactions.MobCompactionRequest.CompactionType type,
                                       boolean isForceAllFiles,
                                       boolean createDelFiles)
                                throws Exception

Throws:

Exception

testCompactionAtMergeSize

private void testCompactionAtMergeSize(String tableName,
                                       long mergeSize,
                                       org.apache.hadoop.hbase.mob.compactions.MobCompactionRequest.CompactionType type,
                                       boolean isForceAllFiles,
                                       boolean createDelFiles,
                                       Date date)
                                throws Exception

Throws:

Exception

testCompactionAtMergeSize

private void testCompactionAtMergeSize(String tableName,
                                       long mergeSize,
                                       org.apache.hadoop.hbase.mob.compactions.MobCompactionRequest.CompactionType type,
                                       boolean isForceAllFiles,
                                       boolean createDelFiles,
                                       Date date,
                                       org.apache.hadoop.hbase.client.MobCompactPartitionPolicy policy,
                                       long mergeSizeMultiFactor)
                                throws Exception

Throws:

Exception

testCompactDelFilesWithDefaultBatchSize

public void testCompactDelFilesWithDefaultBatchSize()
                                             throws Exception

Throws:

Exception

testCompactDelFilesWithSmallBatchSize

public void testCompactDelFilesWithSmallBatchSize()
                                           throws Exception

Throws:

Exception

testCompactDelFilesChangeMaxDelFileCount

public void testCompactDelFilesChangeMaxDelFileCount()
                                              throws Exception

Throws:

Exception

testCompactFilesWithDstDirFull

public void testCompactFilesWithDstDirFull()
                                    throws Exception

Throws:

Exception

createMobFile

private void createMobFile(org.apache.hadoop.fs.Path basePath)
                    throws IOException

Create mulitple partition files

Throws:

IOException

createMobDelFile

private void createMobDelFile(org.apache.hadoop.fs.Path basePath,
                              int startKey)
                       throws IOException

Create mulitple partition delete files

Throws:

IOException

testCompactFilesWithoutDelFile

public void testCompactFilesWithoutDelFile()
                                    throws Exception

Throws:

Exception

testCompactFilesWithOneDelFile

public void testCompactFilesWithOneDelFile()
                                    throws Exception

Throws:

Exception

testCompactFilesWithMultiDelFiles

public void testCompactFilesWithMultiDelFiles()
                                       throws Exception

Throws:

Exception

testCompactDelFilesAtBatchSize

private void testCompactDelFilesAtBatchSize(String tableName,
                                            int batchSize,
                                            int delfileMaxCount)
                                     throws Exception

Throws:

Exception

testSelectFiles

private void testSelectFiles(String tableName,
                             org.apache.hadoop.hbase.mob.compactions.MobCompactionRequest.CompactionType type,
                             boolean isForceAllFiles,
                             List<String> expected)
                      throws IOException

Tests the selectFiles

Parameters:

tableName - the table name

type - the expected compaction type

isForceAllFiles - whether all the mob files are selected

expected - the expected start keys

Throws:

IOException

testCompactDelFiles

private void testCompactDelFiles(String tableName,
                                 int expectedFileCount,
                                 int expectedCellCount,
                                 boolean isForceAllFiles)
                          throws IOException

Tests the compacteDelFile

Parameters:

tableName - the table name

expectedFileCount - the expected file count

expectedCellCount - the expected cell count

isForceAllFiles - whether all the mob files are selected

Throws:

IOException

listFiles

private void listFiles()
                throws IOException

Lists the files in the path

Throws:

IOException

compareCompactedPartitions

private void compareCompactedPartitions(List<String> expected,
                                        Collection<org.apache.hadoop.hbase.mob.compactions.PartitionedMobCompactionRequest.CompactionPartition> partitions)

Compares the compacted partitions.

Parameters:

partitions - the collection of CompactedPartitions

compareDelFiles

private void compareDelFiles(List<org.apache.hadoop.hbase.mob.compactions.PartitionedMobCompactionRequest.CompactionDelPartition> delPartitions)

Compares the del files.

Parameters:

delPartitions - all del partitions

createStoreFiles

private void createStoreFiles(org.apache.hadoop.fs.Path basePath,
                              String family,
                              String qualifier,
                              int count,
                              org.apache.hadoop.hbase.KeyValue.Type type,
                              Date date)
                       throws IOException

Creates store files.

Parameters:

basePath - the path to create file

Throws:

IOException

createStoreFiles

private void createStoreFiles(org.apache.hadoop.fs.Path basePath,
                              String family,
                              String qualifier,
                              int count,
                              org.apache.hadoop.hbase.KeyValue.Type type,
                              boolean sameStartKey,
                              Date date)
                       throws IOException

Throws:

IOException

writeStoreFile

private static void writeStoreFile(org.apache.hadoop.hbase.regionserver.StoreFileWriter writer,
                                   byte[] row,
                                   byte[] family,
                                   byte[] qualifier,
                                   org.apache.hadoop.hbase.KeyValue.Type type,
                                   int size)
                            throws IOException

Writes data to store file.

Parameters:

writer - the store file writer

row - the row key

family - the family name

qualifier - the column qualifier

type - the key type

size - the size of value

Throws:

IOException

countDelCellsInDelFiles

private int countDelCellsInDelFiles(List<org.apache.hadoop.fs.Path> paths)
                             throws IOException

Gets the number of del cell in the del files

Parameters:

paths - the del file paths

Returns:

the cell size

Throws:

IOException

createThreadPool

private static ExecutorService createThreadPool()

resetConf

private void resetConf()

Resets the configuration.