Package org.apache.hadoop.hbase.regionserver.compactions

Class TestStripeCompactionPolicy

java.lang.Object

org.apache.hadoop.hbase.regionserver.compactions.TestStripeCompactionPolicy

@Tag("org.apache.hadoop.hbase.testclassification.RegionServerTests") @Tag("org.apache.hadoop.hbase.testclassification.MediumTests")
public class TestStripeCompactionPolicy
extends Object

Nested Class Summary

Nested Classes

Modifier and Type	Class	Description
private static class	TestStripeCompactionPolicy.Scanner

Field Summary

Fields

Modifier and Type	Field	Description
private static final int	defaultInitialCount
private static float	defaultSplitCount
private static long	defaultSplitSize
private static long	defaultTtl
private static final byte[]	KEY_A
private static final byte[]	KEY_B
private static final byte[]	KEY_C
private static final byte[]	KEY_D
private static final byte[]	KEY_E
private static final org.apache.hadoop.hbase.KeyValue	KV_A
private static final org.apache.hadoop.hbase.KeyValue	KV_B
private static final org.apache.hadoop.hbase.KeyValue	KV_C
private static final org.apache.hadoop.hbase.KeyValue	KV_D
private static final org.apache.hadoop.hbase.KeyValue	KV_E
private final boolean	usePrivateReaders

Constructor Summary

Constructors

Constructor	Description
TestStripeCompactionPolicy(boolean usePrivateReaders)

Method Summary

Modifier and Type	Method	Description
private static ArrayList<org.apache.hadoop.hbase.regionserver.HStoreFile>	al(org.apache.hadoop.hbase.regionserver.HStoreFile... sfs)
private static List<byte[]>	createBoundaries(int stripeCount)
private org.apache.hadoop.hbase.regionserver.compactions.StripeCompactor	createCompactor()
private static org.apache.hadoop.hbase.regionserver.HStoreFile	createFile()
private static org.apache.hadoop.hbase.regionserver.HStoreFile	createFile(long size)
private static org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy	createPolicy(org.apache.hadoop.conf.Configuration conf)	HELPER METHODS
private static org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy	createPolicy(org.apache.hadoop.conf.Configuration conf, long splitSize, float splitCount, int initialCount, boolean hasTtl)
private static org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy.StripeInformationProvider	createStripes(int l0Count, byte[]... boundaries)
private static org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy.StripeInformationProvider	createStripes(List<byte[]> boundaries, List<List<Long>> stripeSizes, List<Long> l0Sizes)
private static org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy.StripeInformationProvider	createStripesL0Only(int l0Count, long l0Size)
private static org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy.StripeInformationProvider	createStripesWithFiles(List<byte[]> boundaries, List<List<org.apache.hadoop.hbase.regionserver.HStoreFile>> stripeFiles, List<org.apache.hadoop.hbase.regionserver.HStoreFile> l0Files)	This method actually does all the work.
private static org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy.StripeInformationProvider	createStripesWithFiles(List<org.apache.hadoop.hbase.regionserver.HStoreFile>... stripeFiles)
private static org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy.StripeInformationProvider	createStripesWithSizes(int l0Count, long l0Size, Long[]... sizes)
private static org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy.StripeInformationProvider	createStripesWithSizes(int l0Count, long l0Size, List<List<Long>> sizes)
private byte[]	dropDeletesMatcher(Boolean dropDeletes, byte[] value)
private static List<org.apache.hadoop.hbase.regionserver.HStoreFile>	getAllFiles(org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy.StripeInformationProvider si, int fromStripe, int toStripe)
static Stream<org.junit.jupiter.params.provider.Arguments>	parameters()
private static void	setFileStripe(org.apache.hadoop.hbase.regionserver.HStoreFile sf, byte[] startKey, byte[] endKey)
void	testCheckExpiredL0Compaction()
void	testCheckExpiredStripeCompaction()
void	testExistingStripesFromL0()
void	testInitialCountFromL0()
void	testMergeExpiredFiles()
void	testMergeExpiredStripes()
void	testNewStripesFromFlush()
void	testNoStripesFromFlush()
void	testNothingToCompactFromL0()
void	testOldStripesFromFlush()
void	testSelectL0Compaction()
void	testSingleStripeCompaction()
void	testSingleStripeDropDeletes()
void	testSplitOffStripe()
void	testSplitOffStripeDropDeletes()
void	testSplitOffStripeOffPeak()
void	testWithParallelCompaction()
void	testWithReferences()
private void	verifyCollectionsEqual(Collection<org.apache.hadoop.hbase.regionserver.HStoreFile> sfs, Collection<org.apache.hadoop.hbase.regionserver.HStoreFile> scr)
private void	verifyCompaction(org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy policy, org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy.StripeInformationProvider si, Collection<org.apache.hadoop.hbase.regionserver.HStoreFile> sfs, byte[] dropDeletesFrom, byte[] dropDeletesTo, List<byte[]> boundaries)	Verify arbitrary compaction.
private void	verifyCompaction(org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy policy, org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy.StripeInformationProvider si, Collection<org.apache.hadoop.hbase.regionserver.HStoreFile> sfs, Boolean dropDeletes, Integer count, Long size, byte[] start, byte[] end, boolean needsCompaction)	Verify arbitrary compaction.
protected void	verifyFlush(org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy policy, org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy.StripeInformationProvider si, org.apache.hadoop.hbase.KeyValue[] input, org.apache.hadoop.hbase.KeyValue[][] expected, byte[][] boundaries)	Verify arbitrary flush.
private void	verifyMergeCompatcion(org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy policy, org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy.StripeInformationProvider si, int from, int to)
private void	verifyNoCompaction(org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy policy, org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy.StripeInformationProvider si)	Verify no compaction is needed or selected.
private void	verifySingleStripeCompaction(org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy policy, org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy.StripeInformationProvider si, int index, Boolean dropDeletes)
private void	verifyWholeStripesCompaction(org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy policy, org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy.StripeInformationProvider si, int from, int to, Boolean dropDeletes, Integer count, Long size)
private void	verifyWholeStripesCompaction(org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy policy, org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy.StripeInformationProvider si, int from, int to, Boolean dropDeletes, Integer count, Long size, boolean needsCompaction)	Verify the compaction that includes several entire stripes.

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Details

KEY_A

private static final byte[] KEY_A

KEY_B

private static final byte[] KEY_B

KEY_C

private static final byte[] KEY_C

KEY_D

private static final byte[] KEY_D

KEY_E

private static final byte[] KEY_E

KV_A

private static final org.apache.hadoop.hbase.KeyValue KV_A

KV_B

private static final org.apache.hadoop.hbase.KeyValue KV_B

KV_C

private static final org.apache.hadoop.hbase.KeyValue KV_C

KV_D

private static final org.apache.hadoop.hbase.KeyValue KV_D

KV_E

private static final org.apache.hadoop.hbase.KeyValue KV_E

defaultSplitSize

private static long defaultSplitSize

defaultSplitCount

private static float defaultSplitCount

defaultInitialCount

private static final int defaultInitialCount

See Also:

• Constant Field Values

defaultTtl

private static long defaultTtl

usePrivateReaders

private final boolean usePrivateReaders

Constructor Details

TestStripeCompactionPolicy

public TestStripeCompactionPolicy(boolean usePrivateReaders)

Method Details

parameters

public static Stream<org.junit.jupiter.params.provider.Arguments> parameters()

testNoStripesFromFlush

@TestTemplate
public void testNoStripesFromFlush()
                            throws Exception

Throws:

Exception

testOldStripesFromFlush

@TestTemplate
public void testOldStripesFromFlush()
                             throws Exception

Throws:

Exception

testNewStripesFromFlush

@TestTemplate
public void testNewStripesFromFlush()
                             throws Exception

Throws:

Exception

testSingleStripeCompaction

@TestTemplate
public void testSingleStripeCompaction()
                                throws Exception

Throws:

Exception

testWithParallelCompaction

@TestTemplate
public void testWithParallelCompaction()
                                throws Exception

Throws:

Exception

testWithReferences

@TestTemplate
public void testWithReferences()
                        throws Exception

Throws:

Exception

testInitialCountFromL0

@TestTemplate
public void testInitialCountFromL0()
                            throws Exception

Throws:

Exception

testSelectL0Compaction

@TestTemplate
public void testSelectL0Compaction()
                            throws Exception

Throws:

Exception

testExistingStripesFromL0

@TestTemplate
public void testExistingStripesFromL0()
                               throws Exception

Throws:

Exception

testNothingToCompactFromL0

@TestTemplate
public void testNothingToCompactFromL0()
                                throws Exception

Throws:

Exception

testCheckExpiredStripeCompaction

@TestTemplate
public void testCheckExpiredStripeCompaction()
                                      throws Exception

Throws:

Exception

testSplitOffStripe

@TestTemplate
public void testSplitOffStripe()
                        throws Exception

Throws:

Exception

testSplitOffStripeOffPeak

@TestTemplate
public void testSplitOffStripeOffPeak()
                               throws Exception

Throws:

Exception

testSplitOffStripeDropDeletes

@TestTemplate
public void testSplitOffStripeDropDeletes()
                                   throws Exception

Throws:

Exception

testMergeExpiredFiles

@TestTemplate
public void testMergeExpiredFiles()
                           throws Exception

Throws:

Exception

testMergeExpiredStripes

@TestTemplate
public void testMergeExpiredStripes()
                             throws Exception

Throws:

Exception

createStripesWithFiles

private static org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy.StripeInformationProvider createStripesWithFiles(List<org.apache.hadoop.hbase.regionserver.HStoreFile>... stripeFiles)
                                                                                                                                 throws Exception

Throws:

Exception

testSingleStripeDropDeletes

@TestTemplate
public void testSingleStripeDropDeletes()
                                 throws Exception

Throws:

Exception

testCheckExpiredL0Compaction

@TestTemplate
public void testCheckExpiredL0Compaction()
                                  throws Exception

Throws:

Exception

createPolicy

private static org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy createPolicy(org.apache.hadoop.conf.Configuration conf)
                                                                                             throws Exception

HELPER METHODS

Throws:

Exception

createPolicy

private static org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy createPolicy(org.apache.hadoop.conf.Configuration conf,
 long splitSize,
 float splitCount,
 int initialCount,
 boolean hasTtl)
                                                                                             throws Exception

Throws:

Exception

al

private static ArrayList<org.apache.hadoop.hbase.regionserver.HStoreFile> al(org.apache.hadoop.hbase.regionserver.HStoreFile... sfs)

verifyMergeCompatcion

private void verifyMergeCompatcion(org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy policy,
 org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy.StripeInformationProvider si,
 int from,
 int to)
                            throws Exception

Throws:

Exception

verifyWholeStripesCompaction

private void verifyWholeStripesCompaction(org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy policy,
 org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy.StripeInformationProvider si,
 int from,
 int to,
 Boolean dropDeletes,
 Integer count,
 Long size,
 boolean needsCompaction)
                                   throws IOException

Verify the compaction that includes several entire stripes.

Parameters:

policy - Policy to test.

si - Stripe information pre-set with stripes to test.

from - Starting stripe.

to - Ending stripe (inclusive).

dropDeletes - Whether to drop deletes from compaction range.

count - Expected # of resulting stripes, null if not checked.

size - Expected target stripe size, null if not checked.

Throws:

IOException

verifyWholeStripesCompaction

private void verifyWholeStripesCompaction(org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy policy,
 org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy.StripeInformationProvider si,
 int from,
 int to,
 Boolean dropDeletes,
 Integer count,
 Long size)
                                   throws IOException

Throws:

IOException

verifySingleStripeCompaction

private void verifySingleStripeCompaction(org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy policy,
 org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy.StripeInformationProvider si,
 int index,
 Boolean dropDeletes)
                                   throws IOException

Throws:

IOException

verifyNoCompaction

private void verifyNoCompaction(org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy policy,
 org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy.StripeInformationProvider si)
                         throws IOException

Verify no compaction is needed or selected.

Parameters:

policy - Policy to test.

si - Stripe information pre-set with stripes to test.

Throws:

IOException

verifyCompaction

private void verifyCompaction(org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy policy,
 org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy.StripeInformationProvider si,
 Collection<org.apache.hadoop.hbase.regionserver.HStoreFile> sfs,
 byte[] dropDeletesFrom,
 byte[] dropDeletesTo,
 List<byte[]> boundaries)
                       throws Exception

Verify arbitrary compaction.

Parameters:

policy - Policy to test.

si - Stripe information pre-set with stripes to test.

sfs - Files that should be compacted.

dropDeletesFrom - Row from which to drop deletes.

dropDeletesTo - Row to which to drop deletes.

boundaries - Expected target stripe boundaries.

Throws:

Exception

verifyCompaction

private void verifyCompaction(org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy policy,
 org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy.StripeInformationProvider si,
 Collection<org.apache.hadoop.hbase.regionserver.HStoreFile> sfs,
 Boolean dropDeletes,
 Integer count,
 Long size,
 byte[] start,
 byte[] end,
 boolean needsCompaction)
                       throws IOException

Verify arbitrary compaction.

Parameters:

policy - Policy to test.

si - Stripe information pre-set with stripes to test.

sfs - Files that should be compacted.

dropDeletes - Whether to drop deletes from compaction range.

count - Expected # of resulting stripes, null if not checked.

size - Expected target stripe size, null if not checked.

start - Left boundary of the compaction.

end - Right boundary of the compaction.

Throws:

IOException

verifyFlush

protected void verifyFlush(org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy policy,
 org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy.StripeInformationProvider si,
 org.apache.hadoop.hbase.KeyValue[] input,
 org.apache.hadoop.hbase.KeyValue[][] expected,
 byte[][] boundaries)
                    throws IOException

Verify arbitrary flush.

Throws:

IOException

dropDeletesMatcher

private byte[] dropDeletesMatcher(Boolean dropDeletes,
 byte[] value)

verifyCollectionsEqual

private void verifyCollectionsEqual(Collection<org.apache.hadoop.hbase.regionserver.HStoreFile> sfs,
 Collection<org.apache.hadoop.hbase.regionserver.HStoreFile> scr)

getAllFiles

private static List<org.apache.hadoop.hbase.regionserver.HStoreFile> getAllFiles(org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy.StripeInformationProvider si,
 int fromStripe,
 int toStripe)

createStripes

private static org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy.StripeInformationProvider createStripes(int l0Count,
 byte[]... boundaries)
                                                                                                                        throws Exception

Parameters:

l0Count - Number of L0 files.

boundaries - Target boundaries.

Returns:

Mock stripes.

Throws:

Exception

createStripesL0Only

private static org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy.StripeInformationProvider createStripesL0Only(int l0Count,
 long l0Size)
                                                                                                                              throws Exception

Parameters:

l0Count - Number of L0 files.

l0Size - Size of each file.

Returns:

Mock stripes.

Throws:

Exception

createStripesWithSizes

private static org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy.StripeInformationProvider createStripesWithSizes(int l0Count,
 long l0Size,
 Long[]... sizes)
                                                                                                                                 throws Exception

Parameters:

l0Count - Number of L0 files.

l0Size - Size of each file.

sizes - Sizes of the files; each sub-array representing a stripe.

Returns:

Mock stripes.

Throws:

Exception

createStripesWithSizes

private static org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy.StripeInformationProvider createStripesWithSizes(int l0Count,
 long l0Size,
 List<List<Long>> sizes)
                                                                                                                                 throws Exception

Throws:

Exception

createBoundaries

private static List<byte[]> createBoundaries(int stripeCount)

createStripes

private static org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy.StripeInformationProvider createStripes(List<byte[]> boundaries,
 List<List<Long>> stripeSizes,
 List<Long> l0Sizes)
                                                                                                                        throws Exception

Throws:

Exception

createStripesWithFiles

private static org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy.StripeInformationProvider createStripesWithFiles(List<byte[]> boundaries,
 List<List<org.apache.hadoop.hbase.regionserver.HStoreFile>> stripeFiles,
 List<org.apache.hadoop.hbase.regionserver.HStoreFile> l0Files)
                                                                                                                                 throws Exception

This method actually does all the work.

Throws:

Exception

createFile

private static org.apache.hadoop.hbase.regionserver.HStoreFile createFile(long size)
                                                                   throws Exception

Throws:

Exception

createFile

private static org.apache.hadoop.hbase.regionserver.HStoreFile createFile()
                                                                   throws Exception

Throws:

Exception

setFileStripe

private static void setFileStripe(org.apache.hadoop.hbase.regionserver.HStoreFile sf,
 byte[] startKey,
 byte[] endKey)

createCompactor

private org.apache.hadoop.hbase.regionserver.compactions.StripeCompactor createCompactor()
                                                                                  throws Exception

Throws:

Exception