Package org.apache.hadoop.hbase.regionserver

Class TestHRegionReplayEvents

java.lang.Object

org.apache.hadoop.hbase.regionserver.TestHRegionReplayEvents

@Tag("org.apache.hadoop.hbase.testclassification.LargeTests")
public class TestHRegionReplayEvents
extends Object

Tests of HRegion methods for replaying flush, compaction, region open, etc events for secondary region replicas

Field Summary

Fields

Modifier and Type	Field	Description
static org.apache.hadoop.conf.Configuration	CONF
protected byte[]	cq
private String	dir
private byte[][]	families
private org.apache.hadoop.hbase.client.TableDescriptor	htd
private static final org.slf4j.Logger	LOG
protected String	method
private String	name
private org.apache.hadoop.hbase.client.RegionInfo	primaryHri
private org.apache.hadoop.hbase.regionserver.HRegion	primaryRegion
private org.apache.hadoop.hbase.wal.WALStreamReader	reader
private org.apache.hadoop.fs.Path	rootDir
protected final byte[]	row
protected final byte[]	row2
private org.apache.hadoop.hbase.regionserver.RegionServerServices	rss
private org.apache.hadoop.hbase.client.RegionInfo	secondaryHri
private org.apache.hadoop.hbase.regionserver.HRegion	secondaryRegion
protected byte[]	tableName
private static HBaseTestingUtil	TEST_UTIL
private org.apache.hadoop.hbase.wal.WAL	walPrimary
private org.apache.hadoop.hbase.wal.WAL	walSecondary

Constructor Summary

Constructors

Constructor	Description
TestHRegionReplayEvents()

Method Summary

Modifier and Type	Method	Description
private void	assertPathListsEqual(List<String> list1, List<String> list2)	Paths can be qualified or not.
private org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor	clone(org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor flush, long flushSeqId)
private String	createHFileForFamilies(org.apache.hadoop.fs.Path testPath, byte[] family, byte[] valueBytes)
private org.apache.hadoop.hbase.wal.WALStreamReader	createWALReaderForPrimary()
private void	disableReads(org.apache.hadoop.hbase.regionserver.HRegion region)
(package private) String	getName()
private static org.apache.hadoop.hbase.regionserver.HRegion	initHRegion(byte[] tableName, byte[]... families)
private void	putDataByReplay(org.apache.hadoop.hbase.regionserver.HRegion region, int startRow, int numRows, byte[] qf, byte[]... families)
private void	putDataWithFlushes(org.apache.hadoop.hbase.regionserver.HRegion region, int flushInterval, int numRows, int numRowsAfterFlush)	Puts a total of numRows + numRowsAfterFlush records indexed with numeric row keys.
private void	replay(org.apache.hadoop.hbase.regionserver.HRegion region, org.apache.hadoop.hbase.client.Put put, long replaySeqId)
(package private) static int	replayEdit(org.apache.hadoop.hbase.regionserver.HRegion region, org.apache.hadoop.hbase.wal.WAL.Entry entry)
void	setUp(org.junit.jupiter.api.TestInfo testInfo)
static void	setUpBeforeClass()
void	tearDown()
static void	tearDownAfterClass()
void	testBatchReplayWithMultipleNonces()
void	testOnlyReplayingFlushStartDoesNotHoldUpRegionClose()	Tests a case where we replay only a flush start marker, then the region is closed.
void	testRefresStoreFiles()
void	testRegionReadsEnabledFlag()	Tests the reads enabled flag for the region.
void	testRegionReplicaSecondaryCannotFlush()
void	testReplayBulkLoadEvent()	Tests replaying region open markers from primary region.
void	testReplayFlushCommitMarkerLargerThanFlushStartMarker()	Tests the case where we prepare a flush with some seqId and we receive a flush commit marker larger than the previous flush start marker.
void	testReplayFlushCommitMarkerSmallerThanFlushStartMarker()	Tests the case where we prepare a flush with some seqId and we receive a flush commit marker less than the previous flush start marker.
void	testReplayFlushCommitMarkerWithoutFlushStartMarker(boolean droppableMemstore)	Tests the case where we receive a flush commit before receiving any flush prepare markers
void	testReplayFlushCommitMarkerWithoutFlushStartMarkerDroppableMemstore()	Tests the case where we receive a flush commit before receiving any flush prepare markers.
void	testReplayFlushCommitMarkerWithoutFlushStartMarkerNonDroppableMemstore()	Tests the case where we receive a flush commit before receiving any flush prepare markers.
void	testReplayFlushesAndCompactions()
void	testReplayFlushSeqIds()
void	testReplayFlushStartMarkers()	Tests cases where we prepare a flush with some seqId and we receive other flush start markers equal to, greater or less than the previous flush start marker.
void	testReplayingBulkLoadEventWithFileAlreadyDeleted()
void	testReplayingCompactionWithFileAlreadyDeleted()
void	testReplayingFlushCommitWithFileAlreadyDeleted()
void	testReplayingFlushRequestRestoresReadsEnabledState()	Test the case where the secondary region replica is not in reads enabled state because it is waiting for a flush or region open marker from primary region.
void	testReplayingFlushRestoresReadsEnabledState()	Test the case where the secondary region replica is not in reads enabled state because it is waiting for a flush or region open marker from primary region.
void	testReplayingFlushWithEmptyMemstoreRestoresReadsEnabledState()	Test the case where the secondary region replica is not in reads enabled state because it is waiting for a flush or region open marker from primary region.
void	testReplayingRegionOpenEventRestoresReadsEnabledState()	Test the case where the secondary region replica is not in reads enabled state because it is waiting for a flush or region open marker from primary region.
void	testReplayingRegionOpenEventWithFileAlreadyDeleted()
void	testReplayRegionOpenEvent()	Tests replaying region open markers from primary region.
void	testReplayRegionOpenEventAfterFlushStart()	Tests the case where we replay a region open event after a flush start but before receiving flush commit
void	testSecondaryRegionDoesNotWriteRegionEventsToWAL()	Tests that a region opened in secondary mode would not write region open / close events to its WAL.
void	testSeqIdsFromReplay()
void	testSkippingEditsWithSmallerSeqIdAfterRegionOpenEvent()	Tests whether edits coming in for replay are skipped which have smaller seq id than the seqId of the last replayed region open event.
void	testWriteFlushRequestMarker()	Tests the case where a request for flush cache is sent to the region, but region cannot flush.

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Details

LOG

private static final org.slf4j.Logger LOG

name

private String name

TEST_UTIL

private static HBaseTestingUtil TEST_UTIL

CONF

public static org.apache.hadoop.conf.Configuration CONF

dir

private String dir

families

private byte[][] families

tableName

protected byte[] tableName

method

protected String method

row

protected final byte[] row

row2

protected final byte[] row2

cq

protected byte[] cq

rootDir

private org.apache.hadoop.fs.Path rootDir

htd

private org.apache.hadoop.hbase.client.TableDescriptor htd

rss

private org.apache.hadoop.hbase.regionserver.RegionServerServices rss

primaryHri

private org.apache.hadoop.hbase.client.RegionInfo primaryHri

secondaryHri

private org.apache.hadoop.hbase.client.RegionInfo secondaryHri

primaryRegion

private org.apache.hadoop.hbase.regionserver.HRegion primaryRegion

secondaryRegion

private org.apache.hadoop.hbase.regionserver.HRegion secondaryRegion

walPrimary

private org.apache.hadoop.hbase.wal.WAL walPrimary

walSecondary

private org.apache.hadoop.hbase.wal.WAL walSecondary

reader

private org.apache.hadoop.hbase.wal.WALStreamReader reader

Constructor Details

TestHRegionReplayEvents

public TestHRegionReplayEvents()

Method Details

setUpBeforeClass

@BeforeAll
public static void setUpBeforeClass()
                             throws Exception

Throws:

Exception

tearDownAfterClass

@AfterAll
public static void tearDownAfterClass()
                               throws Exception

Throws:

Exception

setUp

@BeforeEach
public void setUp(org.junit.jupiter.api.TestInfo testInfo)
           throws Exception

Throws:

Exception

tearDown

@AfterEach
public void tearDown()
              throws Exception

Throws:

Exception

getName

String getName()

testRegionReplicaSecondaryCannotFlush

@Test
public void testRegionReplicaSecondaryCannotFlush()
                                           throws IOException

Throws:

IOException

testOnlyReplayingFlushStartDoesNotHoldUpRegionClose

@Test
public void testOnlyReplayingFlushStartDoesNotHoldUpRegionClose()
                                                         throws IOException

Tests a case where we replay only a flush start marker, then the region is closed. This region should not block indefinitely

Throws:

IOException

replayEdit

static int replayEdit(org.apache.hadoop.hbase.regionserver.HRegion region,
 org.apache.hadoop.hbase.wal.WAL.Entry entry)
               throws IOException

Throws:

IOException

createWALReaderForPrimary

private org.apache.hadoop.hbase.wal.WALStreamReader createWALReaderForPrimary()
                                                                       throws FileNotFoundException,
IOException

Throws:

FileNotFoundException

IOException

testBatchReplayWithMultipleNonces

@Test
public void testBatchReplayWithMultipleNonces()
                                       throws IOException

Throws:

IOException

testReplayFlushesAndCompactions

@Test
public void testReplayFlushesAndCompactions()
                                     throws IOException

Throws:

IOException

testReplayFlushStartMarkers

@Test
public void testReplayFlushStartMarkers()
                                 throws IOException

Tests cases where we prepare a flush with some seqId and we receive other flush start markers equal to, greater or less than the previous flush start marker.

Throws:

IOException

testReplayFlushCommitMarkerSmallerThanFlushStartMarker

@Test
public void testReplayFlushCommitMarkerSmallerThanFlushStartMarker()
                                                            throws IOException

Tests the case where we prepare a flush with some seqId and we receive a flush commit marker less than the previous flush start marker.

Throws:

IOException

testReplayFlushCommitMarkerLargerThanFlushStartMarker

@Test
public void testReplayFlushCommitMarkerLargerThanFlushStartMarker()
                                                           throws IOException

Tests the case where we prepare a flush with some seqId and we receive a flush commit marker larger than the previous flush start marker.

Throws:

IOException

testReplayFlushCommitMarkerWithoutFlushStartMarkerDroppableMemstore

@Test
public void testReplayFlushCommitMarkerWithoutFlushStartMarkerDroppableMemstore()
                                                                         throws IOException

Tests the case where we receive a flush commit before receiving any flush prepare markers. The memstore edits should be dropped after the flush commit replay since they should be in flushed files

Throws:

IOException

testReplayFlushCommitMarkerWithoutFlushStartMarkerNonDroppableMemstore

@Test
public void testReplayFlushCommitMarkerWithoutFlushStartMarkerNonDroppableMemstore()
                                                                            throws IOException

Tests the case where we receive a flush commit before receiving any flush prepare markers. The memstore edits should be not dropped after the flush commit replay since not every edit will be in flushed files (based on seqId)

Throws:

IOException

testReplayFlushCommitMarkerWithoutFlushStartMarker

public void testReplayFlushCommitMarkerWithoutFlushStartMarker(boolean droppableMemstore)
                                                        throws IOException

Tests the case where we receive a flush commit before receiving any flush prepare markers

Throws:

IOException

clone

private org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor clone(org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor flush,
 long flushSeqId)

testReplayRegionOpenEvent

@Test
public void testReplayRegionOpenEvent()
                               throws IOException

Tests replaying region open markers from primary region. Checks whether the files are picked up

Throws:

IOException

testReplayRegionOpenEventAfterFlushStart

@Test
public void testReplayRegionOpenEventAfterFlushStart()
                                              throws IOException

Tests the case where we replay a region open event after a flush start but before receiving flush commit

Throws:

IOException

testSkippingEditsWithSmallerSeqIdAfterRegionOpenEvent

@Test
public void testSkippingEditsWithSmallerSeqIdAfterRegionOpenEvent()
                                                           throws IOException

Tests whether edits coming in for replay are skipped which have smaller seq id than the seqId of the last replayed region open event.

Throws:

IOException

testReplayFlushSeqIds

@Test
public void testReplayFlushSeqIds()
                           throws IOException

Throws:

IOException

testSeqIdsFromReplay

@Test
public void testSeqIdsFromReplay()
                          throws IOException

Throws:

IOException

testSecondaryRegionDoesNotWriteRegionEventsToWAL

@Test
public void testSecondaryRegionDoesNotWriteRegionEventsToWAL()
                                                      throws IOException

Tests that a region opened in secondary mode would not write region open / close events to its WAL.

Throws:

IOException

testRegionReadsEnabledFlag

@Test
public void testRegionReadsEnabledFlag()
                                throws IOException

Tests the reads enabled flag for the region. When unset all reads should be rejected

Throws:

IOException

testWriteFlushRequestMarker

@Test
public void testWriteFlushRequestMarker()
                                 throws IOException

Tests the case where a request for flush cache is sent to the region, but region cannot flush. It should write the flush request marker instead.

Throws:

IOException

testReplayingFlushRequestRestoresReadsEnabledState

@Test
public void testReplayingFlushRequestRestoresReadsEnabledState()
                                                        throws IOException

Test the case where the secondary region replica is not in reads enabled state because it is waiting for a flush or region open marker from primary region. Replaying CANNOT_FLUSH flush marker entry should restore the reads enabled status in the region and allow the reads to continue.

Throws:

IOException

testReplayingFlushRestoresReadsEnabledState

@Test
public void testReplayingFlushRestoresReadsEnabledState()
                                                 throws IOException

Test the case where the secondary region replica is not in reads enabled state because it is waiting for a flush or region open marker from primary region. Replaying flush start and commit entries should restore the reads enabled status in the region and allow the reads to continue.

Throws:

IOException

testReplayingFlushWithEmptyMemstoreRestoresReadsEnabledState

@Test
public void testReplayingFlushWithEmptyMemstoreRestoresReadsEnabledState()
                                                                  throws IOException

Test the case where the secondary region replica is not in reads enabled state because it is waiting for a flush or region open marker from primary region. Replaying flush start and commit entries should restore the reads enabled status in the region and allow the reads to continue.

Throws:

IOException

testReplayingRegionOpenEventRestoresReadsEnabledState

@Test
public void testReplayingRegionOpenEventRestoresReadsEnabledState()
                                                           throws IOException

Test the case where the secondary region replica is not in reads enabled state because it is waiting for a flush or region open marker from primary region. Replaying region open event entry from primary should restore the reads enabled status in the region and allow the reads to continue.

Throws:

IOException

testRefresStoreFiles

@Test
public void testRefresStoreFiles()
                          throws IOException

Throws:

IOException

assertPathListsEqual

private void assertPathListsEqual(List<String> list1,
 List<String> list2)

Paths can be qualified or not. This does the assertion using String->Path conversion.

disableReads

private void disableReads(org.apache.hadoop.hbase.regionserver.HRegion region)

replay

private void replay(org.apache.hadoop.hbase.regionserver.HRegion region,
 org.apache.hadoop.hbase.client.Put put,
 long replaySeqId)
             throws IOException

Throws:

IOException

testReplayBulkLoadEvent

@Test
public void testReplayBulkLoadEvent()
                             throws IOException

Tests replaying region open markers from primary region. Checks whether the files are picked up

Throws:

IOException

testReplayingFlushCommitWithFileAlreadyDeleted

@Test
public void testReplayingFlushCommitWithFileAlreadyDeleted()
                                                    throws IOException

Throws:

IOException

testReplayingCompactionWithFileAlreadyDeleted

@Test
public void testReplayingCompactionWithFileAlreadyDeleted()
                                                   throws IOException

Throws:

IOException

testReplayingRegionOpenEventWithFileAlreadyDeleted

@Test
public void testReplayingRegionOpenEventWithFileAlreadyDeleted()
                                                        throws IOException

Throws:

IOException

testReplayingBulkLoadEventWithFileAlreadyDeleted

@Test
public void testReplayingBulkLoadEventWithFileAlreadyDeleted()
                                                      throws IOException

Throws:

IOException

createHFileForFamilies

private String createHFileForFamilies(org.apache.hadoop.fs.Path testPath,
 byte[] family,
 byte[] valueBytes)
                               throws IOException

Throws:

IOException

putDataWithFlushes

private void putDataWithFlushes(org.apache.hadoop.hbase.regionserver.HRegion region,
 int flushInterval,
 int numRows,
 int numRowsAfterFlush)
                         throws IOException

Puts a total of numRows + numRowsAfterFlush records indexed with numeric row keys. Does a flush every flushInterval number of records. Then it puts numRowsAfterFlush number of more rows but does not execute flush after

Throws:

IOException

putDataByReplay

private void putDataByReplay(org.apache.hadoop.hbase.regionserver.HRegion region,
 int startRow,
 int numRows,
 byte[] qf,
 byte[]... families)
                      throws IOException

Throws:

IOException

initHRegion

private static org.apache.hadoop.hbase.regionserver.HRegion initHRegion(byte[] tableName,
 byte[]... families)
                                                                 throws IOException

Throws:

IOException