Package org.apache.hadoop.hbase.regionserver.wal

Class AbstractTestWALReplay

java.lang.Object

org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay

Direct Known Subclasses:

TestAsyncWALReplay, TestSecureWALReplay, TestWALReplay, TestWALReplayBoundedLogWriterCreation, TestWALReplayCompressed, TestWALReplayValueCompression

public abstract class AbstractTestWALReplay
extends Object

Test replay of edits out of a WAL split.

Nested Class Summary

Nested Classes

Modifier and Type	Class	Description
static class	AbstractTestWALReplay.CustomStoreFlusher
(package private) static class	AbstractTestWALReplay.MockWAL
(package private) static class	AbstractTestWALReplay.TestFlusher

Field Summary

Fields

Modifier and Type	Field	Description
private org.apache.hadoop.conf.Configuration	conf
private String	currentTest
private final org.apache.hadoop.hbase.util.EnvironmentEdge	ee
private org.apache.hadoop.fs.FileSystem	fs
private org.apache.hadoop.fs.Path	hbaseRootDir
private static final org.slf4j.Logger	LOG
private org.apache.hadoop.fs.Path	logDir
private String	logName
private org.apache.hadoop.fs.Path	oldLogDir
(package private) static final HBaseTestingUtil	TEST_UTIL
private org.apache.hadoop.hbase.wal.WALFactory	wals

Constructor Summary

Constructors

Constructor	Description
AbstractTestWALReplay()

Method Summary

Modifier and Type	Method	Description
static List<org.apache.hadoop.hbase.client.Put>	addRegionEdits(byte[] rowName, byte[] family, int count, org.apache.hadoop.hbase.util.EnvironmentEdge ee, org.apache.hadoop.hbase.regionserver.Region r, String qualifierPrefix)
private void	addWALEdits(org.apache.hadoop.hbase.TableName tableName, org.apache.hadoop.hbase.client.RegionInfo hri, byte[] rowName, byte[] family, int count, org.apache.hadoop.hbase.util.EnvironmentEdge ee, org.apache.hadoop.hbase.wal.WAL wal, org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl mvcc, NavigableMap<byte[],Integer> scopes)
private org.apache.hadoop.hbase.client.TableDescriptor	createBasic1FamilyHTD(org.apache.hadoop.hbase.TableName tableName)
private org.apache.hadoop.hbase.client.RegionInfo	createBasic3FamilyHRegionInfo(org.apache.hadoop.hbase.TableName tableName)	Creates an HRI around an HTD that has tableName and three column families named 'a','b', and 'c'.
private org.apache.hadoop.hbase.client.TableDescriptor	createBasic3FamilyHTD(org.apache.hadoop.hbase.TableName tableName)
private org.apache.hadoop.hbase.regionserver.wal.FSWALEntry	createFSWALEntry(org.apache.hadoop.hbase.client.TableDescriptor htd, org.apache.hadoop.hbase.client.RegionInfo hri, long sequence, byte[] rowName, byte[] family, org.apache.hadoop.hbase.util.EnvironmentEdge ee, org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl mvcc, int index, NavigableMap<byte[],Integer> scopes)
private AbstractTestWALReplay.MockWAL	createMockWAL()
protected abstract org.apache.hadoop.hbase.wal.WAL	createWAL(org.apache.hadoop.conf.Configuration c, org.apache.hadoop.fs.Path hbaseRootDir, String logName)
private org.apache.hadoop.hbase.wal.WALEdit	createWALEdit(byte[] rowName, byte[] family, org.apache.hadoop.hbase.util.EnvironmentEdge ee, int index)
private org.apache.hadoop.hbase.wal.WALKeyImpl	createWALKey(org.apache.hadoop.hbase.TableName tableName, org.apache.hadoop.hbase.client.RegionInfo hri, org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl mvcc, NavigableMap<byte[],Integer> scopes)
private void	deleteDir(org.apache.hadoop.fs.Path p)
private int	getScannedCount(org.apache.hadoop.hbase.regionserver.RegionScanner scanner)
private org.apache.hadoop.fs.Path	runWALSplit(org.apache.hadoop.conf.Configuration c)	Run the split.
void	setUp(org.junit.jupiter.api.TestInfo testInfo)
static void	setUpBeforeClass()
void	tearDown()
static void	tearDownAfterClass()
void	test2727()	Tests for hbase-2727.
void	testCompactedBulkLoadedFiles()	HRegion test case that is made of a major compacted HFile (created with three bulk loaded files) and an edit in the memstore.
void	testDatalossWhenInputError()	testcase for https://issues.apache.org/jira/browse/HBASE-15252
private void	testNameConflictWhenSplit(boolean largeFirst)	testcase for https://issues.apache.org/jira/browse/HBASE-14949.
void	testNameConflictWhenSplit0()
void	testNameConflictWhenSplit1()
void	testRegionMadeOfBulkLoadedFilesOnly()	Test case of HRegion that is only made out of bulk loaded files.
void	testReplayEditsAfterAbortingFlush()	Test that we could recover the data correctly after aborting flush.
void	testReplayEditsAfterPartialFlush()	Test that we recover correctly when there is a failure in between the flushes.
void	testReplayEditsAfterRegionMovedWithMultiCF()
void	testReplayEditsWrittenIntoWAL()	Create an HRegion with the result of a WAL split and test we only see the good edits=
void	testReplayEditsWrittenViaHRegion()	Test writing edits into an HRegion, closing it, splitting logs, opening Region again.
void	testSequentialEditLogSeqNum()
private void	writerWALFile(org.apache.hadoop.fs.Path file, List<org.apache.hadoop.hbase.regionserver.wal.FSWALEntry> entries)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Details

LOG

private static final org.slf4j.Logger LOG

TEST_UTIL

static final HBaseTestingUtil TEST_UTIL

ee

private final org.apache.hadoop.hbase.util.EnvironmentEdge ee

hbaseRootDir

private org.apache.hadoop.fs.Path hbaseRootDir

logName

private String logName

oldLogDir

private org.apache.hadoop.fs.Path oldLogDir

logDir

private org.apache.hadoop.fs.Path logDir

fs

private org.apache.hadoop.fs.FileSystem fs

conf

private org.apache.hadoop.conf.Configuration conf

wals

private org.apache.hadoop.hbase.wal.WALFactory wals

currentTest

private String currentTest

Constructor Details

AbstractTestWALReplay

public AbstractTestWALReplay()

Method Details

setUpBeforeClass

public static void setUpBeforeClass()
                             throws Exception

Throws:

Exception

tearDownAfterClass

@AfterAll
public static void tearDownAfterClass()
                               throws Exception

Throws:

Exception

setUp

@BeforeEach
public void setUp(org.junit.jupiter.api.TestInfo testInfo)
           throws Exception

Throws:

Exception

tearDown

@AfterEach
public void tearDown()
              throws Exception

Throws:

Exception

deleteDir

private void deleteDir(org.apache.hadoop.fs.Path p)
                throws IOException

Throws:

IOException

testReplayEditsAfterRegionMovedWithMultiCF

@Test
public void testReplayEditsAfterRegionMovedWithMultiCF()
                                                throws Exception

Throws:

Exception

test2727

@Test
public void test2727()
              throws Exception

Tests for hbase-2727.

Throws:

Exception

See Also:

• HBASE-2727

testRegionMadeOfBulkLoadedFilesOnly

@Test
public void testRegionMadeOfBulkLoadedFilesOnly()
                                         throws IOException,
SecurityException,
IllegalArgumentException,
NoSuchFieldException,
IllegalAccessException,
InterruptedException

Test case of HRegion that is only made out of bulk loaded files. Assert that we don't 'crash'.

Throws:

IOException

SecurityException

IllegalArgumentException

NoSuchFieldException

IllegalAccessException

InterruptedException

testCompactedBulkLoadedFiles

@Test
public void testCompactedBulkLoadedFiles()
                                  throws IOException,
SecurityException,
IllegalArgumentException,
NoSuchFieldException,
IllegalAccessException,
InterruptedException

HRegion test case that is made of a major compacted HFile (created with three bulk loaded files) and an edit in the memstore.

This is for HBASE-10958 "[dataloss] Bulk loading with seqids can prevent some log entries from being replayed"

Throws:

IOException

SecurityException

IllegalArgumentException

NoSuchFieldException

IllegalAccessException

InterruptedException

testReplayEditsWrittenViaHRegion

@Test
public void testReplayEditsWrittenViaHRegion()
                                      throws IOException,
SecurityException,
IllegalArgumentException,
NoSuchFieldException,
IllegalAccessException,
InterruptedException

Test writing edits into an HRegion, closing it, splitting logs, opening Region again. Verify seqids.

Throws:

IOException

SecurityException

IllegalArgumentException

NoSuchFieldException

IllegalAccessException

InterruptedException

testReplayEditsAfterPartialFlush

@Test
public void testReplayEditsAfterPartialFlush()
                                      throws IOException,
SecurityException,
IllegalArgumentException,
NoSuchFieldException,
IllegalAccessException,
InterruptedException

Test that we recover correctly when there is a failure in between the flushes. i.e. Some stores got flushed but others did not.

Unfortunately, there is no easy hook to flush at a store level. The way we get around this is by flushing at the region level, and then deleting the recently flushed store file for one of the Stores. This would put us back in the situation where all but that store got flushed and the region died.

We restart Region again, and verify that the edits were replayed.

Throws:

IOException

SecurityException

IllegalArgumentException

NoSuchFieldException

IllegalAccessException

InterruptedException

testReplayEditsAfterAbortingFlush

@Test
public void testReplayEditsAfterAbortingFlush()
                                       throws IOException

Test that we could recover the data correctly after aborting flush. In the test, first we abort flush after writing some data, then writing more data and flush again, at last verify the data.

Throws:

IOException

getScannedCount

private int getScannedCount(org.apache.hadoop.hbase.regionserver.RegionScanner scanner)
                     throws IOException

Throws:

IOException

testReplayEditsWrittenIntoWAL

@Test
public void testReplayEditsWrittenIntoWAL()
                                   throws Exception

Create an HRegion with the result of a WAL split and test we only see the good edits=

Throws:

Exception

testSequentialEditLogSeqNum

@Test
public void testSequentialEditLogSeqNum()
                                 throws IOException

Throws:

IOException

testDatalossWhenInputError

@Test
public void testDatalossWhenInputError()
                                throws Exception

testcase for https://issues.apache.org/jira/browse/HBASE-15252

Throws:

Exception

testNameConflictWhenSplit

private void testNameConflictWhenSplit(boolean largeFirst)
                                throws IOException,
org.apache.hadoop.hbase.util.CommonFSUtils.StreamLacksCapabilityException

testcase for https://issues.apache.org/jira/browse/HBASE-14949.

Throws:

IOException

org.apache.hadoop.hbase.util.CommonFSUtils.StreamLacksCapabilityException

testNameConflictWhenSplit0

@Test
public void testNameConflictWhenSplit0()
                                throws IOException,
org.apache.hadoop.hbase.util.CommonFSUtils.StreamLacksCapabilityException

Throws:

IOException

org.apache.hadoop.hbase.util.CommonFSUtils.StreamLacksCapabilityException

testNameConflictWhenSplit1

@Test
public void testNameConflictWhenSplit1()
                                throws IOException,
org.apache.hadoop.hbase.util.CommonFSUtils.StreamLacksCapabilityException

Throws:

IOException

org.apache.hadoop.hbase.util.CommonFSUtils.StreamLacksCapabilityException

createBasic1FamilyHTD

private org.apache.hadoop.hbase.client.TableDescriptor createBasic1FamilyHTD(org.apache.hadoop.hbase.TableName tableName)

createMockWAL

private AbstractTestWALReplay.MockWAL createMockWAL()
                                             throws IOException

Throws:

IOException

createWALKey

private org.apache.hadoop.hbase.wal.WALKeyImpl createWALKey(org.apache.hadoop.hbase.TableName tableName,
 org.apache.hadoop.hbase.client.RegionInfo hri,
 org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl mvcc,
 NavigableMap<byte[],Integer> scopes)

createWALEdit

private org.apache.hadoop.hbase.wal.WALEdit createWALEdit(byte[] rowName,
 byte[] family,
 org.apache.hadoop.hbase.util.EnvironmentEdge ee,
 int index)

createFSWALEntry

private org.apache.hadoop.hbase.regionserver.wal.FSWALEntry createFSWALEntry(org.apache.hadoop.hbase.client.TableDescriptor htd,
 org.apache.hadoop.hbase.client.RegionInfo hri,
 long sequence,
 byte[] rowName,
 byte[] family,
 org.apache.hadoop.hbase.util.EnvironmentEdge ee,
 org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl mvcc,
 int index,
 NavigableMap<byte[],Integer> scopes)
                                                                      throws IOException

Throws:

IOException

addWALEdits

private void addWALEdits(org.apache.hadoop.hbase.TableName tableName,
 org.apache.hadoop.hbase.client.RegionInfo hri,
 byte[] rowName,
 byte[] family,
 int count,
 org.apache.hadoop.hbase.util.EnvironmentEdge ee,
 org.apache.hadoop.hbase.wal.WAL wal,
 org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl mvcc,
 NavigableMap<byte[],Integer> scopes)
                  throws IOException

Throws:

IOException

addRegionEdits

public static List<org.apache.hadoop.hbase.client.Put> addRegionEdits(byte[] rowName,
 byte[] family,
 int count,
 org.apache.hadoop.hbase.util.EnvironmentEdge ee,
 org.apache.hadoop.hbase.regionserver.Region r,
 String qualifierPrefix)
                                                               throws IOException

Throws:

IOException

createBasic3FamilyHRegionInfo

private org.apache.hadoop.hbase.client.RegionInfo createBasic3FamilyHRegionInfo(org.apache.hadoop.hbase.TableName tableName)

Creates an HRI around an HTD that has tableName and three column families named 'a','b', and 'c'.

Parameters:

tableName - Name of table to use when we create HTableDescriptor.

runWALSplit

private org.apache.hadoop.fs.Path runWALSplit(org.apache.hadoop.conf.Configuration c)
                                       throws IOException

Run the split. Verify only single split file made.

Returns:

The single split file made

Throws:

IOException

createBasic3FamilyHTD

private org.apache.hadoop.hbase.client.TableDescriptor createBasic3FamilyHTD(org.apache.hadoop.hbase.TableName tableName)

writerWALFile

private void writerWALFile(org.apache.hadoop.fs.Path file,
 List<org.apache.hadoop.hbase.regionserver.wal.FSWALEntry> entries)
                    throws IOException,
org.apache.hadoop.hbase.util.CommonFSUtils.StreamLacksCapabilityException

Throws:

IOException

org.apache.hadoop.hbase.util.CommonFSUtils.StreamLacksCapabilityException

createWAL

protected abstract org.apache.hadoop.hbase.wal.WAL createWAL(org.apache.hadoop.conf.Configuration c,
 org.apache.hadoop.fs.Path hbaseRootDir,
 String logName)
                                                      throws IOException

Throws:

IOException