Package org.apache.hadoop.hbase.backup.replication

Class TestContinuousBackupReplicationEndpoint

java.lang.Object

org.apache.hadoop.hbase.backup.replication.TestContinuousBackupReplicationEndpoint

@Tag("org.apache.hadoop.hbase.testclassification.ReplicationTests") @Tag("org.apache.hadoop.hbase.testclassification.LargeTests")
public class TestContinuousBackupReplicationEndpoint
extends Object

Nested Class Summary

Nested Classes

Modifier and Type	Class	Description
static class	TestContinuousBackupReplicationEndpoint.FailingOnceContinuousBackupReplicationEndpoint	Replication endpoint that fails only once on first upload attempt, then succeeds on retry.
static class	TestContinuousBackupReplicationEndpoint.PartiallyUploadedBulkloadFileEndpoint	Replication endpoint that simulates leaving a partial file behind on first attempt, then succeeds on second attempt by overwriting it.

Field Summary

Fields

Modifier and Type	Field	Description
private static org.apache.hadoop.hbase.client.Admin	admin
private static final String	CF_NAME
private static final org.apache.hadoop.conf.Configuration	conf
(package private) static org.apache.hadoop.fs.FileSystem	fs
private static final org.slf4j.Logger	LOG
private static final byte[]	QUALIFIER
private final String	replicationEndpoint
(package private) static org.apache.hadoop.fs.Path	root
private static final HBaseTestingUtil	TEST_UTIL

Constructor Summary

Constructors

Constructor	Description
TestContinuousBackupReplicationEndpoint()

Method Summary

Modifier and Type	Method	Description
private void	addReplicationPeer(String peerId, org.apache.hadoop.fs.Path backupRootDir, Map<org.apache.hadoop.hbase.TableName,List<String>> tableMap)
private void	addReplicationPeer(String peerId, org.apache.hadoop.fs.Path backupRootDir, Map<org.apache.hadoop.hbase.TableName,List<String>> tableMap, String customReplicationEndpointImpl)
private void	bulkLoadHFiles(org.apache.hadoop.hbase.TableName tableName, Map<byte[],List<org.apache.hadoop.fs.Path>> family2Files)
private void	bulkLoadHFiles(org.apache.hadoop.hbase.TableName tableName, org.apache.hadoop.fs.Path inputDir)
private void	createTable(org.apache.hadoop.hbase.TableName tableName)
private void	deleteReplicationPeer(String peerId)
private void	deleteTable(org.apache.hadoop.hbase.TableName tableName)
private void	generateHFiles(org.apache.hadoop.fs.Path outputDir)
private int	getRowCount(org.apache.hadoop.hbase.TableName tableName)
private void	loadRandomData(org.apache.hadoop.hbase.TableName tableName, int totalRows)
private void	replayBulkLoadHFilesIfPresent(String bulkLoadDir, org.apache.hadoop.hbase.TableName tableName)
private void	replayWALs(String walDir, org.apache.hadoop.hbase.TableName tableName)
static void	setUpBeforeClass()
static void	tearDownAfterClass()
void	testBulkLoadFileUploadRetry()	Simulates a one-time failure during bulk load file upload.
void	testBulkLoadFileUploadWithStaleFileRetry()	Simulates a stale/partial file left behind after a failed bulk load.
void	testCopyWithCleanupDeletesPartialFile()	Unit test for verifying cleanup of partial files.
void	testDayWiseWALBackup()
void	testMultiTableWALBackup()
void	testWALAndBulkLoadFileBackup()
void	testWALBackupWithPeerRestart()
private void	verifyBackup(String backupRootDir, boolean hasBulkLoadFiles, Map<org.apache.hadoop.hbase.TableName,Integer> tablesWithExpectedRows)	Verifies the backup process by: 1.
private void	verifyBulkLoadBackup(String backupRootDir)
private void	verifyWALBackup(String backupRootDir)
private void	waitForReplication(int durationInMillis)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Details

LOG

private static final org.slf4j.Logger LOG

TEST_UTIL

private static final HBaseTestingUtil TEST_UTIL

conf

private static final org.apache.hadoop.conf.Configuration conf

admin

private static org.apache.hadoop.hbase.client.Admin admin

replicationEndpoint

private final String replicationEndpoint

CF_NAME

private static final String CF_NAME

See Also:

• Constant Field Values

QUALIFIER

private static final byte[] QUALIFIER

fs

static org.apache.hadoop.fs.FileSystem fs

root

static org.apache.hadoop.fs.Path root

Constructor Details

TestContinuousBackupReplicationEndpoint

public TestContinuousBackupReplicationEndpoint()

Method Details

setUpBeforeClass

@BeforeAll
public static void setUpBeforeClass()
                             throws Exception

Throws:

Exception

tearDownAfterClass

@AfterAll
public static void tearDownAfterClass()
                               throws Exception

Throws:

Exception

testWALAndBulkLoadFileBackup

@Test
public void testWALAndBulkLoadFileBackup()
                                  throws IOException

Throws:

IOException

testMultiTableWALBackup

@Test
public void testMultiTableWALBackup()
                             throws IOException

Throws:

IOException

testWALBackupWithPeerRestart

@Test
public void testWALBackupWithPeerRestart()
                                  throws IOException,
InterruptedException

Throws:

IOException

InterruptedException

testDayWiseWALBackup

@Test
public void testDayWiseWALBackup()
                          throws IOException

Throws:

IOException

testBulkLoadFileUploadRetry

@Test
public void testBulkLoadFileUploadRetry()
                                 throws IOException

Simulates a one-time failure during bulk load file upload. This validates that the retry logic in the replication endpoint works as expected.

Throws:

IOException

testCopyWithCleanupDeletesPartialFile

@Test
public void testCopyWithCleanupDeletesPartialFile()
                                           throws Exception

Unit test for verifying cleanup of partial files. Simulates a failure during FileUtil.copy(FileSystem, Path, FileSystem, Path, boolean, boolean, Configuration) and checks that the destination file is deleted.

Throws:

Exception

testBulkLoadFileUploadWithStaleFileRetry

@Test
public void testBulkLoadFileUploadWithStaleFileRetry()
                                              throws Exception

Simulates a stale/partial file left behind after a failed bulk load. On retry, the stale file should be overwritten and replication succeeds.

Throws:

Exception

createTable

private void createTable(org.apache.hadoop.hbase.TableName tableName)
                  throws IOException

Throws:

IOException

deleteTable

private void deleteTable(org.apache.hadoop.hbase.TableName tableName)
                  throws IOException

Throws:

IOException

addReplicationPeer

private void addReplicationPeer(String peerId,
 org.apache.hadoop.fs.Path backupRootDir,
 Map<org.apache.hadoop.hbase.TableName,List<String>> tableMap)
                         throws IOException

Throws:

IOException

addReplicationPeer

private void addReplicationPeer(String peerId,
 org.apache.hadoop.fs.Path backupRootDir,
 Map<org.apache.hadoop.hbase.TableName,List<String>> tableMap,
 String customReplicationEndpointImpl)
                         throws IOException

Throws:

IOException

deleteReplicationPeer

private void deleteReplicationPeer(String peerId)
                            throws IOException

Throws:

IOException

loadRandomData

private void loadRandomData(org.apache.hadoop.hbase.TableName tableName,
 int totalRows)
                     throws IOException

Throws:

IOException

bulkLoadHFiles

private void bulkLoadHFiles(org.apache.hadoop.hbase.TableName tableName,
 org.apache.hadoop.fs.Path inputDir)
                     throws IOException

Throws:

IOException

bulkLoadHFiles

private void bulkLoadHFiles(org.apache.hadoop.hbase.TableName tableName,
 Map<byte[],List<org.apache.hadoop.fs.Path>> family2Files)
                     throws IOException

Throws:

IOException

generateHFiles

private void generateHFiles(org.apache.hadoop.fs.Path outputDir)
                     throws IOException

Throws:

IOException

waitForReplication

private void waitForReplication(int durationInMillis)

verifyBackup

private void verifyBackup(String backupRootDir,
 boolean hasBulkLoadFiles,
 Map<org.apache.hadoop.hbase.TableName,Integer> tablesWithExpectedRows)
                   throws IOException

Verifies the backup process by: 1. Checking whether any WAL (Write-Ahead Log) files were generated in the backup directory. 2. Checking whether any bulk-loaded files were generated in the backup directory. 3. Replaying the WAL and bulk-loaded files (if present) to restore data and check consistency by verifying that the restored data matches the expected row count for each table.

Throws:

IOException

verifyWALBackup

private void verifyWALBackup(String backupRootDir)
                      throws IOException

Throws:

IOException

verifyBulkLoadBackup

private void verifyBulkLoadBackup(String backupRootDir)
                           throws IOException

Throws:

IOException

replayWALs

private void replayWALs(String walDir,
 org.apache.hadoop.hbase.TableName tableName)

replayBulkLoadHFilesIfPresent

private void replayBulkLoadHFilesIfPresent(String bulkLoadDir,
 org.apache.hadoop.hbase.TableName tableName)

getRowCount

private int getRowCount(org.apache.hadoop.hbase.TableName tableName)
                 throws IOException

Throws:

IOException