Package org.apache.hadoop.hbase.regionserver

Class TestBulkLoadReplication

java.lang.Object

org.apache.hadoop.hbase.replication.TestReplicationBase

org.apache.hadoop.hbase.regionserver.TestBulkLoadReplication

Direct Known Subclasses:

TestNamespaceReplicationWithBulkLoadedData

public class TestBulkLoadReplication
extends TestReplicationBase

Integration test for bulk load replication. Defines three clusters, with the following replication topology: "1 <-> 2 <-> 3" (active-active between 1 and 2, and active-active between 2 and 3). For each of defined test clusters, it performs a bulk load, asserting values on bulk loaded file gets replicated to other two peers. Since we are doing 3 bulk loads, with the given replication topology all these bulk loads should get replicated only once on each peer. To assert this, this test defines a preBulkLoad coprocessor and adds it to all test table regions, on each of the clusters. This CP counts the amount of times bulk load actually gets invoked, certifying we are not entering the infinite loop condition addressed by HBASE-22380.

Nested Class Summary

Nested Classes

Modifier and Type	Class	Description
static class	TestBulkLoadReplication.BulkReplicationTestObserver

Nested classes/interfaces inherited from class org.apache.hadoop.hbase.replication.TestReplicationBase

TestReplicationBase.ReplicationEndpointTest

Field Summary

Fields

Modifier and Type	Field	Description
private static final org.apache.hadoop.fs.Path	BULK_LOAD_BASE_DIR
private static CountDownLatch	BULK_LOAD_LATCH
private static AtomicInteger	BULK_LOADS_COUNT
static final HBaseClassTestRule	CLASS_RULE
protected static final org.apache.hadoop.conf.Configuration	CONF3
private static org.apache.hadoop.hbase.client.Table	htable3
protected static final org.slf4j.Logger	LOG
org.junit.rules.TestName	name
private static final String	PEER_ID1
private static final String	PEER_ID3
private static final String	PEER1_CLUSTER_ID
private static final String	PEER2_CLUSTER_ID
private static final String	PEER3_CLUSTER_ID
private static org.apache.hadoop.hbase.replication.ReplicationQueueStorage	queueStorage
private static boolean	replicationPeersAdded
static org.junit.rules.TemporaryFolder	testFolder
protected static final HBaseTestingUtil	UTIL3

Fields inherited from class org.apache.hadoop.hbase.replication.TestReplicationBase

CONF_WITH_LOCALFS, CONF1, CONF2, connection1, connection2, famName, hbaseAdmin, htable1, htable2, NB_RETRIES, NB_ROWS_IN_BATCH, NB_ROWS_IN_BIG_BATCH, noRepfamName, NUM_SLAVES1, NUM_SLAVES2, PEER_ID2, replicateCount, replicatedEntries, row, SLEEP_TIME, tableName, UTIL1, UTIL2

Constructor Summary

Constructors

Constructor	Description
TestBulkLoadReplication()

Method Summary

Modifier and Type	Method	Description
protected void	assertBulkLoadConditions(org.apache.hadoop.hbase.TableName tableName, byte[] row, byte[] value, HBaseTestingUtil utility, org.apache.hadoop.hbase.client.Table... tables)
private void	assertBulkLoadConditionsForNoRepFamily(byte[] row, byte[] value, HBaseTestingUtil utility, org.apache.hadoop.hbase.client.Table... tables)
protected void	assertTableHasValue(org.apache.hadoop.hbase.client.Table table, byte[] row, byte[] value)
private void	assertTableNotHasValue(org.apache.hadoop.hbase.client.Table table, byte[] row, byte[] value)
protected void	assertTableNoValue(org.apache.hadoop.hbase.client.Table table, byte[] row, byte[] value)
protected void	bulkLoadOnCluster(org.apache.hadoop.hbase.TableName tableName, byte[] row, byte[] value, HBaseTestingUtil cluster)
private void	bulkLoadOnClusterForNoRepFamily(byte[] row, byte[] value, HBaseTestingUtil cluster)
private void	copyToHdfs(String bulkLoadFilePath, org.apache.hadoop.hdfs.MiniDFSCluster cluster)
private void	copyToHdfsForNoRepFamily(String bulkLoadFilePath, org.apache.hadoop.hdfs.MiniDFSCluster cluster)
private String	createHFileForFamilies(byte[] row, byte[] value, org.apache.hadoop.conf.Configuration clusterConfig)
private String	createHFileForNoRepFamilies(byte[] row, byte[] value, org.apache.hadoop.conf.Configuration clusterConfig)
private org.apache.hadoop.hbase.replication.ReplicationPeerConfig	getPeerConfigForCluster(HBaseTestingUtil util)
void	setUpBase()
static void	setUpBeforeClass()
protected static void	setupBulkLoadConfigsForCluster(org.apache.hadoop.conf.Configuration config, String clusterReplicationId)
private void	setupCoprocessor(HBaseTestingUtil cluster)
private static void	startThirdCluster()
void	testBulkLoadReplicationActiveActive()
void	testBulkloadReplicationActiveActiveForNoRepFamily()

Methods inherited from class org.apache.hadoop.hbase.replication.TestReplicationBase

addPeer, addPeer, cleanUp, createTable, getClusterKey, isSerialPeer, isSyncPeer, loadData, loadData, removePeer, removePeer, restartSourceCluster, runSimplePutDeleteTest, runSmallBatchTest, setupConfig, stopAllRegionServers, tearDownAfterClass, tearDownBase, waitForReplication, waitForReplication

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Details

CLASS_RULE

public static final HBaseClassTestRule CLASS_RULE

LOG

protected static final org.slf4j.Logger LOG

PEER1_CLUSTER_ID

private static final String PEER1_CLUSTER_ID

See Also:

• Constant Field Values

PEER2_CLUSTER_ID

private static final String PEER2_CLUSTER_ID

See Also:

• Constant Field Values

PEER3_CLUSTER_ID

private static final String PEER3_CLUSTER_ID

See Also:

• Constant Field Values

PEER_ID1

private static final String PEER_ID1

See Also:

• Constant Field Values

PEER_ID3

private static final String PEER_ID3

See Also:

• Constant Field Values

BULK_LOADS_COUNT

private static AtomicInteger BULK_LOADS_COUNT

BULK_LOAD_LATCH

private static CountDownLatch BULK_LOAD_LATCH

UTIL3

protected static final HBaseTestingUtil UTIL3

CONF3

protected static final org.apache.hadoop.conf.Configuration CONF3

BULK_LOAD_BASE_DIR

private static final org.apache.hadoop.fs.Path BULK_LOAD_BASE_DIR

htable3

private static org.apache.hadoop.hbase.client.Table htable3

name

public org.junit.rules.TestName name

testFolder

public static org.junit.rules.TemporaryFolder testFolder

queueStorage

private static org.apache.hadoop.hbase.replication.ReplicationQueueStorage queueStorage

replicationPeersAdded

private static boolean replicationPeersAdded

Constructor Details

TestBulkLoadReplication

public TestBulkLoadReplication()

Method Details

setUpBeforeClass

public static void setUpBeforeClass()
                             throws Exception

Throws:

Exception

startThirdCluster

private static void startThirdCluster()
                               throws Exception

Throws:

Exception

setUpBase

public void setUpBase()
               throws Exception

Overrides:

setUpBase in class TestReplicationBase

Throws:

Exception

getPeerConfigForCluster

private org.apache.hadoop.hbase.replication.ReplicationPeerConfig getPeerConfigForCluster(HBaseTestingUtil util)
                                                                                   throws UnknownHostException

Throws:

UnknownHostException

setupCoprocessor

private void setupCoprocessor(HBaseTestingUtil cluster)

setupBulkLoadConfigsForCluster

protected static void setupBulkLoadConfigsForCluster(org.apache.hadoop.conf.Configuration config,
 String clusterReplicationId)
                                              throws Exception

Throws:

Exception

testBulkLoadReplicationActiveActive

public void testBulkLoadReplicationActiveActive()
                                         throws Exception

Throws:

Exception

assertBulkLoadConditions

protected void assertBulkLoadConditions(org.apache.hadoop.hbase.TableName tableName,
 byte[] row,
 byte[] value,
 HBaseTestingUtil utility,
 org.apache.hadoop.hbase.client.Table... tables)
                                 throws Exception

Throws:

Exception

bulkLoadOnCluster

protected void bulkLoadOnCluster(org.apache.hadoop.hbase.TableName tableName,
 byte[] row,
 byte[] value,
 HBaseTestingUtil cluster)
                          throws Exception

Throws:

Exception

copyToHdfs

private void copyToHdfs(String bulkLoadFilePath,
 org.apache.hadoop.hdfs.MiniDFSCluster cluster)
                 throws Exception

Throws:

Exception

assertTableHasValue

protected void assertTableHasValue(org.apache.hadoop.hbase.client.Table table,
 byte[] row,
 byte[] value)
                            throws Exception

Throws:

Exception

assertTableNoValue

protected void assertTableNoValue(org.apache.hadoop.hbase.client.Table table,
 byte[] row,
 byte[] value)
                           throws Exception

Throws:

Exception

createHFileForFamilies

private String createHFileForFamilies(byte[] row,
 byte[] value,
 org.apache.hadoop.conf.Configuration clusterConfig)
                               throws IOException

Throws:

IOException

testBulkloadReplicationActiveActiveForNoRepFamily

public void testBulkloadReplicationActiveActiveForNoRepFamily()
                                                       throws Exception

Throws:

Exception

assertBulkLoadConditionsForNoRepFamily

private void assertBulkLoadConditionsForNoRepFamily(byte[] row,
 byte[] value,
 HBaseTestingUtil utility,
 org.apache.hadoop.hbase.client.Table... tables)
                                             throws Exception

Throws:

Exception

bulkLoadOnClusterForNoRepFamily

private void bulkLoadOnClusterForNoRepFamily(byte[] row,
 byte[] value,
 HBaseTestingUtil cluster)
                                      throws Exception

Throws:

Exception

createHFileForNoRepFamilies

private String createHFileForNoRepFamilies(byte[] row,
 byte[] value,
 org.apache.hadoop.conf.Configuration clusterConfig)
                                    throws IOException

Throws:

IOException

copyToHdfsForNoRepFamily

private void copyToHdfsForNoRepFamily(String bulkLoadFilePath,
 org.apache.hadoop.hdfs.MiniDFSCluster cluster)
                               throws Exception

Throws:

Exception

assertTableNotHasValue

private void assertTableNotHasValue(org.apache.hadoop.hbase.client.Table table,
 byte[] row,
 byte[] value)
                             throws IOException

Throws:

IOException