org.apache.hadoop.hbase.regionserver

Class TestBulkLoadReplication

java.lang.Object

org.apache.hadoop.hbase.replication.TestReplicationBase

org.apache.hadoop.hbase.regionserver.TestBulkLoadReplication

public class TestBulkLoadReplication
extends TestReplicationBase

Integration test for bulk load replication. Defines three clusters, with the following replication topology: "1 <-> 2 <-> 3" (active-active between 1 and 2, and active-active between 2 and 3). For each of defined test clusters, it performs a bulk load, asserting values on bulk loaded file gets replicated to other two peers. Since we are doing 3 bulk loads, with the given replication topology all these bulk loads should get replicated only once on each peer. To assert this, this test defines a preBulkLoad coprocessor and adds it to all test table regions, on each of the clusters. This CP counts the amount of times bulk load actually gets invoked, certifying we are not entering the infinite loop condition addressed by HBASE-22380.

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
static class	TestBulkLoadReplication.BulkReplicationTestObserver

Field Summary

Fields

Modifier and Type	Field and Description
private static org.apache.hadoop.fs.Path	BULK_LOAD_BASE_DIR
private static CountDownLatch	BULK_LOAD_LATCH
private static AtomicInteger	BULK_LOADS_COUNT
static HBaseClassTestRule	CLASS_RULE
private static org.apache.hadoop.conf.Configuration	conf3
private static org.apache.hadoop.conf.Configuration	conf4
private static org.apache.hadoop.hbase.client.Table	htable3
private static org.apache.hadoop.hbase.client.Table	htable4
protected static org.slf4j.Logger	LOG
org.junit.rules.TestName	name
private static String	PEER_ID1
private static String	PEER_ID3
private static String	PEER_ID4
private static String	PEER1_CLUSTER_ID
private static String	PEER3_CLUSTER_ID
private static String	PEER4_CLUSTER_ID
static org.junit.rules.TemporaryFolder	testFolder
private static HBaseTestingUtility	utility3
private static HBaseTestingUtility	utility4

Fields inherited from class org.apache.hadoop.hbase.replication.TestReplicationBase

admin, CONF_WITH_LOCALFS, conf1, conf2, famName, hbaseAdmin, htable1, htable2, NB_RETRIES, NB_ROWS_IN_BATCH, NB_ROWS_IN_BIG_BATCH, noRepfamName, PEER_ID2, row, scopes, SLEEP_TIME, tableName, utility1, utility2, zkw1, zkw2

Constructor Summary

Constructors

Constructor and Description
TestBulkLoadReplication()

Method Summary

Modifier and Type	Method and Description
private void	assertBulkLoadConditions(byte[] row, byte[] value, HBaseTestingUtility utility, org.apache.hadoop.hbase.client.Table... tables)
private void	assertTableHasValue(org.apache.hadoop.hbase.client.Table table, byte[] row, byte[] value)
private void	bulkLoadOnCluster(byte[] row, byte[] value, HBaseTestingUtility cluster)
private void	copyToHdfs(String bulkLoadFilePath, org.apache.hadoop.hdfs.MiniDFSCluster cluster)
private String	createHFileForFamilies(byte[] row, byte[] value, org.apache.hadoop.conf.Configuration clusterConfig)
private org.apache.hadoop.fs.Path	createMobFiles(HBaseTestingUtility util)
private org.apache.hadoop.hbase.replication.ReplicationPeerConfig	getPeerConfigForCluster(HBaseTestingUtility util)
void	setUpBase()
static void	setUpBeforeClass()
private static void	setupBulkLoadConfigsForCluster(org.apache.hadoop.conf.Configuration config, String clusterReplicationId)
private void	setupCoprocessor(HBaseTestingUtility cluster)
private static void	startCluster(HBaseTestingUtility util, org.apache.hadoop.conf.Configuration configuration)
void	tearDownBase()
void	testBulkLoadReplicationActiveActive()
void	testPartionedMOBCompactionBulkLoadDoesntReplicate()

Methods inherited from class org.apache.hadoop.hbase.replication.TestReplicationBase

cleanUp, isSerialPeer, loadData, loadData, runSimplePutDeleteTest, runSmallBatchTest, tearDownAfterClass, waitForReplication

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

CLASS_RULE

public static final HBaseClassTestRule CLASS_RULE

LOG

protected static final org.slf4j.Logger LOG

PEER1_CLUSTER_ID

private static final String PEER1_CLUSTER_ID

See Also:

Constant Field Values

PEER4_CLUSTER_ID

private static final String PEER4_CLUSTER_ID

See Also:

Constant Field Values

PEER3_CLUSTER_ID

private static final String PEER3_CLUSTER_ID

See Also:

Constant Field Values

PEER_ID1

private static final String PEER_ID1

See Also:

Constant Field Values

PEER_ID3

private static final String PEER_ID3

See Also:

Constant Field Values

PEER_ID4

private static final String PEER_ID4

See Also:

Constant Field Values

BULK_LOADS_COUNT

private static AtomicInteger BULK_LOADS_COUNT

BULK_LOAD_LATCH

private static CountDownLatch BULK_LOAD_LATCH

BULK_LOAD_BASE_DIR

private static final org.apache.hadoop.fs.Path BULK_LOAD_BASE_DIR

utility3

private static HBaseTestingUtility utility3

utility4

private static HBaseTestingUtility utility4

conf3

private static org.apache.hadoop.conf.Configuration conf3

conf4

private static org.apache.hadoop.conf.Configuration conf4

htable3

private static org.apache.hadoop.hbase.client.Table htable3

htable4

private static org.apache.hadoop.hbase.client.Table htable4

name

public org.junit.rules.TestName name

testFolder

public static org.junit.rules.TemporaryFolder testFolder

Constructor Detail

TestBulkLoadReplication

public TestBulkLoadReplication()

Method Detail

setUpBeforeClass

public static void setUpBeforeClass()
                             throws Exception

Throws:

Exception

startCluster

private static void startCluster(HBaseTestingUtility util,
                                 org.apache.hadoop.conf.Configuration configuration)
                          throws Exception

Throws:

Exception

setUpBase

public void setUpBase()
               throws Exception

Overrides:

setUpBase in class TestReplicationBase

Throws:

Exception

getPeerConfigForCluster

private org.apache.hadoop.hbase.replication.ReplicationPeerConfig getPeerConfigForCluster(HBaseTestingUtility util)

setupCoprocessor

private void setupCoprocessor(HBaseTestingUtility cluster)

tearDownBase

public void tearDownBase()
                  throws Exception

Overrides:

tearDownBase in class TestReplicationBase

Throws:

Exception

setupBulkLoadConfigsForCluster

private static void setupBulkLoadConfigsForCluster(org.apache.hadoop.conf.Configuration config,
                                                   String clusterReplicationId)
                                            throws Exception

Throws:

Exception

testBulkLoadReplicationActiveActive

public void testBulkLoadReplicationActiveActive()
                                         throws Exception

Throws:

Exception

testPartionedMOBCompactionBulkLoadDoesntReplicate

public void testPartionedMOBCompactionBulkLoadDoesntReplicate()
                                                       throws Exception

Throws:

Exception

assertBulkLoadConditions

private void assertBulkLoadConditions(byte[] row,
                                      byte[] value,
                                      HBaseTestingUtility utility,
                                      org.apache.hadoop.hbase.client.Table... tables)
                               throws Exception

Throws:

Exception

bulkLoadOnCluster

private void bulkLoadOnCluster(byte[] row,
                               byte[] value,
                               HBaseTestingUtility cluster)
                        throws Exception

Throws:

Exception

copyToHdfs

private void copyToHdfs(String bulkLoadFilePath,
                        org.apache.hadoop.hdfs.MiniDFSCluster cluster)
                 throws Exception

Throws:

Exception

assertTableHasValue

private void assertTableHasValue(org.apache.hadoop.hbase.client.Table table,
                                 byte[] row,
                                 byte[] value)
                          throws Exception

Throws:

Exception

createHFileForFamilies

private String createHFileForFamilies(byte[] row,
                                      byte[] value,
                                      org.apache.hadoop.conf.Configuration clusterConfig)
                               throws IOException

Throws:

IOException

createMobFiles

private org.apache.hadoop.fs.Path createMobFiles(HBaseTestingUtility util)
                                          throws IOException

Throws:

IOException