org.apache.hadoop.hbase.master.replication

Class ReplicationPeerManager

java.lang.Object

org.apache.hadoop.hbase.master.replication.ReplicationPeerManager

All Implemented Interfaces:

ConfigurationObserver

@InterfaceAudience.Private
public class ReplicationPeerManager
extends Object
implements ConfigurationObserver

Manages and performs all replication admin operations.

Used to add/remove a replication peer.

Implement ConfigurationObserver mainly for recreating ReplicationPeerStorage, for supporting migrating across different replication peer storages without restarting master.

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
private static interface	ReplicationPeerManager.ExceptionalRunnable
(package private) static interface	ReplicationPeerManager.ReplicationQueueStorageInitializer

Field Summary

Fields

Modifier and Type	Field and Description
private org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap<SyncReplicationState,EnumSet<SyncReplicationState>>	allowedTransition
private String	clusterId
private org.apache.hadoop.conf.Configuration	conf
private org.apache.hadoop.fs.FileSystem	fs
private static org.slf4j.Logger	LOG
private ConcurrentMap<String,ReplicationPeerDescription>	peers
private ReplicationPeerStorage	peerStorage
private ReplicationQueueStorage	queueStorage
private ReplicationPeerManager.ReplicationQueueStorageInitializer	queueStorageInitializer
private ReplicationLogCleanerBarrier	replicationLogCleanerBarrier
private Semaphore	syncReplicationPeerLock
private ZKWatcher	zk

Constructor Summary

Constructors

Constructor and Description
ReplicationPeerManager(org.apache.hadoop.fs.FileSystem fs, ZKWatcher zk, ReplicationPeerStorage peerStorage, ReplicationQueueStorage queueStorage, ConcurrentMap<String,ReplicationPeerDescription> peers, org.apache.hadoop.conf.Configuration conf, String clusterId, ReplicationPeerManager.ReplicationQueueStorageInitializer queueStorageInitializer)

Method Summary

Modifier and Type	Method and Description
void	addPeer(String peerId, ReplicationPeerConfig peerConfig, boolean enabled)
private void	checkClusterKey(String clusterKey)
private void	checkConfiguredWALEntryFilters(ReplicationPeerConfig peerConfig)
private void	checkNamespacesAndTableCfsConfigConflict(Set<String> namespaces, Map<TableName,? extends Collection<String>> tableCfs) Set a namespace in the peer config means that all tables in this namespace will be replicated to the peer cluster.
private void	checkPeerConfig(ReplicationPeerConfig peerConfig)
private void	checkPeerConfigForSyncReplication(ReplicationPeerConfig peerConfig)
private ReplicationPeerDescription	checkPeerExists(String peerId)
private void	checkPeerInDAStateIfSyncReplication(String peerId)
private void	checkQueuesDeleted(String peerId)
private void	checkSameClusterKey(String clusterKey)
private void	checkSyncReplicationPeerConfigConflict(ReplicationPeerConfig peerConfig)
private ReplicationQueueData	convert(ZKReplicationQueueStorageForMigration.ZkReplicationQueueData zkData)
static ReplicationPeerManager	create(MasterServices services, String clusterId)
private static Pair<ReplicationQueueStorage,ReplicationPeerManager.ReplicationQueueStorageInitializer>	createReplicationQueueStorage(MasterServices services)
void	disablePeer(String peerId)
void	enablePeer(String peerId)
Optional<ReplicationPeerConfig>	getPeerConfig(String peerId)
boolean	getPeerState(String peerId)
ReplicationPeerStorage	getPeerStorage()
ReplicationQueueStorage	getQueueStorage()
ReplicationLogCleanerBarrier	getReplicationLogCleanerBarrier()
List<String>	getSerialPeerIdsBelongsTo(TableName tableName)
private void	initializeQueueStorage()
private boolean	isStringEquals(String s1, String s2) For replication peer cluster key or endpoint class, null and empty string is same.
List<ReplicationPeerDescription>	listPeers(Pattern pattern)
private void	migrateHFileRefs(ZKReplicationQueueStorageForMigration oldQueueStorage)
private void	migrateLastPushedSeqIds(ZKReplicationQueueStorageForMigration oldQueueStorage)
private void	migrateQueues(ZKReplicationQueueStorageForMigration oldQueueStorage)
(package private) CompletableFuture<?>	migrateQueuesFromZk(ZKWatcher zookeeper, ExecutorService executor) Submit the migration tasks to the given executor.
void	onConfigurationChange(org.apache.hadoop.conf.Configuration conf) This method would be called by the ConfigurationManager object when the Configuration object is reloaded from disk.
(package private) void	preAddPeer(String peerId, ReplicationPeerConfig peerConfig)
(package private) void	preDisablePeer(String peerId)
(package private) void	preEnablePeer(String peerId)
(package private) ReplicationPeerConfig	preRemovePeer(String peerId)
(package private) ReplicationPeerDescription	preTransitPeerSyncReplicationState(String peerId, SyncReplicationState state) Returns the old desciption of the peer
(package private) ReplicationPeerDescription	preUpdatePeerConfig(String peerId, ReplicationPeerConfig peerConfig) Return the old peer description.
void	releaseSyncReplicationPeerLock()
(package private) void	removeAllLastPushedSeqIds(String peerId)
void	removeAllQueues(String peerId)
void	removeAllQueuesAndHFileRefs(String peerId)
void	removePeer(String peerId)
private CompletableFuture<?>	runAsync(ReplicationPeerManager.ExceptionalRunnable task, ExecutorService executor)
void	setPeerNewSyncReplicationState(String peerId, SyncReplicationState state)
private void	setPeerState(String peerId, boolean enabled)
void	transitPeerSyncReplicationState(String peerId, SyncReplicationState newState)
boolean	tryAcquireSyncReplicationPeerLock()
void	updatePeerConfig(String peerId, ReplicationPeerConfig peerConfig)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

LOG

private static final org.slf4j.Logger LOG

peerStorage

private volatile ReplicationPeerStorage peerStorage

queueStorage

private final ReplicationQueueStorage queueStorage

peers

private final ConcurrentMap<String,ReplicationPeerDescription> peers

allowedTransition

private final org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap<SyncReplicationState,EnumSet<SyncReplicationState>> allowedTransition

syncReplicationPeerLock

private final Semaphore syncReplicationPeerLock

replicationLogCleanerBarrier

private final ReplicationLogCleanerBarrier replicationLogCleanerBarrier

clusterId

private final String clusterId

conf

private volatile org.apache.hadoop.conf.Configuration conf

fs

private final org.apache.hadoop.fs.FileSystem fs

zk

private final ZKWatcher zk

queueStorageInitializer

private final ReplicationPeerManager.ReplicationQueueStorageInitializer queueStorageInitializer

Constructor Detail

ReplicationPeerManager

ReplicationPeerManager(org.apache.hadoop.fs.FileSystem fs,
                       ZKWatcher zk,
                       ReplicationPeerStorage peerStorage,
                       ReplicationQueueStorage queueStorage,
                       ConcurrentMap<String,ReplicationPeerDescription> peers,
                       org.apache.hadoop.conf.Configuration conf,
                       String clusterId,
                       ReplicationPeerManager.ReplicationQueueStorageInitializer queueStorageInitializer)

Method Detail

checkQueuesDeleted

private void checkQueuesDeleted(String peerId)
                         throws ReplicationException,
                                DoNotRetryIOException

Throws:

ReplicationException

DoNotRetryIOException

initializeQueueStorage

private void initializeQueueStorage()
                             throws IOException

Throws:

IOException

preAddPeer

void preAddPeer(String peerId,
                ReplicationPeerConfig peerConfig)
         throws ReplicationException,
                IOException

Throws:

ReplicationException

IOException

checkPeerExists

private ReplicationPeerDescription checkPeerExists(String peerId)
                                            throws DoNotRetryIOException

Throws:

DoNotRetryIOException

checkPeerInDAStateIfSyncReplication

private void checkPeerInDAStateIfSyncReplication(String peerId)
                                          throws DoNotRetryIOException

Throws:

DoNotRetryIOException

preRemovePeer

ReplicationPeerConfig preRemovePeer(String peerId)
                             throws DoNotRetryIOException

Throws:

DoNotRetryIOException

preEnablePeer

void preEnablePeer(String peerId)
            throws DoNotRetryIOException

Throws:

DoNotRetryIOException

preDisablePeer

void preDisablePeer(String peerId)
             throws DoNotRetryIOException

Throws:

DoNotRetryIOException

preUpdatePeerConfig

ReplicationPeerDescription preUpdatePeerConfig(String peerId,
                                               ReplicationPeerConfig peerConfig)
                                        throws DoNotRetryIOException

Return the old peer description. Can never be null.

Throws:

DoNotRetryIOException

preTransitPeerSyncReplicationState

ReplicationPeerDescription preTransitPeerSyncReplicationState(String peerId,
                                                              SyncReplicationState state)
                                                       throws DoNotRetryIOException

Returns the old desciption of the peer

Throws:

DoNotRetryIOException

addPeer

public void addPeer(String peerId,
                    ReplicationPeerConfig peerConfig,
                    boolean enabled)
             throws ReplicationException

Throws:

ReplicationException

removePeer

public void removePeer(String peerId)
                throws ReplicationException

Throws:

ReplicationException

setPeerState

private void setPeerState(String peerId,
                          boolean enabled)
                   throws ReplicationException

Throws:

ReplicationException

getPeerState

public boolean getPeerState(String peerId)
                     throws ReplicationException

Throws:

ReplicationException

enablePeer

public void enablePeer(String peerId)
                throws ReplicationException

Throws:

ReplicationException

disablePeer

public void disablePeer(String peerId)
                 throws ReplicationException

Throws:

ReplicationException

updatePeerConfig

public void updatePeerConfig(String peerId,
                             ReplicationPeerConfig peerConfig)
                      throws ReplicationException

Throws:

ReplicationException

listPeers

public List<ReplicationPeerDescription> listPeers(Pattern pattern)

getPeerConfig

public Optional<ReplicationPeerConfig> getPeerConfig(String peerId)

removeAllLastPushedSeqIds

void removeAllLastPushedSeqIds(String peerId)
                        throws ReplicationException

Throws:

ReplicationException

setPeerNewSyncReplicationState

public void setPeerNewSyncReplicationState(String peerId,
                                           SyncReplicationState state)
                                    throws ReplicationException

Throws:

ReplicationException

transitPeerSyncReplicationState

public void transitPeerSyncReplicationState(String peerId,
                                            SyncReplicationState newState)
                                     throws ReplicationException

Throws:

ReplicationException

removeAllQueues

public void removeAllQueues(String peerId)
                     throws ReplicationException

Throws:

ReplicationException

removeAllQueuesAndHFileRefs

public void removeAllQueuesAndHFileRefs(String peerId)
                                 throws ReplicationException

Throws:

ReplicationException

checkPeerConfig

private void checkPeerConfig(ReplicationPeerConfig peerConfig)
                      throws DoNotRetryIOException

Throws:

DoNotRetryIOException

checkPeerConfigForSyncReplication

private void checkPeerConfigForSyncReplication(ReplicationPeerConfig peerConfig)
                                        throws DoNotRetryIOException

Throws:

DoNotRetryIOException

checkSyncReplicationPeerConfigConflict

private void checkSyncReplicationPeerConfigConflict(ReplicationPeerConfig peerConfig)
                                             throws DoNotRetryIOException

Throws:

DoNotRetryIOException

checkNamespacesAndTableCfsConfigConflict

private void checkNamespacesAndTableCfsConfigConflict(Set<String> namespaces,
                                                      Map<TableName,? extends Collection<String>> tableCfs)
                                               throws DoNotRetryIOException

Set a namespace in the peer config means that all tables in this namespace will be replicated to the peer cluster.

1. If peer config already has a namespace, then not allow set any table of this namespace to the peer config.
2. If peer config already has a table, then not allow set this table's namespace to the peer config.

Set a exclude namespace in the peer config means that all tables in this namespace can't be replicated to the peer cluster.

1. If peer config already has a exclude namespace, then not allow set any exclude table of this namespace to the peer config.
2. If peer config already has a exclude table, then not allow set this table's namespace as a exclude namespace.

Throws:

DoNotRetryIOException

checkConfiguredWALEntryFilters

private void checkConfiguredWALEntryFilters(ReplicationPeerConfig peerConfig)
                                     throws DoNotRetryIOException

Throws:

DoNotRetryIOException

checkClusterKey

private void checkClusterKey(String clusterKey)
                      throws DoNotRetryIOException

Throws:

DoNotRetryIOException

checkSameClusterKey

private void checkSameClusterKey(String clusterKey)
                          throws DoNotRetryIOException

Throws:

DoNotRetryIOException

getSerialPeerIdsBelongsTo

public List<String> getSerialPeerIdsBelongsTo(TableName tableName)

getPeerStorage

public ReplicationPeerStorage getPeerStorage()

getQueueStorage

public ReplicationQueueStorage getQueueStorage()

createReplicationQueueStorage

private static Pair<ReplicationQueueStorage,ReplicationPeerManager.ReplicationQueueStorageInitializer> createReplicationQueueStorage(MasterServices services)
                                                                                                                              throws IOException

Throws:

IOException

create

public static ReplicationPeerManager create(MasterServices services,
                                            String clusterId)
                                     throws ReplicationException,
                                            IOException

Throws:

ReplicationException

IOException

isStringEquals

private boolean isStringEquals(String s1,
                               String s2)

For replication peer cluster key or endpoint class, null and empty string is same. So here don't use StringUtils.equals(CharSequence, CharSequence) directly.

tryAcquireSyncReplicationPeerLock

public boolean tryAcquireSyncReplicationPeerLock()

releaseSyncReplicationPeerLock

public void releaseSyncReplicationPeerLock()

getReplicationLogCleanerBarrier

public ReplicationLogCleanerBarrier getReplicationLogCleanerBarrier()

onConfigurationChange

public void onConfigurationChange(org.apache.hadoop.conf.Configuration conf)

Description copied from interface: ConfigurationObserver

This method would be called by the ConfigurationManager object when the Configuration object is reloaded from disk.

Specified by:

onConfigurationChange in interface ConfigurationObserver

convert

private ReplicationQueueData convert(ZKReplicationQueueStorageForMigration.ZkReplicationQueueData zkData)

migrateQueues

private void migrateQueues(ZKReplicationQueueStorageForMigration oldQueueStorage)
                    throws Exception

Throws:

Exception

migrateLastPushedSeqIds

private void migrateLastPushedSeqIds(ZKReplicationQueueStorageForMigration oldQueueStorage)
                              throws Exception

Throws:

Exception

migrateHFileRefs

private void migrateHFileRefs(ZKReplicationQueueStorageForMigration oldQueueStorage)
                       throws Exception

Throws:

Exception

runAsync

private CompletableFuture<?> runAsync(ReplicationPeerManager.ExceptionalRunnable task,
                                      ExecutorService executor)

migrateQueuesFromZk

CompletableFuture<?> migrateQueuesFromZk(ZKWatcher zookeeper,
                                         ExecutorService executor)

Submit the migration tasks to the given executor.