org.apache.hadoop.hbase.replication.regionserver

Class HFileReplicator

java.lang.Object

org.apache.hadoop.hbase.replication.regionserver.HFileReplicator

All Implemented Interfaces:

Closeable, AutoCloseable

@InterfaceAudience.Private
public class HFileReplicator
extends Object
implements Closeable

It is used for replicating HFile entries. It will first copy parallely all the hfiles to a local staging directory and then it will use (LoadIncrementalHFiles to prepare a collection of LoadIncrementalHFiles.LoadQueueItem which will finally be loaded(replicated) into the table of this cluster. Call close() when done.

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
private class	HFileReplicator.Copier This class will copy the given hfiles from the given source file system to the given local file system staging directory.

Field Summary

Fields

Modifier and Type	Field and Description
private Map<String,List<Pair<byte[],List<String>>>>	bulkLoadHFileMap
private org.apache.hadoop.conf.Configuration	conf
private Connection	connection
private int	copiesPerThread
private ThreadPoolExecutor	exec
private FsDelegationToken	fsDelegationToken
private org.apache.hadoop.fs.Path	hbaseStagingDir
private static org.slf4j.Logger	LOG
private int	maxCopyThreads
private static org.apache.hadoop.fs.permission.FsPermission	PERM_ALL_ACCESS
static int	REPLICATION_BULKLOAD_COPY_HFILES_PERTHREAD_DEFAULT
static String	REPLICATION_BULKLOAD_COPY_HFILES_PERTHREAD_KEY Number of hfiles to copy per thread during replication
static int	REPLICATION_BULKLOAD_COPY_MAXTHREADS_DEFAULT
static String	REPLICATION_BULKLOAD_COPY_MAXTHREADS_KEY Maximum number of threads to allow in pool to copy hfiles during replication
private org.apache.hadoop.fs.FileSystem	sinkFs
private String	sourceBaseNamespaceDirPath
private org.apache.hadoop.conf.Configuration	sourceClusterConf
private List<String>	sourceClusterIds
private String	sourceHFileArchiveDirPath
private static String	UNDERSCORE
private UserProvider	userProvider

Constructor Summary

Constructors

Constructor and Description
HFileReplicator(org.apache.hadoop.conf.Configuration sourceClusterConf, String sourceBaseNamespaceDirPath, String sourceHFileArchiveDirPath, Map<String,List<Pair<byte[],List<String>>>> tableQueueMap, org.apache.hadoop.conf.Configuration conf, Connection connection, List<String> sourceClusterIds)

Method Summary

Modifier and Type	Method and Description
private void	cleanup(String stagingDir, Table table)
void	close()
private Map<String,org.apache.hadoop.fs.Path>	copyHFilesToStagingDir()
private org.apache.hadoop.fs.Path	createStagingDir(org.apache.hadoop.fs.Path baseDir, User user, String randomDir)
private org.apache.hadoop.fs.Path	createStagingDir(org.apache.hadoop.fs.Path baseDir, User user, TableName tableName)
private void	doBulkLoad(LoadIncrementalHFiles loadHFiles, Table table, Deque<LoadIncrementalHFiles.LoadQueueItem> queue, RegionLocator locator, int maxRetries)
Void	replicate()

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

REPLICATION_BULKLOAD_COPY_MAXTHREADS_KEY

public static final String REPLICATION_BULKLOAD_COPY_MAXTHREADS_KEY

Maximum number of threads to allow in pool to copy hfiles during replication

See Also:

Constant Field Values

REPLICATION_BULKLOAD_COPY_MAXTHREADS_DEFAULT

public static final int REPLICATION_BULKLOAD_COPY_MAXTHREADS_DEFAULT

See Also:

Constant Field Values

REPLICATION_BULKLOAD_COPY_HFILES_PERTHREAD_KEY

public static final String REPLICATION_BULKLOAD_COPY_HFILES_PERTHREAD_KEY

Number of hfiles to copy per thread during replication

See Also:

Constant Field Values

REPLICATION_BULKLOAD_COPY_HFILES_PERTHREAD_DEFAULT

public static final int REPLICATION_BULKLOAD_COPY_HFILES_PERTHREAD_DEFAULT

See Also:

Constant Field Values

LOG

private static final org.slf4j.Logger LOG

UNDERSCORE

private static final String UNDERSCORE

See Also:

Constant Field Values

PERM_ALL_ACCESS

private static final org.apache.hadoop.fs.permission.FsPermission PERM_ALL_ACCESS

sourceClusterConf

private org.apache.hadoop.conf.Configuration sourceClusterConf

sourceBaseNamespaceDirPath

private String sourceBaseNamespaceDirPath

sourceHFileArchiveDirPath

private String sourceHFileArchiveDirPath

bulkLoadHFileMap

private Map<String,List<Pair<byte[],List<String>>>> bulkLoadHFileMap

sinkFs

private org.apache.hadoop.fs.FileSystem sinkFs

fsDelegationToken

private FsDelegationToken fsDelegationToken

userProvider

private UserProvider userProvider

conf

private org.apache.hadoop.conf.Configuration conf

connection

private Connection connection

hbaseStagingDir

private org.apache.hadoop.fs.Path hbaseStagingDir

exec

private ThreadPoolExecutor exec

maxCopyThreads

private int maxCopyThreads

copiesPerThread

private int copiesPerThread

sourceClusterIds

private List<String> sourceClusterIds

Constructor Detail

HFileReplicator

public HFileReplicator(org.apache.hadoop.conf.Configuration sourceClusterConf,
                       String sourceBaseNamespaceDirPath,
                       String sourceHFileArchiveDirPath,
                       Map<String,List<Pair<byte[],List<String>>>> tableQueueMap,
                       org.apache.hadoop.conf.Configuration conf,
                       Connection connection,
                       List<String> sourceClusterIds)
                throws IOException

Throws:

IOException

Method Detail

close

public void close()
           throws IOException

Specified by:

close in interface Closeable

Specified by:

close in interface AutoCloseable

Throws:

IOException

replicate

public Void replicate()
               throws IOException

Throws:

IOException

doBulkLoad

private void doBulkLoad(LoadIncrementalHFiles loadHFiles,
                        Table table,
                        Deque<LoadIncrementalHFiles.LoadQueueItem> queue,
                        RegionLocator locator,
                        int maxRetries)
                 throws IOException

Throws:

IOException

cleanup

private void cleanup(String stagingDir,
                     Table table)

copyHFilesToStagingDir

private Map<String,org.apache.hadoop.fs.Path> copyHFilesToStagingDir()
                                                              throws IOException

Throws:

IOException

createStagingDir

private org.apache.hadoop.fs.Path createStagingDir(org.apache.hadoop.fs.Path baseDir,
                                                   User user,
                                                   TableName tableName)
                                            throws IOException

Throws:

IOException

createStagingDir

private org.apache.hadoop.fs.Path createStagingDir(org.apache.hadoop.fs.Path baseDir,
                                                   User user,
                                                   String randomDir)
                                            throws IOException

Throws:

IOException