org.apache.hadoop.hbase.replication.regionserver

Class ReplicationSourceWALReader

java.lang.Object

java.lang.Thread

org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader

All Implemented Interfaces:

Runnable

Direct Known Subclasses:

SerialReplicationSourceWALReader

@InterfaceAudience.Private
 @InterfaceStability.Evolving
class ReplicationSourceWALReader
extends Thread

Reads and filters WAL entries, groups the filtered entries into batches, and puts the batches onto a queue

Nested Class Summary

Nested classes/interfaces inherited from class java.lang.Thread

Thread.State, Thread.UncaughtExceptionHandler

Field Summary

Fields

Modifier and Type	Field and Description
private org.apache.hadoop.conf.Configuration	conf
private long	currentPosition
(package private) BlockingQueue<WALEntryBatch>	entryBatchQueue
private boolean	eofAutoRecovery
private WALEntryFilter	filter
private org.apache.hadoop.fs.FileSystem	fs
private boolean	isReaderRunning
private static org.slf4j.Logger	LOG
private ReplicationSourceLogQueue	logQueue
private int	maxRetriesMultiplier
private int	replicationBatchCountCapacity
private long	replicationBatchSizeCapacity
private long	sleepForRetries
private ReplicationSource	source
private long	totalBufferQuota
private AtomicLong	totalBufferUsed
private String	walGroupId

Fields inherited from class java.lang.Thread

MAX_PRIORITY, MIN_PRIORITY, NORM_PRIORITY

Constructor Summary

Constructors

Constructor and Description
ReplicationSourceWALReader(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.conf.Configuration conf, ReplicationSourceLogQueue logQueue, long startPosition, WALEntryFilter filter, ReplicationSource source, String walGroupId) Creates a reader worker for a given WAL queue.

Method Summary

Modifier and Type	Method and Description
private boolean	acquireBufferQuota(long size)
private void	addBatchToShippingQueue(WALEntryBatch batch) Update the batch try to ship and return true if shipped
protected boolean	addEntryToBatch(WALEntryBatch batch, WAL.Entry entry)
private boolean	checkQuota()
private Pair<Integer,Integer>	countDistinctRowKeysAndHFiles(WALEdit edit) Count the number of different row keys in the given edit because of mini-batching.
private WALEntryBatch	createBatch(WALEntryStream entryStream)
protected WAL.Entry	filterEntry(WAL.Entry entry)
org.apache.hadoop.fs.Path	getCurrentPath()
static long	getEntrySizeExcludeBulkLoad(WAL.Entry entry)
private long	getEntrySizeIncludeBulkLoad(WAL.Entry entry)
private void	handleEmptyWALEntryBatch()
private boolean	handleEofException(Exception e, WALEntryBatch batch) This is to handle the EOFException from the WAL entry stream.
boolean	isReaderRunning() Returns whether the reader thread is running
WALEntryBatch	poll(long timeout)
protected void	readWALEntries(WALEntryStream entryStream, WALEntryBatch batch)
void	run()
void	setReaderRunning(boolean readerRunning)
private int	sizeOfStoreFilesIncludeBulkLoad(WALEdit edit) Calculate the total size of all the store files
protected static boolean	switched(WALEntryStream entryStream, org.apache.hadoop.fs.Path path)
WALEntryBatch	take() Retrieves the next batch of WAL entries from the queue, waiting up to the specified time for a batch to become available
private WALEntryBatch	tryAdvanceStreamAndCreateWALBatch(WALEntryStream entryStream)
private void	updateBatchStats(WALEntryBatch batch, WAL.Entry entry, long entrySize)

Methods inherited from class java.lang.Thread

activeCount, checkAccess, clone, countStackFrames, currentThread, destroy, dumpStack, enumerate, getAllStackTraces, getContextClassLoader, getDefaultUncaughtExceptionHandler, getId, getName, getPriority, getStackTrace, getState, getThreadGroup, getUncaughtExceptionHandler, holdsLock, interrupt, interrupted, isAlive, isDaemon, isInterrupted, join, join, join, resume, setContextClassLoader, setDaemon, setDefaultUncaughtExceptionHandler, setName, setPriority, setUncaughtExceptionHandler, sleep, sleep, start, stop, stop, suspend, toString, yield

Methods inherited from class java.lang.Object

equals, finalize, getClass, hashCode, notify, notifyAll, wait, wait, wait

Field Detail

LOG

private static final org.slf4j.Logger LOG

logQueue

private final ReplicationSourceLogQueue logQueue

fs

private final org.apache.hadoop.fs.FileSystem fs

conf

private final org.apache.hadoop.conf.Configuration conf

filter

private final WALEntryFilter filter

source

private final ReplicationSource source

entryBatchQueue

@InterfaceAudience.Private
final BlockingQueue<WALEntryBatch> entryBatchQueue

replicationBatchSizeCapacity

private final long replicationBatchSizeCapacity

replicationBatchCountCapacity

private final int replicationBatchCountCapacity

currentPosition

private long currentPosition

sleepForRetries

private final long sleepForRetries

maxRetriesMultiplier

private final int maxRetriesMultiplier

eofAutoRecovery

private final boolean eofAutoRecovery

isReaderRunning

private boolean isReaderRunning

totalBufferUsed

private AtomicLong totalBufferUsed

totalBufferQuota

private long totalBufferQuota

walGroupId

private final String walGroupId

Constructor Detail

ReplicationSourceWALReader

public ReplicationSourceWALReader(org.apache.hadoop.fs.FileSystem fs,
                                  org.apache.hadoop.conf.Configuration conf,
                                  ReplicationSourceLogQueue logQueue,
                                  long startPosition,
                                  WALEntryFilter filter,
                                  ReplicationSource source,
                                  String walGroupId)

Creates a reader worker for a given WAL queue. Reads WAL entries off a given queue, batches the entries, and puts them on a batch queue.

Parameters:

fs - the files system to use

conf - configuration to use

logQueue - The WAL queue to read off of

startPosition - position in the first WAL to start reading from

filter - The filter to use while reading

source - replication source

Method Detail

run

public void run()

Specified by:

run in interface Runnable

Overrides:

run in class Thread

addEntryToBatch

protected final boolean addEntryToBatch(WALEntryBatch batch,
                                        WAL.Entry entry)

switched

protected static final boolean switched(WALEntryStream entryStream,
                                        org.apache.hadoop.fs.Path path)

readWALEntries

protected void readWALEntries(WALEntryStream entryStream,
                              WALEntryBatch batch)
                       throws IOException,
                              InterruptedException

Throws:

IOException

InterruptedException

handleEmptyWALEntryBatch

private void handleEmptyWALEntryBatch()
                               throws InterruptedException

Throws:

InterruptedException

tryAdvanceStreamAndCreateWALBatch

private WALEntryBatch tryAdvanceStreamAndCreateWALBatch(WALEntryStream entryStream)
                                                 throws IOException

Throws:

IOException

handleEofException

private boolean handleEofException(Exception e,
                                   WALEntryBatch batch)

This is to handle the EOFException from the WAL entry stream. EOFException should be handled carefully because there are chances of data loss because of never replicating the data. Thus we should always try to ship existing batch of entries here. If there was only one log in the queue before EOF, we ship the empty batch here and since reader is still active, in the next iteration of reader we will stop the reader.

If there was more than one log in the queue before EOF, we ship the existing batch and reset the wal patch and position to the log with EOF, so shipper can remove logs from replication queue

Returns:

true only the IOE can be handled

addBatchToShippingQueue

private void addBatchToShippingQueue(WALEntryBatch batch)
                              throws InterruptedException

Update the batch try to ship and return true if shipped

Parameters:

batch - Batch of entries to ship

Throws:

InterruptedException - throws interrupted exception

getCurrentPath

public org.apache.hadoop.fs.Path getCurrentPath()

checkQuota

private boolean checkQuota()

createBatch

private WALEntryBatch createBatch(WALEntryStream entryStream)

filterEntry

protected final WAL.Entry filterEntry(WAL.Entry entry)

take

public WALEntryBatch take()
                   throws InterruptedException

Retrieves the next batch of WAL entries from the queue, waiting up to the specified time for a batch to become available

Returns:

A batch of entries, along with the position in the log after reading the batch

Throws:

InterruptedException - if interrupted while waiting

poll

public WALEntryBatch poll(long timeout)
                   throws InterruptedException

Throws:

InterruptedException

getEntrySizeIncludeBulkLoad

private long getEntrySizeIncludeBulkLoad(WAL.Entry entry)

getEntrySizeExcludeBulkLoad

public static long getEntrySizeExcludeBulkLoad(WAL.Entry entry)

updateBatchStats

private void updateBatchStats(WALEntryBatch batch,
                              WAL.Entry entry,
                              long entrySize)

countDistinctRowKeysAndHFiles

private Pair<Integer,Integer> countDistinctRowKeysAndHFiles(WALEdit edit)

Count the number of different row keys in the given edit because of mini-batching. We assume that there's at least one Cell in the WALEdit.

Parameters:

edit - edit to count row keys from

Returns:

number of different row keys and HFiles

sizeOfStoreFilesIncludeBulkLoad

private int sizeOfStoreFilesIncludeBulkLoad(WALEdit edit)

Calculate the total size of all the store files

Parameters:

edit - edit to count row keys from

Returns:

the total size of the store files

acquireBufferQuota

private boolean acquireBufferQuota(long size)

Parameters:

size - delta size for grown buffer

Returns:

true if we should clear buffer and push all

isReaderRunning

public boolean isReaderRunning()

Returns whether the reader thread is running

setReaderRunning

public void setReaderRunning(boolean readerRunning)

Parameters:

readerRunning - the readerRunning to set