@InterfaceAudience.Private @InterfaceStability.Evolving class ReplicationSourceWALReader extends Thread
Thread.State, Thread.UncaughtExceptionHandler
Modifier and Type | Field and Description |
---|---|
private org.apache.hadoop.conf.Configuration |
conf |
private long |
currentPosition |
(package private) BlockingQueue<WALEntryBatch> |
entryBatchQueue |
private WALEntryFilter |
filter |
private org.apache.hadoop.fs.FileSystem |
fs |
private boolean |
isReaderRunning |
private static org.slf4j.Logger |
LOG |
private ReplicationSourceLogQueue |
logQueue |
private int |
maxRetriesMultiplier |
private int |
replicationBatchCountCapacity |
private long |
replicationBatchSizeCapacity |
private long |
sleepForRetries |
private ReplicationSource |
source |
private String |
walGroupId |
MAX_PRIORITY, MIN_PRIORITY, NORM_PRIORITY
Constructor and Description |
---|
ReplicationSourceWALReader(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.conf.Configuration conf,
ReplicationSourceLogQueue logQueue,
long startPosition,
WALEntryFilter filter,
ReplicationSource source,
String walGroupId)
Creates a reader worker for a given WAL queue.
|
Modifier and Type | Method and Description |
---|---|
protected boolean |
addEntryToBatch(WALEntryBatch batch,
WAL.Entry entry) |
private boolean |
checkBufferQuota() |
private Pair<Integer,Integer> |
countDistinctRowKeysAndHFiles(WALEdit edit)
Count the number of different row keys in the given edit because of mini-batching.
|
private WALEntryBatch |
createBatch(WALEntryStream entryStream) |
protected WAL.Entry |
filterEntry(WAL.Entry entry) |
org.apache.hadoop.fs.Path |
getCurrentPath() |
private long |
getEntrySizeIncludeBulkLoad(WAL.Entry entry) |
private ReplicationSourceManager |
getSourceManager() |
boolean |
isReaderRunning()
Returns whether the reader thread is running
|
WALEntryBatch |
poll(long timeout) |
protected void |
readWALEntries(WALEntryStream entryStream,
WALEntryBatch batch) |
private void |
replicationDone() |
void |
run() |
void |
setReaderRunning(boolean readerRunning) |
private int |
sizeOfStoreFilesIncludeBulkLoad(WALEdit edit)
Calculate the total size of all the store files
|
protected int |
sleep(int sleepMultiplier) |
protected static boolean |
switched(WALEntryStream entryStream,
org.apache.hadoop.fs.Path path) |
WALEntryBatch |
take()
Retrieves the next batch of WAL entries from the queue, waiting up to the specified time for a
batch to become available
|
private void |
updateBatchStats(WALEntryBatch batch,
WAL.Entry entry,
long entrySize) |
private void |
updateReplicationMarkerEdit(WAL.Entry entry,
long offset) |
activeCount, checkAccess, clone, countStackFrames, currentThread, destroy, dumpStack, enumerate, getAllStackTraces, getContextClassLoader, getDefaultUncaughtExceptionHandler, getId, getName, getPriority, getStackTrace, getState, getThreadGroup, getUncaughtExceptionHandler, holdsLock, interrupt, interrupted, isAlive, isDaemon, isInterrupted, join, join, join, resume, setContextClassLoader, setDaemon, setDefaultUncaughtExceptionHandler, setName, setPriority, setUncaughtExceptionHandler, sleep, sleep, start, stop, stop, suspend, toString, yield
private static final org.slf4j.Logger LOG
private final ReplicationSourceLogQueue logQueue
private final org.apache.hadoop.fs.FileSystem fs
private final org.apache.hadoop.conf.Configuration conf
private final WALEntryFilter filter
private final ReplicationSource source
@InterfaceAudience.Private final BlockingQueue<WALEntryBatch> entryBatchQueue
private final long replicationBatchSizeCapacity
private final int replicationBatchCountCapacity
private long currentPosition
private final long sleepForRetries
private final int maxRetriesMultiplier
private boolean isReaderRunning
private final String walGroupId
public ReplicationSourceWALReader(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.conf.Configuration conf, ReplicationSourceLogQueue logQueue, long startPosition, WALEntryFilter filter, ReplicationSource source, String walGroupId)
fs
- the files system to useconf
- configuration to uselogQueue
- The WAL queue to read off ofstartPosition
- position in the first WAL to start reading fromfilter
- The filter to use while readingsource
- replication sourceprivate void replicationDone() throws InterruptedException
InterruptedException
protected final int sleep(int sleepMultiplier)
protected final boolean addEntryToBatch(WALEntryBatch batch, WAL.Entry entry)
protected static final boolean switched(WALEntryStream entryStream, org.apache.hadoop.fs.Path path)
protected void readWALEntries(WALEntryStream entryStream, WALEntryBatch batch) throws InterruptedException
InterruptedException
public org.apache.hadoop.fs.Path getCurrentPath()
private boolean checkBufferQuota()
private WALEntryBatch createBatch(WALEntryStream entryStream)
protected final WAL.Entry filterEntry(WAL.Entry entry)
public WALEntryBatch take() throws InterruptedException
InterruptedException
- if interrupted while waitingpublic WALEntryBatch poll(long timeout) throws InterruptedException
InterruptedException
private long getEntrySizeIncludeBulkLoad(WAL.Entry entry)
private void updateBatchStats(WALEntryBatch batch, WAL.Entry entry, long entrySize)
private Pair<Integer,Integer> countDistinctRowKeysAndHFiles(WALEdit edit)
edit
- edit to count row keys fromprivate int sizeOfStoreFilesIncludeBulkLoad(WALEdit edit)
edit
- edit to count row keys fromprivate void updateReplicationMarkerEdit(WAL.Entry entry, long offset)
public boolean isReaderRunning()
public void setReaderRunning(boolean readerRunning)
readerRunning
- the readerRunning to setprivate ReplicationSourceManager getSourceManager()
Copyright © 2007–2020 The Apache Software Foundation. All rights reserved.