@InterfaceAudience.LimitedPrivate(value="Configuration") public class AsyncFSWAL extends AbstractFSWAL<WALProvider.AsyncWriter>
Here 'waitingConsumePayloads' acts as the RingBuffer in FSHLog.
For append, we process it as follow:
AbstractFSWAL.shouldScheduleConsumer()
for more details.
AbstractFSWAL.waitingConsumePayloads
and insert it into
AbstractFSWAL.toWriteAppends
AbstractFSWAL.toWriteAppends
, append it to the AsyncWriter, and insert it into
AbstractFSWAL.unackedAppends
AbstractFSWAL.batchSize
, or there is a sync request, then we call
sync on the AsyncWriter.AbstractFSWAL.unackedAppends
and drop it.AbstractFSWAL.unackedAppends
back to AbstractFSWAL.toWriteAppends
and
wait for writing them again.
Here we only describe the logic of doReplaceWriter. The main logic of rollWriter is same with
FSHLog.
For a normal roll request(for example, we have reached the log roll size):
AbstractFSWAL.waitingRoll(int)
to true and
AbstractFSWAL.readyForRolling
to false, and then wait on AbstractFSWAL.readyForRolling
(see
AbstractFSWAL.waitForSafePoint()
).AbstractFSWAL.waitingConsumePayloads
if
AbstractFSWAL.waitingRoll(int)
is true, and also stop writing the entries in AbstractFSWAL.toWriteAppends
out.
AbstractFSWAL.unackedAppends
is empty,
signal the AbstractFSWAL.readyForRollingCond
.AbstractFSWAL.writerBroken(int)
and AbstractFSWAL.waitingRoll(int)
to false.Modifier and Type | Field and Description |
---|---|
static String |
ASYNC_WAL_USE_SHARED_EVENT_LOOP |
static String |
ASYNC_WAL_WAIT_ON_SHUTDOWN_IN_SECONDS |
private Class<? extends org.apache.hbase.thirdparty.io.netty.channel.Channel> |
channelClass |
static boolean |
DEFAULT_ASYNC_WAL_USE_SHARED_EVENT_LOOP |
static int |
DEFAULT_ASYNC_WAL_WAIT_ON_SHUTDOWN_IN_SECONDS |
static long |
DEFAULT_WAL_BATCH_SIZE |
private org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup |
eventLoopGroup |
private AsyncFSOutput |
fsOut |
private static org.slf4j.Logger |
LOG |
private StreamSlowMonitor |
streamSlowMonitor |
static String |
WAL_BATCH_SIZE |
abortable, blocksize, closed, closeExecutor, conf, consumeExecutor, consumer, coprocessorHost, DEFAULT_ROLL_ON_SYNC_TIME_MS, DEFAULT_SLOW_SYNC_ROLL_INTERVAL_MS, DEFAULT_SLOW_SYNC_ROLL_THRESHOLD, DEFAULT_SLOW_SYNC_TIME_MS, DEFAULT_WAL_SHUTDOWN_WAIT_TIMEOUT_MS, DEFAULT_WAL_SYNC_TIMEOUT_MS, filenum, fs, hasConsumerTask, highestProcessedAppendTxid, highestSyncedTxid, highestUnsyncedTxid, implClassName, inflightWALClosures, listeners, LOG_NAME_COMPARATOR, logrollsize, MAX_LOGS, maxLogs, numEntries, ourFiles, prefixPathStr, RING_BUFFER_SLOT_COUNT, ROLL_ON_SYNC_TIME_MS, rollOnSyncNs, rollRequested, rollWriterLock, sequenceIdAccounting, shouldShutDownConsumeExecutorWhenClose, shutdown, SLOW_SYNC_ROLL_INTERVAL_MS, SLOW_SYNC_ROLL_THRESHOLD, SLOW_SYNC_TIME_MS, slowSyncCheckInterval, slowSyncCount, slowSyncNs, slowSyncRollThreshold, syncFutureCache, syncFutures, totalLogSize, toWriteAppends, unackedAppends, useHsync, WAL_AVOID_LOCAL_WRITES_DEFAULT, WAL_AVOID_LOCAL_WRITES_KEY, WAL_ROLL_MULTIPLIER, WAL_SHUTDOWN_WAIT_TIMEOUT_MS, WAL_SYNC_TIMEOUT_MS, walArchiveDir, walDir, walFile2Props, walFilePrefix, walFileSuffix, walShutdownTimeout, writer
Constructor and Description |
---|
AsyncFSWAL(org.apache.hadoop.fs.FileSystem fs,
Abortable abortable,
org.apache.hadoop.fs.Path rootDir,
String logDir,
String archiveDir,
org.apache.hadoop.conf.Configuration conf,
List<WALActionsListener> listeners,
boolean failIfWALExists,
String prefix,
String suffix,
org.apache.hadoop.fs.FileSystem remoteFs,
org.apache.hadoop.fs.Path remoteWALDir,
org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup eventLoopGroup,
Class<? extends org.apache.hbase.thirdparty.io.netty.channel.Channel> channelClass,
StreamSlowMonitor monitor) |
Modifier and Type | Method and Description |
---|---|
protected WALProvider.AsyncWriter |
createAsyncWriter(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path path) |
protected WALProvider.AsyncWriter |
createCombinedWriter(WALProvider.AsyncWriter localWriter,
WALProvider.AsyncWriter remoteWriter) |
protected WALProvider.AsyncWriter |
createWriterInstance(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path path) |
protected void |
doAppend(WALProvider.AsyncWriter writer,
FSWALEntry entry) |
protected boolean |
doCheckLogLowReplication() |
protected CompletableFuture<Long> |
doWriterSync(WALProvider.AsyncWriter writer,
boolean shouldUseHsync,
long txidWhenSyn) |
(package private) int |
getLogReplication()
This method gets the datanode replication count for the current WAL.
|
(package private) org.apache.hadoop.hdfs.protocol.DatanodeInfo[] |
getPipeline()
This method gets the pipeline for the current WAL.
|
protected void |
onWriterReplaced(WALProvider.AsyncWriter nextWriter) |
abortCacheFlush, append, appendData, appendEntry, appendMarker, archive, archiveLogFile, atHeadOfRingBufferEventHandlerAppend, blockOnSync, checkLogLowReplication, checkSlowSyncCount, close, closeWriter, completeCacheFlush, computeFilename, createSingleThreadPoolConsumeExecutor, doCheckSlowSync, doCleanUpResources, doReplaceWriter, doShutdown, doSync, doSync, findRegionsToForceFlush, getCoprocessorHost, getCurrentFileName, getEarliestMemStoreSeqNum, getEarliestMemStoreSeqNum, getFilenum, getFileNumFromFileName, getFiles, getInflightWALCloseCount, getLogFileSize, getLogFileSizeIfBeingWritten, getNumLogFiles, getNumRolledLogFiles, getOldPath, getPreallocatedEventCount, getSyncedTxid, getSyncFuture, getUnflushedEntriesCount, getWALArchivePath, getWriter, init, isLogRollRequested, isUnflushedEntries, isWriterBroken, logRollAndSetupWalProps, main, markFutureDoneAndOffer, postSync, registerWALActionsListener, replaceWriter, requestLogRoll, requestLogRoll, rollWriter, rollWriter, setWaitOnShutdownInSeconds, shutdown, skipRemoteWAL, stampSequenceIdAndPublishToRingBuffer, startCacheFlush, startCacheFlush, sync, sync, sync, sync, toString, unregisterWALActionsListener, updateStore, waitForSafePoint
private static final org.slf4j.Logger LOG
public static final String WAL_BATCH_SIZE
public static final long DEFAULT_WAL_BATCH_SIZE
public static final String ASYNC_WAL_USE_SHARED_EVENT_LOOP
public static final boolean DEFAULT_ASYNC_WAL_USE_SHARED_EVENT_LOOP
public static final String ASYNC_WAL_WAIT_ON_SHUTDOWN_IN_SECONDS
public static final int DEFAULT_ASYNC_WAL_WAIT_ON_SHUTDOWN_IN_SECONDS
private final org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup eventLoopGroup
private final Class<? extends org.apache.hbase.thirdparty.io.netty.channel.Channel> channelClass
private volatile AsyncFSOutput fsOut
private final StreamSlowMonitor streamSlowMonitor
public AsyncFSWAL(org.apache.hadoop.fs.FileSystem fs, Abortable abortable, org.apache.hadoop.fs.Path rootDir, String logDir, String archiveDir, org.apache.hadoop.conf.Configuration conf, List<WALActionsListener> listeners, boolean failIfWALExists, String prefix, String suffix, org.apache.hadoop.fs.FileSystem remoteFs, org.apache.hadoop.fs.Path remoteWALDir, org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup eventLoopGroup, Class<? extends org.apache.hbase.thirdparty.io.netty.channel.Channel> channelClass, StreamSlowMonitor monitor) throws FailedLogCloseException, IOException
FailedLogCloseException
IOException
protected CompletableFuture<Long> doWriterSync(WALProvider.AsyncWriter writer, boolean shouldUseHsync, long txidWhenSyn)
doWriterSync
in class AbstractFSWAL<WALProvider.AsyncWriter>
protected final WALProvider.AsyncWriter createAsyncWriter(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path path) throws IOException
IOException
protected WALProvider.AsyncWriter createWriterInstance(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path path) throws IOException
createWriterInstance
in class AbstractFSWAL<WALProvider.AsyncWriter>
IOException
protected void onWriterReplaced(WALProvider.AsyncWriter nextWriter)
onWriterReplaced
in class AbstractFSWAL<WALProvider.AsyncWriter>
protected void doAppend(WALProvider.AsyncWriter writer, FSWALEntry entry)
doAppend
in class AbstractFSWAL<WALProvider.AsyncWriter>
org.apache.hadoop.hdfs.protocol.DatanodeInfo[] getPipeline()
AbstractFSWAL
getPipeline
in class AbstractFSWAL<WALProvider.AsyncWriter>
int getLogReplication()
AbstractFSWAL
getLogReplication
in class AbstractFSWAL<WALProvider.AsyncWriter>
protected boolean doCheckLogLowReplication()
doCheckLogLowReplication
in class AbstractFSWAL<WALProvider.AsyncWriter>
protected WALProvider.AsyncWriter createCombinedWriter(WALProvider.AsyncWriter localWriter, WALProvider.AsyncWriter remoteWriter)
createCombinedWriter
in class AbstractFSWAL<WALProvider.AsyncWriter>
Copyright © 2007–2020 The Apache Software Foundation. All rights reserved.