@InterfaceAudience.Private public class FSHLog extends AbstractFSWAL<WALProvider.Writer>
Modifier and Type | Class and Description |
---|---|
(package private) static class |
FSHLog.SyncRequest |
private class |
FSHLog.SyncRunner
Thread to runs the hdfs sync call.
|
abortable, blocksize, closed, closeExecutor, conf, consumeExecutor, consumer, coprocessorHost, DEFAULT_ROLL_ON_SYNC_TIME_MS, DEFAULT_SLOW_SYNC_ROLL_INTERVAL_MS, DEFAULT_SLOW_SYNC_ROLL_THRESHOLD, DEFAULT_SLOW_SYNC_TIME_MS, DEFAULT_WAL_BATCH_SIZE, DEFAULT_WAL_SHUTDOWN_WAIT_TIMEOUT_MS, DEFAULT_WAL_SYNC_TIMEOUT_MS, filenum, fs, hasConsumerTask, highestProcessedAppendTxid, highestSyncedTxid, highestUnsyncedTxid, implClassName, inflightWALClosures, listeners, LOG_NAME_COMPARATOR, logrollsize, MAX_LOGS, maxLogs, numEntries, ourFiles, prefixPathStr, RING_BUFFER_SLOT_COUNT, ROLL_ON_SYNC_TIME_MS, rollOnSyncNs, rollRequested, rollWriterLock, sequenceIdAccounting, shouldShutDownConsumeExecutorWhenClose, shutdown, SLOW_SYNC_ROLL_INTERVAL_MS, SLOW_SYNC_ROLL_THRESHOLD, SLOW_SYNC_TIME_MS, slowSyncCheckInterval, slowSyncCount, slowSyncNs, slowSyncRollThreshold, syncFutureCache, syncFutures, totalLogSize, toWriteAppends, unackedAppends, useHsync, WAL_BATCH_SIZE, WAL_ROLL_MULTIPLIER, WAL_SHUTDOWN_WAIT_TIMEOUT_MS, WAL_SYNC_TIMEOUT_MS, walArchiveDir, walDir, walFile2Props, walFilePrefix, walFileSuffix, walShutdownTimeout, writer
Constructor and Description |
---|
FSHLog(org.apache.hadoop.fs.FileSystem fs,
Abortable abortable,
org.apache.hadoop.fs.Path root,
String logDir,
org.apache.hadoop.conf.Configuration conf) |
FSHLog(org.apache.hadoop.fs.FileSystem fs,
Abortable abortable,
org.apache.hadoop.fs.Path rootDir,
String logDir,
String archiveDir,
org.apache.hadoop.conf.Configuration conf,
List<WALActionsListener> listeners,
boolean failIfWALExists,
String prefix,
String suffix,
org.apache.hadoop.fs.FileSystem remoteFs,
org.apache.hadoop.fs.Path remoteWALDir)
Create an edit log at the given
dir location. |
FSHLog(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path root,
String logDir,
org.apache.hadoop.conf.Configuration conf)
Constructor.
|
FSHLog(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path rootDir,
String logDir,
String archiveDir,
org.apache.hadoop.conf.Configuration conf,
List<WALActionsListener> listeners,
boolean failIfWALExists,
String prefix,
String suffix) |
Modifier and Type | Method and Description |
---|---|
protected void |
checkSlowSyncCount() |
protected WALProvider.Writer |
createCombinedWriter(WALProvider.Writer localWriter,
WALProvider.Writer remoteWriter) |
private void |
createSyncRunnersAndStart() |
protected WALProvider.Writer |
createWriterInstance(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path path)
This method allows subclasses to inject different writers without having to extend other
methods like rollWriter().
|
protected void |
doAppend(WALProvider.Writer writer,
FSWALEntry entry) |
protected boolean |
doCheckLogLowReplication()
Returns true if number of replicas for the WAL is lower than threshold
|
protected void |
doCleanUpResources() |
protected CompletableFuture<Long> |
doWriterSync(WALProvider.Writer writer,
boolean shouldUseHSync,
long txidWhenSync) |
(package private) int |
getLogReplication()
This method gets the datanode replication count for the current WAL.
|
(package private) OutputStream |
getOutputStream()
Currently, we need to expose the writer's OutputStream to tests so that they can manipulate the
default behavior (such as setting the maxRecoveryErrorCount value).
|
(package private) org.apache.hadoop.hdfs.protocol.DatanodeInfo[] |
getPipeline()
This method gets the pipeline for the current WAL.
|
(package private) WALProvider.Writer |
getWriter() |
void |
init()
Used to initialize the WAL.
|
(package private) boolean |
isLowReplicationRollEnabled() |
private void |
offerSyncRequest(FSHLog.SyncRequest syncRequest) |
protected void |
onWriterReplaced(WALProvider.Writer nextWriter) |
private void |
preemptiveSync(ProtobufLogWriter nextWriter)
Run a sync after opening to set up the pipeline.
|
(package private) void |
setWriter(WALProvider.Writer writer) |
private void |
shutDownSyncRunners() |
abortCacheFlush, append, appendData, appendEntry, appendMarker, archive, archiveLogFile, atHeadOfRingBufferEventHandlerAppend, blockOnSync, checkLogLowReplication, close, closeWriter, completeCacheFlush, computeFilename, createSingleThreadPoolConsumeExecutor, doCheckSlowSync, doReplaceWriter, doShutdown, doSync, doSync, findRegionsToForceFlush, getCoprocessorHost, getCurrentFileName, getEarliestMemStoreSeqNum, getEarliestMemStoreSeqNum, getFilenum, getFileNumFromFileName, getFiles, getInflightWALCloseCount, getLogFileSize, getLogFileSizeIfBeingWritten, getNumLogFiles, getNumRolledLogFiles, getOldPath, getPreallocatedEventCount, getSyncedTxid, getSyncFuture, getUnflushedEntriesCount, getWALArchivePath, isLogRollRequested, isUnflushedEntries, isWriterBroken, logRollAndSetupWalProps, main, markFutureDoneAndOffer, postSync, registerWALActionsListener, replaceWriter, requestLogRoll, requestLogRoll, rollWriter, rollWriter, setWaitOnShutdownInSeconds, shutdown, skipRemoteWAL, stampSequenceIdAndPublishToRingBuffer, startCacheFlush, startCacheFlush, sync, sync, sync, sync, toString, unregisterWALActionsListener, updateStore, waitForSafePoint
private static final org.slf4j.Logger LOG
private static final String TOLERABLE_LOW_REPLICATION
private static final String LOW_REPLICATION_ROLL_LIMIT
private static final int DEFAULT_LOW_REPLICATION_ROLL_LIMIT
private static final String SYNCER_COUNT
private static final int DEFAULT_SYNCER_COUNT
private static final String MAX_BATCH_COUNT
private static final int DEFAULT_MAX_BATCH_COUNT
private static final String FSHLOG_WAIT_ON_SHUTDOWN_IN_SECONDS
private static final int DEFAULT_FSHLOG_WAIT_ON_SHUTDOWN_IN_SECONDS
private static final IOException WITER_REPLACED_EXCEPTION
private static final IOException WITER_BROKEN_EXCEPTION
private static final IOException WAL_CLOSE_EXCEPTION
private org.apache.hadoop.fs.FSDataOutputStream hdfs_out
private final int minTolerableReplication
private final AtomicInteger consecutiveLogRolls
private final int lowReplicationRollLimit
private volatile boolean lowReplicationRollEnabled
private final int syncerCount
private final int maxSyncRequestCount
private int syncRunnerIndex
private FSHLog.SyncRunner[] syncRunners
public static final long FIXED_OVERHEAD
public FSHLog(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path root, String logDir, org.apache.hadoop.conf.Configuration conf) throws IOException
fs
- filesystem handleroot
- path for stored and archived walslogDir
- dir where wals are storedconf
- configuration to useIOException
public FSHLog(org.apache.hadoop.fs.FileSystem fs, Abortable abortable, org.apache.hadoop.fs.Path root, String logDir, org.apache.hadoop.conf.Configuration conf) throws IOException
IOException
public FSHLog(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path rootDir, String logDir, String archiveDir, org.apache.hadoop.conf.Configuration conf, List<WALActionsListener> listeners, boolean failIfWALExists, String prefix, String suffix) throws IOException
IOException
public FSHLog(org.apache.hadoop.fs.FileSystem fs, Abortable abortable, org.apache.hadoop.fs.Path rootDir, String logDir, String archiveDir, org.apache.hadoop.conf.Configuration conf, List<WALActionsListener> listeners, boolean failIfWALExists, String prefix, String suffix, org.apache.hadoop.fs.FileSystem remoteFs, org.apache.hadoop.fs.Path remoteWALDir) throws IOException
dir
location. You should never have to load an
existing log. If there is a log at startup, it should have already been processed and deleted
by the time the WAL object is started up.fs
- filesystem handleabortable
- Abortable - the server hererootDir
- path to where logs and oldlogslogDir
- dir where wals are storedarchiveDir
- dir where wals are archivedconf
- configuration to uselisteners
- Listeners on WAL events. Listeners passed here will be registered before
we do anything else; e.g. the Constructor AbstractFSWAL.rollWriter()
.failIfWALExists
- If true IOException will be thrown if files related to this wal already
exist.prefix
- should always be hostname and port in distributed env and it will be URL
encoded before being used. If prefix is null, "wal" will be usedsuffix
- will be url encoded. null is treated as empty. non-empty must start with
AbstractFSWALProvider.WAL_FILE_NAME_DELIMITER
IOException
public void init() throws IOException
AbstractFSWAL
init
in interface WAL
init
in class AbstractFSWAL<WALProvider.Writer>
IOException
private void createSyncRunnersAndStart()
OutputStream getOutputStream()
private void preemptiveSync(ProtobufLogWriter nextWriter)
protected WALProvider.Writer createWriterInstance(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path path) throws IOException
createWriterInstance
in class AbstractFSWAL<WALProvider.Writer>
IOException
protected void doAppend(WALProvider.Writer writer, FSWALEntry entry) throws IOException
doAppend
in class AbstractFSWAL<WALProvider.Writer>
IOException
protected void onWriterReplaced(WALProvider.Writer nextWriter)
onWriterReplaced
in class AbstractFSWAL<WALProvider.Writer>
protected void doCleanUpResources()
doCleanUpResources
in class AbstractFSWAL<WALProvider.Writer>
private void shutDownSyncRunners()
protected CompletableFuture<Long> doWriterSync(WALProvider.Writer writer, boolean shouldUseHSync, long txidWhenSync)
doWriterSync
in class AbstractFSWAL<WALProvider.Writer>
private void offerSyncRequest(FSHLog.SyncRequest syncRequest)
protected void checkSlowSyncCount()
checkSlowSyncCount
in class AbstractFSWAL<WALProvider.Writer>
protected boolean doCheckLogLowReplication()
doCheckLogLowReplication
in class AbstractFSWAL<WALProvider.Writer>
int getLogReplication()
If the pipeline isn't started yet or is empty, you will get the default replication factor. Therefore, if this function returns 0, it means you are not properly running with the HDFS-826 patch.
getLogReplication
in class AbstractFSWAL<WALProvider.Writer>
boolean isLowReplicationRollEnabled()
org.apache.hadoop.hdfs.protocol.DatanodeInfo[] getPipeline()
getPipeline
in class AbstractFSWAL<WALProvider.Writer>
WALProvider.Writer getWriter()
void setWriter(WALProvider.Writer writer)
protected WALProvider.Writer createCombinedWriter(WALProvider.Writer localWriter, WALProvider.Writer remoteWriter)
createCombinedWriter
in class AbstractFSWAL<WALProvider.Writer>
Copyright © 2007–2020 The Apache Software Foundation. All rights reserved.