@InterfaceAudience.Private public class WALSplitter extends Object
splitWAL(FileStatus, CancelableProgressable)
per file or use static helper methods.Modifier and Type | Class and Description |
---|---|
(package private) static class |
WALSplitter.CorruptedLogFileException |
static class |
WALSplitter.PipelineController
Contains some methods to control WAL-entries producer / consumer interactions
|
(package private) static class |
WALSplitter.SplitWALResult
Data structure returned as result by #splitWAL(FileStatus, CancelableProgressable).
|
Modifier and Type | Field and Description |
---|---|
private long |
bufferSize |
protected org.apache.hadoop.conf.Configuration |
conf |
static boolean |
DEFAULT_WAL_SPLIT_TO_HFILE |
private EntryBuffers |
entryBuffers |
private org.apache.hadoop.fs.FileStatus |
fileBeingSplit |
private boolean |
hfile |
protected Map<String,Long> |
lastFlushedSequenceIds |
private static org.slf4j.Logger |
LOG |
private int |
numWriterThreads |
(package private) OutputSink |
outputSink |
protected Map<String,Map<byte[],Long>> |
regionMaxSeqIdInStores |
(package private) org.apache.hadoop.fs.Path |
rootDir |
(package private) org.apache.hadoop.fs.FileSystem |
rootFS |
(package private) RegionServerServices |
rsServices |
protected LastSequenceId |
sequenceIdChecker |
private boolean |
skipErrors |
static boolean |
SPLIT_SKIP_ERRORS_DEFAULT
By default we retry errors in splitting, rather than skipping.
|
static String |
SPLIT_SKIP_ERRORS_KEY |
static String |
SPLIT_WAL_BUFFER_SIZE |
static String |
SPLIT_WAL_WRITER_THREADS |
static String |
SPLIT_WRITER_CREATION_BOUNDED
True if we are to run with bounded amount of writers rather than let the count blossom.
|
private SplitLogWorkerCoordination |
splitLogWorkerCoordination
Coordinator for split log.
|
private boolean |
splitWriterCreationBounded |
private String |
tmpDirName |
static String |
WAL_SPLIT_TO_HFILE
Split WAL directly to hfiles instead of into intermediary 'recovered.edits' files.
|
private WALFactory |
walFactory |
protected org.apache.hadoop.fs.FileSystem |
walFS |
protected org.apache.hadoop.fs.Path |
walRootDir |
Constructor and Description |
---|
WALSplitter(WALFactory factory,
org.apache.hadoop.conf.Configuration conf,
org.apache.hadoop.fs.Path walRootDir,
org.apache.hadoop.fs.FileSystem walFS,
org.apache.hadoop.fs.Path rootDir,
org.apache.hadoop.fs.FileSystem rootFS) |
WALSplitter(WALFactory factory,
org.apache.hadoop.conf.Configuration conf,
org.apache.hadoop.fs.Path walRootDir,
org.apache.hadoop.fs.FileSystem walFS,
org.apache.hadoop.fs.Path rootDir,
org.apache.hadoop.fs.FileSystem rootFS,
LastSequenceId idChecker,
SplitLogWorkerCoordination splitLogWorkerCoordination,
RegionServerServices rsServices) |
Modifier and Type | Method and Description |
---|---|
private void |
createOutputSinkAndEntryBuffers()
Setup the output sinks and entry buffers ahead of splitting WAL.
|
protected WALProvider.Writer |
createWriter(org.apache.hadoop.fs.Path logfile)
Create a new
WALProvider.Writer for writing log splits. |
(package private) org.apache.hadoop.fs.FileStatus |
getFileBeingSplit() |
private WAL.Entry |
getNextLogLine(WAL.Reader in,
org.apache.hadoop.fs.Path path,
boolean skipErrors) |
private int |
getNumOpenWriters()
Get current open writers
|
protected WAL.Reader |
getReader(org.apache.hadoop.fs.FileStatus walStatus,
boolean skipErrors,
CancelableProgressable cancel)
Create a new
WAL.Reader for reading logs to split. |
private WAL.Reader |
getReader(org.apache.hadoop.fs.Path curLogFile,
CancelableProgressable reporter)
Create a new
WAL.Reader for reading logs to split. |
(package private) Map<String,Map<byte[],Long>> |
getRegionMaxSeqIdInStores() |
(package private) String |
getTmpDirName() |
(package private) WALFactory |
getWalFactory() |
private boolean |
isRegionDirPresentUnderRoot(TableName tn,
String region) |
static List<org.apache.hadoop.fs.Path> |
split(org.apache.hadoop.fs.Path walRootDir,
org.apache.hadoop.fs.Path walsDir,
org.apache.hadoop.fs.Path archiveDir,
org.apache.hadoop.fs.FileSystem walFS,
org.apache.hadoop.conf.Configuration conf,
WALFactory factory)
Split a folder of WAL files.
|
static boolean |
splitLogFile(org.apache.hadoop.fs.Path walDir,
org.apache.hadoop.fs.FileStatus logfile,
org.apache.hadoop.fs.FileSystem walFS,
org.apache.hadoop.conf.Configuration conf,
CancelableProgressable reporter,
LastSequenceId idChecker,
SplitLogWorkerCoordination splitLogWorkerCoordination,
WALFactory factory,
RegionServerServices rsServices)
Splits a WAL file.
|
(package private) WALSplitter.SplitWALResult |
splitWAL(org.apache.hadoop.fs.FileStatus walStatus,
CancelableProgressable cancel)
WAL splitting implementation, splits one WAL file.
|
private static final org.slf4j.Logger LOG
public static final String SPLIT_SKIP_ERRORS_KEY
public static final boolean SPLIT_SKIP_ERRORS_DEFAULT
protected final org.apache.hadoop.fs.Path walRootDir
protected final org.apache.hadoop.fs.FileSystem walFS
protected final org.apache.hadoop.conf.Configuration conf
final org.apache.hadoop.fs.Path rootDir
final org.apache.hadoop.fs.FileSystem rootFS
final RegionServerServices rsServices
OutputSink outputSink
private EntryBuffers entryBuffers
private SplitLogWorkerCoordination splitLogWorkerCoordination
private final WALFactory walFactory
protected final LastSequenceId sequenceIdChecker
protected Map<String,Long> lastFlushedSequenceIds
protected Map<String,Map<byte[],Long>> regionMaxSeqIdInStores
private org.apache.hadoop.fs.FileStatus fileBeingSplit
private final String tmpDirName
public static final String WAL_SPLIT_TO_HFILE
public static final boolean DEFAULT_WAL_SPLIT_TO_HFILE
public static final String SPLIT_WRITER_CREATION_BOUNDED
public static final String SPLIT_WAL_BUFFER_SIZE
public static final String SPLIT_WAL_WRITER_THREADS
private final int numWriterThreads
private final long bufferSize
private final boolean splitWriterCreationBounded
private final boolean hfile
private final boolean skipErrors
WALSplitter(WALFactory factory, org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.Path walRootDir, org.apache.hadoop.fs.FileSystem walFS, org.apache.hadoop.fs.Path rootDir, org.apache.hadoop.fs.FileSystem rootFS)
WALSplitter(WALFactory factory, org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.Path walRootDir, org.apache.hadoop.fs.FileSystem walFS, org.apache.hadoop.fs.Path rootDir, org.apache.hadoop.fs.FileSystem rootFS, LastSequenceId idChecker, SplitLogWorkerCoordination splitLogWorkerCoordination, RegionServerServices rsServices)
WALFactory getWalFactory()
org.apache.hadoop.fs.FileStatus getFileBeingSplit()
String getTmpDirName()
Map<String,Map<byte[],Long>> getRegionMaxSeqIdInStores()
public static boolean splitLogFile(org.apache.hadoop.fs.Path walDir, org.apache.hadoop.fs.FileStatus logfile, org.apache.hadoop.fs.FileSystem walFS, org.apache.hadoop.conf.Configuration conf, CancelableProgressable reporter, LastSequenceId idChecker, SplitLogWorkerCoordination splitLogWorkerCoordination, WALFactory factory, RegionServerServices rsServices) throws IOException
SplitLogWorker
and
tests. Not used by new procedure-based WAL splitter.IOException
public static List<org.apache.hadoop.fs.Path> split(org.apache.hadoop.fs.Path walRootDir, org.apache.hadoop.fs.Path walsDir, org.apache.hadoop.fs.Path archiveDir, org.apache.hadoop.fs.FileSystem walFS, org.apache.hadoop.conf.Configuration conf, WALFactory factory) throws IOException
IOException
private void createOutputSinkAndEntryBuffers()
WALSplitter.SplitWALResult splitWAL(org.apache.hadoop.fs.FileStatus walStatus, CancelableProgressable cancel) throws IOException
walStatus
- should be for an actual WAL file.IOException
private boolean isRegionDirPresentUnderRoot(TableName tn, String region) throws IOException
IOException
protected WAL.Reader getReader(org.apache.hadoop.fs.FileStatus walStatus, boolean skipErrors, CancelableProgressable cancel) throws IOException, WALSplitter.CorruptedLogFileException
WAL.Reader
for reading logs to split.IOException
WALSplitter.CorruptedLogFileException
private WAL.Entry getNextLogLine(WAL.Reader in, org.apache.hadoop.fs.Path path, boolean skipErrors) throws WALSplitter.CorruptedLogFileException, IOException
protected WALProvider.Writer createWriter(org.apache.hadoop.fs.Path logfile) throws IOException
WALProvider.Writer
for writing log splits.IOException
private WAL.Reader getReader(org.apache.hadoop.fs.Path curLogFile, CancelableProgressable reporter) throws IOException
WAL.Reader
for reading logs to split.IOException
private int getNumOpenWriters()
Copyright © 2007–2020 The Apache Software Foundation. All rights reserved.