@InterfaceAudience.Private public abstract class Compactor<T extends CellSink> extends Object
Compactions might be concurrent against a given store and the Compactor is shared among them. Do not put mutable state into class fields. All Compactor class fields should be final or effectively final. 'keepSeqIdPeriod' is an exception to this rule because unit tests may set it.
Modifier and Type | Class and Description |
---|---|
protected static interface |
Compactor.CellSinkFactory<S> |
protected static class |
Compactor.FileDetails
The sole reason this class exists is that java has no ref/out/pointer parameters.
|
protected static interface |
Compactor.InternalScannerFactory |
Modifier and Type | Field and Description |
---|---|
protected static long |
COMPACTION_PROGRESS_LOG_INTERVAL |
protected int |
compactionKVMax |
protected org.apache.hadoop.conf.Configuration |
conf |
protected Compactor.InternalScannerFactory |
defaultScannerFactory |
protected boolean |
dropCacheMajor |
protected boolean |
dropCacheMinor |
protected int |
keepSeqIdPeriod
specify how many days to keep MVCC values during major compaction
|
private static org.slf4j.Logger |
LOG |
protected static String |
MAJOR_COMPACTION_DROP_CACHE |
protected Compression.Algorithm |
majorCompactionCompression |
protected static String |
MINOR_COMPACTION_DROP_CACHE |
protected Compression.Algorithm |
minorCompactionCompression |
private Set<CompactionProgress> |
progressSet |
protected HStore |
store |
Constructor and Description |
---|
Compactor(org.apache.hadoop.conf.Configuration conf,
HStore store) |
Modifier and Type | Method and Description |
---|---|
protected abstract void |
abortWriter(T writer) |
protected abstract List<org.apache.hadoop.fs.Path> |
commitWriter(T writer,
Compactor.FileDetails fd,
CompactionRequestImpl request) |
protected List<org.apache.hadoop.fs.Path> |
compact(CompactionRequestImpl request,
Compactor.InternalScannerFactory scannerFactory,
Compactor.CellSinkFactory<T> sinkFactory,
ThroughputController throughputController,
User user) |
private List<StoreFileScanner> |
createFileScanners(Collection<HStoreFile> filesToCompact,
long smallestReadPoint,
boolean useDropBehind)
Creates file scanners for compaction.
|
protected CreateStoreFileWriterParams |
createParams(Compactor.FileDetails fd,
boolean shouldDropBehind,
boolean major,
Consumer<org.apache.hadoop.fs.Path> writerCreationTracker) |
protected InternalScanner |
createScanner(HStore store,
ScanInfo scanInfo,
List<StoreFileScanner> scanners,
long smallestReadPoint,
long earliestPutTs,
byte[] dropDeletesFromRow,
byte[] dropDeletesToRow) |
protected InternalScanner |
createScanner(HStore store,
ScanInfo scanInfo,
List<StoreFileScanner> scanners,
ScanType scanType,
long smallestReadPoint,
long earliestPutTs) |
protected StoreFileWriter |
createWriter(Compactor.FileDetails fd,
boolean shouldDropBehind,
boolean major,
Consumer<org.apache.hadoop.fs.Path> writerCreationTracker)
Creates a writer for a new file.
|
protected StoreFileWriter |
createWriter(Compactor.FileDetails fd,
boolean shouldDropBehind,
String fileStoragePolicy,
boolean major,
Consumer<org.apache.hadoop.fs.Path> writerCreationTracker) |
private Compactor.FileDetails |
getFileDetails(Collection<HStoreFile> filesToCompact,
boolean allFiles,
boolean major)
Extracts some details about the files to compact that are commonly needed by compactors.
|
CompactionProgress |
getProgress()
Return the aggregate progress for all currently active compactions.
|
private long |
getSmallestReadPoint() |
boolean |
isCompacting() |
protected boolean |
performCompaction(Compactor.FileDetails fd,
InternalScanner scanner,
CellSink writer,
long smallestReadPoint,
boolean cleanSeqId,
ThroughputController throughputController,
CompactionRequestImpl request,
CompactionProgress progress)
Performs the compaction.
|
private InternalScanner |
postCompactScannerOpen(CompactionRequestImpl request,
ScanType scanType,
InternalScanner scanner,
User user)
Calls coprocessor, if any, to create scanners - after normal scanner creation.
|
private ScanInfo |
preCompactScannerOpen(CompactionRequestImpl request,
ScanType scanType,
User user) |
private static final org.slf4j.Logger LOG
protected static final long COMPACTION_PROGRESS_LOG_INTERVAL
protected final org.apache.hadoop.conf.Configuration conf
protected final int compactionKVMax
protected final Compression.Algorithm majorCompactionCompression
protected final Compression.Algorithm minorCompactionCompression
protected int keepSeqIdPeriod
protected static final String MAJOR_COMPACTION_DROP_CACHE
protected static final String MINOR_COMPACTION_DROP_CACHE
protected final boolean dropCacheMajor
protected final boolean dropCacheMinor
private final Set<CompactionProgress> progressSet
protected final Compactor.InternalScannerFactory defaultScannerFactory
private Compactor.FileDetails getFileDetails(Collection<HStoreFile> filesToCompact, boolean allFiles, boolean major) throws IOException
filesToCompact
- Files.allFiles
- Whether all files are included for compactionIOException
private List<StoreFileScanner> createFileScanners(Collection<HStoreFile> filesToCompact, long smallestReadPoint, boolean useDropBehind) throws IOException
filesToCompact
- Files.IOException
private long getSmallestReadPoint()
protected final CreateStoreFileWriterParams createParams(Compactor.FileDetails fd, boolean shouldDropBehind, boolean major, Consumer<org.apache.hadoop.fs.Path> writerCreationTracker)
protected final StoreFileWriter createWriter(Compactor.FileDetails fd, boolean shouldDropBehind, boolean major, Consumer<org.apache.hadoop.fs.Path> writerCreationTracker) throws IOException
fd
- The file details.IOException
- if creation failedprotected final StoreFileWriter createWriter(Compactor.FileDetails fd, boolean shouldDropBehind, String fileStoragePolicy, boolean major, Consumer<org.apache.hadoop.fs.Path> writerCreationTracker) throws IOException
IOException
private ScanInfo preCompactScannerOpen(CompactionRequestImpl request, ScanType scanType, User user) throws IOException
IOException
private InternalScanner postCompactScannerOpen(CompactionRequestImpl request, ScanType scanType, InternalScanner scanner, User user) throws IOException
request
- Compaction request.scanType
- Scan type.scanner
- The default scanner created for compaction.IOException
protected final List<org.apache.hadoop.fs.Path> compact(CompactionRequestImpl request, Compactor.InternalScannerFactory scannerFactory, Compactor.CellSinkFactory<T> sinkFactory, ThroughputController throughputController, User user) throws IOException
IOException
protected abstract List<org.apache.hadoop.fs.Path> commitWriter(T writer, Compactor.FileDetails fd, CompactionRequestImpl request) throws IOException
IOException
protected abstract void abortWriter(T writer) throws IOException
IOException
protected boolean performCompaction(Compactor.FileDetails fd, InternalScanner scanner, CellSink writer, long smallestReadPoint, boolean cleanSeqId, ThroughputController throughputController, CompactionRequestImpl request, CompactionProgress progress) throws IOException
fd
- FileDetails of cell sink writerscanner
- Where to read from.writer
- Where to write to.smallestReadPoint
- Smallest read point.cleanSeqId
- When true, remove seqId(used to be mvcc) value which is <=
smallestReadPointrequest
- compaction request.progress
- Progress reporter.IOException
protected InternalScanner createScanner(HStore store, ScanInfo scanInfo, List<StoreFileScanner> scanners, ScanType scanType, long smallestReadPoint, long earliestPutTs) throws IOException
store
- storescanners
- Store file scanners.scanType
- Scan type.smallestReadPoint
- Smallest MVCC read point.earliestPutTs
- Earliest put across all files.IOException
protected InternalScanner createScanner(HStore store, ScanInfo scanInfo, List<StoreFileScanner> scanners, long smallestReadPoint, long earliestPutTs, byte[] dropDeletesFromRow, byte[] dropDeletesToRow) throws IOException
store
- The store.scanners
- Store file scanners.smallestReadPoint
- Smallest MVCC read point.earliestPutTs
- Earliest put across all files.dropDeletesFromRow
- Drop deletes starting with this row, inclusive. Can be null.dropDeletesToRow
- Drop deletes ending with this row, exclusive. Can be null.IOException
public CompactionProgress getProgress()
public boolean isCompacting()
Copyright © 2007–2020 The Apache Software Foundation. All rights reserved.