@InterfaceAudience.Private public abstract class StoreEngine<SF extends StoreFlusher,CP extends CompactionPolicy,C extends Compactor<?>,SFM extends StoreFileManager> extends Object
HRegionFileSystem
and HStore
before we have SFT. And since SFM is designed to only holds in memory states,
we will hold write lock when updating it, the lock is also used to protect the normal read/write
requests. This means we'd better not add IO operations to SFM. And also, no matter what the in
memory state is, stripe or not, it does not effect how we track the store files. So consider all
these facts, here we introduce a separated SFT to track the store files.
Here, since we always need to update SFM and SFT almost at the same time, we introduce methods in
StoreEngine directly to update them both, so upper layer just need to update StoreEngine once, to
reduce the possible misuse.Modifier and Type | Class and Description |
---|---|
static interface |
StoreEngine.IOExceptionRunnable |
Modifier and Type | Field and Description |
---|---|
private BloomFilterMetrics |
bloomFilterMetrics |
protected CP |
compactionPolicy |
protected C |
compactor |
private org.apache.hadoop.conf.Configuration |
conf |
private RegionCoprocessorHost |
coprocessorHost |
private StoreContext |
ctx |
private static Class<? extends StoreEngine<?,?,?,?>> |
DEFAULT_STORE_ENGINE_CLASS |
private static org.slf4j.Logger |
LOG |
private Function<String,ExecutorService> |
openStoreFileThreadPoolCreator |
static String |
STORE_ENGINE_CLASS_KEY
The name of the configuration parameter that specifies the class of a store engine that is used
to manage and compact HBase store files.
|
protected SFM |
storeFileManager |
private StoreFileTracker |
storeFileTracker |
protected SF |
storeFlusher |
private ReadWriteLock |
storeLock |
Constructor and Description |
---|
StoreEngine() |
Modifier and Type | Method and Description |
---|---|
void |
addStoreFiles(Collection<HStoreFile> storeFiles,
StoreEngine.IOExceptionRunnable actionAfterAdding)
Add the store files to store file manager, and also record it in the store file tracker.
|
List<HStoreFile> |
commitStoreFiles(List<org.apache.hadoop.fs.Path> files,
boolean validate)
Commit the given
files . |
static StoreEngine<?,?,?,?> |
create(HStore store,
org.apache.hadoop.conf.Configuration conf,
CellComparator cellComparator)
Create the StoreEngine configured for the given Store.
|
abstract CompactionContext |
createCompaction()
Creates an instance of a compaction context specific to this engine.
|
protected abstract void |
createComponents(org.apache.hadoop.conf.Configuration conf,
HStore store,
CellComparator cellComparator)
Create the StoreEngine's components.
|
protected void |
createComponentsOnce(org.apache.hadoop.conf.Configuration conf,
HStore store,
CellComparator cellComparator) |
HStoreFile |
createStoreFileAndReader(org.apache.hadoop.fs.Path p) |
HStoreFile |
createStoreFileAndReader(StoreFileInfo info) |
private StoreFileTracker |
createStoreFileTracker(org.apache.hadoop.conf.Configuration conf,
HStore store) |
StoreFileWriter |
createWriter(CreateStoreFileWriterParams params)
Create a writer for writing new store files.
|
BloomFilterMetrics |
getBloomFilterMetrics() |
CompactionPolicy |
getCompactionPolicy()
Returns Compaction policy to use.
|
Compactor<?> |
getCompactor()
Returns Compactor to use.
|
(package private) ReadWriteLock |
getLock() |
StoreFileManager |
getStoreFileManager()
Returns Store file manager to use.
|
StoreFlusher |
getStoreFlusher()
Returns Store flusher to use.
|
void |
initialize(boolean warmup) |
abstract boolean |
needsCompaction(List<HStoreFile> filesCompacting) |
private List<HStoreFile> |
openStoreFiles(Collection<StoreFileInfo> files,
boolean warmup) |
void |
readLock()
Acquire read lock of this store.
|
void |
readUnlock()
Release read lock of this store.
|
void |
refreshStoreFiles() |
void |
refreshStoreFiles(Collection<String> newFiles) |
private void |
refreshStoreFilesInternal(Collection<StoreFileInfo> newFiles)
Checks the underlying store files, and opens the files that have not been opened, and removes
the store file readers for store files no longer available.
|
void |
removeCompactedFiles(Collection<HStoreFile> compactedFiles) |
void |
replaceStoreFiles(Collection<HStoreFile> compactedFiles,
Collection<HStoreFile> newFiles,
StoreEngine.IOExceptionRunnable walMarkerWriter,
Runnable actionUnderLock) |
boolean |
requireWritingToTmpDirFirst()
Whether the implementation of the used storefile tracker requires you to write to temp
directory first, i.e, does not allow broken store files under the actual data directory.
|
void |
validateStoreFile(org.apache.hadoop.fs.Path path)
Validates a store file by opening and closing it.
|
void |
writeLock()
Acquire write lock of this store.
|
void |
writeUnlock()
Release write lock of this store.
|
private static final org.slf4j.Logger LOG
protected SF extends StoreFlusher storeFlusher
protected CP extends CompactionPolicy compactionPolicy
protected SFM extends StoreFileManager storeFileManager
private final BloomFilterMetrics bloomFilterMetrics
private org.apache.hadoop.conf.Configuration conf
private StoreContext ctx
private RegionCoprocessorHost coprocessorHost
private Function<String,ExecutorService> openStoreFileThreadPoolCreator
private StoreFileTracker storeFileTracker
private final ReadWriteLock storeLock
public static final String STORE_ENGINE_CLASS_KEY
private static final Class<? extends StoreEngine<?,?,?,?>> DEFAULT_STORE_ENGINE_CLASS
public StoreEngine()
public void readLock()
public void readUnlock()
public void writeLock()
public void writeUnlock()
public CompactionPolicy getCompactionPolicy()
public Compactor<?> getCompactor()
public StoreFileManager getStoreFileManager()
public StoreFlusher getStoreFlusher()
private StoreFileTracker createStoreFileTracker(org.apache.hadoop.conf.Configuration conf, HStore store)
public abstract boolean needsCompaction(List<HStoreFile> filesCompacting)
filesCompacting
- Files currently compactingpublic abstract CompactionContext createCompaction() throws IOException
IOException
protected abstract void createComponents(org.apache.hadoop.conf.Configuration conf, HStore store, CellComparator cellComparator) throws IOException
IOException
protected final void createComponentsOnce(org.apache.hadoop.conf.Configuration conf, HStore store, CellComparator cellComparator) throws IOException
IOException
public StoreFileWriter createWriter(CreateStoreFileWriterParams params) throws IOException
IOException
public HStoreFile createStoreFileAndReader(org.apache.hadoop.fs.Path p) throws IOException
IOException
public HStoreFile createStoreFileAndReader(StoreFileInfo info) throws IOException
IOException
public void validateStoreFile(org.apache.hadoop.fs.Path path) throws IOException
path
- the path to the store fileIOException
private List<HStoreFile> openStoreFiles(Collection<StoreFileInfo> files, boolean warmup) throws IOException
IOException
public void initialize(boolean warmup) throws IOException
IOException
public void refreshStoreFiles() throws IOException
IOException
public void refreshStoreFiles(Collection<String> newFiles) throws IOException
IOException
private void refreshStoreFilesInternal(Collection<StoreFileInfo> newFiles) throws IOException
IOException
public List<HStoreFile> commitStoreFiles(List<org.apache.hadoop.fs.Path> files, boolean validate) throws IOException
files
.
We will move the file into data directory, and open it.files
- the files want to commitvalidate
- whether to validate the store filesIOException
public void addStoreFiles(Collection<HStoreFile> storeFiles, StoreEngine.IOExceptionRunnable actionAfterAdding) throws IOException
actionAfterAdding
will be executed after the insertion to store file manager, under
the lock protection. Usually this is for clear the memstore snapshot.IOException
public void replaceStoreFiles(Collection<HStoreFile> compactedFiles, Collection<HStoreFile> newFiles, StoreEngine.IOExceptionRunnable walMarkerWriter, Runnable actionUnderLock) throws IOException
IOException
public void removeCompactedFiles(Collection<HStoreFile> compactedFiles)
public static StoreEngine<?,?,?,?> create(HStore store, org.apache.hadoop.conf.Configuration conf, CellComparator cellComparator) throws IOException
store
- The store. An unfortunate dependency needed due to it being passed to
coprocessors via the compactor.conf
- Store configuration.cellComparator
- CellComparator for storeFileManager.IOException
public boolean requireWritingToTmpDirFirst()
ReadWriteLock getLock()
public BloomFilterMetrics getBloomFilterMetrics()
Copyright © 2007–2020 The Apache Software Foundation. All rights reserved.