Package org.apache.hadoop.hbase.regionserver

Class StoreFileWriter

java.lang.Object

org.apache.hadoop.hbase.regionserver.StoreFileWriter

All Implemented Interfaces:

CellSink, ShipperListener

@Private
public class StoreFileWriter
extends Object
implements CellSink, ShipperListener

A StoreFile writer. Use this to read/write HBase Store Files. It is package local because it is an implementation detail of the HBase regionserver.

Nested Class Summary

Nested Classes

Modifier and Type	Class	Description
static class	StoreFileWriter.Builder
private static class	StoreFileWriter.SingleStoreFileWriter

Field Summary

Fields

Modifier and Type	Field	Description
private final BloomType	bloomType
private final CacheConfig	cacheConf
private final Supplier<Collection<HStoreFile>>	compactedFilesSupplier
private final CellComparator	comparator
private final org.apache.hadoop.conf.Configuration	conf
private static final Pattern	dash
static final boolean	DEFAULT_ENABLE_HISTORICAL_COMPACTION_FILES
private ExtendedCell	deleteColumn
private List<ExtendedCell>	deleteColumnVersionList
private ExtendedCell	deleteFamily
private List<ExtendedCell>	deleteFamilyVersionList
static final String	ENABLE_HISTORICAL_COMPACTION_FILES
private final InetSocketAddress[]	favoredNodes
private final HFileContext	fileContext
private final org.apache.hadoop.fs.FileSystem	fs
private final org.apache.hadoop.fs.Path	historicalFilePath
private StoreFileWriter.SingleStoreFileWriter	historicalFileWriter
private ExtendedCell	lastCell
private StoreFileWriter.SingleStoreFileWriter	liveFileWriter
private int	livePutCellCount
private static final org.slf4j.Logger	LOG
private final long	maxKeys
private final int	maxVersions
private final boolean	newVersionBehavior
private final boolean	shouldDropCacheBehind

Constructor Summary

Constructors

Modifier	Constructor	Description
private	StoreFileWriter(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path liveFilePath, org.apache.hadoop.fs.Path historicalFilePath, org.apache.hadoop.conf.Configuration conf, CacheConfig cacheConf, BloomType bloomType, long maxKeys, InetSocketAddress[] favoredNodes, HFileContext fileContext, boolean shouldDropCacheBehind, Supplier<Collection<HStoreFile>> compactedFilesSupplier, CellComparator comparator, int maxVersions, boolean newVersionBehavior)	Creates an HFile.Writer that also write helpful meta data.

Method Summary

Modifier and Type	Method	Description
void	append(ExtendedCell cell)	Append the given cell
void	appendAll(List<ExtendedCell> cellList)	Append the given (possibly partial) list of cells of a row
private void	appendCell(ExtendedCell cell)
void	appendFileInfo(byte[] key, byte[] value)
void	appendMetadata(long maxSequenceId, boolean majorCompaction)	Writes meta data.
void	appendMetadata(long maxSequenceId, boolean majorCompaction, long mobCellsCount)	Writes meta data.
void	appendMetadata(long maxSequenceId, boolean majorCompaction, Collection<HStoreFile> storeFiles)	Writes meta data.
void	appendMobMetadata(org.apache.hbase.thirdparty.com.google.common.collect.SetMultimap<TableName,String> mobRefSet)	Appends MOB - specific metadata (even if it is empty)
void	appendTrackedTimestampsToMetadata()	Add TimestampRange and earliest put timestamp to Metadata
void	beforeShipped()	The action that needs to be performed before Shipper.shipped() is performed
void	close()
(package private) BloomFilterWriter	getGeneralBloomWriter()	For unit testing only.
private StoreFileWriter.SingleStoreFileWriter	getHistoricalFileWriter()
(package private) HFile.Writer	getLiveFileWriter()	For use in testing.
org.apache.hadoop.fs.Path	getPath()
List<org.apache.hadoop.fs.Path>	getPaths()
long	getPos()
static org.apache.hadoop.fs.Path	getUniqueFile(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path dir)
boolean	hasGeneralBloom()
private void	initColumnState()
private void	initRowState()
private boolean	isDeleted(ExtendedCell cell)
private boolean	isDeletedByDeleteColumn(ExtendedCell cell)
private boolean	isDeletedByDeleteColumnVersion(ExtendedCell cell)
private boolean	isDeletedByDeleteFamily(ExtendedCell cell)
private boolean	isDeletedByDeleteFamilyVersion(ExtendedCell cell)
static boolean	shouldEnableHistoricalCompactionFiles(org.apache.hadoop.conf.Configuration conf)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Details

LOG

private static final org.slf4j.Logger LOG

ENABLE_HISTORICAL_COMPACTION_FILES

public static final String ENABLE_HISTORICAL_COMPACTION_FILES

See Also:

• Constant Field Values

DEFAULT_ENABLE_HISTORICAL_COMPACTION_FILES

public static final boolean DEFAULT_ENABLE_HISTORICAL_COMPACTION_FILES

See Also:

• Constant Field Values

dash

private static final Pattern dash

liveFileWriter

private StoreFileWriter.SingleStoreFileWriter liveFileWriter

historicalFileWriter

private StoreFileWriter.SingleStoreFileWriter historicalFileWriter

fs

private final org.apache.hadoop.fs.FileSystem fs

historicalFilePath

private final org.apache.hadoop.fs.Path historicalFilePath

conf

private final org.apache.hadoop.conf.Configuration conf

cacheConf

private final CacheConfig cacheConf

bloomType

private final BloomType bloomType

maxKeys

private final long maxKeys

favoredNodes

private final InetSocketAddress[] favoredNodes

fileContext

private final HFileContext fileContext

shouldDropCacheBehind

private final boolean shouldDropCacheBehind

compactedFilesSupplier

private final Supplier<Collection<HStoreFile>> compactedFilesSupplier

comparator

private final CellComparator comparator

lastCell

private ExtendedCell lastCell

deleteFamily

private ExtendedCell deleteFamily

deleteFamilyVersionList

private List<ExtendedCell> deleteFamilyVersionList

deleteColumn

private ExtendedCell deleteColumn

deleteColumnVersionList

private List<ExtendedCell> deleteColumnVersionList

livePutCellCount

private int livePutCellCount

maxVersions

private final int maxVersions

newVersionBehavior

private final boolean newVersionBehavior

Constructor Details

StoreFileWriter

private StoreFileWriter(org.apache.hadoop.fs.FileSystem fs,
 org.apache.hadoop.fs.Path liveFilePath,
 org.apache.hadoop.fs.Path historicalFilePath,
 org.apache.hadoop.conf.Configuration conf,
 CacheConfig cacheConf,
 BloomType bloomType,
 long maxKeys,
 InetSocketAddress[] favoredNodes,
 HFileContext fileContext,
 boolean shouldDropCacheBehind,
 Supplier<Collection<HStoreFile>> compactedFilesSupplier,
 CellComparator comparator,
 int maxVersions,
 boolean newVersionBehavior)
                 throws IOException

Creates an HFile.Writer that also write helpful meta data.

Parameters:

fs - file system to write to

liveFilePath - the name of the live file to create

historicalFilePath - the name of the historical file name to create

conf - user configuration

bloomType - bloom filter setting

maxKeys - the expected maximum number of keys to be added. Was used for Bloom filter size in HFile format version 1.

favoredNodes - an array of favored nodes or possibly null

fileContext - The HFile context

shouldDropCacheBehind - Drop pages written to page cache after writing the store file.

compactedFilesSupplier - Returns the HStore compacted files which not archived

comparator - Cell comparator

maxVersions - max cell versions

newVersionBehavior - enable new version behavior

Throws:

IOException - problem writing to FS

Method Details

shouldEnableHistoricalCompactionFiles

public static boolean shouldEnableHistoricalCompactionFiles(org.apache.hadoop.conf.Configuration conf)

getPos

public long getPos()
            throws IOException

Throws:

IOException

appendMetadata

public void appendMetadata(long maxSequenceId,
 boolean majorCompaction)
                    throws IOException

Writes meta data. Call before close() since its written as meta data to this file.

Parameters:

maxSequenceId - Maximum sequence id.

majorCompaction - True if this file is product of a major compaction

Throws:

IOException - problem writing to FS

appendMetadata

public void appendMetadata(long maxSequenceId,
 boolean majorCompaction,
 Collection<HStoreFile> storeFiles)
                    throws IOException

Writes meta data. Call before close() since its written as meta data to this file.

Parameters:

maxSequenceId - Maximum sequence id.

majorCompaction - True if this file is product of a major compaction

storeFiles - The compacted store files to generate this new file

Throws:

IOException - problem writing to FS

appendMetadata

public void appendMetadata(long maxSequenceId,
 boolean majorCompaction,
 long mobCellsCount)
                    throws IOException

Writes meta data. Call before close() since its written as meta data to this file.

Parameters:

maxSequenceId - Maximum sequence id.

majorCompaction - True if this file is product of a major compaction

mobCellsCount - The number of mob cells.

Throws:

IOException - problem writing to FS

appendMobMetadata

public void appendMobMetadata(org.apache.hbase.thirdparty.com.google.common.collect.SetMultimap<TableName,String> mobRefSet)
                       throws IOException

Appends MOB - specific metadata (even if it is empty)

Parameters:

mobRefSet - - original table -> set of MOB file names

Throws:

IOException - problem writing to FS

appendTrackedTimestampsToMetadata

public void appendTrackedTimestampsToMetadata()
                                       throws IOException

Add TimestampRange and earliest put timestamp to Metadata

Throws:

IOException

beforeShipped

public void beforeShipped()
                   throws IOException

Description copied from interface: ShipperListener

The action that needs to be performed before Shipper.shipped() is performed

Specified by:

beforeShipped in interface ShipperListener

Throws:

IOException

getPath

public org.apache.hadoop.fs.Path getPath()

getPaths

public List<org.apache.hadoop.fs.Path> getPaths()

hasGeneralBloom

public boolean hasGeneralBloom()

getGeneralBloomWriter

BloomFilterWriter getGeneralBloomWriter()

For unit testing only.

Returns:

the Bloom filter used by this writer.

close

public void close()
           throws IOException

Throws:

IOException

appendFileInfo

public void appendFileInfo(byte[] key,
 byte[] value)
                    throws IOException

Throws:

IOException

getLiveFileWriter

HFile.Writer getLiveFileWriter()

For use in testing.

getUniqueFile

public static org.apache.hadoop.fs.Path getUniqueFile(org.apache.hadoop.fs.FileSystem fs,
 org.apache.hadoop.fs.Path dir)
                                               throws IOException

Parameters:

dir - Directory to create file in.

Returns:

random filename inside passed dir

Throws:

IOException

getHistoricalFileWriter

private StoreFileWriter.SingleStoreFileWriter getHistoricalFileWriter()
                                                               throws IOException

Throws:

IOException

initRowState

private void initRowState()

initColumnState

private void initColumnState()

isDeletedByDeleteFamily

private boolean isDeletedByDeleteFamily(ExtendedCell cell)

isDeletedByDeleteFamilyVersion

private boolean isDeletedByDeleteFamilyVersion(ExtendedCell cell)

isDeletedByDeleteColumn

private boolean isDeletedByDeleteColumn(ExtendedCell cell)

isDeletedByDeleteColumnVersion

private boolean isDeletedByDeleteColumnVersion(ExtendedCell cell)

isDeleted

private boolean isDeleted(ExtendedCell cell)

appendCell

private void appendCell(ExtendedCell cell)
                 throws IOException

Throws:

IOException

appendAll

public void appendAll(List<ExtendedCell> cellList)
               throws IOException

Description copied from interface: CellSink

Append the given (possibly partial) list of cells of a row

Specified by:

appendAll in interface CellSink

Parameters:

cellList - the cell list to be added

Throws:

IOException

append

public void append(ExtendedCell cell)
            throws IOException

Description copied from interface: CellSink

Append the given cell

Specified by:

append in interface CellSink

Parameters:

cell - the cell to be added

Throws:

IOException