@InterfaceAudience.Private public class ProtobufLogWriter extends AbstractProtobufLogWriter implements FSHLogProvider.Writer
Modifier and Type | Field and Description |
---|---|
private static org.slf4j.Logger |
LOG |
protected org.apache.hadoop.fs.FSDataOutputStream |
output |
private AtomicLong |
syncedLength |
cellEncoder, compressionContext, compressor, conf, length, trailer, trailerWarnSize, trailerWritten
Constructor and Description |
---|
ProtobufLogWriter() |
Modifier and Type | Method and Description |
---|---|
void |
append(WAL.Entry entry) |
void |
close() |
protected OutputStream |
getOutputStreamForCellEncoder() |
org.apache.hadoop.fs.FSDataOutputStream |
getStream() |
long |
getSyncedLength()
NOTE: We add this method for
WALFileLengthProvider used for replication,
considering the case if we use AsyncFSWAL ,we write to 3 DNs concurrently,
according to the visibility guarantee of HDFS, the data will be available immediately
when arriving at DN since all the DNs will be considered as the last one in pipeline. |
protected void |
initOutput(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path path,
boolean overwritable,
int bufferSize,
short replication,
long blockSize) |
void |
sync(boolean forceSync) |
protected long |
writeMagicAndWALHeader(byte[] magic,
org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.WALHeader header)
return the file length after written.
|
protected long |
writeWALTrailerAndMagic(org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.WALTrailer trailer,
byte[] magic) |
buildSecureWALHeader, buildWALHeader, getLength, getWriterClassName, init, initAfterHeader, secureInitAfterHeader, setEncryptor, setWALTrailer, writeWALTrailer
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
init
getLength
private static final org.slf4j.Logger LOG
protected org.apache.hadoop.fs.FSDataOutputStream output
private final AtomicLong syncedLength
public ProtobufLogWriter()
public void append(WAL.Entry entry) throws IOException
append
in interface WALProvider.Writer
IOException
public void close() throws IOException
close
in interface Closeable
close
in interface AutoCloseable
IOException
public void sync(boolean forceSync) throws IOException
sync
in interface WALProvider.Writer
IOException
public long getSyncedLength()
WALProvider.WriterBase
WALFileLengthProvider
used for replication,
considering the case if we use AsyncFSWAL
,we write to 3 DNs concurrently,
according to the visibility guarantee of HDFS, the data will be available immediately
when arriving at DN since all the DNs will be considered as the last one in pipeline.
This means replication may read uncommitted data and replicate it to the remote cluster
and cause data inconsistency.
The method WALProvider.WriterBase.getLength()
may return length which just in hdfs client
buffer and not successfully synced to HDFS, so we use this method to return the length
successfully synced to HDFS and replication thread could only read writing WAL file
limited by this length.
see also HBASE-14004 and this document for more details:
https://docs.google.com/document/d/11AyWtGhItQs6vsLRIx32PwTxmBY3libXwGXI25obVEY/edit#getSyncedLength
in interface WALProvider.WriterBase
public org.apache.hadoop.fs.FSDataOutputStream getStream()
protected void initOutput(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path path, boolean overwritable, int bufferSize, short replication, long blockSize) throws IOException, CommonFSUtils.StreamLacksCapabilityException
initOutput
in class AbstractProtobufLogWriter
IOException
CommonFSUtils.StreamLacksCapabilityException
protected long writeMagicAndWALHeader(byte[] magic, org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.WALHeader header) throws IOException
AbstractProtobufLogWriter
writeMagicAndWALHeader
in class AbstractProtobufLogWriter
IOException
protected OutputStream getOutputStreamForCellEncoder()
getOutputStreamForCellEncoder
in class AbstractProtobufLogWriter
protected long writeWALTrailerAndMagic(org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.WALTrailer trailer, byte[] magic) throws IOException
writeWALTrailerAndMagic
in class AbstractProtobufLogWriter
IOException
Copyright © 2007–2021 The Apache Software Foundation. All rights reserved.