org.apache.hadoop.hbase.io.asyncfs

Class FanOutOneBlockAsyncDFSOutputHelper

java.lang.Object

org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper

@InterfaceAudience.Private
public final class FanOutOneBlockAsyncDFSOutputHelper
extends Object

Helper class for implementing FanOutOneBlockAsyncDFSOutput.

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
private static interface	FanOutOneBlockAsyncDFSOutputHelper.BlockAdder
(package private) static class	FanOutOneBlockAsyncDFSOutputHelper.CancelOnClose
private static interface	FanOutOneBlockAsyncDFSOutputHelper.ChecksumCreater
private static interface	FanOutOneBlockAsyncDFSOutputHelper.DFSClientAdaptor
private static interface	FanOutOneBlockAsyncDFSOutputHelper.FileCreator
private static interface	FanOutOneBlockAsyncDFSOutputHelper.LeaseManager
static class	FanOutOneBlockAsyncDFSOutputHelper.NameNodeException Exception other than RemoteException thrown when calling create on namenode
private static interface	FanOutOneBlockAsyncDFSOutputHelper.PBHelper
private static interface	FanOutOneBlockAsyncDFSOutputHelper.PipelineAckStatusGetter
private static interface	FanOutOneBlockAsyncDFSOutputHelper.StorageTypeSetter

Field Summary

Fields

Modifier and Type	Field and Description
private static org.apache.hbase.thirdparty.io.netty.buffer.ByteBufAllocator	ALLOC
static String	ASYNC_DFS_OUTPUT_CREATE_MAX_RETRIES
private static FanOutOneBlockAsyncDFSOutputHelper.BlockAdder	BLOCK_ADDER
private static FanOutOneBlockAsyncDFSOutputHelper.ChecksumCreater	CHECKSUM_CREATER
static int	DEFAULT_ASYNC_DFS_OUTPUT_CREATE_MAX_RETRIES
private static FanOutOneBlockAsyncDFSOutputHelper.DFSClientAdaptor	DFS_CLIENT_ADAPTOR
private static org.apache.hadoop.hdfs.protocol.DatanodeInfo[]	EMPTY_DN_ARRAY
private static FanOutOneBlockAsyncDFSOutputHelper.FileCreator	FILE_CREATOR
static long	HEART_BEAT_SEQNO
private static FanOutOneBlockAsyncDFSOutputHelper.LeaseManager	LEASE_MANAGER
private static org.slf4j.Logger	LOG
private static FanOutOneBlockAsyncDFSOutputHelper.PBHelper	PB_HELPER
private static FanOutOneBlockAsyncDFSOutputHelper.PipelineAckStatusGetter	PIPELINE_ACK_STATUS_GETTER
static int	READ_TIMEOUT
private static FanOutOneBlockAsyncDFSOutputHelper.StorageTypeSetter	STORAGE_TYPE_SETTER

Constructor Summary

Constructors

Modifier	Constructor and Description
private	FanOutOneBlockAsyncDFSOutputHelper()

Method Summary

Modifier and Type	Method and Description
(package private) static void	beginFileLease(org.apache.hadoop.hdfs.DFSClient client, long inodeId)
(package private) static void	completeFile(org.apache.hadoop.hdfs.DFSClient client, org.apache.hadoop.hdfs.protocol.ClientProtocol namenode, String src, String clientName, org.apache.hadoop.hdfs.protocol.ExtendedBlock block, long fileId)
private static List<org.apache.hbase.thirdparty.io.netty.util.concurrent.Future<org.apache.hbase.thirdparty.io.netty.channel.Channel>>	connectToDataNodes(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.hdfs.DFSClient client, String clientName, org.apache.hadoop.hdfs.protocol.LocatedBlock locatedBlock, long maxBytesRcvd, long latestGS, org.apache.hadoop.hdfs.protocol.datatransfer.BlockConstructionStage stage, org.apache.hadoop.util.DataChecksum summer, org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup eventLoopGroup, Class<? extends org.apache.hbase.thirdparty.io.netty.channel.Channel> channelClass)
private static FanOutOneBlockAsyncDFSOutputHelper.BlockAdder	createBlockAdder()
(package private) static org.apache.hadoop.util.DataChecksum	createChecksum(org.apache.hadoop.hdfs.DFSClient client)
private static FanOutOneBlockAsyncDFSOutputHelper.ChecksumCreater	createChecksumCreater()
private static FanOutOneBlockAsyncDFSOutputHelper.ChecksumCreater	createChecksumCreater27(Method getConfMethod, Class<?> confClass)
private static FanOutOneBlockAsyncDFSOutputHelper.ChecksumCreater	createChecksumCreater28(Method getConfMethod, Class<?> confClass)
private static FanOutOneBlockAsyncDFSOutputHelper.DFSClientAdaptor	createDFSClientAdaptor()
private static FanOutOneBlockAsyncDFSOutputHelper.FileCreator	createFileCreator()
private static FanOutOneBlockAsyncDFSOutputHelper.FileCreator	createFileCreator2()
private static FanOutOneBlockAsyncDFSOutputHelper.FileCreator	createFileCreator3()
private static FanOutOneBlockAsyncDFSOutputHelper.LeaseManager	createLeaseManager()
static FanOutOneBlockAsyncDFSOutput	createOutput(org.apache.hadoop.hdfs.DistributedFileSystem dfs, org.apache.hadoop.fs.Path f, boolean overwrite, boolean createParent, short replication, long blockSize, org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup eventLoopGroup, Class<? extends org.apache.hbase.thirdparty.io.netty.channel.Channel> channelClass) Create a FanOutOneBlockAsyncDFSOutput.
private static FanOutOneBlockAsyncDFSOutput	createOutput(org.apache.hadoop.hdfs.DistributedFileSystem dfs, String src, boolean overwrite, boolean createParent, short replication, long blockSize, org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup eventLoopGroup, Class<? extends org.apache.hbase.thirdparty.io.netty.channel.Channel> channelClass)
private static FanOutOneBlockAsyncDFSOutputHelper.PBHelper	createPBHelper()
private static FanOutOneBlockAsyncDFSOutputHelper.PipelineAckStatusGetter	createPipelineAckStatusGetter()
private static FanOutOneBlockAsyncDFSOutputHelper.PipelineAckStatusGetter	createPipelineAckStatusGetter26()
private static FanOutOneBlockAsyncDFSOutputHelper.PipelineAckStatusGetter	createPipelineAckStatusGetter27()
private static FanOutOneBlockAsyncDFSOutputHelper.StorageTypeSetter	createStorageTypeSetter()
(package private) static void	endFileLease(org.apache.hadoop.hdfs.DFSClient client, long inodeId)
(package private) static org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.Status	getStatus(org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.PipelineAckProto ack)
private static void	initialize(org.apache.hadoop.conf.Configuration conf, org.apache.hbase.thirdparty.io.netty.channel.Channel channel, org.apache.hadoop.hdfs.protocol.DatanodeInfo dnInfo, Enum<?> storageType, org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.OpWriteBlockProto.Builder writeBlockProtoBuilder, int timeoutMs, org.apache.hadoop.hdfs.DFSClient client, org.apache.hadoop.security.token.Token<org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier> accessToken, org.apache.hbase.thirdparty.io.netty.util.concurrent.Promise<org.apache.hbase.thirdparty.io.netty.channel.Channel> promise)
private static void	processWriteBlockResponse(org.apache.hbase.thirdparty.io.netty.channel.Channel channel, org.apache.hadoop.hdfs.protocol.DatanodeInfo dnInfo, org.apache.hbase.thirdparty.io.netty.util.concurrent.Promise<org.apache.hbase.thirdparty.io.netty.channel.Channel> promise, int timeoutMs)
private static void	requestWriteBlock(org.apache.hbase.thirdparty.io.netty.channel.Channel channel, Enum<?> storageType, org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.OpWriteBlockProto.Builder writeBlockProtoBuilder)
static boolean	shouldRetryCreate(org.apache.hadoop.ipc.RemoteException e)
(package private) static void	sleepIgnoreInterrupt(int retry)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

LOG

private static final org.slf4j.Logger LOG

ASYNC_DFS_OUTPUT_CREATE_MAX_RETRIES

public static final String ASYNC_DFS_OUTPUT_CREATE_MAX_RETRIES

See Also:

Constant Field Values

DEFAULT_ASYNC_DFS_OUTPUT_CREATE_MAX_RETRIES

public static final int DEFAULT_ASYNC_DFS_OUTPUT_CREATE_MAX_RETRIES

See Also:

Constant Field Values

ALLOC

private static final org.apache.hbase.thirdparty.io.netty.buffer.ByteBufAllocator ALLOC

HEART_BEAT_SEQNO

public static final long HEART_BEAT_SEQNO

See Also:

Constant Field Values

READ_TIMEOUT

public static final int READ_TIMEOUT

See Also:

Constant Field Values

EMPTY_DN_ARRAY

private static final org.apache.hadoop.hdfs.protocol.DatanodeInfo[] EMPTY_DN_ARRAY

PIPELINE_ACK_STATUS_GETTER

private static final FanOutOneBlockAsyncDFSOutputHelper.PipelineAckStatusGetter PIPELINE_ACK_STATUS_GETTER

STORAGE_TYPE_SETTER

private static final FanOutOneBlockAsyncDFSOutputHelper.StorageTypeSetter STORAGE_TYPE_SETTER

BLOCK_ADDER

private static final FanOutOneBlockAsyncDFSOutputHelper.BlockAdder BLOCK_ADDER

LEASE_MANAGER

private static final FanOutOneBlockAsyncDFSOutputHelper.LeaseManager LEASE_MANAGER

DFS_CLIENT_ADAPTOR

private static final FanOutOneBlockAsyncDFSOutputHelper.DFSClientAdaptor DFS_CLIENT_ADAPTOR

PB_HELPER

private static final FanOutOneBlockAsyncDFSOutputHelper.PBHelper PB_HELPER

CHECKSUM_CREATER

private static final FanOutOneBlockAsyncDFSOutputHelper.ChecksumCreater CHECKSUM_CREATER

FILE_CREATOR

private static final FanOutOneBlockAsyncDFSOutputHelper.FileCreator FILE_CREATOR

Constructor Detail

FanOutOneBlockAsyncDFSOutputHelper

private FanOutOneBlockAsyncDFSOutputHelper()

Method Detail

createDFSClientAdaptor

private static FanOutOneBlockAsyncDFSOutputHelper.DFSClientAdaptor createDFSClientAdaptor()
                                                                                   throws NoSuchMethodException

Throws:

NoSuchMethodException

createLeaseManager

private static FanOutOneBlockAsyncDFSOutputHelper.LeaseManager createLeaseManager()
                                                                           throws NoSuchMethodException

Throws:

NoSuchMethodException

createPipelineAckStatusGetter27

private static FanOutOneBlockAsyncDFSOutputHelper.PipelineAckStatusGetter createPipelineAckStatusGetter27()
                                                                                                   throws NoSuchMethodException

Throws:

NoSuchMethodException

createPipelineAckStatusGetter26

private static FanOutOneBlockAsyncDFSOutputHelper.PipelineAckStatusGetter createPipelineAckStatusGetter26()
                                                                                                   throws NoSuchMethodException

Throws:

NoSuchMethodException

createPipelineAckStatusGetter

private static FanOutOneBlockAsyncDFSOutputHelper.PipelineAckStatusGetter createPipelineAckStatusGetter()
                                                                                                 throws NoSuchMethodException

Throws:

NoSuchMethodException

createStorageTypeSetter

private static FanOutOneBlockAsyncDFSOutputHelper.StorageTypeSetter createStorageTypeSetter()
                                                                                     throws NoSuchMethodException

Throws:

NoSuchMethodException

createBlockAdder

private static FanOutOneBlockAsyncDFSOutputHelper.BlockAdder createBlockAdder()
                                                                       throws NoSuchMethodException

Throws:

NoSuchMethodException

createPBHelper

private static FanOutOneBlockAsyncDFSOutputHelper.PBHelper createPBHelper()
                                                                   throws NoSuchMethodException

Throws:

NoSuchMethodException

createChecksumCreater28

private static FanOutOneBlockAsyncDFSOutputHelper.ChecksumCreater createChecksumCreater28(Method getConfMethod,
                                                                                          Class<?> confClass)
                                                                                   throws NoSuchMethodException

Throws:

NoSuchMethodException

createChecksumCreater27

private static FanOutOneBlockAsyncDFSOutputHelper.ChecksumCreater createChecksumCreater27(Method getConfMethod,
                                                                                          Class<?> confClass)
                                                                                   throws NoSuchMethodException

Throws:

NoSuchMethodException

createChecksumCreater

private static FanOutOneBlockAsyncDFSOutputHelper.ChecksumCreater createChecksumCreater()
                                                                                 throws NoSuchMethodException,
                                                                                        ClassNotFoundException

Throws:

NoSuchMethodException

ClassNotFoundException

createFileCreator3

private static FanOutOneBlockAsyncDFSOutputHelper.FileCreator createFileCreator3()
                                                                          throws NoSuchMethodException

Throws:

NoSuchMethodException

createFileCreator2

private static FanOutOneBlockAsyncDFSOutputHelper.FileCreator createFileCreator2()
                                                                          throws NoSuchMethodException

Throws:

NoSuchMethodException

createFileCreator

private static FanOutOneBlockAsyncDFSOutputHelper.FileCreator createFileCreator()
                                                                         throws NoSuchMethodException

Throws:

NoSuchMethodException

beginFileLease

static void beginFileLease(org.apache.hadoop.hdfs.DFSClient client,
                           long inodeId)

endFileLease

static void endFileLease(org.apache.hadoop.hdfs.DFSClient client,
                         long inodeId)

createChecksum

static org.apache.hadoop.util.DataChecksum createChecksum(org.apache.hadoop.hdfs.DFSClient client)

getStatus

static org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.Status getStatus(org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.PipelineAckProto ack)

processWriteBlockResponse

private static void processWriteBlockResponse(org.apache.hbase.thirdparty.io.netty.channel.Channel channel,
                                              org.apache.hadoop.hdfs.protocol.DatanodeInfo dnInfo,
                                              org.apache.hbase.thirdparty.io.netty.util.concurrent.Promise<org.apache.hbase.thirdparty.io.netty.channel.Channel> promise,
                                              int timeoutMs)

requestWriteBlock

private static void requestWriteBlock(org.apache.hbase.thirdparty.io.netty.channel.Channel channel,
                                      Enum<?> storageType,
                                      org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.OpWriteBlockProto.Builder writeBlockProtoBuilder)
                               throws IOException

Throws:

IOException

initialize

private static void initialize(org.apache.hadoop.conf.Configuration conf,
                               org.apache.hbase.thirdparty.io.netty.channel.Channel channel,
                               org.apache.hadoop.hdfs.protocol.DatanodeInfo dnInfo,
                               Enum<?> storageType,
                               org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.OpWriteBlockProto.Builder writeBlockProtoBuilder,
                               int timeoutMs,
                               org.apache.hadoop.hdfs.DFSClient client,
                               org.apache.hadoop.security.token.Token<org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier> accessToken,
                               org.apache.hbase.thirdparty.io.netty.util.concurrent.Promise<org.apache.hbase.thirdparty.io.netty.channel.Channel> promise)
                        throws IOException

Throws:

IOException

connectToDataNodes

private static List<org.apache.hbase.thirdparty.io.netty.util.concurrent.Future<org.apache.hbase.thirdparty.io.netty.channel.Channel>> connectToDataNodes(org.apache.hadoop.conf.Configuration conf,
                                                                                                                                                          org.apache.hadoop.hdfs.DFSClient client,
                                                                                                                                                          String clientName,
                                                                                                                                                          org.apache.hadoop.hdfs.protocol.LocatedBlock locatedBlock,
                                                                                                                                                          long maxBytesRcvd,
                                                                                                                                                          long latestGS,
                                                                                                                                                          org.apache.hadoop.hdfs.protocol.datatransfer.BlockConstructionStage stage,
                                                                                                                                                          org.apache.hadoop.util.DataChecksum summer,
                                                                                                                                                          org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup eventLoopGroup,
                                                                                                                                                          Class<? extends org.apache.hbase.thirdparty.io.netty.channel.Channel> channelClass)

createOutput

private static FanOutOneBlockAsyncDFSOutput createOutput(org.apache.hadoop.hdfs.DistributedFileSystem dfs,
                                                         String src,
                                                         boolean overwrite,
                                                         boolean createParent,
                                                         short replication,
                                                         long blockSize,
                                                         org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup eventLoopGroup,
                                                         Class<? extends org.apache.hbase.thirdparty.io.netty.channel.Channel> channelClass)
                                                  throws IOException

Throws:

IOException

createOutput

public static FanOutOneBlockAsyncDFSOutput createOutput(org.apache.hadoop.hdfs.DistributedFileSystem dfs,
                                                        org.apache.hadoop.fs.Path f,
                                                        boolean overwrite,
                                                        boolean createParent,
                                                        short replication,
                                                        long blockSize,
                                                        org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup eventLoopGroup,
                                                        Class<? extends org.apache.hbase.thirdparty.io.netty.channel.Channel> channelClass)
                                                 throws IOException

Create a FanOutOneBlockAsyncDFSOutput. The method maybe blocked so do not call it inside an EventLoop.

Throws:

IOException

shouldRetryCreate

public static boolean shouldRetryCreate(org.apache.hadoop.ipc.RemoteException e)

completeFile

static void completeFile(org.apache.hadoop.hdfs.DFSClient client,
                         org.apache.hadoop.hdfs.protocol.ClientProtocol namenode,
                         String src,
                         String clientName,
                         org.apache.hadoop.hdfs.protocol.ExtendedBlock block,
                         long fileId)

sleepIgnoreInterrupt

static void sleepIgnoreInterrupt(int retry)