org.apache.hadoop.hbase.io.asyncfs

Class FanOutOneBlockAsyncDFSOutputHelper

java.lang.Object

org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper

@InterfaceAudience.Private
public final class FanOutOneBlockAsyncDFSOutputHelper
extends Object

Helper class for implementing FanOutOneBlockAsyncDFSOutput.

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
(package private) static class	FanOutOneBlockAsyncDFSOutputHelper.CancelOnClose
private static interface	FanOutOneBlockAsyncDFSOutputHelper.DFSClientAdaptor
private static interface	FanOutOneBlockAsyncDFSOutputHelper.FileCreator
private static interface	FanOutOneBlockAsyncDFSOutputHelper.LeaseManager
static class	FanOutOneBlockAsyncDFSOutputHelper.NameNodeException Exception other than RemoteException thrown when calling create on namenode

Field Summary

Fields

Modifier and Type	Field and Description
private static org.apache.hbase.thirdparty.io.netty.buffer.ByteBufAllocator	ALLOC
static String	ASYNC_DFS_OUTPUT_CREATE_MAX_RETRIES
static int	DEFAULT_ASYNC_DFS_OUTPUT_CREATE_MAX_RETRIES
private static FanOutOneBlockAsyncDFSOutputHelper.DFSClientAdaptor	DFS_CLIENT_ADAPTOR
private static FanOutOneBlockAsyncDFSOutputHelper.FileCreator	FILE_CREATOR
static long	HEART_BEAT_SEQNO
private static FanOutOneBlockAsyncDFSOutputHelper.LeaseManager	LEASE_MANAGER
private static org.slf4j.Logger	LOG
static int	READ_TIMEOUT
private static org.apache.hadoop.fs.CreateFlag	SHOULD_REPLICATE_FLAG

Constructor Summary

Constructors

Modifier	Constructor and Description
private	FanOutOneBlockAsyncDFSOutputHelper()

Method Summary

Modifier and Type	Method and Description
(package private) static void	beginFileLease(org.apache.hadoop.hdfs.DFSClient client, long inodeId)
(package private) static void	completeFile(org.apache.hadoop.hdfs.DFSClient client, org.apache.hadoop.hdfs.protocol.ClientProtocol namenode, String src, String clientName, org.apache.hadoop.hdfs.protocol.ExtendedBlock block, long fileId)
private static List<org.apache.hbase.thirdparty.io.netty.util.concurrent.Future<org.apache.hbase.thirdparty.io.netty.channel.Channel>>	connectToDataNodes(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.hdfs.DFSClient client, String clientName, org.apache.hadoop.hdfs.protocol.LocatedBlock locatedBlock, long maxBytesRcvd, long latestGS, org.apache.hadoop.hdfs.protocol.datatransfer.BlockConstructionStage stage, org.apache.hadoop.util.DataChecksum summer, org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup eventLoopGroup, Class<? extends org.apache.hbase.thirdparty.io.netty.channel.Channel> channelClass)
(package private) static org.apache.hadoop.util.DataChecksum	createChecksum(org.apache.hadoop.hdfs.DFSClient client)
private static FanOutOneBlockAsyncDFSOutputHelper.DFSClientAdaptor	createDFSClientAdaptor()
private static FanOutOneBlockAsyncDFSOutputHelper.FileCreator	createFileCreator()
private static FanOutOneBlockAsyncDFSOutputHelper.FileCreator	createFileCreator2()
private static FanOutOneBlockAsyncDFSOutputHelper.FileCreator	createFileCreator3_3()
private static FanOutOneBlockAsyncDFSOutputHelper.FileCreator	createFileCreator3()
private static FanOutOneBlockAsyncDFSOutputHelper.LeaseManager	createLeaseManager()
static FanOutOneBlockAsyncDFSOutput	createOutput(org.apache.hadoop.hdfs.DistributedFileSystem dfs, org.apache.hadoop.fs.Path f, boolean overwrite, boolean createParent, short replication, long blockSize, org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup eventLoopGroup, Class<? extends org.apache.hbase.thirdparty.io.netty.channel.Channel> channelClass, StreamSlowMonitor monitor) Create a FanOutOneBlockAsyncDFSOutput.
private static FanOutOneBlockAsyncDFSOutput	createOutput(org.apache.hadoop.hdfs.DistributedFileSystem dfs, String src, boolean overwrite, boolean createParent, short replication, long blockSize, org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup eventLoopGroup, Class<? extends org.apache.hbase.thirdparty.io.netty.channel.Channel> channelClass, StreamSlowMonitor monitor)
(package private) static void	endFileLease(org.apache.hadoop.hdfs.DFSClient client, long inodeId)
private static org.apache.hadoop.io.EnumSetWritable<org.apache.hadoop.fs.CreateFlag>	getCreateFlags(boolean overwrite)
(package private) static org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.Status	getStatus(org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.PipelineAckProto ack)
private static void	initialize(org.apache.hadoop.conf.Configuration conf, org.apache.hbase.thirdparty.io.netty.channel.Channel channel, org.apache.hadoop.hdfs.protocol.DatanodeInfo dnInfo, org.apache.hadoop.fs.StorageType storageType, org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.OpWriteBlockProto.Builder writeBlockProtoBuilder, int timeoutMs, org.apache.hadoop.hdfs.DFSClient client, org.apache.hadoop.security.token.Token<org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier> accessToken, org.apache.hbase.thirdparty.io.netty.util.concurrent.Promise<org.apache.hbase.thirdparty.io.netty.channel.Channel> promise)
private static org.apache.hadoop.fs.CreateFlag	loadShouldReplicateFlag()
private static void	processWriteBlockResponse(org.apache.hbase.thirdparty.io.netty.channel.Channel channel, org.apache.hadoop.hdfs.protocol.DatanodeInfo dnInfo, org.apache.hbase.thirdparty.io.netty.util.concurrent.Promise<org.apache.hbase.thirdparty.io.netty.channel.Channel> promise, int timeoutMs)
private static void	requestWriteBlock(org.apache.hbase.thirdparty.io.netty.channel.Channel channel, org.apache.hadoop.fs.StorageType storageType, org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.OpWriteBlockProto.Builder writeBlockProtoBuilder)
static boolean	shouldRetryCreate(org.apache.hadoop.ipc.RemoteException e)
(package private) static void	sleepIgnoreInterrupt(int retry)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

LOG

private static final org.slf4j.Logger LOG

ASYNC_DFS_OUTPUT_CREATE_MAX_RETRIES

public static final String ASYNC_DFS_OUTPUT_CREATE_MAX_RETRIES

See Also:

Constant Field Values

DEFAULT_ASYNC_DFS_OUTPUT_CREATE_MAX_RETRIES

public static final int DEFAULT_ASYNC_DFS_OUTPUT_CREATE_MAX_RETRIES

See Also:

Constant Field Values

ALLOC

private static final org.apache.hbase.thirdparty.io.netty.buffer.ByteBufAllocator ALLOC

HEART_BEAT_SEQNO

public static final long HEART_BEAT_SEQNO

See Also:

Constant Field Values

READ_TIMEOUT

public static final int READ_TIMEOUT

See Also:

Constant Field Values

LEASE_MANAGER

private static final FanOutOneBlockAsyncDFSOutputHelper.LeaseManager LEASE_MANAGER

DFS_CLIENT_ADAPTOR

private static final FanOutOneBlockAsyncDFSOutputHelper.DFSClientAdaptor DFS_CLIENT_ADAPTOR

FILE_CREATOR

private static final FanOutOneBlockAsyncDFSOutputHelper.FileCreator FILE_CREATOR

SHOULD_REPLICATE_FLAG

private static final org.apache.hadoop.fs.CreateFlag SHOULD_REPLICATE_FLAG

Constructor Detail

FanOutOneBlockAsyncDFSOutputHelper

private FanOutOneBlockAsyncDFSOutputHelper()

Method Detail

createDFSClientAdaptor

private static FanOutOneBlockAsyncDFSOutputHelper.DFSClientAdaptor createDFSClientAdaptor()
                                                                                   throws NoSuchMethodException

Throws:

NoSuchMethodException

createLeaseManager

private static FanOutOneBlockAsyncDFSOutputHelper.LeaseManager createLeaseManager()
                                                                           throws NoSuchMethodException

Throws:

NoSuchMethodException

createFileCreator3_3

private static FanOutOneBlockAsyncDFSOutputHelper.FileCreator createFileCreator3_3()
                                                                            throws NoSuchMethodException

Throws:

NoSuchMethodException

createFileCreator3

private static FanOutOneBlockAsyncDFSOutputHelper.FileCreator createFileCreator3()
                                                                          throws NoSuchMethodException

Throws:

NoSuchMethodException

createFileCreator2

private static FanOutOneBlockAsyncDFSOutputHelper.FileCreator createFileCreator2()
                                                                          throws NoSuchMethodException

Throws:

NoSuchMethodException

createFileCreator

private static FanOutOneBlockAsyncDFSOutputHelper.FileCreator createFileCreator()
                                                                         throws NoSuchMethodException

Throws:

NoSuchMethodException

loadShouldReplicateFlag

private static org.apache.hadoop.fs.CreateFlag loadShouldReplicateFlag()

beginFileLease

static void beginFileLease(org.apache.hadoop.hdfs.DFSClient client,
                           long inodeId)

endFileLease

static void endFileLease(org.apache.hadoop.hdfs.DFSClient client,
                         long inodeId)

createChecksum

static org.apache.hadoop.util.DataChecksum createChecksum(org.apache.hadoop.hdfs.DFSClient client)

getStatus

static org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.Status getStatus(org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.PipelineAckProto ack)

processWriteBlockResponse

private static void processWriteBlockResponse(org.apache.hbase.thirdparty.io.netty.channel.Channel channel,
                                              org.apache.hadoop.hdfs.protocol.DatanodeInfo dnInfo,
                                              org.apache.hbase.thirdparty.io.netty.util.concurrent.Promise<org.apache.hbase.thirdparty.io.netty.channel.Channel> promise,
                                              int timeoutMs)

requestWriteBlock

private static void requestWriteBlock(org.apache.hbase.thirdparty.io.netty.channel.Channel channel,
                                      org.apache.hadoop.fs.StorageType storageType,
                                      org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.OpWriteBlockProto.Builder writeBlockProtoBuilder)
                               throws IOException

Throws:

IOException

initialize

private static void initialize(org.apache.hadoop.conf.Configuration conf,
                               org.apache.hbase.thirdparty.io.netty.channel.Channel channel,
                               org.apache.hadoop.hdfs.protocol.DatanodeInfo dnInfo,
                               org.apache.hadoop.fs.StorageType storageType,
                               org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.OpWriteBlockProto.Builder writeBlockProtoBuilder,
                               int timeoutMs,
                               org.apache.hadoop.hdfs.DFSClient client,
                               org.apache.hadoop.security.token.Token<org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier> accessToken,
                               org.apache.hbase.thirdparty.io.netty.util.concurrent.Promise<org.apache.hbase.thirdparty.io.netty.channel.Channel> promise)
                        throws IOException

Throws:

IOException

connectToDataNodes

private static List<org.apache.hbase.thirdparty.io.netty.util.concurrent.Future<org.apache.hbase.thirdparty.io.netty.channel.Channel>> connectToDataNodes(org.apache.hadoop.conf.Configuration conf,
                                                                                                                                                          org.apache.hadoop.hdfs.DFSClient client,
                                                                                                                                                          String clientName,
                                                                                                                                                          org.apache.hadoop.hdfs.protocol.LocatedBlock locatedBlock,
                                                                                                                                                          long maxBytesRcvd,
                                                                                                                                                          long latestGS,
                                                                                                                                                          org.apache.hadoop.hdfs.protocol.datatransfer.BlockConstructionStage stage,
                                                                                                                                                          org.apache.hadoop.util.DataChecksum summer,
                                                                                                                                                          org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup eventLoopGroup,
                                                                                                                                                          Class<? extends org.apache.hbase.thirdparty.io.netty.channel.Channel> channelClass)

getCreateFlags

private static org.apache.hadoop.io.EnumSetWritable<org.apache.hadoop.fs.CreateFlag> getCreateFlags(boolean overwrite)

createOutput

private static FanOutOneBlockAsyncDFSOutput createOutput(org.apache.hadoop.hdfs.DistributedFileSystem dfs,
                                                         String src,
                                                         boolean overwrite,
                                                         boolean createParent,
                                                         short replication,
                                                         long blockSize,
                                                         org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup eventLoopGroup,
                                                         Class<? extends org.apache.hbase.thirdparty.io.netty.channel.Channel> channelClass,
                                                         StreamSlowMonitor monitor)
                                                  throws IOException

Throws:

IOException

createOutput

public static FanOutOneBlockAsyncDFSOutput createOutput(org.apache.hadoop.hdfs.DistributedFileSystem dfs,
                                                        org.apache.hadoop.fs.Path f,
                                                        boolean overwrite,
                                                        boolean createParent,
                                                        short replication,
                                                        long blockSize,
                                                        org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup eventLoopGroup,
                                                        Class<? extends org.apache.hbase.thirdparty.io.netty.channel.Channel> channelClass,
                                                        StreamSlowMonitor monitor)
                                                 throws IOException

Create a FanOutOneBlockAsyncDFSOutput. The method maybe blocked so do not call it inside an EventLoop.

Throws:

IOException

shouldRetryCreate

public static boolean shouldRetryCreate(org.apache.hadoop.ipc.RemoteException e)

completeFile

static void completeFile(org.apache.hadoop.hdfs.DFSClient client,
                         org.apache.hadoop.hdfs.protocol.ClientProtocol namenode,
                         String src,
                         String clientName,
                         org.apache.hadoop.hdfs.protocol.ExtendedBlock block,
                         long fileId)

sleepIgnoreInterrupt

static void sleepIgnoreInterrupt(int retry)