org.apache.hadoop.hbase.util

Class LoadTestTool

java.lang.Object

org.apache.hadoop.hbase.util.AbstractHBaseTool

org.apache.hadoop.hbase.util.LoadTestTool

All Implemented Interfaces:

org.apache.hadoop.conf.Configurable, org.apache.hadoop.util.Tool

@InterfaceAudience.LimitedPrivate(value="Tools")
public class LoadTestTool
extends org.apache.hadoop.hbase.util.AbstractHBaseTool

A command-line utility that reads, writes, and verifies data. Unlike PerformanceEvaluation, this tool validates the data written, and supports simultaneously writing and reading the same set of keys.

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
(package private) class	LoadTestTool.WorkerThread

Nested classes/interfaces inherited from class org.apache.hadoop.hbase.util.AbstractHBaseTool

org.apache.hadoop.hbase.util.AbstractHBaseTool.OptionsOrderComparator

Field Summary

Fields

Modifier and Type	Field and Description
private String	authnFileName
protected org.apache.hadoop.hbase.regionserver.BloomType	bloomType
protected org.apache.hadoop.hbase.io.crypto.Cipher	cipher
protected org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine	cmd This will be removed as we factor out the dependency on command line
private static String	COLON
protected org.apache.hadoop.hbase.io.compress.Compression.Algorithm	compressAlgo
protected org.apache.hadoop.hbase.io.encoding.DataBlockEncoding	dataBlockEncodingAlgo
protected static int	DEFAULT_DATA_SIZE The default data size if not specified
static int	DEFAULT_NUM_REGIONS_PER_SERVER
protected static int	DEFAULT_NUM_THREADS The number of reader/writer threads if not specified
protected static long	DEFAULT_START_KEY
protected static String	DEFAULT_TABLE_NAME Table name to use of not overridden on the command line
protected boolean	deferredLogFlush
protected long	endKey
private byte[][]	families Column families for the test
protected boolean	ignoreConflicts
private boolean	inMemoryCF
protected boolean	isBatchUpdate
protected boolean	isInitOnly
protected boolean	isMultiPut
protected boolean	isRead
protected boolean	isSkipInit
protected boolean	isUpdate
protected boolean	isVerbose
protected boolean	isWrite
private int	keyWindow
private static org.slf4j.Logger	LOG
protected int	maxColDataSize
protected int	maxColsPerKey
private int	maxReadErrors
protected int	minColDataSize
protected int	minColsPerKey
private int	mobThreshold
private int	multiGetBatchSize
protected static String	NUM_TABLES
private int	numReaderThreads
private int	numRegionsPerServer
private int	numTables
protected int	numUpdaterThreads
protected int	numWriterThreads
protected static String	OPT_BATCHUPDATE
static String	OPT_BLOOM
static String	OPT_BLOOM_PARAM
static String	OPT_COLUMN_FAMILIES
static String	OPT_COMPRESSION
static String	OPT_DEFERRED_LOG_FLUSH
static String	OPT_DEFERRED_LOG_FLUSH_USAGE
static String	OPT_ENCRYPTION
protected static String	OPT_ENCRYPTION_USAGE
static String	OPT_GENERATOR
static String	OPT_GENERATOR_USAGE
protected static String	OPT_INIT_ONLY
static String	OPT_INMEMORY
protected static String	OPT_KEY_WINDOW
protected static String	OPT_MAX_READ_ERRORS
static String	OPT_MOB_THRESHOLD
protected static String	OPT_MOB_THRESHOLD_USAGE
static String	OPT_MULTIGET
static String	OPT_MULTIPUT
protected static String	OPT_NUM_KEYS
static String	OPT_NUM_REGIONS_PER_SERVER
protected static String	OPT_NUM_REGIONS_PER_SERVER_USAGE
protected static String	OPT_READ
static String	OPT_READER
static String	OPT_READER_USAGE
static String	OPT_REGION_REPLICA_ID
protected static String	OPT_REGION_REPLICA_ID_USAGE
static String	OPT_REGION_REPLICATION
protected static String	OPT_REGION_REPLICATION_USAGE
protected static String	OPT_SKIP_INIT
protected static String	OPT_START_KEY
static String	OPT_TABLE_NAME
protected static String	OPT_UPDATE
static String	OPT_UPDATER
static String	OPT_UPDATER_USAGE
protected static String	OPT_USAGE_BLOOM
protected static String	OPT_USAGE_COMPRESSION
static String	OPT_USAGE_IN_MEMORY
protected static String	OPT_USAGE_LOAD Usage string for the load option
protected static String	OPT_USAGE_READ Usage string for the read option
protected static String	OPT_USAGE_UPDATE Usage string for the update option
protected static String	OPT_VERBOSE
protected static String	OPT_WRITE
static String	OPT_WRITER
static String	OPT_WRITER_USAGE
protected static String	OPT_ZK_PARENT_NODE
protected static String	OPT_ZK_QUORUM
protected MultiThreadedReader	readerThreads
private int	regionReplicaId
private int	regionReplication
protected long	startKey
private String	superUser
private org.apache.hadoop.hbase.TableName	tableName Table name for the test
protected AtomicReference<Throwable>	thrown
protected int	updatePercent
protected MultiThreadedUpdater	updaterThreads
private String	userNames
private org.apache.hadoop.hbase.security.User	userOwner
private int	verifyPercent
protected MultiThreadedWriter	writerThreads

Fields inherited from class org.apache.hadoop.hbase.util.AbstractHBaseTool

cmdLineArgs, conf, EXIT_FAILURE, EXIT_SUCCESS, LONG_HELP_OPTION, options, SHORT_HELP_OPTION

Constructor Summary

Constructors

Constructor and Description
LoadTestTool()

Method Summary

Modifier and Type	Method and Description
private void	addAuthInfoToConf(Properties authConfig, org.apache.hadoop.conf.Configuration conf, String owner, String userList)
protected void	addOptions()
protected void	applyColumnFamilyOptions(org.apache.hadoop.hbase.TableName tableName, byte[][] columnFamilies) Apply column family options such as Bloom filters, compression, and data block encoding.
private void	checkForErrors() Check for errors in the writer threads.
protected int	doWork()
byte[][]	getColumnFamilies()
private LoadTestDataGenerator	getLoadGeneratorInstance(String clazzName)
private MultiThreadedReader	getMultiThreadedReaderInstance(String clazzName, LoadTestDataGenerator dataGen)
private MultiThreadedUpdater	getMultiThreadedUpdaterInstance(String clazzName, LoadTestDataGenerator dataGen)
private MultiThreadedWriter	getMultiThreadedWriterInstance(String clazzName, LoadTestDataGenerator dataGen)
protected int	getNumThreads(String numThreadsStr)
void	initTestTable()
protected int	loadTable()
static void	main(String[] args)
protected org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLineParser	newParser()
private int	parallelLoadTables() When NUM_TABLES is specified, the function starts multiple worker threads which individually start a LoadTestTool instance to load a table.
private void	parseColumnFamilyOptions(org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine cmd)
protected void	processOptions(org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine cmd)
protected String[]	splitColonSeparated(String option, int minNumCols, int maxNumCols)
private void	workerThreadError(Throwable t)

Methods inherited from class org.apache.hadoop.hbase.util.AbstractHBaseTool

addOption, addOptNoArg, addOptNoArg, addOptWithArg, addOptWithArg, addRequiredOption, addRequiredOptWithArg, addRequiredOptWithArg, doStaticMain, getConf, getOptionAsDouble, getOptionAsInt, getOptionAsInt, getOptionAsLong, getOptionAsLong, parseArgs, parseInt, parseLong, printUsage, printUsage, processOldArgs, run, setConf

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

LOG

private static final org.slf4j.Logger LOG

COLON

private static final String COLON

See Also:

Constant Field Values

tableName

private org.apache.hadoop.hbase.TableName tableName

Table name for the test

families

private byte[][] families

Column families for the test

DEFAULT_TABLE_NAME

protected static final String DEFAULT_TABLE_NAME

Table name to use of not overridden on the command line

See Also:

Constant Field Values

DEFAULT_DATA_SIZE

protected static final int DEFAULT_DATA_SIZE

The default data size if not specified

See Also:

Constant Field Values

DEFAULT_NUM_THREADS

protected static final int DEFAULT_NUM_THREADS

The number of reader/writer threads if not specified

See Also:

Constant Field Values

OPT_USAGE_LOAD

protected static final String OPT_USAGE_LOAD

Usage string for the load option

See Also:

Constant Field Values

OPT_USAGE_READ

protected static final String OPT_USAGE_READ

Usage string for the read option

See Also:

Constant Field Values

OPT_USAGE_UPDATE

protected static final String OPT_USAGE_UPDATE

Usage string for the update option

See Also:

Constant Field Values

OPT_USAGE_BLOOM

protected static final String OPT_USAGE_BLOOM

OPT_USAGE_COMPRESSION

protected static final String OPT_USAGE_COMPRESSION

OPT_VERBOSE

protected static final String OPT_VERBOSE

See Also:

Constant Field Values

OPT_BLOOM

public static final String OPT_BLOOM

See Also:

Constant Field Values

OPT_BLOOM_PARAM

public static final String OPT_BLOOM_PARAM

See Also:

Constant Field Values

OPT_COMPRESSION

public static final String OPT_COMPRESSION

See Also:

Constant Field Values

OPT_DEFERRED_LOG_FLUSH

public static final String OPT_DEFERRED_LOG_FLUSH

See Also:

Constant Field Values

OPT_DEFERRED_LOG_FLUSH_USAGE

public static final String OPT_DEFERRED_LOG_FLUSH_USAGE

See Also:

Constant Field Values

OPT_INMEMORY

public static final String OPT_INMEMORY

See Also:

Constant Field Values

OPT_USAGE_IN_MEMORY

public static final String OPT_USAGE_IN_MEMORY

See Also:

Constant Field Values

OPT_GENERATOR

public static final String OPT_GENERATOR

See Also:

Constant Field Values

OPT_GENERATOR_USAGE

public static final String OPT_GENERATOR_USAGE

See Also:

Constant Field Values

OPT_WRITER

public static final String OPT_WRITER

See Also:

Constant Field Values

OPT_WRITER_USAGE

public static final String OPT_WRITER_USAGE

See Also:

Constant Field Values

OPT_UPDATER

public static final String OPT_UPDATER

See Also:

Constant Field Values

OPT_UPDATER_USAGE

public static final String OPT_UPDATER_USAGE

See Also:

Constant Field Values

OPT_READER

public static final String OPT_READER

See Also:

Constant Field Values

OPT_READER_USAGE

public static final String OPT_READER_USAGE

See Also:

Constant Field Values

OPT_KEY_WINDOW

protected static final String OPT_KEY_WINDOW

See Also:

Constant Field Values

OPT_WRITE

protected static final String OPT_WRITE

See Also:

Constant Field Values

OPT_MAX_READ_ERRORS

protected static final String OPT_MAX_READ_ERRORS

See Also:

Constant Field Values

OPT_MULTIPUT

public static final String OPT_MULTIPUT

See Also:

Constant Field Values

OPT_MULTIGET

public static final String OPT_MULTIGET

See Also:

Constant Field Values

OPT_NUM_KEYS

protected static final String OPT_NUM_KEYS

See Also:

Constant Field Values

OPT_READ

protected static final String OPT_READ

See Also:

Constant Field Values

OPT_START_KEY

protected static final String OPT_START_KEY

See Also:

Constant Field Values

OPT_TABLE_NAME

public static final String OPT_TABLE_NAME

See Also:

Constant Field Values

OPT_COLUMN_FAMILIES

public static final String OPT_COLUMN_FAMILIES

See Also:

Constant Field Values

OPT_ZK_QUORUM

protected static final String OPT_ZK_QUORUM

See Also:

Constant Field Values

OPT_ZK_PARENT_NODE

protected static final String OPT_ZK_PARENT_NODE

See Also:

Constant Field Values

OPT_SKIP_INIT

protected static final String OPT_SKIP_INIT

See Also:

Constant Field Values

OPT_INIT_ONLY

protected static final String OPT_INIT_ONLY

See Also:

Constant Field Values

NUM_TABLES

protected static final String NUM_TABLES

See Also:

Constant Field Values

OPT_BATCHUPDATE

protected static final String OPT_BATCHUPDATE

See Also:

Constant Field Values

OPT_UPDATE

protected static final String OPT_UPDATE

See Also:

Constant Field Values

OPT_ENCRYPTION

public static final String OPT_ENCRYPTION

See Also:

Constant Field Values

OPT_ENCRYPTION_USAGE

protected static final String OPT_ENCRYPTION_USAGE

OPT_NUM_REGIONS_PER_SERVER

public static final String OPT_NUM_REGIONS_PER_SERVER

See Also:

Constant Field Values

OPT_NUM_REGIONS_PER_SERVER_USAGE

protected static final String OPT_NUM_REGIONS_PER_SERVER_USAGE

See Also:

Constant Field Values

DEFAULT_NUM_REGIONS_PER_SERVER

public static int DEFAULT_NUM_REGIONS_PER_SERVER

OPT_REGION_REPLICATION

public static final String OPT_REGION_REPLICATION

See Also:

Constant Field Values

OPT_REGION_REPLICATION_USAGE

protected static final String OPT_REGION_REPLICATION_USAGE

See Also:

Constant Field Values

OPT_REGION_REPLICA_ID

public static final String OPT_REGION_REPLICA_ID

See Also:

Constant Field Values

OPT_REGION_REPLICA_ID_USAGE

protected static final String OPT_REGION_REPLICA_ID_USAGE

See Also:

Constant Field Values

OPT_MOB_THRESHOLD

public static final String OPT_MOB_THRESHOLD

See Also:

Constant Field Values

OPT_MOB_THRESHOLD_USAGE

protected static final String OPT_MOB_THRESHOLD_USAGE

See Also:

Constant Field Values

DEFAULT_START_KEY

protected static final long DEFAULT_START_KEY

See Also:

Constant Field Values

cmd

protected org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine cmd

This will be removed as we factor out the dependency on command line

writerThreads

protected MultiThreadedWriter writerThreads

readerThreads

protected MultiThreadedReader readerThreads

updaterThreads

protected MultiThreadedUpdater updaterThreads

startKey

protected long startKey

endKey

protected long endKey

isVerbose

protected boolean isVerbose

isWrite

protected boolean isWrite

isRead

protected boolean isRead

isUpdate

protected boolean isUpdate

deferredLogFlush

protected boolean deferredLogFlush

dataBlockEncodingAlgo

protected org.apache.hadoop.hbase.io.encoding.DataBlockEncoding dataBlockEncodingAlgo

compressAlgo

protected org.apache.hadoop.hbase.io.compress.Compression.Algorithm compressAlgo

bloomType

protected org.apache.hadoop.hbase.regionserver.BloomType bloomType

inMemoryCF

private boolean inMemoryCF

userOwner

private org.apache.hadoop.hbase.security.User userOwner

numWriterThreads

protected int numWriterThreads

minColsPerKey

protected int minColsPerKey

maxColsPerKey

protected int maxColsPerKey

minColDataSize

protected int minColDataSize

maxColDataSize

protected int maxColDataSize

isMultiPut

protected boolean isMultiPut

numUpdaterThreads

protected int numUpdaterThreads

updatePercent

protected int updatePercent

ignoreConflicts

protected boolean ignoreConflicts

isBatchUpdate

protected boolean isBatchUpdate

numReaderThreads

private int numReaderThreads

keyWindow

private int keyWindow

multiGetBatchSize

private int multiGetBatchSize

maxReadErrors

private int maxReadErrors

verifyPercent

private int verifyPercent

numTables

private int numTables

superUser

private String superUser

userNames

private String userNames

authnFileName

private String authnFileName

numRegionsPerServer

private int numRegionsPerServer

regionReplication

private int regionReplication

regionReplicaId

private int regionReplicaId

mobThreshold

private int mobThreshold

isSkipInit

protected boolean isSkipInit

isInitOnly

protected boolean isInitOnly

cipher

protected org.apache.hadoop.hbase.io.crypto.Cipher cipher

thrown

protected AtomicReference<Throwable> thrown

Constructor Detail

LoadTestTool

public LoadTestTool()

Method Detail

splitColonSeparated

protected String[] splitColonSeparated(String option,
                                       int minNumCols,
                                       int maxNumCols)

getNumThreads

protected int getNumThreads(String numThreadsStr)

getColumnFamilies

public byte[][] getColumnFamilies()

applyColumnFamilyOptions

protected void applyColumnFamilyOptions(org.apache.hadoop.hbase.TableName tableName,
                                        byte[][] columnFamilies)
                                 throws IOException

Apply column family options such as Bloom filters, compression, and data block encoding.

Throws:

IOException

addOptions

protected void addOptions()

Specified by:

addOptions in class org.apache.hadoop.hbase.util.AbstractHBaseTool

newParser

protected org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLineParser newParser()

Overrides:

newParser in class org.apache.hadoop.hbase.util.AbstractHBaseTool

processOptions

protected void processOptions(org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine cmd)

Specified by:

processOptions in class org.apache.hadoop.hbase.util.AbstractHBaseTool

parseColumnFamilyOptions

private void parseColumnFamilyOptions(org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine cmd)

initTestTable

public void initTestTable()
                   throws IOException

Throws:

IOException

doWork

protected int doWork()
              throws IOException

Specified by:

doWork in class org.apache.hadoop.hbase.util.AbstractHBaseTool

Throws:

IOException

loadTable

protected int loadTable()
                 throws IOException

Throws:

IOException

getLoadGeneratorInstance

private LoadTestDataGenerator getLoadGeneratorInstance(String clazzName)
                                                throws IOException

Throws:

IOException

getMultiThreadedWriterInstance

private MultiThreadedWriter getMultiThreadedWriterInstance(String clazzName,
                                                           LoadTestDataGenerator dataGen)
                                                    throws IOException

Throws:

IOException

getMultiThreadedUpdaterInstance

private MultiThreadedUpdater getMultiThreadedUpdaterInstance(String clazzName,
                                                             LoadTestDataGenerator dataGen)
                                                      throws IOException

Throws:

IOException

getMultiThreadedReaderInstance

private MultiThreadedReader getMultiThreadedReaderInstance(String clazzName,
                                                           LoadTestDataGenerator dataGen)
                                                    throws IOException

Throws:

IOException

main

public static void main(String[] args)

parallelLoadTables

private int parallelLoadTables()
                        throws IOException

When NUM_TABLES is specified, the function starts multiple worker threads which individually start a LoadTestTool instance to load a table. Each table name is in format <tn>_<index>. For example, "-tn test -num_tables 2" , table names will be "test_1", "test_2"

Throws:

IOException - if one of the load tasks is unable to complete

workerThreadError

private void workerThreadError(Throwable t)

checkForErrors

private void checkForErrors()
                     throws IOException

Check for errors in the writer threads. If any is found, rethrow it.

Throws:

IOException

addAuthInfoToConf

private void addAuthInfoToConf(Properties authConfig,
                               org.apache.hadoop.conf.Configuration conf,
                               String owner,
                               String userList)
                        throws IOException

Throws:

IOException