org.apache.hadoop.hbase.client

Class MetricsConnection

java.lang.Object

org.apache.hadoop.hbase.client.MetricsConnection

All Implemented Interfaces:

StatisticTrackable

@InterfaceAudience.Private
public final class MetricsConnection
extends Object
implements StatisticTrackable

This class is for maintaining the various connection statistics and publishing them through the metrics interfaces. This class manages its own MetricRegistry and JmxReporter so as to not conflict with other uses of Yammer Metrics within the client application. Calling getMetricsConnection(Configuration, String, Supplier, Supplier) implicitly creates and "starts" instances of these classes; be sure to call deleteMetricsConnection(String) to terminate the thread pools they allocate. The metrics reporter will be shutdown shutdown() when all connections within this metrics instances are closed.

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
static class	MetricsConnection.CallStats A container class for collecting details about the RPC call as it percolates.
protected static class	MetricsConnection.CallTracker
private static interface	MetricsConnection.NewMetric<T> A lambda for dispatching to the appropriate metric factory method
protected static class	MetricsConnection.RegionStats
protected static class	MetricsConnection.RunnerStats

Field Summary

Fields

Modifier and Type	Field and Description
private MetricsConnection.CallTracker	appendTracker
private List<Supplier<ThreadPoolExecutor>>	batchPools
private static String	CACHE_BASE
private ConcurrentMap<String,com.codahale.metrics.Counter>	cacheDroppingExceptions
private static int	CAPACITY Anticipated number of metric entries
static String	CLIENT_SIDE_METRICS_ENABLED_KEY Set this key to true to enable metrics collection of client requests.
static String	CLIENT_SIDE_TABLE_METRICS_ENABLED_KEY Set this key to true to enable table metrics collection of client requests.
private static String	CLIENT_SVC
private static String	CNT_BASE
private static int	CONCURRENCY_LEVEL Anticipated number of concurrent accessor threads
private com.codahale.metrics.Histogram	concurrentCallsPerServerHist
private com.codahale.metrics.Counter	connectionCount
private MetricsConnection.NewMetric<com.codahale.metrics.Counter>	counterFactory
private MetricsConnection.CallTracker	deleteTracker
private static String	DRTN_BASE
private static String	FAILURE_CNT_BASE
private MetricsConnection.CallTracker	getTracker
private static String	HEAP_BASE
private com.codahale.metrics.Counter	hedgedReadOps
private com.codahale.metrics.Counter	hedgedReadWin
private MetricsConnection.NewMetric<com.codahale.metrics.Histogram>	histogramFactory
private MetricsConnection.CallTracker	incrementTracker
private static float	LOAD_FACTOR Default load factor from HashMap.DEFAULT_LOAD_FACTOR
private static String	LOCAL_EXCEPTION_CNT_BASE
private static String	MEMLOAD_BASE
private com.codahale.metrics.Counter	metaCacheHits
private com.codahale.metrics.Counter	metaCacheMisses
private com.codahale.metrics.Counter	metaCacheNumClearRegion
private com.codahale.metrics.Counter	metaCacheNumClearServer
private List<Supplier<ThreadPoolExecutor>>	metaPools
private static ConcurrentMap<String,MetricsConnection>	METRICS_INSTANCES
static String	METRICS_SCOPE_KEY Set to specify a custom scope for the metrics published through MetricsConnection.
private MetricsConnection.CallTracker	multiTracker
private static String	NS_LOOKUPS
private static String	NS_LOOKUPS_FAILED
private com.codahale.metrics.Counter	nsLookups
private com.codahale.metrics.Counter	nsLookupsFailed
private com.codahale.metrics.Histogram	numActionsPerServerHist
private com.codahale.metrics.Timer	overloadedBackoffTimer
private MetricsConnection.CallTracker	putTracker
private com.codahale.metrics.MetricRegistry	registry
private static String	REMOTE_EXCEPTION_CNT_BASE
private com.codahale.metrics.JmxReporter	reporter
private static String	REQ_BASE
private static String	RESP_BASE
private ConcurrentMap<String,com.codahale.metrics.Counter>	rpcCounters
private ConcurrentMap<String,com.codahale.metrics.Histogram>	rpcHistograms
private ConcurrentMap<String,com.codahale.metrics.Timer>	rpcTimers
private MetricsConnection.RunnerStats	runnerStats
private MetricsConnection.CallTracker	scanTracker
private String	scope
private ConcurrentHashMap<ServerName,ConcurrentMap<byte[],MetricsConnection.RegionStats>>	serverStats
private boolean	tableMetricsEnabled
private MetricsConnection.NewMetric<com.codahale.metrics.Timer>	timerFactory
private static String	TOTAL_EXCEPTION_CNT
private static String	UNKNOWN_EXCEPTION

Constructor Summary

Constructors

Modifier	Constructor and Description
private	MetricsConnection(org.apache.hadoop.conf.Configuration conf, String scope, Supplier<ThreadPoolExecutor> batchPool, Supplier<ThreadPoolExecutor> metaPool)

Method Summary

Modifier and Type	Method and Description
private void	addThreadPools(Supplier<ThreadPoolExecutor> batchPool, Supplier<ThreadPoolExecutor> metaPool) Add thread pools of additional connections to the metrics
private void	decrConnectionCount() Decrement the connection count of the metrics within a scope
(package private) static void	deleteMetricsConnection(String scope)
MetricsConnection.CallTracker	getAppendTracker() appendTracker metric
long	getConnectionCount() Return the connection count of the metrics within a scope
MetricsConnection.CallTracker	getDeleteTracker() deleteTracker metric
(package private) String	getExecutorPoolName()
MetricsConnection.CallTracker	getGetTracker() getTracker metric
com.codahale.metrics.Counter	getHedgedReadOps() hedgedReadOps metric
com.codahale.metrics.Counter	getHedgedReadWin() hedgedReadWin metric
MetricsConnection.CallTracker	getIncrementTracker() incrementTracker metric
long	getMetaCacheMisses()
com.codahale.metrics.Counter	getMetaCacheNumClearRegion() metaCacheNumClearRegion metric
com.codahale.metrics.Counter	getMetaCacheNumClearServer() metaCacheNumClearServer metric
(package private) String	getMetaPoolName()
private <T> T	getMetric(String key, ConcurrentMap<String,T> map, MetricsConnection.NewMetric<T> factory) Get a metric for key from map, or create it with factory.
(package private) com.codahale.metrics.MetricRegistry	getMetricRegistry()
(package private) static MetricsConnection	getMetricsConnection(org.apache.hadoop.conf.Configuration conf, String scope, Supplier<ThreadPoolExecutor> batchPool, Supplier<ThreadPoolExecutor> metaPool)
String	getMetricScope() scope of the metrics object
MetricsConnection.CallTracker	getMultiTracker() multiTracker metric
com.codahale.metrics.Histogram	getNumActionsPerServerHist() numActionsPerServerHist metric
MetricsConnection.CallTracker	getPutTracker() putTracker metric
ConcurrentMap<String,com.codahale.metrics.Counter>	getRpcCounters() rpcCounters metric
ConcurrentMap<String,com.codahale.metrics.Histogram>	getRpcHistograms() rpcHistograms metric
ConcurrentMap<String,com.codahale.metrics.Timer>	getRpcTimers() rpcTimers metric
MetricsConnection.RunnerStats	getRunnerStats() runnerStats metric
MetricsConnection.CallTracker	getScanTracker() scanTracker metric
(package private) static String	getScope(org.apache.hadoop.conf.Configuration conf, String clusterId, Object connectionObj) Returns the scope for a MetricsConnection based on the configured METRICS_SCOPE_KEY or by generating a default from the passed clusterId and connectionObj's hashCode.
ConcurrentHashMap<ServerName,ConcurrentMap<byte[],MetricsConnection.RegionStats>>	getServerStats() serverStats metric
void	incrCacheDroppingExceptions(Object exception)
private void	incrConnectionCount() Increment the connection count of the metrics within a scope
void	incrDelayRunnersAndUpdateDelayInterval(long interval) Increment the number of delay runner counts and update delay interval of delay runner.
void	incrementServerOverloadedBackoffTime(long time, TimeUnit timeUnit) Update the overloaded backoff time
void	incrHedgedReadOps() Increment the number of hedged read that have occurred.
void	incrHedgedReadWin() Increment the number of hedged read returned faster than the original read.
void	incrMetaCacheHit() Increment the number of meta cache hits.
void	incrMetaCacheMiss() Increment the number of meta cache misses.
void	incrMetaCacheNumClearRegion() Increment the number of meta cache drops requested for individual region.
void	incrMetaCacheNumClearRegion(int count) Increment the number of meta cache drops requested for individual region.
void	incrMetaCacheNumClearServer() Increment the number of meta cache drops requested for entire RegionServer.
void	incrNormalRunners() Increment the number of normal runner counts.
void	incrNsLookups()
void	incrNsLookupsFailed()
static MetricsConnection.CallStats	newCallStats() Produce an instance of MetricsConnection.CallStats for clients to attach to RPCs.
private void	shutdown()
void	updateRegionStats(ServerName serverName, byte[] regionName, RegionLoadStats stats) Update stats per region.
void	updateRpc(org.apache.hbase.thirdparty.com.google.protobuf.Descriptors.MethodDescriptor method, TableName tableName, org.apache.hbase.thirdparty.com.google.protobuf.Message param, MetricsConnection.CallStats stats, Throwable e) Report RPC context to metrics system.
private void	updateRpcGeneric(String methodName, MetricsConnection.CallStats stats) Update call stats for non-critical-path methods
void	updateServerStats(ServerName serverName, byte[] regionName, Object r)
private void	updateTableMetric(String methodName, TableName tableName, MetricsConnection.CallStats stats, Throwable e) Report table rpc context to metrics system.

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

METRICS_INSTANCES

private static final ConcurrentMap<String,MetricsConnection> METRICS_INSTANCES

CLIENT_SIDE_METRICS_ENABLED_KEY

public static final String CLIENT_SIDE_METRICS_ENABLED_KEY

Set this key to true to enable metrics collection of client requests.

See Also:

Constant Field Values

CLIENT_SIDE_TABLE_METRICS_ENABLED_KEY

public static final String CLIENT_SIDE_TABLE_METRICS_ENABLED_KEY

Set this key to true to enable table metrics collection of client requests.

See Also:

Constant Field Values

METRICS_SCOPE_KEY

public static final String METRICS_SCOPE_KEY

Set to specify a custom scope for the metrics published through MetricsConnection. The scope is added to JMX MBean objectName, and defaults to a combination of the Connection's clusterId and hashCode. For example, a default value for a connection to cluster "foo" might be "foo-7d9d0818", where "7d9d0818" is the hashCode of the underlying AsyncConnectionImpl. Users may set this key to give a more contextual name for this scope. For example, one might want to differentiate a read connection from a write connection by setting the scopes to "foo-read" and "foo-write" respectively. Scope is the only thing that lends any uniqueness to the metrics. Care should be taken to avoid using the same scope for multiple Connections, otherwise the metrics may aggregate in unforeseen ways.

See Also:

Constant Field Values

CNT_BASE

private static final String CNT_BASE

See Also:

Constant Field Values

FAILURE_CNT_BASE

private static final String FAILURE_CNT_BASE

See Also:

Constant Field Values

TOTAL_EXCEPTION_CNT

private static final String TOTAL_EXCEPTION_CNT

See Also:

Constant Field Values

LOCAL_EXCEPTION_CNT_BASE

private static final String LOCAL_EXCEPTION_CNT_BASE

See Also:

Constant Field Values

REMOTE_EXCEPTION_CNT_BASE

private static final String REMOTE_EXCEPTION_CNT_BASE

See Also:

Constant Field Values

DRTN_BASE

private static final String DRTN_BASE

See Also:

Constant Field Values

REQ_BASE

private static final String REQ_BASE

See Also:

Constant Field Values

RESP_BASE

private static final String RESP_BASE

See Also:

Constant Field Values

MEMLOAD_BASE

private static final String MEMLOAD_BASE

See Also:

Constant Field Values

HEAP_BASE

private static final String HEAP_BASE

See Also:

Constant Field Values

CACHE_BASE

private static final String CACHE_BASE

See Also:

Constant Field Values

UNKNOWN_EXCEPTION

private static final String UNKNOWN_EXCEPTION

See Also:

Constant Field Values

NS_LOOKUPS

private static final String NS_LOOKUPS

See Also:

Constant Field Values

NS_LOOKUPS_FAILED

private static final String NS_LOOKUPS_FAILED

See Also:

Constant Field Values

CLIENT_SVC

private static final String CLIENT_SVC

serverStats

private ConcurrentHashMap<ServerName,ConcurrentMap<byte[],MetricsConnection.RegionStats>> serverStats

CAPACITY

private static final int CAPACITY

Anticipated number of metric entries

See Also:

Constant Field Values

LOAD_FACTOR

private static final float LOAD_FACTOR

Default load factor from HashMap.DEFAULT_LOAD_FACTOR

See Also:

Constant Field Values

CONCURRENCY_LEVEL

private static final int CONCURRENCY_LEVEL

Anticipated number of concurrent accessor threads

See Also:

Constant Field Values

registry

private final com.codahale.metrics.MetricRegistry registry

reporter

private final com.codahale.metrics.JmxReporter reporter

scope

private final String scope

tableMetricsEnabled

private final boolean tableMetricsEnabled

timerFactory

private final MetricsConnection.NewMetric<com.codahale.metrics.Timer> timerFactory

histogramFactory

private final MetricsConnection.NewMetric<com.codahale.metrics.Histogram> histogramFactory

counterFactory

private final MetricsConnection.NewMetric<com.codahale.metrics.Counter> counterFactory

batchPools

private final List<Supplier<ThreadPoolExecutor>> batchPools

metaPools

private final List<Supplier<ThreadPoolExecutor>> metaPools

connectionCount

private final com.codahale.metrics.Counter connectionCount

metaCacheHits

private final com.codahale.metrics.Counter metaCacheHits

metaCacheMisses

private final com.codahale.metrics.Counter metaCacheMisses

getTracker

private final MetricsConnection.CallTracker getTracker

scanTracker

private final MetricsConnection.CallTracker scanTracker

appendTracker

private final MetricsConnection.CallTracker appendTracker

deleteTracker

private final MetricsConnection.CallTracker deleteTracker

incrementTracker

private final MetricsConnection.CallTracker incrementTracker

putTracker

private final MetricsConnection.CallTracker putTracker

multiTracker

private final MetricsConnection.CallTracker multiTracker

runnerStats

private final MetricsConnection.RunnerStats runnerStats

metaCacheNumClearServer

private final com.codahale.metrics.Counter metaCacheNumClearServer

metaCacheNumClearRegion

private final com.codahale.metrics.Counter metaCacheNumClearRegion

hedgedReadOps

private final com.codahale.metrics.Counter hedgedReadOps

hedgedReadWin

private final com.codahale.metrics.Counter hedgedReadWin

concurrentCallsPerServerHist

private final com.codahale.metrics.Histogram concurrentCallsPerServerHist

numActionsPerServerHist

private final com.codahale.metrics.Histogram numActionsPerServerHist

nsLookups

private final com.codahale.metrics.Counter nsLookups

nsLookupsFailed

private final com.codahale.metrics.Counter nsLookupsFailed

overloadedBackoffTimer

private final com.codahale.metrics.Timer overloadedBackoffTimer

rpcTimers

private final ConcurrentMap<String,com.codahale.metrics.Timer> rpcTimers

rpcHistograms

private final ConcurrentMap<String,com.codahale.metrics.Histogram> rpcHistograms

cacheDroppingExceptions

private final ConcurrentMap<String,com.codahale.metrics.Counter> cacheDroppingExceptions

rpcCounters

private final ConcurrentMap<String,com.codahale.metrics.Counter> rpcCounters

Constructor Detail

MetricsConnection

private MetricsConnection(org.apache.hadoop.conf.Configuration conf,
                          String scope,
                          Supplier<ThreadPoolExecutor> batchPool,
                          Supplier<ThreadPoolExecutor> metaPool)

Method Detail

getMetricsConnection

static MetricsConnection getMetricsConnection(org.apache.hadoop.conf.Configuration conf,
                                              String scope,
                                              Supplier<ThreadPoolExecutor> batchPool,
                                              Supplier<ThreadPoolExecutor> metaPool)

deleteMetricsConnection

static void deleteMetricsConnection(String scope)

getScope

static String getScope(org.apache.hadoop.conf.Configuration conf,
                       String clusterId,
                       Object connectionObj)

Returns the scope for a MetricsConnection based on the configured METRICS_SCOPE_KEY or by generating a default from the passed clusterId and connectionObj's hashCode.

Parameters:

conf - configuration for the connection

clusterId - clusterId for the connection

connectionObj - either a Connection or AsyncConnectionImpl, the instance creating this MetricsConnection.

updateServerStats

public void updateServerStats(ServerName serverName,
                              byte[] regionName,
                              Object r)

updateRegionStats

public void updateRegionStats(ServerName serverName,
                              byte[] regionName,
                              RegionLoadStats stats)

Description copied from interface: StatisticTrackable

Update stats per region.

Specified by:

updateRegionStats in interface StatisticTrackable

getExecutorPoolName

final String getExecutorPoolName()

getMetaPoolName

final String getMetaPoolName()

getMetricRegistry

com.codahale.metrics.MetricRegistry getMetricRegistry()

getMetricScope

public String getMetricScope()

scope of the metrics object

getServerStats

public ConcurrentHashMap<ServerName,ConcurrentMap<byte[],MetricsConnection.RegionStats>> getServerStats()

serverStats metric

getRunnerStats

public MetricsConnection.RunnerStats getRunnerStats()

runnerStats metric

getMetaCacheNumClearServer

public com.codahale.metrics.Counter getMetaCacheNumClearServer()

metaCacheNumClearServer metric

getMetaCacheNumClearRegion

public com.codahale.metrics.Counter getMetaCacheNumClearRegion()

metaCacheNumClearRegion metric

getHedgedReadOps

public com.codahale.metrics.Counter getHedgedReadOps()

hedgedReadOps metric

getHedgedReadWin

public com.codahale.metrics.Counter getHedgedReadWin()

hedgedReadWin metric

getNumActionsPerServerHist

public com.codahale.metrics.Histogram getNumActionsPerServerHist()

numActionsPerServerHist metric

getRpcCounters

public ConcurrentMap<String,com.codahale.metrics.Counter> getRpcCounters()

rpcCounters metric

getRpcTimers

public ConcurrentMap<String,com.codahale.metrics.Timer> getRpcTimers()

rpcTimers metric

getRpcHistograms

public ConcurrentMap<String,com.codahale.metrics.Histogram> getRpcHistograms()

rpcHistograms metric

getGetTracker

public MetricsConnection.CallTracker getGetTracker()

getTracker metric

getScanTracker

public MetricsConnection.CallTracker getScanTracker()

scanTracker metric

getMultiTracker

public MetricsConnection.CallTracker getMultiTracker()

multiTracker metric

getAppendTracker

public MetricsConnection.CallTracker getAppendTracker()

appendTracker metric

getDeleteTracker

public MetricsConnection.CallTracker getDeleteTracker()

deleteTracker metric

getIncrementTracker

public MetricsConnection.CallTracker getIncrementTracker()

incrementTracker metric

getPutTracker

public MetricsConnection.CallTracker getPutTracker()

putTracker metric

newCallStats

public static MetricsConnection.CallStats newCallStats()

Produce an instance of MetricsConnection.CallStats for clients to attach to RPCs.

incrMetaCacheHit

public void incrMetaCacheHit()

Increment the number of meta cache hits.

incrMetaCacheMiss

public void incrMetaCacheMiss()

Increment the number of meta cache misses.

getMetaCacheMisses

public long getMetaCacheMisses()

incrMetaCacheNumClearServer

public void incrMetaCacheNumClearServer()

Increment the number of meta cache drops requested for entire RegionServer.

incrMetaCacheNumClearRegion

public void incrMetaCacheNumClearRegion()

Increment the number of meta cache drops requested for individual region.

incrMetaCacheNumClearRegion

public void incrMetaCacheNumClearRegion(int count)

Increment the number of meta cache drops requested for individual region.

incrHedgedReadOps

public void incrHedgedReadOps()

Increment the number of hedged read that have occurred.

incrHedgedReadWin

public void incrHedgedReadWin()

Increment the number of hedged read returned faster than the original read.

incrNormalRunners

public void incrNormalRunners()

Increment the number of normal runner counts.

incrDelayRunnersAndUpdateDelayInterval

public void incrDelayRunnersAndUpdateDelayInterval(long interval)

Increment the number of delay runner counts and update delay interval of delay runner.

incrementServerOverloadedBackoffTime

public void incrementServerOverloadedBackoffTime(long time,
                                                 TimeUnit timeUnit)

Update the overloaded backoff time

getConnectionCount

public long getConnectionCount()

Return the connection count of the metrics within a scope

incrConnectionCount

private void incrConnectionCount()

Increment the connection count of the metrics within a scope

decrConnectionCount

private void decrConnectionCount()

Decrement the connection count of the metrics within a scope

addThreadPools

private void addThreadPools(Supplier<ThreadPoolExecutor> batchPool,
                            Supplier<ThreadPoolExecutor> metaPool)

Add thread pools of additional connections to the metrics

getMetric

private <T> T getMetric(String key,
                        ConcurrentMap<String,T> map,
                        MetricsConnection.NewMetric<T> factory)

Get a metric for key from map, or create it with factory.

updateRpcGeneric

private void updateRpcGeneric(String methodName,
                              MetricsConnection.CallStats stats)

Update call stats for non-critical-path methods

shutdown

private void shutdown()

updateRpc

public void updateRpc(org.apache.hbase.thirdparty.com.google.protobuf.Descriptors.MethodDescriptor method,
                      TableName tableName,
                      org.apache.hbase.thirdparty.com.google.protobuf.Message param,
                      MetricsConnection.CallStats stats,
                      Throwable e)

Report RPC context to metrics system.

updateTableMetric

private void updateTableMetric(String methodName,
                               TableName tableName,
                               MetricsConnection.CallStats stats,
                               Throwable e)

Report table rpc context to metrics system.

incrCacheDroppingExceptions

public void incrCacheDroppingExceptions(Object exception)

incrNsLookups

public void incrNsLookups()

incrNsLookupsFailed

public void incrNsLookupsFailed()