org.apache.hadoop.hbase.backup.example

Class TestZooKeeperTableArchiveClient

java.lang.Object

org.apache.hadoop.hbase.backup.example.TestZooKeeperTableArchiveClient

public class TestZooKeeperTableArchiveClient
extends Object

Spin up a small cluster and check that the hfiles of region are properly long-term archived as specified via the ZKTableArchiveClient.

Field Summary

Fields

Modifier and Type	Field and Description
private static org.apache.hadoop.hbase.backup.example.ZKTableArchiveClient	archivingClient
static HBaseClassTestRule	CLASS_RULE
private static org.apache.hadoop.hbase.client.ClusterConnection	CONNECTION
private static org.slf4j.Logger	LOG
private static org.apache.hadoop.hbase.master.cleaner.DirScanPool	POOL
private static org.apache.hadoop.hbase.regionserver.RegionServerServices	rss
private static String	STRING_TABLE_NAME
private static byte[]	TABLE_NAME
private static byte[]	TEST_FAM
private List<org.apache.hadoop.fs.Path>	toCleanup
private static HBaseTestingUtility	UTIL

Constructor Summary

Constructors

Constructor and Description
TestZooKeeperTableArchiveClient()

Method Summary

Modifier and Type	Method and Description
static void	cleanupTest()
private void	createArchiveDirectory()
private void	createHFileInRegion(org.apache.hadoop.hbase.regionserver.HRegion region, byte[] columnFamily) Create a new hfile in the passed region
private List<org.apache.hadoop.fs.Path>	getAllFiles(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path dir) Get all the files (non-directory entries) in the file system under the passed directory
private org.apache.hadoop.fs.Path	getArchiveDir()
private org.apache.hadoop.fs.Path	getTableDir(String tableName)
private void	loadFlushAndCompact(org.apache.hadoop.hbase.regionserver.HRegion region, byte[] family)
private void	runCleaner(org.apache.hadoop.hbase.master.cleaner.HFileCleaner cleaner, CountDownLatch finished, org.apache.hadoop.hbase.Stoppable stop)
private org.apache.hadoop.hbase.master.cleaner.HFileCleaner	setupAndCreateCleaner(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path archiveDir, org.apache.hadoop.hbase.Stoppable stop)
private CountDownLatch	setupCleanerWatching(org.apache.hadoop.hbase.backup.example.LongTermArchivingHFileCleaner cleaner, List<org.apache.hadoop.hbase.master.cleaner.BaseHFileCleanerDelegate> cleaners, int expected) Spy on the LongTermArchivingHFileCleaner to ensure we can catch when the cleaner has seen all the files
static void	setupCluster() Setup the config for the cluster
private static void	setupConf(org.apache.hadoop.conf.Configuration conf)
void	tearDown()
void	testArchivingEnableDisable() Test turning on/off archiving
void	testArchivingOnSingleTable()
void	testMultipleTables() Test archiving/cleaning across multiple tables, where some are retained, and others aren't
private List<org.apache.hadoop.hbase.master.cleaner.BaseHFileCleanerDelegate>	turnOnArchiving(String tableName, org.apache.hadoop.hbase.master.cleaner.HFileCleaner cleaner) Start archiving table for given hfile cleaner

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

CLASS_RULE

public static final HBaseClassTestRule CLASS_RULE

LOG

private static final org.slf4j.Logger LOG

UTIL

private static final HBaseTestingUtility UTIL

STRING_TABLE_NAME

private static final String STRING_TABLE_NAME

See Also:

Constant Field Values

TEST_FAM

private static final byte[] TEST_FAM

TABLE_NAME

private static final byte[] TABLE_NAME

archivingClient

private static org.apache.hadoop.hbase.backup.example.ZKTableArchiveClient archivingClient

toCleanup

private final List<org.apache.hadoop.fs.Path> toCleanup

CONNECTION

private static org.apache.hadoop.hbase.client.ClusterConnection CONNECTION

rss

private static org.apache.hadoop.hbase.regionserver.RegionServerServices rss

POOL

private static org.apache.hadoop.hbase.master.cleaner.DirScanPool POOL

Constructor Detail

TestZooKeeperTableArchiveClient

public TestZooKeeperTableArchiveClient()

Method Detail

setupCluster

public static void setupCluster()
                         throws Exception

Setup the config for the cluster

Throws:

Exception

setupConf

private static void setupConf(org.apache.hadoop.conf.Configuration conf)

tearDown

public void tearDown()
              throws Exception

Throws:

Exception

cleanupTest

public static void cleanupTest()
                        throws Exception

Throws:

Exception

testArchivingEnableDisable

public void testArchivingEnableDisable()
                                throws Exception

Test turning on/off archiving

Throws:

Exception

testArchivingOnSingleTable

public void testArchivingOnSingleTable()
                                throws Exception

Throws:

Exception

testMultipleTables

public void testMultipleTables()
                        throws Exception

Test archiving/cleaning across multiple tables, where some are retained, and others aren't

Throws:

Exception - on failure

createArchiveDirectory

private void createArchiveDirectory()
                             throws IOException

Throws:

IOException

getArchiveDir

private org.apache.hadoop.fs.Path getArchiveDir()
                                         throws IOException

Throws:

IOException

getTableDir

private org.apache.hadoop.fs.Path getTableDir(String tableName)
                                       throws IOException

Throws:

IOException

setupAndCreateCleaner

private org.apache.hadoop.hbase.master.cleaner.HFileCleaner setupAndCreateCleaner(org.apache.hadoop.conf.Configuration conf,
                                                                                  org.apache.hadoop.fs.FileSystem fs,
                                                                                  org.apache.hadoop.fs.Path archiveDir,
                                                                                  org.apache.hadoop.hbase.Stoppable stop)

turnOnArchiving

private List<org.apache.hadoop.hbase.master.cleaner.BaseHFileCleanerDelegate> turnOnArchiving(String tableName,
                                                                                              org.apache.hadoop.hbase.master.cleaner.HFileCleaner cleaner)
                                                                                       throws IOException,
                                                                                              org.apache.zookeeper.KeeperException

Start archiving table for given hfile cleaner

Parameters:

tableName - table to archive

cleaner - cleaner to check to make sure change propagated

Returns:

underlying LongTermArchivingHFileCleaner that is managing archiving

Throws:

IOException - on failure

org.apache.zookeeper.KeeperException - on failure

setupCleanerWatching

private CountDownLatch setupCleanerWatching(org.apache.hadoop.hbase.backup.example.LongTermArchivingHFileCleaner cleaner,
                                            List<org.apache.hadoop.hbase.master.cleaner.BaseHFileCleanerDelegate> cleaners,
                                            int expected)

Spy on the LongTermArchivingHFileCleaner to ensure we can catch when the cleaner has seen all the files

Returns:

a CountDownLatch to wait on that releases when the cleaner has been called at least the expected number of times.

getAllFiles

private List<org.apache.hadoop.fs.Path> getAllFiles(org.apache.hadoop.fs.FileSystem fs,
                                                    org.apache.hadoop.fs.Path dir)
                                             throws IOException

Get all the files (non-directory entries) in the file system under the passed directory

Parameters:

dir - directory to investigate

Returns:

all files under the directory

Throws:

IOException

loadFlushAndCompact

private void loadFlushAndCompact(org.apache.hadoop.hbase.regionserver.HRegion region,
                                 byte[] family)
                          throws IOException

Throws:

IOException

createHFileInRegion

private void createHFileInRegion(org.apache.hadoop.hbase.regionserver.HRegion region,
                                 byte[] columnFamily)
                          throws IOException

Create a new hfile in the passed region

Parameters:

region - region to operate on

columnFamily - family for which to add data

Throws:

IOException - if doing the put or flush fails

runCleaner

private void runCleaner(org.apache.hadoop.hbase.master.cleaner.HFileCleaner cleaner,
                        CountDownLatch finished,
                        org.apache.hadoop.hbase.Stoppable stop)
                 throws InterruptedException

Parameters:

cleaner - the cleaner to use

Throws:

InterruptedException