Package org.apache.hadoop.hbase.backup

Class TestHFileArchiving

java.lang.Object

org.apache.hadoop.hbase.backup.TestHFileArchiving

@Tag("org.apache.hadoop.hbase.testclassification.LargeTests") @Tag("org.apache.hadoop.hbase.testclassification.MiscTests")
public class TestHFileArchiving
extends Object

Test that the HFileArchiver correctly removes all the parts of a region when cleaning up a region

Nested Class Summary

Nested Classes

Modifier and Type	Class	Description
private static interface	TestHFileArchiving.ArchivingFunction<Configuration,FS,Region,Dir,Family,Files>

Field Summary

Fields

Modifier and Type	Field	Description
private static final org.slf4j.Logger	LOG
private static org.apache.hadoop.hbase.master.cleaner.DirScanPool	POOL
private static final byte[]	TEST_FAM
private static final HBaseTestingUtil	UTIL

Constructor Summary

Constructors

Constructor	Description
TestHFileArchiving()

Method Summary

Modifier and Type	Method	Description
private void	assertArchiveFiles(org.apache.hadoop.fs.FileSystem fs, List<String> storeFiles, long timeout)
static void	cleanupTest()
private void	clearArchiveDirectory()
private List<String>	getAllFileNames(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path archiveDir)	Get the names of all the files below the given directory
private org.apache.hadoop.hbase.master.cleaner.HFileCleaner	getHFileCleaner(org.apache.hadoop.hbase.Stoppable stoppable, org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path archiveDir)
private List<org.apache.hadoop.hbase.regionserver.HRegion>	initTableForArchivingRegions(org.apache.hadoop.hbase.TableName tableName)
private List<String>	recurseOnFiles(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.FileStatus[] files, List<String> fileNames)	Recursively lookup all the file names under the file[] array
static void	setupCluster()	Setup the config for the cluster
private static void	setupConf(org.apache.hadoop.conf.Configuration conf)
void	tearDown()
void	testArchiveOnTableDelete(org.junit.jupiter.api.TestInfo testInfo)
void	testArchiveOnTableFamilyDelete(org.junit.jupiter.api.TestInfo testInfo)	Test that the store files are archived when a column family is removed.
void	testArchiveRecoveredEditsWalDirDifferentFS()
void	testArchiveRecoveredEditsWalDirNull()
private void	testArchiveRecoveredEditsWalDirNullOrSame(String walDir)
void	testArchiveRecoveredEditsWalDirSameFsStoreFiles()
void	testArchiveRecoveredEditsWrongFS()
void	testArchiveRegions(org.junit.jupiter.api.TestInfo testInfo)
void	testArchiveRegionsWhenPermissionDenied(org.junit.jupiter.api.TestInfo testInfo)
void	testArchiveRegionTableAndRegionDirsNull()
void	testArchiveRegionWithRegionDirNull(org.junit.jupiter.api.TestInfo testInfo)
void	testArchiveRegionWithTableDirNull(org.junit.jupiter.api.TestInfo testInfo)
private void	testArchiveStoreFilesDifferentFileSystems(String walDir, String expectedBase, boolean archiveFileExists, boolean sourceFileExists, boolean archiveFileDifferentLength, TestHFileArchiving.ArchivingFunction<org.apache.hadoop.conf.Configuration,org.apache.hadoop.fs.FileSystem,org.apache.hadoop.hbase.client.RegionInfo,org.apache.hadoop.fs.Path,byte[],Collection<org.apache.hadoop.hbase.regionserver.HStoreFile>> archivingFunction)
private void	testArchiveStoreFilesDifferentFileSystems(String walDir, String expectedBase, TestHFileArchiving.ArchivingFunction<org.apache.hadoop.conf.Configuration,org.apache.hadoop.fs.FileSystem,org.apache.hadoop.hbase.client.RegionInfo,org.apache.hadoop.fs.Path,byte[],Collection<org.apache.hadoop.hbase.regionserver.HStoreFile>> archivingFunction)
void	testArchiveStoreFilesDifferentFileSystemsArchiveFileMatchCurrent()
void	testArchiveStoreFilesDifferentFileSystemsArchiveFileMismatch()
void	testArchiveStoreFilesDifferentFileSystemsFileAlreadyArchived()
void	testArchiveStoreFilesDifferentFileSystemsWallAndRootSame()
void	testArchiveStoreFilesDifferentFileSystemsWallNullPlainRoot()
void	testArchiveStoreFilesDifferentFileSystemsWallWithSchemaPlainRoot()
void	testCleaningRace(org.junit.jupiter.api.TestInfo testInfo)	Test HFileArchiver.resolveAndArchive() race condition HBASE-7643
void	testDeleteRegionWithNoStoreFiles(org.junit.jupiter.api.TestInfo testInfo)	Test that the region directory is removed when we archive a region without store files, but still has hidden files.
private String	testMethodName(org.junit.jupiter.api.TestInfo testInfo)
void	testRemoveRegionDirOnArchive(org.junit.jupiter.api.TestInfo testInfo)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Details

LOG

private static final org.slf4j.Logger LOG

UTIL

private static final HBaseTestingUtil UTIL

TEST_FAM

private static final byte[] TEST_FAM

POOL

private static org.apache.hadoop.hbase.master.cleaner.DirScanPool POOL

Constructor Details

TestHFileArchiving

public TestHFileArchiving()

Method Details

testMethodName

private String testMethodName(org.junit.jupiter.api.TestInfo testInfo)

setupCluster

@BeforeAll
public static void setupCluster()
                         throws Exception

Setup the config for the cluster

Throws:

Exception

setupConf

private static void setupConf(org.apache.hadoop.conf.Configuration conf)

tearDown

@AfterEach
public void tearDown()
              throws Exception

Throws:

Exception

cleanupTest

@AfterAll
public static void cleanupTest()
                        throws Exception

Throws:

Exception

testArchiveStoreFilesDifferentFileSystemsWallWithSchemaPlainRoot

@Test
public void testArchiveStoreFilesDifferentFileSystemsWallWithSchemaPlainRoot()
                                                                      throws Exception

Throws:

Exception

testArchiveStoreFilesDifferentFileSystemsWallNullPlainRoot

@Test
public void testArchiveStoreFilesDifferentFileSystemsWallNullPlainRoot()
                                                                throws Exception

Throws:

Exception

testArchiveStoreFilesDifferentFileSystemsWallAndRootSame

@Test
public void testArchiveStoreFilesDifferentFileSystemsWallAndRootSame()
                                                              throws Exception

Throws:

Exception

testArchiveStoreFilesDifferentFileSystemsFileAlreadyArchived

@Test
public void testArchiveStoreFilesDifferentFileSystemsFileAlreadyArchived()
                                                                  throws Exception

Throws:

Exception

testArchiveStoreFilesDifferentFileSystemsArchiveFileMatchCurrent

@Test
public void testArchiveStoreFilesDifferentFileSystemsArchiveFileMatchCurrent()
                                                                      throws Exception

Throws:

Exception

testArchiveStoreFilesDifferentFileSystemsArchiveFileMismatch

@Test
public void testArchiveStoreFilesDifferentFileSystemsArchiveFileMismatch()
                                                                  throws Exception

Throws:

Exception

testArchiveStoreFilesDifferentFileSystems

private void testArchiveStoreFilesDifferentFileSystems(String walDir,
 String expectedBase,
 TestHFileArchiving.ArchivingFunction<org.apache.hadoop.conf.Configuration,org.apache.hadoop.fs.FileSystem,org.apache.hadoop.hbase.client.RegionInfo,org.apache.hadoop.fs.Path,byte[],Collection<org.apache.hadoop.hbase.regionserver.HStoreFile>> archivingFunction)
                                                throws IOException

Throws:

IOException

testArchiveStoreFilesDifferentFileSystems

private void testArchiveStoreFilesDifferentFileSystems(String walDir,
 String expectedBase,
 boolean archiveFileExists,
 boolean sourceFileExists,
 boolean archiveFileDifferentLength,
 TestHFileArchiving.ArchivingFunction<org.apache.hadoop.conf.Configuration,org.apache.hadoop.fs.FileSystem,org.apache.hadoop.hbase.client.RegionInfo,org.apache.hadoop.fs.Path,byte[],Collection<org.apache.hadoop.hbase.regionserver.HStoreFile>> archivingFunction)
                                                throws IOException

Throws:

IOException

testArchiveRecoveredEditsWalDirNull

@Test
public void testArchiveRecoveredEditsWalDirNull()
                                         throws Exception

Throws:

Exception

testArchiveRecoveredEditsWalDirSameFsStoreFiles

@Test
public void testArchiveRecoveredEditsWalDirSameFsStoreFiles()
                                                     throws Exception

Throws:

Exception

testArchiveRecoveredEditsWalDirNullOrSame

private void testArchiveRecoveredEditsWalDirNullOrSame(String walDir)
                                                throws Exception

Throws:

Exception

testArchiveRecoveredEditsWrongFS

@Test
public void testArchiveRecoveredEditsWrongFS()
                                      throws Exception

Throws:

Exception

testArchiveRecoveredEditsWalDirDifferentFS

@Test
public void testArchiveRecoveredEditsWalDirDifferentFS()
                                                throws Exception

Throws:

Exception

testRemoveRegionDirOnArchive

@Test
public void testRemoveRegionDirOnArchive(org.junit.jupiter.api.TestInfo testInfo)
                                  throws Exception

Throws:

Exception

testDeleteRegionWithNoStoreFiles

@Test
public void testDeleteRegionWithNoStoreFiles(org.junit.jupiter.api.TestInfo testInfo)
                                      throws IOException

Test that the region directory is removed when we archive a region without store files, but still has hidden files.

Throws:

IOException - throws an IOException if there's problem creating a table or if there's an issue with accessing FileSystem.

initTableForArchivingRegions

private List<org.apache.hadoop.hbase.regionserver.HRegion> initTableForArchivingRegions(org.apache.hadoop.hbase.TableName tableName)
                                                                                 throws IOException

Throws:

IOException

testArchiveRegions

@Test
public void testArchiveRegions(org.junit.jupiter.api.TestInfo testInfo)
                        throws Exception

Throws:

Exception

testArchiveRegionsWhenPermissionDenied

@Test
public void testArchiveRegionsWhenPermissionDenied(org.junit.jupiter.api.TestInfo testInfo)
                                            throws Exception

Throws:

Exception

testArchiveOnTableDelete

@Test
public void testArchiveOnTableDelete(org.junit.jupiter.api.TestInfo testInfo)
                              throws Exception

Throws:

Exception

assertArchiveFiles

private void assertArchiveFiles(org.apache.hadoop.fs.FileSystem fs,
 List<String> storeFiles,
 long timeout)
                         throws IOException

Throws:

IOException

testArchiveOnTableFamilyDelete

@Test
public void testArchiveOnTableFamilyDelete(org.junit.jupiter.api.TestInfo testInfo)
                                    throws IOException,
InterruptedException

Test that the store files are archived when a column family is removed.

Throws:

IOException - if there's a problem creating a table.

InterruptedException - problem getting a RegionServer.

testCleaningRace

@Test
public void testCleaningRace(org.junit.jupiter.api.TestInfo testInfo)
                      throws Exception

Test HFileArchiver.resolveAndArchive() race condition HBASE-7643

Throws:

Exception

testArchiveRegionTableAndRegionDirsNull

@Test
public void testArchiveRegionTableAndRegionDirsNull()
                                             throws IOException

Throws:

IOException

testArchiveRegionWithTableDirNull

@Test
public void testArchiveRegionWithTableDirNull(org.junit.jupiter.api.TestInfo testInfo)
                                       throws IOException

Throws:

IOException

testArchiveRegionWithRegionDirNull

@Test
public void testArchiveRegionWithRegionDirNull(org.junit.jupiter.api.TestInfo testInfo)
                                        throws IOException

Throws:

IOException

getHFileCleaner

private org.apache.hadoop.hbase.master.cleaner.HFileCleaner getHFileCleaner(org.apache.hadoop.hbase.Stoppable stoppable,
 org.apache.hadoop.conf.Configuration conf,
 org.apache.hadoop.fs.FileSystem fs,
 org.apache.hadoop.fs.Path archiveDir)
                                                                     throws IOException

Throws:

IOException

clearArchiveDirectory

private void clearArchiveDirectory()
                            throws IOException

Throws:

IOException

getAllFileNames

private List<String> getAllFileNames(org.apache.hadoop.fs.FileSystem fs,
 org.apache.hadoop.fs.Path archiveDir)
                              throws IOException

Get the names of all the files below the given directory

Parameters:

fs - the file system to inspect

archiveDir - the directory in which to look

Returns:

a list of all files in the directory and sub-directories

Throws:

IOException - throws IOException in case FS is unavailable

recurseOnFiles

private List<String> recurseOnFiles(org.apache.hadoop.fs.FileSystem fs,
 org.apache.hadoop.fs.FileStatus[] files,
 List<String> fileNames)
                             throws IOException

Recursively lookup all the file names under the file[] array

Throws:

IOException