Package org.apache.hadoop.hbase.regionserver.wal

Class AbstractTestFSWAL

java.lang.Object

org.apache.hadoop.hbase.regionserver.wal.AbstractTestFSWAL

Direct Known Subclasses:

TestAsyncFSWAL, TestFSHLog

public abstract class AbstractTestFSWAL
extends Object

Nested Class Summary

Nested Classes

Modifier and Type	Class	Description
static final class	AbstractTestFSWAL.FlushSpecificStoresPolicy

Field Summary

Fields

Modifier and Type	Field	Description
protected static org.apache.hadoop.conf.Configuration	CONF
final org.junit.rules.TestName	currentTest
protected static org.apache.hadoop.fs.Path	DIR
protected static org.apache.hadoop.fs.FileSystem	FS
protected static final org.slf4j.Logger	LOG
private static final Set<byte[]>	STORES_TO_FLUSH
protected static final HBaseTestingUtil	TEST_UTIL

Constructor Summary

Constructors

Constructor	Description
AbstractTestFSWAL()

Method Summary

Modifier and Type	Method	Description
protected void	addEdits(org.apache.hadoop.hbase.wal.WAL log, org.apache.hadoop.hbase.client.RegionInfo hri, org.apache.hadoop.hbase.client.TableDescriptor htd, int times, org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl mvcc, NavigableMap<byte[],Integer> scopes, String cf)
private org.apache.hadoop.hbase.regionserver.HRegion	createHoldingHRegion(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.hbase.client.TableDescriptor htd, org.apache.hadoop.hbase.wal.WAL wal)
private org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL<?>	createHoldingWAL(String testName, AtomicBoolean startHoldingForAppend, CountDownLatch holdAppend)
private void	doPutWithAsyncWAL(ExecutorService exec, org.apache.hadoop.hbase.regionserver.HRegion region, org.apache.hadoop.hbase.client.Put put, Runnable flushOrCloseRegion, AtomicBoolean startHoldingForAppend, CountDownLatch flushOrCloseFinished, CountDownLatch holdAppend)
protected void	flushRegion(org.apache.hadoop.hbase.wal.WAL wal, byte[] regionEncodedName, Set<byte[]> flushedFamilyNames)	helper method to simulate region flush for a WAL.
static long	getEarliestMemStoreSeqNum(org.apache.hadoop.hbase.wal.WAL wal, byte[] encodedRegionName)
protected abstract org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL<?>	newSlowWAL(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path rootDir, String WALDir, String archiveDir, org.apache.hadoop.conf.Configuration conf, List<org.apache.hadoop.hbase.regionserver.wal.WALActionsListener> listeners, boolean failIfWALExists, String prefix, String suffix, Runnable action)
protected abstract org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL<?>	newWAL(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path rootDir, String WALDir, String archiveDir, org.apache.hadoop.conf.Configuration conf, List<org.apache.hadoop.hbase.regionserver.wal.WALActionsListener> listeners, boolean failIfWALExists, String prefix, String suffix)
void	setUp()
static void	setUpBeforeClass()
static void	tearDownAfterClass()
void	testFailedToCreateWALIfParentRenamed()
void	testFindMemStoresEligibleForFlush()	On rolling a wal after reaching the threshold, WAL.rollWriter() returns the list of regions which should be flushed in order to archive the oldest wal file.
private void	testFindMemStoresEligibleForFlush(org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL<?> wal)
void	testFlushSequenceIdIsGreaterThanAllEditsInHFile()	Test flush for sure has a sequence id that is beyond the last edit appended.
void	testMaxFlushedSequenceIdGoBackwards()
void	testRollWriterForClosedWAL()
void	testSyncNoAppend()
void	testUnflushedSeqIdTrackingWithAsyncWal()
void	testWALComparator()	tests the log comparator.
void	testWALCoprocessorLoaded()	A loaded WAL coprocessor won't break existing WAL test cases.
void	testWriteEntryCanBeNull()
private void	waitNumRolledLogFiles(org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL<?> wal, int expected)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Details

LOG

protected static final org.slf4j.Logger LOG

CONF

protected static org.apache.hadoop.conf.Configuration CONF

FS

protected static org.apache.hadoop.fs.FileSystem FS

DIR

protected static org.apache.hadoop.fs.Path DIR

TEST_UTIL

protected static final HBaseTestingUtil TEST_UTIL

currentTest

public final org.junit.rules.TestName currentTest

STORES_TO_FLUSH

private static final Set<byte[]> STORES_TO_FLUSH

Constructor Details

AbstractTestFSWAL

public AbstractTestFSWAL()

Method Details

setUp

public void setUp()
           throws Exception

Throws:

Exception

setUpBeforeClass

public static void setUpBeforeClass()
                             throws Exception

Throws:

Exception

tearDownAfterClass

public static void tearDownAfterClass()
                               throws Exception

Throws:

Exception

newWAL

protected abstract org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL<?> newWAL(org.apache.hadoop.fs.FileSystem fs,
 org.apache.hadoop.fs.Path rootDir,
 String WALDir,
 String archiveDir,
 org.apache.hadoop.conf.Configuration conf,
 List<org.apache.hadoop.hbase.regionserver.wal.WALActionsListener> listeners,
 boolean failIfWALExists,
 String prefix,
 String suffix)
                                                                             throws IOException

Throws:

IOException

newSlowWAL

protected abstract org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL<?> newSlowWAL(org.apache.hadoop.fs.FileSystem fs,
 org.apache.hadoop.fs.Path rootDir,
 String WALDir,
 String archiveDir,
 org.apache.hadoop.conf.Configuration conf,
 List<org.apache.hadoop.hbase.regionserver.wal.WALActionsListener> listeners,
 boolean failIfWALExists,
 String prefix,
 String suffix,
 Runnable action)
                                                                                 throws IOException

Throws:

IOException

testWALCoprocessorLoaded

public void testWALCoprocessorLoaded()
                              throws Exception

A loaded WAL coprocessor won't break existing WAL test cases.

Throws:

Exception

addEdits

protected void addEdits(org.apache.hadoop.hbase.wal.WAL log,
 org.apache.hadoop.hbase.client.RegionInfo hri,
 org.apache.hadoop.hbase.client.TableDescriptor htd,
 int times,
 org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl mvcc,
 NavigableMap<byte[],Integer> scopes,
 String cf)
                 throws IOException

Throws:

IOException

flushRegion

protected void flushRegion(org.apache.hadoop.hbase.wal.WAL wal,
 byte[] regionEncodedName,
 Set<byte[]> flushedFamilyNames)

helper method to simulate region flush for a WAL.

testWALComparator

public void testWALComparator()
                       throws Exception

tests the log comparator. Ensure that we are not mixing meta logs with non-meta logs (throws exception if we do). Comparison is based on the timestamp present in the wal name.

Throws:

Exception

waitNumRolledLogFiles

private void waitNumRolledLogFiles(org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL<?> wal,
 int expected)

testFindMemStoresEligibleForFlush

private void testFindMemStoresEligibleForFlush(org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL<?> wal)
                                        throws IOException

Throws:

IOException

testFindMemStoresEligibleForFlush

public void testFindMemStoresEligibleForFlush()
                                       throws Exception

On rolling a wal after reaching the threshold, WAL.rollWriter() returns the list of regions which should be flushed in order to archive the oldest wal file.

This method tests this behavior by inserting edits and rolling the wal enough times to reach the max number of logs threshold. It checks whether we get the "right regions and stores" for flush on rolling the wal.

Throws:

Exception

testFailedToCreateWALIfParentRenamed

public void testFailedToCreateWALIfParentRenamed()
                                          throws IOException,
org.apache.hadoop.hbase.util.CommonFSUtils.StreamLacksCapabilityException

Throws:

IOException

org.apache.hadoop.hbase.util.CommonFSUtils.StreamLacksCapabilityException

testFlushSequenceIdIsGreaterThanAllEditsInHFile

public void testFlushSequenceIdIsGreaterThanAllEditsInHFile()
                                                     throws IOException

Test flush for sure has a sequence id that is beyond the last edit appended. We do this by slowing appends in the background ring buffer thread while in foreground we call flush. The addition of the sync over HRegion in flush should fix an issue where flush was returning before all of its appends had made it out to the WAL (HBASE-11109).

Throws:

IOException

See Also:

• HBASE-11109

testSyncNoAppend

public void testSyncNoAppend()
                      throws IOException

Throws:

IOException

testWriteEntryCanBeNull

public void testWriteEntryCanBeNull()
                             throws IOException

Throws:

IOException

testRollWriterForClosedWAL

public void testRollWriterForClosedWAL()
                                throws IOException

Throws:

IOException

createHoldingWAL

private org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL<?> createHoldingWAL(String testName,
 AtomicBoolean startHoldingForAppend,
 CountDownLatch holdAppend)
                                                                            throws IOException

Throws:

IOException

createHoldingHRegion

private org.apache.hadoop.hbase.regionserver.HRegion createHoldingHRegion(org.apache.hadoop.conf.Configuration conf,
 org.apache.hadoop.hbase.client.TableDescriptor htd,
 org.apache.hadoop.hbase.wal.WAL wal)
                                                                   throws IOException

Throws:

IOException

doPutWithAsyncWAL

private void doPutWithAsyncWAL(ExecutorService exec,
 org.apache.hadoop.hbase.regionserver.HRegion region,
 org.apache.hadoop.hbase.client.Put put,
 Runnable flushOrCloseRegion,
 AtomicBoolean startHoldingForAppend,
 CountDownLatch flushOrCloseFinished,
 CountDownLatch holdAppend)
                        throws InterruptedException,
IOException

Throws:

InterruptedException

IOException

testUnflushedSeqIdTrackingWithAsyncWal

public void testUnflushedSeqIdTrackingWithAsyncWal()
                                            throws IOException,
InterruptedException

Throws:

IOException

InterruptedException

getEarliestMemStoreSeqNum

public static long getEarliestMemStoreSeqNum(org.apache.hadoop.hbase.wal.WAL wal,
 byte[] encodedRegionName)

testMaxFlushedSequenceIdGoBackwards

public void testMaxFlushedSequenceIdGoBackwards()
                                         throws IOException,
InterruptedException

Throws:

IOException

InterruptedException