T
- Cleaner delegate class that is dynamically loaded from configuration@InterfaceAudience.Private public abstract class CleanerChore<T extends FileCleanerDelegate> extends ScheduledChore
Modifier and Type | Class and Description |
---|---|
private static interface |
CleanerChore.Action<T> |
Modifier and Type | Field and Description |
---|---|
private static int |
AVAIL_PROCESSORS |
static String |
CHORE_POOL_SIZE
Configures the threadpool used for scanning the archive directory for the HFileCleaner If it is
an integer and >= 1, it would be the size; if 0.0 < size <= 1.0, size would be available
processors * size.
|
protected List<T> |
cleanersChain |
private org.apache.hadoop.conf.Configuration |
conf |
(package private) static String |
DEFAULT_CHORE_POOL_SIZE |
(package private) static boolean |
DEFAULT_LOG_CLEANER_CHORE_DIRECTORY_SORTING |
(package private) static String |
DEFAULT_LOG_CLEANER_CHORE_POOL_SIZE |
private AtomicBoolean |
enabled |
protected List<String> |
excludeDirs |
private boolean |
forceRun |
protected org.apache.hadoop.fs.FileSystem |
fs |
private CompletableFuture<Boolean> |
future |
private static org.slf4j.Logger |
LOG |
static String |
LOG_CLEANER_CHORE_DIRECTORY_SORTING
Enable the CleanerChore to sort the subdirectories by consumed space and start the cleaning
with the largest subdirectory.
|
static String |
LOG_CLEANER_CHORE_SIZE
Configures the threadpool used for scanning the Old logs directory for the LogCleaner Follows
the same configuration mechanism as CHORE_POOL_SIZE, but has a default of 1 thread.
|
private org.apache.hadoop.fs.Path |
oldFileDir |
protected Map<String,Object> |
params |
private DirScanPool |
pool |
private boolean |
sortDirectories |
Constructor and Description |
---|
CleanerChore(String name,
int sleepPeriod,
Stoppable s,
org.apache.hadoop.conf.Configuration conf,
org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path oldFileDir,
String confKey,
DirScanPool pool) |
CleanerChore(String name,
int sleepPeriod,
Stoppable s,
org.apache.hadoop.conf.Configuration conf,
org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path oldFileDir,
String confKey,
DirScanPool pool,
Map<String,Object> params,
List<org.apache.hadoop.fs.Path> excludePaths) |
Modifier and Type | Method and Description |
---|---|
(package private) static int |
calculatePoolSize(String poolSize)
Calculate size for cleaner pool.
|
private boolean |
checkAndDeleteFiles(List<org.apache.hadoop.fs.FileStatus> files)
Run the given files through each of the cleaners to see if it should be deleted, deleting it if
necessary.
|
protected void |
chore()
The task to execute on each scheduled execution of the Chore
|
void |
cleanup()
Override to run cleanup tasks when the Chore encounters an error and must stop running
|
private boolean |
deleteAction(CleanerChore.Action<Boolean> deletion,
String type,
org.apache.hadoop.fs.Path dir)
Perform a delete on a specified type.
|
protected int |
deleteFiles(Iterable<org.apache.hadoop.fs.FileStatus> filesToDelete)
Delete the given files
|
(package private) int |
getChorePoolSize() |
boolean |
getEnabled() |
private void |
initCleanerChain(String confKey)
Instantiate and initialize all the file cleaners set in the configuration
|
protected boolean |
initialChore()
Override to run a task before we start looping.
|
private boolean |
isEmptyDirDeletable(org.apache.hadoop.fs.Path dir)
Check if a empty directory with no subdirs or subfiles can be deleted
|
private T |
newFileCleaner(String className,
org.apache.hadoop.conf.Configuration conf)
A utility method to create new instances of LogCleanerDelegate based on the class name of the
LogCleanerDelegate.
|
private void |
postRunCleaner() |
private void |
preRunCleaner() |
boolean |
setEnabled(boolean enabled) |
private boolean |
shouldExclude(org.apache.hadoop.fs.FileStatus f)
Check if a path should not perform clear
|
private void |
sortByConsumedSpace(List<org.apache.hadoop.fs.FileStatus> dirs)
Sort the given list in (descending) order of the space each element takes
|
private void |
traverseAndDelete(org.apache.hadoop.fs.Path dir,
boolean root,
CompletableFuture<Boolean> result)
Attempts to clean up a directory(its subdirectories, and files) in a
ThreadPoolExecutor concurrently. |
CompletableFuture<Boolean> |
triggerCleanerNow()
Trigger the cleaner immediately and return a CompletableFuture for getting the result.
|
protected abstract boolean |
validate(org.apache.hadoop.fs.Path file)
Validate the file to see if it even belongs in the directory.
|
cancel, cancel, choreForTesting, getInitialDelay, getName, getPeriod, getStopper, getTimeUnit, isInitialChoreComplete, isScheduled, run, shutdown, shutdown, toString, triggerNow
private static final org.slf4j.Logger LOG
private static final int AVAIL_PROCESSORS
public static final String CHORE_POOL_SIZE
static final String DEFAULT_CHORE_POOL_SIZE
public static final String LOG_CLEANER_CHORE_SIZE
static final String DEFAULT_LOG_CLEANER_CHORE_POOL_SIZE
public static final String LOG_CLEANER_CHORE_DIRECTORY_SORTING
static final boolean DEFAULT_LOG_CLEANER_CHORE_DIRECTORY_SORTING
private final DirScanPool pool
protected final org.apache.hadoop.fs.FileSystem fs
private final org.apache.hadoop.fs.Path oldFileDir
private final org.apache.hadoop.conf.Configuration conf
private final AtomicBoolean enabled
protected List<T extends FileCleanerDelegate> cleanersChain
protected List<String> excludeDirs
private CompletableFuture<Boolean> future
private boolean forceRun
private boolean sortDirectories
public CleanerChore(String name, int sleepPeriod, Stoppable s, org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path oldFileDir, String confKey, DirScanPool pool)
public CleanerChore(String name, int sleepPeriod, Stoppable s, org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path oldFileDir, String confKey, DirScanPool pool, Map<String,Object> params, List<org.apache.hadoop.fs.Path> excludePaths)
name
- name of the chore being runsleepPeriod
- the period of time to sleep between each runs
- the stopperconf
- configuration to usefs
- handle to the FSoldFileDir
- the path to the archived filesconfKey
- configuration key for the classes to instantiatepool
- the thread pool used to scan directoriesparams
- members could be used in cleanerstatic int calculatePoolSize(String poolSize)
poolSize
- size from configurationprotected abstract boolean validate(org.apache.hadoop.fs.Path file)
file
- full Path
of the file to be checkedprivate void initCleanerChain(String confKey)
confKey
- key to get the file cleaner classes from the configurationprivate T newFileCleaner(String className, org.apache.hadoop.conf.Configuration conf)
className
- fully qualified class name of the LogCleanerDelegateconf
- used configurationprotected boolean initialChore()
ScheduledChore
initialChore
in class ScheduledChore
protected void chore()
ScheduledChore
chore
in class ScheduledChore
private void preRunCleaner()
private void postRunCleaner()
public CompletableFuture<Boolean> triggerCleanerNow() throws InterruptedException
true
means all the old files have been deleted, otherwise false
.InterruptedException
private void sortByConsumedSpace(List<org.apache.hadoop.fs.FileStatus> dirs)
dirs
- the list to sort, element in it should be directory (not file)private boolean checkAndDeleteFiles(List<org.apache.hadoop.fs.FileStatus> files)
files
- List of FileStatus for the files to check (and possibly delete)private boolean isEmptyDirDeletable(org.apache.hadoop.fs.Path dir)
dir
- Path of the directoryprotected int deleteFiles(Iterable<org.apache.hadoop.fs.FileStatus> filesToDelete)
filesToDelete
- files to deletepublic void cleanup()
ScheduledChore
cleanup
in class ScheduledChore
int getChorePoolSize()
public boolean setEnabled(boolean enabled)
public boolean getEnabled()
private void traverseAndDelete(org.apache.hadoop.fs.Path dir, boolean root, CompletableFuture<Boolean> result)
ThreadPoolExecutor
concurrently. We can get the final result by
calling result.get().private boolean shouldExclude(org.apache.hadoop.fs.FileStatus f)
private boolean deleteAction(CleanerChore.Action<Boolean> deletion, String type, org.apache.hadoop.fs.Path dir)
deletion
- a deletetype
- possible values are 'files', 'subdirs', 'dirs'Copyright © 2007–2020 The Apache Software Foundation. All rights reserved.