@InterfaceAudience.Private public final class CommonFSUtils extends Object
setStoragePolicy(FileSystem, Path, String)
is tested in TestFSUtils and
pre-commit will run the hbase-server tests if there's code change in this class. See
HBASE-20838 for more details.Modifier and Type | Class and Description |
---|---|
static class |
CommonFSUtils.StreamLacksCapabilityException
Helper exception for those cases where the place where we need to check a stream capability is
not where we have the needed context to explain the impact and mitigation for a lack.
|
Modifier and Type | Field and Description |
---|---|
static String |
FULL_RWX_PERMISSIONS
Full access permissions (starting point for a umask)
|
static String |
HBASE_WAL_DIR
Parameter name for HBase WAL directory
|
private static org.slf4j.Logger |
LOG |
static String |
UNSAFE_STREAM_CAPABILITY_ENFORCE
Parameter to disable stream capability enforcement checks
|
private static Map<org.apache.hadoop.fs.FileSystem,Boolean> |
warningMap |
Modifier | Constructor and Description |
---|---|
private |
CommonFSUtils() |
Modifier and Type | Method and Description |
---|---|
static void |
checkShortCircuitReadBufferSize(org.apache.hadoop.conf.Configuration conf)
Check if short circuit read buffer size is set and if not, set it to hbase value.
|
static org.apache.hadoop.fs.FSDataOutputStream |
create(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path path,
org.apache.hadoop.fs.permission.FsPermission perm,
boolean overwrite)
Create the specified file on the filesystem.
|
static boolean |
delete(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path path,
boolean recursive)
Calls fs.delete() and returns the value returned by the fs.delete()
|
static boolean |
deleteDirectory(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path dir)
Delete if exists.
|
static org.apache.hadoop.fs.FileSystem |
getCurrentFileSystem(org.apache.hadoop.conf.Configuration conf)
Returns the filesystem of the hbase rootdir.
|
static long |
getDefaultBlockSize(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path path)
Return the number of bytes that large input files should be optimally be split into to minimize
i/o time.
|
static int |
getDefaultBufferSize(org.apache.hadoop.fs.FileSystem fs)
Returns the default buffer size to use during writes.
|
static short |
getDefaultReplication(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path path) |
static String |
getDirUri(org.apache.hadoop.conf.Configuration c,
org.apache.hadoop.fs.Path p)
Returns the URI in the string format
|
static org.apache.hadoop.fs.permission.FsPermission |
getFilePermissions(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.conf.Configuration conf,
String permssionConfKey)
Get the file permissions specified in the configuration, if they are enabled.
|
static org.apache.hadoop.fs.Path |
getNamespaceDir(org.apache.hadoop.fs.Path rootdir,
String namespace)
Returns the
Path object representing the namespace directory under
path rootdir |
static String |
getPath(org.apache.hadoop.fs.Path p)
Return the 'path' component of a Path.
|
static org.apache.hadoop.fs.Path |
getRegionDir(org.apache.hadoop.fs.Path rootdir,
TableName tableName,
String regionName)
Returns the
Path object representing the region directory under
path rootdir |
static org.apache.hadoop.fs.Path |
getRootDir(org.apache.hadoop.conf.Configuration c)
Get the path for the root data directory
|
static org.apache.hadoop.fs.FileSystem |
getRootDirFileSystem(org.apache.hadoop.conf.Configuration c) |
static org.apache.hadoop.fs.Path |
getTableDir(org.apache.hadoop.fs.Path rootdir,
TableName tableName)
Returns the
Path object representing the table directory under
path rootdir |
static TableName |
getTableName(org.apache.hadoop.fs.Path tablePath)
Returns the
TableName object representing the table directory
under path rootdir |
static org.apache.hadoop.fs.FileSystem |
getWALFileSystem(org.apache.hadoop.conf.Configuration c) |
static org.apache.hadoop.fs.Path |
getWALRegionDir(org.apache.hadoop.conf.Configuration conf,
TableName tableName,
String encodedRegionName)
Returns the WAL region directory based on the given table name and region name
|
static org.apache.hadoop.fs.Path |
getWALRootDir(org.apache.hadoop.conf.Configuration c)
Get the path for the root directory for WAL data
|
static org.apache.hadoop.fs.Path |
getWALTableDir(org.apache.hadoop.conf.Configuration conf,
TableName tableName)
Returns the Table directory under the WALRootDir for the specified table name
|
static org.apache.hadoop.fs.Path |
getWrongWALRegionDir(org.apache.hadoop.conf.Configuration conf,
TableName tableName,
String encodedRegionName)
Deprecated.
For compatibility, will be removed in 4.0.0.
|
private static void |
invokeSetStoragePolicy(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path path,
String storagePolicy) |
static boolean |
isExists(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path path)
Calls fs.exists().
|
static boolean |
isHDFS(org.apache.hadoop.conf.Configuration conf)
Return true if this is a filesystem whose scheme is 'hdfs'.
|
static boolean |
isMatchingTail(org.apache.hadoop.fs.Path pathToSearch,
org.apache.hadoop.fs.Path pathTail)
Compare path component of the Path URI; e.g.
|
static boolean |
isMatchingTail(org.apache.hadoop.fs.Path pathToSearch,
String pathTail)
Compare path component of the Path URI; e.g.
|
static boolean |
isRecoveredEdits(org.apache.hadoop.fs.Path path)
Checks if the given path is the one with 'recovered.edits' dir.
|
static boolean |
isStartingWithPath(org.apache.hadoop.fs.Path rootPath,
String path)
Compare of path component.
|
private static boolean |
isValidWALRootDir(org.apache.hadoop.fs.Path walDir,
org.apache.hadoop.conf.Configuration c) |
static List<org.apache.hadoop.fs.LocatedFileStatus> |
listLocatedStatus(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path dir)
Calls fs.listFiles() to get FileStatus and BlockLocations together for reducing rpc call
|
static org.apache.hadoop.fs.FileStatus[] |
listStatus(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path dir)
Calls fs.listStatus() and treats FileNotFoundException as non-fatal This would accommodates
differences between hadoop versions
|
static org.apache.hadoop.fs.FileStatus[] |
listStatus(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path dir,
org.apache.hadoop.fs.PathFilter filter)
Calls fs.listStatus() and treats FileNotFoundException as non-fatal This accommodates
differences between hadoop versions, where hadoop 1 does not throw a FileNotFoundException, and
return an empty FileStatus[] while Hadoop 2 will throw FileNotFoundException.
|
static void |
logFileSystemState(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path root,
org.slf4j.Logger log)
Log the current state of the filesystem from a certain root directory
|
private static void |
logFSTree(org.slf4j.Logger log,
org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path root,
String prefix)
Recursive helper to log the state of the FS
|
static String |
removeWALRootPath(org.apache.hadoop.fs.Path path,
org.apache.hadoop.conf.Configuration conf)
Checks for the presence of the WAL log root path (using the provided conf object) in the given
path.
|
static boolean |
renameAndSetModifyTime(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path src,
org.apache.hadoop.fs.Path dest) |
static void |
setFsDefault(org.apache.hadoop.conf.Configuration c,
org.apache.hadoop.fs.Path root) |
static void |
setFsDefault(org.apache.hadoop.conf.Configuration c,
String uri) |
static void |
setRootDir(org.apache.hadoop.conf.Configuration c,
org.apache.hadoop.fs.Path root) |
static void |
setStoragePolicy(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path path,
String storagePolicy)
Sets storage policy for given path.
|
(package private) static void |
setStoragePolicy(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path path,
String storagePolicy,
boolean throwException) |
static void |
setWALRootDir(org.apache.hadoop.conf.Configuration c,
org.apache.hadoop.fs.Path root) |
static org.apache.hadoop.fs.Path |
validateRootPath(org.apache.hadoop.fs.Path root)
Verifies root directory path is a valid URI with a scheme
|
private static final org.slf4j.Logger LOG
public static final String HBASE_WAL_DIR
public static final String UNSAFE_STREAM_CAPABILITY_ENFORCE
public static final String FULL_RWX_PERMISSIONS
private static final Map<org.apache.hadoop.fs.FileSystem,Boolean> warningMap
private CommonFSUtils()
public static boolean isStartingWithPath(org.apache.hadoop.fs.Path rootPath, String path)
path
starts with rootPath
, then the function returns truerootPath
- value to check forpath
- subject to checkpath
starts with rootPath
public static boolean isMatchingTail(org.apache.hadoop.fs.Path pathToSearch, String pathTail)
pathToSearch
- Path we will be trying to match against.pathTail
- what to matchpathTail
is tail on the path of pathToSearch
public static boolean isMatchingTail(org.apache.hadoop.fs.Path pathToSearch, org.apache.hadoop.fs.Path pathTail)
pathToSearch
- Path we will be trying to match agains againstpathTail
- what to matchpathTail
is tail on the path of pathToSearch
public static boolean deleteDirectory(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path dir) throws IOException
fs
- filesystem objectdir
- directory to deletedir
IOException
- epublic static long getDefaultBlockSize(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path path)
fs
- filesystem objectpublic static short getDefaultReplication(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path path)
public static int getDefaultBufferSize(org.apache.hadoop.fs.FileSystem fs)
fs
- filesystem objectpublic static org.apache.hadoop.fs.FSDataOutputStream create(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path path, org.apache.hadoop.fs.permission.FsPermission perm, boolean overwrite) throws IOException
fs
- FileSystem
on which to write the filepath
- Path
to the file to writeperm
- intial permissionsoverwrite
- Whether or not the created file should be overwritten.IOException
- if the file cannot be createdpublic static org.apache.hadoop.fs.permission.FsPermission getFilePermissions(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.conf.Configuration conf, String permssionConfKey)
fs
- filesystem that the file will be created on.conf
- configuration to read for determining if permissions are enabled and
which to usepermssionConfKey
- property key in the configuration to use when finding the permissionpublic static org.apache.hadoop.fs.Path validateRootPath(org.apache.hadoop.fs.Path root) throws IOException
root
- root directory pathroot
argument.IOException
- if not a valid URI with a schemepublic static String removeWALRootPath(org.apache.hadoop.fs.Path path, org.apache.hadoop.conf.Configuration conf) throws IOException
path
- must not be nullconf
- must not be nullIOException
- from underlying filesystempublic static String getPath(org.apache.hadoop.fs.Path p)
hdfs://example.org:9000/hbase_trunk/TestTable/compaction.dir
, this method returns
/hbase_trunk/TestTable/compaction.dir
. This method is useful if you want to print
out a Path without qualifying Filesystem instance.p
- Filesystem Path whose 'path' component we are to return.public static org.apache.hadoop.fs.Path getRootDir(org.apache.hadoop.conf.Configuration c) throws IOException
c
- configurationPath
to hbase root directory from configuration as a qualified Path.IOException
- epublic static void setRootDir(org.apache.hadoop.conf.Configuration c, org.apache.hadoop.fs.Path root)
public static void setFsDefault(org.apache.hadoop.conf.Configuration c, org.apache.hadoop.fs.Path root)
public static void setFsDefault(org.apache.hadoop.conf.Configuration c, String uri)
public static org.apache.hadoop.fs.FileSystem getRootDirFileSystem(org.apache.hadoop.conf.Configuration c) throws IOException
IOException
public static org.apache.hadoop.fs.Path getWALRootDir(org.apache.hadoop.conf.Configuration c) throws IOException
c
- configurationPath
to hbase log root directory: e.g. "hbase.wal.dir" from
configuration as a qualified Path. Defaults to HBase root dir.IOException
- epublic static String getDirUri(org.apache.hadoop.conf.Configuration c, org.apache.hadoop.fs.Path p) throws IOException
c
- configurationp
- pathIOException
public static void setWALRootDir(org.apache.hadoop.conf.Configuration c, org.apache.hadoop.fs.Path root)
public static org.apache.hadoop.fs.FileSystem getWALFileSystem(org.apache.hadoop.conf.Configuration c) throws IOException
IOException
private static boolean isValidWALRootDir(org.apache.hadoop.fs.Path walDir, org.apache.hadoop.conf.Configuration c) throws IOException
IOException
public static org.apache.hadoop.fs.Path getWALRegionDir(org.apache.hadoop.conf.Configuration conf, TableName tableName, String encodedRegionName) throws IOException
conf
- configuration to determine WALRootDirtableName
- Table that the region is underencodedRegionName
- Region name used for creating the final region directoryIOException
- if there is an exception determining the WALRootDirpublic static org.apache.hadoop.fs.Path getWALTableDir(org.apache.hadoop.conf.Configuration conf, TableName tableName) throws IOException
conf
- configuration used to get the WALRootDirtableName
- Table to get the directory forIOException
- if there is an exception determining the WALRootDir@Deprecated public static org.apache.hadoop.fs.Path getWrongWALRegionDir(org.apache.hadoop.conf.Configuration conf, TableName tableName, String encodedRegionName) throws IOException
IOException
public static org.apache.hadoop.fs.Path getTableDir(org.apache.hadoop.fs.Path rootdir, TableName tableName)
Path
object representing the table directory under
path rootdirrootdir
- qualified path of HBase root directorytableName
- name of tablePath
for tablepublic static org.apache.hadoop.fs.Path getRegionDir(org.apache.hadoop.fs.Path rootdir, TableName tableName, String regionName)
Path
object representing the region directory under
path rootdirrootdir
- qualified path of HBase root directorytableName
- name of tableregionName
- The encoded region namePath
for regionpublic static TableName getTableName(org.apache.hadoop.fs.Path tablePath)
TableName
object representing the table directory
under path rootdirtablePath
- path of tablePath
for tablepublic static org.apache.hadoop.fs.Path getNamespaceDir(org.apache.hadoop.fs.Path rootdir, String namespace)
Path
object representing the namespace directory under
path rootdirrootdir
- qualified path of HBase root directorynamespace
- namespace namePath
for tablepublic static void setStoragePolicy(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path path, String storagePolicy)
fs
- We only do anything it implements a setStoragePolicy methodpath
- the Path whose storage policy is to be setstoragePolicy
- Policy to set on path
; see hadoop 2.6+
org.apache.hadoop.hdfs.protocol.HdfsConstants for possible list e.g
'COLD', 'WARM', 'HOT', 'ONE_SSD', 'ALL_SSD', 'LAZY_PERSIST'.static void setStoragePolicy(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path path, String storagePolicy, boolean throwException) throws IOException
IOException
private static void invokeSetStoragePolicy(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path path, String storagePolicy) throws IOException
IOException
public static boolean isHDFS(org.apache.hadoop.conf.Configuration conf) throws IOException
IOException
- from underlying FileSystempublic static boolean isRecoveredEdits(org.apache.hadoop.fs.Path path)
path
- must not be nullpublic static org.apache.hadoop.fs.FileSystem getCurrentFileSystem(org.apache.hadoop.conf.Configuration conf) throws IOException
IOException
- from underlying FileSystempublic static org.apache.hadoop.fs.FileStatus[] listStatus(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path dir, org.apache.hadoop.fs.PathFilter filter) throws IOException
fs
- file systemdir
- directoryfilter
- path filterIOException
public static org.apache.hadoop.fs.FileStatus[] listStatus(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path dir) throws IOException
fs
- file systemdir
- directoryIOException
public static List<org.apache.hadoop.fs.LocatedFileStatus> listLocatedStatus(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path dir) throws IOException
fs
- file systemdir
- directoryIOException
public static boolean delete(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path path, boolean recursive) throws IOException
fs
- must not be nullpath
- must not be nullrecursive
- delete tree rooted at pathIOException
- from underlying FileSystempublic static boolean isExists(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path path) throws IOException
fs
- must not be nullpath
- must not be nullIOException
- from underlying FileSystempublic static void logFileSystemState(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path root, org.slf4j.Logger log) throws IOException
fs
- filesystem to investigateroot
- root file/directory to start logging fromlog
- log to output informationIOException
- if an unexpected exception occursprivate static void logFSTree(org.slf4j.Logger log, org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path root, String prefix) throws IOException
IOException
logFileSystemState(FileSystem, Path, Logger)
public static boolean renameAndSetModifyTime(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path src, org.apache.hadoop.fs.Path dest) throws IOException
IOException
public static void checkShortCircuitReadBufferSize(org.apache.hadoop.conf.Configuration conf)
conf
- must not be nullCopyright © 2007–2020 The Apache Software Foundation. All rights reserved.