@InterfaceAudience.Private public class HRegionFileSystem extends Object
Modifier and Type | Field and Description |
---|---|
private int |
baseSleepBeforeRetries |
(package private) org.apache.hadoop.conf.Configuration |
conf |
private static int |
DEFAULT_BASE_SLEEP_BEFORE_RETRIES |
private static int |
DEFAULT_HDFS_CLIENT_RETRIES_NUMBER |
(package private) org.apache.hadoop.fs.FileSystem |
fs |
private int |
hdfsClientRetriesNumber
In order to handle NN connectivity hiccups, one need to retry non-idempotent operation at the
client level.
|
private static org.slf4j.Logger |
LOG |
static String |
REGION_INFO_FILE
Name of the region info file that resides just under the region directory.
|
static String |
REGION_MERGES_DIR
Temporary subdirectory of the region directory used for merges.
|
static String |
REGION_SPLITS_DIR
Temporary subdirectory of the region directory used for splits.
|
(package private) static String |
REGION_TEMP_DIR
Temporary subdirectory of the region directory used for compaction output.
|
private org.apache.hadoop.fs.Path |
regionDir |
private RegionInfo |
regionInfo |
(package private) RegionInfo |
regionInfoForFs |
private org.apache.hadoop.fs.Path |
tableDir |
Constructor and Description |
---|
HRegionFileSystem(org.apache.hadoop.conf.Configuration conf,
org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path tableDir,
RegionInfo regionInfo)
Create a view to the on-disk region
|
Modifier and Type | Method and Description |
---|---|
(package private) Pair<org.apache.hadoop.fs.Path,org.apache.hadoop.fs.Path> |
bulkLoadStoreFile(String familyName,
org.apache.hadoop.fs.Path srcPath,
long seqNum)
Bulk load: Add a specified store file to the specified family.
|
(package private) void |
checkRegionInfoOnFilesystem()
Write out an info file under the stored region directory.
|
(package private) void |
cleanupDaughterRegion(RegionInfo regionInfo)
Remove daughter region
|
void |
cleanupMergedRegion(RegionInfo mergedRegion)
Remove merged region
|
(package private) void |
cleanupTempDir()
Clean up any temp detritus that may have been left around from previous operation attempts.
|
org.apache.hadoop.fs.Path |
commitDaughterRegion(RegionInfo regionInfo,
List<org.apache.hadoop.fs.Path> allRegionFiles,
MasterProcedureEnv env)
Commit a daughter region, moving it from the split temporary directory to the proper location
in the filesystem.
|
void |
commitMergedRegion(List<org.apache.hadoop.fs.Path> allMergedFiles,
MasterProcedureEnv env)
Commit a merged region, making it ready for use.
|
(package private) org.apache.hadoop.fs.Path |
commitStoreFile(org.apache.hadoop.fs.Path buildPath,
org.apache.hadoop.fs.Path dstPath) |
org.apache.hadoop.fs.Path |
commitStoreFile(String familyName,
org.apache.hadoop.fs.Path buildPath)
Move the file from a build/temp location to the main family store directory.
|
(package private) boolean |
createDir(org.apache.hadoop.fs.Path dir)
Creates a directory.
|
private static boolean |
createDirOnFileSystem(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.conf.Configuration conf,
org.apache.hadoop.fs.Path dir)
Creates a directory for a filesystem and configuration object.
|
static HRegionFileSystem |
createRegionOnFileSystem(org.apache.hadoop.conf.Configuration conf,
org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path tableDir,
RegionInfo regionInfo)
Create a new Region on file-system.
|
void |
createSplitsDir(RegionInfo daughterA,
RegionInfo daughterB)
Creates region split daughter directories under the table dir.
|
(package private) org.apache.hadoop.fs.Path |
createStoreDir(String familyName)
Create the store directory for the specified family name
|
org.apache.hadoop.fs.Path |
createTempName()
Generate a unique temporary Path.
|
org.apache.hadoop.fs.Path |
createTempName(String suffix)
Generate a unique temporary Path.
|
(package private) boolean |
deleteDir(org.apache.hadoop.fs.Path dir)
Deletes a directory.
|
void |
deleteFamily(String familyName)
Remove the region family from disk, archiving the store files.
|
static void |
deleteRegionFromFileSystem(org.apache.hadoop.conf.Configuration conf,
org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path tableDir,
RegionInfo regionInfo)
Remove the region from the table directory, archiving the region's hfiles.
|
private static String |
generateUniqueName(String suffix)
Generate a unique file name, used by createTempName() and commitStoreFile()
|
Collection<String> |
getFamilies()
Returns the set of families present on disk n
|
org.apache.hadoop.fs.FileSystem |
getFileSystem()
Returns the underlying
FileSystem |
(package private) org.apache.hadoop.fs.Path |
getMergesDir(RegionInfo hri) |
org.apache.hadoop.fs.Path |
getRegionDir()
Returns
Path to the region directory. |
RegionInfo |
getRegionInfo()
Returns the
RegionInfo that describe this on-disk region view |
private static byte[] |
getRegionInfoFileContent(RegionInfo hri)
n * @return Content of the file we write out to the filesystem under a region n
|
RegionInfo |
getRegionInfoForFS() |
org.apache.hadoop.fs.Path |
getSplitsDir(RegionInfo hri) |
String |
getStoragePolicyName(String familyName)
Get the storage policy of the directory of CF.
|
org.apache.hadoop.fs.Path |
getStoreDir(String familyName)
Returns the directory path of the specified family
|
(package private) StoreFileInfo |
getStoreFileInfo(String familyName,
String fileName)
Return the store file information of the specified family/file.
|
(package private) org.apache.hadoop.fs.Path |
getStoreFilePath(String familyName,
String fileName)
Return Qualified Path of the specified family/file
|
List<StoreFileInfo> |
getStoreFiles(String familyName)
Returns the store files available for the family.
|
List<StoreFileInfo> |
getStoreFiles(String familyName,
boolean validate)
Returns the store files available for the family.
|
static List<org.apache.hadoop.fs.LocatedFileStatus> |
getStoreFilesLocatedStatus(HRegionFileSystem regionfs,
String familyName,
boolean validate)
Returns the store files' LocatedFileStatus which available for the family.
|
static org.apache.hadoop.fs.Path |
getStoreHomedir(org.apache.hadoop.fs.Path tabledir,
RegionInfo hri,
byte[] family) |
static org.apache.hadoop.fs.Path |
getStoreHomedir(org.apache.hadoop.fs.Path tabledir,
String encodedName,
byte[] family) |
org.apache.hadoop.fs.Path |
getTableDir()
Returns
Path to the region's root directory. |
org.apache.hadoop.fs.Path |
getTempDir()
Returns
Path to the region's temp directory, used for file creations |
boolean |
hasReferences(String familyName)
Returns true if the specified family has reference files
|
boolean |
hasReferences(TableDescriptor htd)
Check whether region has Reference file
|
private void |
insertRegionFilesIntoStoreTracker(List<org.apache.hadoop.fs.Path> allFiles,
MasterProcedureEnv env,
HRegionFileSystem regionFs) |
static RegionInfo |
loadRegionInfoFileContent(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path regionDir)
Create a
RegionInfo from the serialized version on-disk. |
org.apache.hadoop.fs.Path |
mergeStoreFile(RegionInfo mergingRegion,
String familyName,
HStoreFile f)
Write out a merge reference under the given merges directory.
|
(package private) static boolean |
mkdirs(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.conf.Configuration conf,
org.apache.hadoop.fs.Path dir) |
static HRegionFileSystem |
openRegionFromFileSystem(org.apache.hadoop.conf.Configuration conf,
org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path tableDir,
RegionInfo regionInfo,
boolean readOnly)
Open Region from file-system.
|
private org.apache.hadoop.fs.Path |
preCommitStoreFile(String familyName,
org.apache.hadoop.fs.Path buildPath,
long seqNum,
boolean generateNewName)
Generate the filename in the main family store directory for moving the file from a build/temp
location.
|
void |
removeStoreFile(String familyName,
org.apache.hadoop.fs.Path filePath)
Archives the specified store file from the specified family.
|
void |
removeStoreFiles(String familyName,
Collection<HStoreFile> storeFiles)
Closes and archives the specified store files from the specified family.
|
(package private) boolean |
rename(org.apache.hadoop.fs.Path srcpath,
org.apache.hadoop.fs.Path dstPath)
Renames a directory.
|
void |
setStoragePolicy(String policyName)
Set storage policy for a whole region.
|
void |
setStoragePolicy(String familyName,
String policyName)
Set the directory of CF to the specified storage policy.
|
private void |
sleepBeforeRetry(String msg,
int sleepMultiplier)
sleeping logic; handles the interrupt exception.
|
private static void |
sleepBeforeRetry(String msg,
int sleepMultiplier,
int baseSleepBeforeRetries,
int hdfsClientRetriesNumber)
sleeping logic for static methods; handles the interrupt exception.
|
org.apache.hadoop.fs.Path |
splitStoreFile(RegionInfo hri,
String familyName,
HStoreFile f,
byte[] splitRow,
boolean top,
RegionSplitPolicy splitPolicy)
Write out a split reference.
|
private static void |
writeRegionInfoFileContent(org.apache.hadoop.conf.Configuration conf,
org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path regionInfoFile,
byte[] content)
Write the .regioninfo file on-disk.
|
private void |
writeRegionInfoOnFilesystem(boolean useTempDir)
Write out an info file under the region directory.
|
private void |
writeRegionInfoOnFilesystem(byte[] regionInfoContent,
boolean useTempDir)
Write out an info file under the region directory.
|
private static final org.slf4j.Logger LOG
public static final String REGION_INFO_FILE
public static final String REGION_MERGES_DIR
public static final String REGION_SPLITS_DIR
static final String REGION_TEMP_DIR
private final RegionInfo regionInfo
final RegionInfo regionInfoForFs
final org.apache.hadoop.conf.Configuration conf
private final org.apache.hadoop.fs.Path tableDir
final org.apache.hadoop.fs.FileSystem fs
private final org.apache.hadoop.fs.Path regionDir
private final int hdfsClientRetriesNumber
private final int baseSleepBeforeRetries
private static final int DEFAULT_HDFS_CLIENT_RETRIES_NUMBER
private static final int DEFAULT_BASE_SLEEP_BEFORE_RETRIES
HRegionFileSystem(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path tableDir, RegionInfo regionInfo)
conf
- the Configuration
to usefs
- FileSystem
that contains the regiontableDir
- Path
to where the table is being storedregionInfo
- RegionInfo
for regionpublic org.apache.hadoop.fs.FileSystem getFileSystem()
FileSystem
public RegionInfo getRegionInfo()
RegionInfo
that describe this on-disk region viewpublic RegionInfo getRegionInfoForFS()
public org.apache.hadoop.fs.Path getTableDir()
Path
to the region's root directory.public org.apache.hadoop.fs.Path getRegionDir()
Path
to the region directory.public org.apache.hadoop.fs.Path getTempDir()
Path
to the region's temp directory, used for file creationsvoid cleanupTempDir() throws IOException
IOException
public org.apache.hadoop.fs.Path getStoreDir(String familyName)
familyName
- Column Family NamePath
to the directory of the specified familypublic static org.apache.hadoop.fs.Path getStoreHomedir(org.apache.hadoop.fs.Path tabledir, RegionInfo hri, byte[] family)
tabledir
- Path
to where the table is being storedhri
- RegionInfo
for the region.family
- ColumnFamilyDescriptor
describing the column familypublic static org.apache.hadoop.fs.Path getStoreHomedir(org.apache.hadoop.fs.Path tabledir, String encodedName, byte[] family)
tabledir
- Path
to where the table is being storedencodedName
- Encoded region name.family
- ColumnFamilyDescriptor
describing the column familyorg.apache.hadoop.fs.Path createStoreDir(String familyName) throws IOException
familyName
- Column Family NamePath
to the directory of the specified familyIOException
- if the directory creation fails.public void setStoragePolicy(String familyName, String policyName)
HdfsConstants
for more details.familyName
- The name of column family.policyName
- The name of the storage policy: 'HOT', 'COLD', etc. See hadoop 2.6+
org.apache.hadoop.hdfs.protocol.HdfsConstants for possible list e.g 'COLD',
'WARM', 'HOT', 'ONE_SSD', 'ALL_SSD', 'LAZY_PERSIST'.public void setStoragePolicy(String policyName)
HdfsConstants
for more details.policyName
- The name of the storage policy: 'HOT', 'COLD', etc. See hadoop 2.6+
org.apache.hadoop.hdfs.protocol.HdfsConstants for possible list e.g 'COLD',
'WARM', 'HOT', 'ONE_SSD', 'ALL_SSD', 'LAZY_PERSIST'.@Nullable public String getStoragePolicyName(String familyName)
familyName
- The name of column family.null
if not using HFileSystem
or exception
thrown when trying to get policypublic List<StoreFileInfo> getStoreFiles(String familyName) throws IOException
familyName
- Column Family NameStoreFileInfo
for the specified family.IOException
public List<StoreFileInfo> getStoreFiles(String familyName, boolean validate) throws IOException
familyName
- Column Family NameStoreFileInfo
for the specified family.IOException
public static List<org.apache.hadoop.fs.LocatedFileStatus> getStoreFilesLocatedStatus(HRegionFileSystem regionfs, String familyName, boolean validate) throws IOException
familyName
- Column Family NameIOException
org.apache.hadoop.fs.Path getStoreFilePath(String familyName, String fileName)
familyName
- Column Family NamefileName
- File NameStoreFileInfo getStoreFileInfo(String familyName, String fileName) throws IOException
familyName
- Column Family NamefileName
- File NameStoreFileInfo
for the specified family/fileIOException
public boolean hasReferences(String familyName) throws IOException
familyName
- Column Family NameIOException
public boolean hasReferences(TableDescriptor htd) throws IOException
htd
- table desciptor of the regionIOException
public Collection<String> getFamilies() throws IOException
IOException
public void deleteFamily(String familyName) throws IOException
familyName
- Column Family NameIOException
- if an error occours during the archivingprivate static String generateUniqueName(String suffix)
suffix
- extra information to append to the generated namepublic org.apache.hadoop.fs.Path createTempName()
Path file = fs.createTempName();
...StoreFile.Writer(file)...
fs.commitStoreFile("family", file);
Path
of the temporary filepublic org.apache.hadoop.fs.Path createTempName(String suffix)
Path file = fs.createTempName();
...StoreFile.Writer(file)...
fs.commitStoreFile("family", file);
suffix
- extra information to append to the generated namePath
of the temporary filepublic org.apache.hadoop.fs.Path commitStoreFile(String familyName, org.apache.hadoop.fs.Path buildPath) throws IOException
familyName
- Family that will gain the filebuildPath
- Path
to the file to commit.Path
of the committed file nIOException
private org.apache.hadoop.fs.Path preCommitStoreFile(String familyName, org.apache.hadoop.fs.Path buildPath, long seqNum, boolean generateNewName) throws IOException
familyName
- Family that will gain the filebuildPath
- Path
to the file to commit.seqNum
- Sequence Number to append to the file name (less then 0 if no sequence
number)generateNewName
- False if you want to keep the buildPath namePath
of the to be committed file nIOException
org.apache.hadoop.fs.Path commitStoreFile(org.apache.hadoop.fs.Path buildPath, org.apache.hadoop.fs.Path dstPath) throws IOException
IOException
public void removeStoreFile(String familyName, org.apache.hadoop.fs.Path filePath) throws IOException
familyName
- Family that contains the store filesfilePath
- Path
to the store file to removeIOException
- if the archiving failspublic void removeStoreFiles(String familyName, Collection<HStoreFile> storeFiles) throws IOException
familyName
- Family that contains the store filesstoreFiles
- set of store files to removeIOException
- if the archiving failsPair<org.apache.hadoop.fs.Path,org.apache.hadoop.fs.Path> bulkLoadStoreFile(String familyName, org.apache.hadoop.fs.Path srcPath, long seqNum) throws IOException
familyName
- Family that will gain the filesrcPath
- Path
to the file to importseqNum
- Bulk Load sequence numberPath
of the bulk loaded file nIOException
public org.apache.hadoop.fs.Path getSplitsDir(RegionInfo hri)
void cleanupDaughterRegion(RegionInfo regionInfo) throws IOException
regionInfo
- daughter RegionInfo
nIOException
public org.apache.hadoop.fs.Path commitDaughterRegion(RegionInfo regionInfo, List<org.apache.hadoop.fs.Path> allRegionFiles, MasterProcedureEnv env) throws IOException
regionInfo
- daughter RegionInfo
IOException
private void insertRegionFilesIntoStoreTracker(List<org.apache.hadoop.fs.Path> allFiles, MasterProcedureEnv env, HRegionFileSystem regionFs) throws IOException
IOException
public void createSplitsDir(RegionInfo daughterA, RegionInfo daughterB) throws IOException
IOException
public org.apache.hadoop.fs.Path splitStoreFile(RegionInfo hri, String familyName, HStoreFile f, byte[] splitRow, boolean top, RegionSplitPolicy splitPolicy) throws IOException
hri
- RegionInfo
of the destinationfamilyName
- Column Family Namef
- File to split.splitRow
- Split Rowtop
- True if we are referring to the top half of the hfile.splitPolicy
- A split policy instance; be careful! May not be full populated; e.g. if this
method is invoked on the Master side, then the RegionSplitPolicy will NOT
have a reference to a Region.IOException
org.apache.hadoop.fs.Path getMergesDir(RegionInfo hri)
public void cleanupMergedRegion(RegionInfo mergedRegion) throws IOException
mergedRegion
- RegionInfo
nIOException
static boolean mkdirs(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.Path dir) throws IOException
IOException
public org.apache.hadoop.fs.Path mergeStoreFile(RegionInfo mergingRegion, String familyName, HStoreFile f) throws IOException
mergingRegion
- RegionInfo
for one of the regions being merged.familyName
- Column Family Namef
- File to create reference.IOException
- if the merge write fails.public void commitMergedRegion(List<org.apache.hadoop.fs.Path> allMergedFiles, MasterProcedureEnv env) throws IOException
IOException
private static byte[] getRegionInfoFileContent(RegionInfo hri) throws IOException
IOException
public static RegionInfo loadRegionInfoFileContent(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path regionDir) throws IOException
RegionInfo
from the serialized version on-disk.fs
- FileSystem
that contains the Region Info fileregionDir
- Path
to the Region Directory that contains the Info fileRegionInfo
instance gotten from the Region Info file.IOException
- if an error occurred during file open/read operation.private static void writeRegionInfoFileContent(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path regionInfoFile, byte[] content) throws IOException
IOException
void checkRegionInfoOnFilesystem() throws IOException
IOException
private void writeRegionInfoOnFilesystem(boolean useTempDir) throws IOException
useTempDir
- indicate whether or not using the region .tmp dir for a safer file creation.IOException
private void writeRegionInfoOnFilesystem(byte[] regionInfoContent, boolean useTempDir) throws IOException
regionInfoContent
- serialized version of the RegionInfo
useTempDir
- indicate whether or not using the region .tmp dir for a safer file
creation.IOException
public static HRegionFileSystem createRegionOnFileSystem(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path tableDir, RegionInfo regionInfo) throws IOException
conf
- the Configuration
to usefs
- FileSystem
from which to add the regiontableDir
- Path
to where the table is being storedregionInfo
- RegionInfo
for region to be addedIOException
- if the region creation fails due to a FileSystem exception.public static HRegionFileSystem openRegionFromFileSystem(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path tableDir, RegionInfo regionInfo, boolean readOnly) throws IOException
conf
- the Configuration
to usefs
- FileSystem
from which to add the regiontableDir
- Path
to where the table is being storedregionInfo
- RegionInfo
for region to be addedreadOnly
- True if you don't want to edit the region dataIOException
- if the region creation fails due to a FileSystem exception.public static void deleteRegionFromFileSystem(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path tableDir, RegionInfo regionInfo) throws IOException
conf
- the Configuration
to usefs
- FileSystem
from which to remove the regiontableDir
- Path
to where the table is being storedregionInfo
- RegionInfo
for region to be deletedIOException
- if the request cannot be completedboolean createDir(org.apache.hadoop.fs.Path dir) throws IOException
IOException
boolean rename(org.apache.hadoop.fs.Path srcpath, org.apache.hadoop.fs.Path dstPath) throws IOException
IOException
boolean deleteDir(org.apache.hadoop.fs.Path dir) throws IOException
IOException
private void sleepBeforeRetry(String msg, int sleepMultiplier) throws InterruptedException
InterruptedException
private static boolean createDirOnFileSystem(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.Path dir) throws IOException
IOException
private static void sleepBeforeRetry(String msg, int sleepMultiplier, int baseSleepBeforeRetries, int hdfsClientRetriesNumber) throws InterruptedException
InterruptedException
Copyright © 2007–2020 The Apache Software Foundation. All rights reserved.