org.apache.hadoop.hbase.snapshot

Class SnapshotManifest

java.lang.Object

org.apache.hadoop.hbase.snapshot.SnapshotManifest

@InterfaceAudience.Private
public final class SnapshotManifest
extends Object

Utility class to help read/write the Snapshot Manifest. The snapshot format is transparent for the users of this class, once the snapshot is written, it will never be modified. On open() the snapshot will be loaded to the current in-memory format.

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
(package private) static interface	SnapshotManifest.RegionVisitor<TRegion,TFamily>

Field Summary

Fields

Modifier and Type	Field and Description
private org.apache.hadoop.conf.Configuration	conf
static String	DATA_MANIFEST_NAME
private org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription	desc
private TableDescriptor	htd
private static org.slf4j.Logger	LOG
private int	manifestSizeLimit
private ForeignExceptionSnare	monitor
private List<org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest>	regionManifests
private org.apache.hadoop.fs.FileSystem	rootFs
static String	SNAPSHOT_MANIFEST_SIZE_LIMIT_CONF_KEY
private MonitoredTask	statusTask
private org.apache.hadoop.fs.Path	workingDir
private org.apache.hadoop.fs.FileSystem	workingDirFs

Constructor Summary

Constructors

Modifier	Constructor and Description
private	SnapshotManifest(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.FileSystem rootFs, org.apache.hadoop.fs.Path workingDir, org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription desc, ForeignExceptionSnare monitor, MonitoredTask statusTask)

Method Summary

Modifier and Type	Method and Description
void	addMobRegion(RegionInfo regionInfo)
protected void	addMobRegion(RegionInfo regionInfo, SnapshotManifest.RegionVisitor visitor)
private void	addReferenceFiles(SnapshotManifest.RegionVisitor visitor, Object regionData, Object familyData, Collection<StoreFileInfo> storeFiles, boolean isMob)
void	addRegion(HRegion region) Creates a 'manifest' for the specified region, by reading directly from the HRegion object.
protected void	addRegion(HRegion region, SnapshotManifest.RegionVisitor visitor)
void	addRegion(org.apache.hadoop.fs.Path tableDir, RegionInfo regionInfo) Creates a 'manifest' for the specified region, by reading directly from the disk.
protected void	addRegion(org.apache.hadoop.fs.Path tableDir, RegionInfo regionInfo, SnapshotManifest.RegionVisitor visitor)
void	addTableDescriptor(TableDescriptor htd) Add the table descriptor to the snapshot manifest
void	consolidate()
private void	convertToV2SingleManifest()
static SnapshotManifest	create(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path workingDir, org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription desc, ForeignExceptionSnare monitor) Return a SnapshotManifest instance, used for writing a snapshot.
static SnapshotManifest	create(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path workingDir, org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription desc, ForeignExceptionSnare monitor, MonitoredTask statusTask)
static ThreadPoolExecutor	createExecutor(org.apache.hadoop.conf.Configuration conf, String name)
private ThreadPoolExecutor	createExecutor(String name)
private SnapshotManifest.RegionVisitor	createRegionVisitor(org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription desc)
List<org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest>	getRegionManifests() Get all the Region Manifest from the snapshot
Map<String,org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest>	getRegionManifestsMap() Get all the Region Manifest from the snapshot.
(package private) static String	getRegionNameFromManifest(org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest manifest) Extract the region encoded name from the region manifest
org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription	getSnapshotDescription() Get the SnapshotDescription
org.apache.hadoop.fs.Path	getSnapshotDir() Get the current snapshot working dir
private static int	getSnapshotFormat(org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription desc)
private List<StoreFileInfo>	getStoreFiles(org.apache.hadoop.fs.Path storeDir)
TableDescriptor	getTableDescriptor() Get the table descriptor from the Snapshot
private void	load() Load the information in the SnapshotManifest.
static SnapshotManifest	open(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path workingDir, org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription desc) Return a SnapshotManifest instance with the information already loaded in-memory.
private org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDataManifest	readDataManifest()
private void	setStatusMsg(String msg)
private void	writeDataManifest(org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDataManifest manifest)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

LOG

private static final org.slf4j.Logger LOG

SNAPSHOT_MANIFEST_SIZE_LIMIT_CONF_KEY

public static final String SNAPSHOT_MANIFEST_SIZE_LIMIT_CONF_KEY

See Also:

Constant Field Values

DATA_MANIFEST_NAME

public static final String DATA_MANIFEST_NAME

See Also:

Constant Field Values

regionManifests

private List<org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest> regionManifests

desc

private org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription desc

htd

private TableDescriptor htd

monitor

private final ForeignExceptionSnare monitor

conf

private final org.apache.hadoop.conf.Configuration conf

workingDir

private final org.apache.hadoop.fs.Path workingDir

rootFs

private final org.apache.hadoop.fs.FileSystem rootFs

workingDirFs

private final org.apache.hadoop.fs.FileSystem workingDirFs

manifestSizeLimit

private int manifestSizeLimit

statusTask

private final MonitoredTask statusTask

Constructor Detail

SnapshotManifest

private SnapshotManifest(org.apache.hadoop.conf.Configuration conf,
                         org.apache.hadoop.fs.FileSystem rootFs,
                         org.apache.hadoop.fs.Path workingDir,
                         org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription desc,
                         ForeignExceptionSnare monitor,
                         MonitoredTask statusTask)
                  throws IOException

Parameters:

conf - configuration file for HBase setup

rootFs - root filesystem containing HFiles

workingDir - file path of where the manifest should be located

desc - description of snapshot being taken

monitor - monitor of foreign exceptions

Throws:

IOException - if the working directory file system cannot be determined from the config file

Method Detail

create

public static SnapshotManifest create(org.apache.hadoop.conf.Configuration conf,
                                      org.apache.hadoop.fs.FileSystem fs,
                                      org.apache.hadoop.fs.Path workingDir,
                                      org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription desc,
                                      ForeignExceptionSnare monitor)
                               throws IOException

Return a SnapshotManifest instance, used for writing a snapshot. There are two usage pattern: - The Master will create a manifest, add the descriptor, offline regions and consolidate the snapshot by writing all the pending stuff on-disk. manifest = SnapshotManifest.create(...) manifest.addRegion(tableDir, hri) manifest.consolidate() - The RegionServer will create a single region manifest manifest = SnapshotManifest.create(...) manifest.addRegion(region)

Throws:

IOException

create

public static SnapshotManifest create(org.apache.hadoop.conf.Configuration conf,
                                      org.apache.hadoop.fs.FileSystem fs,
                                      org.apache.hadoop.fs.Path workingDir,
                                      org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription desc,
                                      ForeignExceptionSnare monitor,
                                      MonitoredTask statusTask)
                               throws IOException

Throws:

IOException

open

public static SnapshotManifest open(org.apache.hadoop.conf.Configuration conf,
                                    org.apache.hadoop.fs.FileSystem fs,
                                    org.apache.hadoop.fs.Path workingDir,
                                    org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription desc)
                             throws IOException

Return a SnapshotManifest instance with the information already loaded in-memory. SnapshotManifest manifest = SnapshotManifest.open(...) TableDescriptor htd = manifest.getTableDescriptor() for (SnapshotRegionManifest regionManifest: manifest.getRegionManifests()) hri = regionManifest.getRegionInfo() for (regionManifest.getFamilyFiles()) ...

Throws:

IOException

addTableDescriptor

public void addTableDescriptor(TableDescriptor htd)
                        throws IOException

Add the table descriptor to the snapshot manifest

Throws:

IOException

createRegionVisitor

private SnapshotManifest.RegionVisitor createRegionVisitor(org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription desc)
                                                    throws IOException

Throws:

IOException

addMobRegion

public void addMobRegion(RegionInfo regionInfo)
                  throws IOException

Throws:

IOException

addMobRegion

protected void addMobRegion(RegionInfo regionInfo,
                            SnapshotManifest.RegionVisitor visitor)
                     throws IOException

Throws:

IOException

addRegion

public void addRegion(HRegion region)
               throws IOException

Creates a 'manifest' for the specified region, by reading directly from the HRegion object. This is used by the "online snapshot" when the table is enabled.

Throws:

IOException

addRegion

protected void addRegion(HRegion region,
                         SnapshotManifest.RegionVisitor visitor)
                  throws IOException

Throws:

IOException

addRegion

public void addRegion(org.apache.hadoop.fs.Path tableDir,
                      RegionInfo regionInfo)
               throws IOException

Creates a 'manifest' for the specified region, by reading directly from the disk. This is used by the "offline snapshot" when the table is disabled.

Throws:

IOException

addRegion

protected void addRegion(org.apache.hadoop.fs.Path tableDir,
                         RegionInfo regionInfo,
                         SnapshotManifest.RegionVisitor visitor)
                  throws IOException

Throws:

IOException

getStoreFiles

private List<StoreFileInfo> getStoreFiles(org.apache.hadoop.fs.Path storeDir)
                                   throws IOException

Throws:

IOException

addReferenceFiles

private void addReferenceFiles(SnapshotManifest.RegionVisitor visitor,
                               Object regionData,
                               Object familyData,
                               Collection<StoreFileInfo> storeFiles,
                               boolean isMob)
                        throws IOException

Throws:

IOException

load

private void load()
           throws IOException

Load the information in the SnapshotManifest. Called by SnapshotManifest.open() If the format is v2 and there is no data-manifest, means that we are loading an in-progress snapshot. Since we support rolling-upgrades, we loook for v1 and v2 regions format.

Throws:

IOException

getSnapshotDir

public org.apache.hadoop.fs.Path getSnapshotDir()

Get the current snapshot working dir

getSnapshotDescription

public org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription getSnapshotDescription()

Get the SnapshotDescription

getTableDescriptor

public TableDescriptor getTableDescriptor()

Get the table descriptor from the Snapshot

getRegionManifests

public List<org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest> getRegionManifests()

Get all the Region Manifest from the snapshot

setStatusMsg

private void setStatusMsg(String msg)

getRegionManifestsMap

public Map<String,org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest> getRegionManifestsMap()

Get all the Region Manifest from the snapshot. This is an helper to get a map with the region encoded name

consolidate

public void consolidate()
                 throws IOException

Throws:

IOException

convertToV2SingleManifest

private void convertToV2SingleManifest()
                                throws IOException

Throws:

IOException

writeDataManifest

private void writeDataManifest(org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDataManifest manifest)
                        throws IOException

Throws:

IOException

readDataManifest

private org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDataManifest readDataManifest()
                                                                                                        throws IOException

Throws:

IOException

createExecutor

private ThreadPoolExecutor createExecutor(String name)

createExecutor

public static ThreadPoolExecutor createExecutor(org.apache.hadoop.conf.Configuration conf,
                                                String name)

getRegionNameFromManifest

static String getRegionNameFromManifest(org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest manifest)

Extract the region encoded name from the region manifest

getSnapshotFormat

private static int getSnapshotFormat(org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription desc)