Package org.apache.hadoop.hbase.regionserver

Class SecureBulkLoadManager

java.lang.Object

org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager

@Private
public class SecureBulkLoadManager
extends Object

Bulk loads in secure mode. This service addresses two issues:

1. Moving files in a secure filesystem wherein the HBase Client and HBase Server are different filesystem users.
2. Does moving in a secure manner. Assuming that the filesystem is POSIX compliant.

The algorithm is as follows:

1. Create an hbase owned staging directory which is world traversable (711): /hbase/staging
2. A user writes out data to his secure output directory: /user/foo/data
3. A call is made to hbase to create a secret staging directory which globally rwx (777): /user/staging/averylongandrandomdirectoryname
4. The user moves the data into the random staging directory, then calls bulkLoadHFiles()

Like delegation tokens the strength of the security lies in the length and randomness of the secret directory.

Nested Class Summary

Nested Classes

Modifier and Type	Class	Description
(package private) static class	SecureBulkLoadManager.SecureBulkLoadListener

Field Summary

Fields

Modifier and Type	Field	Description
private org.apache.hadoop.fs.Path	baseStagingDir
private org.apache.hadoop.conf.Configuration	conf
private Connection	conn
private org.apache.hadoop.fs.FileSystem	fs
private Consumer<HRegion>	fsCreatedListener
private static final org.slf4j.Logger	LOG
private static final org.apache.hadoop.fs.permission.FsPermission	PERM_ALL_ACCESS
private static final org.apache.hadoop.fs.permission.FsPermission	PERM_HIDDEN
private SecureRandom	random
private static final int	RANDOM_RADIX
private static final int	RANDOM_WIDTH
private ConcurrentHashMap<org.apache.hadoop.security.UserGroupInformation,org.apache.commons.lang3.mutable.MutableInt>	ugiReferenceCounter
private UserProvider	userProvider
static final long	VERSION

Constructor Summary

Constructors

Constructor	Description
SecureBulkLoadManager(org.apache.hadoop.conf.Configuration conf, Connection conn)

Method Summary

Modifier and Type	Method	Description
void	cleanupBulkLoad(HRegion region, org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.CleanupBulkLoadRequest request)
private org.apache.hadoop.fs.Path	createStagingDir(org.apache.hadoop.fs.Path baseDir, User user, String randomDir)
private org.apache.hadoop.fs.Path	createStagingDir(org.apache.hadoop.fs.Path baseDir, User user, TableName tableName)
private void	decrementUgiReference(org.apache.hadoop.security.UserGroupInformation ugi)
private User	getActiveUser()
private void	incrementUgiReference(org.apache.hadoop.security.UserGroupInformation ugi)
private boolean	isUserReferenced(org.apache.hadoop.security.UserGroupInformation ugi)
String	prepareBulkLoad(HRegion region, org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.PrepareBulkLoadRequest request)
Map<byte[],List<org.apache.hadoop.fs.Path>>	secureBulkLoadHFiles(HRegion region, org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.BulkLoadHFileRequest request)
Map<byte[],List<org.apache.hadoop.fs.Path>>	secureBulkLoadHFiles(HRegion region, org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.BulkLoadHFileRequest request, List<String> clusterIds)
(package private) void	setFsCreatedListener(Consumer<HRegion> fsCreatedListener)
void	start()
void	stop()

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Details

VERSION

public static final long VERSION

See Also:

• Constant Field Values

RANDOM_WIDTH

private static final int RANDOM_WIDTH

See Also:

• Constant Field Values

RANDOM_RADIX

private static final int RANDOM_RADIX

See Also:

• Constant Field Values

LOG

private static final org.slf4j.Logger LOG

PERM_ALL_ACCESS

private static final org.apache.hadoop.fs.permission.FsPermission PERM_ALL_ACCESS

PERM_HIDDEN

private static final org.apache.hadoop.fs.permission.FsPermission PERM_HIDDEN

random

private SecureRandom random

fs

private org.apache.hadoop.fs.FileSystem fs

conf

private org.apache.hadoop.conf.Configuration conf

baseStagingDir

private org.apache.hadoop.fs.Path baseStagingDir

userProvider

private UserProvider userProvider

ugiReferenceCounter

private ConcurrentHashMap<org.apache.hadoop.security.UserGroupInformation,org.apache.commons.lang3.mutable.MutableInt> ugiReferenceCounter

conn

private Connection conn

fsCreatedListener

private Consumer<HRegion> fsCreatedListener

Constructor Details

SecureBulkLoadManager

SecureBulkLoadManager(org.apache.hadoop.conf.Configuration conf,
 Connection conn)

Method Details

start

public void start()
           throws IOException

Throws:

IOException

stop

public void stop()
          throws IOException

Throws:

IOException

prepareBulkLoad

public String prepareBulkLoad(HRegion region,
 org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.PrepareBulkLoadRequest request)
                       throws IOException

Throws:

IOException

cleanupBulkLoad

public void cleanupBulkLoad(HRegion region,
 org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.CleanupBulkLoadRequest request)
                     throws IOException

Throws:

IOException

setFsCreatedListener

void setFsCreatedListener(Consumer<HRegion> fsCreatedListener)

incrementUgiReference

private void incrementUgiReference(org.apache.hadoop.security.UserGroupInformation ugi)

decrementUgiReference

private void decrementUgiReference(org.apache.hadoop.security.UserGroupInformation ugi)

isUserReferenced

private boolean isUserReferenced(org.apache.hadoop.security.UserGroupInformation ugi)

secureBulkLoadHFiles

public Map<byte[],List<org.apache.hadoop.fs.Path>> secureBulkLoadHFiles(HRegion region,
 org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.BulkLoadHFileRequest request)
                                                                 throws IOException

Throws:

IOException

secureBulkLoadHFiles

public Map<byte[],List<org.apache.hadoop.fs.Path>> secureBulkLoadHFiles(HRegion region,
 org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.BulkLoadHFileRequest request,
 List<String> clusterIds)
                                                                 throws IOException

Throws:

IOException

createStagingDir

private org.apache.hadoop.fs.Path createStagingDir(org.apache.hadoop.fs.Path baseDir,
 User user,
 TableName tableName)
                                            throws IOException

Throws:

IOException

createStagingDir

private org.apache.hadoop.fs.Path createStagingDir(org.apache.hadoop.fs.Path baseDir,
 User user,
 String randomDir)
                                            throws IOException

Throws:

IOException

getActiveUser

private User getActiveUser()
                    throws IOException

Throws:

IOException