Package org.apache.hadoop.hbase.snapshot

Class ExportSnapshot.ExportMapper

java.lang.Object
org.apache.hadoop.mapreduce.Mapper<org.apache.hadoop.io.BytesWritable,org.apache.hadoop.io.NullWritable,org.apache.hadoop.io.NullWritable,org.apache.hadoop.io.NullWritable>
org.apache.hadoop.hbase.snapshot.ExportSnapshot.ExportMapper
Enclosing class:
ExportSnapshot
private static class ExportSnapshot.ExportMapper extends org.apache.hadoop.mapreduce.Mapper<org.apache.hadoop.io.BytesWritable,org.apache.hadoop.io.NullWritable,org.apache.hadoop.io.NullWritable,org.apache.hadoop.io.NullWritable>

  • Nested Class Summary

    Nested classes/interfaces inherited from class org.apache.hadoop.mapreduce.Mapper

    org.apache.hadoop.mapreduce.Mapper.Context

  • Field Summary

    Fields
    Modifier and Type
    Field
    Description
    (package private) static final int
    BUFFER_SIZE
     
    private int
    bufferSize
     
    private String
    filesGroup
     
    private short
    filesMode
     
    private String
    filesUser
     
    private org.apache.hadoop.fs.Path
    inputArchive
     
    private org.apache.hadoop.fs.FileSystem
    inputFs
     
    private org.apache.hadoop.fs.Path
    inputRoot
     
    private static final org.slf4j.Logger
    LOG
     
    private org.apache.hadoop.fs.Path
    outputArchive
     
    private org.apache.hadoop.fs.FileSystem
    outputFs
     
    private org.apache.hadoop.fs.Path
    outputRoot
     
    (package private) static final int
    REPORT_SIZE
     
    private int
    reportSize
     
    private static ExportSnapshot.Testing
    testing
     
    private boolean
    verifyChecksum
     

  • Constructor Summary

    Constructors
    Modifier
    Constructor
    Description
    private
    ExportMapper()
     

  • Method Summary

    Modifier and Type
    Method
    Description
    private long
    copyData(org.apache.hadoop.mapreduce.Mapper<org.apache.hadoop.io.BytesWritable,org.apache.hadoop.io.NullWritable,org.apache.hadoop.io.NullWritable,org.apache.hadoop.io.NullWritable>.org.apache.hadoop.mapreduce.Mapper.Context context, org.apache.hadoop.fs.Path inputPath, InputStream in, org.apache.hadoop.fs.Path outputPath, org.apache.hadoop.fs.FSDataOutputStream out, long inputFileSize)
     
    private void
    copyFile(org.apache.hadoop.mapreduce.Mapper<org.apache.hadoop.io.BytesWritable,org.apache.hadoop.io.NullWritable,org.apache.hadoop.io.NullWritable,org.apache.hadoop.io.NullWritable>.org.apache.hadoop.mapreduce.Mapper.Context context, org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotFileInfo inputInfo, org.apache.hadoop.fs.Path outputPath)
     
    private void
    createOutputPath(org.apache.hadoop.fs.Path path)
    Create the output folder and optionally set ownership.
    private org.apache.hadoop.fs.FileChecksum
    getFileChecksum(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path path)
     
    private FileLink
    getFileLink(org.apache.hadoop.fs.Path path, org.apache.hadoop.conf.Configuration conf)
     
    private org.apache.hadoop.fs.Path
    getOutputPath(org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotFileInfo inputInfo)
    Returns the location where the inputPath will be copied.
    private org.apache.hadoop.fs.FileStatus
    getSourceFileStatus(org.apache.hadoop.mapreduce.Mapper<org.apache.hadoop.io.BytesWritable,org.apache.hadoop.io.NullWritable,org.apache.hadoop.io.NullWritable,org.apache.hadoop.io.NullWritable>.org.apache.hadoop.mapreduce.Mapper.Context context, org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotFileInfo fileInfo)
     
    private void
    injectTestFailure(org.apache.hadoop.mapreduce.Mapper<org.apache.hadoop.io.BytesWritable,org.apache.hadoop.io.NullWritable,org.apache.hadoop.io.NullWritable,org.apache.hadoop.io.NullWritable>.org.apache.hadoop.mapreduce.Mapper.Context context, org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotFileInfo inputInfo)
     
    void
    map(org.apache.hadoop.io.BytesWritable key, org.apache.hadoop.io.NullWritable value, org.apache.hadoop.mapreduce.Mapper<org.apache.hadoop.io.BytesWritable,org.apache.hadoop.io.NullWritable,org.apache.hadoop.io.NullWritable,org.apache.hadoop.io.NullWritable>.org.apache.hadoop.mapreduce.Mapper.Context context)
     
    private org.apache.hadoop.fs.FSDataInputStream
    openSourceFile(org.apache.hadoop.mapreduce.Mapper<org.apache.hadoop.io.BytesWritable,org.apache.hadoop.io.NullWritable,org.apache.hadoop.io.NullWritable,org.apache.hadoop.io.NullWritable>.org.apache.hadoop.mapreduce.Mapper.Context context, org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotFileInfo fileInfo)
    Try to open the "source" file.
    private boolean
    preserveAttributes(org.apache.hadoop.fs.Path path, org.apache.hadoop.fs.FileStatus refStat)
    Try to Preserve the files attribute selected by the user copying them from the source file This is only required when you are exporting as a different user than "hbase" or on a system that doesn't have the "hbase" user.
    private boolean
    sameFile(org.apache.hadoop.fs.FileStatus inputStat, org.apache.hadoop.fs.FileStatus outputStat)
    Check if the two files are equal by looking at the file length, and at the checksum (if user has specified the verifyChecksum flag).
    void
    setup(org.apache.hadoop.mapreduce.Mapper<org.apache.hadoop.io.BytesWritable,org.apache.hadoop.io.NullWritable,org.apache.hadoop.io.NullWritable,org.apache.hadoop.io.NullWritable>.org.apache.hadoop.mapreduce.Mapper.Context context)
     
    private boolean
    stringIsNotEmpty(String str)
     
    private ExportSnapshot.ChecksumComparison
    verifyChecksum(org.apache.hadoop.fs.FileChecksum inChecksum, org.apache.hadoop.fs.FileChecksum outChecksum)
    Utility to compare checksums
    private void
    verifyCopyResult(org.apache.hadoop.fs.FileStatus inputStat, org.apache.hadoop.fs.FileStatus outputStat)
    Utility to compare the file length and checksums for the paths specified.

    Methods inherited from class org.apache.hadoop.mapreduce.Mapper

    cleanup, run

    Methods inherited from class java.lang.Object

    clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

  • Field Details

  • Constructor Details

  • Method Details

    • setup

      public void setup(org.apache.hadoop.mapreduce.Mapper<org.apache.hadoop.io.BytesWritable,org.apache.hadoop.io.NullWritable,org.apache.hadoop.io.NullWritable,org.apache.hadoop.io.NullWritable>.org.apache.hadoop.mapreduce.Mapper.Context context) throws IOException
      Overrides:
      setup in class org.apache.hadoop.mapreduce.Mapper<org.apache.hadoop.io.BytesWritable,org.apache.hadoop.io.NullWritable,org.apache.hadoop.io.NullWritable,org.apache.hadoop.io.NullWritable>
      Throws:
      IOException

    • map

      public void map(org.apache.hadoop.io.BytesWritable key, org.apache.hadoop.io.NullWritable value, org.apache.hadoop.mapreduce.Mapper<org.apache.hadoop.io.BytesWritable,org.apache.hadoop.io.NullWritable,org.apache.hadoop.io.NullWritable,org.apache.hadoop.io.NullWritable>.org.apache.hadoop.mapreduce.Mapper.Context context) throws InterruptedException, IOException
      Overrides:
      map in class org.apache.hadoop.mapreduce.Mapper<org.apache.hadoop.io.BytesWritable,org.apache.hadoop.io.NullWritable,org.apache.hadoop.io.NullWritable,org.apache.hadoop.io.NullWritable>
      Throws:
      InterruptedException
      IOException

    • getOutputPath

      private org.apache.hadoop.fs.Path getOutputPath(org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotFileInfo inputInfo) throws IOException
      Returns the location where the inputPath will be copied.
      Throws:
      IOException

    • injectTestFailure

      private void injectTestFailure(org.apache.hadoop.mapreduce.Mapper<org.apache.hadoop.io.BytesWritable,org.apache.hadoop.io.NullWritable,org.apache.hadoop.io.NullWritable,org.apache.hadoop.io.NullWritable>.org.apache.hadoop.mapreduce.Mapper.Context context, org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotFileInfo inputInfo) throws IOException
      Throws:
      IOException

    • copyFile

      private void copyFile(org.apache.hadoop.mapreduce.Mapper<org.apache.hadoop.io.BytesWritable,org.apache.hadoop.io.NullWritable,org.apache.hadoop.io.NullWritable,org.apache.hadoop.io.NullWritable>.org.apache.hadoop.mapreduce.Mapper.Context context, org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotFileInfo inputInfo, org.apache.hadoop.fs.Path outputPath) throws IOException
      Throws:
      IOException

    • createOutputPath

      private void createOutputPath(org.apache.hadoop.fs.Path path) throws IOException
      Create the output folder and optionally set ownership.
      Throws:
      IOException

    • preserveAttributes

      private boolean preserveAttributes(org.apache.hadoop.fs.Path path, org.apache.hadoop.fs.FileStatus refStat)
      Try to Preserve the files attribute selected by the user copying them from the source file This is only required when you are exporting as a different user than "hbase" or on a system that doesn't have the "hbase" user. This is not considered a blocking failure since the user can force a chmod with the user that knows is available on the system.

    • stringIsNotEmpty

      private boolean stringIsNotEmpty(String str)

    • copyData

      private long copyData(org.apache.hadoop.mapreduce.Mapper<org.apache.hadoop.io.BytesWritable,org.apache.hadoop.io.NullWritable,org.apache.hadoop.io.NullWritable,org.apache.hadoop.io.NullWritable>.org.apache.hadoop.mapreduce.Mapper.Context context, org.apache.hadoop.fs.Path inputPath, InputStream in, org.apache.hadoop.fs.Path outputPath, org.apache.hadoop.fs.FSDataOutputStream out, long inputFileSize) throws IOException
      Throws:
      IOException

    • openSourceFile

      private org.apache.hadoop.fs.FSDataInputStream openSourceFile(org.apache.hadoop.mapreduce.Mapper<org.apache.hadoop.io.BytesWritable,org.apache.hadoop.io.NullWritable,org.apache.hadoop.io.NullWritable,org.apache.hadoop.io.NullWritable>.org.apache.hadoop.mapreduce.Mapper.Context context, org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotFileInfo fileInfo) throws IOException
      Try to open the "source" file. Throws an IOException if the communication with the inputFs fail or if the file is not found.
      Throws:
      IOException

    • getSourceFileStatus

      private org.apache.hadoop.fs.FileStatus getSourceFileStatus(org.apache.hadoop.mapreduce.Mapper<org.apache.hadoop.io.BytesWritable,org.apache.hadoop.io.NullWritable,org.apache.hadoop.io.NullWritable,org.apache.hadoop.io.NullWritable>.org.apache.hadoop.mapreduce.Mapper.Context context, org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotFileInfo fileInfo) throws IOException
      Throws:
      IOException

    • getFileLink

      private FileLink getFileLink(org.apache.hadoop.fs.Path path, org.apache.hadoop.conf.Configuration conf) throws IOException
      Throws:
      IOException

    • getFileChecksum

      private org.apache.hadoop.fs.FileChecksum getFileChecksum(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path path)

    • verifyCopyResult

      private void verifyCopyResult(org.apache.hadoop.fs.FileStatus inputStat, org.apache.hadoop.fs.FileStatus outputStat) throws IOException
      Utility to compare the file length and checksums for the paths specified.
      Throws:
      IOException

    • verifyChecksum

      private ExportSnapshot.ChecksumComparison verifyChecksum(org.apache.hadoop.fs.FileChecksum inChecksum, org.apache.hadoop.fs.FileChecksum outChecksum)
      Utility to compare checksums

    • sameFile

      private boolean sameFile(org.apache.hadoop.fs.FileStatus inputStat, org.apache.hadoop.fs.FileStatus outputStat)
      Check if the two files are equal by looking at the file length, and at the checksum (if user has specified the verifyChecksum flag).