001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.io.hfile;
019
020import java.io.DataInput;
021import java.io.IOException;
022import java.nio.ByteBuffer;
023import java.security.Key;
024import java.util.ArrayList;
025import java.util.List;
026import java.util.Optional;
027import java.util.concurrent.atomic.AtomicInteger;
028
029import org.apache.hadoop.conf.Configurable;
030import org.apache.hadoop.conf.Configuration;
031import org.apache.hadoop.fs.Path;
032import org.apache.hadoop.hbase.ByteBufferKeyOnlyKeyValue;
033import org.apache.hadoop.hbase.Cell;
034import org.apache.hadoop.hbase.CellComparator;
035import org.apache.hadoop.hbase.CellUtil;
036import org.apache.hadoop.hbase.HConstants;
037import org.apache.hadoop.hbase.NoTagsByteBufferKeyValue;
038import org.apache.hadoop.hbase.PrivateCellUtil;
039import org.apache.hadoop.hbase.KeyValue;
040import org.apache.hadoop.hbase.ByteBufferKeyValue;
041import org.apache.hadoop.hbase.SizeCachedKeyValue;
042import org.apache.hadoop.hbase.SizeCachedNoTagsKeyValue;
043import org.apache.hadoop.hbase.trace.TraceUtil;
044import org.apache.yetus.audience.InterfaceAudience;
045import org.slf4j.Logger;
046import org.slf4j.LoggerFactory;
047import org.apache.hadoop.hbase.fs.HFileSystem;
048import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
049import org.apache.hadoop.hbase.io.compress.Compression;
050import org.apache.hadoop.hbase.io.crypto.Cipher;
051import org.apache.hadoop.hbase.io.crypto.Encryption;
052import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder;
053import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
054import org.apache.hadoop.hbase.io.encoding.HFileBlockDecodingContext;
055import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
056import org.apache.hadoop.hbase.nio.ByteBuff;
057import org.apache.hadoop.hbase.regionserver.KeyValueScanner;
058import org.apache.hadoop.hbase.security.EncryptionUtil;
059import org.apache.hadoop.hbase.util.ByteBufferUtils;
060import org.apache.hadoop.hbase.util.Bytes;
061import org.apache.hadoop.hbase.util.IdLock;
062import org.apache.hadoop.hbase.util.ObjectIntPair;
063import org.apache.hadoop.io.WritableUtils;
064import org.apache.htrace.core.TraceScope;
065
066import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
067
068/**
069 * Implementation that can handle all hfile versions of {@link HFile.Reader}.
070 */
071@InterfaceAudience.Private
072@edu.umd.cs.findbugs.annotations.SuppressWarnings(value="URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD")
073public class HFileReaderImpl implements HFile.Reader, Configurable {
074  // This class is HFileReaderV3 + HFileReaderV2 + AbstractHFileReader all squashed together into
075  // one file.  Ditto for all the HFileReader.ScannerV? implementations. I was running up against
076  // the MaxInlineLevel limit because too many tiers involved reading from an hfile. Was also hard
077  // to navigate the source code when so many classes participating in read.
078  private static final Logger LOG = LoggerFactory.getLogger(HFileReaderImpl.class);
079
080  /** Data block index reader keeping the root data index in memory */
081  private HFileBlockIndex.CellBasedKeyBlockIndexReader dataBlockIndexReader;
082
083  /** Meta block index reader -- always single level */
084  private HFileBlockIndex.ByteArrayKeyBlockIndexReader metaBlockIndexReader;
085
086  private final FixedFileTrailer trailer;
087
088  /** Filled when we read in the trailer. */
089  private final Compression.Algorithm compressAlgo;
090
091  private final boolean primaryReplicaReader;
092
093  /**
094   * What kind of data block encoding should be used while reading, writing,
095   * and handling cache.
096   */
097  private HFileDataBlockEncoder dataBlockEncoder = NoOpDataBlockEncoder.INSTANCE;
098
099  /** Last key in the file. Filled in when we read in the file info */
100  private Cell lastKeyCell = null;
101
102  /** Average key length read from file info */
103  private int avgKeyLen = -1;
104
105  /** Average value length read from file info */
106  private int avgValueLen = -1;
107
108  /** Key comparator */
109  private CellComparator comparator = CellComparator.getInstance();
110
111  /** Size of this file. */
112  private final long fileSize;
113
114  /** Block cache configuration. */
115  private final CacheConfig cacheConf;
116
117  /** Path of file */
118  private final Path path;
119
120  /** File name to be used for block names */
121  private final String name;
122
123  private FileInfo fileInfo;
124
125  private Configuration conf;
126
127  private HFileContext hfileContext;
128
129  /** Filesystem-level block reader. */
130  private HFileBlock.FSReader fsBlockReader;
131
132  /**
133   * A "sparse lock" implementation allowing to lock on a particular block
134   * identified by offset. The purpose of this is to avoid two clients loading
135   * the same block, and have all but one client wait to get the block from the
136   * cache.
137   */
138  private IdLock offsetLock = new IdLock();
139
140  /**
141   * Blocks read from the load-on-open section, excluding data root index, meta
142   * index, and file info.
143   */
144  private List<HFileBlock> loadOnOpenBlocks = new ArrayList<>();
145
146  /** Minimum minor version supported by this HFile format */
147  static final int MIN_MINOR_VERSION = 0;
148
149  /** Maximum minor version supported by this HFile format */
150  // We went to version 2 when we moved to pb'ing fileinfo and the trailer on
151  // the file. This version can read Writables version 1.
152  static final int MAX_MINOR_VERSION = 3;
153
154  /**
155   * We can read files whose major version is v2 IFF their minor version is at least 3.
156   */
157  private static final int MIN_V2_MINOR_VERSION_WITH_PB = 3;
158
159  /** Minor versions starting with this number have faked index key */
160  static final int MINOR_VERSION_WITH_FAKED_KEY = 3;
161
162  @VisibleForTesting
163  @Deprecated
164  public HFileReaderImpl(Path path, FixedFileTrailer trailer, FSDataInputStreamWrapper fsdis,
165      long fileSize, CacheConfig cacheConf, HFileSystem hfs, Configuration conf)
166      throws IOException {
167    this(path, trailer, fsdis, fileSize, cacheConf, hfs, true, conf);
168  }
169
170  /**
171   * Opens a HFile. You must load the index before you can use it by calling
172   * {@link #loadFileInfo()}.
173   * @param path
174   *          Path to HFile.
175   * @param trailer
176   *          File trailer.
177   * @param fsdis
178   *          input stream.
179   * @param fileSize
180   *          Length of the stream.
181   * @param cacheConf
182   *          Cache configuration.
183   * @param hfs
184   *          The file system.
185   * @param conf
186   *          Configuration
187   */
188  @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD")
189  public HFileReaderImpl(Path path, FixedFileTrailer trailer, FSDataInputStreamWrapper fsdis,
190      long fileSize, CacheConfig cacheConf, HFileSystem hfs, boolean primaryReplicaReader,
191      Configuration conf) throws IOException {
192    this.trailer = trailer;
193    this.compressAlgo = trailer.getCompressionCodec();
194    this.cacheConf = cacheConf;
195    this.fileSize = fileSize;
196    this.path = path;
197    this.name = path.getName();
198    this.conf = conf;
199    this.primaryReplicaReader = primaryReplicaReader;
200    checkFileVersion();
201    this.hfileContext = createHFileContext(fsdis, fileSize, hfs, path, trailer);
202    this.fsBlockReader = new HFileBlock.FSReaderImpl(fsdis, fileSize, hfs, path, hfileContext);
203
204    // Comparator class name is stored in the trailer in version 2.
205    comparator = trailer.createComparator();
206    dataBlockIndexReader = new HFileBlockIndex.CellBasedKeyBlockIndexReader(comparator,
207        trailer.getNumDataIndexLevels(), this);
208    metaBlockIndexReader = new HFileBlockIndex.ByteArrayKeyBlockIndexReader(1);
209
210    // Parse load-on-open data.
211
212    HFileBlock.BlockIterator blockIter = fsBlockReader.blockRange(
213        trailer.getLoadOnOpenDataOffset(),
214        fileSize - trailer.getTrailerSize());
215
216    // Data index. We also read statistics about the block index written after
217    // the root level.
218    dataBlockIndexReader.readMultiLevelIndexRoot(
219        blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX),
220        trailer.getDataIndexCount());
221
222    // Meta index.
223    metaBlockIndexReader.readRootIndex(
224        blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX),
225        trailer.getMetaIndexCount());
226
227    // File info
228    fileInfo = new FileInfo();
229    fileInfo.read(blockIter.nextBlockWithBlockType(BlockType.FILE_INFO).getByteStream());
230    byte[] creationTimeBytes = fileInfo.get(FileInfo.CREATE_TIME_TS);
231    this.hfileContext.setFileCreateTime(creationTimeBytes == null?  0:
232        Bytes.toLong(creationTimeBytes));
233    if (fileInfo.get(FileInfo.LASTKEY) != null) {
234      lastKeyCell = new KeyValue.KeyOnlyKeyValue(fileInfo.get(FileInfo.LASTKEY));
235    }
236    avgKeyLen = Bytes.toInt(fileInfo.get(FileInfo.AVG_KEY_LEN));
237    avgValueLen = Bytes.toInt(fileInfo.get(FileInfo.AVG_VALUE_LEN));
238    byte [] keyValueFormatVersion = fileInfo.get(HFileWriterImpl.KEY_VALUE_VERSION);
239    includesMemstoreTS = keyValueFormatVersion != null &&
240        Bytes.toInt(keyValueFormatVersion) == HFileWriterImpl.KEY_VALUE_VER_WITH_MEMSTORE;
241    fsBlockReader.setIncludesMemStoreTS(includesMemstoreTS);
242    if (includesMemstoreTS) {
243      decodeMemstoreTS = Bytes.toLong(fileInfo.get(HFileWriterImpl.MAX_MEMSTORE_TS_KEY)) > 0;
244    }
245
246    // Read data block encoding algorithm name from file info.
247    dataBlockEncoder = HFileDataBlockEncoderImpl.createFromFileInfo(fileInfo);
248    fsBlockReader.setDataBlockEncoder(dataBlockEncoder);
249
250    // Store all other load-on-open blocks for further consumption.
251    HFileBlock b;
252    while ((b = blockIter.nextBlock()) != null) {
253      loadOnOpenBlocks.add(b);
254    }
255
256    // Prefetch file blocks upon open if requested
257    if (cacheConf.shouldPrefetchOnOpen()) {
258      PrefetchExecutor.request(path, new Runnable() {
259        @Override
260        public void run() {
261          long offset = 0;
262          long end = 0;
263          try {
264            end = getTrailer().getLoadOnOpenDataOffset();
265            if (LOG.isTraceEnabled()) {
266              LOG.trace("Prefetch start " + getPathOffsetEndStr(path, offset, end));
267            }
268            // TODO: Could we use block iterator in here? Would that get stuff into the cache?
269            HFileBlock prevBlock = null;
270            while (offset < end) {
271              if (Thread.interrupted()) {
272                break;
273              }
274              // Perhaps we got our block from cache? Unlikely as this may be, if it happens, then
275              // the internal-to-hfileblock thread local which holds the overread that gets the
276              // next header, will not have happened...so, pass in the onDiskSize gotten from the
277              // cached block. This 'optimization' triggers extremely rarely I'd say.
278              long onDiskSize = prevBlock != null? prevBlock.getNextBlockOnDiskSize(): -1;
279              HFileBlock block = readBlock(offset, onDiskSize, /*cacheBlock=*/true,
280                  /*pread=*/true, false, false, null, null);
281              // Need not update the current block. Ideally here the readBlock won't find the
282              // block in cache. We call this readBlock so that block data is read from FS and
283              // cached in BC. So there is no reference count increment that happens here.
284              // The return will ideally be a noop because the block is not of MemoryType SHARED.
285              returnBlock(block);
286              prevBlock = block;
287              offset += block.getOnDiskSizeWithHeader();
288            }
289          } catch (IOException e) {
290            // IOExceptions are probably due to region closes (relocation, etc.)
291            if (LOG.isTraceEnabled()) {
292              LOG.trace("Prefetch " + getPathOffsetEndStr(path, offset, end), e);
293            }
294          } catch (NullPointerException e) {
295            LOG.warn("Stream moved/closed or prefetch cancelled?" +
296                getPathOffsetEndStr(path, offset, end), e);
297          } catch (Exception e) {
298            // Other exceptions are interesting
299            LOG.warn("Prefetch " + getPathOffsetEndStr(path, offset, end), e);
300          } finally {
301            PrefetchExecutor.complete(path);
302          }
303        }
304      });
305    }
306
307    byte[] tmp = fileInfo.get(FileInfo.MAX_TAGS_LEN);
308    // max tag length is not present in the HFile means tags were not at all written to file.
309    if (tmp != null) {
310      hfileContext.setIncludesTags(true);
311      tmp = fileInfo.get(FileInfo.TAGS_COMPRESSED);
312      if (tmp != null && Bytes.toBoolean(tmp)) {
313        hfileContext.setCompressTags(true);
314      }
315    }
316  }
317
318  private static String getPathOffsetEndStr(final Path path, final long offset, final long end) {
319    return "path=" + path.toString() + ", offset=" + offset + ", end=" + end;
320  }
321
322  /**
323   * File version check is a little sloppy. We read v3 files but can also read v2 files if their
324   * content has been pb'd; files written with 0.98.
325   */
326  private void checkFileVersion() {
327    int majorVersion = trailer.getMajorVersion();
328    if (majorVersion == getMajorVersion()) return;
329    int minorVersion = trailer.getMinorVersion();
330    if (majorVersion == 2 && minorVersion >= MIN_V2_MINOR_VERSION_WITH_PB) return;
331    // We can read v3 or v2 versions of hfile.
332    throw new IllegalArgumentException("Invalid HFile version: major=" +
333      trailer.getMajorVersion() + ", minor=" + trailer.getMinorVersion() + ": expected at least " +
334      "major=2 and minor=" + MAX_MINOR_VERSION + ", path=" + path);
335  }
336
337  @SuppressWarnings("serial")
338  public static class BlockIndexNotLoadedException extends IllegalStateException {
339    public BlockIndexNotLoadedException(Path path) {
340      // Add a message in case anyone relies on it as opposed to class name.
341      super(path + " block index not loaded");
342    }
343  }
344
345  private Optional<String> toStringFirstKey() {
346    return getFirstKey().map(CellUtil::getCellKeyAsString);
347  }
348
349  private Optional<String> toStringLastKey() {
350    return getLastKey().map(CellUtil::getCellKeyAsString);
351  }
352
353  @Override
354  public String toString() {
355    return "reader=" + path.toString() +
356        (!isFileInfoLoaded()? "":
357          ", compression=" + compressAlgo.getName() +
358          ", cacheConf=" + cacheConf +
359          ", firstKey=" + toStringFirstKey() +
360          ", lastKey=" + toStringLastKey()) +
361          ", avgKeyLen=" + avgKeyLen +
362          ", avgValueLen=" + avgValueLen +
363          ", entries=" + trailer.getEntryCount() +
364          ", length=" + fileSize;
365  }
366
367  @Override
368  public long length() {
369    return fileSize;
370  }
371
372  @Override
373  public void returnBlock(HFileBlock block) {
374    if (block != null) {
375      this.cacheConf.getBlockCache().ifPresent(blockCache -> {
376        BlockCacheKey cacheKey =
377            new BlockCacheKey(this.getFileContext().getHFileName(), block.getOffset(),
378                this.isPrimaryReplicaReader(), block.getBlockType());
379        blockCache.returnBlock(cacheKey, block);
380      });
381    }
382  }
383
384  /**
385   * @return the first key in the file. May be null if file has no entries. Note
386   *         that this is not the first row key, but rather the byte form of the
387   *         first KeyValue.
388   */
389  @Override
390  public Optional<Cell> getFirstKey() {
391    if (dataBlockIndexReader == null) {
392      throw new BlockIndexNotLoadedException(path);
393    }
394    return dataBlockIndexReader.isEmpty() ? Optional.empty()
395        : Optional.of(dataBlockIndexReader.getRootBlockKey(0));
396  }
397
398  /**
399   * TODO left from {@link HFile} version 1: move this to StoreFile after Ryan's
400   * patch goes in to eliminate {@link KeyValue} here.
401   *
402   * @return the first row key, or null if the file is empty.
403   */
404  @Override
405  public Optional<byte[]> getFirstRowKey() {
406    // We have to copy the row part to form the row key alone
407    return getFirstKey().map(CellUtil::cloneRow);
408  }
409
410  /**
411   * TODO left from {@link HFile} version 1: move this to StoreFile after
412   * Ryan's patch goes in to eliminate {@link KeyValue} here.
413   *
414   * @return the last row key, or null if the file is empty.
415   */
416  @Override
417  public Optional<byte[]> getLastRowKey() {
418    // We have to copy the row part to form the row key alone
419    return getLastKey().map(CellUtil::cloneRow);
420  }
421
422  /** @return number of KV entries in this HFile */
423  @Override
424  public long getEntries() {
425    return trailer.getEntryCount();
426  }
427
428  /** @return comparator */
429  @Override
430  public CellComparator getComparator() {
431    return comparator;
432  }
433
434  /** @return compression algorithm */
435  @Override
436  public Compression.Algorithm getCompressionAlgorithm() {
437    return compressAlgo;
438  }
439
440  /**
441   * @return the total heap size of data and meta block indexes in bytes. Does
442   *         not take into account non-root blocks of a multilevel data index.
443   */
444  @Override
445  public long indexSize() {
446    return (dataBlockIndexReader != null ? dataBlockIndexReader.heapSize() : 0)
447        + ((metaBlockIndexReader != null) ? metaBlockIndexReader.heapSize()
448            : 0);
449  }
450
451  @Override
452  public String getName() {
453    return name;
454  }
455
456  @Override
457  public HFileBlockIndex.BlockIndexReader getDataBlockIndexReader() {
458    return dataBlockIndexReader;
459  }
460
461  @Override
462  public FixedFileTrailer getTrailer() {
463    return trailer;
464  }
465
466  @Override
467  public boolean isPrimaryReplicaReader() {
468    return primaryReplicaReader;
469  }
470
471  @Override
472  public FileInfo loadFileInfo() throws IOException {
473    return fileInfo;
474  }
475
476  /**
477   * An exception thrown when an operation requiring a scanner to be seeked
478   * is invoked on a scanner that is not seeked.
479   */
480  @SuppressWarnings("serial")
481  public static class NotSeekedException extends IllegalStateException {
482    public NotSeekedException(Path path) {
483      super(path + " not seeked to a key/value");
484    }
485  }
486
487  protected static class HFileScannerImpl implements HFileScanner {
488    private ByteBuff blockBuffer;
489    protected final boolean cacheBlocks;
490    protected final boolean pread;
491    protected final boolean isCompaction;
492    private int currKeyLen;
493    private int currValueLen;
494    private int currMemstoreTSLen;
495    private long currMemstoreTS;
496    // Updated but never read?
497    protected AtomicInteger blockFetches = new AtomicInteger(0);
498    protected final HFile.Reader reader;
499    private int currTagsLen;
500    // buffer backed keyonlyKV
501    private ByteBufferKeyOnlyKeyValue bufBackedKeyOnlyKv = new ByteBufferKeyOnlyKeyValue();
502    // A pair for reusing in blockSeek() so that we don't garbage lot of objects
503    final ObjectIntPair<ByteBuffer> pair = new ObjectIntPair<>();
504
505    /**
506     * The next indexed key is to keep track of the indexed key of the next data block.
507     * If the nextIndexedKey is HConstants.NO_NEXT_INDEXED_KEY, it means that the
508     * current data block is the last data block.
509     *
510     * If the nextIndexedKey is null, it means the nextIndexedKey has not been loaded yet.
511     */
512    protected Cell nextIndexedKey;
513    // Current block being used
514    protected HFileBlock curBlock;
515    // Previous blocks that were used in the course of the read
516    protected final ArrayList<HFileBlock> prevBlocks = new ArrayList<>();
517
518    public HFileScannerImpl(final HFile.Reader reader, final boolean cacheBlocks,
519        final boolean pread, final boolean isCompaction) {
520      this.reader = reader;
521      this.cacheBlocks = cacheBlocks;
522      this.pread = pread;
523      this.isCompaction = isCompaction;
524    }
525
526    void updateCurrBlockRef(HFileBlock block) {
527      if (block != null && this.curBlock != null &&
528          block.getOffset() == this.curBlock.getOffset()) {
529        return;
530      }
531      // We don't have to keep ref to EXCLUSIVE type of block
532      if (this.curBlock != null && this.curBlock.usesSharedMemory()) {
533        prevBlocks.add(this.curBlock);
534      }
535      this.curBlock = block;
536    }
537
538    void reset() {
539      // We don't have to keep ref to EXCLUSIVE type of block
540      if (this.curBlock != null && this.curBlock.usesSharedMemory()) {
541        this.prevBlocks.add(this.curBlock);
542      }
543      this.curBlock = null;
544    }
545
546    private void returnBlockToCache(HFileBlock block) {
547      if (LOG.isTraceEnabled()) {
548        LOG.trace("Returning the block : " + block);
549      }
550      this.reader.returnBlock(block);
551    }
552
553    private void returnBlocks(boolean returnAll) {
554      for (int i = 0; i < this.prevBlocks.size(); i++) {
555        returnBlockToCache(this.prevBlocks.get(i));
556      }
557      this.prevBlocks.clear();
558      if (returnAll && this.curBlock != null) {
559        returnBlockToCache(this.curBlock);
560        this.curBlock = null;
561      }
562    }
563
564    @Override
565    public boolean isSeeked(){
566      return blockBuffer != null;
567    }
568
569    @Override
570    public String toString() {
571      return "HFileScanner for reader " + String.valueOf(getReader());
572    }
573
574    protected void assertSeeked() {
575      if (!isSeeked())
576        throw new NotSeekedException(reader.getPath());
577    }
578
579    @Override
580    public HFile.Reader getReader() {
581      return reader;
582    }
583
584    // From non encoded HFiles, we always read back KeyValue or its descendant.(Note: When HFile
585    // block is in DBB, it will be OffheapKV). So all parts of the Cell is in a contiguous
586    // array/buffer. How many bytes we should wrap to make the KV is what this method returns.
587    private int getKVBufSize() {
588      int kvBufSize = KEY_VALUE_LEN_SIZE + currKeyLen + currValueLen;
589      if (currTagsLen > 0) {
590        kvBufSize += Bytes.SIZEOF_SHORT + currTagsLen;
591      }
592      return kvBufSize;
593    }
594
595    @Override
596    public void close() {
597      if (!pread) {
598        // For seek + pread stream socket should be closed when the scanner is closed. HBASE-9393
599        reader.unbufferStream();
600      }
601      this.returnBlocks(true);
602    }
603
604    // Returns the #bytes in HFile for the current cell. Used to skip these many bytes in current
605    // HFile block's buffer so as to position to the next cell.
606    private int getCurCellSerializedSize() {
607      int curCellSize =  KEY_VALUE_LEN_SIZE + currKeyLen + currValueLen
608          + currMemstoreTSLen;
609      if (this.reader.getFileContext().isIncludesTags()) {
610        curCellSize += Bytes.SIZEOF_SHORT + currTagsLen;
611      }
612      return curCellSize;
613    }
614
615    protected void readKeyValueLen() {
616      // This is a hot method. We go out of our way to make this method short so it can be
617      // inlined and is not too big to compile. We also manage position in ByteBuffer ourselves
618      // because it is faster than going via range-checked ByteBuffer methods or going through a
619      // byte buffer array a byte at a time.
620      // Get a long at a time rather than read two individual ints. In micro-benchmarking, even
621      // with the extra bit-fiddling, this is order-of-magnitude faster than getting two ints.
622      // Trying to imitate what was done - need to profile if this is better or
623      // earlier way is better by doing mark and reset?
624      // But ensure that you read long instead of two ints
625      long ll = blockBuffer.getLongAfterPosition(0);
626      // Read top half as an int of key length and bottom int as value length
627      this.currKeyLen = (int)(ll >> Integer.SIZE);
628      this.currValueLen = (int)(Bytes.MASK_FOR_LOWER_INT_IN_LONG ^ ll);
629      checkKeyValueLen();
630      // Move position past the key and value lengths and then beyond the key and value
631      int p = (Bytes.SIZEOF_LONG + currKeyLen + currValueLen);
632      if (reader.getFileContext().isIncludesTags()) {
633        // Tags length is a short.
634        this.currTagsLen = blockBuffer.getShortAfterPosition(p);
635        checkTagsLen();
636        p += (Bytes.SIZEOF_SHORT + currTagsLen);
637      }
638      readMvccVersion(p);
639    }
640
641    private final void checkTagsLen() {
642      if (checkLen(this.currTagsLen)) {
643        throw new IllegalStateException("Invalid currTagsLen " + this.currTagsLen +
644          ". Block offset: " + curBlock.getOffset() + ", block length: " +
645            this.blockBuffer.limit() +
646          ", position: " + this.blockBuffer.position() + " (without header)." +
647          " path=" + reader.getPath());
648      }
649    }
650
651    /**
652     * Read mvcc. Does checks to see if we even need to read the mvcc at all.
653     * @param offsetFromPos
654     */
655    protected void readMvccVersion(final int offsetFromPos) {
656      // See if we even need to decode mvcc.
657      if (!this.reader.shouldIncludeMemStoreTS()) return;
658      if (!this.reader.isDecodeMemStoreTS()) {
659        currMemstoreTS = 0;
660        currMemstoreTSLen = 1;
661        return;
662      }
663      _readMvccVersion(offsetFromPos);
664    }
665
666    /**
667     * Actually do the mvcc read. Does no checks.
668     * @param offsetFromPos
669     */
670    private void _readMvccVersion(int offsetFromPos) {
671      // This is Bytes#bytesToVint inlined so can save a few instructions in this hot method; i.e.
672      // previous if one-byte vint, we'd redo the vint call to find int size.
673      // Also the method is kept small so can be inlined.
674      byte firstByte = blockBuffer.getByteAfterPosition(offsetFromPos);
675      int len = WritableUtils.decodeVIntSize(firstByte);
676      if (len == 1) {
677        this.currMemstoreTS = firstByte;
678      } else {
679        int remaining = len -1;
680        long i = 0;
681        offsetFromPos++;
682        if (remaining >= Bytes.SIZEOF_INT) {
683          // The int read has to be converted to unsigned long so the & op
684          i = (blockBuffer.getIntAfterPosition(offsetFromPos) & 0x00000000ffffffffL);
685          remaining -= Bytes.SIZEOF_INT;
686          offsetFromPos += Bytes.SIZEOF_INT;
687        }
688        if (remaining >= Bytes.SIZEOF_SHORT) {
689          short s = blockBuffer.getShortAfterPosition(offsetFromPos);
690          i = i << 16;
691          i = i | (s & 0xFFFF);
692          remaining -= Bytes.SIZEOF_SHORT;
693          offsetFromPos += Bytes.SIZEOF_SHORT;
694        }
695        for (int idx = 0; idx < remaining; idx++) {
696          byte b = blockBuffer.getByteAfterPosition(offsetFromPos + idx);
697          i = i << 8;
698          i = i | (b & 0xFF);
699        }
700        currMemstoreTS = (WritableUtils.isNegativeVInt(firstByte) ? ~i : i);
701      }
702      this.currMemstoreTSLen = len;
703    }
704
705    /**
706     * Within a loaded block, seek looking for the last key that is smaller than
707     * (or equal to?) the key we are interested in.
708     * A note on the seekBefore: if you have seekBefore = true, AND the first
709     * key in the block = key, then you'll get thrown exceptions. The caller has
710     * to check for that case and load the previous block as appropriate.
711     * @param key
712     *          the key to find
713     * @param seekBefore
714     *          find the key before the given key in case of exact match.
715     * @return 0 in case of an exact key match, 1 in case of an inexact match,
716     *         -2 in case of an inexact match and furthermore, the input key
717     *         less than the first key of current block(e.g. using a faked index
718     *         key)
719     */
720    protected int blockSeek(Cell key, boolean seekBefore) {
721      int klen, vlen, tlen = 0;
722      int lastKeyValueSize = -1;
723      int offsetFromPos;
724      do {
725        offsetFromPos = 0;
726        // Better to ensure that we use the BB Utils here
727        long ll = blockBuffer.getLongAfterPosition(offsetFromPos);
728        klen = (int)(ll >> Integer.SIZE);
729        vlen = (int)(Bytes.MASK_FOR_LOWER_INT_IN_LONG ^ ll);
730        if (checkKeyLen(klen) || checkLen(vlen)) {
731          throw new IllegalStateException("Invalid klen " + klen + " or vlen "
732              + vlen + ". Block offset: "
733              + curBlock.getOffset() + ", block length: " + blockBuffer.limit() + ", position: "
734              + blockBuffer.position() + " (without header)."
735              + " path=" + reader.getPath());
736        }
737        offsetFromPos += Bytes.SIZEOF_LONG;
738        blockBuffer.asSubByteBuffer(blockBuffer.position() + offsetFromPos, klen, pair);
739        bufBackedKeyOnlyKv.setKey(pair.getFirst(), pair.getSecond(), klen);
740        int comp =
741            PrivateCellUtil.compareKeyIgnoresMvcc(reader.getComparator(), key, bufBackedKeyOnlyKv);
742        offsetFromPos += klen + vlen;
743        if (this.reader.getFileContext().isIncludesTags()) {
744          // Read short as unsigned, high byte first
745          tlen = ((blockBuffer.getByteAfterPosition(offsetFromPos) & 0xff) << 8)
746              ^ (blockBuffer.getByteAfterPosition(offsetFromPos + 1) & 0xff);
747          if (checkLen(tlen)) {
748            throw new IllegalStateException("Invalid tlen " + tlen + ". Block offset: "
749                + curBlock.getOffset() + ", block length: " + blockBuffer.limit() + ", position: "
750                + blockBuffer.position() + " (without header)."
751                + " path=" + reader.getPath());
752          }
753          // add the two bytes read for the tags.
754          offsetFromPos += tlen + (Bytes.SIZEOF_SHORT);
755        }
756        if (this.reader.shouldIncludeMemStoreTS()) {
757          // Directly read the mvcc based on current position
758          readMvccVersion(offsetFromPos);
759        }
760        if (comp == 0) {
761          if (seekBefore) {
762            if (lastKeyValueSize < 0) {
763              throw new IllegalStateException("blockSeek with seekBefore "
764                  + "at the first key of the block: key=" + CellUtil.getCellKeyAsString(key)
765                  + ", blockOffset=" + curBlock.getOffset() + ", onDiskSize="
766                  + curBlock.getOnDiskSizeWithHeader()
767                  + ", path=" + reader.getPath());
768            }
769            blockBuffer.moveBack(lastKeyValueSize);
770            readKeyValueLen();
771            return 1; // non exact match.
772          }
773          currKeyLen = klen;
774          currValueLen = vlen;
775          currTagsLen = tlen;
776          return 0; // indicate exact match
777        } else if (comp < 0) {
778          if (lastKeyValueSize > 0) {
779            blockBuffer.moveBack(lastKeyValueSize);
780          }
781          readKeyValueLen();
782          if (lastKeyValueSize == -1 && blockBuffer.position() == 0) {
783            return HConstants.INDEX_KEY_MAGIC;
784          }
785          return 1;
786        }
787        // The size of this key/value tuple, including key/value length fields.
788        lastKeyValueSize = klen + vlen + currMemstoreTSLen + KEY_VALUE_LEN_SIZE;
789        // include tag length also if tags included with KV
790        if (reader.getFileContext().isIncludesTags()) {
791          lastKeyValueSize += tlen + Bytes.SIZEOF_SHORT;
792        }
793        blockBuffer.skip(lastKeyValueSize);
794      } while (blockBuffer.hasRemaining());
795
796      // Seek to the last key we successfully read. This will happen if this is
797      // the last key/value pair in the file, in which case the following call
798      // to next() has to return false.
799      blockBuffer.moveBack(lastKeyValueSize);
800      readKeyValueLen();
801      return 1; // didn't exactly find it.
802    }
803
804    @Override
805    public Cell getNextIndexedKey() {
806      return nextIndexedKey;
807    }
808
809    @Override
810    public int seekTo(Cell key) throws IOException {
811      return seekTo(key, true);
812    }
813
814    @Override
815    public int reseekTo(Cell key) throws IOException {
816      int compared;
817      if (isSeeked()) {
818        compared = compareKey(reader.getComparator(), key);
819        if (compared < 1) {
820          // If the required key is less than or equal to current key, then
821          // don't do anything.
822          return compared;
823        } else {
824          // The comparison with no_next_index_key has to be checked
825          if (this.nextIndexedKey != null &&
826              (this.nextIndexedKey == KeyValueScanner.NO_NEXT_INDEXED_KEY || PrivateCellUtil
827                  .compareKeyIgnoresMvcc(reader.getComparator(), key, nextIndexedKey) < 0)) {
828            // The reader shall continue to scan the current data block instead
829            // of querying the
830            // block index as long as it knows the target key is strictly
831            // smaller than
832            // the next indexed key or the current data block is the last data
833            // block.
834            return loadBlockAndSeekToKey(this.curBlock, nextIndexedKey, false, key,
835                false);
836          }
837
838        }
839      }
840      // Don't rewind on a reseek operation, because reseek implies that we are
841      // always going forward in the file.
842      return seekTo(key, false);
843    }
844
845    /**
846     * An internal API function. Seek to the given key, optionally rewinding to
847     * the first key of the block before doing the seek.
848     *
849     * @param key - a cell representing the key that we need to fetch
850     * @param rewind whether to rewind to the first key of the block before
851     *        doing the seek. If this is false, we are assuming we never go
852     *        back, otherwise the result is undefined.
853     * @return -1 if the key is earlier than the first key of the file,
854     *         0 if we are at the given key, 1 if we are past the given key
855     *         -2 if the key is earlier than the first key of the file while
856     *         using a faked index key
857     * @throws IOException
858     */
859    public int seekTo(Cell key, boolean rewind) throws IOException {
860      HFileBlockIndex.BlockIndexReader indexReader = reader.getDataBlockIndexReader();
861      BlockWithScanInfo blockWithScanInfo = indexReader.loadDataBlockWithScanInfo(key, curBlock,
862          cacheBlocks, pread, isCompaction, getEffectiveDataBlockEncoding());
863      if (blockWithScanInfo == null || blockWithScanInfo.getHFileBlock() == null) {
864        // This happens if the key e.g. falls before the beginning of the
865        // file.
866        return -1;
867      }
868      return loadBlockAndSeekToKey(blockWithScanInfo.getHFileBlock(),
869          blockWithScanInfo.getNextIndexedKey(), rewind, key, false);
870    }
871
872    @Override
873    public boolean seekBefore(Cell key) throws IOException {
874      HFileBlock seekToBlock = reader.getDataBlockIndexReader().seekToDataBlock(key, curBlock,
875          cacheBlocks, pread, isCompaction, reader.getEffectiveEncodingInCache(isCompaction));
876      if (seekToBlock == null) {
877        return false;
878      }
879      Cell firstKey = getFirstKeyCellInBlock(seekToBlock);
880      if (PrivateCellUtil.compareKeyIgnoresMvcc(reader.getComparator(), firstKey, key) >= 0) {
881        long previousBlockOffset = seekToBlock.getPrevBlockOffset();
882        // The key we are interested in
883        if (previousBlockOffset == -1) {
884          // we have a 'problem', the key we want is the first of the file.
885          releaseIfNotCurBlock(seekToBlock);
886          return false;
887        }
888
889        // The first key in the current block 'seekToBlock' is greater than the given
890        // seekBefore key. We will go ahead by reading the next block that satisfies the
891        // given key. Return the current block before reading the next one.
892        releaseIfNotCurBlock(seekToBlock);
893
894        // It is important that we compute and pass onDiskSize to the block
895        // reader so that it does not have to read the header separately to
896        // figure out the size.  Currently, we do not have a way to do this
897        // correctly in the general case however.
898        // TODO: See https://issues.apache.org/jira/browse/HBASE-14576
899        int prevBlockSize = -1;
900        seekToBlock = reader.readBlock(previousBlockOffset,
901            prevBlockSize, cacheBlocks,
902            pread, isCompaction, true, BlockType.DATA, getEffectiveDataBlockEncoding());
903        // TODO shortcut: seek forward in this block to the last key of the
904        // block.
905      }
906      loadBlockAndSeekToKey(seekToBlock, firstKey, true, key, true);
907      return true;
908    }
909
910    /**
911     * The curBlock will be released by shipping or close method, so only need to consider releasing
912     * the block, which was read from HFile before and not referenced by curBlock.
913     */
914    protected void releaseIfNotCurBlock(HFileBlock block) {
915      if (curBlock != block) {
916        reader.returnBlock(block);
917      }
918    }
919
920    /**
921     * Scans blocks in the "scanned" section of the {@link HFile} until the next
922     * data block is found.
923     *
924     * @return the next block, or null if there are no more data blocks
925     * @throws IOException
926     */
927    @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="NP_NULL_ON_SOME_PATH",
928        justification="Yeah, unnecessary null check; could do w/ clean up")
929    protected HFileBlock readNextDataBlock() throws IOException {
930      long lastDataBlockOffset = reader.getTrailer().getLastDataBlockOffset();
931      if (curBlock == null)
932        return null;
933
934      HFileBlock block = this.curBlock;
935
936      do {
937        if (block.getOffset() >= lastDataBlockOffset) {
938          return null;
939        }
940
941        if (block.getOffset() < 0) {
942          throw new IOException(
943              "Invalid block file offset: " + block + ", path=" + reader.getPath());
944        }
945
946        // We are reading the next block without block type validation, because
947        // it might turn out to be a non-data block.
948        block = reader.readBlock(block.getOffset() + block.getOnDiskSizeWithHeader(),
949            block.getNextBlockOnDiskSize(), cacheBlocks, pread,
950            isCompaction, true, null, getEffectiveDataBlockEncoding());
951        if (block != null && !block.getBlockType().isData()) { // Findbugs: NP_NULL_ON_SOME_PATH
952          // Whatever block we read we will be returning it unless
953          // it is a datablock. Just in case the blocks are non data blocks
954          reader.returnBlock(block);
955        }
956      } while (!block.getBlockType().isData());
957
958      return block;
959    }
960
961    public DataBlockEncoding getEffectiveDataBlockEncoding() {
962      return this.reader.getEffectiveEncodingInCache(isCompaction);
963    }
964
965    @Override
966    public Cell getCell() {
967      if (!isSeeked())
968        return null;
969
970      Cell ret;
971      int cellBufSize = getKVBufSize();
972      long seqId = 0L;
973      if (this.reader.shouldIncludeMemStoreTS()) {
974        seqId = currMemstoreTS;
975      }
976      if (blockBuffer.hasArray()) {
977        // TODO : reduce the varieties of KV here. Check if based on a boolean
978        // we can handle the 'no tags' case.
979        if (currTagsLen > 0) {
980          ret = new SizeCachedKeyValue(blockBuffer.array(),
981              blockBuffer.arrayOffset() + blockBuffer.position(), cellBufSize, seqId);
982        } else {
983          ret = new SizeCachedNoTagsKeyValue(blockBuffer.array(),
984              blockBuffer.arrayOffset() + blockBuffer.position(), cellBufSize, seqId);
985        }
986      } else {
987        ByteBuffer buf = blockBuffer.asSubByteBuffer(cellBufSize);
988        if (buf.isDirect()) {
989          ret = currTagsLen > 0 ? new ByteBufferKeyValue(buf, buf.position(), cellBufSize, seqId)
990              : new NoTagsByteBufferKeyValue(buf, buf.position(), cellBufSize, seqId);
991        } else {
992          if (currTagsLen > 0) {
993            ret = new SizeCachedKeyValue(buf.array(), buf.arrayOffset() + buf.position(),
994                cellBufSize, seqId);
995          } else {
996            ret = new SizeCachedNoTagsKeyValue(buf.array(), buf.arrayOffset() + buf.position(),
997                cellBufSize, seqId);
998          }
999        }
1000      }
1001      return ret;
1002    }
1003
1004    @Override
1005    public Cell getKey() {
1006      assertSeeked();
1007      // Create a new object so that this getKey is cached as firstKey, lastKey
1008      ObjectIntPair<ByteBuffer> keyPair = new ObjectIntPair<>();
1009      blockBuffer.asSubByteBuffer(blockBuffer.position() + KEY_VALUE_LEN_SIZE, currKeyLen, keyPair);
1010      ByteBuffer keyBuf = keyPair.getFirst();
1011      if (keyBuf.hasArray()) {
1012        return new KeyValue.KeyOnlyKeyValue(keyBuf.array(), keyBuf.arrayOffset()
1013            + keyPair.getSecond(), currKeyLen);
1014      } else {
1015        // Better to do a copy here instead of holding on to this BB so that
1016        // we could release the blocks referring to this key. This key is specifically used
1017        // in HalfStoreFileReader to get the firstkey and lastkey by creating a new scanner
1018        // every time. So holding onto the BB (incase of DBB) is not advised here.
1019        byte[] key = new byte[currKeyLen];
1020        ByteBufferUtils.copyFromBufferToArray(key, keyBuf, keyPair.getSecond(), 0, currKeyLen);
1021        return new KeyValue.KeyOnlyKeyValue(key, 0, currKeyLen);
1022      }
1023    }
1024
1025    @Override
1026    public ByteBuffer getValue() {
1027      assertSeeked();
1028      // Okie to create new Pair. Not used in hot path
1029      ObjectIntPair<ByteBuffer> valuePair = new ObjectIntPair<>();
1030      this.blockBuffer.asSubByteBuffer(blockBuffer.position() + KEY_VALUE_LEN_SIZE + currKeyLen,
1031        currValueLen, valuePair);
1032      ByteBuffer valBuf = valuePair.getFirst().duplicate();
1033      valBuf.position(valuePair.getSecond());
1034      valBuf.limit(currValueLen + valuePair.getSecond());
1035      return valBuf.slice();
1036    }
1037
1038    protected void setNonSeekedState() {
1039      reset();
1040      blockBuffer = null;
1041      currKeyLen = 0;
1042      currValueLen = 0;
1043      currMemstoreTS = 0;
1044      currMemstoreTSLen = 0;
1045      currTagsLen = 0;
1046    }
1047
1048    /**
1049     * Set the position on current backing blockBuffer.
1050     */
1051    private void positionThisBlockBuffer() {
1052      try {
1053        blockBuffer.skip(getCurCellSerializedSize());
1054      } catch (IllegalArgumentException e) {
1055        LOG.error("Current pos = " + blockBuffer.position()
1056            + "; currKeyLen = " + currKeyLen + "; currValLen = "
1057            + currValueLen + "; block limit = " + blockBuffer.limit()
1058            + "; currBlock currBlockOffset = " + this.curBlock.getOffset()
1059            + "; path=" + reader.getPath());
1060        throw e;
1061      }
1062    }
1063
1064    /**
1065     * Set our selves up for the next 'next' invocation, set up next block.
1066     * @return True is more to read else false if at the end.
1067     * @throws IOException
1068     */
1069    private boolean positionForNextBlock() throws IOException {
1070      // Methods are small so they get inlined because they are 'hot'.
1071      long lastDataBlockOffset = reader.getTrailer().getLastDataBlockOffset();
1072      if (this.curBlock.getOffset() >= lastDataBlockOffset) {
1073        setNonSeekedState();
1074        return false;
1075      }
1076      return isNextBlock();
1077    }
1078
1079
1080    private boolean isNextBlock() throws IOException {
1081      // Methods are small so they get inlined because they are 'hot'.
1082      HFileBlock nextBlock = readNextDataBlock();
1083      if (nextBlock == null) {
1084        setNonSeekedState();
1085        return false;
1086      }
1087      updateCurrentBlock(nextBlock);
1088      return true;
1089    }
1090
1091    private final boolean _next() throws IOException {
1092      // Small method so can be inlined. It is a hot one.
1093      if (blockBuffer.remaining() <= 0) {
1094        return positionForNextBlock();
1095      }
1096
1097      // We are still in the same block.
1098      readKeyValueLen();
1099      return true;
1100    }
1101
1102    /**
1103     * Go to the next key/value in the block section. Loads the next block if
1104     * necessary. If successful, {@link #getKey()} and {@link #getValue()} can
1105     * be called.
1106     *
1107     * @return true if successfully navigated to the next key/value
1108     */
1109    @Override
1110    public boolean next() throws IOException {
1111      // This is a hot method so extreme measures taken to ensure it is small and inlineable.
1112      // Checked by setting: -XX:+UnlockDiagnosticVMOptions -XX:+PrintInlining -XX:+PrintCompilation
1113      assertSeeked();
1114      positionThisBlockBuffer();
1115      return _next();
1116    }
1117
1118    /**
1119     * Positions this scanner at the start of the file.
1120     *
1121     * @return false if empty file; i.e. a call to next would return false and
1122     *         the current key and value are undefined.
1123     * @throws IOException
1124     */
1125    @Override
1126    public boolean seekTo() throws IOException {
1127      if (reader == null) {
1128        return false;
1129      }
1130
1131      if (reader.getTrailer().getEntryCount() == 0) {
1132        // No data blocks.
1133        return false;
1134      }
1135
1136      long firstDataBlockOffset = reader.getTrailer().getFirstDataBlockOffset();
1137      if (curBlock != null
1138          && curBlock.getOffset() == firstDataBlockOffset) {
1139        return processFirstDataBlock();
1140      }
1141
1142      readAndUpdateNewBlock(firstDataBlockOffset);
1143      return true;
1144    }
1145
1146    protected boolean processFirstDataBlock() throws IOException{
1147      blockBuffer.rewind();
1148      readKeyValueLen();
1149      return true;
1150    }
1151
1152    protected void readAndUpdateNewBlock(long firstDataBlockOffset) throws IOException,
1153        CorruptHFileException {
1154      HFileBlock newBlock = reader.readBlock(firstDataBlockOffset, -1, cacheBlocks, pread,
1155          isCompaction, true, BlockType.DATA, getEffectiveDataBlockEncoding());
1156      if (newBlock.getOffset() < 0) {
1157        releaseIfNotCurBlock(newBlock);
1158        throw new IOException(
1159            "Invalid block offset: " + newBlock.getOffset() + ", path=" + reader.getPath());
1160      }
1161      updateCurrentBlock(newBlock);
1162    }
1163
1164    protected int loadBlockAndSeekToKey(HFileBlock seekToBlock, Cell nextIndexedKey,
1165        boolean rewind, Cell key, boolean seekBefore) throws IOException {
1166      if (this.curBlock == null
1167          || this.curBlock.getOffset() != seekToBlock.getOffset()) {
1168        updateCurrentBlock(seekToBlock);
1169      } else if (rewind) {
1170        blockBuffer.rewind();
1171      }
1172
1173      // Update the nextIndexedKey
1174      this.nextIndexedKey = nextIndexedKey;
1175      return blockSeek(key, seekBefore);
1176    }
1177
1178    /**
1179     * @param v
1180     * @return True if v &lt;= 0 or v &gt; current block buffer limit.
1181     */
1182    protected final boolean checkKeyLen(final int v) {
1183      return v <= 0 || v > this.blockBuffer.limit();
1184    }
1185
1186    /**
1187     * @param v
1188     * @return True if v &lt; 0 or v &gt; current block buffer limit.
1189     */
1190    protected final boolean checkLen(final int v) {
1191      return v < 0 || v > this.blockBuffer.limit();
1192    }
1193
1194    /**
1195     * Check key and value lengths are wholesome.
1196     */
1197    protected final void checkKeyValueLen() {
1198      if (checkKeyLen(this.currKeyLen) || checkLen(this.currValueLen)) {
1199        throw new IllegalStateException("Invalid currKeyLen " + this.currKeyLen
1200            + " or currValueLen " + this.currValueLen + ". Block offset: "
1201            + this.curBlock.getOffset() + ", block length: "
1202            + this.blockBuffer.limit() + ", position: " + this.blockBuffer.position()
1203            + " (without header)." + ", path=" + reader.getPath());
1204      }
1205    }
1206
1207    /**
1208     * Updates the current block to be the given {@link HFileBlock}. Seeks to
1209     * the the first key/value pair.
1210     *
1211     * @param newBlock the block to make current
1212     */
1213    protected void updateCurrentBlock(HFileBlock newBlock) throws IOException {
1214      try {
1215        // Set the active block on the reader
1216        // sanity check
1217        if (newBlock.getBlockType() != BlockType.DATA) {
1218          throw new IllegalStateException(
1219              "ScannerV2 works only on data " + "blocks, got " + newBlock.getBlockType() + "; "
1220                  + "HFileName=" + reader.getPath() + ", " + "dataBlockEncoder=" + reader
1221                  .getDataBlockEncoding() + ", " + "isCompaction=" + isCompaction);
1222        }
1223
1224        updateCurrBlockRef(newBlock);
1225        blockBuffer = newBlock.getBufferWithoutHeader();
1226        readKeyValueLen();
1227        blockFetches.incrementAndGet();
1228      } finally {
1229        releaseIfNotCurBlock(newBlock);
1230      }
1231
1232      // Reset the next indexed key
1233      this.nextIndexedKey = null;
1234    }
1235
1236    protected Cell getFirstKeyCellInBlock(HFileBlock curBlock) {
1237      ByteBuff buffer = curBlock.getBufferWithoutHeader();
1238      // It is safe to manipulate this buffer because we own the buffer object.
1239      buffer.rewind();
1240      int klen = buffer.getInt();
1241      buffer.skip(Bytes.SIZEOF_INT);// Skip value len part
1242      ByteBuffer keyBuff = buffer.asSubByteBuffer(klen);
1243      if (keyBuff.hasArray()) {
1244        return new KeyValue.KeyOnlyKeyValue(keyBuff.array(), keyBuff.arrayOffset()
1245            + keyBuff.position(), klen);
1246      } else {
1247        return new ByteBufferKeyOnlyKeyValue(keyBuff, keyBuff.position(), klen);
1248      }
1249    }
1250
1251    @Override
1252    public String getKeyString() {
1253      return CellUtil.toString(getKey(), false);
1254    }
1255
1256    @Override
1257    public String getValueString() {
1258      return ByteBufferUtils.toStringBinary(getValue());
1259    }
1260
1261    public int compareKey(CellComparator comparator, Cell key) {
1262      blockBuffer.asSubByteBuffer(blockBuffer.position() + KEY_VALUE_LEN_SIZE, currKeyLen, pair);
1263      this.bufBackedKeyOnlyKv.setKey(pair.getFirst(), pair.getSecond(), currKeyLen);
1264      return PrivateCellUtil.compareKeyIgnoresMvcc(comparator, key, this.bufBackedKeyOnlyKv);
1265    }
1266
1267    @Override
1268    public void shipped() throws IOException {
1269      this.returnBlocks(false);
1270    }
1271  }
1272
1273  @Override
1274  public Path getPath() {
1275    return path;
1276  }
1277
1278  @Override
1279  public DataBlockEncoding getDataBlockEncoding() {
1280    return dataBlockEncoder.getDataBlockEncoding();
1281  }
1282
1283  @Override
1284  public Configuration getConf() {
1285    return conf;
1286  }
1287
1288  @Override
1289  public void setConf(Configuration conf) {
1290    this.conf = conf;
1291  }
1292
1293  /** Minor versions in HFile starting with this number have hbase checksums */
1294  public static final int MINOR_VERSION_WITH_CHECKSUM = 1;
1295  /** In HFile minor version that does not support checksums */
1296  public static final int MINOR_VERSION_NO_CHECKSUM = 0;
1297
1298  /** HFile minor version that introduced pbuf filetrailer */
1299  public static final int PBUF_TRAILER_MINOR_VERSION = 2;
1300
1301  /**
1302   * The size of a (key length, value length) tuple that prefixes each entry in
1303   * a data block.
1304   */
1305  public final static int KEY_VALUE_LEN_SIZE = 2 * Bytes.SIZEOF_INT;
1306
1307  private boolean includesMemstoreTS = false;
1308  protected boolean decodeMemstoreTS = false;
1309
1310
1311  @Override
1312  public boolean isDecodeMemStoreTS() {
1313    return this.decodeMemstoreTS;
1314  }
1315
1316  @Override
1317  public boolean shouldIncludeMemStoreTS() {
1318    return includesMemstoreTS;
1319  }
1320
1321  /**
1322   * Retrieve block from cache. Validates the retrieved block's type vs {@code expectedBlockType}
1323   * and its encoding vs. {@code expectedDataBlockEncoding}. Unpacks the block as necessary.
1324   */
1325  private HFileBlock getCachedBlock(BlockCacheKey cacheKey, boolean cacheBlock, boolean useLock,
1326      boolean isCompaction, boolean updateCacheMetrics, BlockType expectedBlockType,
1327      DataBlockEncoding expectedDataBlockEncoding) throws IOException {
1328    // Check cache for block. If found return.
1329    BlockCache cache = cacheConf.getBlockCache().orElse(null);
1330    if (cache != null) {
1331      HFileBlock cachedBlock =
1332          (HFileBlock) cache.getBlock(cacheKey, cacheBlock, useLock, updateCacheMetrics);
1333      if (cachedBlock != null) {
1334        if (cacheConf.shouldCacheCompressed(cachedBlock.getBlockType().getCategory())) {
1335          HFileBlock compressedBlock = cachedBlock;
1336          cachedBlock = compressedBlock.unpack(hfileContext, fsBlockReader);
1337          // In case of compressed block after unpacking we can return the compressed block
1338          if (compressedBlock != cachedBlock) {
1339            cache.returnBlock(cacheKey, compressedBlock);
1340          }
1341        }
1342        try {
1343          validateBlockType(cachedBlock, expectedBlockType);
1344        } catch (IOException e) {
1345          returnAndEvictBlock(cache, cacheKey, cachedBlock);
1346          throw e;
1347        }
1348
1349        if (expectedDataBlockEncoding == null) {
1350          return cachedBlock;
1351        }
1352        DataBlockEncoding actualDataBlockEncoding = cachedBlock.getDataBlockEncoding();
1353        // Block types other than data blocks always have
1354        // DataBlockEncoding.NONE. To avoid false negative cache misses, only
1355        // perform this check if cached block is a data block.
1356        if (cachedBlock.getBlockType().isData() &&
1357            !actualDataBlockEncoding.equals(expectedDataBlockEncoding)) {
1358          // This mismatch may happen if a Scanner, which is used for say a
1359          // compaction, tries to read an encoded block from the block cache.
1360          // The reverse might happen when an EncodedScanner tries to read
1361          // un-encoded blocks which were cached earlier.
1362          //
1363          // Because returning a data block with an implicit BlockType mismatch
1364          // will cause the requesting scanner to throw a disk read should be
1365          // forced here. This will potentially cause a significant number of
1366          // cache misses, so update so we should keep track of this as it might
1367          // justify the work on a CompoundScanner.
1368          if (!expectedDataBlockEncoding.equals(DataBlockEncoding.NONE) &&
1369              !actualDataBlockEncoding.equals(DataBlockEncoding.NONE)) {
1370            // If the block is encoded but the encoding does not match the
1371            // expected encoding it is likely the encoding was changed but the
1372            // block was not yet evicted. Evictions on file close happen async
1373            // so blocks with the old encoding still linger in cache for some
1374            // period of time. This event should be rare as it only happens on
1375            // schema definition change.
1376            LOG.info("Evicting cached block with key " + cacheKey
1377                + " because of a data block encoding mismatch" + "; expected: "
1378                + expectedDataBlockEncoding + ", actual: " + actualDataBlockEncoding + ", path="
1379                + path);
1380            // This is an error scenario. so here we need to decrement the
1381            // count.
1382            returnAndEvictBlock(cache, cacheKey, cachedBlock);
1383          }
1384          return null;
1385        }
1386        return cachedBlock;
1387      }
1388    }
1389    return null;
1390  }
1391
1392  private void returnAndEvictBlock(BlockCache cache, BlockCacheKey cacheKey, Cacheable block) {
1393    cache.returnBlock(cacheKey, block);
1394    cache.evictBlock(cacheKey);
1395  }
1396
1397  /**
1398   * @param metaBlockName
1399   * @param cacheBlock Add block to cache, if found
1400   * @return block wrapped in a ByteBuffer, with header skipped
1401   * @throws IOException
1402   */
1403  @Override
1404  public HFileBlock getMetaBlock(String metaBlockName, boolean cacheBlock)
1405      throws IOException {
1406    if (trailer.getMetaIndexCount() == 0) {
1407      return null; // there are no meta blocks
1408    }
1409    if (metaBlockIndexReader == null) {
1410      throw new IOException(path + " meta index not loaded");
1411    }
1412
1413    byte[] mbname = Bytes.toBytes(metaBlockName);
1414    int block = metaBlockIndexReader.rootBlockContainingKey(mbname,
1415        0, mbname.length);
1416    if (block == -1)
1417      return null;
1418    long blockSize = metaBlockIndexReader.getRootBlockDataSize(block);
1419
1420    // Per meta key from any given file, synchronize reads for said block. This
1421    // is OK to do for meta blocks because the meta block index is always
1422    // single-level.
1423    synchronized (metaBlockIndexReader
1424        .getRootBlockKey(block)) {
1425      // Check cache for block. If found return.
1426      long metaBlockOffset = metaBlockIndexReader.getRootBlockOffset(block);
1427      BlockCacheKey cacheKey = new BlockCacheKey(name, metaBlockOffset,
1428        this.isPrimaryReplicaReader(), BlockType.META);
1429
1430      cacheBlock &= cacheConf.shouldCacheBlockOnRead(BlockType.META.getCategory());
1431      HFileBlock cachedBlock =
1432          getCachedBlock(cacheKey, cacheBlock, false, true, true, BlockType.META, null);
1433      if (cachedBlock != null) {
1434        assert cachedBlock.isUnpacked() : "Packed block leak.";
1435        // Return a distinct 'shallow copy' of the block,
1436        // so pos does not get messed by the scanner
1437        return cachedBlock;
1438      }
1439      // Cache Miss, please load.
1440
1441      HFileBlock metaBlock = fsBlockReader.readBlockData(metaBlockOffset, blockSize, true, false).
1442          unpack(hfileContext, fsBlockReader);
1443
1444      // Cache the block
1445      if (cacheBlock) {
1446        cacheConf.getBlockCache()
1447            .ifPresent(cache -> cache.cacheBlock(cacheKey, metaBlock, cacheConf.isInMemory()));
1448      }
1449      return metaBlock;
1450    }
1451  }
1452
1453  @Override
1454  public HFileBlock readBlock(long dataBlockOffset, long onDiskBlockSize,
1455      final boolean cacheBlock, boolean pread, final boolean isCompaction,
1456      boolean updateCacheMetrics, BlockType expectedBlockType,
1457      DataBlockEncoding expectedDataBlockEncoding)
1458      throws IOException {
1459    if (dataBlockIndexReader == null) {
1460      throw new IOException(path + " block index not loaded");
1461    }
1462    long trailerOffset = trailer.getLoadOnOpenDataOffset();
1463    if (dataBlockOffset < 0 || dataBlockOffset >= trailerOffset) {
1464      throw new IOException("Requested block is out of range: " + dataBlockOffset +
1465        ", lastDataBlockOffset: " + trailer.getLastDataBlockOffset() +
1466        ", trailer.getLoadOnOpenDataOffset: " + trailerOffset +
1467        ", path=" + path);
1468    }
1469    // For any given block from any given file, synchronize reads for said
1470    // block.
1471    // Without a cache, this synchronizing is needless overhead, but really
1472    // the other choice is to duplicate work (which the cache would prevent you
1473    // from doing).
1474
1475    BlockCacheKey cacheKey = new BlockCacheKey(name, dataBlockOffset,
1476      this.isPrimaryReplicaReader(), expectedBlockType);
1477
1478    boolean useLock = false;
1479    IdLock.Entry lockEntry = null;
1480    try (TraceScope traceScope = TraceUtil.createTrace("HFileReaderImpl.readBlock")) {
1481      while (true) {
1482        // Check cache for block. If found return.
1483        if (cacheConf.shouldReadBlockFromCache(expectedBlockType)) {
1484          if (useLock) {
1485            lockEntry = offsetLock.getLockEntry(dataBlockOffset);
1486          }
1487          // Try and get the block from the block cache. If the useLock variable is true then this
1488          // is the second time through the loop and it should not be counted as a block cache miss.
1489          HFileBlock cachedBlock = getCachedBlock(cacheKey, cacheBlock, useLock, isCompaction,
1490            updateCacheMetrics, expectedBlockType, expectedDataBlockEncoding);
1491          if (cachedBlock != null) {
1492            if (LOG.isTraceEnabled()) {
1493              LOG.trace("From Cache " + cachedBlock);
1494            }
1495            TraceUtil.addTimelineAnnotation("blockCacheHit");
1496            assert cachedBlock.isUnpacked() : "Packed block leak.";
1497            if (cachedBlock.getBlockType().isData()) {
1498              if (updateCacheMetrics) {
1499                HFile.DATABLOCK_READ_COUNT.increment();
1500              }
1501              // Validate encoding type for data blocks. We include encoding
1502              // type in the cache key, and we expect it to match on a cache hit.
1503              if (cachedBlock.getDataBlockEncoding() != dataBlockEncoder.getDataBlockEncoding()) {
1504                cacheConf.getBlockCache().ifPresent(cache -> {
1505                  returnAndEvictBlock(cache, cacheKey, cachedBlock);
1506                });
1507                throw new IOException(
1508                    "Cached block under key " + cacheKey + " " + "has wrong encoding: "
1509                        + cachedBlock.getDataBlockEncoding() + " (expected: " + dataBlockEncoder
1510                        .getDataBlockEncoding() + ")" + ", path=" + path);
1511              }
1512            }
1513            // Cache-hit. Return!
1514            return cachedBlock;
1515          }
1516
1517          if (!useLock && cacheBlock && cacheConf.shouldLockOnCacheMiss(expectedBlockType)) {
1518            // check cache again with lock
1519            useLock = true;
1520            continue;
1521          }
1522          // Carry on, please load.
1523        }
1524
1525        TraceUtil.addTimelineAnnotation("blockCacheMiss");
1526        // Load block from filesystem.
1527        HFileBlock hfileBlock =
1528            fsBlockReader.readBlockData(dataBlockOffset, onDiskBlockSize, pread, !isCompaction);
1529        validateBlockType(hfileBlock, expectedBlockType);
1530        HFileBlock unpacked = hfileBlock.unpack(hfileContext, fsBlockReader);
1531        BlockType.BlockCategory category = hfileBlock.getBlockType().getCategory();
1532
1533        // Cache the block if necessary
1534        cacheConf.getBlockCache().ifPresent(cache -> {
1535          if (cacheBlock && cacheConf.shouldCacheBlockOnRead(category)) {
1536            cache.cacheBlock(cacheKey,
1537                cacheConf.shouldCacheCompressed(category) ? hfileBlock : unpacked,
1538                cacheConf.isInMemory());
1539          }
1540        });
1541
1542        if (updateCacheMetrics && hfileBlock.getBlockType().isData()) {
1543          HFile.DATABLOCK_READ_COUNT.increment();
1544        }
1545
1546        return unpacked;
1547      }
1548    } finally {
1549      if (lockEntry != null) {
1550        offsetLock.releaseLockEntry(lockEntry);
1551      }
1552    }
1553  }
1554
1555  @Override
1556  public boolean hasMVCCInfo() {
1557    return includesMemstoreTS && decodeMemstoreTS;
1558  }
1559
1560  /**
1561   * Compares the actual type of a block retrieved from cache or disk with its
1562   * expected type and throws an exception in case of a mismatch. Expected
1563   * block type of {@link BlockType#DATA} is considered to match the actual
1564   * block type [@link {@link BlockType#ENCODED_DATA} as well.
1565   * @param block a block retrieved from cache or disk
1566   * @param expectedBlockType the expected block type, or null to skip the
1567   *          check
1568   */
1569  private void validateBlockType(HFileBlock block,
1570      BlockType expectedBlockType) throws IOException {
1571    if (expectedBlockType == null) {
1572      return;
1573    }
1574    BlockType actualBlockType = block.getBlockType();
1575    if (expectedBlockType.isData() && actualBlockType.isData()) {
1576      // We consider DATA to match ENCODED_DATA for the purpose of this
1577      // verification.
1578      return;
1579    }
1580    if (actualBlockType != expectedBlockType) {
1581      throw new IOException("Expected block type " + expectedBlockType + ", " +
1582          "but got " + actualBlockType + ": " + block + ", path=" + path);
1583    }
1584  }
1585
1586  /**
1587   * @return Last key as cell in the file. May be null if file has no entries. Note that
1588   *         this is not the last row key, but it is the Cell representation of the last
1589   *         key
1590   */
1591  @Override
1592  public Optional<Cell> getLastKey() {
1593    return dataBlockIndexReader.isEmpty() ? Optional.empty() : Optional.of(lastKeyCell);
1594  }
1595
1596  /**
1597   * @return Midkey for this file. We work with block boundaries only so
1598   *         returned midkey is an approximation only.
1599   * @throws IOException
1600   */
1601  @Override
1602  public Optional<Cell> midKey() throws IOException {
1603    return Optional.ofNullable(dataBlockIndexReader.midkey());
1604  }
1605
1606  @Override
1607  public void close() throws IOException {
1608    close(cacheConf.shouldEvictOnClose());
1609  }
1610
1611  @Override
1612  public void close(boolean evictOnClose) throws IOException {
1613    PrefetchExecutor.cancel(path);
1614    cacheConf.getBlockCache().ifPresent(cache -> {
1615      if (evictOnClose) {
1616        int numEvicted = cache.evictBlocksByHfileName(name);
1617        if (LOG.isTraceEnabled()) {
1618          LOG.trace("On close, file=" + name + " evicted=" + numEvicted + " block(s)");
1619        }
1620      }
1621    });
1622    fsBlockReader.closeStreams();
1623  }
1624
1625  @Override
1626  public DataBlockEncoding getEffectiveEncodingInCache(boolean isCompaction) {
1627    return dataBlockEncoder.getEffectiveEncodingInCache(isCompaction);
1628  }
1629
1630  /** For testing */
1631  @Override
1632  public HFileBlock.FSReader getUncachedBlockReader() {
1633    return fsBlockReader;
1634  }
1635
1636  /**
1637   * Scanner that operates on encoded data blocks.
1638   */
1639  protected static class EncodedScanner extends HFileScannerImpl {
1640    private final HFileBlockDecodingContext decodingCtx;
1641    private final DataBlockEncoder.EncodedSeeker seeker;
1642    private final DataBlockEncoder dataBlockEncoder;
1643
1644    public EncodedScanner(HFile.Reader reader, boolean cacheBlocks,
1645        boolean pread, boolean isCompaction, HFileContext meta) {
1646      super(reader, cacheBlocks, pread, isCompaction);
1647      DataBlockEncoding encoding = reader.getDataBlockEncoding();
1648      dataBlockEncoder = encoding.getEncoder();
1649      decodingCtx = dataBlockEncoder.newDataBlockDecodingContext(meta);
1650      seeker = dataBlockEncoder.createSeeker(
1651        reader.getComparator(), decodingCtx);
1652    }
1653
1654    @Override
1655    public boolean isSeeked(){
1656      return curBlock != null;
1657    }
1658
1659    @Override
1660    public void setNonSeekedState() {
1661      reset();
1662    }
1663
1664    /**
1665     * Updates the current block to be the given {@link HFileBlock}. Seeks to
1666     * the the first key/value pair.
1667     *
1668     * @param newBlock the block to make current
1669     * @throws CorruptHFileException
1670     */
1671    @Override
1672    protected void updateCurrentBlock(HFileBlock newBlock) throws CorruptHFileException {
1673      try {
1674        // sanity checks
1675        if (newBlock.getBlockType() != BlockType.ENCODED_DATA) {
1676          throw new IllegalStateException("EncodedScanner works only on encoded data blocks");
1677        }
1678        short dataBlockEncoderId = newBlock.getDataBlockEncodingId();
1679        if (!DataBlockEncoding.isCorrectEncoder(dataBlockEncoder, dataBlockEncoderId)) {
1680          String encoderCls = dataBlockEncoder.getClass().getName();
1681          throw new CorruptHFileException(
1682              "Encoder " + encoderCls + " doesn't support data block encoding " + DataBlockEncoding
1683                  .getNameFromId(dataBlockEncoderId) + ", path=" + reader.getPath());
1684        }
1685        updateCurrBlockRef(newBlock);
1686        ByteBuff encodedBuffer = getEncodedBuffer(newBlock);
1687        seeker.setCurrentBuffer(encodedBuffer);
1688        blockFetches.incrementAndGet();
1689      } finally {
1690        releaseIfNotCurBlock(newBlock);
1691      }
1692
1693      // Reset the next indexed key
1694      this.nextIndexedKey = null;
1695    }
1696
1697    private ByteBuff getEncodedBuffer(HFileBlock newBlock) {
1698      ByteBuff origBlock = newBlock.getBufferReadOnly();
1699      int pos = newBlock.headerSize() + DataBlockEncoding.ID_SIZE;
1700      origBlock.position(pos);
1701      origBlock
1702          .limit(pos + newBlock.getUncompressedSizeWithoutHeader() - DataBlockEncoding.ID_SIZE);
1703      return origBlock.slice();
1704    }
1705
1706    @Override
1707    protected boolean processFirstDataBlock() throws IOException {
1708      seeker.rewind();
1709      return true;
1710    }
1711
1712    @Override
1713    public boolean next() throws IOException {
1714      boolean isValid = seeker.next();
1715      if (!isValid) {
1716        HFileBlock newBlock = readNextDataBlock();
1717        isValid = newBlock != null;
1718        if (isValid) {
1719          updateCurrentBlock(newBlock);
1720        } else {
1721          setNonSeekedState();
1722        }
1723      }
1724      return isValid;
1725    }
1726
1727    @Override
1728    public Cell getKey() {
1729      assertValidSeek();
1730      return seeker.getKey();
1731    }
1732
1733    @Override
1734    public ByteBuffer getValue() {
1735      assertValidSeek();
1736      return seeker.getValueShallowCopy();
1737    }
1738
1739    @Override
1740    public Cell getCell() {
1741      if (this.curBlock == null) {
1742        return null;
1743      }
1744      return seeker.getCell();
1745    }
1746
1747    @Override
1748    public String getKeyString() {
1749      return CellUtil.toString(getKey(), true);
1750    }
1751
1752    @Override
1753    public String getValueString() {
1754      ByteBuffer valueBuffer = getValue();
1755      return ByteBufferUtils.toStringBinary(valueBuffer);
1756    }
1757
1758    private void assertValidSeek() {
1759      if (this.curBlock == null) {
1760        throw new NotSeekedException(reader.getPath());
1761      }
1762    }
1763
1764    @Override
1765    protected Cell getFirstKeyCellInBlock(HFileBlock curBlock) {
1766      return dataBlockEncoder.getFirstKeyCellInBlock(getEncodedBuffer(curBlock));
1767    }
1768
1769    @Override
1770    protected int loadBlockAndSeekToKey(HFileBlock seekToBlock, Cell nextIndexedKey,
1771        boolean rewind, Cell key, boolean seekBefore) throws IOException {
1772      if (this.curBlock == null
1773          || this.curBlock.getOffset() != seekToBlock.getOffset()) {
1774        updateCurrentBlock(seekToBlock);
1775      } else if (rewind) {
1776        seeker.rewind();
1777      }
1778      this.nextIndexedKey = nextIndexedKey;
1779      return seeker.seekToKeyInBlock(key, seekBefore);
1780    }
1781
1782    @Override
1783    public int compareKey(CellComparator comparator, Cell key) {
1784      return seeker.compareKey(comparator, key);
1785    }
1786  }
1787
1788  /**
1789   * Returns a buffer with the Bloom filter metadata. The caller takes
1790   * ownership of the buffer.
1791   */
1792  @Override
1793  public DataInput getGeneralBloomFilterMetadata() throws IOException {
1794    return this.getBloomFilterMetadata(BlockType.GENERAL_BLOOM_META);
1795  }
1796
1797  @Override
1798  public DataInput getDeleteBloomFilterMetadata() throws IOException {
1799    return this.getBloomFilterMetadata(BlockType.DELETE_FAMILY_BLOOM_META);
1800  }
1801
1802  private DataInput getBloomFilterMetadata(BlockType blockType)
1803  throws IOException {
1804    if (blockType != BlockType.GENERAL_BLOOM_META &&
1805        blockType != BlockType.DELETE_FAMILY_BLOOM_META) {
1806      throw new RuntimeException("Block Type: " + blockType.toString() +
1807          " is not supported, path=" + path) ;
1808    }
1809
1810    for (HFileBlock b : loadOnOpenBlocks)
1811      if (b.getBlockType() == blockType)
1812        return b.getByteStream();
1813    return null;
1814  }
1815
1816  public boolean isFileInfoLoaded() {
1817    return true; // We load file info in constructor in version 2.
1818  }
1819
1820  @Override
1821  public HFileContext getFileContext() {
1822    return hfileContext;
1823  }
1824
1825  /**
1826   * Returns false if block prefetching was requested for this file and has
1827   * not completed, true otherwise
1828   */
1829  @Override
1830  @VisibleForTesting
1831  public boolean prefetchComplete() {
1832    return PrefetchExecutor.isCompleted(path);
1833  }
1834
1835  protected HFileContext createHFileContext(FSDataInputStreamWrapper fsdis, long fileSize,
1836      HFileSystem hfs, Path path, FixedFileTrailer trailer) throws IOException {
1837    HFileContextBuilder builder = new HFileContextBuilder()
1838      .withIncludesMvcc(shouldIncludeMemStoreTS())
1839      .withHBaseCheckSum(true)
1840      .withHFileName(this.getName())
1841      .withCompression(this.compressAlgo);
1842
1843    // Check for any key material available
1844    byte[] keyBytes = trailer.getEncryptionKey();
1845    if (keyBytes != null) {
1846      Encryption.Context cryptoContext = Encryption.newContext(conf);
1847      Key key;
1848      key = EncryptionUtil.unwrapKey(conf, keyBytes);
1849      // Use the algorithm the key wants
1850      Cipher cipher = Encryption.getCipher(conf, key.getAlgorithm());
1851      if (cipher == null) {
1852        throw new IOException("Cipher '" + key.getAlgorithm() + "' is not available"
1853            + ", path=" + path);
1854      }
1855      cryptoContext.setCipher(cipher);
1856      cryptoContext.setKey(key);
1857      builder.withEncryptionContext(cryptoContext);
1858    }
1859
1860    HFileContext context = builder.build();
1861
1862    if (LOG.isTraceEnabled()) {
1863      LOG.trace("Reader" + (path != null? " for " + path: "") +
1864        " initialized with cacheConf: " + cacheConf +
1865        " comparator: " + comparator.getClass().getSimpleName() +
1866        " fileContext: " + context);
1867    }
1868
1869    return context;
1870  }
1871
1872  /**
1873   * Create a Scanner on this file. No seeks or reads are done on creation. Call
1874   * {@link HFileScanner#seekTo(Cell)} to position an start the read. There is
1875   * nothing to clean up in a Scanner. Letting go of your references to the
1876   * scanner is sufficient. NOTE: Do not use this overload of getScanner for
1877   * compactions. See {@link #getScanner(boolean, boolean, boolean)}
1878   *
1879   * @param cacheBlocks True if we should cache blocks read in by this scanner.
1880   * @param pread Use positional read rather than seek+read if true (pread is
1881   *          better for random reads, seek+read is better scanning).
1882   * @return Scanner on this file.
1883   */
1884  @Override
1885  @VisibleForTesting
1886  public HFileScanner getScanner(boolean cacheBlocks, final boolean pread) {
1887    return getScanner(cacheBlocks, pread, false);
1888  }
1889
1890  /**
1891   * Create a Scanner on this file. No seeks or reads are done on creation. Call
1892   * {@link HFileScanner#seekTo(Cell)} to position an start the read. There is
1893   * nothing to clean up in a Scanner. Letting go of your references to the
1894   * scanner is sufficient.
1895   * @param cacheBlocks
1896   *          True if we should cache blocks read in by this scanner.
1897   * @param pread
1898   *          Use positional read rather than seek+read if true (pread is better
1899   *          for random reads, seek+read is better scanning).
1900   * @param isCompaction
1901   *          is scanner being used for a compaction?
1902   * @return Scanner on this file.
1903   */
1904  @Override
1905  public HFileScanner getScanner(boolean cacheBlocks, final boolean pread,
1906      final boolean isCompaction) {
1907    if (dataBlockEncoder.useEncodedScanner()) {
1908      return new EncodedScanner(this, cacheBlocks, pread, isCompaction, this.hfileContext);
1909    }
1910    return new HFileScannerImpl(this, cacheBlocks, pread, isCompaction);
1911  }
1912
1913  public int getMajorVersion() {
1914    return 3;
1915  }
1916
1917  @Override
1918  public void unbufferStream() {
1919    fsBlockReader.unbufferStream();
1920  }
1921}