001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.io.hfile;
019
020import java.io.DataInput;
021import java.io.IOException;
022import java.nio.ByteBuffer;
023import java.security.Key;
024import java.util.ArrayList;
025import java.util.List;
026import java.util.Optional;
027import java.util.concurrent.atomic.AtomicInteger;
028
029import org.apache.hadoop.conf.Configurable;
030import org.apache.hadoop.conf.Configuration;
031import org.apache.hadoop.fs.Path;
032import org.apache.hadoop.hbase.ByteBufferKeyOnlyKeyValue;
033import org.apache.hadoop.hbase.Cell;
034import org.apache.hadoop.hbase.CellComparator;
035import org.apache.hadoop.hbase.CellUtil;
036import org.apache.hadoop.hbase.HConstants;
037import org.apache.hadoop.hbase.PrivateCellUtil;
038import org.apache.hadoop.hbase.KeyValue;
039import org.apache.hadoop.hbase.ByteBufferKeyValue;
040import org.apache.hadoop.hbase.SizeCachedKeyValue;
041import org.apache.hadoop.hbase.SizeCachedNoTagsKeyValue;
042import org.apache.hadoop.hbase.trace.TraceUtil;
043import org.apache.yetus.audience.InterfaceAudience;
044import org.slf4j.Logger;
045import org.slf4j.LoggerFactory;
046import org.apache.hadoop.hbase.fs.HFileSystem;
047import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
048import org.apache.hadoop.hbase.io.compress.Compression;
049import org.apache.hadoop.hbase.io.crypto.Cipher;
050import org.apache.hadoop.hbase.io.crypto.Encryption;
051import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder;
052import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
053import org.apache.hadoop.hbase.io.encoding.HFileBlockDecodingContext;
054import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
055import org.apache.hadoop.hbase.nio.ByteBuff;
056import org.apache.hadoop.hbase.regionserver.KeyValueScanner;
057import org.apache.hadoop.hbase.security.EncryptionUtil;
058import org.apache.hadoop.hbase.util.ByteBufferUtils;
059import org.apache.hadoop.hbase.util.Bytes;
060import org.apache.hadoop.hbase.util.IdLock;
061import org.apache.hadoop.hbase.util.ObjectIntPair;
062import org.apache.hadoop.io.WritableUtils;
063import org.apache.htrace.core.TraceScope;
064
065import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
066
067/**
068 * Implementation that can handle all hfile versions of {@link HFile.Reader}.
069 */
070@InterfaceAudience.Private
071@edu.umd.cs.findbugs.annotations.SuppressWarnings(value="URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD")
072public class HFileReaderImpl implements HFile.Reader, Configurable {
073  // This class is HFileReaderV3 + HFileReaderV2 + AbstractHFileReader all squashed together into
074  // one file.  Ditto for all the HFileReader.ScannerV? implementations. I was running up against
075  // the MaxInlineLevel limit because too many tiers involved reading from an hfile. Was also hard
076  // to navigate the source code when so many classes participating in read.
077  private static final Logger LOG = LoggerFactory.getLogger(HFileReaderImpl.class);
078
079  /** Data block index reader keeping the root data index in memory */
080  private HFileBlockIndex.CellBasedKeyBlockIndexReader dataBlockIndexReader;
081
082  /** Meta block index reader -- always single level */
083  private HFileBlockIndex.ByteArrayKeyBlockIndexReader metaBlockIndexReader;
084
085  private final FixedFileTrailer trailer;
086
087  /** Filled when we read in the trailer. */
088  private final Compression.Algorithm compressAlgo;
089
090  private final boolean primaryReplicaReader;
091
092  /**
093   * What kind of data block encoding should be used while reading, writing,
094   * and handling cache.
095   */
096  private HFileDataBlockEncoder dataBlockEncoder = NoOpDataBlockEncoder.INSTANCE;
097
098  /** Last key in the file. Filled in when we read in the file info */
099  private Cell lastKeyCell = null;
100
101  /** Average key length read from file info */
102  private int avgKeyLen = -1;
103
104  /** Average value length read from file info */
105  private int avgValueLen = -1;
106
107  /** Key comparator */
108  private CellComparator comparator = CellComparator.getInstance();
109
110  /** Size of this file. */
111  private final long fileSize;
112
113  /** Block cache configuration. */
114  private final CacheConfig cacheConf;
115
116  /** Path of file */
117  private final Path path;
118
119  /** File name to be used for block names */
120  private final String name;
121
122  private FileInfo fileInfo;
123
124  private Configuration conf;
125
126  private HFileContext hfileContext;
127
128  /** Filesystem-level block reader. */
129  private HFileBlock.FSReader fsBlockReader;
130
131  /**
132   * A "sparse lock" implementation allowing to lock on a particular block
133   * identified by offset. The purpose of this is to avoid two clients loading
134   * the same block, and have all but one client wait to get the block from the
135   * cache.
136   */
137  private IdLock offsetLock = new IdLock();
138
139  /**
140   * Blocks read from the load-on-open section, excluding data root index, meta
141   * index, and file info.
142   */
143  private List<HFileBlock> loadOnOpenBlocks = new ArrayList<>();
144
145  /** Minimum minor version supported by this HFile format */
146  static final int MIN_MINOR_VERSION = 0;
147
148  /** Maximum minor version supported by this HFile format */
149  // We went to version 2 when we moved to pb'ing fileinfo and the trailer on
150  // the file. This version can read Writables version 1.
151  static final int MAX_MINOR_VERSION = 3;
152
153  /**
154   * We can read files whose major version is v2 IFF their minor version is at least 3.
155   */
156  private static final int MIN_V2_MINOR_VERSION_WITH_PB = 3;
157
158  /** Minor versions starting with this number have faked index key */
159  static final int MINOR_VERSION_WITH_FAKED_KEY = 3;
160
161  @VisibleForTesting
162  @Deprecated
163  public HFileReaderImpl(Path path, FixedFileTrailer trailer, FSDataInputStreamWrapper fsdis,
164      long fileSize, CacheConfig cacheConf, HFileSystem hfs, Configuration conf)
165      throws IOException {
166    this(path, trailer, fsdis, fileSize, cacheConf, hfs, true, conf);
167  }
168
169  /**
170   * Opens a HFile. You must load the index before you can use it by calling
171   * {@link #loadFileInfo()}.
172   * @param path
173   *          Path to HFile.
174   * @param trailer
175   *          File trailer.
176   * @param fsdis
177   *          input stream.
178   * @param fileSize
179   *          Length of the stream.
180   * @param cacheConf
181   *          Cache configuration.
182   * @param hfs
183   *          The file system.
184   * @param conf
185   *          Configuration
186   */
187  @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD")
188  public HFileReaderImpl(Path path, FixedFileTrailer trailer, FSDataInputStreamWrapper fsdis,
189      long fileSize, CacheConfig cacheConf, HFileSystem hfs, boolean primaryReplicaReader,
190      Configuration conf) throws IOException {
191    this.trailer = trailer;
192    this.compressAlgo = trailer.getCompressionCodec();
193    this.cacheConf = cacheConf;
194    this.fileSize = fileSize;
195    this.path = path;
196    this.name = path.getName();
197    this.conf = conf;
198    this.primaryReplicaReader = primaryReplicaReader;
199    checkFileVersion();
200    this.hfileContext = createHFileContext(fsdis, fileSize, hfs, path, trailer);
201    this.fsBlockReader = new HFileBlock.FSReaderImpl(fsdis, fileSize, hfs, path, hfileContext);
202
203    // Comparator class name is stored in the trailer in version 2.
204    comparator = trailer.createComparator();
205    dataBlockIndexReader = new HFileBlockIndex.CellBasedKeyBlockIndexReader(comparator,
206        trailer.getNumDataIndexLevels(), this);
207    metaBlockIndexReader = new HFileBlockIndex.ByteArrayKeyBlockIndexReader(1);
208
209    // Parse load-on-open data.
210
211    HFileBlock.BlockIterator blockIter = fsBlockReader.blockRange(
212        trailer.getLoadOnOpenDataOffset(),
213        fileSize - trailer.getTrailerSize());
214
215    // Data index. We also read statistics about the block index written after
216    // the root level.
217    dataBlockIndexReader.readMultiLevelIndexRoot(
218        blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX),
219        trailer.getDataIndexCount());
220
221    // Meta index.
222    metaBlockIndexReader.readRootIndex(
223        blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX),
224        trailer.getMetaIndexCount());
225
226    // File info
227    fileInfo = new FileInfo();
228    fileInfo.read(blockIter.nextBlockWithBlockType(BlockType.FILE_INFO).getByteStream());
229    byte[] creationTimeBytes = fileInfo.get(FileInfo.CREATE_TIME_TS);
230    this.hfileContext.setFileCreateTime(creationTimeBytes == null?  0:
231        Bytes.toLong(creationTimeBytes));
232    if (fileInfo.get(FileInfo.LASTKEY) != null) {
233      lastKeyCell = new KeyValue.KeyOnlyKeyValue(fileInfo.get(FileInfo.LASTKEY));
234    }
235    avgKeyLen = Bytes.toInt(fileInfo.get(FileInfo.AVG_KEY_LEN));
236    avgValueLen = Bytes.toInt(fileInfo.get(FileInfo.AVG_VALUE_LEN));
237    byte [] keyValueFormatVersion = fileInfo.get(HFileWriterImpl.KEY_VALUE_VERSION);
238    includesMemstoreTS = keyValueFormatVersion != null &&
239        Bytes.toInt(keyValueFormatVersion) == HFileWriterImpl.KEY_VALUE_VER_WITH_MEMSTORE;
240    fsBlockReader.setIncludesMemStoreTS(includesMemstoreTS);
241    if (includesMemstoreTS) {
242      decodeMemstoreTS = Bytes.toLong(fileInfo.get(HFileWriterImpl.MAX_MEMSTORE_TS_KEY)) > 0;
243    }
244
245    // Read data block encoding algorithm name from file info.
246    dataBlockEncoder = HFileDataBlockEncoderImpl.createFromFileInfo(fileInfo);
247    fsBlockReader.setDataBlockEncoder(dataBlockEncoder);
248
249    // Store all other load-on-open blocks for further consumption.
250    HFileBlock b;
251    while ((b = blockIter.nextBlock()) != null) {
252      loadOnOpenBlocks.add(b);
253    }
254
255    // Prefetch file blocks upon open if requested
256    if (cacheConf.shouldPrefetchOnOpen()) {
257      PrefetchExecutor.request(path, new Runnable() {
258        @Override
259        public void run() {
260          long offset = 0;
261          long end = 0;
262          try {
263            end = getTrailer().getLoadOnOpenDataOffset();
264            if (LOG.isTraceEnabled()) {
265              LOG.trace("Prefetch start " + getPathOffsetEndStr(path, offset, end));
266            }
267            // TODO: Could we use block iterator in here? Would that get stuff into the cache?
268            HFileBlock prevBlock = null;
269            while (offset < end) {
270              if (Thread.interrupted()) {
271                break;
272              }
273              // Perhaps we got our block from cache? Unlikely as this may be, if it happens, then
274              // the internal-to-hfileblock thread local which holds the overread that gets the
275              // next header, will not have happened...so, pass in the onDiskSize gotten from the
276              // cached block. This 'optimization' triggers extremely rarely I'd say.
277              long onDiskSize = prevBlock != null? prevBlock.getNextBlockOnDiskSize(): -1;
278              HFileBlock block = readBlock(offset, onDiskSize, /*cacheBlock=*/true,
279                  /*pread=*/true, false, false, null, null);
280              // Need not update the current block. Ideally here the readBlock won't find the
281              // block in cache. We call this readBlock so that block data is read from FS and
282              // cached in BC. So there is no reference count increment that happens here.
283              // The return will ideally be a noop because the block is not of MemoryType SHARED.
284              returnBlock(block);
285              prevBlock = block;
286              offset += block.getOnDiskSizeWithHeader();
287            }
288          } catch (IOException e) {
289            // IOExceptions are probably due to region closes (relocation, etc.)
290            if (LOG.isTraceEnabled()) {
291              LOG.trace("Prefetch " + getPathOffsetEndStr(path, offset, end), e);
292            }
293          } catch (NullPointerException e) {
294            LOG.warn("Stream moved/closed or prefetch cancelled?" +
295                getPathOffsetEndStr(path, offset, end), e);
296          } catch (Exception e) {
297            // Other exceptions are interesting
298            LOG.warn("Prefetch " + getPathOffsetEndStr(path, offset, end), e);
299          } finally {
300            PrefetchExecutor.complete(path);
301          }
302        }
303      });
304    }
305
306    byte[] tmp = fileInfo.get(FileInfo.MAX_TAGS_LEN);
307    // max tag length is not present in the HFile means tags were not at all written to file.
308    if (tmp != null) {
309      hfileContext.setIncludesTags(true);
310      tmp = fileInfo.get(FileInfo.TAGS_COMPRESSED);
311      if (tmp != null && Bytes.toBoolean(tmp)) {
312        hfileContext.setCompressTags(true);
313      }
314    }
315  }
316
317  private static String getPathOffsetEndStr(final Path path, final long offset, final long end) {
318    return "path=" + path.toString() + ", offset=" + offset + ", end=" + end;
319  }
320
321  /**
322   * File version check is a little sloppy. We read v3 files but can also read v2 files if their
323   * content has been pb'd; files written with 0.98.
324   */
325  private void checkFileVersion() {
326    int majorVersion = trailer.getMajorVersion();
327    if (majorVersion == getMajorVersion()) return;
328    int minorVersion = trailer.getMinorVersion();
329    if (majorVersion == 2 && minorVersion >= MIN_V2_MINOR_VERSION_WITH_PB) return;
330    // We can read v3 or v2 versions of hfile.
331    throw new IllegalArgumentException("Invalid HFile version: major=" +
332      trailer.getMajorVersion() + ", minor=" + trailer.getMinorVersion() + ": expected at least " +
333      "major=2 and minor=" + MAX_MINOR_VERSION);
334  }
335
336  @SuppressWarnings("serial")
337  public static class BlockIndexNotLoadedException extends IllegalStateException {
338    public BlockIndexNotLoadedException() {
339      // Add a message in case anyone relies on it as opposed to class name.
340      super("Block index not loaded");
341    }
342  }
343
344  private Optional<String> toStringFirstKey() {
345    return getFirstKey().map(CellUtil::getCellKeyAsString);
346  }
347
348  private Optional<String> toStringLastKey() {
349    return getLastKey().map(CellUtil::getCellKeyAsString);
350  }
351
352  @Override
353  public String toString() {
354    return "reader=" + path.toString() +
355        (!isFileInfoLoaded()? "":
356          ", compression=" + compressAlgo.getName() +
357          ", cacheConf=" + cacheConf +
358          ", firstKey=" + toStringFirstKey() +
359          ", lastKey=" + toStringLastKey()) +
360          ", avgKeyLen=" + avgKeyLen +
361          ", avgValueLen=" + avgValueLen +
362          ", entries=" + trailer.getEntryCount() +
363          ", length=" + fileSize;
364  }
365
366  @Override
367  public long length() {
368    return fileSize;
369  }
370
371  @Override
372  public void returnBlock(HFileBlock block) {
373    BlockCache blockCache = this.cacheConf.getBlockCache();
374    if (blockCache != null && block != null) {
375      BlockCacheKey cacheKey = new BlockCacheKey(this.getFileContext().getHFileName(),
376          block.getOffset(), this.isPrimaryReplicaReader(), block.getBlockType());
377      blockCache.returnBlock(cacheKey, block);
378    }
379  }
380  /**
381   * @return the first key in the file. May be null if file has no entries. Note
382   *         that this is not the first row key, but rather the byte form of the
383   *         first KeyValue.
384   */
385  @Override
386  public Optional<Cell> getFirstKey() {
387    if (dataBlockIndexReader == null) {
388      throw new BlockIndexNotLoadedException();
389    }
390    return dataBlockIndexReader.isEmpty() ? Optional.empty()
391        : Optional.of(dataBlockIndexReader.getRootBlockKey(0));
392  }
393
394  /**
395   * TODO left from {@link HFile} version 1: move this to StoreFile after Ryan's
396   * patch goes in to eliminate {@link KeyValue} here.
397   *
398   * @return the first row key, or null if the file is empty.
399   */
400  @Override
401  public Optional<byte[]> getFirstRowKey() {
402    // We have to copy the row part to form the row key alone
403    return getFirstKey().map(CellUtil::cloneRow);
404  }
405
406  /**
407   * TODO left from {@link HFile} version 1: move this to StoreFile after
408   * Ryan's patch goes in to eliminate {@link KeyValue} here.
409   *
410   * @return the last row key, or null if the file is empty.
411   */
412  @Override
413  public Optional<byte[]> getLastRowKey() {
414    // We have to copy the row part to form the row key alone
415    return getLastKey().map(CellUtil::cloneRow);
416  }
417
418  /** @return number of KV entries in this HFile */
419  @Override
420  public long getEntries() {
421    return trailer.getEntryCount();
422  }
423
424  /** @return comparator */
425  @Override
426  public CellComparator getComparator() {
427    return comparator;
428  }
429
430  /** @return compression algorithm */
431  @Override
432  public Compression.Algorithm getCompressionAlgorithm() {
433    return compressAlgo;
434  }
435
436  /**
437   * @return the total heap size of data and meta block indexes in bytes. Does
438   *         not take into account non-root blocks of a multilevel data index.
439   */
440  @Override
441  public long indexSize() {
442    return (dataBlockIndexReader != null ? dataBlockIndexReader.heapSize() : 0)
443        + ((metaBlockIndexReader != null) ? metaBlockIndexReader.heapSize()
444            : 0);
445  }
446
447  @Override
448  public String getName() {
449    return name;
450  }
451
452  @Override
453  public HFileBlockIndex.BlockIndexReader getDataBlockIndexReader() {
454    return dataBlockIndexReader;
455  }
456
457  @Override
458  public FixedFileTrailer getTrailer() {
459    return trailer;
460  }
461
462  @Override
463  public boolean isPrimaryReplicaReader() {
464    return primaryReplicaReader;
465  }
466
467  @Override
468  public FileInfo loadFileInfo() throws IOException {
469    return fileInfo;
470  }
471
472  /**
473   * An exception thrown when an operation requiring a scanner to be seeked
474   * is invoked on a scanner that is not seeked.
475   */
476  @SuppressWarnings("serial")
477  public static class NotSeekedException extends IllegalStateException {
478    public NotSeekedException() {
479      super("Not seeked to a key/value");
480    }
481  }
482
483  protected static class HFileScannerImpl implements HFileScanner {
484    private ByteBuff blockBuffer;
485    protected final boolean cacheBlocks;
486    protected final boolean pread;
487    protected final boolean isCompaction;
488    private int currKeyLen;
489    private int currValueLen;
490    private int currMemstoreTSLen;
491    private long currMemstoreTS;
492    // Updated but never read?
493    protected AtomicInteger blockFetches = new AtomicInteger(0);
494    protected final HFile.Reader reader;
495    private int currTagsLen;
496    // buffer backed keyonlyKV
497    private ByteBufferKeyOnlyKeyValue bufBackedKeyOnlyKv = new ByteBufferKeyOnlyKeyValue();
498    // A pair for reusing in blockSeek() so that we don't garbage lot of objects
499    final ObjectIntPair<ByteBuffer> pair = new ObjectIntPair<>();
500
501    /**
502     * The next indexed key is to keep track of the indexed key of the next data block.
503     * If the nextIndexedKey is HConstants.NO_NEXT_INDEXED_KEY, it means that the
504     * current data block is the last data block.
505     *
506     * If the nextIndexedKey is null, it means the nextIndexedKey has not been loaded yet.
507     */
508    protected Cell nextIndexedKey;
509    // Current block being used
510    protected HFileBlock curBlock;
511    // Previous blocks that were used in the course of the read
512    protected final ArrayList<HFileBlock> prevBlocks = new ArrayList<>();
513
514    public HFileScannerImpl(final HFile.Reader reader, final boolean cacheBlocks,
515        final boolean pread, final boolean isCompaction) {
516      this.reader = reader;
517      this.cacheBlocks = cacheBlocks;
518      this.pread = pread;
519      this.isCompaction = isCompaction;
520    }
521
522    void updateCurrBlockRef(HFileBlock block) {
523      if (block != null && this.curBlock != null &&
524          block.getOffset() == this.curBlock.getOffset()) {
525        return;
526      }
527      // We don't have to keep ref to EXCLUSIVE type of block
528      if (this.curBlock != null && this.curBlock.usesSharedMemory()) {
529        prevBlocks.add(this.curBlock);
530      }
531      this.curBlock = block;
532    }
533
534    void reset() {
535      // We don't have to keep ref to EXCLUSIVE type of block
536      if (this.curBlock != null && this.curBlock.usesSharedMemory()) {
537        this.prevBlocks.add(this.curBlock);
538      }
539      this.curBlock = null;
540    }
541
542    private void returnBlockToCache(HFileBlock block) {
543      if (LOG.isTraceEnabled()) {
544        LOG.trace("Returning the block : " + block);
545      }
546      this.reader.returnBlock(block);
547    }
548
549    private void returnBlocks(boolean returnAll) {
550      for (int i = 0; i < this.prevBlocks.size(); i++) {
551        returnBlockToCache(this.prevBlocks.get(i));
552      }
553      this.prevBlocks.clear();
554      if (returnAll && this.curBlock != null) {
555        returnBlockToCache(this.curBlock);
556        this.curBlock = null;
557      }
558    }
559    @Override
560    public boolean isSeeked(){
561      return blockBuffer != null;
562    }
563
564    @Override
565    public String toString() {
566      return "HFileScanner for reader " + String.valueOf(getReader());
567    }
568
569    protected void assertSeeked() {
570      if (!isSeeked())
571        throw new NotSeekedException();
572    }
573
574    @Override
575    public HFile.Reader getReader() {
576      return reader;
577    }
578
579    // From non encoded HFiles, we always read back KeyValue or its descendant.(Note: When HFile
580    // block is in DBB, it will be OffheapKV). So all parts of the Cell is in a contiguous
581    // array/buffer. How many bytes we should wrap to make the KV is what this method returns.
582    private int getKVBufSize() {
583      int kvBufSize = KEY_VALUE_LEN_SIZE + currKeyLen + currValueLen;
584      if (currTagsLen > 0) {
585        kvBufSize += Bytes.SIZEOF_SHORT + currTagsLen;
586      }
587      return kvBufSize;
588    }
589
590    @Override
591    public void close() {
592      if (!pread) {
593        // For seek + pread stream socket should be closed when the scanner is closed. HBASE-9393
594        reader.unbufferStream();
595      }
596      this.returnBlocks(true);
597    }
598
599    // Returns the #bytes in HFile for the current cell. Used to skip these many bytes in current
600    // HFile block's buffer so as to position to the next cell.
601    private int getCurCellSerializedSize() {
602      int curCellSize =  KEY_VALUE_LEN_SIZE + currKeyLen + currValueLen
603          + currMemstoreTSLen;
604      if (this.reader.getFileContext().isIncludesTags()) {
605        curCellSize += Bytes.SIZEOF_SHORT + currTagsLen;
606      }
607      return curCellSize;
608    }
609
610    protected void readKeyValueLen() {
611      // This is a hot method. We go out of our way to make this method short so it can be
612      // inlined and is not too big to compile. We also manage position in ByteBuffer ourselves
613      // because it is faster than going via range-checked ByteBuffer methods or going through a
614      // byte buffer array a byte at a time.
615      // Get a long at a time rather than read two individual ints. In micro-benchmarking, even
616      // with the extra bit-fiddling, this is order-of-magnitude faster than getting two ints.
617      // Trying to imitate what was done - need to profile if this is better or
618      // earlier way is better by doing mark and reset?
619      // But ensure that you read long instead of two ints
620      long ll = blockBuffer.getLongAfterPosition(0);
621      // Read top half as an int of key length and bottom int as value length
622      this.currKeyLen = (int)(ll >> Integer.SIZE);
623      this.currValueLen = (int)(Bytes.MASK_FOR_LOWER_INT_IN_LONG ^ ll);
624      checkKeyValueLen();
625      // Move position past the key and value lengths and then beyond the key and value
626      int p = (Bytes.SIZEOF_LONG + currKeyLen + currValueLen);
627      if (reader.getFileContext().isIncludesTags()) {
628        // Tags length is a short.
629        this.currTagsLen = blockBuffer.getShortAfterPosition(p);
630        checkTagsLen();
631        p += (Bytes.SIZEOF_SHORT + currTagsLen);
632      }
633      readMvccVersion(p);
634    }
635
636    private final void checkTagsLen() {
637      if (checkLen(this.currTagsLen)) {
638        throw new IllegalStateException("Invalid currTagsLen " + this.currTagsLen +
639          ". Block offset: " + curBlock.getOffset() + ", block length: " +
640            this.blockBuffer.limit() +
641          ", position: " + this.blockBuffer.position() + " (without header).");
642      }
643    }
644
645    /**
646     * Read mvcc. Does checks to see if we even need to read the mvcc at all.
647     * @param offsetFromPos
648     */
649    protected void readMvccVersion(final int offsetFromPos) {
650      // See if we even need to decode mvcc.
651      if (!this.reader.shouldIncludeMemStoreTS()) return;
652      if (!this.reader.isDecodeMemStoreTS()) {
653        currMemstoreTS = 0;
654        currMemstoreTSLen = 1;
655        return;
656      }
657      _readMvccVersion(offsetFromPos);
658    }
659
660    /**
661     * Actually do the mvcc read. Does no checks.
662     * @param offsetFromPos
663     */
664    private void _readMvccVersion(int offsetFromPos) {
665      // This is Bytes#bytesToVint inlined so can save a few instructions in this hot method; i.e.
666      // previous if one-byte vint, we'd redo the vint call to find int size.
667      // Also the method is kept small so can be inlined.
668      byte firstByte = blockBuffer.getByteAfterPosition(offsetFromPos);
669      int len = WritableUtils.decodeVIntSize(firstByte);
670      if (len == 1) {
671        this.currMemstoreTS = firstByte;
672      } else {
673        int remaining = len -1;
674        long i = 0;
675        offsetFromPos++;
676        if (remaining >= Bytes.SIZEOF_INT) {
677          // The int read has to be converted to unsigned long so the & op
678          i = (blockBuffer.getIntAfterPosition(offsetFromPos) & 0x00000000ffffffffL);
679          remaining -= Bytes.SIZEOF_INT;
680          offsetFromPos += Bytes.SIZEOF_INT;
681        }
682        if (remaining >= Bytes.SIZEOF_SHORT) {
683          short s = blockBuffer.getShortAfterPosition(offsetFromPos);
684          i = i << 16;
685          i = i | (s & 0xFFFF);
686          remaining -= Bytes.SIZEOF_SHORT;
687          offsetFromPos += Bytes.SIZEOF_SHORT;
688        }
689        for (int idx = 0; idx < remaining; idx++) {
690          byte b = blockBuffer.getByteAfterPosition(offsetFromPos + idx);
691          i = i << 8;
692          i = i | (b & 0xFF);
693        }
694        currMemstoreTS = (WritableUtils.isNegativeVInt(firstByte) ? ~i : i);
695      }
696      this.currMemstoreTSLen = len;
697    }
698
699    /**
700     * Within a loaded block, seek looking for the last key that is smaller than
701     * (or equal to?) the key we are interested in.
702     * A note on the seekBefore: if you have seekBefore = true, AND the first
703     * key in the block = key, then you'll get thrown exceptions. The caller has
704     * to check for that case and load the previous block as appropriate.
705     * @param key
706     *          the key to find
707     * @param seekBefore
708     *          find the key before the given key in case of exact match.
709     * @return 0 in case of an exact key match, 1 in case of an inexact match,
710     *         -2 in case of an inexact match and furthermore, the input key
711     *         less than the first key of current block(e.g. using a faked index
712     *         key)
713     */
714    protected int blockSeek(Cell key, boolean seekBefore) {
715      int klen, vlen, tlen = 0;
716      int lastKeyValueSize = -1;
717      int offsetFromPos;
718      do {
719        offsetFromPos = 0;
720        // Better to ensure that we use the BB Utils here
721        long ll = blockBuffer.getLongAfterPosition(offsetFromPos);
722        klen = (int)(ll >> Integer.SIZE);
723        vlen = (int)(Bytes.MASK_FOR_LOWER_INT_IN_LONG ^ ll);
724        if (checkKeyLen(klen) || checkLen(vlen)) {
725          throw new IllegalStateException("Invalid klen " + klen + " or vlen "
726              + vlen + ". Block offset: "
727              + curBlock.getOffset() + ", block length: " + blockBuffer.limit() + ", position: "
728              + blockBuffer.position() + " (without header).");
729        }
730        offsetFromPos += Bytes.SIZEOF_LONG;
731        blockBuffer.asSubByteBuffer(blockBuffer.position() + offsetFromPos, klen, pair);
732        bufBackedKeyOnlyKv.setKey(pair.getFirst(), pair.getSecond(), klen);
733        int comp =
734            PrivateCellUtil.compareKeyIgnoresMvcc(reader.getComparator(), key, bufBackedKeyOnlyKv);
735        offsetFromPos += klen + vlen;
736        if (this.reader.getFileContext().isIncludesTags()) {
737          // Read short as unsigned, high byte first
738          tlen = ((blockBuffer.getByteAfterPosition(offsetFromPos) & 0xff) << 8)
739              ^ (blockBuffer.getByteAfterPosition(offsetFromPos + 1) & 0xff);
740          if (checkLen(tlen)) {
741            throw new IllegalStateException("Invalid tlen " + tlen + ". Block offset: "
742                + curBlock.getOffset() + ", block length: " + blockBuffer.limit() + ", position: "
743                + blockBuffer.position() + " (without header).");
744          }
745          // add the two bytes read for the tags.
746          offsetFromPos += tlen + (Bytes.SIZEOF_SHORT);
747        }
748        if (this.reader.shouldIncludeMemStoreTS()) {
749          // Directly read the mvcc based on current position
750          readMvccVersion(offsetFromPos);
751        }
752        if (comp == 0) {
753          if (seekBefore) {
754            if (lastKeyValueSize < 0) {
755              throw new IllegalStateException("blockSeek with seekBefore "
756                  + "at the first key of the block: key=" + CellUtil.getCellKeyAsString(key)
757                  + ", blockOffset=" + curBlock.getOffset() + ", onDiskSize="
758                  + curBlock.getOnDiskSizeWithHeader());
759            }
760            blockBuffer.moveBack(lastKeyValueSize);
761            readKeyValueLen();
762            return 1; // non exact match.
763          }
764          currKeyLen = klen;
765          currValueLen = vlen;
766          currTagsLen = tlen;
767          return 0; // indicate exact match
768        } else if (comp < 0) {
769          if (lastKeyValueSize > 0) {
770            blockBuffer.moveBack(lastKeyValueSize);
771          }
772          readKeyValueLen();
773          if (lastKeyValueSize == -1 && blockBuffer.position() == 0) {
774            return HConstants.INDEX_KEY_MAGIC;
775          }
776          return 1;
777        }
778        // The size of this key/value tuple, including key/value length fields.
779        lastKeyValueSize = klen + vlen + currMemstoreTSLen + KEY_VALUE_LEN_SIZE;
780        // include tag length also if tags included with KV
781        if (reader.getFileContext().isIncludesTags()) {
782          lastKeyValueSize += tlen + Bytes.SIZEOF_SHORT;
783        }
784        blockBuffer.skip(lastKeyValueSize);
785      } while (blockBuffer.hasRemaining());
786
787      // Seek to the last key we successfully read. This will happen if this is
788      // the last key/value pair in the file, in which case the following call
789      // to next() has to return false.
790      blockBuffer.moveBack(lastKeyValueSize);
791      readKeyValueLen();
792      return 1; // didn't exactly find it.
793    }
794
795    @Override
796    public Cell getNextIndexedKey() {
797      return nextIndexedKey;
798    }
799
800    @Override
801    public int seekTo(Cell key) throws IOException {
802      return seekTo(key, true);
803    }
804
805    @Override
806    public int reseekTo(Cell key) throws IOException {
807      int compared;
808      if (isSeeked()) {
809        compared = compareKey(reader.getComparator(), key);
810        if (compared < 1) {
811          // If the required key is less than or equal to current key, then
812          // don't do anything.
813          return compared;
814        } else {
815          // The comparison with no_next_index_key has to be checked
816          if (this.nextIndexedKey != null &&
817              (this.nextIndexedKey == KeyValueScanner.NO_NEXT_INDEXED_KEY || PrivateCellUtil
818                  .compareKeyIgnoresMvcc(reader.getComparator(), key, nextIndexedKey) < 0)) {
819            // The reader shall continue to scan the current data block instead
820            // of querying the
821            // block index as long as it knows the target key is strictly
822            // smaller than
823            // the next indexed key or the current data block is the last data
824            // block.
825            return loadBlockAndSeekToKey(this.curBlock, nextIndexedKey, false, key,
826                false);
827          }
828
829        }
830      }
831      // Don't rewind on a reseek operation, because reseek implies that we are
832      // always going forward in the file.
833      return seekTo(key, false);
834    }
835
836    /**
837     * An internal API function. Seek to the given key, optionally rewinding to
838     * the first key of the block before doing the seek.
839     *
840     * @param key - a cell representing the key that we need to fetch
841     * @param rewind whether to rewind to the first key of the block before
842     *        doing the seek. If this is false, we are assuming we never go
843     *        back, otherwise the result is undefined.
844     * @return -1 if the key is earlier than the first key of the file,
845     *         0 if we are at the given key, 1 if we are past the given key
846     *         -2 if the key is earlier than the first key of the file while
847     *         using a faked index key
848     * @throws IOException
849     */
850    public int seekTo(Cell key, boolean rewind) throws IOException {
851      HFileBlockIndex.BlockIndexReader indexReader = reader.getDataBlockIndexReader();
852      BlockWithScanInfo blockWithScanInfo = indexReader.loadDataBlockWithScanInfo(key, curBlock,
853          cacheBlocks, pread, isCompaction, getEffectiveDataBlockEncoding());
854      if (blockWithScanInfo == null || blockWithScanInfo.getHFileBlock() == null) {
855        // This happens if the key e.g. falls before the beginning of the
856        // file.
857        return -1;
858      }
859      return loadBlockAndSeekToKey(blockWithScanInfo.getHFileBlock(),
860          blockWithScanInfo.getNextIndexedKey(), rewind, key, false);
861    }
862
863    @Override
864    public boolean seekBefore(Cell key) throws IOException {
865      HFileBlock seekToBlock = reader.getDataBlockIndexReader().seekToDataBlock(key, curBlock,
866          cacheBlocks, pread, isCompaction, reader.getEffectiveEncodingInCache(isCompaction));
867      if (seekToBlock == null) {
868        return false;
869      }
870      Cell firstKey = getFirstKeyCellInBlock(seekToBlock);
871      if (PrivateCellUtil.compareKeyIgnoresMvcc(reader.getComparator(), firstKey, key) >= 0) {
872        long previousBlockOffset = seekToBlock.getPrevBlockOffset();
873        // The key we are interested in
874        if (previousBlockOffset == -1) {
875          // we have a 'problem', the key we want is the first of the file.
876          return false;
877        }
878
879        // The first key in the current block 'seekToBlock' is greater than the given
880        // seekBefore key. We will go ahead by reading the next block that satisfies the
881        // given key. Return the current block before reading the next one.
882        reader.returnBlock(seekToBlock);
883        // It is important that we compute and pass onDiskSize to the block
884        // reader so that it does not have to read the header separately to
885        // figure out the size.  Currently, we do not have a way to do this
886        // correctly in the general case however.
887        // TODO: See https://issues.apache.org/jira/browse/HBASE-14576
888        int prevBlockSize = -1;
889        seekToBlock = reader.readBlock(previousBlockOffset,
890            prevBlockSize, cacheBlocks,
891            pread, isCompaction, true, BlockType.DATA, getEffectiveDataBlockEncoding());
892        // TODO shortcut: seek forward in this block to the last key of the
893        // block.
894      }
895      loadBlockAndSeekToKey(seekToBlock, firstKey, true, key, true);
896      return true;
897    }
898
899    /**
900     * Scans blocks in the "scanned" section of the {@link HFile} until the next
901     * data block is found.
902     *
903     * @return the next block, or null if there are no more data blocks
904     * @throws IOException
905     */
906    @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="NP_NULL_ON_SOME_PATH",
907        justification="Yeah, unnecessary null check; could do w/ clean up")
908    protected HFileBlock readNextDataBlock() throws IOException {
909      long lastDataBlockOffset = reader.getTrailer().getLastDataBlockOffset();
910      if (curBlock == null)
911        return null;
912
913      HFileBlock block = this.curBlock;
914
915      do {
916        if (block.getOffset() >= lastDataBlockOffset) {
917          return null;
918        }
919
920        if (block.getOffset() < 0) {
921          throw new IOException("Invalid block file offset: " + block);
922        }
923
924        // We are reading the next block without block type validation, because
925        // it might turn out to be a non-data block.
926        block = reader.readBlock(block.getOffset() + block.getOnDiskSizeWithHeader(),
927            block.getNextBlockOnDiskSize(), cacheBlocks, pread,
928            isCompaction, true, null, getEffectiveDataBlockEncoding());
929        if (block != null && !block.getBlockType().isData()) { // Findbugs: NP_NULL_ON_SOME_PATH
930          // Whatever block we read we will be returning it unless
931          // it is a datablock. Just in case the blocks are non data blocks
932          reader.returnBlock(block);
933        }
934      } while (!block.getBlockType().isData());
935
936      return block;
937    }
938
939    public DataBlockEncoding getEffectiveDataBlockEncoding() {
940      return this.reader.getEffectiveEncodingInCache(isCompaction);
941    }
942
943    @Override
944    public Cell getCell() {
945      if (!isSeeked())
946        return null;
947
948      Cell ret;
949      int cellBufSize = getKVBufSize();
950      long seqId = 0L;
951      if (this.reader.shouldIncludeMemStoreTS()) {
952        seqId = currMemstoreTS;
953      }
954      if (blockBuffer.hasArray()) {
955        // TODO : reduce the varieties of KV here. Check if based on a boolean
956        // we can handle the 'no tags' case.
957        if (currTagsLen > 0) {
958          ret = new SizeCachedKeyValue(blockBuffer.array(),
959              blockBuffer.arrayOffset() + blockBuffer.position(), cellBufSize, seqId);
960        } else {
961          ret = new SizeCachedNoTagsKeyValue(blockBuffer.array(),
962              blockBuffer.arrayOffset() + blockBuffer.position(), cellBufSize, seqId);
963        }
964      } else {
965        ByteBuffer buf = blockBuffer.asSubByteBuffer(cellBufSize);
966        if (buf.isDirect()) {
967          ret = new ByteBufferKeyValue(buf, buf.position(), cellBufSize, seqId);
968        } else {
969          if (currTagsLen > 0) {
970            ret = new SizeCachedKeyValue(buf.array(), buf.arrayOffset() + buf.position(),
971                cellBufSize, seqId);
972          } else {
973            ret = new SizeCachedNoTagsKeyValue(buf.array(), buf.arrayOffset() + buf.position(),
974                cellBufSize, seqId);
975          }
976        }
977      }
978      return ret;
979    }
980
981    @Override
982    public Cell getKey() {
983      assertSeeked();
984      // Create a new object so that this getKey is cached as firstKey, lastKey
985      ObjectIntPair<ByteBuffer> keyPair = new ObjectIntPair<>();
986      blockBuffer.asSubByteBuffer(blockBuffer.position() + KEY_VALUE_LEN_SIZE, currKeyLen, keyPair);
987      ByteBuffer keyBuf = keyPair.getFirst();
988      if (keyBuf.hasArray()) {
989        return new KeyValue.KeyOnlyKeyValue(keyBuf.array(), keyBuf.arrayOffset()
990            + keyPair.getSecond(), currKeyLen);
991      } else {
992        // Better to do a copy here instead of holding on to this BB so that
993        // we could release the blocks referring to this key. This key is specifically used
994        // in HalfStoreFileReader to get the firstkey and lastkey by creating a new scanner
995        // every time. So holding onto the BB (incase of DBB) is not advised here.
996        byte[] key = new byte[currKeyLen];
997        ByteBufferUtils.copyFromBufferToArray(key, keyBuf, keyPair.getSecond(), 0, currKeyLen);
998        return new KeyValue.KeyOnlyKeyValue(key, 0, currKeyLen);
999      }
1000    }
1001
1002    @Override
1003    public ByteBuffer getValue() {
1004      assertSeeked();
1005      // Okie to create new Pair. Not used in hot path
1006      ObjectIntPair<ByteBuffer> valuePair = new ObjectIntPair<>();
1007      this.blockBuffer.asSubByteBuffer(blockBuffer.position() + KEY_VALUE_LEN_SIZE + currKeyLen,
1008        currValueLen, valuePair);
1009      ByteBuffer valBuf = valuePair.getFirst().duplicate();
1010      valBuf.position(valuePair.getSecond());
1011      valBuf.limit(currValueLen + valuePair.getSecond());
1012      return valBuf.slice();
1013    }
1014
1015    protected void setNonSeekedState() {
1016      reset();
1017      blockBuffer = null;
1018      currKeyLen = 0;
1019      currValueLen = 0;
1020      currMemstoreTS = 0;
1021      currMemstoreTSLen = 0;
1022      currTagsLen = 0;
1023    }
1024
1025    /**
1026     * Set the position on current backing blockBuffer.
1027     */
1028    private void positionThisBlockBuffer() {
1029      try {
1030        blockBuffer.skip(getCurCellSerializedSize());
1031      } catch (IllegalArgumentException e) {
1032        LOG.error("Current pos = " + blockBuffer.position()
1033            + "; currKeyLen = " + currKeyLen + "; currValLen = "
1034            + currValueLen + "; block limit = " + blockBuffer.limit()
1035            + "; HFile name = " + reader.getName()
1036            + "; currBlock currBlockOffset = " + this.curBlock.getOffset());
1037        throw e;
1038      }
1039    }
1040
1041    /**
1042     * Set our selves up for the next 'next' invocation, set up next block.
1043     * @return True is more to read else false if at the end.
1044     * @throws IOException
1045     */
1046    private boolean positionForNextBlock() throws IOException {
1047      // Methods are small so they get inlined because they are 'hot'.
1048      long lastDataBlockOffset = reader.getTrailer().getLastDataBlockOffset();
1049      if (this.curBlock.getOffset() >= lastDataBlockOffset) {
1050        setNonSeekedState();
1051        return false;
1052      }
1053      return isNextBlock();
1054    }
1055
1056
1057    private boolean isNextBlock() throws IOException {
1058      // Methods are small so they get inlined because they are 'hot'.
1059      HFileBlock nextBlock = readNextDataBlock();
1060      if (nextBlock == null) {
1061        setNonSeekedState();
1062        return false;
1063      }
1064      updateCurrentBlock(nextBlock);
1065      return true;
1066    }
1067
1068    private final boolean _next() throws IOException {
1069      // Small method so can be inlined. It is a hot one.
1070      if (blockBuffer.remaining() <= 0) {
1071        return positionForNextBlock();
1072      }
1073
1074      // We are still in the same block.
1075      readKeyValueLen();
1076      return true;
1077    }
1078
1079    /**
1080     * Go to the next key/value in the block section. Loads the next block if
1081     * necessary. If successful, {@link #getKey()} and {@link #getValue()} can
1082     * be called.
1083     *
1084     * @return true if successfully navigated to the next key/value
1085     */
1086    @Override
1087    public boolean next() throws IOException {
1088      // This is a hot method so extreme measures taken to ensure it is small and inlineable.
1089      // Checked by setting: -XX:+UnlockDiagnosticVMOptions -XX:+PrintInlining -XX:+PrintCompilation
1090      assertSeeked();
1091      positionThisBlockBuffer();
1092      return _next();
1093    }
1094
1095    /**
1096     * Positions this scanner at the start of the file.
1097     *
1098     * @return false if empty file; i.e. a call to next would return false and
1099     *         the current key and value are undefined.
1100     * @throws IOException
1101     */
1102    @Override
1103    public boolean seekTo() throws IOException {
1104      if (reader == null) {
1105        return false;
1106      }
1107
1108      if (reader.getTrailer().getEntryCount() == 0) {
1109        // No data blocks.
1110        return false;
1111      }
1112
1113      long firstDataBlockOffset = reader.getTrailer().getFirstDataBlockOffset();
1114      if (curBlock != null
1115          && curBlock.getOffset() == firstDataBlockOffset) {
1116        return processFirstDataBlock();
1117      }
1118
1119      readAndUpdateNewBlock(firstDataBlockOffset);
1120      return true;
1121    }
1122
1123    protected boolean processFirstDataBlock() throws IOException{
1124      blockBuffer.rewind();
1125      readKeyValueLen();
1126      return true;
1127    }
1128
1129    protected void readAndUpdateNewBlock(long firstDataBlockOffset) throws IOException,
1130        CorruptHFileException {
1131      HFileBlock newBlock = reader.readBlock(firstDataBlockOffset, -1, cacheBlocks, pread,
1132          isCompaction, true, BlockType.DATA, getEffectiveDataBlockEncoding());
1133      if (newBlock.getOffset() < 0) {
1134        throw new IOException("Invalid block offset: " + newBlock.getOffset());
1135      }
1136      updateCurrentBlock(newBlock);
1137    }
1138
1139    protected int loadBlockAndSeekToKey(HFileBlock seekToBlock, Cell nextIndexedKey,
1140        boolean rewind, Cell key, boolean seekBefore) throws IOException {
1141      if (this.curBlock == null
1142          || this.curBlock.getOffset() != seekToBlock.getOffset()) {
1143        updateCurrentBlock(seekToBlock);
1144      } else if (rewind) {
1145        blockBuffer.rewind();
1146      }
1147
1148      // Update the nextIndexedKey
1149      this.nextIndexedKey = nextIndexedKey;
1150      return blockSeek(key, seekBefore);
1151    }
1152
1153    /**
1154     * @param v
1155     * @return True if v &lt;= 0 or v &gt; current block buffer limit.
1156     */
1157    protected final boolean checkKeyLen(final int v) {
1158      return v <= 0 || v > this.blockBuffer.limit();
1159    }
1160
1161    /**
1162     * @param v
1163     * @return True if v &lt; 0 or v &gt; current block buffer limit.
1164     */
1165    protected final boolean checkLen(final int v) {
1166      return v < 0 || v > this.blockBuffer.limit();
1167    }
1168
1169    /**
1170     * Check key and value lengths are wholesome.
1171     */
1172    protected final void checkKeyValueLen() {
1173      if (checkKeyLen(this.currKeyLen) || checkLen(this.currValueLen)) {
1174        throw new IllegalStateException("Invalid currKeyLen " + this.currKeyLen
1175            + " or currValueLen " + this.currValueLen + ". Block offset: "
1176            + this.curBlock.getOffset() + ", block length: "
1177            + this.blockBuffer.limit() + ", position: " + this.blockBuffer.position()
1178            + " (without header).");
1179      }
1180    }
1181
1182    /**
1183     * Updates the current block to be the given {@link HFileBlock}. Seeks to
1184     * the the first key/value pair.
1185     *
1186     * @param newBlock the block to make current
1187     */
1188    protected void updateCurrentBlock(HFileBlock newBlock) throws IOException {
1189      // Set the active block on the reader
1190      // sanity check
1191      if (newBlock.getBlockType() != BlockType.DATA) {
1192        throw new IllegalStateException("ScannerV2 works only on data " + "blocks, got "
1193            + newBlock.getBlockType() + "; " + "fileName=" + reader.getName()
1194            + ", " + "dataBlockEncoder=" + reader.getDataBlockEncoding() + ", " + "isCompaction="
1195            + isCompaction);
1196      }
1197
1198      updateCurrBlockRef(newBlock);
1199      blockBuffer = newBlock.getBufferWithoutHeader();
1200      readKeyValueLen();
1201      blockFetches.incrementAndGet();
1202
1203      // Reset the next indexed key
1204      this.nextIndexedKey = null;
1205    }
1206
1207    protected Cell getFirstKeyCellInBlock(HFileBlock curBlock) {
1208      ByteBuff buffer = curBlock.getBufferWithoutHeader();
1209      // It is safe to manipulate this buffer because we own the buffer object.
1210      buffer.rewind();
1211      int klen = buffer.getInt();
1212      buffer.skip(Bytes.SIZEOF_INT);// Skip value len part
1213      ByteBuffer keyBuff = buffer.asSubByteBuffer(klen);
1214      if (keyBuff.hasArray()) {
1215        return new KeyValue.KeyOnlyKeyValue(keyBuff.array(), keyBuff.arrayOffset()
1216            + keyBuff.position(), klen);
1217      } else {
1218        return new ByteBufferKeyOnlyKeyValue(keyBuff, keyBuff.position(), klen);
1219      }
1220    }
1221
1222    @Override
1223    public String getKeyString() {
1224      return CellUtil.toString(getKey(), false);
1225    }
1226
1227    @Override
1228    public String getValueString() {
1229      return ByteBufferUtils.toStringBinary(getValue());
1230    }
1231
1232    public int compareKey(CellComparator comparator, Cell key) {
1233      blockBuffer.asSubByteBuffer(blockBuffer.position() + KEY_VALUE_LEN_SIZE, currKeyLen, pair);
1234      this.bufBackedKeyOnlyKv.setKey(pair.getFirst(), pair.getSecond(), currKeyLen);
1235      return PrivateCellUtil.compareKeyIgnoresMvcc(comparator, key, this.bufBackedKeyOnlyKv);
1236    }
1237
1238    @Override
1239    public void shipped() throws IOException {
1240      this.returnBlocks(false);
1241    }
1242  }
1243
1244  @Override
1245  public Path getPath() {
1246    return path;
1247  }
1248
1249  @Override
1250  public DataBlockEncoding getDataBlockEncoding() {
1251    return dataBlockEncoder.getDataBlockEncoding();
1252  }
1253
1254  @Override
1255  public Configuration getConf() {
1256    return conf;
1257  }
1258
1259  @Override
1260  public void setConf(Configuration conf) {
1261    this.conf = conf;
1262  }
1263
1264  /** Minor versions in HFile starting with this number have hbase checksums */
1265  public static final int MINOR_VERSION_WITH_CHECKSUM = 1;
1266  /** In HFile minor version that does not support checksums */
1267  public static final int MINOR_VERSION_NO_CHECKSUM = 0;
1268
1269  /** HFile minor version that introduced pbuf filetrailer */
1270  public static final int PBUF_TRAILER_MINOR_VERSION = 2;
1271
1272  /**
1273   * The size of a (key length, value length) tuple that prefixes each entry in
1274   * a data block.
1275   */
1276  public final static int KEY_VALUE_LEN_SIZE = 2 * Bytes.SIZEOF_INT;
1277
1278  private boolean includesMemstoreTS = false;
1279  protected boolean decodeMemstoreTS = false;
1280
1281
1282  @Override
1283  public boolean isDecodeMemStoreTS() {
1284    return this.decodeMemstoreTS;
1285  }
1286
1287  @Override
1288  public boolean shouldIncludeMemStoreTS() {
1289    return includesMemstoreTS;
1290  }
1291
1292  /**
1293   * Retrieve block from cache. Validates the retrieved block's type vs {@code expectedBlockType}
1294   * and its encoding vs. {@code expectedDataBlockEncoding}. Unpacks the block as necessary.
1295   */
1296   private HFileBlock getCachedBlock(BlockCacheKey cacheKey, boolean cacheBlock, boolean useLock,
1297       boolean isCompaction, boolean updateCacheMetrics, BlockType expectedBlockType,
1298       DataBlockEncoding expectedDataBlockEncoding) throws IOException {
1299     // Check cache for block. If found return.
1300     if (cacheConf.isBlockCacheEnabled()) {
1301       BlockCache cache = cacheConf.getBlockCache();
1302       HFileBlock cachedBlock = (HFileBlock) cache.getBlock(cacheKey, cacheBlock, useLock,
1303         updateCacheMetrics);
1304       if (cachedBlock != null) {
1305         if (cacheConf.shouldCacheCompressed(cachedBlock.getBlockType().getCategory())) {
1306           HFileBlock compressedBlock = cachedBlock;
1307           cachedBlock = compressedBlock.unpack(hfileContext, fsBlockReader);
1308           // In case of compressed block after unpacking we can return the compressed block
1309          if (compressedBlock != cachedBlock) {
1310            cache.returnBlock(cacheKey, compressedBlock);
1311          }
1312        }
1313         validateBlockType(cachedBlock, expectedBlockType);
1314
1315         if (expectedDataBlockEncoding == null) {
1316           return cachedBlock;
1317         }
1318         DataBlockEncoding actualDataBlockEncoding =
1319                 cachedBlock.getDataBlockEncoding();
1320         // Block types other than data blocks always have
1321         // DataBlockEncoding.NONE. To avoid false negative cache misses, only
1322         // perform this check if cached block is a data block.
1323         if (cachedBlock.getBlockType().isData() &&
1324                 !actualDataBlockEncoding.equals(expectedDataBlockEncoding)) {
1325           // This mismatch may happen if a Scanner, which is used for say a
1326           // compaction, tries to read an encoded block from the block cache.
1327           // The reverse might happen when an EncodedScanner tries to read
1328           // un-encoded blocks which were cached earlier.
1329           //
1330           // Because returning a data block with an implicit BlockType mismatch
1331           // will cause the requesting scanner to throw a disk read should be
1332           // forced here. This will potentially cause a significant number of
1333           // cache misses, so update so we should keep track of this as it might
1334           // justify the work on a CompoundScanner.
1335           if (!expectedDataBlockEncoding.equals(DataBlockEncoding.NONE) &&
1336                   !actualDataBlockEncoding.equals(DataBlockEncoding.NONE)) {
1337             // If the block is encoded but the encoding does not match the
1338             // expected encoding it is likely the encoding was changed but the
1339             // block was not yet evicted. Evictions on file close happen async
1340             // so blocks with the old encoding still linger in cache for some
1341             // period of time. This event should be rare as it only happens on
1342             // schema definition change.
1343             LOG.info("Evicting cached block with key " + cacheKey +
1344                     " because of a data block encoding mismatch" +
1345                     "; expected: " + expectedDataBlockEncoding +
1346                     ", actual: " + actualDataBlockEncoding);
1347             // This is an error scenario. so here we need to decrement the
1348             // count.
1349             cache.returnBlock(cacheKey, cachedBlock);
1350             cache.evictBlock(cacheKey);
1351           }
1352           return null;
1353         }
1354         return cachedBlock;
1355       }
1356     }
1357     return null;
1358   }
1359
1360  /**
1361   * @param metaBlockName
1362   * @param cacheBlock Add block to cache, if found
1363   * @return block wrapped in a ByteBuffer, with header skipped
1364   * @throws IOException
1365   */
1366  @Override
1367  public HFileBlock getMetaBlock(String metaBlockName, boolean cacheBlock)
1368      throws IOException {
1369    if (trailer.getMetaIndexCount() == 0) {
1370      return null; // there are no meta blocks
1371    }
1372    if (metaBlockIndexReader == null) {
1373      throw new IOException("Meta index not loaded");
1374    }
1375
1376    byte[] mbname = Bytes.toBytes(metaBlockName);
1377    int block = metaBlockIndexReader.rootBlockContainingKey(mbname,
1378        0, mbname.length);
1379    if (block == -1)
1380      return null;
1381    long blockSize = metaBlockIndexReader.getRootBlockDataSize(block);
1382
1383    // Per meta key from any given file, synchronize reads for said block. This
1384    // is OK to do for meta blocks because the meta block index is always
1385    // single-level.
1386    synchronized (metaBlockIndexReader
1387        .getRootBlockKey(block)) {
1388      // Check cache for block. If found return.
1389      long metaBlockOffset = metaBlockIndexReader.getRootBlockOffset(block);
1390      BlockCacheKey cacheKey = new BlockCacheKey(name, metaBlockOffset,
1391        this.isPrimaryReplicaReader(), BlockType.META);
1392
1393      cacheBlock &= cacheConf.shouldCacheBlockOnRead(BlockType.META.getCategory());
1394      if (cacheConf.isBlockCacheEnabled()) {
1395        HFileBlock cachedBlock = getCachedBlock(cacheKey, cacheBlock, false, true, true,
1396          BlockType.META, null);
1397        if (cachedBlock != null) {
1398          assert cachedBlock.isUnpacked() : "Packed block leak.";
1399          // Return a distinct 'shallow copy' of the block,
1400          // so pos does not get messed by the scanner
1401          return cachedBlock;
1402        }
1403        // Cache Miss, please load.
1404      }
1405
1406      HFileBlock metaBlock = fsBlockReader.readBlockData(metaBlockOffset, blockSize, true, false).
1407          unpack(hfileContext, fsBlockReader);
1408
1409      // Cache the block
1410      if (cacheBlock) {
1411        cacheConf.getBlockCache().cacheBlock(cacheKey, metaBlock, cacheConf.isInMemory());
1412      }
1413
1414      return metaBlock;
1415    }
1416  }
1417
1418  @Override
1419  public HFileBlock readBlock(long dataBlockOffset, long onDiskBlockSize,
1420      final boolean cacheBlock, boolean pread, final boolean isCompaction,
1421      boolean updateCacheMetrics, BlockType expectedBlockType,
1422      DataBlockEncoding expectedDataBlockEncoding)
1423      throws IOException {
1424    if (dataBlockIndexReader == null) {
1425      throw new IOException("Block index not loaded");
1426    }
1427    long trailerOffset = trailer.getLoadOnOpenDataOffset();
1428    if (dataBlockOffset < 0 || dataBlockOffset >= trailerOffset) {
1429      throw new IOException("Requested block is out of range: " + dataBlockOffset +
1430        ", lastDataBlockOffset: " + trailer.getLastDataBlockOffset() +
1431        ", trailer.getLoadOnOpenDataOffset: " + trailerOffset);
1432    }
1433    // For any given block from any given file, synchronize reads for said
1434    // block.
1435    // Without a cache, this synchronizing is needless overhead, but really
1436    // the other choice is to duplicate work (which the cache would prevent you
1437    // from doing).
1438
1439    BlockCacheKey cacheKey = new BlockCacheKey(name, dataBlockOffset,
1440      this.isPrimaryReplicaReader(), expectedBlockType);
1441
1442    boolean useLock = false;
1443    IdLock.Entry lockEntry = null;
1444    try (TraceScope traceScope = TraceUtil.createTrace("HFileReaderImpl.readBlock")) {
1445      while (true) {
1446        // Check cache for block. If found return.
1447        if (cacheConf.shouldReadBlockFromCache(expectedBlockType)) {
1448          if (useLock) {
1449            lockEntry = offsetLock.getLockEntry(dataBlockOffset);
1450          }
1451          // Try and get the block from the block cache. If the useLock variable is true then this
1452          // is the second time through the loop and it should not be counted as a block cache miss.
1453          HFileBlock cachedBlock = getCachedBlock(cacheKey, cacheBlock, useLock, isCompaction,
1454            updateCacheMetrics, expectedBlockType, expectedDataBlockEncoding);
1455          if (cachedBlock != null) {
1456            if (LOG.isTraceEnabled()) {
1457              LOG.trace("From Cache " + cachedBlock);
1458            }
1459            TraceUtil.addTimelineAnnotation("blockCacheHit");
1460            assert cachedBlock.isUnpacked() : "Packed block leak.";
1461            if (cachedBlock.getBlockType().isData()) {
1462              if (updateCacheMetrics) {
1463                HFile.DATABLOCK_READ_COUNT.increment();
1464              }
1465              // Validate encoding type for data blocks. We include encoding
1466              // type in the cache key, and we expect it to match on a cache hit.
1467              if (cachedBlock.getDataBlockEncoding() != dataBlockEncoder.getDataBlockEncoding()) {
1468                throw new IOException("Cached block under key " + cacheKey + " "
1469                  + "has wrong encoding: " + cachedBlock.getDataBlockEncoding() + " (expected: "
1470                  + dataBlockEncoder.getDataBlockEncoding() + ")");
1471              }
1472            }
1473            // Cache-hit. Return!
1474            return cachedBlock;
1475          }
1476
1477          if (!useLock && cacheBlock && cacheConf.shouldLockOnCacheMiss(expectedBlockType)) {
1478            // check cache again with lock
1479            useLock = true;
1480            continue;
1481          }
1482          // Carry on, please load.
1483        }
1484
1485        TraceUtil.addTimelineAnnotation("blockCacheMiss");
1486        // Load block from filesystem.
1487        HFileBlock hfileBlock =
1488            fsBlockReader.readBlockData(dataBlockOffset, onDiskBlockSize, pread, !isCompaction);
1489        validateBlockType(hfileBlock, expectedBlockType);
1490        HFileBlock unpacked = hfileBlock.unpack(hfileContext, fsBlockReader);
1491        BlockType.BlockCategory category = hfileBlock.getBlockType().getCategory();
1492
1493        // Cache the block if necessary
1494        if (cacheBlock && cacheConf.shouldCacheBlockOnRead(category)) {
1495          cacheConf.getBlockCache().cacheBlock(cacheKey,
1496            cacheConf.shouldCacheCompressed(category) ? hfileBlock : unpacked,
1497            cacheConf.isInMemory());
1498        }
1499
1500        if (updateCacheMetrics && hfileBlock.getBlockType().isData()) {
1501          HFile.DATABLOCK_READ_COUNT.increment();
1502        }
1503
1504        return unpacked;
1505      }
1506    } finally {
1507      if (lockEntry != null) {
1508        offsetLock.releaseLockEntry(lockEntry);
1509      }
1510    }
1511  }
1512
1513  @Override
1514  public boolean hasMVCCInfo() {
1515    return includesMemstoreTS && decodeMemstoreTS;
1516  }
1517
1518  /**
1519   * Compares the actual type of a block retrieved from cache or disk with its
1520   * expected type and throws an exception in case of a mismatch. Expected
1521   * block type of {@link BlockType#DATA} is considered to match the actual
1522   * block type [@link {@link BlockType#ENCODED_DATA} as well.
1523   * @param block a block retrieved from cache or disk
1524   * @param expectedBlockType the expected block type, or null to skip the
1525   *          check
1526   */
1527  private void validateBlockType(HFileBlock block,
1528      BlockType expectedBlockType) throws IOException {
1529    if (expectedBlockType == null) {
1530      return;
1531    }
1532    BlockType actualBlockType = block.getBlockType();
1533    if (expectedBlockType.isData() && actualBlockType.isData()) {
1534      // We consider DATA to match ENCODED_DATA for the purpose of this
1535      // verification.
1536      return;
1537    }
1538    if (actualBlockType != expectedBlockType) {
1539      throw new IOException("Expected block type " + expectedBlockType + ", " +
1540          "but got " + actualBlockType + ": " + block);
1541    }
1542  }
1543
1544  /**
1545   * @return Last key as cell in the file. May be null if file has no entries. Note that
1546   *         this is not the last row key, but it is the Cell representation of the last
1547   *         key
1548   */
1549  @Override
1550  public Optional<Cell> getLastKey() {
1551    return dataBlockIndexReader.isEmpty() ? Optional.empty() : Optional.of(lastKeyCell);
1552  }
1553
1554  /**
1555   * @return Midkey for this file. We work with block boundaries only so
1556   *         returned midkey is an approximation only.
1557   * @throws IOException
1558   */
1559  @Override
1560  public Optional<Cell> midKey() throws IOException {
1561    return Optional.ofNullable(dataBlockIndexReader.midkey());
1562  }
1563
1564  @Override
1565  public void close() throws IOException {
1566    close(cacheConf.shouldEvictOnClose());
1567  }
1568
1569  @Override
1570  public void close(boolean evictOnClose) throws IOException {
1571    PrefetchExecutor.cancel(path);
1572    if (evictOnClose && cacheConf.isBlockCacheEnabled()) {
1573      int numEvicted = cacheConf.getBlockCache().evictBlocksByHfileName(name);
1574      if (LOG.isTraceEnabled()) {
1575        LOG.trace("On close, file=" + name + " evicted=" + numEvicted
1576          + " block(s)");
1577      }
1578    }
1579    fsBlockReader.closeStreams();
1580  }
1581
1582  @Override
1583  public DataBlockEncoding getEffectiveEncodingInCache(boolean isCompaction) {
1584    return dataBlockEncoder.getEffectiveEncodingInCache(isCompaction);
1585  }
1586
1587  /** For testing */
1588  @Override
1589  public HFileBlock.FSReader getUncachedBlockReader() {
1590    return fsBlockReader;
1591  }
1592
1593  /**
1594   * Scanner that operates on encoded data blocks.
1595   */
1596  protected static class EncodedScanner extends HFileScannerImpl {
1597    private final HFileBlockDecodingContext decodingCtx;
1598    private final DataBlockEncoder.EncodedSeeker seeker;
1599    private final DataBlockEncoder dataBlockEncoder;
1600
1601    public EncodedScanner(HFile.Reader reader, boolean cacheBlocks,
1602        boolean pread, boolean isCompaction, HFileContext meta) {
1603      super(reader, cacheBlocks, pread, isCompaction);
1604      DataBlockEncoding encoding = reader.getDataBlockEncoding();
1605      dataBlockEncoder = encoding.getEncoder();
1606      decodingCtx = dataBlockEncoder.newDataBlockDecodingContext(meta);
1607      seeker = dataBlockEncoder.createSeeker(
1608        reader.getComparator(), decodingCtx);
1609    }
1610
1611    @Override
1612    public boolean isSeeked(){
1613      return curBlock != null;
1614    }
1615
1616    @Override
1617    public void setNonSeekedState() {
1618      reset();
1619    }
1620
1621    /**
1622     * Updates the current block to be the given {@link HFileBlock}. Seeks to
1623     * the the first key/value pair.
1624     *
1625     * @param newBlock the block to make current
1626     * @throws CorruptHFileException
1627     */
1628    @Override
1629    protected void updateCurrentBlock(HFileBlock newBlock) throws CorruptHFileException {
1630
1631      // sanity checks
1632      if (newBlock.getBlockType() != BlockType.ENCODED_DATA) {
1633        throw new IllegalStateException("EncodedScanner works only on encoded data blocks");
1634      }
1635      short dataBlockEncoderId = newBlock.getDataBlockEncodingId();
1636      if (!DataBlockEncoding.isCorrectEncoder(dataBlockEncoder, dataBlockEncoderId)) {
1637        String encoderCls = dataBlockEncoder.getClass().getName();
1638        throw new CorruptHFileException("Encoder " + encoderCls
1639          + " doesn't support data block encoding "
1640          + DataBlockEncoding.getNameFromId(dataBlockEncoderId));
1641      }
1642      updateCurrBlockRef(newBlock);
1643      ByteBuff encodedBuffer = getEncodedBuffer(newBlock);
1644      seeker.setCurrentBuffer(encodedBuffer);
1645      blockFetches.incrementAndGet();
1646
1647      // Reset the next indexed key
1648      this.nextIndexedKey = null;
1649    }
1650
1651    private ByteBuff getEncodedBuffer(HFileBlock newBlock) {
1652      ByteBuff origBlock = newBlock.getBufferReadOnly();
1653      int pos = newBlock.headerSize() + DataBlockEncoding.ID_SIZE;
1654      origBlock.position(pos);
1655      origBlock
1656          .limit(pos + newBlock.getUncompressedSizeWithoutHeader() - DataBlockEncoding.ID_SIZE);
1657      return origBlock.slice();
1658    }
1659
1660    @Override
1661    protected boolean processFirstDataBlock() throws IOException {
1662      seeker.rewind();
1663      return true;
1664    }
1665
1666    @Override
1667    public boolean next() throws IOException {
1668      boolean isValid = seeker.next();
1669      if (!isValid) {
1670        HFileBlock newBlock = readNextDataBlock();
1671        isValid = newBlock != null;
1672        if (isValid) {
1673          updateCurrentBlock(newBlock);
1674        } else {
1675          setNonSeekedState();
1676        }
1677      }
1678      return isValid;
1679    }
1680
1681    @Override
1682    public Cell getKey() {
1683      assertValidSeek();
1684      return seeker.getKey();
1685    }
1686
1687    @Override
1688    public ByteBuffer getValue() {
1689      assertValidSeek();
1690      return seeker.getValueShallowCopy();
1691    }
1692
1693    @Override
1694    public Cell getCell() {
1695      if (this.curBlock == null) {
1696        return null;
1697      }
1698      return seeker.getCell();
1699    }
1700
1701    @Override
1702    public String getKeyString() {
1703      return CellUtil.toString(getKey(), true);
1704    }
1705
1706    @Override
1707    public String getValueString() {
1708      ByteBuffer valueBuffer = getValue();
1709      return ByteBufferUtils.toStringBinary(valueBuffer);
1710    }
1711
1712    private void assertValidSeek() {
1713      if (this.curBlock == null) {
1714        throw new NotSeekedException();
1715      }
1716    }
1717
1718    @Override
1719    protected Cell getFirstKeyCellInBlock(HFileBlock curBlock) {
1720      return dataBlockEncoder.getFirstKeyCellInBlock(getEncodedBuffer(curBlock));
1721    }
1722
1723    @Override
1724    protected int loadBlockAndSeekToKey(HFileBlock seekToBlock, Cell nextIndexedKey,
1725        boolean rewind, Cell key, boolean seekBefore) throws IOException {
1726      if (this.curBlock == null
1727          || this.curBlock.getOffset() != seekToBlock.getOffset()) {
1728        updateCurrentBlock(seekToBlock);
1729      } else if (rewind) {
1730        seeker.rewind();
1731      }
1732      this.nextIndexedKey = nextIndexedKey;
1733      return seeker.seekToKeyInBlock(key, seekBefore);
1734    }
1735
1736    @Override
1737    public int compareKey(CellComparator comparator, Cell key) {
1738      return seeker.compareKey(comparator, key);
1739    }
1740  }
1741
1742  /**
1743   * Returns a buffer with the Bloom filter metadata. The caller takes
1744   * ownership of the buffer.
1745   */
1746  @Override
1747  public DataInput getGeneralBloomFilterMetadata() throws IOException {
1748    return this.getBloomFilterMetadata(BlockType.GENERAL_BLOOM_META);
1749  }
1750
1751  @Override
1752  public DataInput getDeleteBloomFilterMetadata() throws IOException {
1753    return this.getBloomFilterMetadata(BlockType.DELETE_FAMILY_BLOOM_META);
1754  }
1755
1756  private DataInput getBloomFilterMetadata(BlockType blockType)
1757  throws IOException {
1758    if (blockType != BlockType.GENERAL_BLOOM_META &&
1759        blockType != BlockType.DELETE_FAMILY_BLOOM_META) {
1760      throw new RuntimeException("Block Type: " + blockType.toString() +
1761          " is not supported") ;
1762    }
1763
1764    for (HFileBlock b : loadOnOpenBlocks)
1765      if (b.getBlockType() == blockType)
1766        return b.getByteStream();
1767    return null;
1768  }
1769
1770  public boolean isFileInfoLoaded() {
1771    return true; // We load file info in constructor in version 2.
1772  }
1773
1774  @Override
1775  public HFileContext getFileContext() {
1776    return hfileContext;
1777  }
1778
1779  /**
1780   * Returns false if block prefetching was requested for this file and has
1781   * not completed, true otherwise
1782   */
1783  @Override
1784  @VisibleForTesting
1785  public boolean prefetchComplete() {
1786    return PrefetchExecutor.isCompleted(path);
1787  }
1788
1789  protected HFileContext createHFileContext(FSDataInputStreamWrapper fsdis, long fileSize,
1790      HFileSystem hfs, Path path, FixedFileTrailer trailer) throws IOException {
1791    HFileContextBuilder builder = new HFileContextBuilder()
1792      .withIncludesMvcc(shouldIncludeMemStoreTS())
1793      .withHBaseCheckSum(true)
1794      .withHFileName(this.getName())
1795      .withCompression(this.compressAlgo);
1796
1797    // Check for any key material available
1798    byte[] keyBytes = trailer.getEncryptionKey();
1799    if (keyBytes != null) {
1800      Encryption.Context cryptoContext = Encryption.newContext(conf);
1801      Key key;
1802      key = EncryptionUtil.unwrapKey(conf, keyBytes);
1803      // Use the algorithm the key wants
1804      Cipher cipher = Encryption.getCipher(conf, key.getAlgorithm());
1805      if (cipher == null) {
1806        throw new IOException("Cipher '" + key.getAlgorithm() + "' is not available");
1807      }
1808      cryptoContext.setCipher(cipher);
1809      cryptoContext.setKey(key);
1810      builder.withEncryptionContext(cryptoContext);
1811    }
1812
1813    HFileContext context = builder.build();
1814
1815    if (LOG.isTraceEnabled()) {
1816      LOG.trace("Reader" + (path != null? " for " + path: "") +
1817        " initialized with cacheConf: " + cacheConf +
1818        " comparator: " + comparator.getClass().getSimpleName() +
1819        " fileContext: " + context);
1820    }
1821
1822    return context;
1823  }
1824
1825  /**
1826   * Create a Scanner on this file. No seeks or reads are done on creation. Call
1827   * {@link HFileScanner#seekTo(Cell)} to position an start the read. There is
1828   * nothing to clean up in a Scanner. Letting go of your references to the
1829   * scanner is sufficient. NOTE: Do not use this overload of getScanner for
1830   * compactions. See {@link #getScanner(boolean, boolean, boolean)}
1831   *
1832   * @param cacheBlocks True if we should cache blocks read in by this scanner.
1833   * @param pread Use positional read rather than seek+read if true (pread is
1834   *          better for random reads, seek+read is better scanning).
1835   * @return Scanner on this file.
1836   */
1837  @Override
1838  @VisibleForTesting
1839  public HFileScanner getScanner(boolean cacheBlocks, final boolean pread) {
1840    return getScanner(cacheBlocks, pread, false);
1841  }
1842
1843  /**
1844   * Create a Scanner on this file. No seeks or reads are done on creation. Call
1845   * {@link HFileScanner#seekTo(Cell)} to position an start the read. There is
1846   * nothing to clean up in a Scanner. Letting go of your references to the
1847   * scanner is sufficient.
1848   * @param cacheBlocks
1849   *          True if we should cache blocks read in by this scanner.
1850   * @param pread
1851   *          Use positional read rather than seek+read if true (pread is better
1852   *          for random reads, seek+read is better scanning).
1853   * @param isCompaction
1854   *          is scanner being used for a compaction?
1855   * @return Scanner on this file.
1856   */
1857  @Override
1858  public HFileScanner getScanner(boolean cacheBlocks, final boolean pread,
1859      final boolean isCompaction) {
1860    if (dataBlockEncoder.useEncodedScanner()) {
1861      return new EncodedScanner(this, cacheBlocks, pread, isCompaction, this.hfileContext);
1862    }
1863    return new HFileScannerImpl(this, cacheBlocks, pread, isCompaction);
1864  }
1865
1866  public int getMajorVersion() {
1867    return 3;
1868  }
1869
1870  @Override
1871  public void unbufferStream() {
1872    fsBlockReader.unbufferStream();
1873  }
1874}