View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.io.hfile;
19  
20  import java.io.DataInput;
21  import java.io.IOException;
22  import java.nio.ByteBuffer;
23  import java.util.ArrayList;
24  import java.util.List;
25  
26  import org.apache.commons.logging.Log;
27  import org.apache.commons.logging.LogFactory;
28  import org.apache.hadoop.hbase.classification.InterfaceAudience;
29  import org.apache.hadoop.conf.Configuration;
30  import org.apache.hadoop.fs.Path;
31  import org.apache.hadoop.hbase.Cell;
32  import org.apache.hadoop.hbase.CellUtil;
33  import org.apache.hadoop.hbase.HConstants;
34  import org.apache.hadoop.hbase.KeyValue;
35  import org.apache.hadoop.hbase.KeyValue.KVComparator;
36  import org.apache.hadoop.hbase.fs.HFileSystem;
37  import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
38  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder;
39  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
40  import org.apache.hadoop.hbase.io.encoding.HFileBlockDecodingContext;
41  import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
42  import org.apache.hadoop.hbase.util.ByteBufferUtils;
43  import org.apache.hadoop.hbase.util.Bytes;
44  import org.apache.hadoop.hbase.util.IdLock;
45  import org.apache.hadoop.io.WritableUtils;
46  import org.apache.htrace.Trace;
47  import org.apache.htrace.TraceScope;
48  
49  import com.google.common.annotations.VisibleForTesting;
50  
51  /**
52   * {@link HFile} reader for version 2.
53   */
54  @InterfaceAudience.Private
55  public class HFileReaderV2 extends AbstractHFileReader {
56  
57    private static final Log LOG = LogFactory.getLog(HFileReaderV2.class);
58  
59    /** Minor versions in HFile V2 starting with this number have hbase checksums */
60    public static final int MINOR_VERSION_WITH_CHECKSUM = 1;
61    /** In HFile V2 minor version that does not support checksums */
62    public static final int MINOR_VERSION_NO_CHECKSUM = 0;
63  
64    /** HFile minor version that introduced pbuf filetrailer */
65    public static final int PBUF_TRAILER_MINOR_VERSION = 2;
66  
67    /**
68     * The size of a (key length, value length) tuple that prefixes each entry in
69     * a data block.
70     */
71    public final static int KEY_VALUE_LEN_SIZE = 2 * Bytes.SIZEOF_INT;
72  
73    protected boolean includesMemstoreTS = false;
74    protected boolean decodeMemstoreTS = false;
75    protected boolean shouldIncludeMemstoreTS() {
76      return includesMemstoreTS;
77    }
78  
79    /** Filesystem-level block reader. */
80    protected HFileBlock.FSReader fsBlockReader;
81  
82    /**
83     * A "sparse lock" implementation allowing to lock on a particular block
84     * identified by offset. The purpose of this is to avoid two clients loading
85     * the same block, and have all but one client wait to get the block from the
86     * cache.
87     */
88    private IdLock offsetLock = new IdLock();
89  
90    /**
91     * Blocks read from the load-on-open section, excluding data root index, meta
92     * index, and file info.
93     */
94    private List<HFileBlock> loadOnOpenBlocks = new ArrayList<HFileBlock>();
95  
96    /** Minimum minor version supported by this HFile format */
97    static final int MIN_MINOR_VERSION = 0;
98  
99    /** Maximum minor version supported by this HFile format */
100   // We went to version 2 when we moved to pb'ing fileinfo and the trailer on
101   // the file. This version can read Writables version 1.
102   static final int MAX_MINOR_VERSION = 3;
103 
104   /** Minor versions starting with this number have faked index key */
105   static final int MINOR_VERSION_WITH_FAKED_KEY = 3;
106 
107   protected HFileContext hfileContext;
108 
109   /**
110    * Opens a HFile. You must load the index before you can use it by calling
111    * {@link #loadFileInfo()}.
112    *
113    * @param path Path to HFile.
114    * @param trailer File trailer.
115    * @param fsdis input stream.
116    * @param size Length of the stream.
117    * @param cacheConf Cache configuration.
118    * @param hfs
119    * @param conf
120    */
121   public HFileReaderV2(final Path path, final FixedFileTrailer trailer,
122       final FSDataInputStreamWrapper fsdis, final long size, final CacheConfig cacheConf,
123       final HFileSystem hfs, final Configuration conf) throws IOException {
124     super(path, trailer, size, cacheConf, hfs, conf);
125     this.conf = conf;
126     trailer.expectMajorVersion(getMajorVersion());
127     validateMinorVersion(path, trailer.getMinorVersion());
128     this.hfileContext = createHFileContext(fsdis, fileSize, hfs, path, trailer);
129     HFileBlock.FSReaderImpl fsBlockReaderV2 =
130       new HFileBlock.FSReaderImpl(fsdis, fileSize, hfs, path, hfileContext);
131     this.fsBlockReader = fsBlockReaderV2; // upcast
132 
133     // Comparator class name is stored in the trailer in version 2.
134     comparator = trailer.createComparator();
135     dataBlockIndexReader = new HFileBlockIndex.BlockIndexReader(comparator,
136         trailer.getNumDataIndexLevels(), this);
137     metaBlockIndexReader = new HFileBlockIndex.BlockIndexReader(
138         KeyValue.RAW_COMPARATOR, 1);
139 
140     // Parse load-on-open data.
141 
142     HFileBlock.BlockIterator blockIter = fsBlockReaderV2.blockRange(
143         trailer.getLoadOnOpenDataOffset(),
144         fileSize - trailer.getTrailerSize());
145 
146     // Data index. We also read statistics about the block index written after
147     // the root level.
148     dataBlockIndexReader.readMultiLevelIndexRoot(
149         blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX),
150         trailer.getDataIndexCount());
151 
152     // Meta index.
153     metaBlockIndexReader.readRootIndex(
154         blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX),
155         trailer.getMetaIndexCount());
156 
157     // File info
158     fileInfo = new FileInfo();
159     fileInfo.read(blockIter.nextBlockWithBlockType(BlockType.FILE_INFO).getByteStream());
160     byte[] creationTimeBytes = fileInfo.get(FileInfo.CREATE_TIME_TS);
161     this.hfileContext.setFileCreateTime(creationTimeBytes == null ? 0 : Bytes.toLong(creationTimeBytes));
162     lastKey = fileInfo.get(FileInfo.LASTKEY);
163     avgKeyLen = Bytes.toInt(fileInfo.get(FileInfo.AVG_KEY_LEN));
164     avgValueLen = Bytes.toInt(fileInfo.get(FileInfo.AVG_VALUE_LEN));
165     byte [] keyValueFormatVersion =
166         fileInfo.get(HFileWriterV2.KEY_VALUE_VERSION);
167     includesMemstoreTS = keyValueFormatVersion != null &&
168         Bytes.toInt(keyValueFormatVersion) ==
169             HFileWriterV2.KEY_VALUE_VER_WITH_MEMSTORE;
170     fsBlockReaderV2.setIncludesMemstoreTS(includesMemstoreTS);
171     if (includesMemstoreTS) {
172       decodeMemstoreTS = Bytes.toLong(fileInfo.get(HFileWriterV2.MAX_MEMSTORE_TS_KEY)) > 0;
173     }
174 
175     // Read data block encoding algorithm name from file info.
176     dataBlockEncoder = HFileDataBlockEncoderImpl.createFromFileInfo(fileInfo);
177     fsBlockReaderV2.setDataBlockEncoder(dataBlockEncoder);
178 
179     // Store all other load-on-open blocks for further consumption.
180     HFileBlock b;
181     while ((b = blockIter.nextBlock()) != null) {
182       loadOnOpenBlocks.add(b);
183     }
184 
185     // Prefetch file blocks upon open if requested
186     if (cacheConf.shouldPrefetchOnOpen()) {
187       PrefetchExecutor.request(path, new Runnable() {
188         public void run() {
189           try {
190             long offset = 0;
191             long end = fileSize - getTrailer().getTrailerSize();
192             HFileBlock prevBlock = null;
193             while (offset < end) {
194               if (Thread.interrupted()) {
195                 break;
196               }
197               long onDiskSize = -1;
198               if (prevBlock != null) {
199                 onDiskSize = prevBlock.getNextBlockOnDiskSizeWithHeader();
200               }
201               HFileBlock block = readBlock(offset, onDiskSize, true, false, false, false,
202                 null, null);
203               prevBlock = block;
204               offset += block.getOnDiskSizeWithHeader();
205             }
206           } catch (IOException e) {
207             // IOExceptions are probably due to region closes (relocation, etc.)
208             if (LOG.isTraceEnabled()) {
209               LOG.trace("Exception encountered while prefetching " + path + ":", e);
210             }
211           } catch (Exception e) {
212             // Other exceptions are interesting
213             LOG.warn("Exception encountered while prefetching " + path + ":", e);
214           } finally {
215             PrefetchExecutor.complete(path);
216           }
217         }
218       });
219     }
220   }
221 
222   protected HFileContext createHFileContext(FSDataInputStreamWrapper fsdis, long fileSize,
223       HFileSystem hfs, Path path, FixedFileTrailer trailer) throws IOException {
224     return new HFileContextBuilder()
225       .withIncludesMvcc(this.includesMemstoreTS)
226       .withCompression(this.compressAlgo)
227       .withHBaseCheckSum(trailer.getMinorVersion() >= MINOR_VERSION_WITH_CHECKSUM)
228       .build();
229   }
230 
231   /**
232    * Create a Scanner on this file. No seeks or reads are done on creation. Call
233    * {@link HFileScanner#seekTo(byte[])} to position an start the read. There is
234    * nothing to clean up in a Scanner. Letting go of your references to the
235    * scanner is sufficient.
236    *
237    * @param cacheBlocks True if we should cache blocks read in by this scanner.
238    * @param pread Use positional read rather than seek+read if true (pread is
239    *          better for random reads, seek+read is better scanning).
240    * @param isCompaction is scanner being used for a compaction?
241    * @return Scanner on this file.
242    */
243    @Override
244    public HFileScanner getScanner(boolean cacheBlocks, final boolean pread,
245       final boolean isCompaction) {
246     if (dataBlockEncoder.useEncodedScanner()) {
247       return new EncodedScannerV2(this, cacheBlocks, pread, isCompaction,
248           hfileContext);
249     }
250 
251     return new ScannerV2(this, cacheBlocks, pread, isCompaction);
252   }
253 
254   /**
255    * Retrieve block from cache. Validates the retrieved block's type vs {@code expectedBlockType}
256    * and its encoding vs. {@code expectedDataBlockEncoding}. Unpacks the block as necessary.
257    */
258    private HFileBlock getCachedBlock(BlockCacheKey cacheKey, boolean cacheBlock, boolean useLock,
259        boolean isCompaction, boolean updateCacheMetrics, BlockType expectedBlockType,
260        DataBlockEncoding expectedDataBlockEncoding) throws IOException {
261      // Check cache for block. If found return.
262      if (cacheConf.isBlockCacheEnabled()) {
263        BlockCache cache = cacheConf.getBlockCache();
264        HFileBlock cachedBlock = (HFileBlock) cache.getBlock(cacheKey, cacheBlock, useLock,
265          updateCacheMetrics);
266        if (cachedBlock != null) {
267          if (cacheConf.shouldCacheCompressed(cachedBlock.getBlockType().getCategory())) {
268            cachedBlock = cachedBlock.unpack(hfileContext, fsBlockReader);
269          }
270          validateBlockType(cachedBlock, expectedBlockType);
271 
272          if (expectedDataBlockEncoding == null) {
273            return cachedBlock;
274          }
275          DataBlockEncoding actualDataBlockEncoding =
276                  cachedBlock.getDataBlockEncoding();
277          // Block types other than data blocks always have
278          // DataBlockEncoding.NONE. To avoid false negative cache misses, only
279          // perform this check if cached block is a data block.
280          if (cachedBlock.getBlockType().isData() &&
281                  !actualDataBlockEncoding.equals(expectedDataBlockEncoding)) {
282            // This mismatch may happen if a ScannerV2, which is used for say a
283            // compaction, tries to read an encoded block from the block cache.
284            // The reverse might happen when an EncodedScannerV2 tries to read
285            // un-encoded blocks which were cached earlier.
286            //
287            // Because returning a data block with an implicit BlockType mismatch
288            // will cause the requesting scanner to throw a disk read should be
289            // forced here. This will potentially cause a significant number of
290            // cache misses, so update so we should keep track of this as it might
291            // justify the work on a CompoundScannerV2.
292            if (!expectedDataBlockEncoding.equals(DataBlockEncoding.NONE) &&
293                    !actualDataBlockEncoding.equals(DataBlockEncoding.NONE)) {
294              // If the block is encoded but the encoding does not match the
295              // expected encoding it is likely the encoding was changed but the
296              // block was not yet evicted. Evictions on file close happen async
297              // so blocks with the old encoding still linger in cache for some
298              // period of time. This event should be rare as it only happens on
299              // schema definition change.
300              LOG.info("Evicting cached block with key " + cacheKey +
301                      " because of a data block encoding mismatch" +
302                      "; expected: " + expectedDataBlockEncoding +
303                      ", actual: " + actualDataBlockEncoding);
304              cache.evictBlock(cacheKey);
305            }
306            return null;
307          }
308          return cachedBlock;
309        }
310      }
311      return null;
312    }
313   /**
314    * @param metaBlockName
315    * @param cacheBlock Add block to cache, if found
316    * @return block wrapped in a ByteBuffer, with header skipped
317    * @throws IOException
318    */
319   @Override
320   public ByteBuffer getMetaBlock(String metaBlockName, boolean cacheBlock)
321       throws IOException {
322     if (trailer.getMetaIndexCount() == 0) {
323       return null; // there are no meta blocks
324     }
325     if (metaBlockIndexReader == null) {
326       throw new IOException("Meta index not loaded");
327     }
328 
329     byte[] mbname = Bytes.toBytes(metaBlockName);
330     int block = metaBlockIndexReader.rootBlockContainingKey(mbname,
331         0, mbname.length);
332     if (block == -1)
333       return null;
334     long blockSize = metaBlockIndexReader.getRootBlockDataSize(block);
335 
336     // Per meta key from any given file, synchronize reads for said block. This
337     // is OK to do for meta blocks because the meta block index is always
338     // single-level.
339     synchronized (metaBlockIndexReader.getRootBlockKey(block)) {
340       // Check cache for block. If found return.
341       long metaBlockOffset = metaBlockIndexReader.getRootBlockOffset(block);
342       BlockCacheKey cacheKey = new BlockCacheKey(name, metaBlockOffset);
343 
344       cacheBlock &= cacheConf.shouldCacheDataOnRead();
345       if (cacheConf.isBlockCacheEnabled()) {
346         HFileBlock cachedBlock = getCachedBlock(cacheKey, cacheBlock, false, true, true,
347           BlockType.META, null);
348         if (cachedBlock != null) {
349           assert cachedBlock.isUnpacked() : "Packed block leak.";
350           // Return a distinct 'shallow copy' of the block,
351           // so pos does not get messed by the scanner
352           return cachedBlock.getBufferWithoutHeader();
353         }
354         // Cache Miss, please load.
355       }
356 
357       HFileBlock metaBlock = fsBlockReader.readBlockData(metaBlockOffset,
358           blockSize, -1, true).unpack(hfileContext, fsBlockReader);
359 
360       // Cache the block
361       if (cacheBlock) {
362         cacheConf.getBlockCache().cacheBlock(cacheKey, metaBlock,
363             cacheConf.isInMemory(), this.cacheConf.isCacheDataInL1());
364       }
365 
366       return metaBlock.getBufferWithoutHeader();
367     }
368   }
369 
370   @Override
371   public HFileBlock readBlock(long dataBlockOffset, long onDiskBlockSize,
372       final boolean cacheBlock, boolean pread, final boolean isCompaction,
373       boolean updateCacheMetrics, BlockType expectedBlockType,
374       DataBlockEncoding expectedDataBlockEncoding)
375       throws IOException {
376     if (dataBlockIndexReader == null) {
377       throw new IOException("Block index not loaded");
378     }
379     if (dataBlockOffset < 0 || dataBlockOffset >= trailer.getLoadOnOpenDataOffset()) {
380       throw new IOException("Requested block is out of range: " + dataBlockOffset +
381         ", lastDataBlockOffset: " + trailer.getLastDataBlockOffset());
382     }
383 
384     // For any given block from any given file, synchronize reads for said block.
385     // Without a cache, this synchronizing is needless overhead, but really
386     // the other choice is to duplicate work (which the cache would prevent you
387     // from doing).
388     BlockCacheKey cacheKey = new BlockCacheKey(name, dataBlockOffset);
389     boolean useLock = false;
390     IdLock.Entry lockEntry = null;
391     TraceScope traceScope = Trace.startSpan("HFileReaderV2.readBlock");
392     try {
393       while (true) {
394         if (useLock) {
395           lockEntry = offsetLock.getLockEntry(dataBlockOffset);
396         }
397 
398         // Check cache for block. If found return.
399         if (cacheConf.isBlockCacheEnabled()) {
400           // Try and get the block from the block cache. If the useLock variable is true then this
401           // is the second time through the loop and it should not be counted as a block cache miss.
402           HFileBlock cachedBlock = getCachedBlock(cacheKey, cacheBlock, useLock, isCompaction,
403             updateCacheMetrics, expectedBlockType, expectedDataBlockEncoding);
404           if (cachedBlock != null) {
405             assert cachedBlock.isUnpacked() : "Packed block leak.";
406             if (cachedBlock.getBlockType().isData()) {
407               if (updateCacheMetrics) {
408                 HFile.dataBlockReadCnt.incrementAndGet();
409               }
410               // Validate encoding type for data blocks. We include encoding
411               // type in the cache key, and we expect it to match on a cache hit.
412               if (cachedBlock.getDataBlockEncoding() != dataBlockEncoder.getDataBlockEncoding()) {
413                 throw new IOException("Cached block under key " + cacheKey + " "
414                   + "has wrong encoding: " + cachedBlock.getDataBlockEncoding() + " (expected: "
415                   + dataBlockEncoder.getDataBlockEncoding() + ")");
416               }
417             }
418             // Cache-hit. Return!
419             return cachedBlock;
420           }
421           // Carry on, please load.
422         }
423         if (!useLock) {
424           // check cache again with lock
425           useLock = true;
426           continue;
427         }
428         if (Trace.isTracing()) {
429           traceScope.getSpan().addTimelineAnnotation("blockCacheMiss");
430         }
431         // Load block from filesystem.
432         HFileBlock hfileBlock = fsBlockReader.readBlockData(dataBlockOffset, onDiskBlockSize, -1,
433             pread);
434         validateBlockType(hfileBlock, expectedBlockType);
435         HFileBlock unpacked = hfileBlock.unpack(hfileContext, fsBlockReader);
436         BlockType.BlockCategory category = hfileBlock.getBlockType().getCategory();
437 
438         // Cache the block if necessary
439         if (cacheBlock && cacheConf.shouldCacheBlockOnRead(category)) {
440           cacheConf.getBlockCache().cacheBlock(cacheKey,
441             cacheConf.shouldCacheCompressed(category) ? hfileBlock : unpacked,
442             cacheConf.isInMemory(), this.cacheConf.isCacheDataInL1());
443         }
444 
445         if (updateCacheMetrics && hfileBlock.getBlockType().isData()) {
446           HFile.dataBlockReadCnt.incrementAndGet();
447         }
448 
449         return unpacked;
450       }
451     } finally {
452       traceScope.close();
453       if (lockEntry != null) {
454         offsetLock.releaseLockEntry(lockEntry);
455       }
456     }
457   }
458 
459   @Override
460   public boolean hasMVCCInfo() {
461     return includesMemstoreTS && decodeMemstoreTS;
462   }
463 
464   /**
465    * Compares the actual type of a block retrieved from cache or disk with its
466    * expected type and throws an exception in case of a mismatch. Expected
467    * block type of {@link BlockType#DATA} is considered to match the actual
468    * block type [@link {@link BlockType#ENCODED_DATA} as well.
469    * @param block a block retrieved from cache or disk
470    * @param expectedBlockType the expected block type, or null to skip the
471    *          check
472    */
473   private void validateBlockType(HFileBlock block,
474       BlockType expectedBlockType) throws IOException {
475     if (expectedBlockType == null) {
476       return;
477     }
478     BlockType actualBlockType = block.getBlockType();
479     if (expectedBlockType.isData() && actualBlockType.isData()) {
480       // We consider DATA to match ENCODED_DATA for the purpose of this
481       // verification.
482       return;
483     }
484     if (actualBlockType != expectedBlockType) {
485       throw new IOException("Expected block type " + expectedBlockType + ", " +
486           "but got " + actualBlockType + ": " + block);
487     }
488   }
489 
490   /**
491    * @return Last key in the file. May be null if file has no entries. Note that
492    *         this is not the last row key, but rather the byte form of the last
493    *         KeyValue.
494    */
495   @Override
496   public byte[] getLastKey() {
497     return dataBlockIndexReader.isEmpty() ? null : lastKey;
498   }
499 
500   /**
501    * @return Midkey for this file. We work with block boundaries only so
502    *         returned midkey is an approximation only.
503    * @throws IOException
504    */
505   @Override
506   public byte[] midkey() throws IOException {
507     return dataBlockIndexReader.midkey();
508   }
509 
510   @Override
511   public void close() throws IOException {
512     close(cacheConf.shouldEvictOnClose());
513   }
514 
515   public void close(boolean evictOnClose) throws IOException {
516     PrefetchExecutor.cancel(path);
517     if (evictOnClose && cacheConf.isBlockCacheEnabled()) {
518       int numEvicted = cacheConf.getBlockCache().evictBlocksByHfileName(name);
519       if (LOG.isTraceEnabled()) {
520         LOG.trace("On close, file=" + name + " evicted=" + numEvicted
521           + " block(s)");
522       }
523     }
524     fsBlockReader.closeStreams();
525   }
526 
527   public DataBlockEncoding getEffectiveEncodingInCache(boolean isCompaction) {
528     return dataBlockEncoder.getEffectiveEncodingInCache(isCompaction);
529   }
530 
531   /** For testing */
532   @Override
533   HFileBlock.FSReader getUncachedBlockReader() {
534     return fsBlockReader;
535   }
536 
537 
538   protected abstract static class AbstractScannerV2
539       extends AbstractHFileReader.Scanner {
540     protected HFileBlock block;
541 
542     /**
543      * The next indexed key is to keep track of the indexed key of the next data block.
544      * If the nextIndexedKey is HConstants.NO_NEXT_INDEXED_KEY, it means that the
545      * current data block is the last data block.
546      *
547      * If the nextIndexedKey is null, it means the nextIndexedKey has not been loaded yet.
548      */
549     protected byte[] nextIndexedKey;
550 
551     public AbstractScannerV2(HFileReaderV2 r, boolean cacheBlocks,
552         final boolean pread, final boolean isCompaction) {
553       super(r, cacheBlocks, pread, isCompaction);
554     }
555 
556     protected abstract ByteBuffer getFirstKeyInBlock(HFileBlock curBlock);
557 
558     protected abstract int loadBlockAndSeekToKey(HFileBlock seekToBlock, byte[] nextIndexedKey,
559         boolean rewind, Cell key, boolean seekBefore) throws IOException;
560 
561     @Override
562     public int seekTo(byte[] key, int offset, int length) throws IOException {
563       // Always rewind to the first key of the block, because the given key
564       // might be before or after the current key.
565       return seekTo(new KeyValue.KeyOnlyKeyValue(key, offset, length));
566     }
567 
568     @Override
569     public int reseekTo(byte[] key, int offset, int length) throws IOException {
570       return reseekTo(new KeyValue.KeyOnlyKeyValue(key, offset, length));
571     }
572 
573     @Override
574     public int seekTo(Cell key) throws IOException {
575       return seekTo(key, true);
576     }
577 
578     @Override
579     public int reseekTo(Cell key) throws IOException {
580       int compared;
581       if (isSeeked()) {
582         compared = compareKey(reader.getComparator(), key);
583         if (compared < 1) {
584           // If the required key is less than or equal to current key, then
585           // don't do anything.
586           return compared;
587         } else {
588           // The comparison with no_next_index_key has to be checked
589           if (this.nextIndexedKey != null &&
590               (this.nextIndexedKey == HConstants.NO_NEXT_INDEXED_KEY || reader
591               .getComparator()
592                   .compareOnlyKeyPortion(key,
593                       new KeyValue.KeyOnlyKeyValue(nextIndexedKey, 0,
594                           nextIndexedKey.length)) < 0)) {
595             // The reader shall continue to scan the current data block instead
596             // of querying the
597             // block index as long as it knows the target key is strictly
598             // smaller than
599             // the next indexed key or the current data block is the last data
600             // block.
601             return loadBlockAndSeekToKey(this.block, nextIndexedKey, false, key, false);
602           }
603         }
604       }
605       // Don't rewind on a reseek operation, because reseek implies that we are
606       // always going forward in the file.
607       return seekTo(key, false);
608     }
609 
610 
611     /**
612      * An internal API function. Seek to the given key, optionally rewinding to
613      * the first key of the block before doing the seek.
614      *
615      * @param key - a cell representing the key that we need to fetch
616      * @param rewind whether to rewind to the first key of the block before
617      *        doing the seek. If this is false, we are assuming we never go
618      *        back, otherwise the result is undefined.
619      * @return -1 if the key is earlier than the first key of the file,
620      *         0 if we are at the given key, 1 if we are past the given key
621      *         -2 if the key is earlier than the first key of the file while
622      *         using a faked index key
623      * @throws IOException
624      */
625     public int seekTo(Cell key, boolean rewind) throws IOException {
626       HFileBlockIndex.BlockIndexReader indexReader = reader.getDataBlockIndexReader();
627       BlockWithScanInfo blockWithScanInfo = indexReader.loadDataBlockWithScanInfo(key, block,
628           cacheBlocks, pread, isCompaction, getEffectiveDataBlockEncoding());
629       if (blockWithScanInfo == null || blockWithScanInfo.getHFileBlock() == null) {
630         // This happens if the key e.g. falls before the beginning of the file.
631         return -1;
632       }
633       return loadBlockAndSeekToKey(blockWithScanInfo.getHFileBlock(),
634           blockWithScanInfo.getNextIndexedKey(), rewind, key, false);
635     }
636 
637     @Override
638     public boolean seekBefore(byte[] key, int offset, int length) throws IOException {
639       return seekBefore(new KeyValue.KeyOnlyKeyValue(key, offset, length));
640     }
641 
642     @Override
643     public boolean seekBefore(Cell key) throws IOException {
644       HFileBlock seekToBlock = reader.getDataBlockIndexReader().seekToDataBlock(key, block,
645           cacheBlocks, pread, isCompaction,
646           ((HFileReaderV2) reader).getEffectiveEncodingInCache(isCompaction));
647       if (seekToBlock == null) {
648         return false;
649       }
650       ByteBuffer firstKey = getFirstKeyInBlock(seekToBlock);
651 
652       if (reader.getComparator()
653           .compareOnlyKeyPortion(
654               new KeyValue.KeyOnlyKeyValue(firstKey.array(), firstKey.arrayOffset(),
655                   firstKey.limit()), key) >= 0) {
656         long previousBlockOffset = seekToBlock.getPrevBlockOffset();
657         // The key we are interested in
658         if (previousBlockOffset == -1) {
659           // we have a 'problem', the key we want is the first of the file.
660           return false;
661         }
662 
663         // It is important that we compute and pass onDiskSize to the block
664         // reader so that it does not have to read the header separately to
665         // figure out the size.
666         seekToBlock = reader.readBlock(previousBlockOffset,
667             seekToBlock.getOffset() - previousBlockOffset, cacheBlocks,
668             pread, isCompaction, true, BlockType.DATA, getEffectiveDataBlockEncoding());
669         // TODO shortcut: seek forward in this block to the last key of the
670         // block.
671       }
672       byte[] firstKeyInCurrentBlock = Bytes.getBytes(firstKey);
673       loadBlockAndSeekToKey(seekToBlock, firstKeyInCurrentBlock, true, key, true);
674       return true;
675     }
676 
677     /**
678      * Scans blocks in the "scanned" section of the {@link HFile} until the next
679      * data block is found.
680      *
681      * @return the next block, or null if there are no more data blocks
682      * @throws IOException
683      */
684     protected HFileBlock readNextDataBlock() throws IOException {
685       long lastDataBlockOffset = reader.getTrailer().getLastDataBlockOffset();
686       if (block == null)
687         return null;
688 
689       HFileBlock curBlock = block;
690 
691       do {
692         if (curBlock.getOffset() >= lastDataBlockOffset)
693           return null;
694 
695         if (curBlock.getOffset() < 0) {
696           throw new IOException("Invalid block file offset: " + block);
697         }
698 
699         // We are reading the next block without block type validation, because
700         // it might turn out to be a non-data block.
701         curBlock = reader.readBlock(curBlock.getOffset()
702             + curBlock.getOnDiskSizeWithHeader(),
703             curBlock.getNextBlockOnDiskSizeWithHeader(), cacheBlocks, pread,
704             isCompaction, true, null, getEffectiveDataBlockEncoding());
705       } while (!curBlock.getBlockType().isData());
706 
707       return curBlock;
708     }
709 
710     public DataBlockEncoding getEffectiveDataBlockEncoding() {
711       return ((HFileReaderV2)reader).getEffectiveEncodingInCache(isCompaction);
712     }
713     /**
714      * Compare the given key against the current key
715      * @param comparator
716      * @param key
717      * @param offset
718      * @param length
719      * @return -1 is the passed key is smaller than the current key, 0 if equal and 1 if greater
720      */
721     public abstract int compareKey(KVComparator comparator, byte[] key, int offset,
722         int length);
723 
724     public abstract int compareKey(KVComparator comparator, Cell kv);
725   }
726 
727   /**
728    * Implementation of {@link HFileScanner} interface.
729    */
730   protected static class ScannerV2 extends AbstractScannerV2 {
731     private HFileReaderV2 reader;
732 
733     public ScannerV2(HFileReaderV2 r, boolean cacheBlocks,
734         final boolean pread, final boolean isCompaction) {
735       super(r, cacheBlocks, pread, isCompaction);
736       this.reader = r;
737     }
738 
739     @Override
740     public Cell getKeyValue() {
741       if (!isSeeked())
742         return null;
743 
744       KeyValue ret = new KeyValue(blockBuffer.array(), blockBuffer.arrayOffset()
745           + blockBuffer.position(), getCellBufSize());
746       if (this.reader.shouldIncludeMemstoreTS()) {
747         ret.setSequenceId(currMemstoreTS);
748       }
749       return ret;
750     }
751 
752     protected int getCellBufSize() {
753       return KEY_VALUE_LEN_SIZE + currKeyLen + currValueLen;
754     }
755 
756     @Override
757     public ByteBuffer getKey() {
758       assertSeeked();
759       return ByteBuffer.wrap(
760           blockBuffer.array(),
761           blockBuffer.arrayOffset() + blockBuffer.position()
762               + KEY_VALUE_LEN_SIZE, currKeyLen).slice();
763     }
764 
765     @Override
766     public int compareKey(KVComparator comparator, byte[] key, int offset, int length) {
767       return comparator.compareFlatKey(key, offset, length, blockBuffer.array(),
768           blockBuffer.arrayOffset() + blockBuffer.position() + KEY_VALUE_LEN_SIZE, currKeyLen);
769     }
770 
771     @Override
772     public ByteBuffer getValue() {
773       assertSeeked();
774       return ByteBuffer.wrap(
775           blockBuffer.array(),
776           blockBuffer.arrayOffset() + blockBuffer.position()
777               + KEY_VALUE_LEN_SIZE + currKeyLen, currValueLen).slice();
778     }
779 
780     protected void setNonSeekedState() {
781       block = null;
782       blockBuffer = null;
783       currKeyLen = 0;
784       currValueLen = 0;
785       currMemstoreTS = 0;
786       currMemstoreTSLen = 0;
787     }
788 
789     /**
790      * Go to the next key/value in the block section. Loads the next block if
791      * necessary. If successful, {@link #getKey()} and {@link #getValue()} can
792      * be called.
793      *
794      * @return true if successfully navigated to the next key/value
795      */
796     @Override
797     public boolean next() throws IOException {
798       assertSeeked();
799 
800       try {
801         blockBuffer.position(getNextCellStartPosition());
802       } catch (IllegalArgumentException e) {
803         LOG.error("Current pos = " + blockBuffer.position()
804             + "; currKeyLen = " + currKeyLen + "; currValLen = "
805             + currValueLen + "; block limit = " + blockBuffer.limit()
806             + "; HFile name = " + reader.getName()
807             + "; currBlock currBlockOffset = " + block.getOffset());
808         throw e;
809       }
810 
811       if (blockBuffer.remaining() <= 0) {
812         long lastDataBlockOffset =
813             reader.getTrailer().getLastDataBlockOffset();
814 
815         if (block.getOffset() >= lastDataBlockOffset) {
816           setNonSeekedState();
817           return false;
818         }
819 
820         // read the next block
821         HFileBlock nextBlock = readNextDataBlock();
822         if (nextBlock == null) {
823           setNonSeekedState();
824           return false;
825         }
826 
827         updateCurrBlock(nextBlock);
828         return true;
829       }
830 
831       // We are still in the same block.
832       readKeyValueLen();
833       return true;
834     }
835 
836     protected int getNextCellStartPosition() {
837       return blockBuffer.position() + KEY_VALUE_LEN_SIZE + currKeyLen + currValueLen
838           + currMemstoreTSLen;
839     }
840 
841     /**
842      * Positions this scanner at the start of the file.
843      *
844      * @return false if empty file; i.e. a call to next would return false and
845      *         the current key and value are undefined.
846      * @throws IOException
847      */
848     @Override
849     public boolean seekTo() throws IOException {
850       if (reader == null) {
851         return false;
852       }
853 
854       if (reader.getTrailer().getEntryCount() == 0) {
855         // No data blocks.
856         return false;
857       }
858 
859       long firstDataBlockOffset =
860           reader.getTrailer().getFirstDataBlockOffset();
861       if (block != null && block.getOffset() == firstDataBlockOffset) {
862         blockBuffer.rewind();
863         readKeyValueLen();
864         return true;
865       }
866 
867       block = reader.readBlock(firstDataBlockOffset, -1, cacheBlocks, pread,
868           isCompaction, true, BlockType.DATA, getEffectiveDataBlockEncoding());
869       if (block.getOffset() < 0) {
870         throw new IOException("Invalid block offset: " + block.getOffset());
871       }
872       updateCurrBlock(block);
873       return true;
874     }
875 
876     @Override
877     protected int loadBlockAndSeekToKey(HFileBlock seekToBlock, byte[] nextIndexedKey,
878         boolean rewind, Cell key, boolean seekBefore) throws IOException {
879       if (block == null || block.getOffset() != seekToBlock.getOffset()) {
880         updateCurrBlock(seekToBlock);
881       } else if (rewind) {
882         blockBuffer.rewind();
883       }
884 
885       // Update the nextIndexedKey
886       this.nextIndexedKey = nextIndexedKey;
887       return blockSeek(key, seekBefore);
888     }
889 
890     /**
891      * Updates the current block to be the given {@link HFileBlock}. Seeks to
892      * the the first key/value pair.
893      *
894      * @param newBlock the block to make current
895      */
896     protected void updateCurrBlock(HFileBlock newBlock) {
897       block = newBlock;
898 
899       // sanity check
900       if (block.getBlockType() != BlockType.DATA) {
901         throw new IllegalStateException("ScannerV2 works only on data " +
902             "blocks, got " + block.getBlockType() + "; " +
903             "fileName=" + reader.name + ", " +
904             "dataBlockEncoder=" + reader.dataBlockEncoder + ", " +
905             "isCompaction=" + isCompaction);
906       }
907 
908       blockBuffer = block.getBufferWithoutHeader();
909       readKeyValueLen();
910       blockFetches++;
911 
912       // Reset the next indexed key
913       this.nextIndexedKey = null;
914     }
915 
916     protected void readKeyValueLen() {
917       blockBuffer.mark();
918       currKeyLen = blockBuffer.getInt();
919       currValueLen = blockBuffer.getInt();
920       ByteBufferUtils.skip(blockBuffer, currKeyLen + currValueLen);
921       readMvccVersion();
922       if (currKeyLen < 0 || currValueLen < 0
923           || currKeyLen > blockBuffer.limit()
924           || currValueLen > blockBuffer.limit()) {
925         throw new IllegalStateException("Invalid currKeyLen " + currKeyLen
926             + " or currValueLen " + currValueLen + ". Block offset: "
927             + block.getOffset() + ", block length: " + blockBuffer.limit()
928             + ", position: " + blockBuffer.position() + " (without header).");
929       }
930       blockBuffer.reset();
931     }
932 
933     protected void readMvccVersion() {
934       if (this.reader.shouldIncludeMemstoreTS()) {
935         if (this.reader.decodeMemstoreTS) {
936           try {
937             currMemstoreTS = Bytes.readVLong(blockBuffer.array(), blockBuffer.arrayOffset()
938                 + blockBuffer.position());
939             currMemstoreTSLen = WritableUtils.getVIntSize(currMemstoreTS);
940           } catch (Exception e) {
941             throw new RuntimeException("Error reading memstore timestamp", e);
942           }
943         } else {
944           currMemstoreTS = 0;
945           currMemstoreTSLen = 1;
946         }
947       }
948     }
949 
950     /**
951      * Within a loaded block, seek looking for the last key that is smaller than
952      * (or equal to?) the key we are interested in.
953      *
954      * A note on the seekBefore: if you have seekBefore = true, AND the first
955      * key in the block = key, then you'll get thrown exceptions. The caller has
956      * to check for that case and load the previous block as appropriate.
957      *
958      * @param key
959      *          the key to find
960      * @param seekBefore
961      *          find the key before the given key in case of exact match.
962      * @return 0 in case of an exact key match, 1 in case of an inexact match,
963      *         -2 in case of an inexact match and furthermore, the input key
964      *         less than the first key of current block(e.g. using a faked index
965      *         key)
966      */
967     protected int blockSeek(Cell key, boolean seekBefore) {
968       int klen, vlen;
969       long memstoreTS = 0;
970       int memstoreTSLen = 0;
971       int lastKeyValueSize = -1;
972       KeyValue.KeyOnlyKeyValue keyOnlykv = new KeyValue.KeyOnlyKeyValue();
973       do {
974         blockBuffer.mark();
975         klen = blockBuffer.getInt();
976         vlen = blockBuffer.getInt();
977         blockBuffer.reset();
978         if (this.reader.shouldIncludeMemstoreTS()) {
979           if (this.reader.decodeMemstoreTS) {
980             try {
981               int memstoreTSOffset = blockBuffer.arrayOffset() + blockBuffer.position()
982                   + KEY_VALUE_LEN_SIZE + klen + vlen;
983               memstoreTS = Bytes.readVLong(blockBuffer.array(), memstoreTSOffset);
984               memstoreTSLen = WritableUtils.getVIntSize(memstoreTS);
985             } catch (Exception e) {
986               throw new RuntimeException("Error reading memstore timestamp", e);
987             }
988           } else {
989             memstoreTS = 0;
990             memstoreTSLen = 1;
991           }
992         }
993 
994         int keyOffset = blockBuffer.arrayOffset() + blockBuffer.position() + KEY_VALUE_LEN_SIZE;
995         keyOnlykv.setKey(blockBuffer.array(), keyOffset, klen);
996         int comp = reader.getComparator().compareOnlyKeyPortion(key, keyOnlykv);
997 
998         if (comp == 0) {
999           if (seekBefore) {
1000             if (lastKeyValueSize < 0) {
1001               throw new IllegalStateException("blockSeek with seekBefore "
1002                   + "at the first key of the block: key="
1003                   + CellUtil.getCellKeyAsString(key)
1004                   + ", blockOffset=" + block.getOffset() + ", onDiskSize="
1005                   + block.getOnDiskSizeWithHeader());
1006             }
1007             blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
1008             readKeyValueLen();
1009             return 1; // non exact match.
1010           }
1011           currKeyLen = klen;
1012           currValueLen = vlen;
1013           if (this.reader.shouldIncludeMemstoreTS()) {
1014             currMemstoreTS = memstoreTS;
1015             currMemstoreTSLen = memstoreTSLen;
1016           }
1017           return 0; // indicate exact match
1018         } else if (comp < 0) {
1019           if (lastKeyValueSize > 0)
1020             blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
1021           readKeyValueLen();
1022           if (lastKeyValueSize == -1 && blockBuffer.position() == 0
1023               && this.reader.trailer.getMinorVersion() >= MINOR_VERSION_WITH_FAKED_KEY) {
1024             return HConstants.INDEX_KEY_MAGIC;
1025           }
1026           return 1;
1027         }
1028 
1029         // The size of this key/value tuple, including key/value length fields.
1030         lastKeyValueSize = klen + vlen + memstoreTSLen + KEY_VALUE_LEN_SIZE;
1031         blockBuffer.position(blockBuffer.position() + lastKeyValueSize);
1032       } while (blockBuffer.remaining() > 0);
1033 
1034       // Seek to the last key we successfully read. This will happen if this is
1035       // the last key/value pair in the file, in which case the following call
1036       // to next() has to return false.
1037       blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
1038       readKeyValueLen();
1039       return 1; // didn't exactly find it.
1040     }
1041 
1042     @Override
1043     protected ByteBuffer getFirstKeyInBlock(HFileBlock curBlock) {
1044       ByteBuffer buffer = curBlock.getBufferWithoutHeader();
1045       // It is safe to manipulate this buffer because we own the buffer object.
1046       buffer.rewind();
1047       int klen = buffer.getInt();
1048       buffer.getInt();
1049       ByteBuffer keyBuff = buffer.slice();
1050       keyBuff.limit(klen);
1051       keyBuff.rewind();
1052       return keyBuff;
1053     }
1054 
1055     @Override
1056     public String getKeyString() {
1057       return Bytes.toStringBinary(blockBuffer.array(),
1058           blockBuffer.arrayOffset() + blockBuffer.position()
1059               + KEY_VALUE_LEN_SIZE, currKeyLen);
1060     }
1061 
1062     @Override
1063     public String getValueString() {
1064       return Bytes.toString(blockBuffer.array(), blockBuffer.arrayOffset()
1065           + blockBuffer.position() + KEY_VALUE_LEN_SIZE + currKeyLen,
1066           currValueLen);
1067     }
1068 
1069     @Override
1070     public int compareKey(KVComparator comparator, Cell key) {
1071       return comparator.compareOnlyKeyPortion(
1072           key,
1073           new KeyValue.KeyOnlyKeyValue(blockBuffer.array(), blockBuffer.arrayOffset()
1074               + blockBuffer.position() + KEY_VALUE_LEN_SIZE, currKeyLen));
1075     }
1076   }
1077 
1078   /**
1079    * ScannerV2 that operates on encoded data blocks.
1080    */
1081   protected static class EncodedScannerV2 extends AbstractScannerV2 {
1082     private final HFileBlockDecodingContext decodingCtx;
1083     private final DataBlockEncoder.EncodedSeeker seeker;
1084     private final DataBlockEncoder dataBlockEncoder;
1085     protected final HFileContext meta;
1086 
1087     public EncodedScannerV2(HFileReaderV2 reader, boolean cacheBlocks,
1088         boolean pread, boolean isCompaction, HFileContext meta) {
1089       super(reader, cacheBlocks, pread, isCompaction);
1090       DataBlockEncoding encoding = reader.dataBlockEncoder.getDataBlockEncoding();
1091       dataBlockEncoder = encoding.getEncoder();
1092       decodingCtx = dataBlockEncoder.newDataBlockDecodingContext(meta);
1093       seeker = dataBlockEncoder.createSeeker(
1094         reader.getComparator(), decodingCtx);
1095       this.meta = meta;
1096     }
1097 
1098     @Override
1099     public boolean isSeeked(){
1100       return this.block != null;
1101     }
1102 
1103     /**
1104      * Updates the current block to be the given {@link HFileBlock}. Seeks to
1105      * the the first key/value pair.
1106      *
1107      * @param newBlock the block to make current
1108      * @throws CorruptHFileException
1109      */
1110     private void updateCurrentBlock(HFileBlock newBlock) throws CorruptHFileException {
1111       block = newBlock;
1112 
1113       // sanity checks
1114       if (block.getBlockType() != BlockType.ENCODED_DATA) {
1115         throw new IllegalStateException(
1116             "EncodedScanner works only on encoded data blocks");
1117       }
1118       short dataBlockEncoderId = block.getDataBlockEncodingId();
1119       if (!DataBlockEncoding.isCorrectEncoder(dataBlockEncoder, dataBlockEncoderId)) {
1120         String encoderCls = dataBlockEncoder.getClass().getName();
1121         throw new CorruptHFileException("Encoder " + encoderCls
1122           + " doesn't support data block encoding "
1123           + DataBlockEncoding.getNameFromId(dataBlockEncoderId));
1124       }
1125 
1126       seeker.setCurrentBuffer(getEncodedBuffer(newBlock));
1127       blockFetches++;
1128 
1129       // Reset the next indexed key
1130       this.nextIndexedKey = null;
1131     }
1132 
1133     private ByteBuffer getEncodedBuffer(HFileBlock newBlock) {
1134       ByteBuffer origBlock = newBlock.getBufferReadOnly();
1135       ByteBuffer encodedBlock = ByteBuffer.wrap(origBlock.array(),
1136           origBlock.arrayOffset() + newBlock.headerSize() +
1137           DataBlockEncoding.ID_SIZE,
1138           newBlock.getUncompressedSizeWithoutHeader() -
1139           DataBlockEncoding.ID_SIZE).slice();
1140       return encodedBlock;
1141     }
1142 
1143     @Override
1144     public boolean seekTo() throws IOException {
1145       if (reader == null) {
1146         return false;
1147       }
1148 
1149       if (reader.getTrailer().getEntryCount() == 0) {
1150         // No data blocks.
1151         return false;
1152       }
1153 
1154       long firstDataBlockOffset =
1155           reader.getTrailer().getFirstDataBlockOffset();
1156       if (block != null && block.getOffset() == firstDataBlockOffset) {
1157         seeker.rewind();
1158         return true;
1159       }
1160 
1161       block = reader.readBlock(firstDataBlockOffset, -1, cacheBlocks, pread,
1162           isCompaction, true, BlockType.DATA, getEffectiveDataBlockEncoding());
1163       if (block.getOffset() < 0) {
1164         throw new IOException("Invalid block offset: " + block.getOffset());
1165       }
1166       updateCurrentBlock(block);
1167       return true;
1168     }
1169 
1170     @Override
1171     public boolean next() throws IOException {
1172       boolean isValid = seeker.next();
1173       if (!isValid) {
1174         block = readNextDataBlock();
1175         isValid = block != null;
1176         if (isValid) {
1177           updateCurrentBlock(block);
1178         }
1179       }
1180       return isValid;
1181     }
1182 
1183     @Override
1184     public ByteBuffer getKey() {
1185       assertValidSeek();
1186       return seeker.getKeyDeepCopy();
1187     }
1188 
1189     @Override
1190     public int compareKey(KVComparator comparator, byte[] key, int offset, int length) {
1191       return seeker.compareKey(comparator, key, offset, length);
1192     }
1193 
1194     @Override
1195     public ByteBuffer getValue() {
1196       assertValidSeek();
1197       return seeker.getValueShallowCopy();
1198     }
1199 
1200     @Override
1201     public Cell getKeyValue() {
1202       if (block == null) {
1203         return null;
1204       }
1205       return seeker.getKeyValue();
1206     }
1207 
1208     @Override
1209     public String getKeyString() {
1210       ByteBuffer keyBuffer = getKey();
1211       return Bytes.toStringBinary(keyBuffer.array(),
1212           keyBuffer.arrayOffset(), keyBuffer.limit());
1213     }
1214 
1215     @Override
1216     public String getValueString() {
1217       ByteBuffer valueBuffer = getValue();
1218       return Bytes.toStringBinary(valueBuffer.array(),
1219           valueBuffer.arrayOffset(), valueBuffer.limit());
1220     }
1221 
1222     private void assertValidSeek() {
1223       if (block == null) {
1224         throw new NotSeekedException();
1225       }
1226     }
1227 
1228     @Override
1229     protected ByteBuffer getFirstKeyInBlock(HFileBlock curBlock) {
1230       return dataBlockEncoder.getFirstKeyInBlock(getEncodedBuffer(curBlock));
1231     }
1232 
1233     @Override
1234     protected int loadBlockAndSeekToKey(HFileBlock seekToBlock, byte[] nextIndexedKey,
1235         boolean rewind, Cell key, boolean seekBefore) throws IOException {
1236       if (block == null || block.getOffset() != seekToBlock.getOffset()) {
1237         updateCurrentBlock(seekToBlock);
1238       } else if (rewind) {
1239         seeker.rewind();
1240       }
1241       this.nextIndexedKey = nextIndexedKey;
1242       return seeker.seekToKeyInBlock(key, seekBefore);
1243     }
1244 
1245     @Override
1246     public int compareKey(KVComparator comparator, Cell key) {
1247       return seeker.compareKey(comparator, key);
1248     }
1249   }
1250 
1251   /**
1252    * Returns a buffer with the Bloom filter metadata. The caller takes
1253    * ownership of the buffer.
1254    */
1255   @Override
1256   public DataInput getGeneralBloomFilterMetadata() throws IOException {
1257     return this.getBloomFilterMetadata(BlockType.GENERAL_BLOOM_META);
1258   }
1259 
1260   @Override
1261   public DataInput getDeleteBloomFilterMetadata() throws IOException {
1262     return this.getBloomFilterMetadata(BlockType.DELETE_FAMILY_BLOOM_META);
1263   }
1264 
1265   private DataInput getBloomFilterMetadata(BlockType blockType)
1266   throws IOException {
1267     if (blockType != BlockType.GENERAL_BLOOM_META &&
1268         blockType != BlockType.DELETE_FAMILY_BLOOM_META) {
1269       throw new RuntimeException("Block Type: " + blockType.toString() +
1270           " is not supported") ;
1271     }
1272 
1273     for (HFileBlock b : loadOnOpenBlocks)
1274       if (b.getBlockType() == blockType)
1275         return b.getByteStream();
1276     return null;
1277   }
1278 
1279   @Override
1280   public boolean isFileInfoLoaded() {
1281     return true; // We load file info in constructor in version 2.
1282   }
1283 
1284   /**
1285    * Validates that the minor version is within acceptable limits.
1286    * Otherwise throws an Runtime exception
1287    */
1288   private void validateMinorVersion(Path path, int minorVersion) {
1289     if (minorVersion < MIN_MINOR_VERSION ||
1290         minorVersion > MAX_MINOR_VERSION) {
1291       String msg = "Minor version for path " + path +
1292                    " is expected to be between " +
1293                    MIN_MINOR_VERSION + " and " + MAX_MINOR_VERSION +
1294                    " but is found to be " + minorVersion;
1295       LOG.error(msg);
1296       throw new RuntimeException(msg);
1297     }
1298   }
1299 
1300   @Override
1301   public int getMajorVersion() {
1302     return 2;
1303   }
1304 
1305   @Override
1306   public HFileContext getFileContext() {
1307     return hfileContext;
1308   }
1309 
1310   /**
1311    * Returns false if block prefetching was requested for this file and has
1312    * not completed, true otherwise
1313    */
1314   @VisibleForTesting
1315   boolean prefetchComplete() {
1316     return PrefetchExecutor.isCompleted(path);
1317   }
1318 }