View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.io.hfile;
19  
20  import java.io.DataInput;
21  import java.io.IOException;
22  import java.nio.ByteBuffer;
23  import java.util.ArrayList;
24  import java.util.List;
25  
26  import org.apache.commons.logging.Log;
27  import org.apache.commons.logging.LogFactory;
28  import org.apache.hadoop.classification.InterfaceAudience;
29  import org.apache.hadoop.conf.Configuration;
30  import org.apache.hadoop.fs.Path;
31  import org.apache.hadoop.hbase.Cell;
32  import org.apache.hadoop.hbase.HConstants;
33  import org.apache.hadoop.hbase.KeyValue;
34  import org.apache.hadoop.hbase.KeyValue.KVComparator;
35  import org.apache.hadoop.hbase.KeyValueUtil;
36  import org.apache.hadoop.hbase.fs.HFileSystem;
37  import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
38  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder;
39  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
40  import org.apache.hadoop.hbase.io.encoding.HFileBlockDecodingContext;
41  import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
42  import org.apache.hadoop.hbase.util.ByteBufferUtils;
43  import org.apache.hadoop.hbase.util.Bytes;
44  import org.apache.hadoop.hbase.util.IdLock;
45  import org.apache.hadoop.io.WritableUtils;
46  import org.htrace.Trace;
47  import org.htrace.TraceScope;
48  
49  import com.google.common.annotations.VisibleForTesting;
50  
51  /**
52   * {@link HFile} reader for version 2.
53   */
54  @InterfaceAudience.Private
55  public class HFileReaderV2 extends AbstractHFileReader {
56  
57    private static final Log LOG = LogFactory.getLog(HFileReaderV2.class);
58  
59    /** Minor versions in HFile V2 starting with this number have hbase checksums */
60    public static final int MINOR_VERSION_WITH_CHECKSUM = 1;
61    /** In HFile V2 minor version that does not support checksums */
62    public static final int MINOR_VERSION_NO_CHECKSUM = 0;
63  
64    /** HFile minor version that introduced pbuf filetrailer */
65    public static final int PBUF_TRAILER_MINOR_VERSION = 2;
66  
67    /**
68     * The size of a (key length, value length) tuple that prefixes each entry in
69     * a data block.
70     */
71    public final static int KEY_VALUE_LEN_SIZE = 2 * Bytes.SIZEOF_INT;
72  
73    protected boolean includesMemstoreTS = false;
74    protected boolean decodeMemstoreTS = false;
75    protected boolean shouldIncludeMemstoreTS() {
76      return includesMemstoreTS;
77    }
78  
79    /** Filesystem-level block reader. */
80    protected HFileBlock.FSReader fsBlockReader;
81  
82    /**
83     * A "sparse lock" implementation allowing to lock on a particular block
84     * identified by offset. The purpose of this is to avoid two clients loading
85     * the same block, and have all but one client wait to get the block from the
86     * cache.
87     */
88    private IdLock offsetLock = new IdLock();
89  
90    /**
91     * Blocks read from the load-on-open section, excluding data root index, meta
92     * index, and file info.
93     */
94    private List<HFileBlock> loadOnOpenBlocks = new ArrayList<HFileBlock>();
95  
96    /** Minimum minor version supported by this HFile format */
97    static final int MIN_MINOR_VERSION = 0;
98  
99    /** Maximum minor version supported by this HFile format */
100   // We went to version 2 when we moved to pb'ing fileinfo and the trailer on
101   // the file. This version can read Writables version 1.
102   static final int MAX_MINOR_VERSION = 3;
103 
104   /** Minor versions starting with this number have faked index key */
105   static final int MINOR_VERSION_WITH_FAKED_KEY = 3;
106 
107   protected HFileContext hfileContext;
108 
109   /**
110    * Opens a HFile. You must load the index before you can use it by calling
111    * {@link #loadFileInfo()}.
112    *
113    * @param path Path to HFile.
114    * @param trailer File trailer.
115    * @param fsdis input stream.
116    * @param size Length of the stream.
117    * @param cacheConf Cache configuration.
118    * @param hfs
119    * @param conf
120    */
121   public HFileReaderV2(final Path path, final FixedFileTrailer trailer,
122       final FSDataInputStreamWrapper fsdis, final long size, final CacheConfig cacheConf,
123       final HFileSystem hfs, final Configuration conf) throws IOException {
124     super(path, trailer, size, cacheConf, hfs, conf);
125     this.conf = conf;
126     trailer.expectMajorVersion(getMajorVersion());
127     validateMinorVersion(path, trailer.getMinorVersion());
128     this.hfileContext = createHFileContext(fsdis, fileSize, hfs, path, trailer);
129     HFileBlock.FSReaderV2 fsBlockReaderV2 = new HFileBlock.FSReaderV2(fsdis, fileSize, hfs, path,
130         hfileContext);
131     this.fsBlockReader = fsBlockReaderV2; // upcast
132 
133     // Comparator class name is stored in the trailer in version 2.
134     comparator = trailer.createComparator();
135     dataBlockIndexReader = new HFileBlockIndex.BlockIndexReader(comparator,
136         trailer.getNumDataIndexLevels(), this);
137     metaBlockIndexReader = new HFileBlockIndex.BlockIndexReader(
138         KeyValue.RAW_COMPARATOR, 1);
139 
140     // Parse load-on-open data.
141 
142     HFileBlock.BlockIterator blockIter = fsBlockReaderV2.blockRange(
143         trailer.getLoadOnOpenDataOffset(),
144         fileSize - trailer.getTrailerSize());
145 
146     // Data index. We also read statistics about the block index written after
147     // the root level.
148     dataBlockIndexReader.readMultiLevelIndexRoot(
149         blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX),
150         trailer.getDataIndexCount());
151 
152     // Meta index.
153     metaBlockIndexReader.readRootIndex(
154         blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX),
155         trailer.getMetaIndexCount());
156 
157     // File info
158     fileInfo = new FileInfo();
159     fileInfo.read(blockIter.nextBlockWithBlockType(BlockType.FILE_INFO).getByteStream());
160     lastKey = fileInfo.get(FileInfo.LASTKEY);
161     avgKeyLen = Bytes.toInt(fileInfo.get(FileInfo.AVG_KEY_LEN));
162     avgValueLen = Bytes.toInt(fileInfo.get(FileInfo.AVG_VALUE_LEN));
163     byte [] keyValueFormatVersion =
164         fileInfo.get(HFileWriterV2.KEY_VALUE_VERSION);
165     includesMemstoreTS = keyValueFormatVersion != null &&
166         Bytes.toInt(keyValueFormatVersion) ==
167             HFileWriterV2.KEY_VALUE_VER_WITH_MEMSTORE;
168     fsBlockReaderV2.setIncludesMemstoreTS(includesMemstoreTS);
169     if (includesMemstoreTS) {
170       decodeMemstoreTS = Bytes.toLong(fileInfo.get(HFileWriterV2.MAX_MEMSTORE_TS_KEY)) > 0;
171     }
172 
173     // Read data block encoding algorithm name from file info.
174     dataBlockEncoder = HFileDataBlockEncoderImpl.createFromFileInfo(fileInfo);
175     fsBlockReaderV2.setDataBlockEncoder(dataBlockEncoder);
176 
177     // Store all other load-on-open blocks for further consumption.
178     HFileBlock b;
179     while ((b = blockIter.nextBlock()) != null) {
180       loadOnOpenBlocks.add(b);
181     }
182 
183     // Prefetch file blocks upon open if requested
184     if (cacheConf.shouldPrefetchOnOpen()) {
185       PrefetchExecutor.request(path, new Runnable() {
186         public void run() {
187           try {
188             long offset = 0;
189             long end = fileSize - getTrailer().getTrailerSize();
190             HFileBlock prevBlock = null;
191             while (offset < end) {
192               if (Thread.interrupted()) {
193                 break;
194               }
195               long onDiskSize = -1;
196               if (prevBlock != null) {
197                 onDiskSize = prevBlock.getNextBlockOnDiskSizeWithHeader();
198               }
199               HFileBlock block = readBlock(offset, onDiskSize, true, false, false, false,
200                 null, null);
201               prevBlock = block;
202               offset += block.getOnDiskSizeWithHeader();
203             }
204           } catch (IOException e) {
205             // IOExceptions are probably due to region closes (relocation, etc.)
206             if (LOG.isTraceEnabled()) {
207               LOG.trace("Exception encountered while prefetching " + path + ":", e);
208             }
209           } catch (Exception e) {
210             // Other exceptions are interesting
211             LOG.warn("Exception encountered while prefetching " + path + ":", e);
212           } finally {
213             PrefetchExecutor.complete(path);
214           }
215         }
216       });
217     }
218   }
219 
220   protected HFileContext createHFileContext(FSDataInputStreamWrapper fsdis, long fileSize,
221       HFileSystem hfs, Path path, FixedFileTrailer trailer) throws IOException {
222     return new HFileContextBuilder()
223       .withIncludesMvcc(this.includesMemstoreTS)
224       .withCompression(this.compressAlgo)
225       .withHBaseCheckSum(trailer.getMinorVersion() >= MINOR_VERSION_WITH_CHECKSUM)
226       .build();
227   }
228 
229   /**
230    * Create a Scanner on this file. No seeks or reads are done on creation. Call
231    * {@link HFileScanner#seekTo(byte[])} to position an start the read. There is
232    * nothing to clean up in a Scanner. Letting go of your references to the
233    * scanner is sufficient.
234    *
235    * @param cacheBlocks True if we should cache blocks read in by this scanner.
236    * @param pread Use positional read rather than seek+read if true (pread is
237    *          better for random reads, seek+read is better scanning).
238    * @param isCompaction is scanner being used for a compaction?
239    * @return Scanner on this file.
240    */
241    @Override
242    public HFileScanner getScanner(boolean cacheBlocks, final boolean pread,
243       final boolean isCompaction) {
244     if (dataBlockEncoder.useEncodedScanner()) {
245       return new EncodedScannerV2(this, cacheBlocks, pread, isCompaction,
246           hfileContext);
247     }
248 
249     return new ScannerV2(this, cacheBlocks, pread, isCompaction);
250   }
251 
252   /**
253    * Retrieve block from cache. Validates the retrieved block's type vs {@code expectedBlockType}
254    * and its encoding vs. {@code expectedDataBlockEncoding}. Unpacks the block as necessary.
255    */
256    private HFileBlock getCachedBlock(BlockCacheKey cacheKey, boolean cacheBlock, boolean useLock,
257        boolean isCompaction, boolean updateCacheMetrics, BlockType expectedBlockType,
258        DataBlockEncoding expectedDataBlockEncoding) throws IOException {
259      // Check cache for block. If found return.
260      if (cacheConf.isBlockCacheEnabled()) {
261        BlockCache cache = cacheConf.getBlockCache();
262        HFileBlock cachedBlock = (HFileBlock) cache.getBlock(cacheKey, cacheBlock, useLock,
263          updateCacheMetrics);
264        if (cachedBlock != null) {
265          if (cacheConf.shouldCacheCompressed(cachedBlock.getBlockType().getCategory())) {
266            cachedBlock = cachedBlock.unpack(hfileContext, fsBlockReader);
267          }
268          validateBlockType(cachedBlock, expectedBlockType);
269 
270          if (expectedDataBlockEncoding == null) {
271            return cachedBlock;
272          }
273          DataBlockEncoding actualDataBlockEncoding =
274                  cachedBlock.getDataBlockEncoding();
275          // Block types other than data blocks always have
276          // DataBlockEncoding.NONE. To avoid false negative cache misses, only
277          // perform this check if cached block is a data block.
278          if (cachedBlock.getBlockType().isData() &&
279                  !actualDataBlockEncoding.equals(expectedDataBlockEncoding)) {
280            // This mismatch may happen if a ScannerV2, which is used for say a
281            // compaction, tries to read an encoded block from the block cache.
282            // The reverse might happen when an EncodedScannerV2 tries to read
283            // un-encoded blocks which were cached earlier.
284            //
285            // Because returning a data block with an implicit BlockType mismatch
286            // will cause the requesting scanner to throw a disk read should be
287            // forced here. This will potentially cause a significant number of
288            // cache misses, so update so we should keep track of this as it might
289            // justify the work on a CompoundScannerV2.
290            if (!expectedDataBlockEncoding.equals(DataBlockEncoding.NONE) &&
291                    !actualDataBlockEncoding.equals(DataBlockEncoding.NONE)) {
292              // If the block is encoded but the encoding does not match the
293              // expected encoding it is likely the encoding was changed but the
294              // block was not yet evicted. Evictions on file close happen async
295              // so blocks with the old encoding still linger in cache for some
296              // period of time. This event should be rare as it only happens on
297              // schema definition change.
298              LOG.info("Evicting cached block with key " + cacheKey +
299                      " because of a data block encoding mismatch" +
300                      "; expected: " + expectedDataBlockEncoding +
301                      ", actual: " + actualDataBlockEncoding);
302              cache.evictBlock(cacheKey);
303            }
304            return null;
305          }
306          return cachedBlock;
307        }
308      }
309      return null;
310    }
311   /**
312    * @param metaBlockName
313    * @param cacheBlock Add block to cache, if found
314    * @return block wrapped in a ByteBuffer, with header skipped
315    * @throws IOException
316    */
317   @Override
318   public ByteBuffer getMetaBlock(String metaBlockName, boolean cacheBlock)
319       throws IOException {
320     if (trailer.getMetaIndexCount() == 0) {
321       return null; // there are no meta blocks
322     }
323     if (metaBlockIndexReader == null) {
324       throw new IOException("Meta index not loaded");
325     }
326 
327     byte[] mbname = Bytes.toBytes(metaBlockName);
328     int block = metaBlockIndexReader.rootBlockContainingKey(mbname,
329         0, mbname.length);
330     if (block == -1)
331       return null;
332     long blockSize = metaBlockIndexReader.getRootBlockDataSize(block);
333 
334     // Per meta key from any given file, synchronize reads for said block. This
335     // is OK to do for meta blocks because the meta block index is always
336     // single-level.
337     synchronized (metaBlockIndexReader.getRootBlockKey(block)) {
338       // Check cache for block. If found return.
339       long metaBlockOffset = metaBlockIndexReader.getRootBlockOffset(block);
340       BlockCacheKey cacheKey = new BlockCacheKey(name, metaBlockOffset);
341 
342       cacheBlock &= cacheConf.shouldCacheDataOnRead();
343       if (cacheConf.isBlockCacheEnabled()) {
344         HFileBlock cachedBlock = getCachedBlock(cacheKey, cacheBlock, false, true, true,
345           BlockType.META, null);
346         if (cachedBlock != null) {
347           assert cachedBlock.isUnpacked() : "Packed block leak.";
348           // Return a distinct 'shallow copy' of the block,
349           // so pos does not get messed by the scanner
350           return cachedBlock.getBufferWithoutHeader();
351         }
352         // Cache Miss, please load.
353       }
354 
355       HFileBlock metaBlock = fsBlockReader.readBlockData(metaBlockOffset,
356           blockSize, -1, true).unpack(hfileContext, fsBlockReader);
357 
358       // Cache the block
359       if (cacheBlock) {
360         cacheConf.getBlockCache().cacheBlock(cacheKey, metaBlock,
361             cacheConf.isInMemory(), this.cacheConf.isCacheDataInL1());
362       }
363 
364       return metaBlock.getBufferWithoutHeader();
365     }
366   }
367 
368   /**
369    * Read in a file block of the given {@link BlockType} and
370    * {@link DataBlockEncoding}. Unpacks the block as necessary.
371    * @param dataBlockOffset offset to read.
372    * @param onDiskBlockSize size of the block
373    * @param cacheBlock
374    * @param pread Use positional read instead of seek+read (positional is
375    *          better doing random reads whereas seek+read is better scanning).
376    * @param isCompaction is this block being read as part of a compaction
377    * @param expectedBlockType the block type we are expecting to read with this
378    *          read operation, or null to read whatever block type is available
379    *          and avoid checking (that might reduce caching efficiency of
380    *          encoded data blocks)
381    * @param expectedDataBlockEncoding the data block encoding the caller is
382    *          expecting data blocks to be in, or null to not perform this
383    *          check and return the block irrespective of the encoding. This
384    *          check only applies to data blocks and can be set to null when
385    *          the caller is expecting to read a non-data block and has set
386    *          expectedBlockType accordingly.
387    * @return Block wrapped in a ByteBuffer.
388    * @throws IOException
389    */
390   @Override
391   public HFileBlock readBlock(long dataBlockOffset, long onDiskBlockSize,
392       final boolean cacheBlock, boolean pread, final boolean isCompaction,
393       boolean updateCacheMetrics, BlockType expectedBlockType,
394       DataBlockEncoding expectedDataBlockEncoding)
395       throws IOException {
396     if (dataBlockIndexReader == null) {
397       throw new IOException("Block index not loaded");
398     }
399     if (dataBlockOffset < 0
400         || dataBlockOffset >= trailer.getLoadOnOpenDataOffset()) {
401       throw new IOException("Requested block is out of range: "
402           + dataBlockOffset + ", lastDataBlockOffset: "
403           + trailer.getLastDataBlockOffset());
404     }
405     // For any given block from any given file, synchronize reads for said
406     // block.
407     // Without a cache, this synchronizing is needless overhead, but really
408     // the other choice is to duplicate work (which the cache would prevent you
409     // from doing).
410 
411     BlockCacheKey cacheKey = new BlockCacheKey(name, dataBlockOffset);
412 
413     boolean useLock = false;
414     IdLock.Entry lockEntry = null;
415     TraceScope traceScope = Trace.startSpan("HFileReaderV2.readBlock");
416     try {
417       while (true) {
418         if (useLock) {
419           lockEntry = offsetLock.getLockEntry(dataBlockOffset);
420         }
421 
422         // Check cache for block. If found return.
423         if (cacheConf.isBlockCacheEnabled()) {
424           // Try and get the block from the block cache. If the useLock variable is true then this
425           // is the second time through the loop and it should not be counted as a block cache miss.
426           HFileBlock cachedBlock = getCachedBlock(cacheKey, cacheBlock, useLock, isCompaction,
427             updateCacheMetrics, expectedBlockType, expectedDataBlockEncoding);
428           if (cachedBlock != null) {
429             assert cachedBlock.isUnpacked() : "Packed block leak.";
430             if (cachedBlock.getBlockType().isData()) {
431               if (updateCacheMetrics) {
432                 HFile.dataBlockReadCnt.incrementAndGet();
433               }
434               // Validate encoding type for data blocks. We include encoding
435               // type in the cache key, and we expect it to match on a cache hit.
436               if (cachedBlock.getDataBlockEncoding() != dataBlockEncoder.getDataBlockEncoding()) {
437                 throw new IOException("Cached block under key " + cacheKey + " "
438                   + "has wrong encoding: " + cachedBlock.getDataBlockEncoding() + " (expected: "
439                   + dataBlockEncoder.getDataBlockEncoding() + ")");
440               }
441             }
442             return cachedBlock;
443           }
444           // Carry on, please load.
445         }
446         if (!useLock) {
447           // check cache again with lock
448           useLock = true;
449           continue;
450         }
451         if (Trace.isTracing()) {
452           traceScope.getSpan().addTimelineAnnotation("blockCacheMiss");
453         }
454         // Load block from filesystem.
455         HFileBlock hfileBlock = fsBlockReader.readBlockData(dataBlockOffset, onDiskBlockSize, -1,
456             pread);
457         validateBlockType(hfileBlock, expectedBlockType);
458         HFileBlock unpacked = hfileBlock.unpack(hfileContext, fsBlockReader);
459         BlockType.BlockCategory category = hfileBlock.getBlockType().getCategory();
460 
461         // Cache the block if necessary
462         if (cacheBlock && cacheConf.shouldCacheBlockOnRead(category)) {
463           cacheConf.getBlockCache().cacheBlock(cacheKey,
464             cacheConf.shouldCacheCompressed(category) ? hfileBlock : unpacked,
465             cacheConf.isInMemory(), this.cacheConf.isCacheDataInL1());
466         }
467 
468         if (updateCacheMetrics && hfileBlock.getBlockType().isData()) {
469           HFile.dataBlockReadCnt.incrementAndGet();
470         }
471 
472         return unpacked;
473       }
474     } finally {
475       traceScope.close();
476       if (lockEntry != null) {
477         offsetLock.releaseLockEntry(lockEntry);
478       }
479     }
480   }
481 
482   @Override
483   public boolean hasMVCCInfo() {
484     return includesMemstoreTS && decodeMemstoreTS;
485   }
486 
487   /**
488    * Compares the actual type of a block retrieved from cache or disk with its
489    * expected type and throws an exception in case of a mismatch. Expected
490    * block type of {@link BlockType#DATA} is considered to match the actual
491    * block type [@link {@link BlockType#ENCODED_DATA} as well.
492    * @param block a block retrieved from cache or disk
493    * @param expectedBlockType the expected block type, or null to skip the
494    *          check
495    */
496   private void validateBlockType(HFileBlock block,
497       BlockType expectedBlockType) throws IOException {
498     if (expectedBlockType == null) {
499       return;
500     }
501     BlockType actualBlockType = block.getBlockType();
502     if (expectedBlockType.isData() && actualBlockType.isData()) {
503       // We consider DATA to match ENCODED_DATA for the purpose of this
504       // verification.
505       return;
506     }
507     if (actualBlockType != expectedBlockType) {
508       throw new IOException("Expected block type " + expectedBlockType + ", " +
509           "but got " + actualBlockType + ": " + block);
510     }
511   }
512 
513   /**
514    * @return Last key in the file. May be null if file has no entries. Note that
515    *         this is not the last row key, but rather the byte form of the last
516    *         KeyValue.
517    */
518   @Override
519   public byte[] getLastKey() {
520     return dataBlockIndexReader.isEmpty() ? null : lastKey;
521   }
522 
523   /**
524    * @return Midkey for this file. We work with block boundaries only so
525    *         returned midkey is an approximation only.
526    * @throws IOException
527    */
528   @Override
529   public byte[] midkey() throws IOException {
530     return dataBlockIndexReader.midkey();
531   }
532 
533   @Override
534   public void close() throws IOException {
535     close(cacheConf.shouldEvictOnClose());
536   }
537 
538   public void close(boolean evictOnClose) throws IOException {
539     PrefetchExecutor.cancel(path);
540     if (evictOnClose && cacheConf.isBlockCacheEnabled()) {
541       int numEvicted = cacheConf.getBlockCache().evictBlocksByHfileName(name);
542       if (LOG.isTraceEnabled()) {
543         LOG.trace("On close, file=" + name + " evicted=" + numEvicted
544           + " block(s)");
545       }
546     }
547     fsBlockReader.closeStreams();
548   }
549 
550   public DataBlockEncoding getEffectiveEncodingInCache(boolean isCompaction) {
551     return dataBlockEncoder.getEffectiveEncodingInCache(isCompaction);
552   }
553 
554   /** For testing */
555   @Override
556   HFileBlock.FSReader getUncachedBlockReader() {
557     return fsBlockReader;
558   }
559 
560 
561   protected abstract static class AbstractScannerV2
562       extends AbstractHFileReader.Scanner {
563     protected HFileBlock block;
564 
565     /**
566      * The next indexed key is to keep track of the indexed key of the next data block.
567      * If the nextIndexedKey is HConstants.NO_NEXT_INDEXED_KEY, it means that the
568      * current data block is the last data block.
569      *
570      * If the nextIndexedKey is null, it means the nextIndexedKey has not been loaded yet.
571      */
572     protected byte[] nextIndexedKey;
573 
574     public AbstractScannerV2(HFileReaderV2 r, boolean cacheBlocks,
575         final boolean pread, final boolean isCompaction) {
576       super(r, cacheBlocks, pread, isCompaction);
577     }
578 
579     protected abstract ByteBuffer getFirstKeyInBlock(HFileBlock curBlock);
580 
581     protected abstract int loadBlockAndSeekToKey(HFileBlock seekToBlock, byte[] nextIndexedKey,
582         boolean rewind, Cell key, boolean seekBefore) throws IOException;
583 
584     @Override
585     public int seekTo(byte[] key, int offset, int length) throws IOException {
586       // Always rewind to the first key of the block, because the given key
587       // might be before or after the current key.
588       return seekTo(new KeyValue.KeyOnlyKeyValue(key, offset, length));
589     }
590 
591     @Override
592     public int reseekTo(byte[] key, int offset, int length) throws IOException {
593       return reseekTo(new KeyValue.KeyOnlyKeyValue(key, offset, length));
594     }
595 
596     @Override
597     public int seekTo(Cell key) throws IOException {
598       return seekTo(key, true);
599     }
600 
601     @Override
602     public int reseekTo(Cell key) throws IOException {
603       int compared;
604       if (isSeeked()) {
605         compared = compareKey(reader.getComparator(), key);
606         if (compared < 1) {
607           // If the required key is less than or equal to current key, then
608           // don't do anything.
609           return compared;
610         } else {
611           // The comparison with no_next_index_key has to be checked
612           if (this.nextIndexedKey != null &&
613               (this.nextIndexedKey == HConstants.NO_NEXT_INDEXED_KEY || reader
614               .getComparator()
615                   .compareOnlyKeyPortion(key,
616                       new KeyValue.KeyOnlyKeyValue(nextIndexedKey, 0, 
617                           nextIndexedKey.length)) < 0)) {
618             // The reader shall continue to scan the current data block instead
619             // of querying the
620             // block index as long as it knows the target key is strictly
621             // smaller than
622             // the next indexed key or the current data block is the last data
623             // block.
624             return loadBlockAndSeekToKey(this.block, nextIndexedKey, false, key, false);
625           }
626         }
627       }
628       // Don't rewind on a reseek operation, because reseek implies that we are
629       // always going forward in the file.
630       return seekTo(key, false);
631     }
632 
633 
634     /**
635      * An internal API function. Seek to the given key, optionally rewinding to
636      * the first key of the block before doing the seek.
637      *
638      * @param key - a cell representing the key that we need to fetch
639      * @param rewind whether to rewind to the first key of the block before
640      *        doing the seek. If this is false, we are assuming we never go
641      *        back, otherwise the result is undefined.
642      * @return -1 if the key is earlier than the first key of the file,
643      *         0 if we are at the given key, 1 if we are past the given key
644      *         -2 if the key is earlier than the first key of the file while
645      *         using a faked index key
646      * @throws IOException
647      */
648     public int seekTo(Cell key, boolean rewind) throws IOException {
649       HFileBlockIndex.BlockIndexReader indexReader = reader.getDataBlockIndexReader();
650       BlockWithScanInfo blockWithScanInfo = indexReader.loadDataBlockWithScanInfo(key, block,
651           cacheBlocks, pread, isCompaction, getEffectiveDataBlockEncoding());
652       if (blockWithScanInfo == null || blockWithScanInfo.getHFileBlock() == null) {
653         // This happens if the key e.g. falls before the beginning of the file.
654         return -1;
655       }
656       return loadBlockAndSeekToKey(blockWithScanInfo.getHFileBlock(),
657           blockWithScanInfo.getNextIndexedKey(), rewind, key, false);
658     }
659 
660     @Override
661     public boolean seekBefore(byte[] key, int offset, int length) throws IOException {
662       return seekBefore(new KeyValue.KeyOnlyKeyValue(key, offset, length));
663     }
664 
665     @Override
666     public boolean seekBefore(Cell key) throws IOException {
667       HFileBlock seekToBlock = reader.getDataBlockIndexReader().seekToDataBlock(key, block,
668           cacheBlocks, pread, isCompaction,
669           ((HFileReaderV2) reader).getEffectiveEncodingInCache(isCompaction));
670       if (seekToBlock == null) {
671         return false;
672       }
673       ByteBuffer firstKey = getFirstKeyInBlock(seekToBlock);
674 
675       if (reader.getComparator()
676           .compareOnlyKeyPortion(
677               new KeyValue.KeyOnlyKeyValue(firstKey.array(), firstKey.arrayOffset(),
678                   firstKey.limit()), key) >= 0) {
679         long previousBlockOffset = seekToBlock.getPrevBlockOffset();
680         // The key we are interested in
681         if (previousBlockOffset == -1) {
682           // we have a 'problem', the key we want is the first of the file.
683           return false;
684         }
685 
686         // It is important that we compute and pass onDiskSize to the block
687         // reader so that it does not have to read the header separately to
688         // figure out the size.
689         seekToBlock = reader.readBlock(previousBlockOffset,
690             seekToBlock.getOffset() - previousBlockOffset, cacheBlocks,
691             pread, isCompaction, true, BlockType.DATA, getEffectiveDataBlockEncoding());
692         // TODO shortcut: seek forward in this block to the last key of the
693         // block.
694       }
695       byte[] firstKeyInCurrentBlock = Bytes.getBytes(firstKey);
696       loadBlockAndSeekToKey(seekToBlock, firstKeyInCurrentBlock, true, key, true);
697       return true;
698     }
699 
700     /**
701      * Scans blocks in the "scanned" section of the {@link HFile} until the next
702      * data block is found.
703      *
704      * @return the next block, or null if there are no more data blocks
705      * @throws IOException
706      */
707     protected HFileBlock readNextDataBlock() throws IOException {
708       long lastDataBlockOffset = reader.getTrailer().getLastDataBlockOffset();
709       if (block == null)
710         return null;
711 
712       HFileBlock curBlock = block;
713 
714       do {
715         if (curBlock.getOffset() >= lastDataBlockOffset)
716           return null;
717 
718         if (curBlock.getOffset() < 0) {
719           throw new IOException("Invalid block file offset: " + block);
720         }
721 
722         // We are reading the next block without block type validation, because
723         // it might turn out to be a non-data block.
724         curBlock = reader.readBlock(curBlock.getOffset()
725             + curBlock.getOnDiskSizeWithHeader(),
726             curBlock.getNextBlockOnDiskSizeWithHeader(), cacheBlocks, pread,
727             isCompaction, true, null, getEffectiveDataBlockEncoding());
728       } while (!curBlock.getBlockType().isData());
729 
730       return curBlock;
731     }
732     
733     public DataBlockEncoding getEffectiveDataBlockEncoding() {
734       return ((HFileReaderV2)reader).getEffectiveEncodingInCache(isCompaction);
735     }
736     /**
737      * Compare the given key against the current key
738      * @param comparator
739      * @param key
740      * @param offset
741      * @param length
742      * @return -1 is the passed key is smaller than the current key, 0 if equal and 1 if greater
743      */
744     public abstract int compareKey(KVComparator comparator, byte[] key, int offset,
745         int length);
746 
747     public abstract int compareKey(KVComparator comparator, Cell kv);
748   }
749 
750   /**
751    * Implementation of {@link HFileScanner} interface.
752    */
753   protected static class ScannerV2 extends AbstractScannerV2 {
754     private HFileReaderV2 reader;
755 
756     public ScannerV2(HFileReaderV2 r, boolean cacheBlocks,
757         final boolean pread, final boolean isCompaction) {
758       super(r, cacheBlocks, pread, isCompaction);
759       this.reader = r;
760     }
761 
762     @Override
763     public Cell getKeyValue() {
764       if (!isSeeked())
765         return null;
766 
767       KeyValue ret = new KeyValue(blockBuffer.array(), blockBuffer.arrayOffset()
768           + blockBuffer.position(), getCellBufSize());
769       if (this.reader.shouldIncludeMemstoreTS()) {
770         ret.setSequenceId(currMemstoreTS);
771       }
772       return ret;
773     }
774 
775     protected int getCellBufSize() {
776       return KEY_VALUE_LEN_SIZE + currKeyLen + currValueLen;
777     }
778 
779     @Override
780     public ByteBuffer getKey() {
781       assertSeeked();
782       return ByteBuffer.wrap(
783           blockBuffer.array(),
784           blockBuffer.arrayOffset() + blockBuffer.position()
785               + KEY_VALUE_LEN_SIZE, currKeyLen).slice();
786     }
787 
788     @Override
789     public int compareKey(KVComparator comparator, byte[] key, int offset, int length) {
790       return comparator.compareFlatKey(key, offset, length, blockBuffer.array(),
791           blockBuffer.arrayOffset() + blockBuffer.position() + KEY_VALUE_LEN_SIZE, currKeyLen);
792     }
793 
794     @Override
795     public ByteBuffer getValue() {
796       assertSeeked();
797       return ByteBuffer.wrap(
798           blockBuffer.array(),
799           blockBuffer.arrayOffset() + blockBuffer.position()
800               + KEY_VALUE_LEN_SIZE + currKeyLen, currValueLen).slice();
801     }
802 
803     protected void setNonSeekedState() {
804       block = null;
805       blockBuffer = null;
806       currKeyLen = 0;
807       currValueLen = 0;
808       currMemstoreTS = 0;
809       currMemstoreTSLen = 0;
810     }
811 
812     /**
813      * Go to the next key/value in the block section. Loads the next block if
814      * necessary. If successful, {@link #getKey()} and {@link #getValue()} can
815      * be called.
816      *
817      * @return true if successfully navigated to the next key/value
818      */
819     @Override
820     public boolean next() throws IOException {
821       assertSeeked();
822 
823       try {
824         blockBuffer.position(getNextCellStartPosition());
825       } catch (IllegalArgumentException e) {
826         LOG.error("Current pos = " + blockBuffer.position()
827             + "; currKeyLen = " + currKeyLen + "; currValLen = "
828             + currValueLen + "; block limit = " + blockBuffer.limit()
829             + "; HFile name = " + reader.getName()
830             + "; currBlock currBlockOffset = " + block.getOffset());
831         throw e;
832       }
833 
834       if (blockBuffer.remaining() <= 0) {
835         long lastDataBlockOffset =
836             reader.getTrailer().getLastDataBlockOffset();
837 
838         if (block.getOffset() >= lastDataBlockOffset) {
839           setNonSeekedState();
840           return false;
841         }
842 
843         // read the next block
844         HFileBlock nextBlock = readNextDataBlock();
845         if (nextBlock == null) {
846           setNonSeekedState();
847           return false;
848         }
849 
850         updateCurrBlock(nextBlock);
851         return true;
852       }
853 
854       // We are still in the same block.
855       readKeyValueLen();
856       return true;
857     }
858 
859     protected int getNextCellStartPosition() {
860       return blockBuffer.position() + KEY_VALUE_LEN_SIZE + currKeyLen + currValueLen
861           + currMemstoreTSLen;
862     }
863 
864     /**
865      * Positions this scanner at the start of the file.
866      *
867      * @return false if empty file; i.e. a call to next would return false and
868      *         the current key and value are undefined.
869      * @throws IOException
870      */
871     @Override
872     public boolean seekTo() throws IOException {
873       if (reader == null) {
874         return false;
875       }
876 
877       if (reader.getTrailer().getEntryCount() == 0) {
878         // No data blocks.
879         return false;
880       }
881 
882       long firstDataBlockOffset =
883           reader.getTrailer().getFirstDataBlockOffset();
884       if (block != null && block.getOffset() == firstDataBlockOffset) {
885         blockBuffer.rewind();
886         readKeyValueLen();
887         return true;
888       }
889 
890       block = reader.readBlock(firstDataBlockOffset, -1, cacheBlocks, pread,
891           isCompaction, true, BlockType.DATA, getEffectiveDataBlockEncoding());
892       if (block.getOffset() < 0) {
893         throw new IOException("Invalid block offset: " + block.getOffset());
894       }
895       updateCurrBlock(block);
896       return true;
897     }
898 
899     @Override
900     protected int loadBlockAndSeekToKey(HFileBlock seekToBlock, byte[] nextIndexedKey,
901         boolean rewind, Cell key, boolean seekBefore) throws IOException {
902       if (block == null || block.getOffset() != seekToBlock.getOffset()) {
903         updateCurrBlock(seekToBlock);
904       } else if (rewind) {
905         blockBuffer.rewind();
906       }
907 
908       // Update the nextIndexedKey
909       this.nextIndexedKey = nextIndexedKey;
910       return blockSeek(key, seekBefore);
911     }
912 
913     /**
914      * Updates the current block to be the given {@link HFileBlock}. Seeks to
915      * the the first key/value pair.
916      *
917      * @param newBlock the block to make current
918      */
919     protected void updateCurrBlock(HFileBlock newBlock) {
920       block = newBlock;
921 
922       // sanity check
923       if (block.getBlockType() != BlockType.DATA) {
924         throw new IllegalStateException("ScannerV2 works only on data " +
925             "blocks, got " + block.getBlockType() + "; " +
926             "fileName=" + reader.name + ", " +
927             "dataBlockEncoder=" + reader.dataBlockEncoder + ", " +
928             "isCompaction=" + isCompaction);
929       }
930 
931       blockBuffer = block.getBufferWithoutHeader();
932       readKeyValueLen();
933       blockFetches++;
934 
935       // Reset the next indexed key
936       this.nextIndexedKey = null;
937     }
938 
939     protected void readKeyValueLen() {
940       blockBuffer.mark();
941       currKeyLen = blockBuffer.getInt();
942       currValueLen = blockBuffer.getInt();
943       ByteBufferUtils.skip(blockBuffer, currKeyLen + currValueLen);
944       readMvccVersion();
945       if (currKeyLen < 0 || currValueLen < 0
946           || currKeyLen > blockBuffer.limit()
947           || currValueLen > blockBuffer.limit()) {
948         throw new IllegalStateException("Invalid currKeyLen " + currKeyLen
949             + " or currValueLen " + currValueLen + ". Block offset: "
950             + block.getOffset() + ", block length: " + blockBuffer.limit()
951             + ", position: " + blockBuffer.position() + " (without header).");
952       }
953       blockBuffer.reset();
954     }
955 
956     protected void readMvccVersion() {
957       if (this.reader.shouldIncludeMemstoreTS()) {
958         if (this.reader.decodeMemstoreTS) {
959           try {
960             currMemstoreTS = Bytes.readVLong(blockBuffer.array(), blockBuffer.arrayOffset()
961                 + blockBuffer.position());
962             currMemstoreTSLen = WritableUtils.getVIntSize(currMemstoreTS);
963           } catch (Exception e) {
964             throw new RuntimeException("Error reading memstore timestamp", e);
965           }
966         } else {
967           currMemstoreTS = 0;
968           currMemstoreTSLen = 1;
969         }
970       }
971     }
972 
973     /**
974      * Within a loaded block, seek looking for the last key that is smaller than
975      * (or equal to?) the key we are interested in.
976      *
977      * A note on the seekBefore: if you have seekBefore = true, AND the first
978      * key in the block = key, then you'll get thrown exceptions. The caller has
979      * to check for that case and load the previous block as appropriate.
980      *
981      * @param key
982      *          the key to find
983      * @param seekBefore
984      *          find the key before the given key in case of exact match.
985      * @return 0 in case of an exact key match, 1 in case of an inexact match,
986      *         -2 in case of an inexact match and furthermore, the input key
987      *         less than the first key of current block(e.g. using a faked index
988      *         key)
989      */
990     protected int blockSeek(Cell key, boolean seekBefore) {
991       int klen, vlen;
992       long memstoreTS = 0;
993       int memstoreTSLen = 0;
994       int lastKeyValueSize = -1;
995       KeyValue.KeyOnlyKeyValue keyOnlykv = new KeyValue.KeyOnlyKeyValue();
996       do {
997         blockBuffer.mark();
998         klen = blockBuffer.getInt();
999         vlen = blockBuffer.getInt();
1000         blockBuffer.reset();
1001         if (this.reader.shouldIncludeMemstoreTS()) {
1002           if (this.reader.decodeMemstoreTS) {
1003             try {
1004               int memstoreTSOffset = blockBuffer.arrayOffset() + blockBuffer.position()
1005                   + KEY_VALUE_LEN_SIZE + klen + vlen;
1006               memstoreTS = Bytes.readVLong(blockBuffer.array(), memstoreTSOffset);
1007               memstoreTSLen = WritableUtils.getVIntSize(memstoreTS);
1008             } catch (Exception e) {
1009               throw new RuntimeException("Error reading memstore timestamp", e);
1010             }
1011           } else {
1012             memstoreTS = 0;
1013             memstoreTSLen = 1;
1014           }
1015         }
1016 
1017         int keyOffset = blockBuffer.arrayOffset() + blockBuffer.position() + KEY_VALUE_LEN_SIZE;
1018         keyOnlykv.setKey(blockBuffer.array(), keyOffset, klen);
1019         int comp = reader.getComparator().compareOnlyKeyPortion(key, keyOnlykv);
1020 
1021         if (comp == 0) {
1022           if (seekBefore) {
1023             if (lastKeyValueSize < 0) {
1024               KeyValue kv = KeyValueUtil.ensureKeyValue(key);
1025               throw new IllegalStateException("blockSeek with seekBefore "
1026                   + "at the first key of the block: key="
1027                   + Bytes.toStringBinary(kv.getKey(), kv.getKeyOffset(), kv.getKeyLength())
1028                   + ", blockOffset=" + block.getOffset() + ", onDiskSize="
1029                   + block.getOnDiskSizeWithHeader());
1030             }
1031             blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
1032             readKeyValueLen();
1033             return 1; // non exact match.
1034           }
1035           currKeyLen = klen;
1036           currValueLen = vlen;
1037           if (this.reader.shouldIncludeMemstoreTS()) {
1038             currMemstoreTS = memstoreTS;
1039             currMemstoreTSLen = memstoreTSLen;
1040           }
1041           return 0; // indicate exact match
1042         } else if (comp < 0) {
1043           if (lastKeyValueSize > 0)
1044             blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
1045           readKeyValueLen();
1046           if (lastKeyValueSize == -1 && blockBuffer.position() == 0
1047               && this.reader.trailer.getMinorVersion() >= MINOR_VERSION_WITH_FAKED_KEY) {
1048             return HConstants.INDEX_KEY_MAGIC;
1049           }
1050           return 1;
1051         }
1052 
1053         // The size of this key/value tuple, including key/value length fields.
1054         lastKeyValueSize = klen + vlen + memstoreTSLen + KEY_VALUE_LEN_SIZE;
1055         blockBuffer.position(blockBuffer.position() + lastKeyValueSize);
1056       } while (blockBuffer.remaining() > 0);
1057 
1058       // Seek to the last key we successfully read. This will happen if this is
1059       // the last key/value pair in the file, in which case the following call
1060       // to next() has to return false.
1061       blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
1062       readKeyValueLen();
1063       return 1; // didn't exactly find it.
1064     }
1065 
1066     @Override
1067     protected ByteBuffer getFirstKeyInBlock(HFileBlock curBlock) {
1068       ByteBuffer buffer = curBlock.getBufferWithoutHeader();
1069       // It is safe to manipulate this buffer because we own the buffer object.
1070       buffer.rewind();
1071       int klen = buffer.getInt();
1072       buffer.getInt();
1073       ByteBuffer keyBuff = buffer.slice();
1074       keyBuff.limit(klen);
1075       keyBuff.rewind();
1076       return keyBuff;
1077     }
1078 
1079     @Override
1080     public String getKeyString() {
1081       return Bytes.toStringBinary(blockBuffer.array(),
1082           blockBuffer.arrayOffset() + blockBuffer.position()
1083               + KEY_VALUE_LEN_SIZE, currKeyLen);
1084     }
1085 
1086     @Override
1087     public String getValueString() {
1088       return Bytes.toString(blockBuffer.array(), blockBuffer.arrayOffset()
1089           + blockBuffer.position() + KEY_VALUE_LEN_SIZE + currKeyLen,
1090           currValueLen);
1091     }
1092 
1093     @Override
1094     public int compareKey(KVComparator comparator, Cell key) {
1095       return comparator.compareOnlyKeyPortion(
1096           key,
1097           new KeyValue.KeyOnlyKeyValue(blockBuffer.array(), blockBuffer.arrayOffset()
1098               + blockBuffer.position() + KEY_VALUE_LEN_SIZE, currKeyLen));
1099     }
1100   }
1101 
1102   /**
1103    * ScannerV2 that operates on encoded data blocks.
1104    */
1105   protected static class EncodedScannerV2 extends AbstractScannerV2 {
1106     private final HFileBlockDecodingContext decodingCtx;
1107     private final DataBlockEncoder.EncodedSeeker seeker;
1108     private final DataBlockEncoder dataBlockEncoder;
1109     protected final HFileContext meta;
1110 
1111     public EncodedScannerV2(HFileReaderV2 reader, boolean cacheBlocks,
1112         boolean pread, boolean isCompaction, HFileContext meta) {
1113       super(reader, cacheBlocks, pread, isCompaction);
1114       DataBlockEncoding encoding = reader.dataBlockEncoder.getDataBlockEncoding();
1115       dataBlockEncoder = encoding.getEncoder();
1116       decodingCtx = dataBlockEncoder.newDataBlockDecodingContext(meta);
1117       seeker = dataBlockEncoder.createSeeker(
1118         reader.getComparator(), decodingCtx);
1119       this.meta = meta;
1120     }
1121 
1122     @Override
1123     public boolean isSeeked(){
1124       return this.block != null;
1125     }
1126 
1127     /**
1128      * Updates the current block to be the given {@link HFileBlock}. Seeks to
1129      * the the first key/value pair.
1130      *
1131      * @param newBlock the block to make current
1132      * @throws CorruptHFileException
1133      */
1134     private void updateCurrentBlock(HFileBlock newBlock) throws CorruptHFileException {
1135       block = newBlock;
1136 
1137       // sanity checks
1138       if (block.getBlockType() != BlockType.ENCODED_DATA) {
1139         throw new IllegalStateException(
1140             "EncodedScanner works only on encoded data blocks");
1141       }
1142       short dataBlockEncoderId = block.getDataBlockEncodingId();
1143       if (!DataBlockEncoding.isCorrectEncoder(dataBlockEncoder, dataBlockEncoderId)) {
1144         String encoderCls = dataBlockEncoder.getClass().getName();
1145         throw new CorruptHFileException("Encoder " + encoderCls
1146           + " doesn't support data block encoding "
1147           + DataBlockEncoding.getNameFromId(dataBlockEncoderId));
1148       }
1149 
1150       seeker.setCurrentBuffer(getEncodedBuffer(newBlock));
1151       blockFetches++;
1152 
1153       // Reset the next indexed key
1154       this.nextIndexedKey = null;
1155     }
1156 
1157     private ByteBuffer getEncodedBuffer(HFileBlock newBlock) {
1158       ByteBuffer origBlock = newBlock.getBufferReadOnly();
1159       ByteBuffer encodedBlock = ByteBuffer.wrap(origBlock.array(),
1160           origBlock.arrayOffset() + newBlock.headerSize() +
1161           DataBlockEncoding.ID_SIZE,
1162           newBlock.getUncompressedSizeWithoutHeader() -
1163           DataBlockEncoding.ID_SIZE).slice();
1164       return encodedBlock;
1165     }
1166 
1167     @Override
1168     public boolean seekTo() throws IOException {
1169       if (reader == null) {
1170         return false;
1171       }
1172 
1173       if (reader.getTrailer().getEntryCount() == 0) {
1174         // No data blocks.
1175         return false;
1176       }
1177 
1178       long firstDataBlockOffset =
1179           reader.getTrailer().getFirstDataBlockOffset();
1180       if (block != null && block.getOffset() == firstDataBlockOffset) {
1181         seeker.rewind();
1182         return true;
1183       }
1184 
1185       block = reader.readBlock(firstDataBlockOffset, -1, cacheBlocks, pread,
1186           isCompaction, true, BlockType.DATA, getEffectiveDataBlockEncoding());
1187       if (block.getOffset() < 0) {
1188         throw new IOException("Invalid block offset: " + block.getOffset());
1189       }
1190       updateCurrentBlock(block);
1191       return true;
1192     }
1193 
1194     @Override
1195     public boolean next() throws IOException {
1196       boolean isValid = seeker.next();
1197       if (!isValid) {
1198         block = readNextDataBlock();
1199         isValid = block != null;
1200         if (isValid) {
1201           updateCurrentBlock(block);
1202         }
1203       }
1204       return isValid;
1205     }
1206 
1207     @Override
1208     public ByteBuffer getKey() {
1209       assertValidSeek();
1210       return seeker.getKeyDeepCopy();
1211     }
1212 
1213     @Override
1214     public int compareKey(KVComparator comparator, byte[] key, int offset, int length) {
1215       return seeker.compareKey(comparator, key, offset, length);
1216     }
1217 
1218     @Override
1219     public ByteBuffer getValue() {
1220       assertValidSeek();
1221       return seeker.getValueShallowCopy();
1222     }
1223 
1224     @Override
1225     public Cell getKeyValue() {
1226       if (block == null) {
1227         return null;
1228       }
1229       return seeker.getKeyValue();
1230     }
1231 
1232     @Override
1233     public String getKeyString() {
1234       ByteBuffer keyBuffer = getKey();
1235       return Bytes.toStringBinary(keyBuffer.array(),
1236           keyBuffer.arrayOffset(), keyBuffer.limit());
1237     }
1238 
1239     @Override
1240     public String getValueString() {
1241       ByteBuffer valueBuffer = getValue();
1242       return Bytes.toStringBinary(valueBuffer.array(),
1243           valueBuffer.arrayOffset(), valueBuffer.limit());
1244     }
1245 
1246     private void assertValidSeek() {
1247       if (block == null) {
1248         throw new NotSeekedException();
1249       }
1250     }
1251 
1252     @Override
1253     protected ByteBuffer getFirstKeyInBlock(HFileBlock curBlock) {
1254       return dataBlockEncoder.getFirstKeyInBlock(getEncodedBuffer(curBlock));
1255     }
1256 
1257     @Override
1258     protected int loadBlockAndSeekToKey(HFileBlock seekToBlock, byte[] nextIndexedKey,
1259         boolean rewind, Cell key, boolean seekBefore) throws IOException {
1260       if (block == null || block.getOffset() != seekToBlock.getOffset()) {
1261         updateCurrentBlock(seekToBlock);
1262       } else if (rewind) {
1263         seeker.rewind();
1264       }
1265       this.nextIndexedKey = nextIndexedKey;
1266       return seeker.seekToKeyInBlock(key, seekBefore);
1267     }
1268 
1269     @Override
1270     public int compareKey(KVComparator comparator, Cell key) {
1271       return seeker.compareKey(comparator, key);
1272     }
1273   }
1274 
1275   /**
1276    * Returns a buffer with the Bloom filter metadata. The caller takes
1277    * ownership of the buffer.
1278    */
1279   @Override
1280   public DataInput getGeneralBloomFilterMetadata() throws IOException {
1281     return this.getBloomFilterMetadata(BlockType.GENERAL_BLOOM_META);
1282   }
1283 
1284   @Override
1285   public DataInput getDeleteBloomFilterMetadata() throws IOException {
1286     return this.getBloomFilterMetadata(BlockType.DELETE_FAMILY_BLOOM_META);
1287   }
1288 
1289   private DataInput getBloomFilterMetadata(BlockType blockType)
1290   throws IOException {
1291     if (blockType != BlockType.GENERAL_BLOOM_META &&
1292         blockType != BlockType.DELETE_FAMILY_BLOOM_META) {
1293       throw new RuntimeException("Block Type: " + blockType.toString() +
1294           " is not supported") ;
1295     }
1296 
1297     for (HFileBlock b : loadOnOpenBlocks)
1298       if (b.getBlockType() == blockType)
1299         return b.getByteStream();
1300     return null;
1301   }
1302 
1303   @Override
1304   public boolean isFileInfoLoaded() {
1305     return true; // We load file info in constructor in version 2.
1306   }
1307 
1308   /**
1309    * Validates that the minor version is within acceptable limits.
1310    * Otherwise throws an Runtime exception
1311    */
1312   private void validateMinorVersion(Path path, int minorVersion) {
1313     if (minorVersion < MIN_MINOR_VERSION ||
1314         minorVersion > MAX_MINOR_VERSION) {
1315       String msg = "Minor version for path " + path + 
1316                    " is expected to be between " +
1317                    MIN_MINOR_VERSION + " and " + MAX_MINOR_VERSION +
1318                    " but is found to be " + minorVersion;
1319       LOG.error(msg);
1320       throw new RuntimeException(msg);
1321     }
1322   }
1323 
1324   @Override
1325   public int getMajorVersion() {
1326     return 2;
1327   }
1328 
1329   @Override
1330   public HFileContext getFileContext() {
1331     return hfileContext;
1332   }
1333 
1334   /**
1335    * Returns false if block prefetching was requested for this file and has
1336    * not completed, true otherwise
1337    */
1338   @VisibleForTesting
1339   boolean prefetchComplete() {
1340     return PrefetchExecutor.isCompleted(path);
1341   }
1342 }