View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.io.hfile;
19  
20  import java.io.DataInput;
21  import java.io.IOException;
22  import java.nio.ByteBuffer;
23  import java.util.ArrayList;
24  import java.util.List;
25  
26  import org.apache.commons.logging.Log;
27  import org.apache.commons.logging.LogFactory;
28  import org.apache.hadoop.classification.InterfaceAudience;
29  import org.apache.hadoop.conf.Configuration;
30  import org.apache.hadoop.fs.Path;
31  import org.apache.hadoop.hbase.Cell;
32  import org.apache.hadoop.hbase.HConstants;
33  import org.apache.hadoop.hbase.KeyValue;
34  import org.apache.hadoop.hbase.KeyValue.KVComparator;
35  import org.apache.hadoop.hbase.KeyValueUtil;
36  import org.apache.hadoop.hbase.fs.HFileSystem;
37  import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
38  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder;
39  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
40  import org.apache.hadoop.hbase.io.encoding.HFileBlockDecodingContext;
41  import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
42  import org.apache.hadoop.hbase.util.ByteBufferUtils;
43  import org.apache.hadoop.hbase.util.Bytes;
44  import org.apache.hadoop.hbase.util.IdLock;
45  import org.apache.hadoop.io.WritableUtils;
46  import org.htrace.Trace;
47  import org.htrace.TraceScope;
48  
49  import com.google.common.annotations.VisibleForTesting;
50  
51  /**
52   * {@link HFile} reader for version 2.
53   */
54  @InterfaceAudience.Private
55  public class HFileReaderV2 extends AbstractHFileReader {
56  
57    private static final Log LOG = LogFactory.getLog(HFileReaderV2.class);
58  
59    /** Minor versions in HFile V2 starting with this number have hbase checksums */
60    public static final int MINOR_VERSION_WITH_CHECKSUM = 1;
61    /** In HFile V2 minor version that does not support checksums */
62    public static final int MINOR_VERSION_NO_CHECKSUM = 0;
63  
64    /** HFile minor version that introduced pbuf filetrailer */
65    public static final int PBUF_TRAILER_MINOR_VERSION = 2;
66  
67    /**
68     * The size of a (key length, value length) tuple that prefixes each entry in
69     * a data block.
70     */
71    public final static int KEY_VALUE_LEN_SIZE = 2 * Bytes.SIZEOF_INT;
72  
73    protected boolean includesMemstoreTS = false;
74    protected boolean decodeMemstoreTS = false;
75    protected boolean shouldIncludeMemstoreTS() {
76      return includesMemstoreTS;
77    }
78  
79    /** Filesystem-level block reader. */
80    protected HFileBlock.FSReader fsBlockReader;
81  
82    /**
83     * A "sparse lock" implementation allowing to lock on a particular block
84     * identified by offset. The purpose of this is to avoid two clients loading
85     * the same block, and have all but one client wait to get the block from the
86     * cache.
87     */
88    private IdLock offsetLock = new IdLock();
89  
90    /**
91     * Blocks read from the load-on-open section, excluding data root index, meta
92     * index, and file info.
93     */
94    private List<HFileBlock> loadOnOpenBlocks = new ArrayList<HFileBlock>();
95  
96    /** Minimum minor version supported by this HFile format */
97    static final int MIN_MINOR_VERSION = 0;
98  
99    /** Maximum minor version supported by this HFile format */
100   // We went to version 2 when we moved to pb'ing fileinfo and the trailer on
101   // the file. This version can read Writables version 1.
102   static final int MAX_MINOR_VERSION = 3;
103 
104   /** Minor versions starting with this number have faked index key */
105   static final int MINOR_VERSION_WITH_FAKED_KEY = 3;
106 
107   protected HFileContext hfileContext;
108 
109   /**
110    * Opens a HFile. You must load the index before you can use it by calling
111    * {@link #loadFileInfo()}.
112    *
113    * @param path Path to HFile.
114    * @param trailer File trailer.
115    * @param fsdis input stream.
116    * @param size Length of the stream.
117    * @param cacheConf Cache configuration.
118    * @param hfs
119    * @param conf
120    */
121   public HFileReaderV2(final Path path, final FixedFileTrailer trailer,
122       final FSDataInputStreamWrapper fsdis, final long size, final CacheConfig cacheConf,
123       final HFileSystem hfs, final Configuration conf) throws IOException {
124     super(path, trailer, size, cacheConf, hfs, conf);
125     this.conf = conf;
126     trailer.expectMajorVersion(getMajorVersion());
127     validateMinorVersion(path, trailer.getMinorVersion());
128     this.hfileContext = createHFileContext(fsdis, fileSize, hfs, path, trailer);
129     HFileBlock.FSReaderV2 fsBlockReaderV2 = new HFileBlock.FSReaderV2(fsdis, fileSize, hfs, path,
130         hfileContext);
131     this.fsBlockReader = fsBlockReaderV2; // upcast
132 
133     // Comparator class name is stored in the trailer in version 2.
134     comparator = trailer.createComparator();
135     dataBlockIndexReader = new HFileBlockIndex.BlockIndexReader(comparator,
136         trailer.getNumDataIndexLevels(), this);
137     metaBlockIndexReader = new HFileBlockIndex.BlockIndexReader(
138         KeyValue.RAW_COMPARATOR, 1);
139 
140     // Parse load-on-open data.
141 
142     HFileBlock.BlockIterator blockIter = fsBlockReaderV2.blockRange(
143         trailer.getLoadOnOpenDataOffset(),
144         fileSize - trailer.getTrailerSize());
145 
146     // Data index. We also read statistics about the block index written after
147     // the root level.
148     dataBlockIndexReader.readMultiLevelIndexRoot(
149         blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX),
150         trailer.getDataIndexCount());
151 
152     // Meta index.
153     metaBlockIndexReader.readRootIndex(
154         blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX),
155         trailer.getMetaIndexCount());
156 
157     // File info
158     fileInfo = new FileInfo();
159     fileInfo.read(blockIter.nextBlockWithBlockType(BlockType.FILE_INFO).getByteStream());
160     lastKey = fileInfo.get(FileInfo.LASTKEY);
161     avgKeyLen = Bytes.toInt(fileInfo.get(FileInfo.AVG_KEY_LEN));
162     avgValueLen = Bytes.toInt(fileInfo.get(FileInfo.AVG_VALUE_LEN));
163     byte [] keyValueFormatVersion =
164         fileInfo.get(HFileWriterV2.KEY_VALUE_VERSION);
165     includesMemstoreTS = keyValueFormatVersion != null &&
166         Bytes.toInt(keyValueFormatVersion) ==
167             HFileWriterV2.KEY_VALUE_VER_WITH_MEMSTORE;
168     fsBlockReaderV2.setIncludesMemstoreTS(includesMemstoreTS);
169     if (includesMemstoreTS) {
170       decodeMemstoreTS = Bytes.toLong(fileInfo.get(HFileWriterV2.MAX_MEMSTORE_TS_KEY)) > 0;
171     }
172 
173     // Read data block encoding algorithm name from file info.
174     dataBlockEncoder = HFileDataBlockEncoderImpl.createFromFileInfo(fileInfo);
175     fsBlockReaderV2.setDataBlockEncoder(dataBlockEncoder);
176 
177     // Store all other load-on-open blocks for further consumption.
178     HFileBlock b;
179     while ((b = blockIter.nextBlock()) != null) {
180       loadOnOpenBlocks.add(b);
181     }
182 
183     // Prefetch file blocks upon open if requested
184     if (cacheConf.shouldPrefetchOnOpen()) {
185       PrefetchExecutor.request(path, new Runnable() {
186         public void run() {
187           try {
188             long offset = 0;
189             long end = fileSize - getTrailer().getTrailerSize();
190             HFileBlock prevBlock = null;
191             while (offset < end) {
192               if (Thread.interrupted()) {
193                 break;
194               }
195               long onDiskSize = -1;
196               if (prevBlock != null) {
197                 onDiskSize = prevBlock.getNextBlockOnDiskSizeWithHeader();
198               }
199               HFileBlock block = readBlock(offset, onDiskSize, true, false, false, false,
200                 null, null);
201               prevBlock = block;
202               offset += block.getOnDiskSizeWithHeader();
203             }
204           } catch (IOException e) {
205             // IOExceptions are probably due to region closes (relocation, etc.)
206             if (LOG.isTraceEnabled()) {
207               LOG.trace("Exception encountered while prefetching " + path + ":", e);
208             }
209           } catch (Exception e) {
210             // Other exceptions are interesting
211             LOG.warn("Exception encountered while prefetching " + path + ":", e);
212           } finally {
213             PrefetchExecutor.complete(path);
214           }
215         }
216       });
217     }
218   }
219 
220   protected HFileContext createHFileContext(FSDataInputStreamWrapper fsdis, long fileSize,
221       HFileSystem hfs, Path path, FixedFileTrailer trailer) throws IOException {
222     return new HFileContextBuilder()
223       .withIncludesMvcc(this.includesMemstoreTS)
224       .withCompression(this.compressAlgo)
225       .withHBaseCheckSum(trailer.getMinorVersion() >= MINOR_VERSION_WITH_CHECKSUM)
226       .build();
227   }
228 
229   /**
230    * Create a Scanner on this file. No seeks or reads are done on creation. Call
231    * {@link HFileScanner#seekTo(byte[])} to position an start the read. There is
232    * nothing to clean up in a Scanner. Letting go of your references to the
233    * scanner is sufficient.
234    *
235    * @param cacheBlocks True if we should cache blocks read in by this scanner.
236    * @param pread Use positional read rather than seek+read if true (pread is
237    *          better for random reads, seek+read is better scanning).
238    * @param isCompaction is scanner being used for a compaction?
239    * @return Scanner on this file.
240    */
241    @Override
242    public HFileScanner getScanner(boolean cacheBlocks, final boolean pread,
243       final boolean isCompaction) {
244     if (dataBlockEncoder.useEncodedScanner()) {
245       return new EncodedScannerV2(this, cacheBlocks, pread, isCompaction,
246           hfileContext);
247     }
248 
249     return new ScannerV2(this, cacheBlocks, pread, isCompaction);
250   }
251 
252    private HFileBlock getCachedBlock(BlockCacheKey cacheKey, boolean cacheBlock, boolean useLock,
253        boolean isCompaction, boolean updateCacheMetrics, BlockType expectedBlockType,
254        DataBlockEncoding expectedDataBlockEncoding) throws IOException {
255      // Check cache for block. If found return.
256      if (cacheConf.isBlockCacheEnabled()) {
257        BlockCache cache = cacheConf.getBlockCache();
258        HFileBlock cachedBlock = (HFileBlock) cache.getBlock(cacheKey, cacheBlock, useLock,
259          updateCacheMetrics);
260        if (cachedBlock != null) {
261          validateBlockType(cachedBlock, expectedBlockType);
262 
263          if (expectedDataBlockEncoding == null) {
264            return cachedBlock;
265          }
266          DataBlockEncoding actualDataBlockEncoding =
267                  cachedBlock.getDataBlockEncoding();
268          // Block types other than data blocks always have
269          // DataBlockEncoding.NONE. To avoid false negative cache misses, only
270          // perform this check if cached block is a data block.
271          if (cachedBlock.getBlockType().isData() &&
272                  !actualDataBlockEncoding.equals(expectedDataBlockEncoding)) {
273            // This mismatch may happen if a ScannerV2, which is used for say a
274            // compaction, tries to read an encoded block from the block cache.
275            // The reverse might happen when an EncodedScannerV2 tries to read
276            // un-encoded blocks which were cached earlier.
277            //
278            // Because returning a data block with an implicit BlockType mismatch
279            // will cause the requesting scanner to throw a disk read should be
280            // forced here. This will potentially cause a significant number of
281            // cache misses, so update so we should keep track of this as it might
282            // justify the work on a CompoundScannerV2.
283            if (!expectedDataBlockEncoding.equals(DataBlockEncoding.NONE) &&
284                    !actualDataBlockEncoding.equals(DataBlockEncoding.NONE)) {
285              // If the block is encoded but the encoding does not match the
286              // expected encoding it is likely the encoding was changed but the
287              // block was not yet evicted. Evictions on file close happen async
288              // so blocks with the old encoding still linger in cache for some
289              // period of time. This event should be rare as it only happens on
290              // schema definition change.
291              LOG.info("Evicting cached block with key " + cacheKey +
292                      " because of a data block encoding mismatch" +
293                      "; expected: " + expectedDataBlockEncoding +
294                      ", actual: " + actualDataBlockEncoding);
295              cache.evictBlock(cacheKey);
296            }
297            return null;
298          }
299          return cachedBlock;
300        }
301      }
302      return null;
303    }
304   /**
305    * @param metaBlockName
306    * @param cacheBlock Add block to cache, if found
307    * @return block wrapped in a ByteBuffer, with header skipped
308    * @throws IOException
309    */
310   @Override
311   public ByteBuffer getMetaBlock(String metaBlockName, boolean cacheBlock)
312       throws IOException {
313     if (trailer.getMetaIndexCount() == 0) {
314       return null; // there are no meta blocks
315     }
316     if (metaBlockIndexReader == null) {
317       throw new IOException("Meta index not loaded");
318     }
319 
320     byte[] mbname = Bytes.toBytes(metaBlockName);
321     int block = metaBlockIndexReader.rootBlockContainingKey(mbname,
322         0, mbname.length);
323     if (block == -1)
324       return null;
325     long blockSize = metaBlockIndexReader.getRootBlockDataSize(block);
326 
327     // Per meta key from any given file, synchronize reads for said block. This
328     // is OK to do for meta blocks because the meta block index is always
329     // single-level.
330     synchronized (metaBlockIndexReader.getRootBlockKey(block)) {
331       // Check cache for block. If found return.
332       long metaBlockOffset = metaBlockIndexReader.getRootBlockOffset(block);
333       BlockCacheKey cacheKey = new BlockCacheKey(name, metaBlockOffset);
334 
335       cacheBlock &= cacheConf.shouldCacheDataOnRead();
336       if (cacheConf.isBlockCacheEnabled()) {
337         HFileBlock cachedBlock = getCachedBlock(cacheKey, cacheBlock, false, true, true,
338           BlockType.META, null);
339         if (cachedBlock != null) {
340           // Return a distinct 'shallow copy' of the block,
341           // so pos does not get messed by the scanner
342           return cachedBlock.getBufferWithoutHeader();
343         }
344         // Cache Miss, please load.
345       }
346 
347       HFileBlock metaBlock = fsBlockReader.readBlockData(metaBlockOffset,
348           blockSize, -1, true);
349 
350       // Cache the block
351       if (cacheBlock) {
352         cacheConf.getBlockCache().cacheBlock(cacheKey, metaBlock,
353             cacheConf.isInMemory(), this.cacheConf.isCacheDataInL1());
354       }
355 
356       return metaBlock.getBufferWithoutHeader();
357     }
358   }
359 
360   /**
361    * Read in a file block of the given {@link BlockType} and
362    * {@link DataBlockEncoding}.
363    * @param dataBlockOffset offset to read.
364    * @param onDiskBlockSize size of the block
365    * @param cacheBlock
366    * @param pread Use positional read instead of seek+read (positional is
367    *          better doing random reads whereas seek+read is better scanning).
368    * @param isCompaction is this block being read as part of a compaction
369    * @param expectedBlockType the block type we are expecting to read with this
370    *          read operation, or null to read whatever block type is available
371    *          and avoid checking (that might reduce caching efficiency of
372    *          encoded data blocks)
373    * @param expectedDataBlockEncoding the data block encoding the caller is
374    *          expecting data blocks to be in, or null to not perform this
375    *          check and return the block irrespective of the encoding. This
376    *          check only applies to data blocks and can be set to null when
377    *          the caller is expecting to read a non-data block and has set
378    *          expectedBlockType accordingly.
379    * @return Block wrapped in a ByteBuffer.
380    * @throws IOException
381    */
382   @Override
383   public HFileBlock readBlock(long dataBlockOffset, long onDiskBlockSize,
384       final boolean cacheBlock, boolean pread, final boolean isCompaction,
385       boolean updateCacheMetrics, BlockType expectedBlockType,
386       DataBlockEncoding expectedDataBlockEncoding)
387       throws IOException {
388     if (dataBlockIndexReader == null) {
389       throw new IOException("Block index not loaded");
390     }
391     if (dataBlockOffset < 0
392         || dataBlockOffset >= trailer.getLoadOnOpenDataOffset()) {
393       throw new IOException("Requested block is out of range: "
394           + dataBlockOffset + ", lastDataBlockOffset: "
395           + trailer.getLastDataBlockOffset());
396     }
397     // For any given block from any given file, synchronize reads for said
398     // block.
399     // Without a cache, this synchronizing is needless overhead, but really
400     // the other choice is to duplicate work (which the cache would prevent you
401     // from doing).
402 
403     BlockCacheKey cacheKey =
404         new BlockCacheKey(name, dataBlockOffset);
405 
406     boolean useLock = false;
407     IdLock.Entry lockEntry = null;
408     TraceScope traceScope = Trace.startSpan("HFileReaderV2.readBlock");
409     try {
410       while (true) {
411         if (useLock) {
412           lockEntry = offsetLock.getLockEntry(dataBlockOffset);
413         }
414 
415         // Check cache for block. If found return.
416         if (cacheConf.isBlockCacheEnabled()) {
417           // Try and get the block from the block cache. If the useLock variable is true then this
418           // is the second time through the loop and it should not be counted as a block cache miss.
419           HFileBlock cachedBlock = getCachedBlock(cacheKey, cacheBlock, useLock, isCompaction,
420             updateCacheMetrics, expectedBlockType, expectedDataBlockEncoding);
421           if (cachedBlock != null) {
422             validateBlockType(cachedBlock, expectedBlockType);
423             if (cachedBlock.getBlockType().isData()) {
424               if (updateCacheMetrics) {
425                 HFile.dataBlockReadCnt.incrementAndGet();
426               }
427               // Validate encoding type for data blocks. We include encoding
428               // type in the cache key, and we expect it to match on a cache hit.
429               if (cachedBlock.getDataBlockEncoding() != dataBlockEncoder.getDataBlockEncoding()) {
430                 throw new IOException("Cached block under key " + cacheKey + " "
431                   + "has wrong encoding: " + cachedBlock.getDataBlockEncoding() + " (expected: "
432                   + dataBlockEncoder.getDataBlockEncoding() + ")");
433               }
434             }
435             return cachedBlock;
436           }
437           // Carry on, please load.
438         }
439         if (!useLock) {
440           // check cache again with lock
441           useLock = true;
442           continue;
443         }
444         if (Trace.isTracing()) {
445           traceScope.getSpan().addTimelineAnnotation("blockCacheMiss");
446         }
447         // Load block from filesystem.
448         HFileBlock hfileBlock = fsBlockReader.readBlockData(dataBlockOffset, onDiskBlockSize, -1,
449             pread);
450         validateBlockType(hfileBlock, expectedBlockType);
451 
452         // Cache the block if necessary
453         if (cacheBlock && cacheConf.shouldCacheBlockOnRead(hfileBlock.getBlockType().getCategory())) {
454           cacheConf.getBlockCache().cacheBlock(cacheKey, hfileBlock, cacheConf.isInMemory(),
455             this.cacheConf.isCacheDataInL1());
456         }
457 
458         if (updateCacheMetrics && hfileBlock.getBlockType().isData()) {
459           HFile.dataBlockReadCnt.incrementAndGet();
460         }
461 
462         return hfileBlock;
463       }
464     } finally {
465       traceScope.close();
466       if (lockEntry != null) {
467         offsetLock.releaseLockEntry(lockEntry);
468       }
469     }
470   }
471 
472   @Override
473   public boolean hasMVCCInfo() {
474     return includesMemstoreTS && decodeMemstoreTS;
475   }
476 
477   /**
478    * Compares the actual type of a block retrieved from cache or disk with its
479    * expected type and throws an exception in case of a mismatch. Expected
480    * block type of {@link BlockType#DATA} is considered to match the actual
481    * block type [@link {@link BlockType#ENCODED_DATA} as well.
482    * @param block a block retrieved from cache or disk
483    * @param expectedBlockType the expected block type, or null to skip the
484    *          check
485    */
486   private void validateBlockType(HFileBlock block,
487       BlockType expectedBlockType) throws IOException {
488     if (expectedBlockType == null) {
489       return;
490     }
491     BlockType actualBlockType = block.getBlockType();
492     if (expectedBlockType.isData() && actualBlockType.isData()) {
493       // We consider DATA to match ENCODED_DATA for the purpose of this
494       // verification.
495       return;
496     }
497     if (actualBlockType != expectedBlockType) {
498       throw new IOException("Expected block type " + expectedBlockType + ", " +
499           "but got " + actualBlockType + ": " + block);
500     }
501   }
502 
503   /**
504    * @return Last key in the file. May be null if file has no entries. Note that
505    *         this is not the last row key, but rather the byte form of the last
506    *         KeyValue.
507    */
508   @Override
509   public byte[] getLastKey() {
510     return dataBlockIndexReader.isEmpty() ? null : lastKey;
511   }
512 
513   /**
514    * @return Midkey for this file. We work with block boundaries only so
515    *         returned midkey is an approximation only.
516    * @throws IOException
517    */
518   @Override
519   public byte[] midkey() throws IOException {
520     return dataBlockIndexReader.midkey();
521   }
522 
523   @Override
524   public void close() throws IOException {
525     close(cacheConf.shouldEvictOnClose());
526   }
527 
528   public void close(boolean evictOnClose) throws IOException {
529     PrefetchExecutor.cancel(path);
530     if (evictOnClose && cacheConf.isBlockCacheEnabled()) {
531       int numEvicted = cacheConf.getBlockCache().evictBlocksByHfileName(name);
532       if (LOG.isTraceEnabled()) {
533         LOG.trace("On close, file=" + name + " evicted=" + numEvicted
534           + " block(s)");
535       }
536     }
537     fsBlockReader.closeStreams();
538   }
539 
540   public DataBlockEncoding getEffectiveEncodingInCache(boolean isCompaction) {
541     return dataBlockEncoder.getEffectiveEncodingInCache(isCompaction);
542   }
543 
544   /** For testing */
545   @Override
546   HFileBlock.FSReader getUncachedBlockReader() {
547     return fsBlockReader;
548   }
549 
550 
551   protected abstract static class AbstractScannerV2
552       extends AbstractHFileReader.Scanner {
553     protected HFileBlock block;
554 
555     /**
556      * The next indexed key is to keep track of the indexed key of the next data block.
557      * If the nextIndexedKey is HConstants.NO_NEXT_INDEXED_KEY, it means that the
558      * current data block is the last data block.
559      *
560      * If the nextIndexedKey is null, it means the nextIndexedKey has not been loaded yet.
561      */
562     protected byte[] nextIndexedKey;
563 
564     public AbstractScannerV2(HFileReaderV2 r, boolean cacheBlocks,
565         final boolean pread, final boolean isCompaction) {
566       super(r, cacheBlocks, pread, isCompaction);
567     }
568 
569     protected abstract ByteBuffer getFirstKeyInBlock(HFileBlock curBlock);
570 
571     protected abstract int loadBlockAndSeekToKey(HFileBlock seekToBlock, byte[] nextIndexedKey,
572         boolean rewind, Cell key, boolean seekBefore) throws IOException;
573 
574     @Override
575     public int seekTo(byte[] key, int offset, int length) throws IOException {
576       // Always rewind to the first key of the block, because the given key
577       // might be before or after the current key.
578       return seekTo(new KeyValue.KeyOnlyKeyValue(key, offset, length));
579     }
580 
581     @Override
582     public int reseekTo(byte[] key, int offset, int length) throws IOException {
583       return reseekTo(new KeyValue.KeyOnlyKeyValue(key, offset, length));
584     }
585 
586     @Override
587     public int seekTo(Cell key) throws IOException {
588       return seekTo(key, true);
589     }
590 
591     @Override
592     public int reseekTo(Cell key) throws IOException {
593       int compared;
594       if (isSeeked()) {
595         compared = compareKey(reader.getComparator(), key);
596         if (compared < 1) {
597           // If the required key is less than or equal to current key, then
598           // don't do anything.
599           return compared;
600         } else {
601           // The comparison with no_next_index_key has to be checked
602           if (this.nextIndexedKey != null &&
603               (this.nextIndexedKey == HConstants.NO_NEXT_INDEXED_KEY || reader
604               .getComparator()
605                   .compareOnlyKeyPortion(key,
606                       new KeyValue.KeyOnlyKeyValue(nextIndexedKey, 0, 
607                           nextIndexedKey.length)) < 0)) {
608             // The reader shall continue to scan the current data block instead
609             // of querying the
610             // block index as long as it knows the target key is strictly
611             // smaller than
612             // the next indexed key or the current data block is the last data
613             // block.
614             return loadBlockAndSeekToKey(this.block, nextIndexedKey, false, key, false);
615           }
616         }
617       }
618       // Don't rewind on a reseek operation, because reseek implies that we are
619       // always going forward in the file.
620       return seekTo(key, false);
621     }
622 
623 
624     /**
625      * An internal API function. Seek to the given key, optionally rewinding to
626      * the first key of the block before doing the seek.
627      *
628      * @param key - a cell representing the key that we need to fetch
629      * @param rewind whether to rewind to the first key of the block before
630      *        doing the seek. If this is false, we are assuming we never go
631      *        back, otherwise the result is undefined.
632      * @return -1 if the key is earlier than the first key of the file,
633      *         0 if we are at the given key, 1 if we are past the given key
634      *         -2 if the key is earlier than the first key of the file while
635      *         using a faked index key
636      * @throws IOException
637      */
638     public int seekTo(Cell key, boolean rewind) throws IOException {
639       HFileBlockIndex.BlockIndexReader indexReader = reader.getDataBlockIndexReader();
640       BlockWithScanInfo blockWithScanInfo = indexReader.loadDataBlockWithScanInfo(key, block,
641           cacheBlocks, pread, isCompaction, getEffectiveDataBlockEncoding());
642       if (blockWithScanInfo == null || blockWithScanInfo.getHFileBlock() == null) {
643         // This happens if the key e.g. falls before the beginning of the file.
644         return -1;
645       }
646       return loadBlockAndSeekToKey(blockWithScanInfo.getHFileBlock(),
647           blockWithScanInfo.getNextIndexedKey(), rewind, key, false);
648     }
649 
650     @Override
651     public boolean seekBefore(byte[] key, int offset, int length) throws IOException {
652       return seekBefore(new KeyValue.KeyOnlyKeyValue(key, offset, length));
653     }
654 
655     @Override
656     public boolean seekBefore(Cell key) throws IOException {
657       HFileBlock seekToBlock = reader.getDataBlockIndexReader().seekToDataBlock(key, block,
658           cacheBlocks, pread, isCompaction,
659           ((HFileReaderV2) reader).getEffectiveEncodingInCache(isCompaction));
660       if (seekToBlock == null) {
661         return false;
662       }
663       ByteBuffer firstKey = getFirstKeyInBlock(seekToBlock);
664 
665       if (reader.getComparator()
666           .compareOnlyKeyPortion(
667               new KeyValue.KeyOnlyKeyValue(firstKey.array(), firstKey.arrayOffset(),
668                   firstKey.limit()), key) >= 0) {
669         long previousBlockOffset = seekToBlock.getPrevBlockOffset();
670         // The key we are interested in
671         if (previousBlockOffset == -1) {
672           // we have a 'problem', the key we want is the first of the file.
673           return false;
674         }
675 
676         // It is important that we compute and pass onDiskSize to the block
677         // reader so that it does not have to read the header separately to
678         // figure out the size.
679         seekToBlock = reader.readBlock(previousBlockOffset,
680             seekToBlock.getOffset() - previousBlockOffset, cacheBlocks,
681             pread, isCompaction, true, BlockType.DATA, getEffectiveDataBlockEncoding());
682         // TODO shortcut: seek forward in this block to the last key of the
683         // block.
684       }
685       byte[] firstKeyInCurrentBlock = Bytes.getBytes(firstKey);
686       loadBlockAndSeekToKey(seekToBlock, firstKeyInCurrentBlock, true, key, true);
687       return true;
688     }
689 
690     /**
691      * Scans blocks in the "scanned" section of the {@link HFile} until the next
692      * data block is found.
693      *
694      * @return the next block, or null if there are no more data blocks
695      * @throws IOException
696      */
697     protected HFileBlock readNextDataBlock() throws IOException {
698       long lastDataBlockOffset = reader.getTrailer().getLastDataBlockOffset();
699       if (block == null)
700         return null;
701 
702       HFileBlock curBlock = block;
703 
704       do {
705         if (curBlock.getOffset() >= lastDataBlockOffset)
706           return null;
707 
708         if (curBlock.getOffset() < 0) {
709           throw new IOException("Invalid block file offset: " + block);
710         }
711 
712         // We are reading the next block without block type validation, because
713         // it might turn out to be a non-data block.
714         curBlock = reader.readBlock(curBlock.getOffset()
715             + curBlock.getOnDiskSizeWithHeader(),
716             curBlock.getNextBlockOnDiskSizeWithHeader(), cacheBlocks, pread,
717             isCompaction, true, null, getEffectiveDataBlockEncoding());
718       } while (!curBlock.getBlockType().isData());
719 
720       return curBlock;
721     }
722     
723     public DataBlockEncoding getEffectiveDataBlockEncoding() {
724       return ((HFileReaderV2)reader).getEffectiveEncodingInCache(isCompaction);
725     }
726     /**
727      * Compare the given key against the current key
728      * @param comparator
729      * @param key
730      * @param offset
731      * @param length
732      * @return -1 is the passed key is smaller than the current key, 0 if equal and 1 if greater
733      */
734     public abstract int compareKey(KVComparator comparator, byte[] key, int offset,
735         int length);
736 
737     public abstract int compareKey(KVComparator comparator, Cell kv);
738   }
739 
740   /**
741    * Implementation of {@link HFileScanner} interface.
742    */
743   protected static class ScannerV2 extends AbstractScannerV2 {
744     private HFileReaderV2 reader;
745 
746     public ScannerV2(HFileReaderV2 r, boolean cacheBlocks,
747         final boolean pread, final boolean isCompaction) {
748       super(r, cacheBlocks, pread, isCompaction);
749       this.reader = r;
750     }
751 
752     @Override
753     public Cell getKeyValue() {
754       if (!isSeeked())
755         return null;
756 
757       KeyValue ret = new KeyValue(blockBuffer.array(), blockBuffer.arrayOffset()
758           + blockBuffer.position(), getCellBufSize());
759       if (this.reader.shouldIncludeMemstoreTS()) {
760         ret.setSequenceId(currMemstoreTS);
761       }
762       return ret;
763     }
764 
765     protected int getCellBufSize() {
766       return KEY_VALUE_LEN_SIZE + currKeyLen + currValueLen;
767     }
768 
769     @Override
770     public ByteBuffer getKey() {
771       assertSeeked();
772       return ByteBuffer.wrap(
773           blockBuffer.array(),
774           blockBuffer.arrayOffset() + blockBuffer.position()
775               + KEY_VALUE_LEN_SIZE, currKeyLen).slice();
776     }
777 
778     @Override
779     public int compareKey(KVComparator comparator, byte[] key, int offset, int length) {
780       return comparator.compareFlatKey(key, offset, length, blockBuffer.array(),
781           blockBuffer.arrayOffset() + blockBuffer.position() + KEY_VALUE_LEN_SIZE, currKeyLen);
782     }
783 
784     @Override
785     public ByteBuffer getValue() {
786       assertSeeked();
787       return ByteBuffer.wrap(
788           blockBuffer.array(),
789           blockBuffer.arrayOffset() + blockBuffer.position()
790               + KEY_VALUE_LEN_SIZE + currKeyLen, currValueLen).slice();
791     }
792 
793     protected void setNonSeekedState() {
794       block = null;
795       blockBuffer = null;
796       currKeyLen = 0;
797       currValueLen = 0;
798       currMemstoreTS = 0;
799       currMemstoreTSLen = 0;
800     }
801 
802     /**
803      * Go to the next key/value in the block section. Loads the next block if
804      * necessary. If successful, {@link #getKey()} and {@link #getValue()} can
805      * be called.
806      *
807      * @return true if successfully navigated to the next key/value
808      */
809     @Override
810     public boolean next() throws IOException {
811       assertSeeked();
812 
813       try {
814         blockBuffer.position(getNextCellStartPosition());
815       } catch (IllegalArgumentException e) {
816         LOG.error("Current pos = " + blockBuffer.position()
817             + "; currKeyLen = " + currKeyLen + "; currValLen = "
818             + currValueLen + "; block limit = " + blockBuffer.limit()
819             + "; HFile name = " + reader.getName()
820             + "; currBlock currBlockOffset = " + block.getOffset());
821         throw e;
822       }
823 
824       if (blockBuffer.remaining() <= 0) {
825         long lastDataBlockOffset =
826             reader.getTrailer().getLastDataBlockOffset();
827 
828         if (block.getOffset() >= lastDataBlockOffset) {
829           setNonSeekedState();
830           return false;
831         }
832 
833         // read the next block
834         HFileBlock nextBlock = readNextDataBlock();
835         if (nextBlock == null) {
836           setNonSeekedState();
837           return false;
838         }
839 
840         updateCurrBlock(nextBlock);
841         return true;
842       }
843 
844       // We are still in the same block.
845       readKeyValueLen();
846       return true;
847     }
848 
849     protected int getNextCellStartPosition() {
850       return blockBuffer.position() + KEY_VALUE_LEN_SIZE + currKeyLen + currValueLen
851           + currMemstoreTSLen;
852     }
853 
854     /**
855      * Positions this scanner at the start of the file.
856      *
857      * @return false if empty file; i.e. a call to next would return false and
858      *         the current key and value are undefined.
859      * @throws IOException
860      */
861     @Override
862     public boolean seekTo() throws IOException {
863       if (reader == null) {
864         return false;
865       }
866 
867       if (reader.getTrailer().getEntryCount() == 0) {
868         // No data blocks.
869         return false;
870       }
871 
872       long firstDataBlockOffset =
873           reader.getTrailer().getFirstDataBlockOffset();
874       if (block != null && block.getOffset() == firstDataBlockOffset) {
875         blockBuffer.rewind();
876         readKeyValueLen();
877         return true;
878       }
879 
880       block = reader.readBlock(firstDataBlockOffset, -1, cacheBlocks, pread,
881           isCompaction, true, BlockType.DATA, getEffectiveDataBlockEncoding());
882       if (block.getOffset() < 0) {
883         throw new IOException("Invalid block offset: " + block.getOffset());
884       }
885       updateCurrBlock(block);
886       return true;
887     }
888 
889     @Override
890     protected int loadBlockAndSeekToKey(HFileBlock seekToBlock, byte[] nextIndexedKey,
891         boolean rewind, Cell key, boolean seekBefore) throws IOException {
892       if (block == null || block.getOffset() != seekToBlock.getOffset()) {
893         updateCurrBlock(seekToBlock);
894       } else if (rewind) {
895         blockBuffer.rewind();
896       }
897 
898       // Update the nextIndexedKey
899       this.nextIndexedKey = nextIndexedKey;
900       return blockSeek(key, seekBefore);
901     }
902 
903     /**
904      * Updates the current block to be the given {@link HFileBlock}. Seeks to
905      * the the first key/value pair.
906      *
907      * @param newBlock the block to make current
908      */
909     protected void updateCurrBlock(HFileBlock newBlock) {
910       block = newBlock;
911 
912       // sanity check
913       if (block.getBlockType() != BlockType.DATA) {
914         throw new IllegalStateException("ScannerV2 works only on data " +
915             "blocks, got " + block.getBlockType() + "; " +
916             "fileName=" + reader.name + ", " +
917             "dataBlockEncoder=" + reader.dataBlockEncoder + ", " +
918             "isCompaction=" + isCompaction);
919       }
920 
921       blockBuffer = block.getBufferWithoutHeader();
922       readKeyValueLen();
923       blockFetches++;
924 
925       // Reset the next indexed key
926       this.nextIndexedKey = null;
927     }
928 
929     protected void readKeyValueLen() {
930       blockBuffer.mark();
931       currKeyLen = blockBuffer.getInt();
932       currValueLen = blockBuffer.getInt();
933       ByteBufferUtils.skip(blockBuffer, currKeyLen + currValueLen);
934       readMvccVersion();
935       if (currKeyLen < 0 || currValueLen < 0
936           || currKeyLen > blockBuffer.limit()
937           || currValueLen > blockBuffer.limit()) {
938         throw new IllegalStateException("Invalid currKeyLen " + currKeyLen
939             + " or currValueLen " + currValueLen + ". Block offset: "
940             + block.getOffset() + ", block length: " + blockBuffer.limit()
941             + ", position: " + blockBuffer.position() + " (without header).");
942       }
943       blockBuffer.reset();
944     }
945 
946     protected void readMvccVersion() {
947       if (this.reader.shouldIncludeMemstoreTS()) {
948         if (this.reader.decodeMemstoreTS) {
949           try {
950             currMemstoreTS = Bytes.readVLong(blockBuffer.array(), blockBuffer.arrayOffset()
951                 + blockBuffer.position());
952             currMemstoreTSLen = WritableUtils.getVIntSize(currMemstoreTS);
953           } catch (Exception e) {
954             throw new RuntimeException("Error reading memstore timestamp", e);
955           }
956         } else {
957           currMemstoreTS = 0;
958           currMemstoreTSLen = 1;
959         }
960       }
961     }
962 
963     /**
964      * Within a loaded block, seek looking for the last key that is smaller than
965      * (or equal to?) the key we are interested in.
966      *
967      * A note on the seekBefore: if you have seekBefore = true, AND the first
968      * key in the block = key, then you'll get thrown exceptions. The caller has
969      * to check for that case and load the previous block as appropriate.
970      *
971      * @param key
972      *          the key to find
973      * @param seekBefore
974      *          find the key before the given key in case of exact match.
975      * @return 0 in case of an exact key match, 1 in case of an inexact match,
976      *         -2 in case of an inexact match and furthermore, the input key
977      *         less than the first key of current block(e.g. using a faked index
978      *         key)
979      */
980     protected int blockSeek(Cell key, boolean seekBefore) {
981       int klen, vlen;
982       long memstoreTS = 0;
983       int memstoreTSLen = 0;
984       int lastKeyValueSize = -1;
985       KeyValue.KeyOnlyKeyValue keyOnlykv = new KeyValue.KeyOnlyKeyValue();
986       do {
987         blockBuffer.mark();
988         klen = blockBuffer.getInt();
989         vlen = blockBuffer.getInt();
990         blockBuffer.reset();
991         if (this.reader.shouldIncludeMemstoreTS()) {
992           if (this.reader.decodeMemstoreTS) {
993             try {
994               int memstoreTSOffset = blockBuffer.arrayOffset() + blockBuffer.position()
995                   + KEY_VALUE_LEN_SIZE + klen + vlen;
996               memstoreTS = Bytes.readVLong(blockBuffer.array(), memstoreTSOffset);
997               memstoreTSLen = WritableUtils.getVIntSize(memstoreTS);
998             } catch (Exception e) {
999               throw new RuntimeException("Error reading memstore timestamp", e);
1000             }
1001           } else {
1002             memstoreTS = 0;
1003             memstoreTSLen = 1;
1004           }
1005         }
1006 
1007         int keyOffset = blockBuffer.arrayOffset() + blockBuffer.position() + KEY_VALUE_LEN_SIZE;
1008         keyOnlykv.setKey(blockBuffer.array(), keyOffset, klen);
1009         int comp = reader.getComparator().compareOnlyKeyPortion(key, keyOnlykv);
1010 
1011         if (comp == 0) {
1012           if (seekBefore) {
1013             if (lastKeyValueSize < 0) {
1014               KeyValue kv = KeyValueUtil.ensureKeyValue(key);
1015               throw new IllegalStateException("blockSeek with seekBefore "
1016                   + "at the first key of the block: key="
1017                   + Bytes.toStringBinary(kv.getKey(), kv.getKeyOffset(), kv.getKeyLength())
1018                   + ", blockOffset=" + block.getOffset() + ", onDiskSize="
1019                   + block.getOnDiskSizeWithHeader());
1020             }
1021             blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
1022             readKeyValueLen();
1023             return 1; // non exact match.
1024           }
1025           currKeyLen = klen;
1026           currValueLen = vlen;
1027           if (this.reader.shouldIncludeMemstoreTS()) {
1028             currMemstoreTS = memstoreTS;
1029             currMemstoreTSLen = memstoreTSLen;
1030           }
1031           return 0; // indicate exact match
1032         } else if (comp < 0) {
1033           if (lastKeyValueSize > 0)
1034             blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
1035           readKeyValueLen();
1036           if (lastKeyValueSize == -1 && blockBuffer.position() == 0
1037               && this.reader.trailer.getMinorVersion() >= MINOR_VERSION_WITH_FAKED_KEY) {
1038             return HConstants.INDEX_KEY_MAGIC;
1039           }
1040           return 1;
1041         }
1042 
1043         // The size of this key/value tuple, including key/value length fields.
1044         lastKeyValueSize = klen + vlen + memstoreTSLen + KEY_VALUE_LEN_SIZE;
1045         blockBuffer.position(blockBuffer.position() + lastKeyValueSize);
1046       } while (blockBuffer.remaining() > 0);
1047 
1048       // Seek to the last key we successfully read. This will happen if this is
1049       // the last key/value pair in the file, in which case the following call
1050       // to next() has to return false.
1051       blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
1052       readKeyValueLen();
1053       return 1; // didn't exactly find it.
1054     }
1055 
1056     @Override
1057     protected ByteBuffer getFirstKeyInBlock(HFileBlock curBlock) {
1058       ByteBuffer buffer = curBlock.getBufferWithoutHeader();
1059       // It is safe to manipulate this buffer because we own the buffer object.
1060       buffer.rewind();
1061       int klen = buffer.getInt();
1062       buffer.getInt();
1063       ByteBuffer keyBuff = buffer.slice();
1064       keyBuff.limit(klen);
1065       keyBuff.rewind();
1066       return keyBuff;
1067     }
1068 
1069     @Override
1070     public String getKeyString() {
1071       return Bytes.toStringBinary(blockBuffer.array(),
1072           blockBuffer.arrayOffset() + blockBuffer.position()
1073               + KEY_VALUE_LEN_SIZE, currKeyLen);
1074     }
1075 
1076     @Override
1077     public String getValueString() {
1078       return Bytes.toString(blockBuffer.array(), blockBuffer.arrayOffset()
1079           + blockBuffer.position() + KEY_VALUE_LEN_SIZE + currKeyLen,
1080           currValueLen);
1081     }
1082 
1083     @Override
1084     public int compareKey(KVComparator comparator, Cell key) {
1085       return comparator.compareOnlyKeyPortion(
1086           key,
1087           new KeyValue.KeyOnlyKeyValue(blockBuffer.array(), blockBuffer.arrayOffset()
1088               + blockBuffer.position() + KEY_VALUE_LEN_SIZE, currKeyLen));
1089     }
1090   }
1091 
1092   /**
1093    * ScannerV2 that operates on encoded data blocks.
1094    */
1095   protected static class EncodedScannerV2 extends AbstractScannerV2 {
1096     private final HFileBlockDecodingContext decodingCtx;
1097     private final DataBlockEncoder.EncodedSeeker seeker;
1098     private final DataBlockEncoder dataBlockEncoder;
1099     protected final HFileContext meta;
1100 
1101     public EncodedScannerV2(HFileReaderV2 reader, boolean cacheBlocks,
1102         boolean pread, boolean isCompaction, HFileContext meta) {
1103       super(reader, cacheBlocks, pread, isCompaction);
1104       DataBlockEncoding encoding = reader.dataBlockEncoder.getDataBlockEncoding();
1105       dataBlockEncoder = encoding.getEncoder();
1106       decodingCtx = dataBlockEncoder.newDataBlockDecodingContext(meta);
1107       seeker = dataBlockEncoder.createSeeker(
1108         reader.getComparator(), decodingCtx);
1109       this.meta = meta;
1110     }
1111 
1112     @Override
1113     public boolean isSeeked(){
1114       return this.block != null;
1115     }
1116 
1117     /**
1118      * Updates the current block to be the given {@link HFileBlock}. Seeks to
1119      * the the first key/value pair.
1120      *
1121      * @param newBlock the block to make current
1122      * @throws CorruptHFileException
1123      */
1124     private void updateCurrentBlock(HFileBlock newBlock) throws CorruptHFileException {
1125       block = newBlock;
1126 
1127       // sanity checks
1128       if (block.getBlockType() != BlockType.ENCODED_DATA) {
1129         throw new IllegalStateException(
1130             "EncodedScanner works only on encoded data blocks");
1131       }
1132       short dataBlockEncoderId = block.getDataBlockEncodingId();
1133       if (!DataBlockEncoding.isCorrectEncoder(dataBlockEncoder, dataBlockEncoderId)) {
1134         String encoderCls = dataBlockEncoder.getClass().getName();
1135         throw new CorruptHFileException("Encoder " + encoderCls
1136           + " doesn't support data block encoding "
1137           + DataBlockEncoding.getNameFromId(dataBlockEncoderId));
1138       }
1139 
1140       seeker.setCurrentBuffer(getEncodedBuffer(newBlock));
1141       blockFetches++;
1142 
1143       // Reset the next indexed key
1144       this.nextIndexedKey = null;
1145     }
1146 
1147     private ByteBuffer getEncodedBuffer(HFileBlock newBlock) {
1148       ByteBuffer origBlock = newBlock.getBufferReadOnly();
1149       ByteBuffer encodedBlock = ByteBuffer.wrap(origBlock.array(),
1150           origBlock.arrayOffset() + newBlock.headerSize() +
1151           DataBlockEncoding.ID_SIZE,
1152           newBlock.getUncompressedSizeWithoutHeader() -
1153           DataBlockEncoding.ID_SIZE).slice();
1154       return encodedBlock;
1155     }
1156 
1157     @Override
1158     public boolean seekTo() throws IOException {
1159       if (reader == null) {
1160         return false;
1161       }
1162 
1163       if (reader.getTrailer().getEntryCount() == 0) {
1164         // No data blocks.
1165         return false;
1166       }
1167 
1168       long firstDataBlockOffset =
1169           reader.getTrailer().getFirstDataBlockOffset();
1170       if (block != null && block.getOffset() == firstDataBlockOffset) {
1171         seeker.rewind();
1172         return true;
1173       }
1174 
1175       block = reader.readBlock(firstDataBlockOffset, -1, cacheBlocks, pread,
1176           isCompaction, true, BlockType.DATA, getEffectiveDataBlockEncoding());
1177       if (block.getOffset() < 0) {
1178         throw new IOException("Invalid block offset: " + block.getOffset());
1179       }
1180       updateCurrentBlock(block);
1181       return true;
1182     }
1183 
1184     @Override
1185     public boolean next() throws IOException {
1186       boolean isValid = seeker.next();
1187       if (!isValid) {
1188         block = readNextDataBlock();
1189         isValid = block != null;
1190         if (isValid) {
1191           updateCurrentBlock(block);
1192         }
1193       }
1194       return isValid;
1195     }
1196 
1197     @Override
1198     public ByteBuffer getKey() {
1199       assertValidSeek();
1200       return seeker.getKeyDeepCopy();
1201     }
1202 
1203     @Override
1204     public int compareKey(KVComparator comparator, byte[] key, int offset, int length) {
1205       return seeker.compareKey(comparator, key, offset, length);
1206     }
1207 
1208     @Override
1209     public ByteBuffer getValue() {
1210       assertValidSeek();
1211       return seeker.getValueShallowCopy();
1212     }
1213 
1214     @Override
1215     public Cell getKeyValue() {
1216       if (block == null) {
1217         return null;
1218       }
1219       return seeker.getKeyValue();
1220     }
1221 
1222     @Override
1223     public String getKeyString() {
1224       ByteBuffer keyBuffer = getKey();
1225       return Bytes.toStringBinary(keyBuffer.array(),
1226           keyBuffer.arrayOffset(), keyBuffer.limit());
1227     }
1228 
1229     @Override
1230     public String getValueString() {
1231       ByteBuffer valueBuffer = getValue();
1232       return Bytes.toStringBinary(valueBuffer.array(),
1233           valueBuffer.arrayOffset(), valueBuffer.limit());
1234     }
1235 
1236     private void assertValidSeek() {
1237       if (block == null) {
1238         throw new NotSeekedException();
1239       }
1240     }
1241 
1242     @Override
1243     protected ByteBuffer getFirstKeyInBlock(HFileBlock curBlock) {
1244       return dataBlockEncoder.getFirstKeyInBlock(getEncodedBuffer(curBlock));
1245     }
1246 
1247     @Override
1248     protected int loadBlockAndSeekToKey(HFileBlock seekToBlock, byte[] nextIndexedKey,
1249         boolean rewind, Cell key, boolean seekBefore) throws IOException {
1250       if (block == null || block.getOffset() != seekToBlock.getOffset()) {
1251         updateCurrentBlock(seekToBlock);
1252       } else if (rewind) {
1253         seeker.rewind();
1254       }
1255       this.nextIndexedKey = nextIndexedKey;
1256       return seeker.seekToKeyInBlock(key, seekBefore);
1257     }
1258 
1259     @Override
1260     public int compareKey(KVComparator comparator, Cell key) {
1261       return seeker.compareKey(comparator, key);
1262     }
1263   }
1264 
1265   /**
1266    * Returns a buffer with the Bloom filter metadata. The caller takes
1267    * ownership of the buffer.
1268    */
1269   @Override
1270   public DataInput getGeneralBloomFilterMetadata() throws IOException {
1271     return this.getBloomFilterMetadata(BlockType.GENERAL_BLOOM_META);
1272   }
1273 
1274   @Override
1275   public DataInput getDeleteBloomFilterMetadata() throws IOException {
1276     return this.getBloomFilterMetadata(BlockType.DELETE_FAMILY_BLOOM_META);
1277   }
1278 
1279   private DataInput getBloomFilterMetadata(BlockType blockType)
1280   throws IOException {
1281     if (blockType != BlockType.GENERAL_BLOOM_META &&
1282         blockType != BlockType.DELETE_FAMILY_BLOOM_META) {
1283       throw new RuntimeException("Block Type: " + blockType.toString() +
1284           " is not supported") ;
1285     }
1286 
1287     for (HFileBlock b : loadOnOpenBlocks)
1288       if (b.getBlockType() == blockType)
1289         return b.getByteStream();
1290     return null;
1291   }
1292 
1293   @Override
1294   public boolean isFileInfoLoaded() {
1295     return true; // We load file info in constructor in version 2.
1296   }
1297 
1298   /**
1299    * Validates that the minor version is within acceptable limits.
1300    * Otherwise throws an Runtime exception
1301    */
1302   private void validateMinorVersion(Path path, int minorVersion) {
1303     if (minorVersion < MIN_MINOR_VERSION ||
1304         minorVersion > MAX_MINOR_VERSION) {
1305       String msg = "Minor version for path " + path + 
1306                    " is expected to be between " +
1307                    MIN_MINOR_VERSION + " and " + MAX_MINOR_VERSION +
1308                    " but is found to be " + minorVersion;
1309       LOG.error(msg);
1310       throw new RuntimeException(msg);
1311     }
1312   }
1313 
1314   @Override
1315   public int getMajorVersion() {
1316     return 2;
1317   }
1318 
1319   @Override
1320   public HFileContext getFileContext() {
1321     return hfileContext;
1322   }
1323 
1324   /**
1325    * Returns false if block prefetching was requested for this file and has
1326    * not completed, true otherwise
1327    */
1328   @VisibleForTesting
1329   boolean prefetchComplete() {
1330     return PrefetchExecutor.isCompleted(path);
1331   }
1332 }