View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.io.hfile;
19  
20  import java.io.DataInput;
21  import java.io.IOException;
22  import java.nio.ByteBuffer;
23  import java.util.ArrayList;
24  import java.util.List;
25  
26  import org.apache.commons.logging.Log;
27  import org.apache.commons.logging.LogFactory;
28  import org.apache.hadoop.hbase.classification.InterfaceAudience;
29  import org.apache.hadoop.conf.Configuration;
30  import org.apache.hadoop.fs.Path;
31  import org.apache.hadoop.hbase.Cell;
32  import org.apache.hadoop.hbase.CellUtil;
33  import org.apache.hadoop.hbase.HConstants;
34  import org.apache.hadoop.hbase.KeyValue;
35  import org.apache.hadoop.hbase.KeyValue.KVComparator;
36  import org.apache.hadoop.hbase.fs.HFileSystem;
37  import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
38  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder;
39  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
40  import org.apache.hadoop.hbase.io.encoding.HFileBlockDecodingContext;
41  import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
42  import org.apache.hadoop.hbase.util.ByteBufferUtils;
43  import org.apache.hadoop.hbase.util.Bytes;
44  import org.apache.hadoop.hbase.util.IdLock;
45  import org.apache.hadoop.io.WritableUtils;
46  import org.apache.htrace.Trace;
47  import org.apache.htrace.TraceScope;
48  
49  import com.google.common.annotations.VisibleForTesting;
50  
51  /**
52   * {@link HFile} reader for version 2.
53   */
54  @InterfaceAudience.Private
55  public class HFileReaderV2 extends AbstractHFileReader {
56  
57    private static final Log LOG = LogFactory.getLog(HFileReaderV2.class);
58  
59    /** Minor versions in HFile V2 starting with this number have hbase checksums */
60    public static final int MINOR_VERSION_WITH_CHECKSUM = 1;
61    /** In HFile V2 minor version that does not support checksums */
62    public static final int MINOR_VERSION_NO_CHECKSUM = 0;
63  
64    /** HFile minor version that introduced pbuf filetrailer */
65    public static final int PBUF_TRAILER_MINOR_VERSION = 2;
66  
67    /**
68     * The size of a (key length, value length) tuple that prefixes each entry in
69     * a data block.
70     */
71    public final static int KEY_VALUE_LEN_SIZE = 2 * Bytes.SIZEOF_INT;
72  
73    protected boolean includesMemstoreTS = false;
74    protected boolean decodeMemstoreTS = false;
75    protected boolean shouldIncludeMemstoreTS() {
76      return includesMemstoreTS;
77    }
78  
79    /** Filesystem-level block reader. */
80    protected HFileBlock.FSReader fsBlockReader;
81  
82    /**
83     * A "sparse lock" implementation allowing to lock on a particular block
84     * identified by offset. The purpose of this is to avoid two clients loading
85     * the same block, and have all but one client wait to get the block from the
86     * cache.
87     */
88    private IdLock offsetLock = new IdLock();
89  
90    /**
91     * Blocks read from the load-on-open section, excluding data root index, meta
92     * index, and file info.
93     */
94    private List<HFileBlock> loadOnOpenBlocks = new ArrayList<HFileBlock>();
95  
96    /** Minimum minor version supported by this HFile format */
97    static final int MIN_MINOR_VERSION = 0;
98  
99    /** Maximum minor version supported by this HFile format */
100   // We went to version 2 when we moved to pb'ing fileinfo and the trailer on
101   // the file. This version can read Writables version 1.
102   static final int MAX_MINOR_VERSION = 3;
103 
104   /** Minor versions starting with this number have faked index key */
105   static final int MINOR_VERSION_WITH_FAKED_KEY = 3;
106 
107   protected HFileContext hfileContext;
108 
109   /**
110    * Opens a HFile. You must load the index before you can use it by calling
111    * {@link #loadFileInfo()}.
112    *
113    * @param path Path to HFile.
114    * @param trailer File trailer.
115    * @param fsdis input stream.
116    * @param size Length of the stream.
117    * @param cacheConf Cache configuration.
118    * @param hfs
119    * @param conf
120    */
121   public HFileReaderV2(final Path path, final FixedFileTrailer trailer,
122       final FSDataInputStreamWrapper fsdis, final long size, final CacheConfig cacheConf,
123       final HFileSystem hfs, final Configuration conf) throws IOException {
124     super(path, trailer, size, cacheConf, hfs, conf);
125     this.conf = conf;
126     trailer.expectMajorVersion(getMajorVersion());
127     validateMinorVersion(path, trailer.getMinorVersion());
128     this.hfileContext = createHFileContext(fsdis, fileSize, hfs, path, trailer);
129     HFileBlock.FSReaderImpl fsBlockReaderV2 =
130       new HFileBlock.FSReaderImpl(fsdis, fileSize, hfs, path, hfileContext);
131     this.fsBlockReader = fsBlockReaderV2; // upcast
132 
133     // Comparator class name is stored in the trailer in version 2.
134     comparator = trailer.createComparator();
135     dataBlockIndexReader = new HFileBlockIndex.BlockIndexReader(comparator,
136         trailer.getNumDataIndexLevels(), this);
137     metaBlockIndexReader = new HFileBlockIndex.BlockIndexReader(
138         KeyValue.RAW_COMPARATOR, 1);
139 
140     // Parse load-on-open data.
141 
142     HFileBlock.BlockIterator blockIter = fsBlockReaderV2.blockRange(
143         trailer.getLoadOnOpenDataOffset(),
144         fileSize - trailer.getTrailerSize());
145 
146     // Data index. We also read statistics about the block index written after
147     // the root level.
148     dataBlockIndexReader.readMultiLevelIndexRoot(
149         blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX),
150         trailer.getDataIndexCount());
151 
152     // Meta index.
153     metaBlockIndexReader.readRootIndex(
154         blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX),
155         trailer.getMetaIndexCount());
156 
157     // File info
158     fileInfo = new FileInfo();
159     fileInfo.read(blockIter.nextBlockWithBlockType(BlockType.FILE_INFO).getByteStream());
160     byte[] creationTimeBytes = fileInfo.get(FileInfo.CREATE_TIME_TS);
161     this.hfileContext.setFileCreateTime(creationTimeBytes == null ? 0 : Bytes.toLong(creationTimeBytes));
162     lastKey = fileInfo.get(FileInfo.LASTKEY);
163     avgKeyLen = Bytes.toInt(fileInfo.get(FileInfo.AVG_KEY_LEN));
164     avgValueLen = Bytes.toInt(fileInfo.get(FileInfo.AVG_VALUE_LEN));
165     byte [] keyValueFormatVersion =
166         fileInfo.get(HFileWriterV2.KEY_VALUE_VERSION);
167     includesMemstoreTS = keyValueFormatVersion != null &&
168         Bytes.toInt(keyValueFormatVersion) ==
169             HFileWriterV2.KEY_VALUE_VER_WITH_MEMSTORE;
170     fsBlockReaderV2.setIncludesMemstoreTS(includesMemstoreTS);
171     if (includesMemstoreTS) {
172       decodeMemstoreTS = Bytes.toLong(fileInfo.get(HFileWriterV2.MAX_MEMSTORE_TS_KEY)) > 0;
173     }
174 
175     // Read data block encoding algorithm name from file info.
176     dataBlockEncoder = HFileDataBlockEncoderImpl.createFromFileInfo(fileInfo);
177     fsBlockReaderV2.setDataBlockEncoder(dataBlockEncoder);
178 
179     // Store all other load-on-open blocks for further consumption.
180     HFileBlock b;
181     while ((b = blockIter.nextBlock()) != null) {
182       loadOnOpenBlocks.add(b);
183     }
184 
185     // Prefetch file blocks upon open if requested
186     if (cacheConf.shouldPrefetchOnOpen()) {
187       PrefetchExecutor.request(path, new Runnable() {
188         public void run() {
189           try {
190             long offset = 0;
191             long end = fileSize - getTrailer().getTrailerSize();
192             HFileBlock prevBlock = null;
193             while (offset < end) {
194               if (Thread.interrupted()) {
195                 break;
196               }
197               long onDiskSize = -1;
198               if (prevBlock != null) {
199                 onDiskSize = prevBlock.getNextBlockOnDiskSizeWithHeader();
200               }
201               HFileBlock block = readBlock(offset, onDiskSize, true, false, false, false,
202                 null, null);
203               prevBlock = block;
204               offset += block.getOnDiskSizeWithHeader();
205             }
206           } catch (IOException e) {
207             // IOExceptions are probably due to region closes (relocation, etc.)
208             if (LOG.isTraceEnabled()) {
209               LOG.trace("Exception encountered while prefetching " + path + ":", e);
210             }
211           } catch (Exception e) {
212             // Other exceptions are interesting
213             LOG.warn("Exception encountered while prefetching " + path + ":", e);
214           } finally {
215             PrefetchExecutor.complete(path);
216           }
217         }
218       });
219     }
220   }
221 
222   protected HFileContext createHFileContext(FSDataInputStreamWrapper fsdis, long fileSize,
223       HFileSystem hfs, Path path, FixedFileTrailer trailer) throws IOException {
224     return new HFileContextBuilder()
225       .withIncludesMvcc(this.includesMemstoreTS)
226       .withCompression(this.compressAlgo)
227       .withHBaseCheckSum(trailer.getMinorVersion() >= MINOR_VERSION_WITH_CHECKSUM)
228       .build();
229   }
230 
231   /**
232    * Create a Scanner on this file. No seeks or reads are done on creation. Call
233    * {@link HFileScanner#seekTo(byte[])} to position an start the read. There is
234    * nothing to clean up in a Scanner. Letting go of your references to the
235    * scanner is sufficient.
236    *
237    * @param cacheBlocks True if we should cache blocks read in by this scanner.
238    * @param pread Use positional read rather than seek+read if true (pread is
239    *          better for random reads, seek+read is better scanning).
240    * @param isCompaction is scanner being used for a compaction?
241    * @return Scanner on this file.
242    */
243    @Override
244    public HFileScanner getScanner(boolean cacheBlocks, final boolean pread,
245       final boolean isCompaction) {
246     if (dataBlockEncoder.useEncodedScanner()) {
247       return new EncodedScannerV2(this, cacheBlocks, pread, isCompaction,
248           hfileContext);
249     }
250 
251     return new ScannerV2(this, cacheBlocks, pread, isCompaction);
252   }
253 
254   /**
255    * Retrieve block from cache. Validates the retrieved block's type vs {@code expectedBlockType}
256    * and its encoding vs. {@code expectedDataBlockEncoding}. Unpacks the block as necessary.
257    */
258    private HFileBlock getCachedBlock(BlockCacheKey cacheKey, boolean cacheBlock, boolean useLock,
259        boolean isCompaction, boolean updateCacheMetrics, BlockType expectedBlockType,
260        DataBlockEncoding expectedDataBlockEncoding) throws IOException {
261      // Check cache for block. If found return.
262      if (cacheConf.isBlockCacheEnabled()) {
263        BlockCache cache = cacheConf.getBlockCache();
264        HFileBlock cachedBlock = (HFileBlock) cache.getBlock(cacheKey, cacheBlock, useLock,
265          updateCacheMetrics);
266        if (cachedBlock != null) {
267          if (cacheConf.shouldCacheCompressed(cachedBlock.getBlockType().getCategory())) {
268            cachedBlock = cachedBlock.unpack(hfileContext, fsBlockReader);
269          }
270          validateBlockType(cachedBlock, expectedBlockType);
271 
272          if (expectedDataBlockEncoding == null) {
273            return cachedBlock;
274          }
275          DataBlockEncoding actualDataBlockEncoding =
276                  cachedBlock.getDataBlockEncoding();
277          // Block types other than data blocks always have
278          // DataBlockEncoding.NONE. To avoid false negative cache misses, only
279          // perform this check if cached block is a data block.
280          if (cachedBlock.getBlockType().isData() &&
281                  !actualDataBlockEncoding.equals(expectedDataBlockEncoding)) {
282            // This mismatch may happen if a ScannerV2, which is used for say a
283            // compaction, tries to read an encoded block from the block cache.
284            // The reverse might happen when an EncodedScannerV2 tries to read
285            // un-encoded blocks which were cached earlier.
286            //
287            // Because returning a data block with an implicit BlockType mismatch
288            // will cause the requesting scanner to throw a disk read should be
289            // forced here. This will potentially cause a significant number of
290            // cache misses, so update so we should keep track of this as it might
291            // justify the work on a CompoundScannerV2.
292            if (!expectedDataBlockEncoding.equals(DataBlockEncoding.NONE) &&
293                    !actualDataBlockEncoding.equals(DataBlockEncoding.NONE)) {
294              // If the block is encoded but the encoding does not match the
295              // expected encoding it is likely the encoding was changed but the
296              // block was not yet evicted. Evictions on file close happen async
297              // so blocks with the old encoding still linger in cache for some
298              // period of time. This event should be rare as it only happens on
299              // schema definition change.
300              LOG.info("Evicting cached block with key " + cacheKey +
301                      " because of a data block encoding mismatch" +
302                      "; expected: " + expectedDataBlockEncoding +
303                      ", actual: " + actualDataBlockEncoding);
304              cache.evictBlock(cacheKey);
305            }
306            return null;
307          }
308          return cachedBlock;
309        }
310      }
311      return null;
312    }
313   /**
314    * @param metaBlockName
315    * @param cacheBlock Add block to cache, if found
316    * @return block wrapped in a ByteBuffer, with header skipped
317    * @throws IOException
318    */
319   @Override
320   public ByteBuffer getMetaBlock(String metaBlockName, boolean cacheBlock)
321       throws IOException {
322     if (trailer.getMetaIndexCount() == 0) {
323       return null; // there are no meta blocks
324     }
325     if (metaBlockIndexReader == null) {
326       throw new IOException("Meta index not loaded");
327     }
328 
329     byte[] mbname = Bytes.toBytes(metaBlockName);
330     int block = metaBlockIndexReader.rootBlockContainingKey(mbname,
331         0, mbname.length);
332     if (block == -1)
333       return null;
334     long blockSize = metaBlockIndexReader.getRootBlockDataSize(block);
335 
336     // Per meta key from any given file, synchronize reads for said block. This
337     // is OK to do for meta blocks because the meta block index is always
338     // single-level.
339     synchronized (metaBlockIndexReader.getRootBlockKey(block)) {
340       // Check cache for block. If found return.
341       long metaBlockOffset = metaBlockIndexReader.getRootBlockOffset(block);
342       BlockCacheKey cacheKey = new BlockCacheKey(name, metaBlockOffset);
343 
344       cacheBlock &= cacheConf.shouldCacheDataOnRead();
345       if (cacheConf.isBlockCacheEnabled()) {
346         HFileBlock cachedBlock = getCachedBlock(cacheKey, cacheBlock, false, true, true,
347           BlockType.META, null);
348         if (cachedBlock != null) {
349           assert cachedBlock.isUnpacked() : "Packed block leak.";
350           // Return a distinct 'shallow copy' of the block,
351           // so pos does not get messed by the scanner
352           return cachedBlock.getBufferWithoutHeader();
353         }
354         // Cache Miss, please load.
355       }
356 
357       HFileBlock metaBlock = fsBlockReader.readBlockData(metaBlockOffset,
358           blockSize, -1, true).unpack(hfileContext, fsBlockReader);
359 
360       // Cache the block
361       if (cacheBlock) {
362         cacheConf.getBlockCache().cacheBlock(cacheKey, metaBlock,
363             cacheConf.isInMemory(), this.cacheConf.isCacheDataInL1());
364       }
365 
366       return metaBlock.getBufferWithoutHeader();
367     }
368   }
369 
370   @Override
371   public HFileBlock readBlock(long dataBlockOffset, long onDiskBlockSize,
372       final boolean cacheBlock, boolean pread, final boolean isCompaction,
373       boolean updateCacheMetrics, BlockType expectedBlockType,
374       DataBlockEncoding expectedDataBlockEncoding)
375       throws IOException {
376     if (dataBlockIndexReader == null) {
377       throw new IOException("Block index not loaded");
378     }
379     if (dataBlockOffset < 0 || dataBlockOffset >= trailer.getLoadOnOpenDataOffset()) {
380       throw new IOException("Requested block is out of range: " + dataBlockOffset +
381         ", lastDataBlockOffset: " + trailer.getLastDataBlockOffset());
382     }
383 
384     // For any given block from any given file, synchronize reads for said block.
385     // Without a cache, this synchronizing is needless overhead, but really
386     // the other choice is to duplicate work (which the cache would prevent you
387     // from doing).
388     BlockCacheKey cacheKey = new BlockCacheKey(name, dataBlockOffset);
389     boolean useLock = false;
390     IdLock.Entry lockEntry = null;
391     TraceScope traceScope = Trace.startSpan("HFileReaderV2.readBlock");
392     try {
393       while (true) {
394         if (useLock) {
395           lockEntry = offsetLock.getLockEntry(dataBlockOffset);
396         }
397 
398         // Check cache for block. If found return.
399         if (cacheConf.isBlockCacheEnabled()) {
400           // Try and get the block from the block cache. If the useLock variable is true then this
401           // is the second time through the loop and it should not be counted as a block cache miss.
402           HFileBlock cachedBlock = getCachedBlock(cacheKey, cacheBlock, useLock, isCompaction,
403             updateCacheMetrics, expectedBlockType, expectedDataBlockEncoding);
404           if (cachedBlock != null) {
405             if (Trace.isTracing()) {
406               traceScope.getSpan().addTimelineAnnotation("blockCacheHit");
407             }
408             assert cachedBlock.isUnpacked() : "Packed block leak.";
409             if (cachedBlock.getBlockType().isData()) {
410               if (updateCacheMetrics) {
411                 HFile.dataBlockReadCnt.incrementAndGet();
412               }
413               // Validate encoding type for data blocks. We include encoding
414               // type in the cache key, and we expect it to match on a cache hit.
415               if (cachedBlock.getDataBlockEncoding() != dataBlockEncoder.getDataBlockEncoding()) {
416                 throw new IOException("Cached block under key " + cacheKey + " "
417                   + "has wrong encoding: " + cachedBlock.getDataBlockEncoding() + " (expected: "
418                   + dataBlockEncoder.getDataBlockEncoding() + ")");
419               }
420             }
421             // Cache-hit. Return!
422             return cachedBlock;
423           }
424           // Carry on, please load.
425         }
426         if (!useLock) {
427           // check cache again with lock
428           useLock = true;
429           continue;
430         }
431         if (Trace.isTracing()) {
432           traceScope.getSpan().addTimelineAnnotation("blockCacheMiss");
433         }
434         // Load block from filesystem.
435         HFileBlock hfileBlock = fsBlockReader.readBlockData(dataBlockOffset, onDiskBlockSize, -1,
436             pread);
437         validateBlockType(hfileBlock, expectedBlockType);
438         HFileBlock unpacked = hfileBlock.unpack(hfileContext, fsBlockReader);
439         BlockType.BlockCategory category = hfileBlock.getBlockType().getCategory();
440 
441         // Cache the block if necessary
442         if (cacheBlock && cacheConf.shouldCacheBlockOnRead(category)) {
443           cacheConf.getBlockCache().cacheBlock(cacheKey,
444             cacheConf.shouldCacheCompressed(category) ? hfileBlock : unpacked,
445             cacheConf.isInMemory(), this.cacheConf.isCacheDataInL1());
446         }
447 
448         if (updateCacheMetrics && hfileBlock.getBlockType().isData()) {
449           HFile.dataBlockReadCnt.incrementAndGet();
450         }
451 
452         return unpacked;
453       }
454     } finally {
455       traceScope.close();
456       if (lockEntry != null) {
457         offsetLock.releaseLockEntry(lockEntry);
458       }
459     }
460   }
461 
462   @Override
463   public boolean hasMVCCInfo() {
464     return includesMemstoreTS && decodeMemstoreTS;
465   }
466 
467   /**
468    * Compares the actual type of a block retrieved from cache or disk with its
469    * expected type and throws an exception in case of a mismatch. Expected
470    * block type of {@link BlockType#DATA} is considered to match the actual
471    * block type [@link {@link BlockType#ENCODED_DATA} as well.
472    * @param block a block retrieved from cache or disk
473    * @param expectedBlockType the expected block type, or null to skip the
474    *          check
475    */
476   private void validateBlockType(HFileBlock block,
477       BlockType expectedBlockType) throws IOException {
478     if (expectedBlockType == null) {
479       return;
480     }
481     BlockType actualBlockType = block.getBlockType();
482     if (expectedBlockType.isData() && actualBlockType.isData()) {
483       // We consider DATA to match ENCODED_DATA for the purpose of this
484       // verification.
485       return;
486     }
487     if (actualBlockType != expectedBlockType) {
488       throw new IOException("Expected block type " + expectedBlockType + ", " +
489           "but got " + actualBlockType + ": " + block);
490     }
491   }
492 
493   /**
494    * @return Last key in the file. May be null if file has no entries. Note that
495    *         this is not the last row key, but rather the byte form of the last
496    *         KeyValue.
497    */
498   @Override
499   public byte[] getLastKey() {
500     return dataBlockIndexReader.isEmpty() ? null : lastKey;
501   }
502 
503   /**
504    * @return Midkey for this file. We work with block boundaries only so
505    *         returned midkey is an approximation only.
506    * @throws IOException
507    */
508   @Override
509   public byte[] midkey() throws IOException {
510     return dataBlockIndexReader.midkey();
511   }
512 
513   @Override
514   public void close() throws IOException {
515     close(cacheConf.shouldEvictOnClose());
516   }
517 
518   public void close(boolean evictOnClose) throws IOException {
519     PrefetchExecutor.cancel(path);
520     if (evictOnClose && cacheConf.isBlockCacheEnabled()) {
521       int numEvicted = cacheConf.getBlockCache().evictBlocksByHfileName(name);
522       if (LOG.isTraceEnabled()) {
523         LOG.trace("On close, file=" + name + " evicted=" + numEvicted
524           + " block(s)");
525       }
526     }
527     fsBlockReader.closeStreams();
528   }
529 
530   public DataBlockEncoding getEffectiveEncodingInCache(boolean isCompaction) {
531     return dataBlockEncoder.getEffectiveEncodingInCache(isCompaction);
532   }
533 
534   /** For testing */
535   @Override
536   HFileBlock.FSReader getUncachedBlockReader() {
537     return fsBlockReader;
538   }
539 
540 
541   protected abstract static class AbstractScannerV2
542       extends AbstractHFileReader.Scanner {
543     protected HFileBlock block;
544 
545     @Override
546     public Cell getNextIndexedKey() {
547       return nextIndexedKey;
548     }
549     /**
550      * The next indexed key is to keep track of the indexed key of the next data block.
551      * If the nextIndexedKey is HConstants.NO_NEXT_INDEXED_KEY, it means that the
552      * current data block is the last data block.
553      *
554      * If the nextIndexedKey is null, it means the nextIndexedKey has not been loaded yet.
555      */
556     protected Cell nextIndexedKey;
557 
558     public AbstractScannerV2(HFileReaderV2 r, boolean cacheBlocks,
559         final boolean pread, final boolean isCompaction) {
560       super(r, cacheBlocks, pread, isCompaction);
561     }
562 
563     protected abstract ByteBuffer getFirstKeyInBlock(HFileBlock curBlock);
564 
565     protected abstract int loadBlockAndSeekToKey(HFileBlock seekToBlock, Cell nextIndexedKey,
566         boolean rewind, Cell key, boolean seekBefore) throws IOException;
567 
568     @Override
569     public int seekTo(byte[] key, int offset, int length) throws IOException {
570       // Always rewind to the first key of the block, because the given key
571       // might be before or after the current key.
572       return seekTo(new KeyValue.KeyOnlyKeyValue(key, offset, length));
573     }
574 
575     @Override
576     public int reseekTo(byte[] key, int offset, int length) throws IOException {
577       return reseekTo(new KeyValue.KeyOnlyKeyValue(key, offset, length));
578     }
579 
580     @Override
581     public int seekTo(Cell key) throws IOException {
582       return seekTo(key, true);
583     }
584 
585     @Override
586     public int reseekTo(Cell key) throws IOException {
587       int compared;
588       if (isSeeked()) {
589         compared = compareKey(reader.getComparator(), key);
590         if (compared < 1) {
591           // If the required key is less than or equal to current key, then
592           // don't do anything.
593           return compared;
594         } else {
595           // The comparison with no_next_index_key has to be checked
596           if (this.nextIndexedKey != null &&
597               (this.nextIndexedKey == HConstants.NO_NEXT_INDEXED_KEY || reader
598               .getComparator()
599                   .compareOnlyKeyPortion(key, nextIndexedKey) < 0)) {
600             // The reader shall continue to scan the current data block instead
601             // of querying the
602             // block index as long as it knows the target key is strictly
603             // smaller than
604             // the next indexed key or the current data block is the last data
605             // block.
606             return loadBlockAndSeekToKey(this.block, nextIndexedKey, false, key, false);
607           }
608         }
609       }
610       // Don't rewind on a reseek operation, because reseek implies that we are
611       // always going forward in the file.
612       return seekTo(key, false);
613     }
614 
615 
616     /**
617      * An internal API function. Seek to the given key, optionally rewinding to
618      * the first key of the block before doing the seek.
619      *
620      * @param key - a cell representing the key that we need to fetch
621      * @param rewind whether to rewind to the first key of the block before
622      *        doing the seek. If this is false, we are assuming we never go
623      *        back, otherwise the result is undefined.
624      * @return -1 if the key is earlier than the first key of the file,
625      *         0 if we are at the given key, 1 if we are past the given key
626      *         -2 if the key is earlier than the first key of the file while
627      *         using a faked index key
628      * @throws IOException
629      */
630     public int seekTo(Cell key, boolean rewind) throws IOException {
631       HFileBlockIndex.BlockIndexReader indexReader = reader.getDataBlockIndexReader();
632       BlockWithScanInfo blockWithScanInfo = indexReader.loadDataBlockWithScanInfo(key, block,
633           cacheBlocks, pread, isCompaction, getEffectiveDataBlockEncoding());
634       if (blockWithScanInfo == null || blockWithScanInfo.getHFileBlock() == null) {
635         // This happens if the key e.g. falls before the beginning of the file.
636         return -1;
637       }
638       return loadBlockAndSeekToKey(blockWithScanInfo.getHFileBlock(),
639           blockWithScanInfo.getNextIndexedKey(), rewind, key, false);
640     }
641 
642     @Override
643     public boolean seekBefore(byte[] key, int offset, int length) throws IOException {
644       return seekBefore(new KeyValue.KeyOnlyKeyValue(key, offset, length));
645     }
646 
647     @Override
648     public boolean seekBefore(Cell key) throws IOException {
649       HFileBlock seekToBlock = reader.getDataBlockIndexReader().seekToDataBlock(key, block,
650           cacheBlocks, pread, isCompaction,
651           ((HFileReaderV2) reader).getEffectiveEncodingInCache(isCompaction));
652       if (seekToBlock == null) {
653         return false;
654       }
655       ByteBuffer firstKey = getFirstKeyInBlock(seekToBlock);
656 
657       if (reader.getComparator()
658           .compareOnlyKeyPortion(
659               new KeyValue.KeyOnlyKeyValue(firstKey.array(), firstKey.arrayOffset(),
660                   firstKey.limit()), key) >= 0) {
661         long previousBlockOffset = seekToBlock.getPrevBlockOffset();
662         // The key we are interested in
663         if (previousBlockOffset == -1) {
664           // we have a 'problem', the key we want is the first of the file.
665           return false;
666         }
667 
668         // It is important that we compute and pass onDiskSize to the block
669         // reader so that it does not have to read the header separately to
670         // figure out the size.
671         seekToBlock = reader.readBlock(previousBlockOffset,
672             seekToBlock.getOffset() - previousBlockOffset, cacheBlocks,
673             pread, isCompaction, true, BlockType.DATA, getEffectiveDataBlockEncoding());
674         // TODO shortcut: seek forward in this block to the last key of the
675         // block.
676       }
677       Cell firstKeyInCurrentBlock = new KeyValue.KeyOnlyKeyValue(Bytes.getBytes(firstKey));
678       loadBlockAndSeekToKey(seekToBlock, firstKeyInCurrentBlock, true, key, true);
679       return true;
680     }
681 
682     /**
683      * Scans blocks in the "scanned" section of the {@link HFile} until the next
684      * data block is found.
685      *
686      * @return the next block, or null if there are no more data blocks
687      * @throws IOException
688      */
689     protected HFileBlock readNextDataBlock() throws IOException {
690       long lastDataBlockOffset = reader.getTrailer().getLastDataBlockOffset();
691       if (block == null)
692         return null;
693 
694       HFileBlock curBlock = block;
695 
696       do {
697         if (curBlock.getOffset() >= lastDataBlockOffset)
698           return null;
699 
700         if (curBlock.getOffset() < 0) {
701           throw new IOException("Invalid block file offset: " + block);
702         }
703 
704         // We are reading the next block without block type validation, because
705         // it might turn out to be a non-data block.
706         curBlock = reader.readBlock(curBlock.getOffset()
707             + curBlock.getOnDiskSizeWithHeader(),
708             curBlock.getNextBlockOnDiskSizeWithHeader(), cacheBlocks, pread,
709             isCompaction, true, null, getEffectiveDataBlockEncoding());
710       } while (!curBlock.getBlockType().isData());
711 
712       return curBlock;
713     }
714 
715     public DataBlockEncoding getEffectiveDataBlockEncoding() {
716       return ((HFileReaderV2)reader).getEffectiveEncodingInCache(isCompaction);
717     }
718     /**
719      * Compare the given key against the current key
720      * @param comparator
721      * @param key
722      * @param offset
723      * @param length
724      * @return -1 is the passed key is smaller than the current key, 0 if equal and 1 if greater
725      */
726     public abstract int compareKey(KVComparator comparator, byte[] key, int offset,
727         int length);
728 
729     public abstract int compareKey(KVComparator comparator, Cell kv);
730   }
731 
732   /**
733    * Implementation of {@link HFileScanner} interface.
734    */
735   protected static class ScannerV2 extends AbstractScannerV2 {
736     private HFileReaderV2 reader;
737 
738     public ScannerV2(HFileReaderV2 r, boolean cacheBlocks,
739         final boolean pread, final boolean isCompaction) {
740       super(r, cacheBlocks, pread, isCompaction);
741       this.reader = r;
742     }
743 
744     @Override
745     public Cell getKeyValue() {
746       if (!isSeeked())
747         return null;
748 
749       KeyValue ret = new KeyValue(blockBuffer.array(), blockBuffer.arrayOffset()
750           + blockBuffer.position(), getCellBufSize());
751       if (this.reader.shouldIncludeMemstoreTS()) {
752         ret.setSequenceId(currMemstoreTS);
753       }
754       return ret;
755     }
756 
757     protected int getCellBufSize() {
758       return KEY_VALUE_LEN_SIZE + currKeyLen + currValueLen;
759     }
760 
761     @Override
762     public ByteBuffer getKey() {
763       assertSeeked();
764       return ByteBuffer.wrap(
765           blockBuffer.array(),
766           blockBuffer.arrayOffset() + blockBuffer.position()
767               + KEY_VALUE_LEN_SIZE, currKeyLen).slice();
768     }
769 
770     @Override
771     public int compareKey(KVComparator comparator, byte[] key, int offset, int length) {
772       return comparator.compareFlatKey(key, offset, length, blockBuffer.array(),
773           blockBuffer.arrayOffset() + blockBuffer.position() + KEY_VALUE_LEN_SIZE, currKeyLen);
774     }
775 
776     @Override
777     public ByteBuffer getValue() {
778       assertSeeked();
779       return ByteBuffer.wrap(
780           blockBuffer.array(),
781           blockBuffer.arrayOffset() + blockBuffer.position()
782               + KEY_VALUE_LEN_SIZE + currKeyLen, currValueLen).slice();
783     }
784 
785     protected void setNonSeekedState() {
786       block = null;
787       blockBuffer = null;
788       currKeyLen = 0;
789       currValueLen = 0;
790       currMemstoreTS = 0;
791       currMemstoreTSLen = 0;
792     }
793 
794     /**
795      * Go to the next key/value in the block section. Loads the next block if
796      * necessary. If successful, {@link #getKey()} and {@link #getValue()} can
797      * be called.
798      *
799      * @return true if successfully navigated to the next key/value
800      */
801     @Override
802     public boolean next() throws IOException {
803       assertSeeked();
804 
805       try {
806         blockBuffer.position(getNextCellStartPosition());
807       } catch (IllegalArgumentException e) {
808         LOG.error("Current pos = " + blockBuffer.position()
809             + "; currKeyLen = " + currKeyLen + "; currValLen = "
810             + currValueLen + "; block limit = " + blockBuffer.limit()
811             + "; HFile name = " + reader.getName()
812             + "; currBlock currBlockOffset = " + block.getOffset());
813         throw e;
814       }
815 
816       if (blockBuffer.remaining() <= 0) {
817         long lastDataBlockOffset =
818             reader.getTrailer().getLastDataBlockOffset();
819 
820         if (block.getOffset() >= lastDataBlockOffset) {
821           setNonSeekedState();
822           return false;
823         }
824 
825         // read the next block
826         HFileBlock nextBlock = readNextDataBlock();
827         if (nextBlock == null) {
828           setNonSeekedState();
829           return false;
830         }
831 
832         updateCurrBlock(nextBlock);
833         return true;
834       }
835 
836       // We are still in the same block.
837       readKeyValueLen();
838       return true;
839     }
840 
841     protected int getNextCellStartPosition() {
842       return blockBuffer.position() + KEY_VALUE_LEN_SIZE + currKeyLen + currValueLen
843           + currMemstoreTSLen;
844     }
845 
846     /**
847      * Positions this scanner at the start of the file.
848      *
849      * @return false if empty file; i.e. a call to next would return false and
850      *         the current key and value are undefined.
851      * @throws IOException
852      */
853     @Override
854     public boolean seekTo() throws IOException {
855       if (reader == null) {
856         return false;
857       }
858 
859       if (reader.getTrailer().getEntryCount() == 0) {
860         // No data blocks.
861         return false;
862       }
863 
864       long firstDataBlockOffset =
865           reader.getTrailer().getFirstDataBlockOffset();
866       if (block != null && block.getOffset() == firstDataBlockOffset) {
867         blockBuffer.rewind();
868         readKeyValueLen();
869         return true;
870       }
871 
872       block = reader.readBlock(firstDataBlockOffset, -1, cacheBlocks, pread,
873           isCompaction, true, BlockType.DATA, getEffectiveDataBlockEncoding());
874       if (block.getOffset() < 0) {
875         throw new IOException("Invalid block offset: " + block.getOffset());
876       }
877       updateCurrBlock(block);
878       return true;
879     }
880 
881     @Override
882     protected int loadBlockAndSeekToKey(HFileBlock seekToBlock, Cell nextIndexedKey,
883         boolean rewind, Cell key, boolean seekBefore) throws IOException {
884       if (block == null || block.getOffset() != seekToBlock.getOffset()) {
885         updateCurrBlock(seekToBlock);
886       } else if (rewind) {
887         blockBuffer.rewind();
888       }
889 
890       // Update the nextIndexedKey
891       this.nextIndexedKey = nextIndexedKey;
892       return blockSeek(key, seekBefore);
893     }
894 
895     /**
896      * Updates the current block to be the given {@link HFileBlock}. Seeks to
897      * the the first key/value pair.
898      *
899      * @param newBlock the block to make current
900      */
901     protected void updateCurrBlock(HFileBlock newBlock) {
902       block = newBlock;
903 
904       // sanity check
905       if (block.getBlockType() != BlockType.DATA) {
906         throw new IllegalStateException("ScannerV2 works only on data " +
907             "blocks, got " + block.getBlockType() + "; " +
908             "fileName=" + reader.name + ", " +
909             "dataBlockEncoder=" + reader.dataBlockEncoder + ", " +
910             "isCompaction=" + isCompaction);
911       }
912 
913       blockBuffer = block.getBufferWithoutHeader();
914       readKeyValueLen();
915       blockFetches++;
916 
917       // Reset the next indexed key
918       this.nextIndexedKey = null;
919     }
920 
921     protected void readKeyValueLen() {
922       blockBuffer.mark();
923       currKeyLen = blockBuffer.getInt();
924       currValueLen = blockBuffer.getInt();
925       ByteBufferUtils.skip(blockBuffer, currKeyLen + currValueLen);
926       readMvccVersion();
927       if (currKeyLen < 0 || currValueLen < 0
928           || currKeyLen > blockBuffer.limit()
929           || currValueLen > blockBuffer.limit()) {
930         throw new IllegalStateException("Invalid currKeyLen " + currKeyLen
931             + " or currValueLen " + currValueLen + ". Block offset: "
932             + block.getOffset() + ", block length: " + blockBuffer.limit()
933             + ", position: " + blockBuffer.position() + " (without header).");
934       }
935       blockBuffer.reset();
936     }
937 
938     protected void readMvccVersion() {
939       if (this.reader.shouldIncludeMemstoreTS()) {
940         if (this.reader.decodeMemstoreTS) {
941           try {
942             currMemstoreTS = Bytes.readVLong(blockBuffer.array(), blockBuffer.arrayOffset()
943                 + blockBuffer.position());
944             currMemstoreTSLen = WritableUtils.getVIntSize(currMemstoreTS);
945           } catch (Exception e) {
946             throw new RuntimeException("Error reading memstore timestamp", e);
947           }
948         } else {
949           currMemstoreTS = 0;
950           currMemstoreTSLen = 1;
951         }
952       }
953     }
954 
955     /**
956      * Within a loaded block, seek looking for the last key that is smaller than
957      * (or equal to?) the key we are interested in.
958      *
959      * A note on the seekBefore: if you have seekBefore = true, AND the first
960      * key in the block = key, then you'll get thrown exceptions. The caller has
961      * to check for that case and load the previous block as appropriate.
962      *
963      * @param key
964      *          the key to find
965      * @param seekBefore
966      *          find the key before the given key in case of exact match.
967      * @return 0 in case of an exact key match, 1 in case of an inexact match,
968      *         -2 in case of an inexact match and furthermore, the input key
969      *         less than the first key of current block(e.g. using a faked index
970      *         key)
971      */
972     protected int blockSeek(Cell key, boolean seekBefore) {
973       int klen, vlen;
974       long memstoreTS = 0;
975       int memstoreTSLen = 0;
976       int lastKeyValueSize = -1;
977       KeyValue.KeyOnlyKeyValue keyOnlykv = new KeyValue.KeyOnlyKeyValue();
978       do {
979         blockBuffer.mark();
980         klen = blockBuffer.getInt();
981         vlen = blockBuffer.getInt();
982         blockBuffer.reset();
983         if (this.reader.shouldIncludeMemstoreTS()) {
984           if (this.reader.decodeMemstoreTS) {
985             try {
986               int memstoreTSOffset = blockBuffer.arrayOffset() + blockBuffer.position()
987                   + KEY_VALUE_LEN_SIZE + klen + vlen;
988               memstoreTS = Bytes.readVLong(blockBuffer.array(), memstoreTSOffset);
989               memstoreTSLen = WritableUtils.getVIntSize(memstoreTS);
990             } catch (Exception e) {
991               throw new RuntimeException("Error reading memstore timestamp", e);
992             }
993           } else {
994             memstoreTS = 0;
995             memstoreTSLen = 1;
996           }
997         }
998 
999         int keyOffset = blockBuffer.arrayOffset() + blockBuffer.position() + KEY_VALUE_LEN_SIZE;
1000         keyOnlykv.setKey(blockBuffer.array(), keyOffset, klen);
1001         int comp = reader.getComparator().compareOnlyKeyPortion(key, keyOnlykv);
1002 
1003         if (comp == 0) {
1004           if (seekBefore) {
1005             if (lastKeyValueSize < 0) {
1006               throw new IllegalStateException("blockSeek with seekBefore "
1007                   + "at the first key of the block: key="
1008                   + CellUtil.getCellKeyAsString(key)
1009                   + ", blockOffset=" + block.getOffset() + ", onDiskSize="
1010                   + block.getOnDiskSizeWithHeader());
1011             }
1012             blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
1013             readKeyValueLen();
1014             return 1; // non exact match.
1015           }
1016           currKeyLen = klen;
1017           currValueLen = vlen;
1018           if (this.reader.shouldIncludeMemstoreTS()) {
1019             currMemstoreTS = memstoreTS;
1020             currMemstoreTSLen = memstoreTSLen;
1021           }
1022           return 0; // indicate exact match
1023         } else if (comp < 0) {
1024           if (lastKeyValueSize > 0)
1025             blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
1026           readKeyValueLen();
1027           if (lastKeyValueSize == -1 && blockBuffer.position() == 0
1028               && this.reader.trailer.getMinorVersion() >= MINOR_VERSION_WITH_FAKED_KEY) {
1029             return HConstants.INDEX_KEY_MAGIC;
1030           }
1031           return 1;
1032         }
1033 
1034         // The size of this key/value tuple, including key/value length fields.
1035         lastKeyValueSize = klen + vlen + memstoreTSLen + KEY_VALUE_LEN_SIZE;
1036         blockBuffer.position(blockBuffer.position() + lastKeyValueSize);
1037       } while (blockBuffer.remaining() > 0);
1038 
1039       // Seek to the last key we successfully read. This will happen if this is
1040       // the last key/value pair in the file, in which case the following call
1041       // to next() has to return false.
1042       blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
1043       readKeyValueLen();
1044       return 1; // didn't exactly find it.
1045     }
1046 
1047     @Override
1048     protected ByteBuffer getFirstKeyInBlock(HFileBlock curBlock) {
1049       ByteBuffer buffer = curBlock.getBufferWithoutHeader();
1050       // It is safe to manipulate this buffer because we own the buffer object.
1051       buffer.rewind();
1052       int klen = buffer.getInt();
1053       buffer.getInt();
1054       ByteBuffer keyBuff = buffer.slice();
1055       keyBuff.limit(klen);
1056       keyBuff.rewind();
1057       return keyBuff;
1058     }
1059 
1060     @Override
1061     public String getKeyString() {
1062       return Bytes.toStringBinary(blockBuffer.array(),
1063           blockBuffer.arrayOffset() + blockBuffer.position()
1064               + KEY_VALUE_LEN_SIZE, currKeyLen);
1065     }
1066 
1067     @Override
1068     public String getValueString() {
1069       return Bytes.toString(blockBuffer.array(), blockBuffer.arrayOffset()
1070           + blockBuffer.position() + KEY_VALUE_LEN_SIZE + currKeyLen,
1071           currValueLen);
1072     }
1073 
1074     @Override
1075     public int compareKey(KVComparator comparator, Cell key) {
1076       return comparator.compareOnlyKeyPortion(
1077           key,
1078           new KeyValue.KeyOnlyKeyValue(blockBuffer.array(), blockBuffer.arrayOffset()
1079               + blockBuffer.position() + KEY_VALUE_LEN_SIZE, currKeyLen));
1080     }
1081   }
1082 
1083   /**
1084    * ScannerV2 that operates on encoded data blocks.
1085    */
1086   protected static class EncodedScannerV2 extends AbstractScannerV2 {
1087     private final HFileBlockDecodingContext decodingCtx;
1088     private final DataBlockEncoder.EncodedSeeker seeker;
1089     private final DataBlockEncoder dataBlockEncoder;
1090     protected final HFileContext meta;
1091 
1092     public EncodedScannerV2(HFileReaderV2 reader, boolean cacheBlocks,
1093         boolean pread, boolean isCompaction, HFileContext meta) {
1094       super(reader, cacheBlocks, pread, isCompaction);
1095       DataBlockEncoding encoding = reader.dataBlockEncoder.getDataBlockEncoding();
1096       dataBlockEncoder = encoding.getEncoder();
1097       decodingCtx = dataBlockEncoder.newDataBlockDecodingContext(meta);
1098       seeker = dataBlockEncoder.createSeeker(
1099         reader.getComparator(), decodingCtx);
1100       this.meta = meta;
1101     }
1102 
1103     @Override
1104     public boolean isSeeked(){
1105       return this.block != null;
1106     }
1107 
1108     /**
1109      * Updates the current block to be the given {@link HFileBlock}. Seeks to
1110      * the the first key/value pair.
1111      *
1112      * @param newBlock the block to make current
1113      * @throws CorruptHFileException
1114      */
1115     private void updateCurrentBlock(HFileBlock newBlock) throws CorruptHFileException {
1116       block = newBlock;
1117 
1118       // sanity checks
1119       if (block.getBlockType() != BlockType.ENCODED_DATA) {
1120         throw new IllegalStateException(
1121             "EncodedScanner works only on encoded data blocks");
1122       }
1123       short dataBlockEncoderId = block.getDataBlockEncodingId();
1124       if (!DataBlockEncoding.isCorrectEncoder(dataBlockEncoder, dataBlockEncoderId)) {
1125         String encoderCls = dataBlockEncoder.getClass().getName();
1126         throw new CorruptHFileException("Encoder " + encoderCls
1127           + " doesn't support data block encoding "
1128           + DataBlockEncoding.getNameFromId(dataBlockEncoderId));
1129       }
1130 
1131       seeker.setCurrentBuffer(getEncodedBuffer(newBlock));
1132       blockFetches++;
1133 
1134       // Reset the next indexed key
1135       this.nextIndexedKey = null;
1136     }
1137 
1138     private ByteBuffer getEncodedBuffer(HFileBlock newBlock) {
1139       ByteBuffer origBlock = newBlock.getBufferReadOnly();
1140       ByteBuffer encodedBlock = ByteBuffer.wrap(origBlock.array(),
1141           origBlock.arrayOffset() + newBlock.headerSize() +
1142           DataBlockEncoding.ID_SIZE,
1143           newBlock.getUncompressedSizeWithoutHeader() -
1144           DataBlockEncoding.ID_SIZE).slice();
1145       return encodedBlock;
1146     }
1147 
1148     @Override
1149     public boolean seekTo() throws IOException {
1150       if (reader == null) {
1151         return false;
1152       }
1153 
1154       if (reader.getTrailer().getEntryCount() == 0) {
1155         // No data blocks.
1156         return false;
1157       }
1158 
1159       long firstDataBlockOffset =
1160           reader.getTrailer().getFirstDataBlockOffset();
1161       if (block != null && block.getOffset() == firstDataBlockOffset) {
1162         seeker.rewind();
1163         return true;
1164       }
1165 
1166       block = reader.readBlock(firstDataBlockOffset, -1, cacheBlocks, pread,
1167           isCompaction, true, BlockType.DATA, getEffectiveDataBlockEncoding());
1168       if (block.getOffset() < 0) {
1169         throw new IOException("Invalid block offset: " + block.getOffset());
1170       }
1171       updateCurrentBlock(block);
1172       return true;
1173     }
1174 
1175     @Override
1176     public boolean next() throws IOException {
1177       boolean isValid = seeker.next();
1178       if (!isValid) {
1179         block = readNextDataBlock();
1180         isValid = block != null;
1181         if (isValid) {
1182           updateCurrentBlock(block);
1183         }
1184       }
1185       return isValid;
1186     }
1187 
1188     @Override
1189     public ByteBuffer getKey() {
1190       assertValidSeek();
1191       return seeker.getKeyDeepCopy();
1192     }
1193 
1194     @Override
1195     public int compareKey(KVComparator comparator, byte[] key, int offset, int length) {
1196       return seeker.compareKey(comparator, key, offset, length);
1197     }
1198 
1199     @Override
1200     public ByteBuffer getValue() {
1201       assertValidSeek();
1202       return seeker.getValueShallowCopy();
1203     }
1204 
1205     @Override
1206     public Cell getKeyValue() {
1207       if (block == null) {
1208         return null;
1209       }
1210       return seeker.getKeyValue();
1211     }
1212 
1213     @Override
1214     public String getKeyString() {
1215       ByteBuffer keyBuffer = getKey();
1216       return Bytes.toStringBinary(keyBuffer.array(),
1217           keyBuffer.arrayOffset(), keyBuffer.limit());
1218     }
1219 
1220     @Override
1221     public String getValueString() {
1222       ByteBuffer valueBuffer = getValue();
1223       return Bytes.toStringBinary(valueBuffer.array(),
1224           valueBuffer.arrayOffset(), valueBuffer.limit());
1225     }
1226 
1227     private void assertValidSeek() {
1228       if (block == null) {
1229         throw new NotSeekedException();
1230       }
1231     }
1232 
1233     @Override
1234     protected ByteBuffer getFirstKeyInBlock(HFileBlock curBlock) {
1235       return dataBlockEncoder.getFirstKeyInBlock(getEncodedBuffer(curBlock));
1236     }
1237 
1238     @Override
1239     protected int loadBlockAndSeekToKey(HFileBlock seekToBlock, Cell nextIndexedKey,
1240         boolean rewind, Cell key, boolean seekBefore) throws IOException {
1241       if (block == null || block.getOffset() != seekToBlock.getOffset()) {
1242         updateCurrentBlock(seekToBlock);
1243       } else if (rewind) {
1244         seeker.rewind();
1245       }
1246       this.nextIndexedKey = nextIndexedKey;
1247       return seeker.seekToKeyInBlock(key, seekBefore);
1248     }
1249 
1250     @Override
1251     public int compareKey(KVComparator comparator, Cell key) {
1252       return seeker.compareKey(comparator, key);
1253     }
1254   }
1255 
1256   /**
1257    * Returns a buffer with the Bloom filter metadata. The caller takes
1258    * ownership of the buffer.
1259    */
1260   @Override
1261   public DataInput getGeneralBloomFilterMetadata() throws IOException {
1262     return this.getBloomFilterMetadata(BlockType.GENERAL_BLOOM_META);
1263   }
1264 
1265   @Override
1266   public DataInput getDeleteBloomFilterMetadata() throws IOException {
1267     return this.getBloomFilterMetadata(BlockType.DELETE_FAMILY_BLOOM_META);
1268   }
1269 
1270   private DataInput getBloomFilterMetadata(BlockType blockType)
1271   throws IOException {
1272     if (blockType != BlockType.GENERAL_BLOOM_META &&
1273         blockType != BlockType.DELETE_FAMILY_BLOOM_META) {
1274       throw new RuntimeException("Block Type: " + blockType.toString() +
1275           " is not supported") ;
1276     }
1277 
1278     for (HFileBlock b : loadOnOpenBlocks)
1279       if (b.getBlockType() == blockType)
1280         return b.getByteStream();
1281     return null;
1282   }
1283 
1284   @Override
1285   public boolean isFileInfoLoaded() {
1286     return true; // We load file info in constructor in version 2.
1287   }
1288 
1289   /**
1290    * Validates that the minor version is within acceptable limits.
1291    * Otherwise throws an Runtime exception
1292    */
1293   private void validateMinorVersion(Path path, int minorVersion) {
1294     if (minorVersion < MIN_MINOR_VERSION ||
1295         minorVersion > MAX_MINOR_VERSION) {
1296       String msg = "Minor version for path " + path +
1297                    " is expected to be between " +
1298                    MIN_MINOR_VERSION + " and " + MAX_MINOR_VERSION +
1299                    " but is found to be " + minorVersion;
1300       LOG.error(msg);
1301       throw new RuntimeException(msg);
1302     }
1303   }
1304 
1305   @Override
1306   public int getMajorVersion() {
1307     return 2;
1308   }
1309 
1310   @Override
1311   public HFileContext getFileContext() {
1312     return hfileContext;
1313   }
1314 
1315   /**
1316    * Returns false if block prefetching was requested for this file and has
1317    * not completed, true otherwise
1318    */
1319   @VisibleForTesting
1320   boolean prefetchComplete() {
1321     return PrefetchExecutor.isCompleted(path);
1322   }
1323 }