View Javadoc

1     /*
2    * Copyright 2011 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.io.hfile;
21  
22  import java.io.IOException;
23  import java.nio.ByteBuffer;
24  
25  import org.apache.hadoop.fs.FSDataInputStream;
26  import org.apache.hadoop.fs.Path;
27  import org.apache.hadoop.hbase.KeyValue;
28  import org.apache.hadoop.hbase.fs.HFileSystem;
29  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
30  import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
31  import org.apache.hadoop.hbase.regionserver.metrics.SchemaConfigured;
32  import org.apache.hadoop.io.RawComparator;
33  
34  /**
35   * Common functionality needed by all versions of {@link HFile} readers.
36   */
37  public abstract class AbstractHFileReader extends SchemaConfigured
38      implements HFile.Reader {
39  
40    /** Filesystem-level block reader for this HFile format version. */
41    protected HFileBlock.FSReader fsBlockReader;
42  
43    /** Stream to read from. Does checksum verifications in file system */
44    protected FSDataInputStream istream;
45  
46    /** The file system stream of the underlying {@link HFile} that
47     * does not do checksum verification in the file system */
48    protected FSDataInputStream istreamNoFsChecksum;
49  
50    /**
51     * True if we should close the input stream when done. We don't close it if we
52     * didn't open it.
53     */
54    protected final boolean closeIStream;
55  
56    /** Data block index reader keeping the root data index in memory */
57    protected HFileBlockIndex.BlockIndexReader dataBlockIndexReader;
58  
59    /** Meta block index reader -- always single level */
60    protected HFileBlockIndex.BlockIndexReader metaBlockIndexReader;
61  
62    protected final FixedFileTrailer trailer;
63  
64    /** Filled when we read in the trailer. */
65    protected final Compression.Algorithm compressAlgo;
66  
67    /**
68     * What kind of data block encoding should be used while reading, writing,
69     * and handling cache.
70     */
71    protected HFileDataBlockEncoder dataBlockEncoder =
72        NoOpDataBlockEncoder.INSTANCE;
73  
74    /** Last key in the file. Filled in when we read in the file info */
75    protected byte [] lastKey = null;
76  
77    /** Average key length read from file info */
78    protected int avgKeyLen = -1;
79  
80    /** Average value length read from file info */
81    protected int avgValueLen = -1;
82  
83    /** Key comparator */
84    protected RawComparator<byte []> comparator;
85  
86    /** Size of this file. */
87    protected final long fileSize;
88  
89    /** Block cache configuration. */
90    protected final CacheConfig cacheConf;
91  
92    /** Path of file */
93    protected final Path path;
94  
95    /** File name to be used for block names */
96    protected final String name;
97  
98    protected FileInfo fileInfo;
99  
100   /** The filesystem used for accesing data */
101   protected HFileSystem hfs;
102 
103   protected AbstractHFileReader(Path path, FixedFileTrailer trailer,
104       final FSDataInputStream fsdis, final long fileSize,
105       final boolean closeIStream,
106       final CacheConfig cacheConf) {
107     this(path, trailer, fsdis, fsdis, fileSize, closeIStream, cacheConf, null);
108   }
109 
110   protected AbstractHFileReader(Path path, FixedFileTrailer trailer,
111       final FSDataInputStream fsdis, final FSDataInputStream fsdisNoFsChecksum,
112       final long fileSize,
113       final boolean closeIStream,
114       final CacheConfig cacheConf, final HFileSystem hfs) {
115     super(null, path);
116     this.trailer = trailer;
117     this.compressAlgo = trailer.getCompressionCodec();
118     this.cacheConf = cacheConf;
119     this.fileSize = fileSize;
120     this.istream = fsdis;
121     this.closeIStream = closeIStream;
122     this.path = path;
123     this.name = path.getName();
124     this.hfs = hfs;
125     this.istreamNoFsChecksum = fsdisNoFsChecksum;
126   }
127 
128   @SuppressWarnings("serial")
129   public static class BlockIndexNotLoadedException
130       extends IllegalStateException {
131     public BlockIndexNotLoadedException() {
132       // Add a message in case anyone relies on it as opposed to class name.
133       super("Block index not loaded");
134     }
135   }
136 
137   protected String toStringFirstKey() {
138     return KeyValue.keyToString(getFirstKey());
139   }
140 
141   protected String toStringLastKey() {
142     return KeyValue.keyToString(getLastKey());
143   }
144 
145   public abstract boolean isFileInfoLoaded();
146 
147   @Override
148   public String toString() {
149     return "reader=" + path.toString() +
150         (!isFileInfoLoaded()? "":
151           ", compression=" + compressAlgo.getName() +
152           ", cacheConf=" + cacheConf +
153           ", firstKey=" + toStringFirstKey() +
154           ", lastKey=" + toStringLastKey()) +
155           ", avgKeyLen=" + avgKeyLen +
156           ", avgValueLen=" + avgValueLen +
157           ", entries=" + trailer.getEntryCount() +
158           ", length=" + fileSize;
159   }
160 
161   @Override
162   public long length() {
163     return fileSize;
164   }
165 
166   /**
167    * Create a Scanner on this file. No seeks or reads are done on creation. Call
168    * {@link HFileScanner#seekTo(byte[])} to position an start the read. There is
169    * nothing to clean up in a Scanner. Letting go of your references to the
170    * scanner is sufficient. NOTE: Do not use this overload of getScanner for
171    * compactions.
172    *
173    * @param cacheBlocks True if we should cache blocks read in by this scanner.
174    * @param pread Use positional read rather than seek+read if true (pread is
175    *          better for random reads, seek+read is better scanning).
176    * @return Scanner on this file.
177    */
178   @Override
179   public HFileScanner getScanner(boolean cacheBlocks, final boolean pread) {
180     return getScanner(cacheBlocks, pread, false);
181   }
182 
183   /**
184    * @return the first key in the file. May be null if file has no entries. Note
185    *         that this is not the first row key, but rather the byte form of the
186    *         first KeyValue.
187    */
188   @Override
189   public byte [] getFirstKey() {
190     if (dataBlockIndexReader == null) {
191       throw new BlockIndexNotLoadedException();
192     }
193     return dataBlockIndexReader.isEmpty() ? null
194         : dataBlockIndexReader.getRootBlockKey(0);
195   }
196 
197   /**
198    * TODO left from {@link HFile} version 1: move this to StoreFile after Ryan's
199    * patch goes in to eliminate {@link KeyValue} here.
200    *
201    * @return the first row key, or null if the file is empty.
202    */
203   @Override
204   public byte[] getFirstRowKey() {
205     byte[] firstKey = getFirstKey();
206     if (firstKey == null)
207       return null;
208     return KeyValue.createKeyValueFromKey(firstKey).getRow();
209   }
210 
211   /**
212    * TODO left from {@link HFile} version 1: move this to StoreFile after
213    * Ryan's patch goes in to eliminate {@link KeyValue} here.
214    *
215    * @return the last row key, or null if the file is empty.
216    */
217   @Override
218   public byte[] getLastRowKey() {
219     byte[] lastKey = getLastKey();
220     if (lastKey == null)
221       return null;
222     return KeyValue.createKeyValueFromKey(lastKey).getRow();
223   }
224 
225   /** @return number of KV entries in this HFile */
226   @Override
227   public long getEntries() {
228     return trailer.getEntryCount();
229   }
230 
231   /** @return comparator */
232   @Override
233   public RawComparator<byte []> getComparator() {
234     return comparator;
235   }
236 
237   /** @return compression algorithm */
238   @Override
239   public Compression.Algorithm getCompressionAlgorithm() {
240     return compressAlgo;
241   }
242 
243   /**
244    * @return the total heap size of data and meta block indexes in bytes. Does
245    *         not take into account non-root blocks of a multilevel data index.
246    */
247   public long indexSize() {
248     return (dataBlockIndexReader != null ? dataBlockIndexReader.heapSize() : 0)
249         + ((metaBlockIndexReader != null) ? metaBlockIndexReader.heapSize()
250             : 0);
251   }
252 
253   @Override
254   public String getName() {
255     return name;
256   }
257 
258   @Override
259   public HFileBlockIndex.BlockIndexReader getDataBlockIndexReader() {
260     return dataBlockIndexReader;
261   }
262 
263   @Override
264   public FixedFileTrailer getTrailer() {
265     return trailer;
266   }
267 
268   @Override
269   public FileInfo loadFileInfo() throws IOException {
270     return fileInfo;
271   }
272 
273   /**
274    * An exception thrown when an operation requiring a scanner to be seeked
275    * is invoked on a scanner that is not seeked.
276    */
277   @SuppressWarnings("serial")
278   public static class NotSeekedException extends IllegalStateException {
279     public NotSeekedException() {
280       super("Not seeked to a key/value");
281     }
282   }
283 
284   protected static abstract class Scanner implements HFileScanner {
285     protected ByteBuffer blockBuffer;
286 
287     protected boolean cacheBlocks;
288     protected final boolean pread;
289     protected final boolean isCompaction;
290 
291     protected int currKeyLen;
292     protected int currValueLen;
293     protected int currMemstoreTSLen;
294     protected long currMemstoreTS;
295 
296     protected int blockFetches;
297 
298     protected final HFile.Reader reader;
299 
300     public Scanner(final HFile.Reader reader, final boolean cacheBlocks,
301         final boolean pread, final boolean isCompaction) {
302       this.reader = reader;
303       this.cacheBlocks = cacheBlocks;
304       this.pread = pread;
305       this.isCompaction = isCompaction;
306     }
307 
308     @Override
309     public boolean isSeeked(){
310       return blockBuffer != null;
311     }
312 
313     @Override
314     public String toString() {
315       return "HFileScanner for reader " + String.valueOf(getReader());
316     }
317 
318     protected void assertSeeked() {
319       if (!isSeeked())
320         throw new NotSeekedException();
321     }
322 
323     @Override
324     public int seekTo(byte[] key) throws IOException {
325       return seekTo(key, 0, key.length);
326     }
327     
328     @Override
329     public boolean seekBefore(byte[] key) throws IOException {
330       return seekBefore(key, 0, key.length);
331     }
332     
333     @Override
334     public int reseekTo(byte[] key) throws IOException {
335       return reseekTo(key, 0, key.length);
336     }
337 
338     @Override
339     public HFile.Reader getReader() {
340       return reader;
341     }
342   }
343 
344   /** For testing */
345   HFileBlock.FSReader getUncachedBlockReader() {
346     return fsBlockReader;
347   }
348 
349   public Path getPath() {
350     return path;
351   }
352 
353   @Override
354   public DataBlockEncoding getEncodingOnDisk() {
355     return dataBlockEncoder.getEncodingOnDisk();
356   }
357 
358   @Override
359   public boolean hasMVCCInfo() {
360     return true;
361   }
362 }