View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.io.hfile;
20  
21  import java.io.IOException;
22  import java.nio.ByteBuffer;
23  
24  import org.apache.hadoop.classification.InterfaceAudience;
25  import org.apache.hadoop.conf.Configurable;
26  import org.apache.hadoop.conf.Configuration;
27  import org.apache.hadoop.fs.FSDataInputStream;
28  import org.apache.hadoop.fs.Path;
29  import org.apache.hadoop.hbase.KeyValue;
30  import org.apache.hadoop.hbase.KeyValue.KVComparator;
31  import org.apache.hadoop.hbase.fs.HFileSystem;
32  import org.apache.hadoop.hbase.io.compress.Compression;
33  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
34  import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
35  
36  /**
37   * Common functionality needed by all versions of {@link HFile} readers.
38   */
39  @InterfaceAudience.Private
40  @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD")
41  public abstract class AbstractHFileReader
42      implements HFile.Reader, Configurable {
43    /** Stream to read from. Does checksum verifications in file system */
44    protected FSDataInputStream istream; // UUF_UNUSED_PUBLIC_OR_PROTECTED_FIELD
45  
46    /** The file system stream of the underlying {@link HFile} that
47     * does not do checksum verification in the file system */
48    protected FSDataInputStream istreamNoFsChecksum;  // UUF_UNUSED_PUBLIC_OR_PROTECTED_FIELD
49  
50    /** Data block index reader keeping the root data index in memory */
51    protected HFileBlockIndex.BlockIndexReader dataBlockIndexReader;
52  
53    /** Meta block index reader -- always single level */
54    protected HFileBlockIndex.BlockIndexReader metaBlockIndexReader;
55  
56    protected final FixedFileTrailer trailer;
57  
58    /** Filled when we read in the trailer. */
59    protected final Compression.Algorithm compressAlgo;
60  
61    /**
62     * What kind of data block encoding should be used while reading, writing,
63     * and handling cache.
64     */
65    protected HFileDataBlockEncoder dataBlockEncoder =
66        NoOpDataBlockEncoder.INSTANCE;
67  
68    /** Last key in the file. Filled in when we read in the file info */
69    protected byte [] lastKey = null;
70  
71    /** Average key length read from file info */
72    protected int avgKeyLen = -1;
73  
74    /** Average value length read from file info */
75    protected int avgValueLen = -1;
76  
77    /** Key comparator */
78    protected KVComparator comparator = new KVComparator();
79  
80    /** Size of this file. */
81    protected final long fileSize;
82  
83    /** Block cache configuration. */
84    protected final CacheConfig cacheConf;
85  
86    /** Path of file */
87    protected final Path path;
88  
89    /** File name to be used for block names */
90    protected final String name;
91  
92    protected FileInfo fileInfo;
93  
94    /** The filesystem used for accesing data */
95    protected HFileSystem hfs;
96  
97    protected Configuration conf;
98  
99    @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD")
100   protected AbstractHFileReader(Path path, FixedFileTrailer trailer,
101       final long fileSize, final CacheConfig cacheConf, final HFileSystem hfs,
102       final Configuration conf) {
103     this.trailer = trailer;
104     this.compressAlgo = trailer.getCompressionCodec();
105     this.cacheConf = cacheConf;
106     this.fileSize = fileSize;
107     this.path = path;
108     this.name = path.getName();
109     this.hfs = hfs; // URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD
110     this.conf = conf;
111   }
112 
113   @SuppressWarnings("serial")
114   public static class BlockIndexNotLoadedException
115       extends IllegalStateException {
116     public BlockIndexNotLoadedException() {
117       // Add a message in case anyone relies on it as opposed to class name.
118       super("Block index not loaded");
119     }
120   }
121 
122   protected String toStringFirstKey() {
123     return KeyValue.keyToString(getFirstKey());
124   }
125 
126   protected String toStringLastKey() {
127     return KeyValue.keyToString(getLastKey());
128   }
129 
130   public abstract boolean isFileInfoLoaded();
131 
132   @Override
133   public String toString() {
134     return "reader=" + path.toString() +
135         (!isFileInfoLoaded()? "":
136           ", compression=" + compressAlgo.getName() +
137           ", cacheConf=" + cacheConf +
138           ", firstKey=" + toStringFirstKey() +
139           ", lastKey=" + toStringLastKey()) +
140           ", avgKeyLen=" + avgKeyLen +
141           ", avgValueLen=" + avgValueLen +
142           ", entries=" + trailer.getEntryCount() +
143           ", length=" + fileSize;
144   }
145 
146   @Override
147   public long length() {
148     return fileSize;
149   }
150 
151   /**
152    * Create a Scanner on this file. No seeks or reads are done on creation. Call
153    * {@link HFileScanner#seekTo(byte[])} to position an start the read. There is
154    * nothing to clean up in a Scanner. Letting go of your references to the
155    * scanner is sufficient. NOTE: Do not use this overload of getScanner for
156    * compactions.
157    *
158    * @param cacheBlocks True if we should cache blocks read in by this scanner.
159    * @param pread Use positional read rather than seek+read if true (pread is
160    *          better for random reads, seek+read is better scanning).
161    * @return Scanner on this file.
162    */
163   @Override
164   public HFileScanner getScanner(boolean cacheBlocks, final boolean pread) {
165     return getScanner(cacheBlocks, pread, false);
166   }
167 
168   /**
169    * @return the first key in the file. May be null if file has no entries. Note
170    *         that this is not the first row key, but rather the byte form of the
171    *         first KeyValue.
172    */
173   @Override
174   public byte [] getFirstKey() {
175     if (dataBlockIndexReader == null) {
176       throw new BlockIndexNotLoadedException();
177     }
178     return dataBlockIndexReader.isEmpty() ? null
179         : dataBlockIndexReader.getRootBlockKey(0);
180   }
181 
182   /**
183    * TODO left from {@link HFile} version 1: move this to StoreFile after Ryan's
184    * patch goes in to eliminate {@link KeyValue} here.
185    *
186    * @return the first row key, or null if the file is empty.
187    */
188   @Override
189   public byte[] getFirstRowKey() {
190     byte[] firstKey = getFirstKey();
191     if (firstKey == null)
192       return null;
193     return KeyValue.createKeyValueFromKey(firstKey).getRow();
194   }
195 
196   /**
197    * TODO left from {@link HFile} version 1: move this to StoreFile after
198    * Ryan's patch goes in to eliminate {@link KeyValue} here.
199    *
200    * @return the last row key, or null if the file is empty.
201    */
202   @Override
203   public byte[] getLastRowKey() {
204     byte[] lastKey = getLastKey();
205     if (lastKey == null)
206       return null;
207     return KeyValue.createKeyValueFromKey(lastKey).getRow();
208   }
209 
210   /** @return number of KV entries in this HFile */
211   @Override
212   public long getEntries() {
213     return trailer.getEntryCount();
214   }
215 
216   /** @return comparator */
217   @Override
218   public KVComparator getComparator() {
219     return comparator;
220   }
221 
222   /** @return compression algorithm */
223   @Override
224   public Compression.Algorithm getCompressionAlgorithm() {
225     return compressAlgo;
226   }
227 
228   /**
229    * @return the total heap size of data and meta block indexes in bytes. Does
230    *         not take into account non-root blocks of a multilevel data index.
231    */
232   public long indexSize() {
233     return (dataBlockIndexReader != null ? dataBlockIndexReader.heapSize() : 0)
234         + ((metaBlockIndexReader != null) ? metaBlockIndexReader.heapSize()
235             : 0);
236   }
237 
238   @Override
239   public String getName() {
240     return name;
241   }
242 
243   @Override
244   public HFileBlockIndex.BlockIndexReader getDataBlockIndexReader() {
245     return dataBlockIndexReader;
246   }
247 
248   @Override
249   public FixedFileTrailer getTrailer() {
250     return trailer;
251   }
252 
253   @Override
254   public FileInfo loadFileInfo() throws IOException {
255     return fileInfo;
256   }
257 
258   /**
259    * An exception thrown when an operation requiring a scanner to be seeked
260    * is invoked on a scanner that is not seeked.
261    */
262   @SuppressWarnings("serial")
263   public static class NotSeekedException extends IllegalStateException {
264     public NotSeekedException() {
265       super("Not seeked to a key/value");
266     }
267   }
268 
269   protected static abstract class Scanner implements HFileScanner {
270     protected ByteBuffer blockBuffer;
271 
272     protected boolean cacheBlocks;
273     protected final boolean pread;
274     protected final boolean isCompaction;
275 
276     protected int currKeyLen;
277     protected int currValueLen;
278     protected int currMemstoreTSLen;
279     protected long currMemstoreTS;
280 
281     protected int blockFetches;
282 
283     protected final HFile.Reader reader;
284 
285     public Scanner(final HFile.Reader reader, final boolean cacheBlocks,
286         final boolean pread, final boolean isCompaction) {
287       this.reader = reader;
288       this.cacheBlocks = cacheBlocks;
289       this.pread = pread;
290       this.isCompaction = isCompaction;
291     }
292 
293     @Override
294     public boolean isSeeked(){
295       return blockBuffer != null;
296     }
297 
298     @Override
299     public String toString() {
300       return "HFileScanner for reader " + String.valueOf(getReader());
301     }
302 
303     protected void assertSeeked() {
304       if (!isSeeked())
305         throw new NotSeekedException();
306     }
307 
308     @Override
309     public int seekTo(byte[] key) throws IOException {
310       return seekTo(key, 0, key.length);
311     }
312     
313     @Override
314     public boolean seekBefore(byte[] key) throws IOException {
315       return seekBefore(key, 0, key.length);
316     }
317     
318     @Override
319     public int reseekTo(byte[] key) throws IOException {
320       return reseekTo(key, 0, key.length);
321     }
322 
323     @Override
324     public HFile.Reader getReader() {
325       return reader;
326     }
327   }
328 
329   /** For testing */
330   abstract HFileBlock.FSReader getUncachedBlockReader();
331 
332   public Path getPath() {
333     return path;
334   }
335 
336   @Override
337   public DataBlockEncoding getDataBlockEncoding() {
338     return dataBlockEncoder.getDataBlockEncoding();
339   }
340 
341   public abstract int getMajorVersion();
342 
343   @Override
344   public Configuration getConf() {
345     return conf;
346   }
347 
348   @Override
349   public void setConf(Configuration conf) {
350     this.conf = conf;
351   }
352 }