View Javadoc

1     /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.io.hfile;
20  
21  import java.io.IOException;
22  import java.nio.ByteBuffer;
23  
24  import org.apache.hadoop.classification.InterfaceAudience;
25  import org.apache.hadoop.conf.Configurable;
26  import org.apache.hadoop.conf.Configuration;
27  import org.apache.hadoop.fs.FSDataInputStream;
28  import org.apache.hadoop.fs.Path;
29  import org.apache.hadoop.hbase.KeyValue;
30  import org.apache.hadoop.hbase.KeyValue.KVComparator;
31  import org.apache.hadoop.hbase.fs.HFileSystem;
32  import org.apache.hadoop.hbase.io.compress.Compression;
33  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
34  import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
35  
36  /**
37   * Common functionality needed by all versions of {@link HFile} readers.
38   */
39  @InterfaceAudience.Private
40  public abstract class AbstractHFileReader
41      implements HFile.Reader, Configurable {
42    /** Stream to read from. Does checksum verifications in file system */
43    protected FSDataInputStream istream;
44  
45    /** The file system stream of the underlying {@link HFile} that
46     * does not do checksum verification in the file system */
47    protected FSDataInputStream istreamNoFsChecksum;
48  
49    /** Data block index reader keeping the root data index in memory */
50    protected HFileBlockIndex.BlockIndexReader dataBlockIndexReader;
51  
52    /** Meta block index reader -- always single level */
53    protected HFileBlockIndex.BlockIndexReader metaBlockIndexReader;
54  
55    protected final FixedFileTrailer trailer;
56  
57    /** Filled when we read in the trailer. */
58    protected final Compression.Algorithm compressAlgo;
59  
60    /**
61     * What kind of data block encoding should be used while reading, writing,
62     * and handling cache.
63     */
64    protected HFileDataBlockEncoder dataBlockEncoder =
65        NoOpDataBlockEncoder.INSTANCE;
66  
67    /** Last key in the file. Filled in when we read in the file info */
68    protected byte [] lastKey = null;
69  
70    /** Average key length read from file info */
71    protected int avgKeyLen = -1;
72  
73    /** Average value length read from file info */
74    protected int avgValueLen = -1;
75  
76    /** Key comparator */
77    protected KVComparator comparator = new KVComparator();
78  
79    /** Size of this file. */
80    protected final long fileSize;
81  
82    /** Block cache configuration. */
83    protected final CacheConfig cacheConf;
84  
85    /** Path of file */
86    protected final Path path;
87  
88    /** File name to be used for block names */
89    protected final String name;
90  
91    protected FileInfo fileInfo;
92  
93    /** The filesystem used for accesing data */
94    protected HFileSystem hfs;
95  
96    protected Configuration conf;
97  
98    protected AbstractHFileReader(Path path, FixedFileTrailer trailer,
99        final long fileSize, final CacheConfig cacheConf, final HFileSystem hfs,
100       final Configuration conf) {
101     this.trailer = trailer;
102     this.compressAlgo = trailer.getCompressionCodec();
103     this.cacheConf = cacheConf;
104     this.fileSize = fileSize;
105     this.path = path;
106     this.name = path.getName();
107     this.hfs = hfs;
108     this.conf = conf;
109   }
110 
111   @SuppressWarnings("serial")
112   public static class BlockIndexNotLoadedException
113       extends IllegalStateException {
114     public BlockIndexNotLoadedException() {
115       // Add a message in case anyone relies on it as opposed to class name.
116       super("Block index not loaded");
117     }
118   }
119 
120   protected String toStringFirstKey() {
121     return KeyValue.keyToString(getFirstKey());
122   }
123 
124   protected String toStringLastKey() {
125     return KeyValue.keyToString(getLastKey());
126   }
127 
128   public abstract boolean isFileInfoLoaded();
129 
130   @Override
131   public String toString() {
132     return "reader=" + path.toString() +
133         (!isFileInfoLoaded()? "":
134           ", compression=" + compressAlgo.getName() +
135           ", cacheConf=" + cacheConf +
136           ", firstKey=" + toStringFirstKey() +
137           ", lastKey=" + toStringLastKey()) +
138           ", avgKeyLen=" + avgKeyLen +
139           ", avgValueLen=" + avgValueLen +
140           ", entries=" + trailer.getEntryCount() +
141           ", length=" + fileSize;
142   }
143 
144   @Override
145   public long length() {
146     return fileSize;
147   }
148 
149   /**
150    * Create a Scanner on this file. No seeks or reads are done on creation. Call
151    * {@link HFileScanner#seekTo(byte[])} to position an start the read. There is
152    * nothing to clean up in a Scanner. Letting go of your references to the
153    * scanner is sufficient. NOTE: Do not use this overload of getScanner for
154    * compactions.
155    *
156    * @param cacheBlocks True if we should cache blocks read in by this scanner.
157    * @param pread Use positional read rather than seek+read if true (pread is
158    *          better for random reads, seek+read is better scanning).
159    * @return Scanner on this file.
160    */
161   @Override
162   public HFileScanner getScanner(boolean cacheBlocks, final boolean pread) {
163     return getScanner(cacheBlocks, pread, false);
164   }
165 
166   /**
167    * @return the first key in the file. May be null if file has no entries. Note
168    *         that this is not the first row key, but rather the byte form of the
169    *         first KeyValue.
170    */
171   @Override
172   public byte [] getFirstKey() {
173     if (dataBlockIndexReader == null) {
174       throw new BlockIndexNotLoadedException();
175     }
176     return dataBlockIndexReader.isEmpty() ? null
177         : dataBlockIndexReader.getRootBlockKey(0);
178   }
179 
180   /**
181    * TODO left from {@link HFile} version 1: move this to StoreFile after Ryan's
182    * patch goes in to eliminate {@link KeyValue} here.
183    *
184    * @return the first row key, or null if the file is empty.
185    */
186   @Override
187   public byte[] getFirstRowKey() {
188     byte[] firstKey = getFirstKey();
189     if (firstKey == null)
190       return null;
191     return KeyValue.createKeyValueFromKey(firstKey).getRow();
192   }
193 
194   /**
195    * TODO left from {@link HFile} version 1: move this to StoreFile after
196    * Ryan's patch goes in to eliminate {@link KeyValue} here.
197    *
198    * @return the last row key, or null if the file is empty.
199    */
200   @Override
201   public byte[] getLastRowKey() {
202     byte[] lastKey = getLastKey();
203     if (lastKey == null)
204       return null;
205     return KeyValue.createKeyValueFromKey(lastKey).getRow();
206   }
207 
208   /** @return number of KV entries in this HFile */
209   @Override
210   public long getEntries() {
211     return trailer.getEntryCount();
212   }
213 
214   /** @return comparator */
215   @Override
216   public KVComparator getComparator() {
217     return comparator;
218   }
219 
220   /** @return compression algorithm */
221   @Override
222   public Compression.Algorithm getCompressionAlgorithm() {
223     return compressAlgo;
224   }
225 
226   /**
227    * @return the total heap size of data and meta block indexes in bytes. Does
228    *         not take into account non-root blocks of a multilevel data index.
229    */
230   public long indexSize() {
231     return (dataBlockIndexReader != null ? dataBlockIndexReader.heapSize() : 0)
232         + ((metaBlockIndexReader != null) ? metaBlockIndexReader.heapSize()
233             : 0);
234   }
235 
236   @Override
237   public String getName() {
238     return name;
239   }
240 
241   @Override
242   public HFileBlockIndex.BlockIndexReader getDataBlockIndexReader() {
243     return dataBlockIndexReader;
244   }
245 
246   @Override
247   public FixedFileTrailer getTrailer() {
248     return trailer;
249   }
250 
251   @Override
252   public FileInfo loadFileInfo() throws IOException {
253     return fileInfo;
254   }
255 
256   /**
257    * An exception thrown when an operation requiring a scanner to be seeked
258    * is invoked on a scanner that is not seeked.
259    */
260   @SuppressWarnings("serial")
261   public static class NotSeekedException extends IllegalStateException {
262     public NotSeekedException() {
263       super("Not seeked to a key/value");
264     }
265   }
266 
267   protected static abstract class Scanner implements HFileScanner {
268     protected ByteBuffer blockBuffer;
269 
270     protected boolean cacheBlocks;
271     protected final boolean pread;
272     protected final boolean isCompaction;
273 
274     protected int currKeyLen;
275     protected int currValueLen;
276     protected int currMemstoreTSLen;
277     protected long currMemstoreTS;
278 
279     protected int blockFetches;
280 
281     protected final HFile.Reader reader;
282 
283     public Scanner(final HFile.Reader reader, final boolean cacheBlocks,
284         final boolean pread, final boolean isCompaction) {
285       this.reader = reader;
286       this.cacheBlocks = cacheBlocks;
287       this.pread = pread;
288       this.isCompaction = isCompaction;
289     }
290 
291     @Override
292     public boolean isSeeked(){
293       return blockBuffer != null;
294     }
295 
296     @Override
297     public String toString() {
298       return "HFileScanner for reader " + String.valueOf(getReader());
299     }
300 
301     protected void assertSeeked() {
302       if (!isSeeked())
303         throw new NotSeekedException();
304     }
305 
306     @Override
307     public int seekTo(byte[] key) throws IOException {
308       return seekTo(key, 0, key.length);
309     }
310     
311     @Override
312     public boolean seekBefore(byte[] key) throws IOException {
313       return seekBefore(key, 0, key.length);
314     }
315     
316     @Override
317     public int reseekTo(byte[] key) throws IOException {
318       return reseekTo(key, 0, key.length);
319     }
320 
321     @Override
322     public HFile.Reader getReader() {
323       return reader;
324     }
325   }
326 
327   /** For testing */
328   abstract HFileBlock.FSReader getUncachedBlockReader();
329 
330   public Path getPath() {
331     return path;
332   }
333 
334   @Override
335   public DataBlockEncoding getDataBlockEncoding() {
336     return dataBlockEncoder.getDataBlockEncoding();
337   }
338 
339   public abstract int getMajorVersion();
340 
341   @Override
342   public Configuration getConf() {
343     return conf;
344   }
345 
346   @Override
347   public void setConf(Configuration conf) {
348     this.conf = conf;
349   }
350 }