View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.io.hfile;
19  
20  import java.io.IOException;
21  import java.security.Key;
22  import java.security.KeyException;
23  
24  import org.apache.commons.logging.Log;
25  import org.apache.commons.logging.LogFactory;
26  import org.apache.hadoop.hbase.classification.InterfaceAudience;
27  import org.apache.hadoop.conf.Configuration;
28  import org.apache.hadoop.fs.Path;
29  import org.apache.hadoop.hbase.Cell;
30  import org.apache.hadoop.hbase.CellUtil;
31  import org.apache.hadoop.hbase.HConstants;
32  import org.apache.hadoop.hbase.KeyValue;
33  import org.apache.hadoop.hbase.fs.HFileSystem;
34  import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
35  import org.apache.hadoop.hbase.io.crypto.Cipher;
36  import org.apache.hadoop.hbase.io.crypto.Encryption;
37  import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
38  import org.apache.hadoop.hbase.security.EncryptionUtil;
39  import org.apache.hadoop.hbase.security.User;
40  import org.apache.hadoop.hbase.util.ByteBufferUtils;
41  import org.apache.hadoop.hbase.util.Bytes;
42  import org.apache.hadoop.io.WritableUtils;
43  
44  /**
45   * {@link HFile} reader for version 3.
46   */
47  @InterfaceAudience.Private
48  public class HFileReaderV3 extends HFileReaderV2 {
49  
50    private static final Log LOG = LogFactory.getLog(HFileReaderV3.class);
51  
52    public static final int MAX_MINOR_VERSION = 0;
53  
54    /**
55     * Opens a HFile. You must load the index before you can use it by calling
56     * {@link #loadFileInfo()}.
57     * @param path
58     *          Path to HFile.
59     * @param trailer
60     *          File trailer.
61     * @param fsdis
62     *          input stream.
63     * @param size
64     *          Length of the stream.
65     * @param cacheConf
66     *          Cache configuration.
67     * @param hfs
68     *          The file system.
69     * @param conf
70     *          Configuration
71     */
72    public HFileReaderV3(final Path path, FixedFileTrailer trailer, final FSDataInputStreamWrapper fsdis,
73        final long size, final CacheConfig cacheConf, final HFileSystem hfs,
74        final Configuration conf) throws IOException {
75      super(path, trailer, fsdis, size, cacheConf, hfs, conf);
76      byte[] tmp = fileInfo.get(FileInfo.MAX_TAGS_LEN);
77      // max tag length is not present in the HFile means tags were not at all written to file.
78      if (tmp != null) {
79        hfileContext.setIncludesTags(true);
80        tmp = fileInfo.get(FileInfo.TAGS_COMPRESSED);
81        if (tmp != null && Bytes.toBoolean(tmp)) {
82          hfileContext.setCompressTags(true);
83        }
84      }
85    }
86  
87    @Override
88    protected HFileContext createHFileContext(FSDataInputStreamWrapper fsdis, long fileSize,
89        HFileSystem hfs, Path path, FixedFileTrailer trailer) throws IOException {
90      trailer.expectMajorVersion(3);
91      HFileContextBuilder builder = new HFileContextBuilder()
92        .withIncludesMvcc(this.includesMemstoreTS)
93        .withHBaseCheckSum(true)
94        .withCompression(this.compressAlgo);
95  
96      // Check for any key material available
97      byte[] keyBytes = trailer.getEncryptionKey();
98      if (keyBytes != null) {
99        Encryption.Context cryptoContext = Encryption.newContext(conf);
100       Key key;
101       String masterKeyName = conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY,
102         User.getCurrent().getShortName());
103       try {
104         // First try the master key
105         key = EncryptionUtil.unwrapKey(conf, masterKeyName, keyBytes);
106       } catch (KeyException e) {
107         // If the current master key fails to unwrap, try the alternate, if
108         // one is configured
109         if (LOG.isDebugEnabled()) {
110           LOG.debug("Unable to unwrap key with current master key '" + masterKeyName + "'");
111         }
112         String alternateKeyName =
113           conf.get(HConstants.CRYPTO_MASTERKEY_ALTERNATE_NAME_CONF_KEY);
114         if (alternateKeyName != null) {
115           try {
116             key = EncryptionUtil.unwrapKey(conf, alternateKeyName, keyBytes);
117           } catch (KeyException ex) {
118             throw new IOException(ex);
119           }
120         } else {
121           throw new IOException(e);
122         }
123       }
124       // Use the algorithm the key wants
125       Cipher cipher = Encryption.getCipher(conf, key.getAlgorithm());
126       if (cipher == null) {
127         throw new IOException("Cipher '" + key.getAlgorithm() + "' is not available");
128       }
129       cryptoContext.setCipher(cipher);
130       cryptoContext.setKey(key);
131       builder.withEncryptionContext(cryptoContext);
132     }
133 
134     HFileContext context = builder.build();
135 
136     if (LOG.isTraceEnabled()) {
137       LOG.trace("Reader" + (path != null ? " for " + path : "" ) +
138         " initialized with cacheConf: " + cacheConf +
139         " comparator: " + comparator.getClass().getSimpleName() +
140         " fileContext: " + context);
141     }
142 
143     return context;
144   }
145 
146   /**
147    * Create a Scanner on this file. No seeks or reads are done on creation. Call
148    * {@link HFileScanner#seekTo(byte[])} to position an start the read. There is
149    * nothing to clean up in a Scanner. Letting go of your references to the
150    * scanner is sufficient.
151    * @param cacheBlocks
152    *          True if we should cache blocks read in by this scanner.
153    * @param pread
154    *          Use positional read rather than seek+read if true (pread is better
155    *          for random reads, seek+read is better scanning).
156    * @param isCompaction
157    *          is scanner being used for a compaction?
158    * @return Scanner on this file.
159    */
160   @Override
161   public HFileScanner getScanner(boolean cacheBlocks, final boolean pread,
162       final boolean isCompaction) {
163     if (dataBlockEncoder.useEncodedScanner()) {
164       return new EncodedScannerV3(this, cacheBlocks, pread, isCompaction, this.hfileContext);
165     }
166     return new ScannerV3(this, cacheBlocks, pread, isCompaction);
167   }
168 
169   /**
170    * Implementation of {@link HFileScanner} interface.
171    */
172   protected static class ScannerV3 extends ScannerV2 {
173 
174     private HFileReaderV3 reader;
175     private int currTagsLen;
176 
177     public ScannerV3(HFileReaderV3 r, boolean cacheBlocks, final boolean pread,
178         final boolean isCompaction) {
179       super(r, cacheBlocks, pread, isCompaction);
180       this.reader = r;
181     }
182 
183     @Override
184     protected int getCellBufSize() {
185       int kvBufSize = super.getCellBufSize();
186       if (reader.hfileContext.isIncludesTags()) {
187         kvBufSize += Bytes.SIZEOF_SHORT + currTagsLen;
188       }
189       return kvBufSize;
190     }
191 
192     protected void setNonSeekedState() {
193       super.setNonSeekedState();
194       currTagsLen = 0;
195     }
196 
197     @Override
198     protected int getNextCellStartPosition() {
199       int nextKvPos = super.getNextCellStartPosition();
200       if (reader.hfileContext.isIncludesTags()) {
201         nextKvPos += Bytes.SIZEOF_SHORT + currTagsLen;
202       }
203       return nextKvPos;
204     }
205 
206     protected void readKeyValueLen() {
207       blockBuffer.mark();
208       currKeyLen = blockBuffer.getInt();
209       currValueLen = blockBuffer.getInt();
210       if (currKeyLen < 0 || currValueLen < 0 || currKeyLen > blockBuffer.limit()
211           || currValueLen > blockBuffer.limit()) {
212         throw new IllegalStateException("Invalid currKeyLen " + currKeyLen + " or currValueLen "
213             + currValueLen + ". Block offset: "
214             + block.getOffset() + ", block length: " + blockBuffer.limit() + ", position: "
215             + blockBuffer.position() + " (without header).");
216       }
217       ByteBufferUtils.skip(blockBuffer, currKeyLen + currValueLen);
218       if (reader.hfileContext.isIncludesTags()) {
219         // Read short as unsigned, high byte first
220         currTagsLen = ((blockBuffer.get() & 0xff) << 8) ^ (blockBuffer.get() & 0xff);
221         if (currTagsLen < 0 || currTagsLen > blockBuffer.limit()) {
222           throw new IllegalStateException("Invalid currTagsLen " + currTagsLen + ". Block offset: "
223               + block.getOffset() + ", block length: " + blockBuffer.limit() + ", position: "
224               + blockBuffer.position() + " (without header).");
225         }
226         ByteBufferUtils.skip(blockBuffer, currTagsLen);
227       }
228       readMvccVersion();
229       blockBuffer.reset();
230     }
231 
232     /**
233      * Within a loaded block, seek looking for the last key that is smaller than
234      * (or equal to?) the key we are interested in.
235      * A note on the seekBefore: if you have seekBefore = true, AND the first
236      * key in the block = key, then you'll get thrown exceptions. The caller has
237      * to check for that case and load the previous block as appropriate.
238      * @param key
239      *          the key to find
240      * @param seekBefore
241      *          find the key before the given key in case of exact match.
242      * @return 0 in case of an exact key match, 1 in case of an inexact match,
243      *         -2 in case of an inexact match and furthermore, the input key
244      *         less than the first key of current block(e.g. using a faked index
245      *         key)
246      */
247     @Override
248     protected int blockSeek(Cell key, boolean seekBefore) {
249       int klen, vlen, tlen = 0;
250       long memstoreTS = 0;
251       int memstoreTSLen = 0;
252       int lastKeyValueSize = -1;
253       KeyValue.KeyOnlyKeyValue keyOnlyKv = new KeyValue.KeyOnlyKeyValue();
254       do {
255         blockBuffer.mark();
256         klen = blockBuffer.getInt();
257         vlen = blockBuffer.getInt();
258         if (klen < 0 || vlen < 0 || klen > blockBuffer.limit()
259             || vlen > blockBuffer.limit()) {
260           throw new IllegalStateException("Invalid klen " + klen + " or vlen "
261               + vlen + ". Block offset: "
262               + block.getOffset() + ", block length: " + blockBuffer.limit() + ", position: "
263               + blockBuffer.position() + " (without header).");
264         }
265         ByteBufferUtils.skip(blockBuffer, klen + vlen);
266         if (reader.hfileContext.isIncludesTags()) {
267           // Read short as unsigned, high byte first
268           tlen = ((blockBuffer.get() & 0xff) << 8) ^ (blockBuffer.get() & 0xff);
269           if (tlen < 0 || tlen > blockBuffer.limit()) {
270             throw new IllegalStateException("Invalid tlen " + tlen + ". Block offset: "
271                 + block.getOffset() + ", block length: " + blockBuffer.limit() + ", position: "
272                 + blockBuffer.position() + " (without header).");
273           }
274           ByteBufferUtils.skip(blockBuffer, tlen);
275         }
276         if (this.reader.shouldIncludeMemstoreTS()) {
277           if (this.reader.decodeMemstoreTS) {
278             try {
279               memstoreTS = Bytes.readVLong(blockBuffer.array(), blockBuffer.arrayOffset()
280                   + blockBuffer.position());
281               memstoreTSLen = WritableUtils.getVIntSize(memstoreTS);
282             } catch (Exception e) {
283               throw new RuntimeException("Error reading memstore timestamp", e);
284             }
285           } else {
286             memstoreTS = 0;
287             memstoreTSLen = 1;
288           }
289         }
290         blockBuffer.reset();
291         int keyOffset = blockBuffer.arrayOffset() + blockBuffer.position() + (Bytes.SIZEOF_INT * 2);
292         keyOnlyKv.setKey(blockBuffer.array(), keyOffset, klen);
293         int comp = reader.getComparator().compareOnlyKeyPortion(key, keyOnlyKv);
294 
295         if (comp == 0) {
296           if (seekBefore) {
297             if (lastKeyValueSize < 0) {
298               throw new IllegalStateException("blockSeek with seekBefore "
299                   + "at the first key of the block: key="
300                   + CellUtil.getCellKeyAsString(key)
301                   + ", blockOffset=" + block.getOffset() + ", onDiskSize="
302                   + block.getOnDiskSizeWithHeader());
303             }
304             blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
305             readKeyValueLen();
306             return 1; // non exact match.
307           }
308           currKeyLen = klen;
309           currValueLen = vlen;
310           currTagsLen = tlen;
311           if (this.reader.shouldIncludeMemstoreTS()) {
312             currMemstoreTS = memstoreTS;
313             currMemstoreTSLen = memstoreTSLen;
314           }
315           return 0; // indicate exact match
316         } else if (comp < 0) {
317           if (lastKeyValueSize > 0)
318             blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
319           readKeyValueLen();
320           if (lastKeyValueSize == -1 && blockBuffer.position() == 0) {
321             return HConstants.INDEX_KEY_MAGIC;
322           }
323           return 1;
324         }
325 
326         // The size of this key/value tuple, including key/value length fields.
327         lastKeyValueSize = klen + vlen + memstoreTSLen + KEY_VALUE_LEN_SIZE;
328         // include tag length also if tags included with KV
329         if (reader.hfileContext.isIncludesTags()) {
330           lastKeyValueSize += tlen + Bytes.SIZEOF_SHORT;
331         }
332         blockBuffer.position(blockBuffer.position() + lastKeyValueSize);
333       } while (blockBuffer.remaining() > 0);
334 
335       // Seek to the last key we successfully read. This will happen if this is
336       // the last key/value pair in the file, in which case the following call
337       // to next() has to return false.
338       blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
339       readKeyValueLen();
340       return 1; // didn't exactly find it.
341     }
342   }
343 
344   /**
345    * ScannerV3 that operates on encoded data blocks.
346    */
347   protected static class EncodedScannerV3 extends EncodedScannerV2 {
348     public EncodedScannerV3(HFileReaderV3 reader, boolean cacheBlocks, boolean pread,
349         boolean isCompaction, HFileContext context) {
350       super(reader, cacheBlocks, pread, isCompaction, context);
351     }
352   }
353 
354   @Override
355   public int getMajorVersion() {
356     return 3;
357   }
358 }