View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.io.hfile;
19  
20  import java.io.IOException;
21  import java.security.Key;
22  import java.security.KeyException;
23  
24  import org.apache.commons.logging.Log;
25  import org.apache.commons.logging.LogFactory;
26  import org.apache.hadoop.classification.InterfaceAudience;
27  import org.apache.hadoop.conf.Configuration;
28  import org.apache.hadoop.fs.Path;
29  import org.apache.hadoop.hbase.HConstants;
30  import org.apache.hadoop.hbase.fs.HFileSystem;
31  import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
32  import org.apache.hadoop.hbase.io.crypto.Cipher;
33  import org.apache.hadoop.hbase.io.crypto.Encryption;
34  import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
35  import org.apache.hadoop.hbase.security.EncryptionUtil;
36  import org.apache.hadoop.hbase.security.User;
37  import org.apache.hadoop.hbase.util.ByteBufferUtils;
38  import org.apache.hadoop.hbase.util.Bytes;
39  import org.apache.hadoop.io.WritableUtils;
40  
41  /**
42   * {@link HFile} reader for version 3.
43   */
44  @InterfaceAudience.Private
45  public class HFileReaderV3 extends HFileReaderV2 {
46  
47    private static final Log LOG = LogFactory.getLog(HFileReaderV3.class);
48  
49    public static final int MAX_MINOR_VERSION = 0;
50  
51    /**
52     * Opens a HFile. You must load the index before you can use it by calling
53     * {@link #loadFileInfo()}.
54     * @param path
55     *          Path to HFile.
56     * @param trailer
57     *          File trailer.
58     * @param fsdis
59     *          input stream.
60     * @param size
61     *          Length of the stream.
62     * @param cacheConf
63     *          Cache configuration.
64     * @param hfs
65     *          The file system.
66     * @param conf
67     *          Configuration
68     */
69    public HFileReaderV3(Path path, FixedFileTrailer trailer, final FSDataInputStreamWrapper fsdis,
70        final long size, final CacheConfig cacheConf, final HFileSystem hfs,
71        final Configuration conf) throws IOException {
72      super(path, trailer, fsdis, size, cacheConf, hfs, conf);
73      byte[] tmp = fileInfo.get(FileInfo.MAX_TAGS_LEN);
74      // max tag length is not present in the HFile means tags were not at all written to file.
75      if (tmp != null) {
76        hfileContext.setIncludesTags(true);
77        tmp = fileInfo.get(FileInfo.TAGS_COMPRESSED);
78        if (tmp != null && Bytes.toBoolean(tmp)) {
79          hfileContext.setCompressTags(true);
80        }
81      }
82    }
83  
84    @Override
85    protected HFileContext createHFileContext(FSDataInputStreamWrapper fsdis, long fileSize,
86        HFileSystem hfs, Path path, FixedFileTrailer trailer) throws IOException {
87      trailer.expectMajorVersion(3);
88      HFileContextBuilder builder = new HFileContextBuilder()
89        .withIncludesMvcc(this.includesMemstoreTS)
90        .withHBaseCheckSum(true)
91        .withCompression(this.compressAlgo);
92  
93      // Check for any key material available
94      byte[] keyBytes = trailer.getEncryptionKey();
95      if (keyBytes != null) {
96        Encryption.Context cryptoContext = Encryption.newContext(conf);
97        Key key;
98        String masterKeyName = conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY,
99          User.getCurrent().getShortName());
100       try {
101         // First try the master key
102         key = EncryptionUtil.unwrapKey(conf, masterKeyName, keyBytes);
103       } catch (KeyException e) {
104         // If the current master key fails to unwrap, try the alternate, if
105         // one is configured
106         if (LOG.isDebugEnabled()) {
107           LOG.debug("Unable to unwrap key with current master key '" + masterKeyName + "'");
108         }
109         String alternateKeyName =
110           conf.get(HConstants.CRYPTO_MASTERKEY_ALTERNATE_NAME_CONF_KEY);
111         if (alternateKeyName != null) {
112           try {
113             key = EncryptionUtil.unwrapKey(conf, alternateKeyName, keyBytes);
114           } catch (KeyException ex) {
115             throw new IOException(ex);
116           }
117         } else {
118           throw new IOException(e);
119         }
120       }
121       // Use the algorithm the key wants
122       Cipher cipher = Encryption.getCipher(conf, key.getAlgorithm());
123       if (cipher == null) {
124         throw new IOException("Cipher '" + key.getAlgorithm() + "' is not available");
125       }
126       cryptoContext.setCipher(cipher);
127       cryptoContext.setKey(key);
128       builder.withEncryptionContext(cryptoContext);
129     }
130 
131     HFileContext context = builder.build();
132 
133     if (LOG.isTraceEnabled()) {
134       LOG.trace("Reader" + (path != null ? " for " + path : "" ) +
135         " initialized with cacheConf: " + cacheConf +
136         " comparator: " + comparator.getClass().getSimpleName() +
137         " fileContext: " + context);
138     }
139 
140     return context;
141   }
142 
143   /**
144    * Create a Scanner on this file. No seeks or reads are done on creation. Call
145    * {@link HFileScanner#seekTo(byte[])} to position an start the read. There is
146    * nothing to clean up in a Scanner. Letting go of your references to the
147    * scanner is sufficient.
148    * @param cacheBlocks
149    *          True if we should cache blocks read in by this scanner.
150    * @param pread
151    *          Use positional read rather than seek+read if true (pread is better
152    *          for random reads, seek+read is better scanning).
153    * @param isCompaction
154    *          is scanner being used for a compaction?
155    * @return Scanner on this file.
156    */
157   @Override
158   public HFileScanner getScanner(boolean cacheBlocks, final boolean pread,
159       final boolean isCompaction) {
160     if (dataBlockEncoder.useEncodedScanner()) {
161       return new EncodedScannerV3(this, cacheBlocks, pread, isCompaction, this.hfileContext);
162     }
163     return new ScannerV3(this, cacheBlocks, pread, isCompaction);
164   }
165 
166   /**
167    * Implementation of {@link HFileScanner} interface.
168    */
169   protected static class ScannerV3 extends ScannerV2 {
170 
171     private HFileReaderV3 reader;
172     private int currTagsLen;
173 
174     public ScannerV3(HFileReaderV3 r, boolean cacheBlocks, final boolean pread,
175         final boolean isCompaction) {
176       super(r, cacheBlocks, pread, isCompaction);
177       this.reader = r;
178     }
179 
180     @Override
181     protected int getCellBufSize() {
182       int kvBufSize = super.getCellBufSize();
183       if (reader.hfileContext.isIncludesTags()) {
184         kvBufSize += Bytes.SIZEOF_SHORT + currTagsLen;
185       }
186       return kvBufSize;
187     }
188 
189     protected void setNonSeekedState() {
190       super.setNonSeekedState();
191       currTagsLen = 0;
192     }
193 
194     @Override
195     protected int getNextCellStartPosition() {
196       int nextKvPos = super.getNextCellStartPosition();
197       if (reader.hfileContext.isIncludesTags()) {
198         nextKvPos += Bytes.SIZEOF_SHORT + currTagsLen;
199       }
200       return nextKvPos;
201     }
202 
203     protected void readKeyValueLen() {
204       blockBuffer.mark();
205       currKeyLen = blockBuffer.getInt();
206       currValueLen = blockBuffer.getInt();
207       if (currKeyLen < 0 || currValueLen < 0 || currKeyLen > blockBuffer.limit()
208           || currValueLen > blockBuffer.limit()) {
209         throw new IllegalStateException("Invalid currKeyLen " + currKeyLen + " or currValueLen "
210             + currValueLen + ". Block offset: "
211             + block.getOffset() + ", block length: " + blockBuffer.limit() + ", position: "
212             + blockBuffer.position() + " (without header).");
213       }
214       ByteBufferUtils.skip(blockBuffer, currKeyLen + currValueLen);
215       if (reader.hfileContext.isIncludesTags()) {
216         currTagsLen = blockBuffer.getShort();
217         if (currTagsLen < 0 || currTagsLen > blockBuffer.limit()) {
218           throw new IllegalStateException("Invalid currTagsLen " + currTagsLen + ". Block offset: "
219               + block.getOffset() + ", block length: " + blockBuffer.limit() + ", position: "
220               + blockBuffer.position() + " (without header).");
221         }
222         ByteBufferUtils.skip(blockBuffer, currTagsLen);
223       }
224       readMvccVersion();
225       blockBuffer.reset();
226     }
227 
228     /**
229      * Within a loaded block, seek looking for the last key that is smaller than
230      * (or equal to?) the key we are interested in.
231      * A note on the seekBefore: if you have seekBefore = true, AND the first
232      * key in the block = key, then you'll get thrown exceptions. The caller has
233      * to check for that case and load the previous block as appropriate.
234      * @param key
235      *          the key to find
236      * @param seekBefore
237      *          find the key before the given key in case of exact match.
238      * @param offset
239      *          Offset to find the key in the given bytebuffer
240      * @param length
241      *          Length of the key to be found
242      * @return 0 in case of an exact key match, 1 in case of an inexact match,
243      *         -2 in case of an inexact match and furthermore, the input key
244      *         less than the first key of current block(e.g. using a faked index
245      *         key)
246      */
247     protected int blockSeek(byte[] key, int offset, int length, boolean seekBefore) {
248       int klen, vlen, tlen = 0;
249       long memstoreTS = 0;
250       int memstoreTSLen = 0;
251       int lastKeyValueSize = -1;
252       do {
253         blockBuffer.mark();
254         klen = blockBuffer.getInt();
255         vlen = blockBuffer.getInt();
256         if (klen < 0 || vlen < 0 || klen > blockBuffer.limit()
257             || vlen > blockBuffer.limit()) {
258           throw new IllegalStateException("Invalid klen " + klen + " or vlen "
259               + vlen + ". Block offset: "
260               + block.getOffset() + ", block length: " + blockBuffer.limit() + ", position: "
261               + blockBuffer.position() + " (without header).");
262         }
263         ByteBufferUtils.skip(blockBuffer, klen + vlen);
264         if (reader.hfileContext.isIncludesTags()) {
265           tlen = blockBuffer.getShort();
266           if (tlen < 0 || tlen > blockBuffer.limit()) {
267             throw new IllegalStateException("Invalid tlen " + tlen + ". Block offset: "
268                 + block.getOffset() + ", block length: " + blockBuffer.limit() + ", position: "
269                 + blockBuffer.position() + " (without header).");
270           }
271           ByteBufferUtils.skip(blockBuffer, tlen);
272         }
273         if (this.reader.shouldIncludeMemstoreTS()) {
274           if (this.reader.decodeMemstoreTS) {
275             try {
276               memstoreTS = Bytes.readVLong(blockBuffer.array(), blockBuffer.arrayOffset()
277                   + blockBuffer.position());
278               memstoreTSLen = WritableUtils.getVIntSize(memstoreTS);
279             } catch (Exception e) {
280               throw new RuntimeException("Error reading memstore timestamp", e);
281             }
282           } else {
283             memstoreTS = 0;
284             memstoreTSLen = 1;
285           }
286         }
287         blockBuffer.reset();
288         int keyOffset = blockBuffer.arrayOffset() + blockBuffer.position() + (Bytes.SIZEOF_INT * 2);
289         int comp = reader.getComparator().compare(key, offset, length, blockBuffer.array(),
290             keyOffset, klen);
291 
292         if (comp == 0) {
293           if (seekBefore) {
294             if (lastKeyValueSize < 0) {
295               throw new IllegalStateException("blockSeek with seekBefore "
296                   + "at the first key of the block: key=" + Bytes.toStringBinary(key)
297                   + ", blockOffset=" + block.getOffset() + ", onDiskSize="
298                   + block.getOnDiskSizeWithHeader());
299             }
300             blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
301             readKeyValueLen();
302             return 1; // non exact match.
303           }
304           currKeyLen = klen;
305           currValueLen = vlen;
306           currTagsLen = tlen;
307           if (this.reader.shouldIncludeMemstoreTS()) {
308             currMemstoreTS = memstoreTS;
309             currMemstoreTSLen = memstoreTSLen;
310           }
311           return 0; // indicate exact match
312         } else if (comp < 0) {
313           if (lastKeyValueSize > 0)
314             blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
315           readKeyValueLen();
316           if (lastKeyValueSize == -1 && blockBuffer.position() == 0) {
317             return HConstants.INDEX_KEY_MAGIC;
318           }
319           return 1;
320         }
321 
322         // The size of this key/value tuple, including key/value length fields.
323         lastKeyValueSize = klen + vlen + memstoreTSLen + KEY_VALUE_LEN_SIZE;
324         // include tag length also if tags included with KV
325         if (reader.hfileContext.isIncludesTags()) {
326           lastKeyValueSize += tlen + Bytes.SIZEOF_SHORT;
327         }
328         blockBuffer.position(blockBuffer.position() + lastKeyValueSize);
329       } while (blockBuffer.remaining() > 0);
330 
331       // Seek to the last key we successfully read. This will happen if this is
332       // the last key/value pair in the file, in which case the following call
333       // to next() has to return false.
334       blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
335       readKeyValueLen();
336       return 1; // didn't exactly find it.
337     }
338 
339   }
340 
341   /**
342    * ScannerV3 that operates on encoded data blocks.
343    */
344   protected static class EncodedScannerV3 extends EncodedScannerV2 {
345     public EncodedScannerV3(HFileReaderV3 reader, boolean cacheBlocks, boolean pread,
346         boolean isCompaction, HFileContext context) {
347       super(reader, cacheBlocks, pread, isCompaction, context);
348     }
349   }
350 
351   @Override
352   public int getMajorVersion() {
353     return 3;
354   }
355 }