View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.io.hfile;
19  
20  import java.io.IOException;
21  import java.security.Key;
22  import java.security.KeyException;
23  
24  import org.apache.commons.logging.Log;
25  import org.apache.commons.logging.LogFactory;
26  import org.apache.hadoop.classification.InterfaceAudience;
27  import org.apache.hadoop.conf.Configuration;
28  import org.apache.hadoop.fs.Path;
29  import org.apache.hadoop.hbase.Cell;
30  import org.apache.hadoop.hbase.HConstants;
31  import org.apache.hadoop.hbase.KeyValue;
32  import org.apache.hadoop.hbase.KeyValueUtil;
33  import org.apache.hadoop.hbase.fs.HFileSystem;
34  import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
35  import org.apache.hadoop.hbase.io.crypto.Cipher;
36  import org.apache.hadoop.hbase.io.crypto.Encryption;
37  import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
38  import org.apache.hadoop.hbase.security.EncryptionUtil;
39  import org.apache.hadoop.hbase.security.User;
40  import org.apache.hadoop.hbase.util.ByteBufferUtils;
41  import org.apache.hadoop.hbase.util.Bytes;
42  import org.apache.hadoop.io.WritableUtils;
43  
44  /**
45   * {@link HFile} reader for version 3.
46   */
47  @InterfaceAudience.Private
48  public class HFileReaderV3 extends HFileReaderV2 {
49  
50    private static final Log LOG = LogFactory.getLog(HFileReaderV3.class);
51  
52    public static final int MAX_MINOR_VERSION = 0;
53  
54    /**
55     * Opens a HFile. You must load the index before you can use it by calling
56     * {@link #loadFileInfo()}.
57     * @param path
58     *          Path to HFile.
59     * @param trailer
60     *          File trailer.
61     * @param fsdis
62     *          input stream.
63     * @param size
64     *          Length of the stream.
65     * @param cacheConf
66     *          Cache configuration.
67     * @param hfs
68     *          The file system.
69     * @param conf
70     *          Configuration
71     */
72    public HFileReaderV3(final Path path, FixedFileTrailer trailer, final FSDataInputStreamWrapper fsdis,
73        final long size, final CacheConfig cacheConf, final HFileSystem hfs,
74        final Configuration conf) throws IOException {
75      super(path, trailer, fsdis, size, cacheConf, hfs, conf);
76      byte[] tmp = fileInfo.get(FileInfo.MAX_TAGS_LEN);
77      // max tag length is not present in the HFile means tags were not at all written to file.
78      if (tmp != null) {
79        hfileContext.setIncludesTags(true);
80        tmp = fileInfo.get(FileInfo.TAGS_COMPRESSED);
81        if (tmp != null && Bytes.toBoolean(tmp)) {
82          hfileContext.setCompressTags(true);
83        }
84      }
85    }
86  
87    @Override
88    protected HFileContext createHFileContext(FSDataInputStreamWrapper fsdis, long fileSize,
89        HFileSystem hfs, Path path, FixedFileTrailer trailer) throws IOException {
90      trailer.expectMajorVersion(3);
91      HFileContextBuilder builder = new HFileContextBuilder()
92        .withIncludesMvcc(this.includesMemstoreTS)
93        .withHBaseCheckSum(true)
94        .withCompression(this.compressAlgo);
95  
96      // Check for any key material available
97      byte[] keyBytes = trailer.getEncryptionKey();
98      if (keyBytes != null) {
99        Encryption.Context cryptoContext = Encryption.newContext(conf);
100       Key key;
101       String masterKeyName = conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY,
102         User.getCurrent().getShortName());
103       try {
104         // First try the master key
105         key = EncryptionUtil.unwrapKey(conf, masterKeyName, keyBytes);
106       } catch (KeyException e) {
107         // If the current master key fails to unwrap, try the alternate, if
108         // one is configured
109         if (LOG.isDebugEnabled()) {
110           LOG.debug("Unable to unwrap key with current master key '" + masterKeyName + "'");
111         }
112         String alternateKeyName =
113           conf.get(HConstants.CRYPTO_MASTERKEY_ALTERNATE_NAME_CONF_KEY);
114         if (alternateKeyName != null) {
115           try {
116             key = EncryptionUtil.unwrapKey(conf, alternateKeyName, keyBytes);
117           } catch (KeyException ex) {
118             throw new IOException(ex);
119           }
120         } else {
121           throw new IOException(e);
122         }
123       }
124       // Use the algorithm the key wants
125       Cipher cipher = Encryption.getCipher(conf, key.getAlgorithm());
126       if (cipher == null) {
127         throw new IOException("Cipher '" + key.getAlgorithm() + "' is not available");
128       }
129       cryptoContext.setCipher(cipher);
130       cryptoContext.setKey(key);
131       builder.withEncryptionContext(cryptoContext);
132     }
133 
134     HFileContext context = builder.build();
135 
136     if (LOG.isTraceEnabled()) {
137       LOG.trace("Reader" + (path != null ? " for " + path : "" ) +
138         " initialized with cacheConf: " + cacheConf +
139         " comparator: " + comparator.getClass().getSimpleName() +
140         " fileContext: " + context);
141     }
142 
143     return context;
144   }
145 
146   /**
147    * Create a Scanner on this file. No seeks or reads are done on creation. Call
148    * {@link HFileScanner#seekTo(byte[])} to position an start the read. There is
149    * nothing to clean up in a Scanner. Letting go of your references to the
150    * scanner is sufficient.
151    * @param cacheBlocks
152    *          True if we should cache blocks read in by this scanner.
153    * @param pread
154    *          Use positional read rather than seek+read if true (pread is better
155    *          for random reads, seek+read is better scanning).
156    * @param isCompaction
157    *          is scanner being used for a compaction?
158    * @return Scanner on this file.
159    */
160   @Override
161   public HFileScanner getScanner(boolean cacheBlocks, final boolean pread,
162       final boolean isCompaction) {
163     if (dataBlockEncoder.useEncodedScanner()) {
164       return new EncodedScannerV3(this, cacheBlocks, pread, isCompaction, this.hfileContext);
165     }
166     return new ScannerV3(this, cacheBlocks, pread, isCompaction);
167   }
168 
169   /**
170    * Implementation of {@link HFileScanner} interface.
171    */
172   protected static class ScannerV3 extends ScannerV2 {
173 
174     private HFileReaderV3 reader;
175     private int currTagsLen;
176 
177     public ScannerV3(HFileReaderV3 r, boolean cacheBlocks, final boolean pread,
178         final boolean isCompaction) {
179       super(r, cacheBlocks, pread, isCompaction);
180       this.reader = r;
181     }
182 
183     @Override
184     protected int getCellBufSize() {
185       int kvBufSize = super.getCellBufSize();
186       if (reader.hfileContext.isIncludesTags()) {
187         kvBufSize += Bytes.SIZEOF_SHORT + currTagsLen;
188       }
189       return kvBufSize;
190     }
191 
192     protected void setNonSeekedState() {
193       super.setNonSeekedState();
194       currTagsLen = 0;
195     }
196 
197     @Override
198     protected int getNextCellStartPosition() {
199       int nextKvPos = super.getNextCellStartPosition();
200       if (reader.hfileContext.isIncludesTags()) {
201         nextKvPos += Bytes.SIZEOF_SHORT + currTagsLen;
202       }
203       return nextKvPos;
204     }
205 
206     protected void readKeyValueLen() {
207       blockBuffer.mark();
208       currKeyLen = blockBuffer.getInt();
209       currValueLen = blockBuffer.getInt();
210       if (currKeyLen < 0 || currValueLen < 0 || currKeyLen > blockBuffer.limit()
211           || currValueLen > blockBuffer.limit()) {
212         throw new IllegalStateException("Invalid currKeyLen " + currKeyLen + " or currValueLen "
213             + currValueLen + ". Block offset: "
214             + block.getOffset() + ", block length: " + blockBuffer.limit() + ", position: "
215             + blockBuffer.position() + " (without header).");
216       }
217       ByteBufferUtils.skip(blockBuffer, currKeyLen + currValueLen);
218       if (reader.hfileContext.isIncludesTags()) {
219         currTagsLen = blockBuffer.getShort();
220         if (currTagsLen < 0 || currTagsLen > blockBuffer.limit()) {
221           throw new IllegalStateException("Invalid currTagsLen " + currTagsLen + ". Block offset: "
222               + block.getOffset() + ", block length: " + blockBuffer.limit() + ", position: "
223               + blockBuffer.position() + " (without header).");
224         }
225         ByteBufferUtils.skip(blockBuffer, currTagsLen);
226       }
227       readMvccVersion();
228       blockBuffer.reset();
229     }
230 
231     /**
232      * Within a loaded block, seek looking for the last key that is smaller than
233      * (or equal to?) the key we are interested in.
234      * A note on the seekBefore: if you have seekBefore = true, AND the first
235      * key in the block = key, then you'll get thrown exceptions. The caller has
236      * to check for that case and load the previous block as appropriate.
237      * @param key
238      *          the key to find
239      * @param seekBefore
240      *          find the key before the given key in case of exact match.
241      * @return 0 in case of an exact key match, 1 in case of an inexact match,
242      *         -2 in case of an inexact match and furthermore, the input key
243      *         less than the first key of current block(e.g. using a faked index
244      *         key)
245      */
246     @Override
247     protected int blockSeek(Cell key, boolean seekBefore) {
248       int klen, vlen, tlen = 0;
249       long memstoreTS = 0;
250       int memstoreTSLen = 0;
251       int lastKeyValueSize = -1;
252       KeyValue.KeyOnlyKeyValue keyOnlyKv = new KeyValue.KeyOnlyKeyValue();
253       do {
254         blockBuffer.mark();
255         klen = blockBuffer.getInt();
256         vlen = blockBuffer.getInt();
257         if (klen < 0 || vlen < 0 || klen > blockBuffer.limit()
258             || vlen > blockBuffer.limit()) {
259           throw new IllegalStateException("Invalid klen " + klen + " or vlen "
260               + vlen + ". Block offset: "
261               + block.getOffset() + ", block length: " + blockBuffer.limit() + ", position: "
262               + blockBuffer.position() + " (without header).");
263         }
264         ByteBufferUtils.skip(blockBuffer, klen + vlen);
265         if (reader.hfileContext.isIncludesTags()) {
266           tlen = blockBuffer.getShort();
267           if (tlen < 0 || tlen > blockBuffer.limit()) {
268             throw new IllegalStateException("Invalid tlen " + tlen + ". Block offset: "
269                 + block.getOffset() + ", block length: " + blockBuffer.limit() + ", position: "
270                 + blockBuffer.position() + " (without header).");
271           }
272           ByteBufferUtils.skip(blockBuffer, tlen);
273         }
274         if (this.reader.shouldIncludeMemstoreTS()) {
275           if (this.reader.decodeMemstoreTS) {
276             try {
277               memstoreTS = Bytes.readVLong(blockBuffer.array(), blockBuffer.arrayOffset()
278                   + blockBuffer.position());
279               memstoreTSLen = WritableUtils.getVIntSize(memstoreTS);
280             } catch (Exception e) {
281               throw new RuntimeException("Error reading memstore timestamp", e);
282             }
283           } else {
284             memstoreTS = 0;
285             memstoreTSLen = 1;
286           }
287         }
288         blockBuffer.reset();
289         int keyOffset = blockBuffer.arrayOffset() + blockBuffer.position() + (Bytes.SIZEOF_INT * 2);
290         keyOnlyKv.setKey(blockBuffer.array(), keyOffset, klen);
291         int comp = reader.getComparator().compareOnlyKeyPortion(key, keyOnlyKv);
292 
293         if (comp == 0) {
294           if (seekBefore) {
295             if (lastKeyValueSize < 0) {
296               KeyValue kv = KeyValueUtil.ensureKeyValue(key);
297               throw new IllegalStateException("blockSeek with seekBefore "
298                   + "at the first key of the block: key="
299                   + Bytes.toStringBinary(kv.getKey(), kv.getKeyOffset(), kv.getKeyLength())
300                   + ", blockOffset=" + block.getOffset() + ", onDiskSize="
301                   + block.getOnDiskSizeWithHeader());
302             }
303             blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
304             readKeyValueLen();
305             return 1; // non exact match.
306           }
307           currKeyLen = klen;
308           currValueLen = vlen;
309           currTagsLen = tlen;
310           if (this.reader.shouldIncludeMemstoreTS()) {
311             currMemstoreTS = memstoreTS;
312             currMemstoreTSLen = memstoreTSLen;
313           }
314           return 0; // indicate exact match
315         } else if (comp < 0) {
316           if (lastKeyValueSize > 0)
317             blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
318           readKeyValueLen();
319           if (lastKeyValueSize == -1 && blockBuffer.position() == 0) {
320             return HConstants.INDEX_KEY_MAGIC;
321           }
322           return 1;
323         }
324 
325         // The size of this key/value tuple, including key/value length fields.
326         lastKeyValueSize = klen + vlen + memstoreTSLen + KEY_VALUE_LEN_SIZE;
327         // include tag length also if tags included with KV
328         if (reader.hfileContext.isIncludesTags()) {
329           lastKeyValueSize += tlen + Bytes.SIZEOF_SHORT;
330         }
331         blockBuffer.position(blockBuffer.position() + lastKeyValueSize);
332       } while (blockBuffer.remaining() > 0);
333 
334       // Seek to the last key we successfully read. This will happen if this is
335       // the last key/value pair in the file, in which case the following call
336       // to next() has to return false.
337       blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
338       readKeyValueLen();
339       return 1; // didn't exactly find it.
340     }
341   }
342 
343   /**
344    * ScannerV3 that operates on encoded data blocks.
345    */
346   protected static class EncodedScannerV3 extends EncodedScannerV2 {
347     public EncodedScannerV3(HFileReaderV3 reader, boolean cacheBlocks, boolean pread,
348         boolean isCompaction, HFileContext context) {
349       super(reader, cacheBlocks, pread, isCompaction, context);
350     }
351   }
352 
353   @Override
354   public int getMajorVersion() {
355     return 3;
356   }
357 }