001/*
002 *
003 * Licensed to the Apache Software Foundation (ASF) under one
004 * or more contributor license agreements.  See the NOTICE file
005 * distributed with this work for additional information
006 * regarding copyright ownership.  The ASF licenses this file
007 * to you under the Apache License, Version 2.0 (the
008 * "License"); you may not use this file except in compliance
009 * with the License.  You may obtain a copy of the License at
010 *
011 *     http://www.apache.org/licenses/LICENSE-2.0
012 *
013 * Unless required by applicable law or agreed to in writing, software
014 * distributed under the License is distributed on an "AS IS" BASIS,
015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
016 * See the License for the specific language governing permissions and
017 * limitations under the License.
018 */
019package org.apache.hadoop.hbase.io.hfile;
020
021import java.io.ByteArrayInputStream;
022import java.io.DataInputStream;
023import java.io.DataOutputStream;
024import java.io.IOException;
025import java.io.SequenceInputStream;
026import java.security.Key;
027import java.util.ArrayList;
028import java.util.Collection;
029import java.util.Comparator;
030import java.util.List;
031import java.util.Map;
032import java.util.Objects;
033import java.util.Set;
034import java.util.SortedMap;
035import java.util.TreeMap;
036import org.apache.commons.io.IOUtils;
037import org.apache.hadoop.conf.Configuration;
038import org.apache.hadoop.fs.Path;
039import org.apache.hadoop.hbase.Cell;
040import org.apache.hadoop.hbase.KeyValue;
041import org.apache.hadoop.hbase.io.crypto.Cipher;
042import org.apache.hadoop.hbase.io.crypto.Encryption;
043import org.apache.hadoop.hbase.protobuf.ProtobufMagic;
044import org.apache.hadoop.hbase.security.EncryptionUtil;
045import org.apache.hadoop.hbase.util.Bytes;
046import org.apache.yetus.audience.InterfaceAudience;
047import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;
048import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
049import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos;
050import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.BytesBytesPair;
051import org.apache.hadoop.hbase.shaded.protobuf.generated.HFileProtos;
052
053/**
054 * Metadata Map of attributes for HFile written out as HFile Trailer. Created by the Writer and
055 * added to the tail of the file just before close. Metadata includes core attributes such as last
056 * key seen, comparator used writing the file, etc. Clients can add their own attributes via
057 * {@link #append(byte[], byte[], boolean)} and they'll be persisted and available at read time.
058 * Reader creates the HFileInfo on open by reading the tail of the HFile. The parse of the HFile
059 * trailer also creates a {@link HFileContext}, a read-only data structure that includes bulk of
060 * the HFileInfo and extras that is safe to pass around when working on HFiles.
061 * @see HFileContext
062 */
063@InterfaceAudience.Private
064public class HFileInfo implements SortedMap<byte[], byte[]> {
065  static final String RESERVED_PREFIX = "hfile.";
066  static final byte[] RESERVED_PREFIX_BYTES = Bytes.toBytes(RESERVED_PREFIX);
067  static final byte [] LASTKEY = Bytes.toBytes(RESERVED_PREFIX + "LASTKEY");
068  static final byte [] AVG_KEY_LEN = Bytes.toBytes(RESERVED_PREFIX + "AVG_KEY_LEN");
069  static final byte [] AVG_VALUE_LEN = Bytes.toBytes(RESERVED_PREFIX + "AVG_VALUE_LEN");
070  static final byte [] CREATE_TIME_TS = Bytes.toBytes(RESERVED_PREFIX + "CREATE_TIME_TS");
071  static final byte [] TAGS_COMPRESSED = Bytes.toBytes(RESERVED_PREFIX + "TAGS_COMPRESSED");
072  public static final byte [] MAX_TAGS_LEN = Bytes.toBytes(RESERVED_PREFIX + "MAX_TAGS_LEN");
073  private final SortedMap<byte [], byte []> map = new TreeMap<>(Bytes.BYTES_COMPARATOR);
074
075  /**
076   * We can read files whose major version is v2 IFF their minor version is at least 3.
077   */
078  private static final int MIN_V2_MINOR_VERSION_WITH_PB = 3;
079
080  /** Maximum minor version supported by this HFile format */
081  // We went to version 2 when we moved to pb'ing fileinfo and the trailer on
082  // the file. This version can read Writables version 1.
083  static final int MAX_MINOR_VERSION = 3;
084
085  /** Last key in the file. Filled in when we read in the file info */
086  private Cell lastKeyCell = null;
087  /** Average key length read from file info */
088  private int avgKeyLen = -1;
089  /** Average value length read from file info */
090  private int avgValueLen = -1;
091  private boolean includesMemstoreTS = false;
092  private boolean decodeMemstoreTS = false;
093
094  /**
095   * Blocks read from the load-on-open section, excluding data root index, meta
096   * index, and file info.
097   */
098  private List<HFileBlock> loadOnOpenBlocks = new ArrayList<>();
099
100  /**
101   * The iterator will track all blocks in load-on-open section, since we use the
102   * {@link org.apache.hadoop.hbase.io.ByteBuffAllocator} to manage the ByteBuffers in block now,
103   * so we must ensure that deallocate all ByteBuffers in the end.
104   */
105  private HFileBlock.BlockIterator blockIter;
106
107  private HFileBlockIndex.CellBasedKeyBlockIndexReader dataIndexReader;
108  private HFileBlockIndex.ByteArrayKeyBlockIndexReader metaIndexReader;
109
110  private FixedFileTrailer trailer;
111  private HFileContext hfileContext;
112
113  public HFileInfo() {
114    super();
115  }
116
117  public HFileInfo(ReaderContext context, Configuration conf) throws IOException {
118    this.initTrailerAndContext(context, conf);
119  }
120
121  /**
122   * Append the given key/value pair to the file info, optionally checking the
123   * key prefix.
124   *
125   * @param k key to add
126   * @param v value to add
127   * @param checkPrefix whether to check that the provided key does not start
128   *          with the reserved prefix
129   * @return this file info object
130   * @throws IOException if the key or value is invalid
131   * @throws NullPointerException if {@code key} or {@code value} is {@code null}
132   */
133  public HFileInfo append(final byte[] k, final byte[] v,
134      final boolean checkPrefix) throws IOException {
135    Objects.requireNonNull(k, "key cannot be null");
136    Objects.requireNonNull(v, "value cannot be null");
137
138    if (checkPrefix && isReservedFileInfoKey(k)) {
139      throw new IOException("Keys with a " + HFileInfo.RESERVED_PREFIX
140          + " are reserved");
141    }
142    put(k, v);
143    return this;
144  }
145
146  /** Return true if the given file info key is reserved for internal use. */
147  public static boolean isReservedFileInfoKey(byte[] key) {
148    return Bytes.startsWith(key, HFileInfo.RESERVED_PREFIX_BYTES);
149  }
150
151  @Override
152  public void clear() {
153    this.map.clear();
154  }
155
156  @Override
157  public Comparator<? super byte[]> comparator() {
158    return map.comparator();
159  }
160
161  @Override
162  public boolean containsKey(Object key) {
163    return map.containsKey(key);
164  }
165
166  @Override
167  public boolean containsValue(Object value) {
168    return map.containsValue(value);
169  }
170
171  @Override
172  public Set<java.util.Map.Entry<byte[], byte[]>> entrySet() {
173    return map.entrySet();
174  }
175
176  @Override
177  public boolean equals(Object o) {
178    return map.equals(o);
179  }
180
181  @Override
182  public byte[] firstKey() {
183    return map.firstKey();
184  }
185
186  @Override
187  public byte[] get(Object key) {
188    return map.get(key);
189  }
190
191  @Override
192  public int hashCode() {
193    return map.hashCode();
194  }
195
196  @Override
197  public SortedMap<byte[], byte[]> headMap(byte[] toKey) {
198    return this.map.headMap(toKey);
199  }
200
201  @Override
202  public boolean isEmpty() {
203    return map.isEmpty();
204  }
205
206  @Override
207  public Set<byte[]> keySet() {
208    return map.keySet();
209  }
210
211  @Override
212  public byte[] lastKey() {
213    return map.lastKey();
214  }
215
216  @Override
217  public byte[] put(byte[] key, byte[] value) {
218    return this.map.put(key, value);
219  }
220
221  @Override
222  public void putAll(Map<? extends byte[], ? extends byte[]> m) {
223    this.map.putAll(m);
224  }
225
226  @Override
227  public byte[] remove(Object key) {
228    return this.map.remove(key);
229  }
230
231  @Override
232  public int size() {
233    return map.size();
234  }
235
236  @Override
237  public SortedMap<byte[], byte[]> subMap(byte[] fromKey, byte[] toKey) {
238    return this.map.subMap(fromKey, toKey);
239  }
240
241  @Override
242  public SortedMap<byte[], byte[]> tailMap(byte[] fromKey) {
243    return this.map.tailMap(fromKey);
244  }
245
246  @Override
247  public Collection<byte[]> values() {
248    return map.values();
249  }
250
251  /**
252   * Write out this instance on the passed in <code>out</code> stream.
253   * We write it as a protobuf.
254   * @see #read(DataInputStream)
255   */
256  void write(final DataOutputStream out) throws IOException {
257    HFileProtos.FileInfoProto.Builder builder = HFileProtos.FileInfoProto.newBuilder();
258    for (Map.Entry<byte [], byte[]> e: this.map.entrySet()) {
259      HBaseProtos.BytesBytesPair.Builder bbpBuilder = HBaseProtos.BytesBytesPair.newBuilder();
260      bbpBuilder.setFirst(UnsafeByteOperations.unsafeWrap(e.getKey()));
261      bbpBuilder.setSecond(UnsafeByteOperations.unsafeWrap(e.getValue()));
262      builder.addMapEntry(bbpBuilder.build());
263    }
264    out.write(ProtobufMagic.PB_MAGIC);
265    builder.build().writeDelimitedTo(out);
266  }
267
268  /**
269   * Populate this instance with what we find on the passed in <code>in</code> stream.
270   * Can deserialize protobuf of old Writables format.
271   * @see #write(DataOutputStream)
272   */
273  void read(final DataInputStream in) throws IOException {
274    // This code is tested over in TestHFileReaderV1 where we read an old hfile w/ this new code.
275    int pblen = ProtobufUtil.lengthOfPBMagic();
276    byte [] pbuf = new byte[pblen];
277    if (in.markSupported()) {
278      in.mark(pblen);
279    }
280    int read = in.read(pbuf);
281    if (read != pblen) {
282      throw new IOException("read=" + read + ", wanted=" + pblen);
283    }
284    if (ProtobufUtil.isPBMagicPrefix(pbuf)) {
285      parsePB(HFileProtos.FileInfoProto.parseDelimitedFrom(in));
286    } else {
287      if (in.markSupported()) {
288        in.reset();
289        parseWritable(in);
290      } else {
291        // We cannot use BufferedInputStream, it consumes more than we read from the underlying IS
292        ByteArrayInputStream bais = new ByteArrayInputStream(pbuf);
293        SequenceInputStream sis = new SequenceInputStream(bais, in); // Concatenate input streams
294        // TODO: Am I leaking anything here wrapping the passed in stream?  We are not calling
295        // close on the wrapped streams but they should be let go after we leave this context?
296        // I see that we keep a reference to the passed in inputstream but since we no longer
297        // have a reference to this after we leave, we should be ok.
298        parseWritable(new DataInputStream(sis));
299      }
300    }
301  }
302
303  /**
304   * Now parse the old Writable format.  It was a list of Map entries.  Each map entry was a
305   * key and a value of a byte [].  The old map format had a byte before each entry that held
306   * a code which was short for the key or value type.  We know it was a byte [] so in below
307   * we just read and dump it.
308   */
309  void parseWritable(final DataInputStream in) throws IOException {
310    // First clear the map.
311    // Otherwise we will just accumulate entries every time this method is called.
312    this.map.clear();
313    // Read the number of entries in the map
314    int entries = in.readInt();
315    // Then read each key/value pair
316    for (int i = 0; i < entries; i++) {
317      byte [] key = Bytes.readByteArray(in);
318      // We used to read a byte that encoded the class type.
319      // Read and ignore it because it is always byte [] in hfile
320      in.readByte();
321      byte [] value = Bytes.readByteArray(in);
322      this.map.put(key, value);
323    }
324  }
325
326  /**
327   * Fill our map with content of the pb we read off disk
328   * @param fip protobuf message to read
329   */
330  void parsePB(final HFileProtos.FileInfoProto fip) {
331    this.map.clear();
332    for (BytesBytesPair pair: fip.getMapEntryList()) {
333      this.map.put(pair.getFirst().toByteArray(), pair.getSecond().toByteArray());
334    }
335  }
336
337  public void initTrailerAndContext(ReaderContext context, Configuration conf) throws IOException {
338    try {
339      boolean isHBaseChecksum = context.getInputStreamWrapper().shouldUseHBaseChecksum();
340      trailer = FixedFileTrailer.readFromStream(context.getInputStreamWrapper()
341          .getStream(isHBaseChecksum), context.getFileSize());
342      Path path = context.getFilePath();
343      checkFileVersion(path);
344      this.hfileContext = createHFileContext(path, trailer, conf);
345    } catch (Throwable t) {
346      context.getInputStreamWrapper().unbuffer();
347      IOUtils.closeQuietly(context.getInputStreamWrapper());
348      throw new CorruptHFileException("Problem reading HFile Trailer from file "
349          + context.getFilePath(), t);
350    }
351  }
352
353  /**
354   * should be called after initTrailerAndContext
355   */
356  public void initMetaAndIndex(HFile.Reader reader) throws IOException {
357    ReaderContext context = reader.getContext();
358    HFileBlock.FSReader blockReader = reader.getUncachedBlockReader();
359    // Initialize an block iterator, and parse load-on-open blocks in the following.
360    blockIter = blockReader.blockRange(trailer.getLoadOnOpenDataOffset(),
361        context.getFileSize() - trailer.getTrailerSize());
362    // Data index. We also read statistics about the block index written after
363    // the root level.
364    this.dataIndexReader = new HFileBlockIndex
365        .CellBasedKeyBlockIndexReader(trailer.createComparator(), trailer.getNumDataIndexLevels());
366    dataIndexReader.readMultiLevelIndexRoot(blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX),
367        trailer.getDataIndexCount());
368    reader.setDataBlockIndexReader(dataIndexReader);
369    // Meta index.
370    this.metaIndexReader = new HFileBlockIndex.ByteArrayKeyBlockIndexReader(1);
371    metaIndexReader.readRootIndex(blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX),
372        trailer.getMetaIndexCount());
373    reader.setMetaBlockIndexReader(metaIndexReader);
374    loadMetaInfo(blockIter, hfileContext);
375    reader.setDataBlockEncoder(HFileDataBlockEncoderImpl.createFromFileInfo(this));
376    // Load-On-Open info
377    HFileBlock b;
378    while ((b = blockIter.nextBlock()) != null) {
379      loadOnOpenBlocks.add(b);
380    }
381  }
382
383  private HFileContext createHFileContext(Path path,
384      FixedFileTrailer trailer, Configuration conf) throws IOException {
385    HFileContextBuilder builder = new HFileContextBuilder()
386      .withHBaseCheckSum(true)
387      .withHFileName(path.getName())
388      .withCompression(trailer.getCompressionCodec())
389      .withCellComparator(trailer.createComparator(trailer.getComparatorClassName()));
390    // Check for any key material available
391    byte[] keyBytes = trailer.getEncryptionKey();
392    if (keyBytes != null) {
393      Encryption.Context cryptoContext = Encryption.newContext(conf);
394      Key key = EncryptionUtil.unwrapKey(conf, keyBytes);
395      // Use the algorithm the key wants
396      Cipher cipher = Encryption.getCipher(conf, key.getAlgorithm());
397      if (cipher == null) {
398        throw new IOException("Cipher '" + key.getAlgorithm() + "' is not available"
399            + ", path=" + path);
400      }
401      cryptoContext.setCipher(cipher);
402      cryptoContext.setKey(key);
403      builder.withEncryptionContext(cryptoContext);
404    }
405    HFileContext context = builder.build();
406    return context;
407  }
408
409  private void loadMetaInfo(HFileBlock.BlockIterator blockIter, HFileContext hfileContext)
410      throws IOException {
411    read(blockIter.nextBlockWithBlockType(BlockType.FILE_INFO).getByteStream());
412    byte[] creationTimeBytes = get(HFileInfo.CREATE_TIME_TS);
413    hfileContext.setFileCreateTime(creationTimeBytes == null ?
414        0 : Bytes.toLong(creationTimeBytes));
415    byte[] tmp = get(HFileInfo.MAX_TAGS_LEN);
416    // max tag length is not present in the HFile means tags were not at all written to file.
417    if (tmp != null) {
418      hfileContext.setIncludesTags(true);
419      tmp = get(HFileInfo.TAGS_COMPRESSED);
420      if (tmp != null && Bytes.toBoolean(tmp)) {
421        hfileContext.setCompressTags(true);
422      }
423    }
424    // parse meta info
425    if (get(HFileInfo.LASTKEY) != null) {
426      lastKeyCell = new KeyValue.KeyOnlyKeyValue(get(HFileInfo.LASTKEY));
427    }
428    avgKeyLen = Bytes.toInt(get(HFileInfo.AVG_KEY_LEN));
429    avgValueLen = Bytes.toInt(get(HFileInfo.AVG_VALUE_LEN));
430    byte [] keyValueFormatVersion = get(HFileWriterImpl.KEY_VALUE_VERSION);
431    includesMemstoreTS = keyValueFormatVersion != null &&
432        Bytes.toInt(keyValueFormatVersion) == HFileWriterImpl.KEY_VALUE_VER_WITH_MEMSTORE;
433    hfileContext.setIncludesMvcc(includesMemstoreTS);
434    if (includesMemstoreTS) {
435      decodeMemstoreTS = Bytes.toLong(get(HFileWriterImpl.MAX_MEMSTORE_TS_KEY)) > 0;
436    }
437  }
438
439  /**
440   * File version check is a little sloppy. We read v3 files but can also read v2 files if their
441   * content has been pb'd; files written with 0.98.
442   */
443  private void checkFileVersion(Path path) {
444    int majorVersion = trailer.getMajorVersion();
445    if (majorVersion == getMajorVersion()) {
446      return;
447    }
448    int minorVersion = trailer.getMinorVersion();
449    if (majorVersion == 2 && minorVersion >= MIN_V2_MINOR_VERSION_WITH_PB) {
450      return;
451    }
452    // We can read v3 or v2 versions of hfile.
453    throw new IllegalArgumentException("Invalid HFile version: major=" +
454      trailer.getMajorVersion() + ", minor=" + trailer.getMinorVersion() + ": expected at least " +
455      "major=2 and minor=" + MAX_MINOR_VERSION + ", path=" + path);
456  }
457
458  public void close() {
459    if (blockIter != null) {
460      blockIter.freeBlocks();
461    }
462  }
463
464  public int getMajorVersion() {
465    return 3;
466  }
467
468  public void setTrailer(FixedFileTrailer trailer) {
469    this.trailer = trailer;
470  }
471
472  public FixedFileTrailer getTrailer() {
473    return this.trailer;
474  }
475
476  public HFileBlockIndex.CellBasedKeyBlockIndexReader getDataBlockIndexReader() {
477    return this.dataIndexReader;
478  }
479
480  public HFileBlockIndex.ByteArrayKeyBlockIndexReader getMetaBlockIndexReader() {
481    return this.metaIndexReader;
482  }
483
484  public HFileContext getHFileContext() {
485    return this.hfileContext;
486  }
487
488  public List<HFileBlock> getLoadOnOpenBlocks() {
489    return loadOnOpenBlocks;
490  }
491
492  public Cell getLastKeyCell() {
493    return lastKeyCell;
494  }
495
496  public int getAvgKeyLen() {
497    return avgKeyLen;
498  }
499
500  public int getAvgValueLen() {
501    return avgValueLen;
502  }
503
504  public boolean shouldIncludeMemStoreTS() {
505    return includesMemstoreTS;
506  }
507
508  public boolean isDecodeMemstoreTS() {
509    return decodeMemstoreTS;
510  }
511}