001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.io.hfile;
019
020import java.io.ByteArrayInputStream;
021import java.io.DataInputStream;
022import java.io.DataOutputStream;
023import java.io.IOException;
024import java.io.SequenceInputStream;
025import java.security.Key;
026import java.util.ArrayList;
027import java.util.Collection;
028import java.util.Comparator;
029import java.util.List;
030import java.util.Map;
031import java.util.Objects;
032import java.util.Set;
033import java.util.SortedMap;
034import java.util.TreeMap;
035import org.apache.commons.io.IOUtils;
036import org.apache.hadoop.conf.Configuration;
037import org.apache.hadoop.fs.Path;
038import org.apache.hadoop.hbase.Cell;
039import org.apache.hadoop.hbase.KeyValue;
040import org.apache.hadoop.hbase.io.crypto.Cipher;
041import org.apache.hadoop.hbase.io.crypto.Encryption;
042import org.apache.hadoop.hbase.protobuf.ProtobufMagic;
043import org.apache.hadoop.hbase.security.EncryptionUtil;
044import org.apache.hadoop.hbase.util.Bytes;
045import org.apache.yetus.audience.InterfaceAudience;
046import org.slf4j.Logger;
047import org.slf4j.LoggerFactory;
048
049import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;
050
051import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
052import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos;
053import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.BytesBytesPair;
054import org.apache.hadoop.hbase.shaded.protobuf.generated.HFileProtos;
055
056/**
057 * Metadata Map of attributes for HFile written out as HFile Trailer. Created by the Writer and
058 * added to the tail of the file just before close. Metadata includes core attributes such as last
059 * key seen, comparator used writing the file, etc. Clients can add their own attributes via
060 * {@link #append(byte[], byte[], boolean)} and they'll be persisted and available at read time.
061 * Reader creates the HFileInfo on open by reading the tail of the HFile. The parse of the HFile
062 * trailer also creates a {@link HFileContext}, a read-only data structure that includes bulk of the
063 * HFileInfo and extras that is safe to pass around when working on HFiles.
064 * @see HFileContext
065 */
066@InterfaceAudience.Private
067public class HFileInfo implements SortedMap<byte[], byte[]> {
068
069  private static final Logger LOG = LoggerFactory.getLogger(HFileInfo.class);
070
071  static final String RESERVED_PREFIX = "hfile.";
072  static final byte[] RESERVED_PREFIX_BYTES = Bytes.toBytes(RESERVED_PREFIX);
073  static final byte[] LASTKEY = Bytes.toBytes(RESERVED_PREFIX + "LASTKEY");
074  static final byte[] AVG_KEY_LEN = Bytes.toBytes(RESERVED_PREFIX + "AVG_KEY_LEN");
075  static final byte[] AVG_VALUE_LEN = Bytes.toBytes(RESERVED_PREFIX + "AVG_VALUE_LEN");
076  static final byte[] CREATE_TIME_TS = Bytes.toBytes(RESERVED_PREFIX + "CREATE_TIME_TS");
077  static final byte[] TAGS_COMPRESSED = Bytes.toBytes(RESERVED_PREFIX + "TAGS_COMPRESSED");
078  public static final byte[] MAX_TAGS_LEN = Bytes.toBytes(RESERVED_PREFIX + "MAX_TAGS_LEN");
079  private final SortedMap<byte[], byte[]> map = new TreeMap<>(Bytes.BYTES_COMPARATOR);
080
081  /**
082   * We can read files whose major version is v2 IFF their minor version is at least 3.
083   */
084  private static final int MIN_V2_MINOR_VERSION_WITH_PB = 3;
085
086  /** Maximum minor version supported by this HFile format */
087  // We went to version 2 when we moved to pb'ing fileinfo and the trailer on
088  // the file. This version can read Writables version 1.
089  static final int MAX_MINOR_VERSION = 3;
090
091  /** Last key in the file. Filled in when we read in the file info */
092  private Cell lastKeyCell = null;
093  /** Average key length read from file info */
094  private int avgKeyLen = -1;
095  /** Average value length read from file info */
096  private int avgValueLen = -1;
097  private boolean includesMemstoreTS = false;
098  private boolean decodeMemstoreTS = false;
099
100  /**
101   * Blocks read from the load-on-open section, excluding data root index, meta index, and file
102   * info.
103   */
104  private List<HFileBlock> loadOnOpenBlocks = new ArrayList<>();
105
106  /**
107   * The iterator will track all blocks in load-on-open section, since we use the
108   * {@link org.apache.hadoop.hbase.io.ByteBuffAllocator} to manage the ByteBuffers in block now, so
109   * we must ensure that deallocate all ByteBuffers in the end.
110   */
111  private HFileBlock.BlockIterator blockIter;
112
113  private HFileBlockIndex.CellBasedKeyBlockIndexReader dataIndexReader;
114  private HFileBlockIndex.ByteArrayKeyBlockIndexReader metaIndexReader;
115
116  private FixedFileTrailer trailer;
117  private HFileContext hfileContext;
118
119  public HFileInfo() {
120    super();
121  }
122
123  public HFileInfo(ReaderContext context, Configuration conf) throws IOException {
124    this.initTrailerAndContext(context, conf);
125  }
126
127  /**
128   * Append the given key/value pair to the file info, optionally checking the key prefix.
129   * @param k           key to add
130   * @param v           value to add
131   * @param checkPrefix whether to check that the provided key does not start with the reserved
132   *                    prefix
133   * @return this file info object
134   * @throws IOException          if the key or value is invalid
135   * @throws NullPointerException if {@code key} or {@code value} is {@code null}
136   */
137  public HFileInfo append(final byte[] k, final byte[] v, final boolean checkPrefix)
138    throws IOException {
139    Objects.requireNonNull(k, "key cannot be null");
140    Objects.requireNonNull(v, "value cannot be null");
141
142    if (checkPrefix && isReservedFileInfoKey(k)) {
143      throw new IOException("Keys with a " + HFileInfo.RESERVED_PREFIX + " are reserved");
144    }
145    put(k, v);
146    return this;
147  }
148
149  /** Return true if the given file info key is reserved for internal use. */
150  public static boolean isReservedFileInfoKey(byte[] key) {
151    return Bytes.startsWith(key, HFileInfo.RESERVED_PREFIX_BYTES);
152  }
153
154  @Override
155  public void clear() {
156    this.map.clear();
157  }
158
159  @Override
160  public Comparator<? super byte[]> comparator() {
161    return map.comparator();
162  }
163
164  @Override
165  public boolean containsKey(Object key) {
166    return map.containsKey(key);
167  }
168
169  @Override
170  public boolean containsValue(Object value) {
171    return map.containsValue(value);
172  }
173
174  @Override
175  public Set<java.util.Map.Entry<byte[], byte[]>> entrySet() {
176    return map.entrySet();
177  }
178
179  @Override
180  public boolean equals(Object o) {
181    return map.equals(o);
182  }
183
184  @Override
185  public byte[] firstKey() {
186    return map.firstKey();
187  }
188
189  @Override
190  public byte[] get(Object key) {
191    return map.get(key);
192  }
193
194  @Override
195  public int hashCode() {
196    return map.hashCode();
197  }
198
199  @Override
200  public SortedMap<byte[], byte[]> headMap(byte[] toKey) {
201    return this.map.headMap(toKey);
202  }
203
204  @Override
205  public boolean isEmpty() {
206    return map.isEmpty();
207  }
208
209  @Override
210  public Set<byte[]> keySet() {
211    return map.keySet();
212  }
213
214  @Override
215  public byte[] lastKey() {
216    return map.lastKey();
217  }
218
219  @Override
220  public byte[] put(byte[] key, byte[] value) {
221    return this.map.put(key, value);
222  }
223
224  @Override
225  public void putAll(Map<? extends byte[], ? extends byte[]> m) {
226    this.map.putAll(m);
227  }
228
229  @Override
230  public byte[] remove(Object key) {
231    return this.map.remove(key);
232  }
233
234  @Override
235  public int size() {
236    return map.size();
237  }
238
239  @Override
240  public SortedMap<byte[], byte[]> subMap(byte[] fromKey, byte[] toKey) {
241    return this.map.subMap(fromKey, toKey);
242  }
243
244  @Override
245  public SortedMap<byte[], byte[]> tailMap(byte[] fromKey) {
246    return this.map.tailMap(fromKey);
247  }
248
249  @Override
250  public Collection<byte[]> values() {
251    return map.values();
252  }
253
254  /**
255   * Write out this instance on the passed in <code>out</code> stream. We write it as a protobuf.
256   * @see #read(DataInputStream)
257   */
258  void write(final DataOutputStream out) throws IOException {
259    HFileProtos.FileInfoProto.Builder builder = HFileProtos.FileInfoProto.newBuilder();
260    for (Map.Entry<byte[], byte[]> e : this.map.entrySet()) {
261      HBaseProtos.BytesBytesPair.Builder bbpBuilder = HBaseProtos.BytesBytesPair.newBuilder();
262      bbpBuilder.setFirst(UnsafeByteOperations.unsafeWrap(e.getKey()));
263      bbpBuilder.setSecond(UnsafeByteOperations.unsafeWrap(e.getValue()));
264      builder.addMapEntry(bbpBuilder.build());
265    }
266    out.write(ProtobufMagic.PB_MAGIC);
267    builder.build().writeDelimitedTo(out);
268  }
269
270  /**
271   * Populate this instance with what we find on the passed in <code>in</code> stream. Can
272   * deserialize protobuf of old Writables format.
273   * @see #write(DataOutputStream)
274   */
275  void read(final DataInputStream in) throws IOException {
276    // This code is tested over in TestHFileReaderV1 where we read an old hfile w/ this new code.
277    int pblen = ProtobufUtil.lengthOfPBMagic();
278    byte[] pbuf = new byte[pblen];
279    if (in.markSupported()) {
280      in.mark(pblen);
281    }
282    int read = in.read(pbuf);
283    if (read != pblen) {
284      throw new IOException("read=" + read + ", wanted=" + pblen);
285    }
286    if (ProtobufUtil.isPBMagicPrefix(pbuf)) {
287      parsePB(HFileProtos.FileInfoProto.parseDelimitedFrom(in));
288    } else {
289      if (in.markSupported()) {
290        in.reset();
291        parseWritable(in);
292      } else {
293        // We cannot use BufferedInputStream, it consumes more than we read from the underlying IS
294        ByteArrayInputStream bais = new ByteArrayInputStream(pbuf);
295        SequenceInputStream sis = new SequenceInputStream(bais, in); // Concatenate input streams
296        // TODO: Am I leaking anything here wrapping the passed in stream? We are not calling
297        // close on the wrapped streams but they should be let go after we leave this context?
298        // I see that we keep a reference to the passed in inputstream but since we no longer
299        // have a reference to this after we leave, we should be ok.
300        parseWritable(new DataInputStream(sis));
301      }
302    }
303  }
304
305  /**
306   * Now parse the old Writable format. It was a list of Map entries. Each map entry was a key and a
307   * value of a byte []. The old map format had a byte before each entry that held a code which was
308   * short for the key or value type. We know it was a byte [] so in below we just read and dump it.
309   */
310  void parseWritable(final DataInputStream in) throws IOException {
311    // First clear the map.
312    // Otherwise we will just accumulate entries every time this method is called.
313    this.map.clear();
314    // Read the number of entries in the map
315    int entries = in.readInt();
316    // Then read each key/value pair
317    for (int i = 0; i < entries; i++) {
318      byte[] key = Bytes.readByteArray(in);
319      // We used to read a byte that encoded the class type.
320      // Read and ignore it because it is always byte [] in hfile
321      in.readByte();
322      byte[] value = Bytes.readByteArray(in);
323      this.map.put(key, value);
324    }
325  }
326
327  /**
328   * Fill our map with content of the pb we read off disk
329   * @param fip protobuf message to read
330   */
331  void parsePB(final HFileProtos.FileInfoProto fip) {
332    this.map.clear();
333    for (BytesBytesPair pair : fip.getMapEntryList()) {
334      this.map.put(pair.getFirst().toByteArray(), pair.getSecond().toByteArray());
335    }
336  }
337
338  public void initTrailerAndContext(ReaderContext context, Configuration conf) throws IOException {
339    try {
340      boolean isHBaseChecksum = context.getInputStreamWrapper().shouldUseHBaseChecksum();
341      trailer = FixedFileTrailer.readFromStream(
342        context.getInputStreamWrapper().getStream(isHBaseChecksum), context.getFileSize());
343      Path path = context.getFilePath();
344      checkFileVersion(path);
345      this.hfileContext = createHFileContext(path, trailer, conf);
346      context.getInputStreamWrapper().unbuffer();
347    } catch (Throwable t) {
348      IOUtils.closeQuietly(context.getInputStreamWrapper(),
349        e -> LOG.warn("failed to close input stream wrapper", e));
350      throw new CorruptHFileException(
351        "Problem reading HFile Trailer from file " + context.getFilePath(), t);
352    }
353  }
354
355  /**
356   * should be called after initTrailerAndContext
357   */
358  public void initMetaAndIndex(HFile.Reader reader) throws IOException {
359    ReaderContext context = reader.getContext();
360    try {
361      HFileBlock.FSReader blockReader = reader.getUncachedBlockReader();
362      // Initialize an block iterator, and parse load-on-open blocks in the following.
363      blockIter = blockReader.blockRange(trailer.getLoadOnOpenDataOffset(),
364        context.getFileSize() - trailer.getTrailerSize());
365      // Data index. We also read statistics about the block index written after
366      // the root level.
367      this.dataIndexReader = new HFileBlockIndex.CellBasedKeyBlockIndexReader(
368        trailer.createComparator(), trailer.getNumDataIndexLevels());
369      dataIndexReader.readMultiLevelIndexRoot(
370        blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX), trailer.getDataIndexCount());
371      reader.setDataBlockIndexReader(dataIndexReader);
372      // Meta index.
373      this.metaIndexReader = new HFileBlockIndex.ByteArrayKeyBlockIndexReader(1);
374      metaIndexReader.readRootIndex(blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX),
375        trailer.getMetaIndexCount());
376      reader.setMetaBlockIndexReader(metaIndexReader);
377      loadMetaInfo(blockIter, hfileContext);
378      reader.setDataBlockEncoder(HFileDataBlockEncoderImpl.createFromFileInfo(this));
379      // Load-On-Open info
380      HFileBlock b;
381      while ((b = blockIter.nextBlock()) != null) {
382        loadOnOpenBlocks.add(b);
383      }
384      // close the block reader
385      context.getInputStreamWrapper().unbuffer();
386    } catch (Throwable t) {
387      IOUtils.closeQuietly(context.getInputStreamWrapper(),
388        e -> LOG.warn("failed to close input stream wrapper", e));
389      throw new CorruptHFileException(
390        "Problem reading data index and meta index from file " + context.getFilePath(), t);
391    }
392  }
393
394  private HFileContext createHFileContext(Path path, FixedFileTrailer trailer, Configuration conf)
395    throws IOException {
396    HFileContextBuilder builder = new HFileContextBuilder().withHBaseCheckSum(true)
397      .withHFileName(path.getName()).withCompression(trailer.getCompressionCodec())
398      .withCellComparator(FixedFileTrailer.createComparator(trailer.getComparatorClassName()));
399    // Check for any key material available
400    byte[] keyBytes = trailer.getEncryptionKey();
401    if (keyBytes != null) {
402      Encryption.Context cryptoContext = Encryption.newContext(conf);
403      Key key = EncryptionUtil.unwrapKey(conf, keyBytes);
404      // Use the algorithm the key wants
405      Cipher cipher = Encryption.getCipher(conf, key.getAlgorithm());
406      if (cipher == null) {
407        throw new IOException(
408          "Cipher '" + key.getAlgorithm() + "' is not available" + ", path=" + path);
409      }
410      cryptoContext.setCipher(cipher);
411      cryptoContext.setKey(key);
412      builder.withEncryptionContext(cryptoContext);
413    }
414    HFileContext context = builder.build();
415    return context;
416  }
417
418  private void loadMetaInfo(HFileBlock.BlockIterator blockIter, HFileContext hfileContext)
419    throws IOException {
420    read(blockIter.nextBlockWithBlockType(BlockType.FILE_INFO).getByteStream());
421    byte[] creationTimeBytes = get(HFileInfo.CREATE_TIME_TS);
422    hfileContext.setFileCreateTime(creationTimeBytes == null ? 0 : Bytes.toLong(creationTimeBytes));
423    byte[] tmp = get(HFileInfo.MAX_TAGS_LEN);
424    // max tag length is not present in the HFile means tags were not at all written to file.
425    if (tmp != null) {
426      hfileContext.setIncludesTags(true);
427      tmp = get(HFileInfo.TAGS_COMPRESSED);
428      if (tmp != null && Bytes.toBoolean(tmp)) {
429        hfileContext.setCompressTags(true);
430      }
431    }
432    // parse meta info
433    if (get(HFileInfo.LASTKEY) != null) {
434      lastKeyCell = new KeyValue.KeyOnlyKeyValue(get(HFileInfo.LASTKEY));
435    }
436    avgKeyLen = Bytes.toInt(get(HFileInfo.AVG_KEY_LEN));
437    avgValueLen = Bytes.toInt(get(HFileInfo.AVG_VALUE_LEN));
438    byte[] keyValueFormatVersion = get(HFileWriterImpl.KEY_VALUE_VERSION);
439    includesMemstoreTS = keyValueFormatVersion != null
440      && Bytes.toInt(keyValueFormatVersion) == HFileWriterImpl.KEY_VALUE_VER_WITH_MEMSTORE;
441    hfileContext.setIncludesMvcc(includesMemstoreTS);
442    if (includesMemstoreTS) {
443      decodeMemstoreTS = Bytes.toLong(get(HFileWriterImpl.MAX_MEMSTORE_TS_KEY)) > 0;
444    }
445  }
446
447  /**
448   * File version check is a little sloppy. We read v3 files but can also read v2 files if their
449   * content has been pb'd; files written with 0.98.
450   */
451  private void checkFileVersion(Path path) {
452    int majorVersion = trailer.getMajorVersion();
453    if (majorVersion == getMajorVersion()) {
454      return;
455    }
456    int minorVersion = trailer.getMinorVersion();
457    if (majorVersion == 2 && minorVersion >= MIN_V2_MINOR_VERSION_WITH_PB) {
458      return;
459    }
460    // We can read v3 or v2 versions of hfile.
461    throw new IllegalArgumentException("Invalid HFile version: major=" + trailer.getMajorVersion()
462      + ", minor=" + trailer.getMinorVersion() + ": expected at least " + "major=2 and minor="
463      + MAX_MINOR_VERSION + ", path=" + path);
464  }
465
466  public void close() {
467    if (blockIter != null) {
468      blockIter.freeBlocks();
469    }
470  }
471
472  public int getMajorVersion() {
473    return 3;
474  }
475
476  public void setTrailer(FixedFileTrailer trailer) {
477    this.trailer = trailer;
478  }
479
480  public FixedFileTrailer getTrailer() {
481    return this.trailer;
482  }
483
484  public HFileBlockIndex.CellBasedKeyBlockIndexReader getDataBlockIndexReader() {
485    return this.dataIndexReader;
486  }
487
488  public HFileBlockIndex.ByteArrayKeyBlockIndexReader getMetaBlockIndexReader() {
489    return this.metaIndexReader;
490  }
491
492  public HFileContext getHFileContext() {
493    return this.hfileContext;
494  }
495
496  public List<HFileBlock> getLoadOnOpenBlocks() {
497    return loadOnOpenBlocks;
498  }
499
500  public Cell getLastKeyCell() {
501    return lastKeyCell;
502  }
503
504  public int getAvgKeyLen() {
505    return avgKeyLen;
506  }
507
508  public int getAvgValueLen() {
509    return avgValueLen;
510  }
511
512  public boolean shouldIncludeMemStoreTS() {
513    return includesMemstoreTS;
514  }
515
516  public boolean isDecodeMemstoreTS() {
517    return decodeMemstoreTS;
518  }
519}