001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.io.hfile; 019 020import java.io.ByteArrayInputStream; 021import java.io.DataInputStream; 022import java.io.DataOutputStream; 023import java.io.IOException; 024import java.io.SequenceInputStream; 025import java.security.Key; 026import java.util.ArrayList; 027import java.util.Collection; 028import java.util.Comparator; 029import java.util.List; 030import java.util.Map; 031import java.util.Set; 032import java.util.SortedMap; 033import java.util.TreeMap; 034import org.apache.commons.io.IOUtils; 035import org.apache.hadoop.conf.Configuration; 036import org.apache.hadoop.fs.Path; 037import org.apache.hadoop.hbase.Cell; 038import org.apache.hadoop.hbase.KeyValue; 039import org.apache.hadoop.hbase.io.crypto.Cipher; 040import org.apache.hadoop.hbase.io.crypto.Encryption; 041import org.apache.hadoop.hbase.protobuf.ProtobufMagic; 042import org.apache.hadoop.hbase.security.EncryptionUtil; 043import org.apache.hadoop.hbase.util.Bytes; 044import org.apache.yetus.audience.InterfaceAudience; 045import org.slf4j.Logger; 046import org.slf4j.LoggerFactory; 047 048import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations; 049 050import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; 051import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; 052import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.BytesBytesPair; 053import org.apache.hadoop.hbase.shaded.protobuf.generated.HFileProtos; 054 055/** 056 * Metadata Map of attributes for HFile written out as HFile Trailer. Created by the Writer and 057 * added to the tail of the file just before close. Metadata includes core attributes such as last 058 * key seen, comparator used writing the file, etc. Clients can add their own attributes via 059 * {@link #append(byte[], byte[], boolean)} and they'll be persisted and available at read time. 060 * Reader creates the HFileInfo on open by reading the tail of the HFile. The parse of the HFile 061 * trailer also creates a {@link HFileContext}, a read-only data structure that includes bulk of the 062 * HFileInfo and extras that is safe to pass around when working on HFiles. 063 * @see HFileContext 064 */ 065@InterfaceAudience.Private 066public class HFileInfo implements SortedMap<byte[], byte[]> { 067 068 private static final Logger LOG = LoggerFactory.getLogger(HFileInfo.class); 069 070 static final String RESERVED_PREFIX = "hfile."; 071 static final byte[] RESERVED_PREFIX_BYTES = Bytes.toBytes(RESERVED_PREFIX); 072 static final byte[] LASTKEY = Bytes.toBytes(RESERVED_PREFIX + "LASTKEY"); 073 static final byte[] AVG_KEY_LEN = Bytes.toBytes(RESERVED_PREFIX + "AVG_KEY_LEN"); 074 static final byte[] AVG_VALUE_LEN = Bytes.toBytes(RESERVED_PREFIX + "AVG_VALUE_LEN"); 075 static final byte[] CREATE_TIME_TS = Bytes.toBytes(RESERVED_PREFIX + "CREATE_TIME_TS"); 076 static final byte[] TAGS_COMPRESSED = Bytes.toBytes(RESERVED_PREFIX + "TAGS_COMPRESSED"); 077 public static final byte[] MAX_TAGS_LEN = Bytes.toBytes(RESERVED_PREFIX + "MAX_TAGS_LEN"); 078 private final SortedMap<byte[], byte[]> map = new TreeMap<>(Bytes.BYTES_COMPARATOR); 079 080 /** 081 * We can read files whose major version is v2 IFF their minor version is at least 3. 082 */ 083 private static final int MIN_V2_MINOR_VERSION_WITH_PB = 3; 084 085 /** Maximum minor version supported by this HFile format */ 086 // We went to version 2 when we moved to pb'ing fileinfo and the trailer on 087 // the file. This version can read Writables version 1. 088 static final int MAX_MINOR_VERSION = 3; 089 090 /** Last key in the file. Filled in when we read in the file info */ 091 private Cell lastKeyCell = null; 092 /** Average key length read from file info */ 093 private int avgKeyLen = -1; 094 /** Average value length read from file info */ 095 private int avgValueLen = -1; 096 private boolean includesMemstoreTS = false; 097 private boolean decodeMemstoreTS = false; 098 099 /** 100 * Blocks read from the load-on-open section, excluding data root index, meta index, and file 101 * info. 102 */ 103 private List<HFileBlock> loadOnOpenBlocks = new ArrayList<>(); 104 105 /** 106 * The iterator will track all blocks in load-on-open section, since we use the 107 * {@link org.apache.hadoop.hbase.io.ByteBuffAllocator} to manage the ByteBuffers in block now, so 108 * we must ensure that deallocate all ByteBuffers in the end. 109 */ 110 private HFileBlock.BlockIterator blockIter; 111 112 private HFileBlockIndex.CellBasedKeyBlockIndexReader dataIndexReader; 113 private HFileBlockIndex.ByteArrayKeyBlockIndexReader metaIndexReader; 114 115 private FixedFileTrailer trailer; 116 private HFileContext hfileContext; 117 118 public HFileInfo() { 119 super(); 120 } 121 122 public HFileInfo(ReaderContext context, Configuration conf) throws IOException { 123 this.initTrailerAndContext(context, conf); 124 } 125 126 /** 127 * Append the given key/value pair to the file info, optionally checking the key prefix. 128 * @param k key to add 129 * @param v value to add 130 * @param checkPrefix whether to check that the provided key does not start with the reserved 131 * prefix 132 * @return this file info object 133 * @throws IOException if the key or value is invalid 134 */ 135 public HFileInfo append(final byte[] k, final byte[] v, final boolean checkPrefix) 136 throws IOException { 137 if (k == null || v == null) { 138 throw new NullPointerException("Key nor value may be null"); 139 } 140 if (checkPrefix && isReservedFileInfoKey(k)) { 141 throw new IOException("Keys with a " + HFileInfo.RESERVED_PREFIX + " are reserved"); 142 } 143 put(k, v); 144 return this; 145 } 146 147 /** Return true if the given file info key is reserved for internal use. */ 148 public static boolean isReservedFileInfoKey(byte[] key) { 149 return Bytes.startsWith(key, HFileInfo.RESERVED_PREFIX_BYTES); 150 } 151 152 @Override 153 public void clear() { 154 this.map.clear(); 155 } 156 157 @Override 158 public Comparator<? super byte[]> comparator() { 159 return map.comparator(); 160 } 161 162 @Override 163 public boolean containsKey(Object key) { 164 return map.containsKey(key); 165 } 166 167 @Override 168 public boolean containsValue(Object value) { 169 return map.containsValue(value); 170 } 171 172 @Override 173 public Set<java.util.Map.Entry<byte[], byte[]>> entrySet() { 174 return map.entrySet(); 175 } 176 177 @Override 178 public boolean equals(Object o) { 179 return map.equals(o); 180 } 181 182 @Override 183 public byte[] firstKey() { 184 return map.firstKey(); 185 } 186 187 @Override 188 public byte[] get(Object key) { 189 return map.get(key); 190 } 191 192 @Override 193 public int hashCode() { 194 return map.hashCode(); 195 } 196 197 @Override 198 public SortedMap<byte[], byte[]> headMap(byte[] toKey) { 199 return this.map.headMap(toKey); 200 } 201 202 @Override 203 public boolean isEmpty() { 204 return map.isEmpty(); 205 } 206 207 @Override 208 public Set<byte[]> keySet() { 209 return map.keySet(); 210 } 211 212 @Override 213 public byte[] lastKey() { 214 return map.lastKey(); 215 } 216 217 @Override 218 public byte[] put(byte[] key, byte[] value) { 219 return this.map.put(key, value); 220 } 221 222 @Override 223 public void putAll(Map<? extends byte[], ? extends byte[]> m) { 224 this.map.putAll(m); 225 } 226 227 @Override 228 public byte[] remove(Object key) { 229 return this.map.remove(key); 230 } 231 232 @Override 233 public int size() { 234 return map.size(); 235 } 236 237 @Override 238 public SortedMap<byte[], byte[]> subMap(byte[] fromKey, byte[] toKey) { 239 return this.map.subMap(fromKey, toKey); 240 } 241 242 @Override 243 public SortedMap<byte[], byte[]> tailMap(byte[] fromKey) { 244 return this.map.tailMap(fromKey); 245 } 246 247 @Override 248 public Collection<byte[]> values() { 249 return map.values(); 250 } 251 252 /** 253 * Write out this instance on the passed in <code>out</code> stream. We write it as a protobuf. 254 * @see #read(DataInputStream) 255 */ 256 void write(final DataOutputStream out) throws IOException { 257 HFileProtos.FileInfoProto.Builder builder = HFileProtos.FileInfoProto.newBuilder(); 258 for (Map.Entry<byte[], byte[]> e : this.map.entrySet()) { 259 HBaseProtos.BytesBytesPair.Builder bbpBuilder = HBaseProtos.BytesBytesPair.newBuilder(); 260 bbpBuilder.setFirst(UnsafeByteOperations.unsafeWrap(e.getKey())); 261 bbpBuilder.setSecond(UnsafeByteOperations.unsafeWrap(e.getValue())); 262 builder.addMapEntry(bbpBuilder.build()); 263 } 264 out.write(ProtobufMagic.PB_MAGIC); 265 builder.build().writeDelimitedTo(out); 266 } 267 268 /** 269 * Populate this instance with what we find on the passed in <code>in</code> stream. Can 270 * deserialize protobuf of old Writables format. 271 * @see #write(DataOutputStream) 272 */ 273 void read(final DataInputStream in) throws IOException { 274 // This code is tested over in TestHFileReaderV1 where we read an old hfile w/ this new code. 275 int pblen = ProtobufUtil.lengthOfPBMagic(); 276 byte[] pbuf = new byte[pblen]; 277 if (in.markSupported()) { 278 in.mark(pblen); 279 } 280 int read = in.read(pbuf); 281 if (read != pblen) { 282 throw new IOException("read=" + read + ", wanted=" + pblen); 283 } 284 if (ProtobufUtil.isPBMagicPrefix(pbuf)) { 285 parsePB(HFileProtos.FileInfoProto.parseDelimitedFrom(in)); 286 } else { 287 if (in.markSupported()) { 288 in.reset(); 289 parseWritable(in); 290 } else { 291 // We cannot use BufferedInputStream, it consumes more than we read from the underlying IS 292 ByteArrayInputStream bais = new ByteArrayInputStream(pbuf); 293 SequenceInputStream sis = new SequenceInputStream(bais, in); // Concatenate input streams 294 // TODO: Am I leaking anything here wrapping the passed in stream? We are not calling 295 // close on the wrapped streams but they should be let go after we leave this context? 296 // I see that we keep a reference to the passed in inputstream but since we no longer 297 // have a reference to this after we leave, we should be ok. 298 parseWritable(new DataInputStream(sis)); 299 } 300 } 301 } 302 303 /** 304 * Now parse the old Writable format. It was a list of Map entries. Each map entry was a key and a 305 * value of a byte []. The old map format had a byte before each entry that held a code which was 306 * short for the key or value type. We know it was a byte [] so in below we just read and dump it. 307 */ 308 void parseWritable(final DataInputStream in) throws IOException { 309 // First clear the map. 310 // Otherwise we will just accumulate entries every time this method is called. 311 this.map.clear(); 312 // Read the number of entries in the map 313 int entries = in.readInt(); 314 // Then read each key/value pair 315 for (int i = 0; i < entries; i++) { 316 byte[] key = Bytes.readByteArray(in); 317 // We used to read a byte that encoded the class type. 318 // Read and ignore it because it is always byte [] in hfile 319 in.readByte(); 320 byte[] value = Bytes.readByteArray(in); 321 this.map.put(key, value); 322 } 323 } 324 325 /** 326 * Fill our map with content of the pb we read off disk 327 * @param fip protobuf message to read 328 */ 329 void parsePB(final HFileProtos.FileInfoProto fip) { 330 this.map.clear(); 331 for (BytesBytesPair pair : fip.getMapEntryList()) { 332 this.map.put(pair.getFirst().toByteArray(), pair.getSecond().toByteArray()); 333 } 334 } 335 336 public void initTrailerAndContext(ReaderContext context, Configuration conf) throws IOException { 337 try { 338 boolean isHBaseChecksum = context.getInputStreamWrapper().shouldUseHBaseChecksum(); 339 trailer = FixedFileTrailer.readFromStream( 340 context.getInputStreamWrapper().getStream(isHBaseChecksum), context.getFileSize()); 341 Path path = context.getFilePath(); 342 checkFileVersion(path); 343 this.hfileContext = createHFileContext(path, trailer, conf); 344 context.getInputStreamWrapper().unbuffer(); 345 } catch (Throwable t) { 346 IOUtils.closeQuietly(context.getInputStreamWrapper(), 347 e -> LOG.warn("failed to close input stream wrapper", e)); 348 throw new CorruptHFileException( 349 "Problem reading HFile Trailer from file " + context.getFilePath(), t); 350 } 351 } 352 353 /** 354 * should be called after initTrailerAndContext 355 */ 356 public void initMetaAndIndex(HFile.Reader reader) throws IOException { 357 ReaderContext context = reader.getContext(); 358 try { 359 HFileBlock.FSReader blockReader = reader.getUncachedBlockReader(); 360 // Initialize an block iterator, and parse load-on-open blocks in the following. 361 blockIter = blockReader.blockRange(trailer.getLoadOnOpenDataOffset(), 362 context.getFileSize() - trailer.getTrailerSize()); 363 // Data index. We also read statistics about the block index written after 364 // the root level. 365 this.dataIndexReader = new HFileBlockIndex.CellBasedKeyBlockIndexReader( 366 trailer.createComparator(), trailer.getNumDataIndexLevels()); 367 dataIndexReader.readMultiLevelIndexRoot( 368 blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX), trailer.getDataIndexCount()); 369 reader.setDataBlockIndexReader(dataIndexReader); 370 // Meta index. 371 this.metaIndexReader = new HFileBlockIndex.ByteArrayKeyBlockIndexReader(1); 372 metaIndexReader.readRootIndex(blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX), 373 trailer.getMetaIndexCount()); 374 reader.setMetaBlockIndexReader(metaIndexReader); 375 loadMetaInfo(blockIter, hfileContext); 376 reader.setDataBlockEncoder(HFileDataBlockEncoderImpl.createFromFileInfo(this)); 377 // Load-On-Open info 378 HFileBlock b; 379 while ((b = blockIter.nextBlock()) != null) { 380 loadOnOpenBlocks.add(b); 381 } 382 // close the block reader 383 context.getInputStreamWrapper().unbuffer(); 384 } catch (Throwable t) { 385 IOUtils.closeQuietly(context.getInputStreamWrapper(), 386 e -> LOG.warn("failed to close input stream wrapper", e)); 387 throw new CorruptHFileException( 388 "Problem reading data index and meta index from file " + context.getFilePath(), t); 389 } 390 } 391 392 private HFileContext createHFileContext(Path path, FixedFileTrailer trailer, Configuration conf) 393 throws IOException { 394 HFileContextBuilder builder = new HFileContextBuilder().withHBaseCheckSum(true) 395 .withHFileName(path.getName()).withCompression(trailer.getCompressionCodec()) 396 .withCellComparator(FixedFileTrailer.createComparator(trailer.getComparatorClassName())); 397 // Check for any key material available 398 byte[] keyBytes = trailer.getEncryptionKey(); 399 if (keyBytes != null) { 400 Encryption.Context cryptoContext = Encryption.newContext(conf); 401 Key key = EncryptionUtil.unwrapKey(conf, keyBytes); 402 // Use the algorithm the key wants 403 Cipher cipher = Encryption.getCipher(conf, key.getAlgorithm()); 404 if (cipher == null) { 405 throw new IOException( 406 "Cipher '" + key.getAlgorithm() + "' is not available" + ", path=" + path); 407 } 408 cryptoContext.setCipher(cipher); 409 cryptoContext.setKey(key); 410 builder.withEncryptionContext(cryptoContext); 411 } 412 HFileContext context = builder.build(); 413 return context; 414 } 415 416 private void loadMetaInfo(HFileBlock.BlockIterator blockIter, HFileContext hfileContext) 417 throws IOException { 418 read(blockIter.nextBlockWithBlockType(BlockType.FILE_INFO).getByteStream()); 419 byte[] creationTimeBytes = get(HFileInfo.CREATE_TIME_TS); 420 hfileContext.setFileCreateTime(creationTimeBytes == null ? 0 : Bytes.toLong(creationTimeBytes)); 421 byte[] tmp = get(HFileInfo.MAX_TAGS_LEN); 422 // max tag length is not present in the HFile means tags were not at all written to file. 423 if (tmp != null) { 424 hfileContext.setIncludesTags(true); 425 tmp = get(HFileInfo.TAGS_COMPRESSED); 426 if (tmp != null && Bytes.toBoolean(tmp)) { 427 hfileContext.setCompressTags(true); 428 } 429 } 430 // parse meta info 431 if (get(HFileInfo.LASTKEY) != null) { 432 lastKeyCell = new KeyValue.KeyOnlyKeyValue(get(HFileInfo.LASTKEY)); 433 } 434 avgKeyLen = Bytes.toInt(get(HFileInfo.AVG_KEY_LEN)); 435 avgValueLen = Bytes.toInt(get(HFileInfo.AVG_VALUE_LEN)); 436 byte[] keyValueFormatVersion = get(HFileWriterImpl.KEY_VALUE_VERSION); 437 includesMemstoreTS = keyValueFormatVersion != null 438 && Bytes.toInt(keyValueFormatVersion) == HFileWriterImpl.KEY_VALUE_VER_WITH_MEMSTORE; 439 hfileContext.setIncludesMvcc(includesMemstoreTS); 440 if (includesMemstoreTS) { 441 decodeMemstoreTS = Bytes.toLong(get(HFileWriterImpl.MAX_MEMSTORE_TS_KEY)) > 0; 442 } 443 } 444 445 /** 446 * File version check is a little sloppy. We read v3 files but can also read v2 files if their 447 * content has been pb'd; files written with 0.98. 448 */ 449 private void checkFileVersion(Path path) { 450 int majorVersion = trailer.getMajorVersion(); 451 if (majorVersion == getMajorVersion()) { 452 return; 453 } 454 int minorVersion = trailer.getMinorVersion(); 455 if (majorVersion == 2 && minorVersion >= MIN_V2_MINOR_VERSION_WITH_PB) { 456 return; 457 } 458 // We can read v3 or v2 versions of hfile. 459 throw new IllegalArgumentException("Invalid HFile version: major=" + trailer.getMajorVersion() 460 + ", minor=" + trailer.getMinorVersion() + ": expected at least " + "major=2 and minor=" 461 + MAX_MINOR_VERSION + ", path=" + path); 462 } 463 464 public void close() { 465 if (blockIter != null) { 466 blockIter.freeBlocks(); 467 } 468 } 469 470 public int getMajorVersion() { 471 return 3; 472 } 473 474 public void setTrailer(FixedFileTrailer trailer) { 475 this.trailer = trailer; 476 } 477 478 public FixedFileTrailer getTrailer() { 479 return this.trailer; 480 } 481 482 public HFileBlockIndex.CellBasedKeyBlockIndexReader getDataBlockIndexReader() { 483 return this.dataIndexReader; 484 } 485 486 public HFileBlockIndex.ByteArrayKeyBlockIndexReader getMetaBlockIndexReader() { 487 return this.metaIndexReader; 488 } 489 490 public HFileContext getHFileContext() { 491 return this.hfileContext; 492 } 493 494 public List<HFileBlock> getLoadOnOpenBlocks() { 495 return loadOnOpenBlocks; 496 } 497 498 public Cell getLastKeyCell() { 499 return lastKeyCell; 500 } 501 502 public int getAvgKeyLen() { 503 return avgKeyLen; 504 } 505 506 public int getAvgValueLen() { 507 return avgValueLen; 508 } 509 510 public boolean shouldIncludeMemStoreTS() { 511 return includesMemstoreTS; 512 } 513 514 public boolean isDecodeMemstoreTS() { 515 return decodeMemstoreTS; 516 } 517}