View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.io.hfile;
20  
21  import java.io.ByteArrayInputStream;
22  import java.io.Closeable;
23  import java.io.DataInput;
24  import java.io.DataInputStream;
25  import java.io.DataOutputStream;
26  import java.io.IOException;
27  import java.io.SequenceInputStream;
28  import java.net.InetSocketAddress;
29  import java.nio.ByteBuffer;
30  import java.util.ArrayList;
31  import java.util.Collection;
32  import java.util.Comparator;
33  import java.util.List;
34  import java.util.Map;
35  import java.util.Set;
36  import java.util.SortedMap;
37  import java.util.TreeMap;
38  import java.util.concurrent.atomic.AtomicLong;
39  
40  import org.apache.hadoop.hbase.util.ByteStringer;
41  import org.apache.commons.logging.Log;
42  import org.apache.commons.logging.LogFactory;
43  import org.apache.hadoop.hbase.classification.InterfaceAudience;
44  import org.apache.hadoop.conf.Configuration;
45  import org.apache.hadoop.fs.FSDataInputStream;
46  import org.apache.hadoop.fs.FSDataOutputStream;
47  import org.apache.hadoop.fs.FileStatus;
48  import org.apache.hadoop.fs.FileSystem;
49  import org.apache.hadoop.fs.Path;
50  import org.apache.hadoop.fs.PathFilter;
51  import org.apache.hadoop.hbase.Cell;
52  import org.apache.hadoop.hbase.CellComparator;
53  import org.apache.hadoop.hbase.HConstants;
54  import org.apache.hadoop.hbase.fs.HFileSystem;
55  import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
56  import org.apache.hadoop.hbase.io.compress.Compression;
57  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
58  import org.apache.hadoop.hbase.protobuf.ProtobufMagic;
59  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
60  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
61  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.BytesBytesPair;
62  import org.apache.hadoop.hbase.protobuf.generated.HFileProtos;
63  import org.apache.hadoop.hbase.util.BloomFilterWriter;
64  import org.apache.hadoop.hbase.util.Bytes;
65  import org.apache.hadoop.hbase.util.FSUtils;
66  import org.apache.hadoop.io.Writable;
67  
68  import com.google.common.annotations.VisibleForTesting;
69  import com.google.common.base.Preconditions;
70  
71  /**
72   * File format for hbase.
73   * A file of sorted key/value pairs. Both keys and values are byte arrays.
74   * <p>
75   * The memory footprint of a HFile includes the following (below is taken from the
76   * <a
77   * href=https://issues.apache.org/jira/browse/HADOOP-3315>TFile</a> documentation
78   * but applies also to HFile):
79   * <ul>
80   * <li>Some constant overhead of reading or writing a compressed block.
81   * <ul>
82   * <li>Each compressed block requires one compression/decompression codec for
83   * I/O.
84   * <li>Temporary space to buffer the key.
85   * <li>Temporary space to buffer the value.
86   * </ul>
87   * <li>HFile index, which is proportional to the total number of Data Blocks.
88   * The total amount of memory needed to hold the index can be estimated as
89   * (56+AvgKeySize)*NumBlocks.
90   * </ul>
91   * Suggestions on performance optimization.
92   * <ul>
93   * <li>Minimum block size. We recommend a setting of minimum block size between
94   * 8KB to 1MB for general usage. Larger block size is preferred if files are
95   * primarily for sequential access. However, it would lead to inefficient random
96   * access (because there are more data to decompress). Smaller blocks are good
97   * for random access, but require more memory to hold the block index, and may
98   * be slower to create (because we must flush the compressor stream at the
99   * conclusion of each data block, which leads to an FS I/O flush). Further, due
100  * to the internal caching in Compression codec, the smallest possible block
101  * size would be around 20KB-30KB.
102  * <li>The current implementation does not offer true multi-threading for
103  * reading. The implementation uses FSDataInputStream seek()+read(), which is
104  * shown to be much faster than positioned-read call in single thread mode.
105  * However, it also means that if multiple threads attempt to access the same
106  * HFile (using multiple scanners) simultaneously, the actual I/O is carried out
107  * sequentially even if they access different DFS blocks (Reexamine! pread seems
108  * to be 10% faster than seek+read in my testing -- stack).
109  * <li>Compression codec. Use "none" if the data is not very compressable (by
110  * compressable, I mean a compression ratio at least 2:1). Generally, use "lzo"
111  * as the starting point for experimenting. "gz" overs slightly better
112  * compression ratio over "lzo" but requires 4x CPU to compress and 2x CPU to
113  * decompress, comparing to "lzo".
114  * </ul>
115  *
116  * For more on the background behind HFile, see <a
117  * href=https://issues.apache.org/jira/browse/HBASE-61>HBASE-61</a>.
118  * <p>
119  * File is made of data blocks followed by meta data blocks (if any), a fileinfo
120  * block, data block index, meta data block index, and a fixed size trailer
121  * which records the offsets at which file changes content type.
122  * <pre>&lt;data blocks&gt;&lt;meta blocks&gt;&lt;fileinfo&gt;&lt;
123  * data index&gt;&lt;meta index&gt;&lt;trailer&gt;</pre>
124  * Each block has a bit of magic at its start.  Block are comprised of
125  * key/values.  In data blocks, they are both byte arrays.  Metadata blocks are
126  * a String key and a byte array value.  An empty file looks like this:
127  * <pre>&lt;fileinfo&gt;&lt;trailer&gt;</pre>.  That is, there are not data nor meta
128  * blocks present.
129  * <p>
130  * TODO: Do scanners need to be able to take a start and end row?
131  * TODO: Should BlockIndex know the name of its file?  Should it have a Path
132  * that points at its file say for the case where an index lives apart from
133  * an HFile instance?
134  */
135 @InterfaceAudience.Private
136 public class HFile {
137   // LOG is being used in HFileBlock and CheckSumUtil
138   static final Log LOG = LogFactory.getLog(HFile.class);
139 
140   /**
141    * Maximum length of key in HFile.
142    */
143   public final static int MAXIMUM_KEY_LENGTH = Integer.MAX_VALUE;
144 
145   /**
146    * Default compression: none.
147    */
148   public final static Compression.Algorithm DEFAULT_COMPRESSION_ALGORITHM =
149     Compression.Algorithm.NONE;
150 
151   /** Minimum supported HFile format version */
152   public static final int MIN_FORMAT_VERSION = 2;
153 
154   /** Maximum supported HFile format version
155    */
156   public static final int MAX_FORMAT_VERSION = 3;
157 
158   /**
159    * Minimum HFile format version with support for persisting cell tags
160    */
161   public static final int MIN_FORMAT_VERSION_WITH_TAGS = 3;
162 
163   /** Default compression name: none. */
164   public final static String DEFAULT_COMPRESSION =
165     DEFAULT_COMPRESSION_ALGORITHM.getName();
166 
167   /** Meta data block name for bloom filter bits. */
168   public static final String BLOOM_FILTER_DATA_KEY = "BLOOM_FILTER_DATA";
169 
170   /**
171    * We assume that HFile path ends with
172    * ROOT_DIR/TABLE_NAME/REGION_NAME/CF_NAME/HFILE, so it has at least this
173    * many levels of nesting. This is needed for identifying table and CF name
174    * from an HFile path.
175    */
176   public final static int MIN_NUM_HFILE_PATH_LEVELS = 5;
177 
178   /**
179    * The number of bytes per checksum.
180    */
181   public static final int DEFAULT_BYTES_PER_CHECKSUM = 16 * 1024;
182   // For measuring number of checksum failures
183   static final AtomicLong checksumFailures = new AtomicLong();
184 
185   // for test purpose
186   public static final AtomicLong dataBlockReadCnt = new AtomicLong(0);
187 
188   /**
189    * Number of checksum verification failures. It also
190    * clears the counter.
191    */
192   public static final long getChecksumFailuresCount() {
193     return checksumFailures.getAndSet(0);
194   }
195 
196   /** API required to write an {@link HFile} */
197   public interface Writer extends Closeable {
198     /** Max memstore (mvcc) timestamp in FileInfo */
199     public static final byte [] MAX_MEMSTORE_TS_KEY = Bytes.toBytes("MAX_MEMSTORE_TS_KEY");
200 
201     /** Add an element to the file info map. */
202     void appendFileInfo(byte[] key, byte[] value) throws IOException;
203 
204     void append(Cell cell) throws IOException;
205 
206     /** @return the path to this {@link HFile} */
207     Path getPath();
208 
209     /**
210      * Adds an inline block writer such as a multi-level block index writer or
211      * a compound Bloom filter writer.
212      */
213     void addInlineBlockWriter(InlineBlockWriter bloomWriter);
214 
215     // The below three methods take Writables.  We'd like to undo Writables but undoing the below would be pretty
216     // painful.  Could take a byte [] or a Message but we want to be backward compatible around hfiles so would need
217     // to map between Message and Writable or byte [] and current Writable serialization.  This would be a bit of work
218     // to little gain.  Thats my thinking at moment.  St.Ack 20121129
219 
220     void appendMetaBlock(String bloomFilterMetaKey, Writable metaWriter);
221 
222     /**
223      * Store general Bloom filter in the file. This does not deal with Bloom filter
224      * internals but is necessary, since Bloom filters are stored differently
225      * in HFile version 1 and version 2.
226      */
227     void addGeneralBloomFilter(BloomFilterWriter bfw);
228 
229     /**
230      * Store delete family Bloom filter in the file, which is only supported in
231      * HFile V2.
232      */
233     void addDeleteFamilyBloomFilter(BloomFilterWriter bfw) throws IOException;
234 
235     /**
236      * Return the file context for the HFile this writer belongs to
237      */
238     HFileContext getFileContext();
239   }
240 
241   /**
242    * This variety of ways to construct writers is used throughout the code, and
243    * we want to be able to swap writer implementations.
244    */
245   public static abstract class WriterFactory {
246     protected final Configuration conf;
247     protected final CacheConfig cacheConf;
248     protected FileSystem fs;
249     protected Path path;
250     protected FSDataOutputStream ostream;
251     protected CellComparator comparator = 
252         CellComparator.COMPARATOR;
253     protected InetSocketAddress[] favoredNodes;
254     private HFileContext fileContext;
255 
256     WriterFactory(Configuration conf, CacheConfig cacheConf) {
257       this.conf = conf;
258       this.cacheConf = cacheConf;
259     }
260 
261     public WriterFactory withPath(FileSystem fs, Path path) {
262       Preconditions.checkNotNull(fs);
263       Preconditions.checkNotNull(path);
264       this.fs = fs;
265       this.path = path;
266       return this;
267     }
268 
269     public WriterFactory withOutputStream(FSDataOutputStream ostream) {
270       Preconditions.checkNotNull(ostream);
271       this.ostream = ostream;
272       return this;
273     }
274 
275     public WriterFactory withComparator(CellComparator comparator) {
276       Preconditions.checkNotNull(comparator);
277       this.comparator = comparator;
278       return this;
279     }
280 
281     public WriterFactory withFavoredNodes(InetSocketAddress[] favoredNodes) {
282       // Deliberately not checking for null here.
283       this.favoredNodes = favoredNodes;
284       return this;
285     }
286 
287     public WriterFactory withFileContext(HFileContext fileContext) {
288       this.fileContext = fileContext;
289       return this;
290     }
291 
292     public Writer create() throws IOException {
293       if ((path != null ? 1 : 0) + (ostream != null ? 1 : 0) != 1) {
294         throw new AssertionError("Please specify exactly one of " +
295             "filesystem/path or path");
296       }
297       if (path != null) {
298         ostream = HFileWriterImpl.createOutputStream(conf, fs, path, favoredNodes);
299       }
300       return createWriter(fs, path, ostream,
301                    comparator, fileContext);
302     }
303 
304     protected abstract Writer createWriter(FileSystem fs, Path path, FSDataOutputStream ostream,
305         CellComparator comparator, HFileContext fileContext) throws IOException;
306   }
307 
308   /** The configuration key for HFile version to use for new files */
309   public static final String FORMAT_VERSION_KEY = "hfile.format.version";
310 
311   public static int getFormatVersion(Configuration conf) {
312     int version = conf.getInt(FORMAT_VERSION_KEY, MAX_FORMAT_VERSION);
313     checkFormatVersion(version);
314     return version;
315   }
316 
317   /**
318    * Returns the factory to be used to create {@link HFile} writers.
319    * Disables block cache access for all writers created through the
320    * returned factory.
321    */
322   public static final WriterFactory getWriterFactoryNoCache(Configuration
323        conf) {
324     Configuration tempConf = new Configuration(conf);
325     tempConf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.0f);
326     return HFile.getWriterFactory(conf, new CacheConfig(tempConf));
327   }
328 
329   /**
330    * Returns the factory to be used to create {@link HFile} writers
331    */
332   public static final WriterFactory getWriterFactory(Configuration conf,
333       CacheConfig cacheConf) {
334     int version = getFormatVersion(conf);
335     switch (version) {
336     case 2:
337       throw new IllegalArgumentException("This should never happen. " +
338         "Did you change hfile.format.version to read v2? This version of the software writes v3" +
339         " hfiles only (but it can read v2 files without having to update hfile.format.version " +
340         "in hbase-site.xml)");
341     case 3:
342       return new HFileWriterFactory(conf, cacheConf);
343     default:
344       throw new IllegalArgumentException("Cannot create writer for HFile " +
345           "format version " + version);
346     }
347   }
348 
349   /**
350    * An abstraction used by the block index.
351    * Implementations will check cache for any asked-for block and return cached block if found.
352    * Otherwise, after reading from fs, will try and put block into cache before returning.
353    */
354   public interface CachingBlockReader {
355     /**
356      * Read in a file block.
357      * @param offset offset to read.
358      * @param onDiskBlockSize size of the block
359      * @param cacheBlock
360      * @param pread
361      * @param isCompaction is this block being read as part of a compaction
362      * @param expectedBlockType the block type we are expecting to read with this read operation,
363      *  or null to read whatever block type is available and avoid checking (that might reduce
364      *  caching efficiency of encoded data blocks)
365      * @param expectedDataBlockEncoding the data block encoding the caller is expecting data blocks
366      *  to be in, or null to not perform this check and return the block irrespective of the
367      *  encoding. This check only applies to data blocks and can be set to null when the caller is
368      *  expecting to read a non-data block and has set expectedBlockType accordingly.
369      * @return Block wrapped in a ByteBuffer.
370      * @throws IOException
371      */
372     HFileBlock readBlock(long offset, long onDiskBlockSize,
373         boolean cacheBlock, final boolean pread, final boolean isCompaction,
374         final boolean updateCacheMetrics, BlockType expectedBlockType,
375         DataBlockEncoding expectedDataBlockEncoding)
376         throws IOException;
377   }
378 
379   /** An interface used by clients to open and iterate an {@link HFile}. */
380   public interface Reader extends Closeable, CachingBlockReader {
381     /**
382      * Returns this reader's "name". Usually the last component of the path.
383      * Needs to be constant as the file is being moved to support caching on
384      * write.
385      */
386     String getName();
387 
388     CellComparator getComparator();
389 
390     HFileScanner getScanner(boolean cacheBlocks, final boolean pread, final boolean isCompaction);
391 
392     ByteBuffer getMetaBlock(String metaBlockName, boolean cacheBlock) throws IOException;
393 
394     Map<byte[], byte[]> loadFileInfo() throws IOException;
395 
396     byte[] getLastKey();
397 
398     Cell midkey() throws IOException;
399 
400     long length();
401 
402     long getEntries();
403 
404     Cell getFirstKey();
405 
406     long indexSize();
407 
408     byte[] getFirstRowKey();
409 
410     byte[] getLastRowKey();
411 
412     FixedFileTrailer getTrailer();
413 
414     HFileBlockIndex.BlockIndexReader getDataBlockIndexReader();
415 
416     HFileScanner getScanner(boolean cacheBlocks, boolean pread);
417 
418     Compression.Algorithm getCompressionAlgorithm();
419 
420     /**
421      * Retrieves general Bloom filter metadata as appropriate for each
422      * {@link HFile} version.
423      * Knows nothing about how that metadata is structured.
424      */
425     DataInput getGeneralBloomFilterMetadata() throws IOException;
426 
427     /**
428      * Retrieves delete family Bloom filter metadata as appropriate for each
429      * {@link HFile}  version.
430      * Knows nothing about how that metadata is structured.
431      */
432     DataInput getDeleteBloomFilterMetadata() throws IOException;
433 
434     Path getPath();
435 
436     /** Close method with optional evictOnClose */
437     void close(boolean evictOnClose) throws IOException;
438 
439     DataBlockEncoding getDataBlockEncoding();
440 
441     boolean hasMVCCInfo();
442 
443     /**
444      * Return the file context of the HFile this reader belongs to
445      */
446     HFileContext getFileContext();
447 
448     boolean shouldIncludeMemstoreTS();
449 
450     boolean isDecodeMemstoreTS();
451 
452     DataBlockEncoding getEffectiveEncodingInCache(boolean isCompaction);
453 
454     @VisibleForTesting
455     HFileBlock.FSReader getUncachedBlockReader();
456 
457     @VisibleForTesting
458     boolean prefetchComplete();
459   }
460 
461   /**
462    * Method returns the reader given the specified arguments.
463    * TODO This is a bad abstraction.  See HBASE-6635.
464    *
465    * @param path hfile's path
466    * @param fsdis stream of path's file
467    * @param size max size of the trailer.
468    * @param cacheConf Cache configuation values, cannot be null.
469    * @param hfs
470    * @return an appropriate instance of HFileReader
471    * @throws IOException If file is invalid, will throw CorruptHFileException flavored IOException
472    */
473   private static Reader pickReaderVersion(Path path, FSDataInputStreamWrapper fsdis,
474       long size, CacheConfig cacheConf, HFileSystem hfs, Configuration conf) throws IOException {
475     FixedFileTrailer trailer = null;
476     try {
477       boolean isHBaseChecksum = fsdis.shouldUseHBaseChecksum();
478       assert !isHBaseChecksum; // Initially we must read with FS checksum.
479       trailer = FixedFileTrailer.readFromStream(fsdis.getStream(isHBaseChecksum), size);
480       switch (trailer.getMajorVersion()) {
481       case 2:
482         LOG.debug("Opening HFile v2 with v3 reader");
483         // Fall through.
484       case 3 :
485         return new HFileReaderImpl(path, trailer, fsdis, size, cacheConf, hfs, conf);
486       default:
487         throw new IllegalArgumentException("Invalid HFile version " + trailer.getMajorVersion());
488       }
489     } catch (Throwable t) {
490       try {
491         fsdis.close();
492       } catch (Throwable t2) {
493         LOG.warn("Error closing fsdis FSDataInputStreamWrapper", t2);
494       }
495       throw new CorruptHFileException("Problem reading HFile Trailer from file " + path, t);
496     }
497   }
498 
499   /**
500    * @param fs A file system
501    * @param path Path to HFile
502    * @param fsdis a stream of path's file
503    * @param size max size of the trailer.
504    * @param cacheConf Cache configuration for hfile's contents
505    * @param conf Configuration
506    * @return A version specific Hfile Reader
507    * @throws IOException If file is invalid, will throw CorruptHFileException flavored IOException
508    */
509   @SuppressWarnings("resource")
510   public static Reader createReader(FileSystem fs, Path path,
511       FSDataInputStreamWrapper fsdis, long size, CacheConfig cacheConf, Configuration conf)
512       throws IOException {
513     HFileSystem hfs = null;
514 
515     // If the fs is not an instance of HFileSystem, then create an
516     // instance of HFileSystem that wraps over the specified fs.
517     // In this case, we will not be able to avoid checksumming inside
518     // the filesystem.
519     if (!(fs instanceof HFileSystem)) {
520       hfs = new HFileSystem(fs);
521     } else {
522       hfs = (HFileSystem)fs;
523     }
524     return pickReaderVersion(path, fsdis, size, cacheConf, hfs, conf);
525   }
526 
527   /**
528    *
529    * @param fs filesystem
530    * @param path Path to file to read
531    * @param cacheConf This must not be null.  @see {@link org.apache.hadoop.hbase.io.hfile.CacheConfig#CacheConfig(Configuration)}
532    * @return an active Reader instance
533    * @throws IOException Will throw a CorruptHFileException (DoNotRetryIOException subtype) if hfile is corrupt/invalid.
534    */
535   public static Reader createReader(
536       FileSystem fs, Path path, CacheConfig cacheConf, Configuration conf) throws IOException {
537     Preconditions.checkNotNull(cacheConf, "Cannot create Reader with null CacheConf");
538     FSDataInputStreamWrapper stream = new FSDataInputStreamWrapper(fs, path);
539     return pickReaderVersion(path, stream, fs.getFileStatus(path).getLen(),
540       cacheConf, stream.getHfs(), conf);
541   }
542 
543   /**
544    * This factory method is used only by unit tests
545    */
546   static Reader createReaderFromStream(Path path,
547       FSDataInputStream fsdis, long size, CacheConfig cacheConf, Configuration conf)
548       throws IOException {
549     FSDataInputStreamWrapper wrapper = new FSDataInputStreamWrapper(fsdis);
550     return pickReaderVersion(path, wrapper, size, cacheConf, null, conf);
551   }
552 
553   /**
554    * Returns true if the specified file has a valid HFile Trailer.
555    * @param fs filesystem
556    * @param path Path to file to verify
557    * @return true if the file has a valid HFile Trailer, otherwise false
558    * @throws IOException if failed to read from the underlying stream
559    */
560   public static boolean isHFileFormat(final FileSystem fs, final Path path) throws IOException {
561     return isHFileFormat(fs, fs.getFileStatus(path));
562   }
563 
564   /**
565    * Returns true if the specified file has a valid HFile Trailer.
566    * @param fs filesystem
567    * @param fileStatus the file to verify
568    * @return true if the file has a valid HFile Trailer, otherwise false
569    * @throws IOException if failed to read from the underlying stream
570    */
571   public static boolean isHFileFormat(final FileSystem fs, final FileStatus fileStatus)
572       throws IOException {
573     final Path path = fileStatus.getPath();
574     final long size = fileStatus.getLen();
575     FSDataInputStreamWrapper fsdis = new FSDataInputStreamWrapper(fs, path);
576     try {
577       boolean isHBaseChecksum = fsdis.shouldUseHBaseChecksum();
578       assert !isHBaseChecksum; // Initially we must read with FS checksum.
579       FixedFileTrailer.readFromStream(fsdis.getStream(isHBaseChecksum), size);
580       return true;
581     } catch (IllegalArgumentException e) {
582       return false;
583     } catch (IOException e) {
584       throw e;
585     } finally {
586       try {
587         fsdis.close();
588       } catch (Throwable t) {
589         LOG.warn("Error closing fsdis FSDataInputStreamWrapper: " + path, t);
590       }
591     }
592   }
593 
594   /**
595    * Metadata for this file. Conjured by the writer. Read in by the reader.
596    */
597   public static class FileInfo implements SortedMap<byte[], byte[]> {
598     static final String RESERVED_PREFIX = "hfile.";
599     static final byte[] RESERVED_PREFIX_BYTES = Bytes.toBytes(RESERVED_PREFIX);
600     static final byte [] LASTKEY = Bytes.toBytes(RESERVED_PREFIX + "LASTKEY");
601     static final byte [] AVG_KEY_LEN = Bytes.toBytes(RESERVED_PREFIX + "AVG_KEY_LEN");
602     static final byte [] AVG_VALUE_LEN = Bytes.toBytes(RESERVED_PREFIX + "AVG_VALUE_LEN");
603     static final byte [] CREATE_TIME_TS = Bytes.toBytes(RESERVED_PREFIX + "CREATE_TIME_TS");
604     static final byte [] COMPARATOR = Bytes.toBytes(RESERVED_PREFIX + "COMPARATOR");
605     static final byte [] TAGS_COMPRESSED = Bytes.toBytes(RESERVED_PREFIX + "TAGS_COMPRESSED");
606     public static final byte [] MAX_TAGS_LEN = Bytes.toBytes(RESERVED_PREFIX + "MAX_TAGS_LEN");
607     private final SortedMap<byte [], byte []> map = new TreeMap<byte [], byte []>(Bytes.BYTES_COMPARATOR);
608 
609     public FileInfo() {
610       super();
611     }
612 
613     /**
614      * Append the given key/value pair to the file info, optionally checking the
615      * key prefix.
616      *
617      * @param k key to add
618      * @param v value to add
619      * @param checkPrefix whether to check that the provided key does not start
620      *          with the reserved prefix
621      * @return this file info object
622      * @throws IOException if the key or value is invalid
623      */
624     public FileInfo append(final byte[] k, final byte[] v,
625         final boolean checkPrefix) throws IOException {
626       if (k == null || v == null) {
627         throw new NullPointerException("Key nor value may be null");
628       }
629       if (checkPrefix && isReservedFileInfoKey(k)) {
630         throw new IOException("Keys with a " + FileInfo.RESERVED_PREFIX
631             + " are reserved");
632       }
633       put(k, v);
634       return this;
635     }
636 
637     public void clear() {
638       this.map.clear();
639     }
640 
641     public Comparator<? super byte[]> comparator() {
642       return map.comparator();
643     }
644 
645     public boolean containsKey(Object key) {
646       return map.containsKey(key);
647     }
648 
649     public boolean containsValue(Object value) {
650       return map.containsValue(value);
651     }
652 
653     public Set<java.util.Map.Entry<byte[], byte[]>> entrySet() {
654       return map.entrySet();
655     }
656 
657     public boolean equals(Object o) {
658       return map.equals(o);
659     }
660 
661     public byte[] firstKey() {
662       return map.firstKey();
663     }
664 
665     public byte[] get(Object key) {
666       return map.get(key);
667     }
668 
669     public int hashCode() {
670       return map.hashCode();
671     }
672 
673     public SortedMap<byte[], byte[]> headMap(byte[] toKey) {
674       return this.map.headMap(toKey);
675     }
676 
677     public boolean isEmpty() {
678       return map.isEmpty();
679     }
680 
681     public Set<byte[]> keySet() {
682       return map.keySet();
683     }
684 
685     public byte[] lastKey() {
686       return map.lastKey();
687     }
688 
689     public byte[] put(byte[] key, byte[] value) {
690       return this.map.put(key, value);
691     }
692 
693     public void putAll(Map<? extends byte[], ? extends byte[]> m) {
694       this.map.putAll(m);
695     }
696 
697     public byte[] remove(Object key) {
698       return this.map.remove(key);
699     }
700 
701     public int size() {
702       return map.size();
703     }
704 
705     public SortedMap<byte[], byte[]> subMap(byte[] fromKey, byte[] toKey) {
706       return this.map.subMap(fromKey, toKey);
707     }
708 
709     public SortedMap<byte[], byte[]> tailMap(byte[] fromKey) {
710       return this.map.tailMap(fromKey);
711     }
712 
713     public Collection<byte[]> values() {
714       return map.values();
715     }
716 
717     /**
718      * Write out this instance on the passed in <code>out</code> stream.
719      * We write it as a protobuf.
720      * @param out
721      * @throws IOException
722      * @see #read(DataInputStream)
723      */
724     void write(final DataOutputStream out) throws IOException {
725       HFileProtos.FileInfoProto.Builder builder = HFileProtos.FileInfoProto.newBuilder();
726       for (Map.Entry<byte [], byte[]> e: this.map.entrySet()) {
727         HBaseProtos.BytesBytesPair.Builder bbpBuilder = HBaseProtos.BytesBytesPair.newBuilder();
728         bbpBuilder.setFirst(ByteStringer.wrap(e.getKey()));
729         bbpBuilder.setSecond(ByteStringer.wrap(e.getValue()));
730         builder.addMapEntry(bbpBuilder.build());
731       }
732       out.write(ProtobufMagic.PB_MAGIC);
733       builder.build().writeDelimitedTo(out);
734     }
735 
736     /**
737      * Populate this instance with what we find on the passed in <code>in</code> stream.
738      * Can deserialize protobuf of old Writables format.
739      * @param in
740      * @throws IOException
741      * @see #write(DataOutputStream)
742      */
743     void read(final DataInputStream in) throws IOException {
744       // This code is tested over in TestHFileReaderV1 where we read an old hfile w/ this new code.
745       int pblen = ProtobufUtil.lengthOfPBMagic();
746       byte [] pbuf = new byte[pblen];
747       if (in.markSupported()) in.mark(pblen);
748       int read = in.read(pbuf);
749       if (read != pblen) throw new IOException("read=" + read + ", wanted=" + pblen);
750       if (ProtobufUtil.isPBMagicPrefix(pbuf)) {
751         parsePB(HFileProtos.FileInfoProto.parseDelimitedFrom(in));
752       } else {
753         if (in.markSupported()) {
754           in.reset();
755           parseWritable(in);
756         } else {
757           // We cannot use BufferedInputStream, it consumes more than we read from the underlying IS
758           ByteArrayInputStream bais = new ByteArrayInputStream(pbuf);
759           SequenceInputStream sis = new SequenceInputStream(bais, in); // Concatenate input streams
760           // TODO: Am I leaking anything here wrapping the passed in stream?  We are not calling close on the wrapped
761           // streams but they should be let go after we leave this context?  I see that we keep a reference to the
762           // passed in inputstream but since we no longer have a reference to this after we leave, we should be ok.
763           parseWritable(new DataInputStream(sis));
764         }
765       }
766     }
767 
768     /** Now parse the old Writable format.  It was a list of Map entries.  Each map entry was a key and a value of
769      * a byte [].  The old map format had a byte before each entry that held a code which was short for the key or
770      * value type.  We know it was a byte [] so in below we just read and dump it.
771      * @throws IOException
772      */
773     void parseWritable(final DataInputStream in) throws IOException {
774       // First clear the map.  Otherwise we will just accumulate entries every time this method is called.
775       this.map.clear();
776       // Read the number of entries in the map
777       int entries = in.readInt();
778       // Then read each key/value pair
779       for (int i = 0; i < entries; i++) {
780         byte [] key = Bytes.readByteArray(in);
781         // We used to read a byte that encoded the class type.  Read and ignore it because it is always byte [] in hfile
782         in.readByte();
783         byte [] value = Bytes.readByteArray(in);
784         this.map.put(key, value);
785       }
786     }
787 
788     /**
789      * Fill our map with content of the pb we read off disk
790      * @param fip protobuf message to read
791      */
792     void parsePB(final HFileProtos.FileInfoProto fip) {
793       this.map.clear();
794       for (BytesBytesPair pair: fip.getMapEntryList()) {
795         this.map.put(pair.getFirst().toByteArray(), pair.getSecond().toByteArray());
796       }
797     }
798   }
799 
800   /** Return true if the given file info key is reserved for internal use. */
801   public static boolean isReservedFileInfoKey(byte[] key) {
802     return Bytes.startsWith(key, FileInfo.RESERVED_PREFIX_BYTES);
803   }
804 
805   /**
806    * Get names of supported compression algorithms. The names are acceptable by
807    * HFile.Writer.
808    *
809    * @return Array of strings, each represents a supported compression
810    *         algorithm. Currently, the following compression algorithms are
811    *         supported.
812    *         <ul>
813    *         <li>"none" - No compression.
814    *         <li>"gz" - GZIP compression.
815    *         </ul>
816    */
817   public static String[] getSupportedCompressionAlgorithms() {
818     return Compression.getSupportedAlgorithms();
819   }
820 
821   // Utility methods.
822   /*
823    * @param l Long to convert to an int.
824    * @return <code>l</code> cast as an int.
825    */
826   static int longToInt(final long l) {
827     // Expecting the size() of a block not exceeding 4GB. Assuming the
828     // size() will wrap to negative integer if it exceeds 2GB (From tfile).
829     return (int)(l & 0x00000000ffffffffL);
830   }
831 
832   /**
833    * Returns all HFiles belonging to the given region directory. Could return an
834    * empty list.
835    *
836    * @param fs  The file system reference.
837    * @param regionDir  The region directory to scan.
838    * @return The list of files found.
839    * @throws IOException When scanning the files fails.
840    */
841   static List<Path> getStoreFiles(FileSystem fs, Path regionDir)
842       throws IOException {
843     List<Path> regionHFiles = new ArrayList<Path>();
844     PathFilter dirFilter = new FSUtils.DirFilter(fs);
845     FileStatus[] familyDirs = fs.listStatus(regionDir, dirFilter);
846     for(FileStatus dir : familyDirs) {
847       FileStatus[] files = fs.listStatus(dir.getPath());
848       for (FileStatus file : files) {
849         if (!file.isDirectory() &&
850             (!file.getPath().toString().contains(HConstants.HREGION_OLDLOGDIR_NAME)) &&
851             (!file.getPath().toString().contains(HConstants.RECOVERED_EDITS_DIR))) {
852           regionHFiles.add(file.getPath());
853         }
854       }
855     }
856     return regionHFiles;
857   }
858 
859   /**
860    * Checks the given {@link HFile} format version, and throws an exception if
861    * invalid. Note that if the version number comes from an input file and has
862    * not been verified, the caller needs to re-throw an {@link IOException} to
863    * indicate that this is not a software error, but corrupted input.
864    *
865    * @param version an HFile version
866    * @throws IllegalArgumentException if the version is invalid
867    */
868   public static void checkFormatVersion(int version)
869       throws IllegalArgumentException {
870     if (version < MIN_FORMAT_VERSION || version > MAX_FORMAT_VERSION) {
871       throw new IllegalArgumentException("Invalid HFile version: " + version
872           + " (expected to be " + "between " + MIN_FORMAT_VERSION + " and "
873           + MAX_FORMAT_VERSION + ")");
874     }
875   }
876 
877 
878   public static void checkHFileVersion(final Configuration c) {
879     int version = c.getInt(FORMAT_VERSION_KEY, MAX_FORMAT_VERSION);
880     if (version < MAX_FORMAT_VERSION || version > MAX_FORMAT_VERSION) {
881       throw new IllegalArgumentException("The setting for " + FORMAT_VERSION_KEY +
882         " (in your hbase-*.xml files) is " + version + " which does not match " +
883         MAX_FORMAT_VERSION +
884         "; are you running with a configuration from an older or newer hbase install (an " +
885         "incompatible hbase-default.xml or hbase-site.xml on your CLASSPATH)?");
886     }
887   }
888 
889   public static void main(String[] args) throws Exception {
890     // delegate to preserve old behavior
891     HFilePrettyPrinter.main(args);
892   }
893 }