001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.io.hfile;
019
020import java.io.Closeable;
021import java.io.DataInput;
022import java.io.IOException;
023import java.net.InetSocketAddress;
024import java.util.ArrayList;
025import java.util.List;
026import java.util.Optional;
027import java.util.concurrent.atomic.LongAdder;
028import org.apache.commons.io.IOUtils;
029import org.apache.hadoop.conf.Configuration;
030import org.apache.hadoop.fs.FSDataOutputStream;
031import org.apache.hadoop.fs.FileStatus;
032import org.apache.hadoop.fs.FileSystem;
033import org.apache.hadoop.fs.Path;
034import org.apache.hadoop.fs.PathFilter;
035import org.apache.hadoop.hbase.Cell;
036import org.apache.hadoop.hbase.CellComparator;
037import org.apache.hadoop.hbase.HConstants;
038import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
039import org.apache.hadoop.hbase.io.MetricsIO;
040import org.apache.hadoop.hbase.io.compress.Compression;
041import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
042import org.apache.hadoop.hbase.io.hfile.ReaderContext.ReaderType;
043import org.apache.hadoop.hbase.ipc.RpcServer;
044import org.apache.hadoop.hbase.regionserver.CellSink;
045import org.apache.hadoop.hbase.regionserver.ShipperListener;
046import org.apache.hadoop.hbase.util.BloomFilterWriter;
047import org.apache.hadoop.hbase.util.Bytes;
048import org.apache.hadoop.hbase.util.FSUtils;
049import org.apache.hadoop.io.Writable;
050import org.apache.yetus.audience.InterfaceAudience;
051import org.slf4j.Logger;
052import org.slf4j.LoggerFactory;
053
054import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
055
056/**
057 * File format for hbase. A file of sorted key/value pairs. Both keys and values are byte arrays.
058 * <p>
059 * The memory footprint of a HFile includes the following (below is taken from the <a
060 * href=https://issues.apache.org/jira/browse/HADOOP-3315>TFile</a> documentation but applies also
061 * to HFile):
062 * <ul>
063 * <li>Some constant overhead of reading or writing a compressed block.
064 * <ul>
065 * <li>Each compressed block requires one compression/decompression codec for I/O.
066 * <li>Temporary space to buffer the key.
067 * <li>Temporary space to buffer the value.
068 * </ul>
069 * <li>HFile index, which is proportional to the total number of Data Blocks. The total amount of
070 * memory needed to hold the index can be estimated as (56+AvgKeySize)*NumBlocks.
071 * </ul>
072 * Suggestions on performance optimization.
073 * <ul>
074 * <li>Minimum block size. We recommend a setting of minimum block size between 8KB to 1MB for
075 * general usage. Larger block size is preferred if files are primarily for sequential access.
076 * However, it would lead to inefficient random access (because there are more data to decompress).
077 * Smaller blocks are good for random access, but require more memory to hold the block index, and
078 * may be slower to create (because we must flush the compressor stream at the conclusion of each
079 * data block, which leads to an FS I/O flush). Further, due to the internal caching in Compression
080 * codec, the smallest possible block size would be around 20KB-30KB.
081 * <li>The current implementation does not offer true multi-threading for reading. The
082 * implementation uses FSDataInputStream seek()+read(), which is shown to be much faster than
083 * positioned-read call in single thread mode. However, it also means that if multiple threads
084 * attempt to access the same HFile (using multiple scanners) simultaneously, the actual I/O is
085 * carried out sequentially even if they access different DFS blocks (Reexamine! pread seems to be
086 * 10% faster than seek+read in my testing -- stack).
087 * <li>Compression codec. Use "none" if the data is not very compressable (by compressable, I mean a
088 * compression ratio at least 2:1). Generally, use "lzo" as the starting point for experimenting.
089 * "gz" overs slightly better compression ratio over "lzo" but requires 4x CPU to compress and 2x
090 * CPU to decompress, comparing to "lzo".
091 * </ul>
092 * For more on the background behind HFile, see <a
093 * href=https://issues.apache.org/jira/browse/HBASE-61>HBASE-61</a>.
094 * <p>
095 * File is made of data blocks followed by meta data blocks (if any), a fileinfo block, data block
096 * index, meta data block index, and a fixed size trailer which records the offsets at which file
097 * changes content type.
098 *
099 * <pre>
100 * &lt;data blocks&gt;&lt;meta blocks&gt;&lt;fileinfo&gt;&lt;
101 * data index&gt;&lt;meta index&gt;&lt;trailer&gt;
102 * </pre>
103 *
104 * Each block has a bit of magic at its start. Block are comprised of key/values. In data blocks,
105 * they are both byte arrays. Metadata blocks are a String key and a byte array value. An empty file
106 * looks like this:
107 *
108 * <pre>
109 * &lt;fileinfo&gt;&lt;trailer&gt;
110 * </pre>
111 *
112 * . That is, there are not data nor meta blocks present.
113 * <p>
114 * TODO: Do scanners need to be able to take a start and end row? TODO: Should BlockIndex know the
115 * name of its file? Should it have a Path that points at its file say for the case where an index
116 * lives apart from an HFile instance?
117 */
118@InterfaceAudience.Private
119public final class HFile {
120  // LOG is being used in HFileBlock and CheckSumUtil
121  static final Logger LOG = LoggerFactory.getLogger(HFile.class);
122
123  /**
124   * Maximum length of key in HFile.
125   */
126  public final static int MAXIMUM_KEY_LENGTH = Integer.MAX_VALUE;
127
128  /**
129   * Default compression: none.
130   */
131  public final static Compression.Algorithm DEFAULT_COMPRESSION_ALGORITHM =
132    Compression.Algorithm.NONE;
133
134  /** Minimum supported HFile format version */
135  public static final int MIN_FORMAT_VERSION = 2;
136
137  /**
138   * Maximum supported HFile format version
139   */
140  public static final int MAX_FORMAT_VERSION = 3;
141
142  /**
143   * Minimum HFile format version with support for persisting cell tags
144   */
145  public static final int MIN_FORMAT_VERSION_WITH_TAGS = 3;
146
147  /** Default compression name: none. */
148  public final static String DEFAULT_COMPRESSION = DEFAULT_COMPRESSION_ALGORITHM.getName();
149
150  /** Meta data block name for bloom filter bits. */
151  public static final String BLOOM_FILTER_DATA_KEY = "BLOOM_FILTER_DATA";
152
153  /**
154   * We assume that HFile path ends with ROOT_DIR/TABLE_NAME/REGION_NAME/CF_NAME/HFILE, so it has at
155   * least this many levels of nesting. This is needed for identifying table and CF name from an
156   * HFile path.
157   */
158  public final static int MIN_NUM_HFILE_PATH_LEVELS = 5;
159
160  /**
161   * The number of bytes per checksum.
162   */
163  public static final int DEFAULT_BYTES_PER_CHECKSUM = 16 * 1024;
164
165  // For measuring number of checksum failures
166  static final LongAdder CHECKSUM_FAILURES = new LongAdder();
167
168  // For tests. Gets incremented when we read a block whether from HDFS or from Cache.
169  public static final LongAdder DATABLOCK_READ_COUNT = new LongAdder();
170
171  /**
172   * Shutdown constructor.
173   */
174  private HFile() {
175  }
176
177  /**
178   * Number of checksum verification failures. It also clears the counter.
179   */
180  public static final long getAndResetChecksumFailuresCount() {
181    return CHECKSUM_FAILURES.sumThenReset();
182  }
183
184  /**
185   * Number of checksum verification failures. It also clears the counter.
186   */
187  public static final long getChecksumFailuresCount() {
188    return CHECKSUM_FAILURES.sum();
189  }
190
191  public static final void updateReadLatency(long latencyMillis, boolean pread, boolean tooSlow) {
192    RpcServer.getCurrentCall().ifPresent(call -> call.updateFsReadTime(latencyMillis));
193    if (pread) {
194      MetricsIO.getInstance().updateFsPreadTime(latencyMillis);
195    } else {
196      MetricsIO.getInstance().updateFsReadTime(latencyMillis);
197    }
198    if (tooSlow) {
199      MetricsIO.getInstance().incrSlowFsRead();
200    }
201  }
202
203  public static final void updateWriteLatency(long latencyMillis) {
204    MetricsIO.getInstance().updateFsWriteTime(latencyMillis);
205  }
206
207  /** API required to write an {@link HFile} */
208  public interface Writer extends Closeable, CellSink, ShipperListener {
209    /** Max memstore (mvcc) timestamp in FileInfo */
210    public static final byte[] MAX_MEMSTORE_TS_KEY = Bytes.toBytes("MAX_MEMSTORE_TS_KEY");
211
212    /** Add an element to the file info map. */
213    void appendFileInfo(byte[] key, byte[] value) throws IOException;
214
215    /** Returns the path to this {@link HFile} */
216    Path getPath();
217
218    /**
219     * Adds an inline block writer such as a multi-level block index writer or a compound Bloom
220     * filter writer.
221     */
222    void addInlineBlockWriter(InlineBlockWriter bloomWriter);
223
224    // The below three methods take Writables. We'd like to undo Writables but undoing the below
225    // would be pretty painful. Could take a byte [] or a Message but we want to be backward
226    // compatible around hfiles so would need to map between Message and Writable or byte [] and
227    // current Writable serialization. This would be a bit of work to little gain. Thats my
228    // thinking at moment. St.Ack 20121129
229
230    void appendMetaBlock(String bloomFilterMetaKey, Writable metaWriter);
231
232    /**
233     * Store general Bloom filter in the file. This does not deal with Bloom filter internals but is
234     * necessary, since Bloom filters are stored differently in HFile version 1 and version 2.
235     */
236    void addGeneralBloomFilter(BloomFilterWriter bfw);
237
238    /**
239     * Store delete family Bloom filter in the file, which is only supported in HFile V2.
240     */
241    void addDeleteFamilyBloomFilter(BloomFilterWriter bfw) throws IOException;
242
243    /**
244     * Return the file context for the HFile this writer belongs to
245     */
246    HFileContext getFileContext();
247  }
248
249  /**
250   * This variety of ways to construct writers is used throughout the code, and we want to be able
251   * to swap writer implementations.
252   */
253  public static class WriterFactory {
254    protected final Configuration conf;
255    protected final CacheConfig cacheConf;
256    protected FileSystem fs;
257    protected Path path;
258    protected FSDataOutputStream ostream;
259    protected InetSocketAddress[] favoredNodes;
260    private HFileContext fileContext;
261    protected boolean shouldDropBehind = false;
262
263    WriterFactory(Configuration conf, CacheConfig cacheConf) {
264      this.conf = conf;
265      this.cacheConf = cacheConf;
266    }
267
268    public WriterFactory withPath(FileSystem fs, Path path) {
269      Preconditions.checkNotNull(fs);
270      Preconditions.checkNotNull(path);
271      this.fs = fs;
272      this.path = path;
273      return this;
274    }
275
276    public WriterFactory withOutputStream(FSDataOutputStream ostream) {
277      Preconditions.checkNotNull(ostream);
278      this.ostream = ostream;
279      return this;
280    }
281
282    public WriterFactory withFavoredNodes(InetSocketAddress[] favoredNodes) {
283      // Deliberately not checking for null here.
284      this.favoredNodes = favoredNodes;
285      return this;
286    }
287
288    public WriterFactory withFileContext(HFileContext fileContext) {
289      this.fileContext = fileContext;
290      return this;
291    }
292
293    public WriterFactory withShouldDropCacheBehind(boolean shouldDropBehind) {
294      this.shouldDropBehind = shouldDropBehind;
295      return this;
296    }
297
298    public Writer create() throws IOException {
299      if ((path != null ? 1 : 0) + (ostream != null ? 1 : 0) != 1) {
300        throw new AssertionError("Please specify exactly one of " + "filesystem/path or path");
301      }
302      if (path != null) {
303        ostream = HFileWriterImpl.createOutputStream(conf, fs, path, favoredNodes);
304        try {
305          ostream.setDropBehind(shouldDropBehind && cacheConf.shouldDropBehindCompaction());
306        } catch (UnsupportedOperationException uoe) {
307          LOG.trace("Unable to set drop behind on {}", path, uoe);
308          LOG.debug("Unable to set drop behind on {}", path.getName());
309        }
310      }
311      return new HFileWriterImpl(conf, cacheConf, path, ostream, fileContext);
312    }
313  }
314
315  /** The configuration key for HFile version to use for new files */
316  public static final String FORMAT_VERSION_KEY = "hfile.format.version";
317
318  public static int getFormatVersion(Configuration conf) {
319    int version = conf.getInt(FORMAT_VERSION_KEY, MAX_FORMAT_VERSION);
320    checkFormatVersion(version);
321    return version;
322  }
323
324  /**
325   * Returns the factory to be used to create {@link HFile} writers. Disables block cache access for
326   * all writers created through the returned factory.
327   */
328  public static final WriterFactory getWriterFactoryNoCache(Configuration conf) {
329    return HFile.getWriterFactory(conf, CacheConfig.DISABLED);
330  }
331
332  /**
333   * Returns the factory to be used to create {@link HFile} writers
334   */
335  public static final WriterFactory getWriterFactory(Configuration conf, CacheConfig cacheConf) {
336    int version = getFormatVersion(conf);
337    switch (version) {
338      case 2:
339        throw new IllegalArgumentException("This should never happen. "
340          + "Did you change hfile.format.version to read v2? This version of the software writes v3"
341          + " hfiles only (but it can read v2 files without having to update hfile.format.version "
342          + "in hbase-site.xml)");
343      case 3:
344        return new HFile.WriterFactory(conf, cacheConf);
345      default:
346        throw new IllegalArgumentException(
347          "Cannot create writer for HFile " + "format version " + version);
348    }
349  }
350
351  /**
352   * An abstraction used by the block index. Implementations will check cache for any asked-for
353   * block and return cached block if found. Otherwise, after reading from fs, will try and put
354   * block into cache before returning.
355   */
356  public interface CachingBlockReader {
357    /**
358     * Read in a file block.
359     * @param offset                    offset to read.
360     * @param onDiskBlockSize           size of the block
361     * @param isCompaction              is this block being read as part of a compaction
362     * @param expectedBlockType         the block type we are expecting to read with this read
363     *                                  operation, or null to read whatever block type is available
364     *                                  and avoid checking (that might reduce caching efficiency of
365     *                                  encoded data blocks)
366     * @param expectedDataBlockEncoding the data block encoding the caller is expecting data blocks
367     *                                  to be in, or null to not perform this check and return the
368     *                                  block irrespective of the encoding. This check only applies
369     *                                  to data blocks and can be set to null when the caller is
370     *                                  expecting to read a non-data block and has set
371     *                                  expectedBlockType accordingly.
372     * @return Block wrapped in a ByteBuffer.
373     */
374    HFileBlock readBlock(long offset, long onDiskBlockSize, boolean cacheBlock, final boolean pread,
375      final boolean isCompaction, final boolean updateCacheMetrics, BlockType expectedBlockType,
376      DataBlockEncoding expectedDataBlockEncoding) throws IOException;
377
378    HFileBlock readBlock(long offset, long onDiskBlockSize, boolean cacheBlock, final boolean pread,
379      final boolean isCompaction, final boolean updateCacheMetrics, BlockType expectedBlockType,
380      DataBlockEncoding expectedDataBlockEncoding, boolean cacheOnly) throws IOException;
381  }
382
383  /** An interface used by clients to open and iterate an {@link HFile}. */
384  public interface Reader extends Closeable, CachingBlockReader {
385    /**
386     * Returns this reader's "name". Usually the last component of the path. Needs to be constant as
387     * the file is being moved to support caching on write.
388     */
389    String getName();
390
391    CellComparator getComparator();
392
393    HFileScanner getScanner(Configuration conf, boolean cacheBlocks, boolean pread,
394      boolean isCompaction);
395
396    HFileBlock getMetaBlock(String metaBlockName, boolean cacheBlock) throws IOException;
397
398    Optional<Cell> getLastKey();
399
400    Optional<Cell> midKey() throws IOException;
401
402    long length();
403
404    long getEntries();
405
406    Optional<Cell> getFirstKey();
407
408    long indexSize();
409
410    Optional<byte[]> getFirstRowKey();
411
412    Optional<byte[]> getLastRowKey();
413
414    FixedFileTrailer getTrailer();
415
416    void setDataBlockIndexReader(HFileBlockIndex.CellBasedKeyBlockIndexReader reader);
417
418    HFileBlockIndex.CellBasedKeyBlockIndexReader getDataBlockIndexReader();
419
420    void setMetaBlockIndexReader(HFileBlockIndex.ByteArrayKeyBlockIndexReader reader);
421
422    HFileBlockIndex.ByteArrayKeyBlockIndexReader getMetaBlockIndexReader();
423
424    HFileScanner getScanner(Configuration conf, boolean cacheBlocks, boolean pread);
425
426    /**
427     * Retrieves general Bloom filter metadata as appropriate for each {@link HFile} version. Knows
428     * nothing about how that metadata is structured.
429     */
430    DataInput getGeneralBloomFilterMetadata() throws IOException;
431
432    /**
433     * Retrieves delete family Bloom filter metadata as appropriate for each {@link HFile} version.
434     * Knows nothing about how that metadata is structured.
435     */
436    DataInput getDeleteBloomFilterMetadata() throws IOException;
437
438    Path getPath();
439
440    /** Close method with optional evictOnClose */
441    void close(boolean evictOnClose) throws IOException;
442
443    DataBlockEncoding getDataBlockEncoding();
444
445    boolean hasMVCCInfo();
446
447    /**
448     * Return the file context of the HFile this reader belongs to
449     */
450    HFileContext getFileContext();
451
452    boolean isPrimaryReplicaReader();
453
454    DataBlockEncoding getEffectiveEncodingInCache(boolean isCompaction);
455
456    HFileBlock.FSReader getUncachedBlockReader();
457
458    boolean prefetchComplete();
459
460    boolean prefetchStarted();
461
462    /**
463     * To close the stream's socket. Note: This can be concurrently called from multiple threads and
464     * implementation should take care of thread safety.
465     */
466    void unbufferStream();
467
468    ReaderContext getContext();
469
470    HFileInfo getHFileInfo();
471
472    void setDataBlockEncoder(HFileDataBlockEncoder dataBlockEncoder);
473  }
474
475  /**
476   * Method returns the reader given the specified arguments. TODO This is a bad abstraction. See
477   * HBASE-6635.
478   * @param context   Reader context info
479   * @param fileInfo  HFile info
480   * @param cacheConf Cache configuation values, cannot be null.
481   * @param conf      Configuration
482   * @return an appropriate instance of HFileReader
483   * @throws IOException If file is invalid, will throw CorruptHFileException flavored IOException
484   */
485  @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "SF_SWITCH_FALLTHROUGH",
486      justification = "Intentional")
487  public static Reader createReader(ReaderContext context, HFileInfo fileInfo,
488    CacheConfig cacheConf, Configuration conf) throws IOException {
489    try {
490      if (context.getReaderType() == ReaderType.STREAM) {
491        // stream reader will share trailer with pread reader, see HFileStreamReader#copyFields
492        return new HFileStreamReader(context, fileInfo, cacheConf, conf);
493      }
494      FixedFileTrailer trailer = fileInfo.getTrailer();
495      switch (trailer.getMajorVersion()) {
496        case 2:
497          LOG.debug("Opening HFile v2 with v3 reader");
498          // Fall through. FindBugs: SF_SWITCH_FALLTHROUGH
499        case 3:
500          return new HFilePreadReader(context, fileInfo, cacheConf, conf);
501        default:
502          throw new IllegalArgumentException("Invalid HFile version " + trailer.getMajorVersion());
503      }
504    } catch (Throwable t) {
505      IOUtils.closeQuietly(context.getInputStreamWrapper(),
506        e -> LOG.warn("failed to close input stream wrapper", e));
507      throw new CorruptHFileException(
508        "Problem reading HFile Trailer from file " + context.getFilePath(), t);
509    } finally {
510      context.getInputStreamWrapper().unbuffer();
511    }
512  }
513
514  /**
515   * Creates reader with cache configuration disabled
516   * @param fs   filesystem
517   * @param path Path to file to read
518   * @param conf Configuration
519   * @return an active Reader instance
520   * @throws IOException Will throw a CorruptHFileException (DoNotRetryIOException subtype) if hfile
521   *                     is corrupt/invalid.
522   */
523  public static Reader createReader(FileSystem fs, Path path, Configuration conf)
524    throws IOException {
525    // The primaryReplicaReader is mainly used for constructing block cache key, so if we do not use
526    // block cache then it is OK to set it as any value. We use true here.
527    return createReader(fs, path, CacheConfig.DISABLED, true, conf);
528  }
529
530  /**
531   * @param fs                   filesystem
532   * @param path                 Path to file to read
533   * @param cacheConf            This must not be null.
534   * @param primaryReplicaReader true if this is a reader for primary replica
535   * @param conf                 Configuration
536   * @return an active Reader instance
537   * @throws IOException Will throw a CorruptHFileException (DoNotRetryIOException subtype) if hfile
538   *                     is corrupt/invalid.
539   * @see CacheConfig#CacheConfig(Configuration)
540   */
541  public static Reader createReader(FileSystem fs, Path path, CacheConfig cacheConf,
542    boolean primaryReplicaReader, Configuration conf) throws IOException {
543    Preconditions.checkNotNull(cacheConf, "Cannot create Reader with null CacheConf");
544    FSDataInputStreamWrapper stream = new FSDataInputStreamWrapper(fs, path);
545    ReaderContext context =
546      new ReaderContextBuilder().withFilePath(path).withInputStreamWrapper(stream)
547        .withFileSize(fs.getFileStatus(path).getLen()).withFileSystem(stream.getHfs())
548        .withPrimaryReplicaReader(primaryReplicaReader).withReaderType(ReaderType.PREAD).build();
549    HFileInfo fileInfo = new HFileInfo(context, conf);
550    Reader reader = createReader(context, fileInfo, cacheConf, conf);
551    fileInfo.initMetaAndIndex(reader);
552    return reader;
553  }
554
555  /**
556   * Returns true if the specified file has a valid HFile Trailer.
557   * @param fs   filesystem
558   * @param path Path to file to verify
559   * @return true if the file has a valid HFile Trailer, otherwise false
560   * @throws IOException if failed to read from the underlying stream
561   */
562  public static boolean isHFileFormat(final FileSystem fs, final Path path) throws IOException {
563    return isHFileFormat(fs, fs.getFileStatus(path));
564  }
565
566  /**
567   * Returns true if the specified file has a valid HFile Trailer.
568   * @param fs         filesystem
569   * @param fileStatus the file to verify
570   * @return true if the file has a valid HFile Trailer, otherwise false
571   * @throws IOException if failed to read from the underlying stream
572   */
573  public static boolean isHFileFormat(final FileSystem fs, final FileStatus fileStatus)
574    throws IOException {
575    final Path path = fileStatus.getPath();
576    final long size = fileStatus.getLen();
577    try (FSDataInputStreamWrapper fsdis = new FSDataInputStreamWrapper(fs, path)) {
578      boolean isHBaseChecksum = fsdis.shouldUseHBaseChecksum();
579      assert !isHBaseChecksum; // Initially we must read with FS checksum.
580      FixedFileTrailer.readFromStream(fsdis.getStream(isHBaseChecksum), size);
581      return true;
582    } catch (IllegalArgumentException e) {
583      return false;
584    }
585  }
586
587  /**
588   * Get names of supported compression algorithms. The names are acceptable by HFile.Writer.
589   * @return Array of strings, each represents a supported compression algorithm. Currently, the
590   *         following compression algorithms are supported.
591   *         <ul>
592   *         <li>"none" - No compression.
593   *         <li>"gz" - GZIP compression.
594   *         </ul>
595   */
596  public static String[] getSupportedCompressionAlgorithms() {
597    return Compression.getSupportedAlgorithms();
598  }
599
600  // Utility methods.
601  /*
602   * @param l Long to convert to an int.
603   * @return <code>l</code> cast as an int.
604   */
605  static int longToInt(final long l) {
606    // Expecting the size() of a block not exceeding 4GB. Assuming the
607    // size() will wrap to negative integer if it exceeds 2GB (From tfile).
608    return (int) (l & 0x00000000ffffffffL);
609  }
610
611  /**
612   * Returns all HFiles belonging to the given region directory. Could return an empty list.
613   * @param fs        The file system reference.
614   * @param regionDir The region directory to scan.
615   * @return The list of files found.
616   * @throws IOException When scanning the files fails.
617   */
618  public static List<Path> getStoreFiles(FileSystem fs, Path regionDir) throws IOException {
619    List<Path> regionHFiles = new ArrayList<>();
620    PathFilter dirFilter = new FSUtils.DirFilter(fs);
621    FileStatus[] familyDirs = fs.listStatus(regionDir, dirFilter);
622    for (FileStatus dir : familyDirs) {
623      FileStatus[] files = fs.listStatus(dir.getPath());
624      for (FileStatus file : files) {
625        if (
626          !file.isDirectory()
627            && (!file.getPath().toString().contains(HConstants.HREGION_OLDLOGDIR_NAME))
628            && (!file.getPath().toString().contains(HConstants.RECOVERED_EDITS_DIR))
629        ) {
630          regionHFiles.add(file.getPath());
631        }
632      }
633    }
634    return regionHFiles;
635  }
636
637  /**
638   * Checks the given {@link HFile} format version, and throws an exception if invalid. Note that if
639   * the version number comes from an input file and has not been verified, the caller needs to
640   * re-throw an {@link IOException} to indicate that this is not a software error, but corrupted
641   * input.
642   * @param version an HFile version
643   * @throws IllegalArgumentException if the version is invalid
644   */
645  public static void checkFormatVersion(int version) throws IllegalArgumentException {
646    if (version < MIN_FORMAT_VERSION || version > MAX_FORMAT_VERSION) {
647      throw new IllegalArgumentException("Invalid HFile version: " + version + " (expected to be "
648        + "between " + MIN_FORMAT_VERSION + " and " + MAX_FORMAT_VERSION + ")");
649    }
650  }
651
652  public static void checkHFileVersion(final Configuration c) {
653    int version = c.getInt(FORMAT_VERSION_KEY, MAX_FORMAT_VERSION);
654    if (version < MAX_FORMAT_VERSION || version > MAX_FORMAT_VERSION) {
655      throw new IllegalArgumentException(
656        "The setting for " + FORMAT_VERSION_KEY + " (in your hbase-*.xml files) is " + version
657          + " which does not match " + MAX_FORMAT_VERSION
658          + "; are you running with a configuration from an older or newer hbase install (an "
659          + "incompatible hbase-default.xml or hbase-site.xml on your CLASSPATH)?");
660    }
661  }
662
663  public static void main(String[] args) throws Exception {
664    // delegate to preserve old behavior
665    HFilePrettyPrinter.main(args);
666  }
667}