View Javadoc

1   /*
2    * Copyright 2011 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.io.hfile;
21  
22  import static org.apache.hadoop.hbase.io.hfile.HFile.MAX_FORMAT_VERSION;
23  import static org.apache.hadoop.hbase.io.hfile.HFile.MIN_FORMAT_VERSION;
24  
25  import java.io.ByteArrayInputStream;
26  import java.io.ByteArrayOutputStream;
27  import java.io.DataInputStream;
28  import java.io.DataOutput;
29  import java.io.DataOutputStream;
30  import java.io.IOException;
31  import java.nio.ByteBuffer;
32  
33  import org.apache.commons.logging.Log;
34  import org.apache.commons.logging.LogFactory;
35  import org.apache.hadoop.fs.FSDataInputStream;
36  import org.apache.hadoop.hbase.util.Bytes;
37  import org.apache.hadoop.io.RawComparator;
38  
39  import com.google.common.io.NullOutputStream;
40  
41  /**
42   * The {@link HFile} has a fixed trailer which contains offsets to other
43   * variable parts of the file. Also includes basic metadata on this file. The
44   * trailer size is fixed within a given {@link HFile} format version only, but
45   * we always store the version number as the last four-byte integer of the file.
46   * The version number itself is split into two portions, a major 
47   * version and a minor version. 
48   * The last three bytes of a file is the major
49   * version and a single preceding byte is the minor number. The major version
50   * determines which readers/writers to use to read/write a hfile while a minor
51   * version determines smaller changes in hfile format that do not need a new
52   * reader/writer type.
53   */
54  public class FixedFileTrailer {
55  
56    private static final Log LOG = LogFactory.getLog(FixedFileTrailer.class);
57  
58    /**
59     * We store the comparator class name as a fixed-length field in the trailer.
60     */
61    private static final int MAX_COMPARATOR_NAME_LENGTH = 128;
62  
63    /**
64     * Offset to the fileinfo data, a small block of vitals. Necessary in v1 but
65     * only potentially useful for pretty-printing in v2.
66     */
67    private long fileInfoOffset;
68  
69    /**
70     * In version 1, the offset to the data block index. Starting from version 2,
71     * the meaning of this field is the offset to the section of the file that
72     * should be loaded at the time the file is being opened, and as of the time
73     * of writing, this happens to be the offset of the file info section.
74     */
75    private long loadOnOpenDataOffset;
76  
77    /** The number of entries in the root data index. */
78    private int dataIndexCount;
79  
80    /** Total uncompressed size of all blocks of the data index */
81    private long uncompressedDataIndexSize;
82  
83    /** The number of entries in the meta index */
84    private int metaIndexCount;
85  
86    /** The total uncompressed size of keys/values stored in the file. */
87    private long totalUncompressedBytes;
88  
89    /**
90     * The number of key/value pairs in the file. This field was int in version 1,
91     * but is now long.
92     */
93    private long entryCount;
94  
95    /** The compression codec used for all blocks. */
96    private Compression.Algorithm compressionCodec = Compression.Algorithm.NONE;
97  
98    /**
99     * The number of levels in the potentially multi-level data index. Used from
100    * version 2 onwards.
101    */
102   private int numDataIndexLevels;
103 
104   /** The offset of the first data block. */
105   private long firstDataBlockOffset;
106 
107   /**
108    * It is guaranteed that no key/value data blocks start after this offset in
109    * the file.
110    */
111   private long lastDataBlockOffset;
112 
113   /** Raw key comparator class name in version 2 */
114   private String comparatorClassName = RawComparator.class.getName();
115 
116   /** The {@link HFile} format major version. */
117   private final int majorVersion;
118 
119   /** The {@link HFile} format minor version. */
120   private final int minorVersion;
121 
122   FixedFileTrailer(int majorVersion, int minorVersion) {
123     this.majorVersion = majorVersion;
124     this.minorVersion = minorVersion;
125     HFile.checkFormatVersion(majorVersion);
126   }
127 
128   private static int[] computeTrailerSizeByVersion() {
129     int versionToSize[] = new int[HFile.MAX_FORMAT_VERSION + 1];
130     for (int version = MIN_FORMAT_VERSION;
131          version <= MAX_FORMAT_VERSION;
132          ++version) {
133       FixedFileTrailer fft = new FixedFileTrailer(version, 
134                                    HFileBlock.MINOR_VERSION_NO_CHECKSUM);
135       DataOutputStream dos = new DataOutputStream(new NullOutputStream());
136       try {
137         fft.serialize(dos);
138       } catch (IOException ex) {
139         // The above has no reason to fail.
140         throw new RuntimeException(ex);
141       }
142       versionToSize[version] = dos.size();
143     }
144     return versionToSize;
145   }
146 
147   private static int getMaxTrailerSize() {
148     int maxSize = 0;
149     for (int version = MIN_FORMAT_VERSION;
150          version <= MAX_FORMAT_VERSION;
151          ++version)
152       maxSize = Math.max(getTrailerSize(version), maxSize);
153     return maxSize;
154   }
155 
156   private static final int TRAILER_SIZE[] = computeTrailerSizeByVersion();
157   private static final int MAX_TRAILER_SIZE = getMaxTrailerSize();
158 
159   static int getTrailerSize(int version) {
160     return TRAILER_SIZE[version];
161   }
162 
163   public int getTrailerSize() {
164     return getTrailerSize(majorVersion);
165   }
166 
167   /**
168    * Write the trailer to a data stream. We support writing version 1 for
169    * testing and for determining version 1 trailer size. It is also easy to see
170    * what fields changed in version 2.
171    *
172    * @param outputStream
173    * @throws IOException
174    */
175   void serialize(DataOutputStream outputStream) throws IOException {
176     HFile.checkFormatVersion(majorVersion);
177 
178     ByteArrayOutputStream baos = new ByteArrayOutputStream();
179     DataOutput baosDos = new DataOutputStream(baos);
180 
181     BlockType.TRAILER.write(baosDos);
182     baosDos.writeLong(fileInfoOffset);
183     baosDos.writeLong(loadOnOpenDataOffset);
184     baosDos.writeInt(dataIndexCount);
185 
186     if (majorVersion == 1) {
187       // This used to be metaIndexOffset, but it was not used in version 1.
188       baosDos.writeLong(0);
189     } else {
190       baosDos.writeLong(uncompressedDataIndexSize);
191     }
192 
193     baosDos.writeInt(metaIndexCount);
194     baosDos.writeLong(totalUncompressedBytes);
195     if (majorVersion == 1) {
196       baosDos.writeInt((int) Math.min(Integer.MAX_VALUE, entryCount));
197     } else {
198       // This field is long from version 2 onwards.
199       baosDos.writeLong(entryCount);
200     }
201     baosDos.writeInt(compressionCodec.ordinal());
202 
203     if (majorVersion > 1) {
204       baosDos.writeInt(numDataIndexLevels);
205       baosDos.writeLong(firstDataBlockOffset);
206       baosDos.writeLong(lastDataBlockOffset);
207       Bytes.writeStringFixedSize(baosDos, comparatorClassName,
208           MAX_COMPARATOR_NAME_LENGTH);
209     }
210 
211     // serialize the major and minor versions
212     baosDos.writeInt(materializeVersion(majorVersion, minorVersion));
213 
214     outputStream.write(baos.toByteArray());
215   }
216 
217   /**
218    * Deserialize the fixed file trailer from the given stream. The version needs
219    * to already be specified. Make sure this is consistent with
220    * {@link #serialize(DataOutputStream)}.
221    *
222    * @param inputStream
223    * @param version
224    * @throws IOException
225    */
226   void deserialize(DataInputStream inputStream) throws IOException {
227     HFile.checkFormatVersion(majorVersion);
228 
229     BlockType.TRAILER.readAndCheck(inputStream);
230 
231     fileInfoOffset = inputStream.readLong();
232     loadOnOpenDataOffset = inputStream.readLong();
233     dataIndexCount = inputStream.readInt();
234 
235     if (majorVersion == 1) {
236       inputStream.readLong(); // Read and skip metaIndexOffset.
237     } else {
238       uncompressedDataIndexSize = inputStream.readLong();
239     }
240     metaIndexCount = inputStream.readInt();
241 
242     totalUncompressedBytes = inputStream.readLong();
243     entryCount = majorVersion == 1 ? inputStream.readInt() : inputStream.readLong();
244     compressionCodec = Compression.Algorithm.values()[inputStream.readInt()];
245     if (majorVersion > 1) {
246       numDataIndexLevels = inputStream.readInt();
247       firstDataBlockOffset = inputStream.readLong();
248       lastDataBlockOffset = inputStream.readLong();
249       comparatorClassName =
250           Bytes.readStringFixedSize(inputStream, MAX_COMPARATOR_NAME_LENGTH);
251     }
252 
253     int version = inputStream.readInt();
254     expectMajorVersion(extractMajorVersion(version));
255     expectMinorVersion(extractMinorVersion(version));
256   }
257 
258   private void append(StringBuilder sb, String s) {
259     if (sb.length() > 0)
260       sb.append(", ");
261     sb.append(s);
262   }
263 
264   @Override
265   public String toString() {
266     StringBuilder sb = new StringBuilder();
267     append(sb, "fileinfoOffset=" + fileInfoOffset);
268     append(sb, "loadOnOpenDataOffset=" + loadOnOpenDataOffset);
269     append(sb, "dataIndexCount=" + dataIndexCount);
270     append(sb, "metaIndexCount=" + metaIndexCount);
271     append(sb, "totalUncomressedBytes=" + totalUncompressedBytes);
272     append(sb, "entryCount=" + entryCount);
273     append(sb, "compressionCodec=" + compressionCodec);
274     if (majorVersion == 2) {
275       append(sb, "uncompressedDataIndexSize=" + uncompressedDataIndexSize);
276       append(sb, "numDataIndexLevels=" + numDataIndexLevels);
277       append(sb, "firstDataBlockOffset=" + firstDataBlockOffset);
278       append(sb, "lastDataBlockOffset=" + lastDataBlockOffset);
279       append(sb, "comparatorClassName=" + comparatorClassName);
280     }
281     append(sb, "majorVersion=" + majorVersion);
282     append(sb, "minorVersion=" + minorVersion);
283 
284     return sb.toString();
285   }
286 
287   /**
288    * Reads a file trailer from the given file.
289    *
290    * @param istream the input stream with the ability to seek. Does not have to
291    *          be buffered, as only one read operation is made.
292    * @param fileSize the file size. Can be obtained using
293    *          {@link org.apache.hadoop.fs.FileSystem#getFileStatus(
294    *          org.apache.hadoop.fs.Path)}.
295    * @return the fixed file trailer read
296    * @throws IOException if failed to read from the underlying stream, or the
297    *           trailer is corrupted, or the version of the trailer is
298    *           unsupported
299    */
300   public static FixedFileTrailer readFromStream(FSDataInputStream istream,
301       long fileSize) throws IOException {
302     int bufferSize = MAX_TRAILER_SIZE;
303     long seekPoint = fileSize - bufferSize;
304     if (seekPoint < 0) {
305       // It is hard to imagine such a small HFile.
306       seekPoint = 0;
307       bufferSize = (int) fileSize;
308     }
309 
310     istream.seek(seekPoint);
311     ByteBuffer buf = ByteBuffer.allocate(bufferSize);
312     istream.readFully(buf.array(), buf.arrayOffset(),
313         buf.arrayOffset() + buf.limit());
314 
315     // Read the version from the last int of the file.
316     buf.position(buf.limit() - Bytes.SIZEOF_INT);
317     int version = buf.getInt();
318 
319     // Extract the major and minor versions.
320     int majorVersion = extractMajorVersion(version);
321     int minorVersion = extractMinorVersion(version);
322 
323     HFile.checkFormatVersion(majorVersion); // throws IAE if invalid
324 
325     int trailerSize = getTrailerSize(majorVersion);
326 
327     FixedFileTrailer fft = new FixedFileTrailer(majorVersion, minorVersion);
328     fft.deserialize(new DataInputStream(new ByteArrayInputStream(buf.array(),
329         buf.arrayOffset() + bufferSize - trailerSize, trailerSize)));
330     return fft;
331   }
332 
333   public void expectMajorVersion(int expected) {
334     if (majorVersion != expected) {
335       throw new IllegalArgumentException("Invalid HFile major version: "
336           + majorVersion 
337           + " (expected: " + expected + ")");
338     }
339   }
340 
341   public void expectMinorVersion(int expected) {
342     if (minorVersion != expected) {
343       throw new IllegalArgumentException("Invalid HFile minor version: "
344           + minorVersion + " (expected: " + expected + ")");
345     }
346   }
347 
348   public void expectAtLeastMajorVersion(int lowerBound) {
349     if (majorVersion < lowerBound) {
350       throw new IllegalArgumentException("Invalid HFile major version: "
351           + majorVersion
352           + " (expected: " + lowerBound + " or higher).");
353     }
354   }
355 
356   public long getFileInfoOffset() {
357     return fileInfoOffset;
358   }
359 
360   public void setFileInfoOffset(long fileInfoOffset) {
361     this.fileInfoOffset = fileInfoOffset;
362   }
363 
364   public long getLoadOnOpenDataOffset() {
365     return loadOnOpenDataOffset;
366   }
367 
368   public void setLoadOnOpenOffset(long loadOnOpenDataOffset) {
369     this.loadOnOpenDataOffset = loadOnOpenDataOffset;
370   }
371 
372   public int getDataIndexCount() {
373     return dataIndexCount;
374   }
375 
376   public void setDataIndexCount(int dataIndexCount) {
377     this.dataIndexCount = dataIndexCount;
378   }
379 
380   public int getMetaIndexCount() {
381     return metaIndexCount;
382   }
383 
384   public void setMetaIndexCount(int metaIndexCount) {
385     this.metaIndexCount = metaIndexCount;
386   }
387 
388   public long getTotalUncompressedBytes() {
389     return totalUncompressedBytes;
390   }
391 
392   public void setTotalUncompressedBytes(long totalUncompressedBytes) {
393     this.totalUncompressedBytes = totalUncompressedBytes;
394   }
395 
396   public long getEntryCount() {
397     return entryCount;
398   }
399 
400   public void setEntryCount(long newEntryCount) {
401     if (majorVersion == 1) {
402       int intEntryCount = (int) Math.min(Integer.MAX_VALUE, newEntryCount);
403       if (intEntryCount != newEntryCount) {
404         LOG.info("Warning: entry count is " + newEntryCount + " but writing "
405             + intEntryCount + " into the version " + majorVersion + " trailer");
406       }
407       entryCount = intEntryCount;
408       return;
409     }
410     entryCount = newEntryCount;
411   }
412 
413   public Compression.Algorithm getCompressionCodec() {
414     return compressionCodec;
415   }
416 
417   public void setCompressionCodec(Compression.Algorithm compressionCodec) {
418     this.compressionCodec = compressionCodec;
419   }
420 
421   public int getNumDataIndexLevels() {
422     expectAtLeastMajorVersion(2);
423     return numDataIndexLevels;
424   }
425 
426   public void setNumDataIndexLevels(int numDataIndexLevels) {
427     expectAtLeastMajorVersion(2);
428     this.numDataIndexLevels = numDataIndexLevels;
429   }
430 
431   public long getLastDataBlockOffset() {
432     expectAtLeastMajorVersion(2);
433     return lastDataBlockOffset;
434   }
435 
436   public void setLastDataBlockOffset(long lastDataBlockOffset) {
437     expectAtLeastMajorVersion(2);
438     this.lastDataBlockOffset = lastDataBlockOffset;
439   }
440 
441   public long getFirstDataBlockOffset() {
442     expectAtLeastMajorVersion(2);
443     return firstDataBlockOffset;
444   }
445 
446   public void setFirstDataBlockOffset(long firstDataBlockOffset) {
447     expectAtLeastMajorVersion(2);
448     this.firstDataBlockOffset = firstDataBlockOffset;
449   }
450 
451   /**
452    * Returns the major version of this HFile format
453    */
454   public int getMajorVersion() {
455     return majorVersion;
456   }
457 
458   /**
459    * Returns the minor version of this HFile format
460    */
461   int getMinorVersion() {
462     return minorVersion;
463   }
464 
465   @SuppressWarnings("rawtypes")
466   public void setComparatorClass(Class<? extends RawComparator> klass) {
467     expectAtLeastMajorVersion(2);
468     comparatorClassName = klass.getName();
469   }
470 
471   @SuppressWarnings("unchecked")
472   private static Class<? extends RawComparator<byte[]>> getComparatorClass(
473       String comparatorClassName) throws IOException {
474     try {
475       return (Class<? extends RawComparator<byte[]>>)
476           Class.forName(comparatorClassName);
477     } catch (ClassNotFoundException ex) {
478       throw new IOException(ex);
479     }
480   }
481 
482   public static RawComparator<byte[]> createComparator(
483       String comparatorClassName) throws IOException {
484     try {
485       return getComparatorClass(comparatorClassName).newInstance();
486     } catch (InstantiationException e) {
487       throw new IOException(e);
488     } catch (IllegalAccessException e) {
489       throw new IOException(e);
490     }
491   }
492 
493   RawComparator<byte[]> createComparator() throws IOException {
494     expectAtLeastMajorVersion(2);
495     return createComparator(comparatorClassName);
496   }
497 
498   public long getUncompressedDataIndexSize() {
499     if (majorVersion == 1)
500       return 0;
501     return uncompressedDataIndexSize;
502   }
503 
504   public void setUncompressedDataIndexSize(
505       long uncompressedDataIndexSize) {
506     expectAtLeastMajorVersion(2);
507     this.uncompressedDataIndexSize = uncompressedDataIndexSize;
508   }
509 
510   /**
511    * Extracts the major version for a 4-byte serialized version data.
512    * The major version is the 3 least significant bytes
513    */
514   private static int extractMajorVersion(int serializedVersion) {
515     return (serializedVersion & 0x00ffffff);
516   }
517 
518   /**
519    * Extracts the minor version for a 4-byte serialized version data.
520    * The major version are the 3 the most significant bytes
521    */
522   private static int extractMinorVersion(int serializedVersion) {
523     return (serializedVersion >>> 24);
524   }
525 
526   /**
527    * Create a 4 byte serialized version number by combining the
528    * minor and major version numbers.
529    */
530   private static int materializeVersion(int majorVersion, int minorVersion) {
531     return ((majorVersion & 0x00ffffff) | (minorVersion << 24));
532   }
533 }