Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.io.hfile;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNull;
022import static org.junit.Assert.assertTrue;
023import static org.junit.Assert.fail;
024
025import java.io.ByteArrayInputStream;
026import java.io.ByteArrayOutputStream;
027import java.io.DataInputStream;
028import java.io.DataOutputStream;
029import java.io.IOException;
030import java.util.ArrayList;
031import java.util.Collection;
032import java.util.List;
033import org.apache.hadoop.fs.FSDataInputStream;
034import org.apache.hadoop.fs.FSDataOutputStream;
035import org.apache.hadoop.fs.FileSystem;
036import org.apache.hadoop.fs.Path;
037import org.apache.hadoop.hbase.CellComparator;
038import org.apache.hadoop.hbase.CellComparatorImpl;
039import org.apache.hadoop.hbase.HBaseClassTestRule;
040import org.apache.hadoop.hbase.HBaseTestingUtility;
041import org.apache.hadoop.hbase.KeyValue;
042import org.apache.hadoop.hbase.MetaCellComparator;
043import org.apache.hadoop.hbase.shaded.protobuf.generated.HFileProtos;
044import org.apache.hadoop.hbase.testclassification.IOTests;
045import org.apache.hadoop.hbase.testclassification.SmallTests;
046import org.apache.hadoop.hbase.util.Bytes;
047import org.junit.Before;
048import org.junit.ClassRule;
049import org.junit.Rule;
050import org.junit.Test;
051import org.junit.experimental.categories.Category;
052import org.junit.rules.ExpectedException;
053import org.junit.runner.RunWith;
054import org.junit.runners.Parameterized;
055import org.junit.runners.Parameterized.Parameters;
056import org.slf4j.Logger;
057import org.slf4j.LoggerFactory;
058
059@RunWith(Parameterized.class)
060@Category({IOTests.class, SmallTests.class})
061public class TestFixedFileTrailer {
062
063  @ClassRule
064  public static final HBaseClassTestRule CLASS_RULE =
065      HBaseClassTestRule.forClass(TestFixedFileTrailer.class);
066
067  private static final Logger LOG = LoggerFactory.getLogger(TestFixedFileTrailer.class);
068  private static final int MAX_COMPARATOR_NAME_LENGTH = 128;
069
070  /**
071   * The number of used fields by version. Indexed by version minus two.
072   * Min version that we support is V2
073   */
074  private static final int[] NUM_FIELDS_BY_VERSION = new int[] { 14, 15 };
075
076  private HBaseTestingUtility util = new HBaseTestingUtility();
077  private FileSystem fs;
078  private ByteArrayOutputStream baos = new ByteArrayOutputStream();
079  private int version;
080
081  static {
082    assert NUM_FIELDS_BY_VERSION.length == HFile.MAX_FORMAT_VERSION
083        - HFile.MIN_FORMAT_VERSION + 1;
084  }
085
086  public TestFixedFileTrailer(int version) {
087    this.version = version;
088  }
089
090  @Rule
091  public ExpectedException expectedEx = ExpectedException.none();
092
093  @Parameters
094  public static Collection<Object[]> getParameters() {
095    List<Object[]> versionsToTest = new ArrayList<>();
096    for (int v = HFile.MIN_FORMAT_VERSION; v <= HFile.MAX_FORMAT_VERSION; ++v)
097      versionsToTest.add(new Integer[] { v } );
098    return versionsToTest;
099  }
100
101  @Before
102  public void setUp() throws IOException {
103    fs = FileSystem.get(util.getConfiguration());
104  }
105
106  @Test
107  public void testComparatorIsHBase1Compatible() {
108    FixedFileTrailer t = new FixedFileTrailer(version, HFileReaderImpl.PBUF_TRAILER_MINOR_VERSION);
109    t.setComparatorClass(CellComparatorImpl.COMPARATOR.getClass());
110    assertEquals(CellComparatorImpl.COMPARATOR.getClass().getName(), t.getComparatorClassName());
111    HFileProtos.FileTrailerProto pb = t.toProtobuf();
112    assertEquals(KeyValue.COMPARATOR.getClass().getName(), pb.getComparatorClassName());
113    t.setComparatorClass(MetaCellComparator.META_COMPARATOR.getClass());
114    pb = t.toProtobuf();
115    assertEquals(KeyValue.META_COMPARATOR.getClass().getName(),
116        pb.getComparatorClassName());
117  }
118
119  @Test
120  public void testCreateComparator() throws IOException {
121    FixedFileTrailer t = new FixedFileTrailer(version, HFileReaderImpl.PBUF_TRAILER_MINOR_VERSION);
122    try {
123      assertEquals(CellComparatorImpl.class,
124          t.createComparator(KeyValue.COMPARATOR.getLegacyKeyComparatorName()).getClass());
125      assertEquals(CellComparatorImpl.class,
126          t.createComparator(KeyValue.COMPARATOR.getClass().getName()).getClass());
127      assertEquals(CellComparatorImpl.class,
128          t.createComparator(CellComparator.class.getName()).getClass());
129      assertEquals(MetaCellComparator.class,
130          t.createComparator(KeyValue.META_COMPARATOR.getLegacyKeyComparatorName()).getClass());
131      assertEquals(MetaCellComparator.class,
132          t.createComparator(KeyValue.META_COMPARATOR.getClass().getName()).getClass());
133      assertEquals(MetaCellComparator.class, t.createComparator(
134          MetaCellComparator.META_COMPARATOR.getClass().getName()).getClass());
135      assertEquals(MetaCellComparator.META_COMPARATOR.getClass(), t.createComparator(
136        MetaCellComparator.META_COMPARATOR.getClass().getName()).getClass());
137      assertEquals(CellComparatorImpl.COMPARATOR.getClass(), t.createComparator(
138        MetaCellComparator.COMPARATOR.getClass().getName()).getClass());
139      assertNull(t.createComparator(Bytes.BYTES_RAWCOMPARATOR.getClass().getName()));
140      assertNull(t.createComparator("org.apache.hadoop.hbase.KeyValue$RawBytesComparator"));
141    } catch (IOException e) {
142      fail("Unexpected exception while testing FixedFileTrailer#createComparator()");
143    }
144
145    // Test an invalid comparatorClassName
146    expectedEx.expect(IOException.class);
147    t.createComparator("");
148
149  }
150
151  @Test
152  public void testTrailer() throws IOException {
153    FixedFileTrailer t = new FixedFileTrailer(version,
154        HFileReaderImpl.PBUF_TRAILER_MINOR_VERSION);
155    t.setDataIndexCount(3);
156    t.setEntryCount(((long) Integer.MAX_VALUE) + 1);
157
158    t.setLastDataBlockOffset(291);
159    t.setNumDataIndexLevels(3);
160    t.setComparatorClass(CellComparatorImpl.COMPARATOR.getClass());
161    t.setFirstDataBlockOffset(9081723123L); // Completely unrealistic.
162    t.setUncompressedDataIndexSize(827398717L); // Something random.
163
164    t.setLoadOnOpenOffset(128);
165    t.setMetaIndexCount(7);
166
167    t.setTotalUncompressedBytes(129731987);
168
169    {
170      DataOutputStream dos = new DataOutputStream(baos); // Limited scope.
171      t.serialize(dos);
172      dos.flush();
173      assertEquals(dos.size(), FixedFileTrailer.getTrailerSize(version));
174    }
175
176    byte[] bytes = baos.toByteArray();
177    baos.reset();
178
179    assertEquals(bytes.length, FixedFileTrailer.getTrailerSize(version));
180
181    ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
182
183    // Finished writing, trying to read.
184    {
185      DataInputStream dis = new DataInputStream(bais);
186      FixedFileTrailer t2 = new FixedFileTrailer(version,
187          HFileReaderImpl.PBUF_TRAILER_MINOR_VERSION);
188      t2.deserialize(dis);
189      assertEquals(-1, bais.read()); // Ensure we have read everything.
190      checkLoadedTrailer(version, t, t2);
191    }
192
193    // Now check what happens if the trailer is corrupted.
194    Path trailerPath = new Path(util.getDataTestDir(), "trailer_" + version);
195
196    {
197      for (byte invalidVersion : new byte[] { HFile.MIN_FORMAT_VERSION - 1,
198          HFile.MAX_FORMAT_VERSION + 1}) {
199        bytes[bytes.length - 1] = invalidVersion;
200        writeTrailer(trailerPath, null, bytes);
201        try {
202          readTrailer(trailerPath);
203          fail("Exception expected");
204        } catch (IllegalArgumentException ex) {
205          // Make it easy to debug this.
206          String msg = ex.getMessage();
207          String cleanMsg = msg.replaceAll(
208              "^(java(\\.[a-zA-Z]+)+:\\s+)?|\\s+\\(.*\\)\\s*$", "");
209          assertEquals("Actual exception message is \"" + msg + "\".\n" +
210              "Cleaned-up message", // will be followed by " expected: ..."
211              "Invalid HFile version: " + invalidVersion, cleanMsg);
212          LOG.info("Got an expected exception: " + msg);
213        }
214      }
215
216    }
217
218    // Now write the trailer into a file and auto-detect the version.
219    writeTrailer(trailerPath, t, null);
220
221    FixedFileTrailer t4 = readTrailer(trailerPath);
222
223    checkLoadedTrailer(version, t, t4);
224
225    String trailerStr = t.toString();
226    assertEquals("Invalid number of fields in the string representation "
227        + "of the trailer: " + trailerStr, NUM_FIELDS_BY_VERSION[version - 2],
228        trailerStr.split(", ").length);
229    assertEquals(trailerStr, t4.toString());
230  }
231
232  @Test
233  public void testTrailerForV2NonPBCompatibility() throws Exception {
234    if (version == 2) {
235      FixedFileTrailer t = new FixedFileTrailer(version,
236          HFileReaderImpl.MINOR_VERSION_NO_CHECKSUM);
237      t.setDataIndexCount(3);
238      t.setEntryCount(((long) Integer.MAX_VALUE) + 1);
239      t.setLastDataBlockOffset(291);
240      t.setNumDataIndexLevels(3);
241      t.setComparatorClass(CellComparatorImpl.COMPARATOR.getClass());
242      t.setFirstDataBlockOffset(9081723123L); // Completely unrealistic.
243      t.setUncompressedDataIndexSize(827398717L); // Something random.
244      t.setLoadOnOpenOffset(128);
245      t.setMetaIndexCount(7);
246      t.setTotalUncompressedBytes(129731987);
247
248      {
249        DataOutputStream dos = new DataOutputStream(baos); // Limited scope.
250        serializeAsWritable(dos, t);
251        dos.flush();
252        assertEquals(FixedFileTrailer.getTrailerSize(version), dos.size());
253      }
254
255      byte[] bytes = baos.toByteArray();
256      baos.reset();
257      assertEquals(bytes.length, FixedFileTrailer.getTrailerSize(version));
258
259      ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
260      {
261        DataInputStream dis = new DataInputStream(bais);
262        FixedFileTrailer t2 = new FixedFileTrailer(version,
263            HFileReaderImpl.MINOR_VERSION_NO_CHECKSUM);
264        t2.deserialize(dis);
265        assertEquals(-1, bais.read()); // Ensure we have read everything.
266        checkLoadedTrailer(version, t, t2);
267      }
268    }
269  }
270
271  // Copied from FixedFileTrailer for testing the reading part of
272  // FixedFileTrailer of non PB
273  // serialized FFTs.
274  private void serializeAsWritable(DataOutputStream output, FixedFileTrailer fft)
275      throws IOException {
276    BlockType.TRAILER.write(output);
277    output.writeLong(fft.getFileInfoOffset());
278    output.writeLong(fft.getLoadOnOpenDataOffset());
279    output.writeInt(fft.getDataIndexCount());
280    output.writeLong(fft.getUncompressedDataIndexSize());
281    output.writeInt(fft.getMetaIndexCount());
282    output.writeLong(fft.getTotalUncompressedBytes());
283    output.writeLong(fft.getEntryCount());
284    output.writeInt(fft.getCompressionCodec().ordinal());
285    output.writeInt(fft.getNumDataIndexLevels());
286    output.writeLong(fft.getFirstDataBlockOffset());
287    output.writeLong(fft.getLastDataBlockOffset());
288    Bytes.writeStringFixedSize(output, fft.getComparatorClassName(), MAX_COMPARATOR_NAME_LENGTH);
289    output.writeInt(FixedFileTrailer.materializeVersion(fft.getMajorVersion(),
290        fft.getMinorVersion()));
291  }
292
293
294  private FixedFileTrailer readTrailer(Path trailerPath) throws IOException {
295    FSDataInputStream fsdis = fs.open(trailerPath);
296    FixedFileTrailer trailerRead = FixedFileTrailer.readFromStream(fsdis,
297        fs.getFileStatus(trailerPath).getLen());
298    fsdis.close();
299    return trailerRead;
300  }
301
302  private void writeTrailer(Path trailerPath, FixedFileTrailer t,
303      byte[] useBytesInstead) throws IOException {
304    assert (t == null) != (useBytesInstead == null); // Expect one non-null.
305
306    FSDataOutputStream fsdos = fs.create(trailerPath);
307    fsdos.write(135); // to make deserializer's job less trivial
308    if (useBytesInstead != null) {
309      fsdos.write(useBytesInstead);
310    } else {
311      t.serialize(fsdos);
312    }
313    fsdos.close();
314  }
315
316  private void checkLoadedTrailer(int version, FixedFileTrailer expected,
317      FixedFileTrailer loaded) throws IOException {
318    assertEquals(version, loaded.getMajorVersion());
319    assertEquals(expected.getDataIndexCount(), loaded.getDataIndexCount());
320
321    assertEquals(Math.min(expected.getEntryCount(),
322        version == 1 ? Integer.MAX_VALUE : Long.MAX_VALUE),
323        loaded.getEntryCount());
324
325    if (version == 1) {
326      assertEquals(expected.getFileInfoOffset(), loaded.getFileInfoOffset());
327    }
328
329    if (version == 2) {
330      assertEquals(expected.getLastDataBlockOffset(),
331          loaded.getLastDataBlockOffset());
332      assertEquals(expected.getNumDataIndexLevels(),
333          loaded.getNumDataIndexLevels());
334      assertEquals(expected.createComparator().getClass().getName(),
335          loaded.createComparator().getClass().getName());
336      assertEquals(expected.getFirstDataBlockOffset(),
337          loaded.getFirstDataBlockOffset());
338      assertTrue(
339          expected.createComparator() instanceof CellComparatorImpl);
340      assertEquals(expected.getUncompressedDataIndexSize(),
341          loaded.getUncompressedDataIndexSize());
342    }
343
344    assertEquals(expected.getLoadOnOpenDataOffset(),
345        loaded.getLoadOnOpenDataOffset());
346    assertEquals(expected.getMetaIndexCount(), loaded.getMetaIndexCount());
347
348    assertEquals(expected.getTotalUncompressedBytes(),
349        loaded.getTotalUncompressedBytes());
350  }
351
352
353}
354