Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.io.hfile;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNull;
022import static org.junit.Assert.assertTrue;
023import static org.junit.Assert.fail;
024
025import java.io.ByteArrayInputStream;
026import java.io.ByteArrayOutputStream;
027import java.io.DataInputStream;
028import java.io.DataOutputStream;
029import java.io.IOException;
030import java.util.ArrayList;
031import java.util.Collection;
032import java.util.List;
033import org.apache.hadoop.fs.FSDataInputStream;
034import org.apache.hadoop.fs.FSDataOutputStream;
035import org.apache.hadoop.fs.FileSystem;
036import org.apache.hadoop.fs.Path;
037import org.apache.hadoop.hbase.CellComparator;
038import org.apache.hadoop.hbase.CellComparatorImpl;
039import org.apache.hadoop.hbase.HBaseClassTestRule;
040import org.apache.hadoop.hbase.HBaseTestingUtility;
041import org.apache.hadoop.hbase.KeyValue;
042import org.apache.hadoop.hbase.shaded.protobuf.generated.HFileProtos;
043import org.apache.hadoop.hbase.testclassification.IOTests;
044import org.apache.hadoop.hbase.testclassification.SmallTests;
045import org.apache.hadoop.hbase.util.Bytes;
046import org.junit.Before;
047import org.junit.ClassRule;
048import org.junit.Rule;
049import org.junit.Test;
050import org.junit.experimental.categories.Category;
051import org.junit.rules.ExpectedException;
052import org.junit.runner.RunWith;
053import org.junit.runners.Parameterized;
054import org.junit.runners.Parameterized.Parameters;
055import org.slf4j.Logger;
056import org.slf4j.LoggerFactory;
057
058@RunWith(Parameterized.class)
059@Category({IOTests.class, SmallTests.class})
060public class TestFixedFileTrailer {
061
062  @ClassRule
063  public static final HBaseClassTestRule CLASS_RULE =
064      HBaseClassTestRule.forClass(TestFixedFileTrailer.class);
065
066  private static final Logger LOG = LoggerFactory.getLogger(TestFixedFileTrailer.class);
067  private static final int MAX_COMPARATOR_NAME_LENGTH = 128;
068
069  /**
070   * The number of used fields by version. Indexed by version minus two.
071   * Min version that we support is V2
072   */
073  private static final int[] NUM_FIELDS_BY_VERSION = new int[] { 14, 15 };
074
075  private HBaseTestingUtility util = new HBaseTestingUtility();
076  private FileSystem fs;
077  private ByteArrayOutputStream baos = new ByteArrayOutputStream();
078  private int version;
079
080  static {
081    assert NUM_FIELDS_BY_VERSION.length == HFile.MAX_FORMAT_VERSION
082        - HFile.MIN_FORMAT_VERSION + 1;
083  }
084
085  public TestFixedFileTrailer(int version) {
086    this.version = version;
087  }
088
089  @Rule
090  public ExpectedException expectedEx = ExpectedException.none();
091
092  @Parameters
093  public static Collection<Object[]> getParameters() {
094    List<Object[]> versionsToTest = new ArrayList<>();
095    for (int v = HFile.MIN_FORMAT_VERSION; v <= HFile.MAX_FORMAT_VERSION; ++v)
096      versionsToTest.add(new Integer[] { v } );
097    return versionsToTest;
098  }
099
100  @Before
101  public void setUp() throws IOException {
102    fs = FileSystem.get(util.getConfiguration());
103  }
104
105  @Test
106  public void testComparatorIsHBase1Compatible() {
107    FixedFileTrailer t = new FixedFileTrailer(version, HFileReaderImpl.PBUF_TRAILER_MINOR_VERSION);
108    t.setComparatorClass(CellComparatorImpl.COMPARATOR.getClass());
109    assertEquals(CellComparatorImpl.COMPARATOR.getClass().getName(), t.getComparatorClassName());
110    HFileProtos.FileTrailerProto pb = t.toProtobuf();
111    assertEquals(KeyValue.COMPARATOR.getClass().getName(), pb.getComparatorClassName());
112    t.setComparatorClass(CellComparatorImpl.MetaCellComparator.META_COMPARATOR.getClass());
113    pb = t.toProtobuf();
114    assertEquals(KeyValue.META_COMPARATOR.getClass().getName(),
115        pb.getComparatorClassName());
116  }
117
118  @Test
119  public void testCreateComparator() throws IOException {
120    FixedFileTrailer t = new FixedFileTrailer(version, HFileReaderImpl.PBUF_TRAILER_MINOR_VERSION);
121    try {
122      assertEquals(CellComparatorImpl.class,
123          t.createComparator(KeyValue.COMPARATOR.getLegacyKeyComparatorName()).getClass());
124      assertEquals(CellComparatorImpl.class,
125          t.createComparator(KeyValue.COMPARATOR.getClass().getName()).getClass());
126      assertEquals(CellComparatorImpl.class,
127          t.createComparator(CellComparator.class.getName()).getClass());
128      assertEquals(CellComparatorImpl.MetaCellComparator.class,
129          t.createComparator(KeyValue.META_COMPARATOR.getLegacyKeyComparatorName()).getClass());
130      assertEquals(CellComparatorImpl.MetaCellComparator.class,
131          t.createComparator(KeyValue.META_COMPARATOR.getClass().getName()).getClass());
132      assertEquals(CellComparatorImpl.MetaCellComparator.class, t.createComparator(
133          CellComparatorImpl.MetaCellComparator.META_COMPARATOR.getClass().getName()).getClass());
134      assertEquals(CellComparatorImpl.META_COMPARATOR.getClass(), t.createComparator(
135        CellComparatorImpl.MetaCellComparator.META_COMPARATOR.getClass().getName()).getClass());
136      assertEquals(CellComparatorImpl.COMPARATOR.getClass(), t.createComparator(
137        CellComparatorImpl.MetaCellComparator.COMPARATOR.getClass().getName()).getClass());
138      assertNull(t.createComparator(Bytes.BYTES_RAWCOMPARATOR.getClass().getName()));
139      assertNull(t.createComparator("org.apache.hadoop.hbase.KeyValue$RawBytesComparator"));
140    } catch (IOException e) {
141      fail("Unexpected exception while testing FixedFileTrailer#createComparator()");
142    }
143
144    // Test an invalid comparatorClassName
145    expectedEx.expect(IOException.class);
146    t.createComparator("");
147
148  }
149
150  @Test
151  public void testTrailer() throws IOException {
152    FixedFileTrailer t = new FixedFileTrailer(version,
153        HFileReaderImpl.PBUF_TRAILER_MINOR_VERSION);
154    t.setDataIndexCount(3);
155    t.setEntryCount(((long) Integer.MAX_VALUE) + 1);
156
157    t.setLastDataBlockOffset(291);
158    t.setNumDataIndexLevels(3);
159    t.setComparatorClass(CellComparatorImpl.COMPARATOR.getClass());
160    t.setFirstDataBlockOffset(9081723123L); // Completely unrealistic.
161    t.setUncompressedDataIndexSize(827398717L); // Something random.
162
163    t.setLoadOnOpenOffset(128);
164    t.setMetaIndexCount(7);
165
166    t.setTotalUncompressedBytes(129731987);
167
168    {
169      DataOutputStream dos = new DataOutputStream(baos); // Limited scope.
170      t.serialize(dos);
171      dos.flush();
172      assertEquals(dos.size(), FixedFileTrailer.getTrailerSize(version));
173    }
174
175    byte[] bytes = baos.toByteArray();
176    baos.reset();
177
178    assertEquals(bytes.length, FixedFileTrailer.getTrailerSize(version));
179
180    ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
181
182    // Finished writing, trying to read.
183    {
184      DataInputStream dis = new DataInputStream(bais);
185      FixedFileTrailer t2 = new FixedFileTrailer(version,
186          HFileReaderImpl.PBUF_TRAILER_MINOR_VERSION);
187      t2.deserialize(dis);
188      assertEquals(-1, bais.read()); // Ensure we have read everything.
189      checkLoadedTrailer(version, t, t2);
190    }
191
192    // Now check what happens if the trailer is corrupted.
193    Path trailerPath = new Path(util.getDataTestDir(), "trailer_" + version);
194
195    {
196      for (byte invalidVersion : new byte[] { HFile.MIN_FORMAT_VERSION - 1,
197          HFile.MAX_FORMAT_VERSION + 1}) {
198        bytes[bytes.length - 1] = invalidVersion;
199        writeTrailer(trailerPath, null, bytes);
200        try {
201          readTrailer(trailerPath);
202          fail("Exception expected");
203        } catch (IllegalArgumentException ex) {
204          // Make it easy to debug this.
205          String msg = ex.getMessage();
206          String cleanMsg = msg.replaceAll(
207              "^(java(\\.[a-zA-Z]+)+:\\s+)?|\\s+\\(.*\\)\\s*$", "");
208          assertEquals("Actual exception message is \"" + msg + "\".\n" +
209              "Cleaned-up message", // will be followed by " expected: ..."
210              "Invalid HFile version: " + invalidVersion, cleanMsg);
211          LOG.info("Got an expected exception: " + msg);
212        }
213      }
214
215    }
216
217    // Now write the trailer into a file and auto-detect the version.
218    writeTrailer(trailerPath, t, null);
219
220    FixedFileTrailer t4 = readTrailer(trailerPath);
221
222    checkLoadedTrailer(version, t, t4);
223
224    String trailerStr = t.toString();
225    assertEquals("Invalid number of fields in the string representation "
226        + "of the trailer: " + trailerStr, NUM_FIELDS_BY_VERSION[version - 2],
227        trailerStr.split(", ").length);
228    assertEquals(trailerStr, t4.toString());
229  }
230
231  @Test
232  public void testTrailerForV2NonPBCompatibility() throws Exception {
233    if (version == 2) {
234      FixedFileTrailer t = new FixedFileTrailer(version,
235          HFileReaderImpl.MINOR_VERSION_NO_CHECKSUM);
236      t.setDataIndexCount(3);
237      t.setEntryCount(((long) Integer.MAX_VALUE) + 1);
238      t.setLastDataBlockOffset(291);
239      t.setNumDataIndexLevels(3);
240      t.setComparatorClass(CellComparatorImpl.COMPARATOR.getClass());
241      t.setFirstDataBlockOffset(9081723123L); // Completely unrealistic.
242      t.setUncompressedDataIndexSize(827398717L); // Something random.
243      t.setLoadOnOpenOffset(128);
244      t.setMetaIndexCount(7);
245      t.setTotalUncompressedBytes(129731987);
246
247      {
248        DataOutputStream dos = new DataOutputStream(baos); // Limited scope.
249        serializeAsWritable(dos, t);
250        dos.flush();
251        assertEquals(FixedFileTrailer.getTrailerSize(version), dos.size());
252      }
253
254      byte[] bytes = baos.toByteArray();
255      baos.reset();
256      assertEquals(bytes.length, FixedFileTrailer.getTrailerSize(version));
257
258      ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
259      {
260        DataInputStream dis = new DataInputStream(bais);
261        FixedFileTrailer t2 = new FixedFileTrailer(version,
262            HFileReaderImpl.MINOR_VERSION_NO_CHECKSUM);
263        t2.deserialize(dis);
264        assertEquals(-1, bais.read()); // Ensure we have read everything.
265        checkLoadedTrailer(version, t, t2);
266      }
267    }
268  }
269
270  // Copied from FixedFileTrailer for testing the reading part of
271  // FixedFileTrailer of non PB
272  // serialized FFTs.
273  private void serializeAsWritable(DataOutputStream output, FixedFileTrailer fft)
274      throws IOException {
275    BlockType.TRAILER.write(output);
276    output.writeLong(fft.getFileInfoOffset());
277    output.writeLong(fft.getLoadOnOpenDataOffset());
278    output.writeInt(fft.getDataIndexCount());
279    output.writeLong(fft.getUncompressedDataIndexSize());
280    output.writeInt(fft.getMetaIndexCount());
281    output.writeLong(fft.getTotalUncompressedBytes());
282    output.writeLong(fft.getEntryCount());
283    output.writeInt(fft.getCompressionCodec().ordinal());
284    output.writeInt(fft.getNumDataIndexLevels());
285    output.writeLong(fft.getFirstDataBlockOffset());
286    output.writeLong(fft.getLastDataBlockOffset());
287    Bytes.writeStringFixedSize(output, fft.getComparatorClassName(), MAX_COMPARATOR_NAME_LENGTH);
288    output.writeInt(FixedFileTrailer.materializeVersion(fft.getMajorVersion(),
289        fft.getMinorVersion()));
290  }
291
292
293  private FixedFileTrailer readTrailer(Path trailerPath) throws IOException {
294    FSDataInputStream fsdis = fs.open(trailerPath);
295    FixedFileTrailer trailerRead = FixedFileTrailer.readFromStream(fsdis,
296        fs.getFileStatus(trailerPath).getLen());
297    fsdis.close();
298    return trailerRead;
299  }
300
301  private void writeTrailer(Path trailerPath, FixedFileTrailer t,
302      byte[] useBytesInstead) throws IOException {
303    assert (t == null) != (useBytesInstead == null); // Expect one non-null.
304
305    FSDataOutputStream fsdos = fs.create(trailerPath);
306    fsdos.write(135); // to make deserializer's job less trivial
307    if (useBytesInstead != null) {
308      fsdos.write(useBytesInstead);
309    } else {
310      t.serialize(fsdos);
311    }
312    fsdos.close();
313  }
314
315  private void checkLoadedTrailer(int version, FixedFileTrailer expected,
316      FixedFileTrailer loaded) throws IOException {
317    assertEquals(version, loaded.getMajorVersion());
318    assertEquals(expected.getDataIndexCount(), loaded.getDataIndexCount());
319
320    assertEquals(Math.min(expected.getEntryCount(),
321        version == 1 ? Integer.MAX_VALUE : Long.MAX_VALUE),
322        loaded.getEntryCount());
323
324    if (version == 1) {
325      assertEquals(expected.getFileInfoOffset(), loaded.getFileInfoOffset());
326    }
327
328    if (version == 2) {
329      assertEquals(expected.getLastDataBlockOffset(),
330          loaded.getLastDataBlockOffset());
331      assertEquals(expected.getNumDataIndexLevels(),
332          loaded.getNumDataIndexLevels());
333      assertEquals(expected.createComparator().getClass().getName(),
334          loaded.createComparator().getClass().getName());
335      assertEquals(expected.getFirstDataBlockOffset(),
336          loaded.getFirstDataBlockOffset());
337      assertTrue(
338          expected.createComparator() instanceof CellComparatorImpl);
339      assertEquals(expected.getUncompressedDataIndexSize(),
340          loaded.getUncompressedDataIndexSize());
341    }
342
343    assertEquals(expected.getLoadOnOpenDataOffset(),
344        loaded.getLoadOnOpenDataOffset());
345    assertEquals(expected.getMetaIndexCount(), loaded.getMetaIndexCount());
346
347    assertEquals(expected.getTotalUncompressedBytes(),
348        loaded.getTotalUncompressedBytes());
349  }
350
351
352}
353