Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.io.hfile;
019
020import static org.apache.hadoop.hbase.io.hfile.FixedFileTrailer.createComparator;
021import static org.junit.Assert.assertEquals;
022import static org.junit.Assert.assertNull;
023import static org.junit.Assert.assertThrows;
024import static org.junit.Assert.assertTrue;
025import static org.junit.Assert.fail;
026
027import java.io.ByteArrayInputStream;
028import java.io.ByteArrayOutputStream;
029import java.io.DataInputStream;
030import java.io.DataOutputStream;
031import java.io.IOException;
032import java.util.ArrayList;
033import java.util.Collection;
034import java.util.List;
035import org.apache.hadoop.fs.FSDataInputStream;
036import org.apache.hadoop.fs.FSDataOutputStream;
037import org.apache.hadoop.fs.FileSystem;
038import org.apache.hadoop.fs.Path;
039import org.apache.hadoop.hbase.CellComparator;
040import org.apache.hadoop.hbase.CellComparatorImpl;
041import org.apache.hadoop.hbase.HBaseClassTestRule;
042import org.apache.hadoop.hbase.HBaseTestingUtil;
043import org.apache.hadoop.hbase.InnerStoreCellComparator;
044import org.apache.hadoop.hbase.KeyValue;
045import org.apache.hadoop.hbase.MetaCellComparator;
046import org.apache.hadoop.hbase.testclassification.IOTests;
047import org.apache.hadoop.hbase.testclassification.SmallTests;
048import org.apache.hadoop.hbase.util.Bytes;
049import org.junit.Before;
050import org.junit.ClassRule;
051import org.junit.Test;
052import org.junit.experimental.categories.Category;
053import org.junit.runner.RunWith;
054import org.junit.runners.Parameterized;
055import org.junit.runners.Parameterized.Parameters;
056import org.slf4j.Logger;
057import org.slf4j.LoggerFactory;
058
059@RunWith(Parameterized.class)
060@Category({ IOTests.class, SmallTests.class })
061public class TestFixedFileTrailer {
062
063  @ClassRule
064  public static final HBaseClassTestRule CLASS_RULE =
065    HBaseClassTestRule.forClass(TestFixedFileTrailer.class);
066
067  private static final Logger LOG = LoggerFactory.getLogger(TestFixedFileTrailer.class);
068  private static final int MAX_COMPARATOR_NAME_LENGTH = 128;
069
070  /**
071   * The number of used fields by version. Indexed by version minus two. Min version that we support
072   * is V2
073   */
074  private static final int[] NUM_FIELDS_BY_VERSION = new int[] { 14, 15 };
075
076  private HBaseTestingUtil util = new HBaseTestingUtil();
077  private FileSystem fs;
078  private ByteArrayOutputStream baos = new ByteArrayOutputStream();
079  private int version;
080
081  static {
082    assert NUM_FIELDS_BY_VERSION.length == HFile.MAX_FORMAT_VERSION - HFile.MIN_FORMAT_VERSION + 1;
083  }
084
085  public TestFixedFileTrailer(int version) {
086    this.version = version;
087  }
088
089  @Parameters
090  public static Collection<Object[]> getParameters() {
091    List<Object[]> versionsToTest = new ArrayList<>();
092    for (int v = HFile.MIN_FORMAT_VERSION; v <= HFile.MAX_FORMAT_VERSION; ++v)
093      versionsToTest.add(new Integer[] { v });
094    return versionsToTest;
095  }
096
097  @Before
098  public void setUp() throws IOException {
099    fs = FileSystem.get(util.getConfiguration());
100  }
101
102  @Test
103  public void testCreateComparator() throws IOException {
104    assertEquals(InnerStoreCellComparator.class,
105      createComparator(KeyValue.COMPARATOR.getLegacyKeyComparatorName()).getClass());
106    assertEquals(InnerStoreCellComparator.class,
107      createComparator(KeyValue.COMPARATOR.getClass().getName()).getClass());
108    assertEquals(InnerStoreCellComparator.class,
109      createComparator(CellComparator.class.getName()).getClass());
110    assertEquals(MetaCellComparator.class,
111      createComparator(KeyValue.META_COMPARATOR.getLegacyKeyComparatorName()).getClass());
112    assertEquals(MetaCellComparator.class,
113      createComparator(KeyValue.META_COMPARATOR.getClass().getName()).getClass());
114    assertEquals(MetaCellComparator.class,
115      createComparator("org.apache.hadoop.hbase.CellComparator$MetaCellComparator").getClass());
116    assertEquals(MetaCellComparator.class,
117      createComparator("org.apache.hadoop.hbase.CellComparatorImpl$MetaCellComparator").getClass());
118    assertEquals(MetaCellComparator.class,
119      createComparator(MetaCellComparator.META_COMPARATOR.getClass().getName()).getClass());
120    assertEquals(MetaCellComparator.META_COMPARATOR.getClass(),
121      createComparator(MetaCellComparator.META_COMPARATOR.getClass().getName()).getClass());
122    assertEquals(CellComparatorImpl.COMPARATOR.getClass(),
123      createComparator(MetaCellComparator.COMPARATOR.getClass().getName()).getClass());
124    assertNull(createComparator(Bytes.BYTES_RAWCOMPARATOR.getClass().getName()));
125    assertNull(createComparator("org.apache.hadoop.hbase.KeyValue$RawBytesComparator"));
126
127    // Test an invalid comparatorClassName
128    assertThrows(IOException.class, () -> createComparator(""));
129  }
130
131  @Test
132  public void testTrailer() throws IOException {
133    FixedFileTrailer t = new FixedFileTrailer(version, HFileReaderImpl.PBUF_TRAILER_MINOR_VERSION);
134    t.setDataIndexCount(3);
135    t.setEntryCount(((long) Integer.MAX_VALUE) + 1);
136
137    t.setLastDataBlockOffset(291);
138    t.setNumDataIndexLevels(3);
139    t.setComparatorClass(InnerStoreCellComparator.INNER_STORE_COMPARATOR.getClass());
140    t.setFirstDataBlockOffset(9081723123L); // Completely unrealistic.
141    t.setUncompressedDataIndexSize(827398717L); // Something random.
142
143    t.setLoadOnOpenOffset(128);
144    t.setMetaIndexCount(7);
145
146    t.setTotalUncompressedBytes(129731987);
147
148    {
149      DataOutputStream dos = new DataOutputStream(baos); // Limited scope.
150      t.serialize(dos);
151      dos.flush();
152      assertEquals(dos.size(), FixedFileTrailer.getTrailerSize(version));
153    }
154
155    byte[] bytes = baos.toByteArray();
156    baos.reset();
157
158    assertEquals(bytes.length, FixedFileTrailer.getTrailerSize(version));
159
160    ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
161
162    // Finished writing, trying to read.
163    {
164      DataInputStream dis = new DataInputStream(bais);
165      FixedFileTrailer t2 =
166        new FixedFileTrailer(version, HFileReaderImpl.PBUF_TRAILER_MINOR_VERSION);
167      t2.deserialize(dis);
168      assertEquals(-1, bais.read()); // Ensure we have read everything.
169      checkLoadedTrailer(version, t, t2);
170    }
171
172    // Now check what happens if the trailer is corrupted.
173    Path trailerPath = new Path(util.getDataTestDir(), "trailer_" + version);
174
175    {
176      for (byte invalidVersion : new byte[] { HFile.MIN_FORMAT_VERSION - 1,
177        HFile.MAX_FORMAT_VERSION + 1 }) {
178        bytes[bytes.length - 1] = invalidVersion;
179        writeTrailer(trailerPath, null, bytes);
180        try {
181          readTrailer(trailerPath);
182          fail("Exception expected");
183        } catch (IllegalArgumentException ex) {
184          // Make it easy to debug this.
185          String msg = ex.getMessage();
186          String cleanMsg = msg.replaceAll("^(java(\\.[a-zA-Z]+)+:\\s+)?|\\s+\\(.*\\)\\s*$", "");
187          // will be followed by " expected: ..."
188          assertEquals("Actual exception message is \"" + msg + "\".\nCleaned-up message",
189            "Invalid HFile version: " + invalidVersion, cleanMsg);
190          LOG.info("Got an expected exception: " + msg);
191        }
192      }
193
194    }
195
196    // Now write the trailer into a file and auto-detect the version.
197    writeTrailer(trailerPath, t, null);
198
199    FixedFileTrailer t4 = readTrailer(trailerPath);
200
201    checkLoadedTrailer(version, t, t4);
202
203    String trailerStr = t.toString();
204    assertEquals(
205      "Invalid number of fields in the string representation " + "of the trailer: " + trailerStr,
206      NUM_FIELDS_BY_VERSION[version - 2], trailerStr.split(", ").length);
207    assertEquals(trailerStr, t4.toString());
208  }
209
210  @Test
211  public void testTrailerForV2NonPBCompatibility() throws Exception {
212    if (version == 2) {
213      FixedFileTrailer t = new FixedFileTrailer(version, HFileReaderImpl.MINOR_VERSION_NO_CHECKSUM);
214      t.setDataIndexCount(3);
215      t.setEntryCount(((long) Integer.MAX_VALUE) + 1);
216      t.setLastDataBlockOffset(291);
217      t.setNumDataIndexLevels(3);
218      t.setComparatorClass(CellComparatorImpl.COMPARATOR.getClass());
219      t.setFirstDataBlockOffset(9081723123L); // Completely unrealistic.
220      t.setUncompressedDataIndexSize(827398717L); // Something random.
221      t.setLoadOnOpenOffset(128);
222      t.setMetaIndexCount(7);
223      t.setTotalUncompressedBytes(129731987);
224
225      {
226        DataOutputStream dos = new DataOutputStream(baos); // Limited scope.
227        serializeAsWritable(dos, t);
228        dos.flush();
229        assertEquals(FixedFileTrailer.getTrailerSize(version), dos.size());
230      }
231
232      byte[] bytes = baos.toByteArray();
233      baos.reset();
234      assertEquals(bytes.length, FixedFileTrailer.getTrailerSize(version));
235
236      ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
237      {
238        DataInputStream dis = new DataInputStream(bais);
239        FixedFileTrailer t2 =
240          new FixedFileTrailer(version, HFileReaderImpl.MINOR_VERSION_NO_CHECKSUM);
241        t2.deserialize(dis);
242        assertEquals(-1, bais.read()); // Ensure we have read everything.
243        checkLoadedTrailer(version, t, t2);
244      }
245    }
246  }
247
248  // Copied from FixedFileTrailer for testing the reading part of
249  // FixedFileTrailer of non PB
250  // serialized FFTs.
251  private void serializeAsWritable(DataOutputStream output, FixedFileTrailer fft)
252    throws IOException {
253    BlockType.TRAILER.write(output);
254    output.writeLong(fft.getFileInfoOffset());
255    output.writeLong(fft.getLoadOnOpenDataOffset());
256    output.writeInt(fft.getDataIndexCount());
257    output.writeLong(fft.getUncompressedDataIndexSize());
258    output.writeInt(fft.getMetaIndexCount());
259    output.writeLong(fft.getTotalUncompressedBytes());
260    output.writeLong(fft.getEntryCount());
261    output.writeInt(fft.getCompressionCodec().ordinal());
262    output.writeInt(fft.getNumDataIndexLevels());
263    output.writeLong(fft.getFirstDataBlockOffset());
264    output.writeLong(fft.getLastDataBlockOffset());
265    Bytes.writeStringFixedSize(output, fft.getComparatorClassName(), MAX_COMPARATOR_NAME_LENGTH);
266    output
267      .writeInt(FixedFileTrailer.materializeVersion(fft.getMajorVersion(), fft.getMinorVersion()));
268  }
269
270  private FixedFileTrailer readTrailer(Path trailerPath) throws IOException {
271    FSDataInputStream fsdis = fs.open(trailerPath);
272    FixedFileTrailer trailerRead =
273      FixedFileTrailer.readFromStream(fsdis, fs.getFileStatus(trailerPath).getLen());
274    fsdis.close();
275    return trailerRead;
276  }
277
278  private void writeTrailer(Path trailerPath, FixedFileTrailer t, byte[] useBytesInstead)
279    throws IOException {
280    assert (t == null) != (useBytesInstead == null); // Expect one non-null.
281
282    FSDataOutputStream fsdos = fs.create(trailerPath);
283    fsdos.write(135); // to make deserializer's job less trivial
284    if (useBytesInstead != null) {
285      fsdos.write(useBytesInstead);
286    } else {
287      t.serialize(fsdos);
288    }
289    fsdos.close();
290  }
291
292  private void checkLoadedTrailer(int version, FixedFileTrailer expected, FixedFileTrailer loaded)
293    throws IOException {
294    assertEquals(version, loaded.getMajorVersion());
295    assertEquals(expected.getDataIndexCount(), loaded.getDataIndexCount());
296
297    assertEquals(
298      Math.min(expected.getEntryCount(), version == 1 ? Integer.MAX_VALUE : Long.MAX_VALUE),
299      loaded.getEntryCount());
300
301    if (version == 1) {
302      assertEquals(expected.getFileInfoOffset(), loaded.getFileInfoOffset());
303    }
304
305    if (version == 2) {
306      assertEquals(expected.getLastDataBlockOffset(), loaded.getLastDataBlockOffset());
307      assertEquals(expected.getNumDataIndexLevels(), loaded.getNumDataIndexLevels());
308      assertEquals(expected.createComparator().getClass().getName(),
309        loaded.createComparator().getClass().getName());
310      assertEquals(expected.getFirstDataBlockOffset(), loaded.getFirstDataBlockOffset());
311      assertTrue(expected.createComparator() instanceof CellComparatorImpl);
312      assertEquals(expected.getUncompressedDataIndexSize(), loaded.getUncompressedDataIndexSize());
313    }
314
315    assertEquals(expected.getLoadOnOpenDataOffset(), loaded.getLoadOnOpenDataOffset());
316    assertEquals(expected.getMetaIndexCount(), loaded.getMetaIndexCount());
317
318    assertEquals(expected.getTotalUncompressedBytes(), loaded.getTotalUncompressedBytes());
319  }
320
321}