001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.io.hfile; 019 020import static org.apache.hadoop.hbase.io.hfile.FixedFileTrailer.createComparator; 021import static org.junit.Assert.assertEquals; 022import static org.junit.Assert.assertNull; 023import static org.junit.Assert.assertThrows; 024import static org.junit.Assert.assertTrue; 025import static org.junit.Assert.fail; 026 027import java.io.ByteArrayInputStream; 028import java.io.ByteArrayOutputStream; 029import java.io.DataInputStream; 030import java.io.DataOutputStream; 031import java.io.IOException; 032import java.util.ArrayList; 033import java.util.Collection; 034import java.util.List; 035import org.apache.hadoop.fs.FSDataInputStream; 036import org.apache.hadoop.fs.FSDataOutputStream; 037import org.apache.hadoop.fs.FileSystem; 038import org.apache.hadoop.fs.Path; 039import org.apache.hadoop.hbase.CellComparator; 040import org.apache.hadoop.hbase.CellComparatorImpl; 041import org.apache.hadoop.hbase.HBaseClassTestRule; 042import org.apache.hadoop.hbase.HBaseTestingUtil; 043import org.apache.hadoop.hbase.InnerStoreCellComparator; 044import org.apache.hadoop.hbase.KeyValue; 045import org.apache.hadoop.hbase.MetaCellComparator; 046import org.apache.hadoop.hbase.testclassification.IOTests; 047import org.apache.hadoop.hbase.testclassification.SmallTests; 048import org.apache.hadoop.hbase.util.Bytes; 049import org.junit.Before; 050import org.junit.ClassRule; 051import org.junit.Test; 052import org.junit.experimental.categories.Category; 053import org.junit.runner.RunWith; 054import org.junit.runners.Parameterized; 055import org.junit.runners.Parameterized.Parameters; 056import org.slf4j.Logger; 057import org.slf4j.LoggerFactory; 058 059@RunWith(Parameterized.class) 060@Category({ IOTests.class, SmallTests.class }) 061public class TestFixedFileTrailer { 062 063 @ClassRule 064 public static final HBaseClassTestRule CLASS_RULE = 065 HBaseClassTestRule.forClass(TestFixedFileTrailer.class); 066 067 private static final Logger LOG = LoggerFactory.getLogger(TestFixedFileTrailer.class); 068 private static final int MAX_COMPARATOR_NAME_LENGTH = 128; 069 070 /** 071 * The number of used fields by version. Indexed by version minus two. Min version that we support 072 * is V2 073 */ 074 private static final int[] NUM_FIELDS_BY_VERSION = new int[] { 14, 15 }; 075 076 private HBaseTestingUtil util = new HBaseTestingUtil(); 077 private FileSystem fs; 078 private ByteArrayOutputStream baos = new ByteArrayOutputStream(); 079 private int version; 080 081 static { 082 assert NUM_FIELDS_BY_VERSION.length == HFile.MAX_FORMAT_VERSION - HFile.MIN_FORMAT_VERSION + 1; 083 } 084 085 public TestFixedFileTrailer(int version) { 086 this.version = version; 087 } 088 089 @Parameters 090 public static Collection<Object[]> getParameters() { 091 List<Object[]> versionsToTest = new ArrayList<>(); 092 for (int v = HFile.MIN_FORMAT_VERSION; v <= HFile.MAX_FORMAT_VERSION; ++v) 093 versionsToTest.add(new Integer[] { v }); 094 return versionsToTest; 095 } 096 097 @Before 098 public void setUp() throws IOException { 099 fs = FileSystem.get(util.getConfiguration()); 100 } 101 102 @Test 103 public void testCreateComparator() throws IOException { 104 assertEquals(InnerStoreCellComparator.class, 105 createComparator(KeyValue.COMPARATOR.getLegacyKeyComparatorName()).getClass()); 106 assertEquals(InnerStoreCellComparator.class, 107 createComparator(KeyValue.COMPARATOR.getClass().getName()).getClass()); 108 assertEquals(InnerStoreCellComparator.class, 109 createComparator(CellComparator.class.getName()).getClass()); 110 assertEquals(MetaCellComparator.class, 111 createComparator(KeyValue.META_COMPARATOR.getLegacyKeyComparatorName()).getClass()); 112 assertEquals(MetaCellComparator.class, 113 createComparator(KeyValue.META_COMPARATOR.getClass().getName()).getClass()); 114 assertEquals(MetaCellComparator.class, 115 createComparator("org.apache.hadoop.hbase.CellComparator$MetaCellComparator").getClass()); 116 assertEquals(MetaCellComparator.class, 117 createComparator("org.apache.hadoop.hbase.CellComparatorImpl$MetaCellComparator").getClass()); 118 assertEquals(MetaCellComparator.class, 119 createComparator(MetaCellComparator.META_COMPARATOR.getClass().getName()).getClass()); 120 assertEquals(MetaCellComparator.META_COMPARATOR.getClass(), 121 createComparator(MetaCellComparator.META_COMPARATOR.getClass().getName()).getClass()); 122 assertEquals(CellComparatorImpl.COMPARATOR.getClass(), 123 createComparator(MetaCellComparator.COMPARATOR.getClass().getName()).getClass()); 124 assertNull(createComparator(Bytes.BYTES_RAWCOMPARATOR.getClass().getName())); 125 assertNull(createComparator("org.apache.hadoop.hbase.KeyValue$RawBytesComparator")); 126 127 // Test an invalid comparatorClassName 128 assertThrows(IOException.class, () -> createComparator("")); 129 } 130 131 @Test 132 public void testTrailer() throws IOException { 133 FixedFileTrailer t = new FixedFileTrailer(version, HFileReaderImpl.PBUF_TRAILER_MINOR_VERSION); 134 t.setDataIndexCount(3); 135 t.setEntryCount(((long) Integer.MAX_VALUE) + 1); 136 137 t.setLastDataBlockOffset(291); 138 t.setNumDataIndexLevels(3); 139 t.setComparatorClass(InnerStoreCellComparator.INNER_STORE_COMPARATOR.getClass()); 140 t.setFirstDataBlockOffset(9081723123L); // Completely unrealistic. 141 t.setUncompressedDataIndexSize(827398717L); // Something random. 142 143 t.setLoadOnOpenOffset(128); 144 t.setMetaIndexCount(7); 145 146 t.setTotalUncompressedBytes(129731987); 147 148 { 149 DataOutputStream dos = new DataOutputStream(baos); // Limited scope. 150 t.serialize(dos); 151 dos.flush(); 152 assertEquals(dos.size(), FixedFileTrailer.getTrailerSize(version)); 153 } 154 155 byte[] bytes = baos.toByteArray(); 156 baos.reset(); 157 158 assertEquals(bytes.length, FixedFileTrailer.getTrailerSize(version)); 159 160 ByteArrayInputStream bais = new ByteArrayInputStream(bytes); 161 162 // Finished writing, trying to read. 163 { 164 DataInputStream dis = new DataInputStream(bais); 165 FixedFileTrailer t2 = 166 new FixedFileTrailer(version, HFileReaderImpl.PBUF_TRAILER_MINOR_VERSION); 167 t2.deserialize(dis); 168 assertEquals(-1, bais.read()); // Ensure we have read everything. 169 checkLoadedTrailer(version, t, t2); 170 } 171 172 // Now check what happens if the trailer is corrupted. 173 Path trailerPath = new Path(util.getDataTestDir(), "trailer_" + version); 174 175 { 176 for (byte invalidVersion : new byte[] { HFile.MIN_FORMAT_VERSION - 1, 177 HFile.MAX_FORMAT_VERSION + 1 }) { 178 bytes[bytes.length - 1] = invalidVersion; 179 writeTrailer(trailerPath, null, bytes); 180 try { 181 readTrailer(trailerPath); 182 fail("Exception expected"); 183 } catch (IllegalArgumentException ex) { 184 // Make it easy to debug this. 185 String msg = ex.getMessage(); 186 String cleanMsg = msg.replaceAll("^(java(\\.[a-zA-Z]+)+:\\s+)?|\\s+\\(.*\\)\\s*$", ""); 187 // will be followed by " expected: ..." 188 assertEquals("Actual exception message is \"" + msg + "\".\nCleaned-up message", 189 "Invalid HFile version: " + invalidVersion, cleanMsg); 190 LOG.info("Got an expected exception: " + msg); 191 } 192 } 193 194 } 195 196 // Now write the trailer into a file and auto-detect the version. 197 writeTrailer(trailerPath, t, null); 198 199 FixedFileTrailer t4 = readTrailer(trailerPath); 200 201 checkLoadedTrailer(version, t, t4); 202 203 String trailerStr = t.toString(); 204 assertEquals( 205 "Invalid number of fields in the string representation " + "of the trailer: " + trailerStr, 206 NUM_FIELDS_BY_VERSION[version - 2], trailerStr.split(", ").length); 207 assertEquals(trailerStr, t4.toString()); 208 } 209 210 @Test 211 public void testTrailerForV2NonPBCompatibility() throws Exception { 212 if (version == 2) { 213 FixedFileTrailer t = new FixedFileTrailer(version, HFileReaderImpl.MINOR_VERSION_NO_CHECKSUM); 214 t.setDataIndexCount(3); 215 t.setEntryCount(((long) Integer.MAX_VALUE) + 1); 216 t.setLastDataBlockOffset(291); 217 t.setNumDataIndexLevels(3); 218 t.setComparatorClass(CellComparatorImpl.COMPARATOR.getClass()); 219 t.setFirstDataBlockOffset(9081723123L); // Completely unrealistic. 220 t.setUncompressedDataIndexSize(827398717L); // Something random. 221 t.setLoadOnOpenOffset(128); 222 t.setMetaIndexCount(7); 223 t.setTotalUncompressedBytes(129731987); 224 225 { 226 DataOutputStream dos = new DataOutputStream(baos); // Limited scope. 227 serializeAsWritable(dos, t); 228 dos.flush(); 229 assertEquals(FixedFileTrailer.getTrailerSize(version), dos.size()); 230 } 231 232 byte[] bytes = baos.toByteArray(); 233 baos.reset(); 234 assertEquals(bytes.length, FixedFileTrailer.getTrailerSize(version)); 235 236 ByteArrayInputStream bais = new ByteArrayInputStream(bytes); 237 { 238 DataInputStream dis = new DataInputStream(bais); 239 FixedFileTrailer t2 = 240 new FixedFileTrailer(version, HFileReaderImpl.MINOR_VERSION_NO_CHECKSUM); 241 t2.deserialize(dis); 242 assertEquals(-1, bais.read()); // Ensure we have read everything. 243 checkLoadedTrailer(version, t, t2); 244 } 245 } 246 } 247 248 // Copied from FixedFileTrailer for testing the reading part of 249 // FixedFileTrailer of non PB 250 // serialized FFTs. 251 private void serializeAsWritable(DataOutputStream output, FixedFileTrailer fft) 252 throws IOException { 253 BlockType.TRAILER.write(output); 254 output.writeLong(fft.getFileInfoOffset()); 255 output.writeLong(fft.getLoadOnOpenDataOffset()); 256 output.writeInt(fft.getDataIndexCount()); 257 output.writeLong(fft.getUncompressedDataIndexSize()); 258 output.writeInt(fft.getMetaIndexCount()); 259 output.writeLong(fft.getTotalUncompressedBytes()); 260 output.writeLong(fft.getEntryCount()); 261 output.writeInt(fft.getCompressionCodec().ordinal()); 262 output.writeInt(fft.getNumDataIndexLevels()); 263 output.writeLong(fft.getFirstDataBlockOffset()); 264 output.writeLong(fft.getLastDataBlockOffset()); 265 Bytes.writeStringFixedSize(output, fft.getComparatorClassName(), MAX_COMPARATOR_NAME_LENGTH); 266 output 267 .writeInt(FixedFileTrailer.materializeVersion(fft.getMajorVersion(), fft.getMinorVersion())); 268 } 269 270 private FixedFileTrailer readTrailer(Path trailerPath) throws IOException { 271 FSDataInputStream fsdis = fs.open(trailerPath); 272 FixedFileTrailer trailerRead = 273 FixedFileTrailer.readFromStream(fsdis, fs.getFileStatus(trailerPath).getLen()); 274 fsdis.close(); 275 return trailerRead; 276 } 277 278 private void writeTrailer(Path trailerPath, FixedFileTrailer t, byte[] useBytesInstead) 279 throws IOException { 280 assert (t == null) != (useBytesInstead == null); // Expect one non-null. 281 282 FSDataOutputStream fsdos = fs.create(trailerPath); 283 fsdos.write(135); // to make deserializer's job less trivial 284 if (useBytesInstead != null) { 285 fsdos.write(useBytesInstead); 286 } else { 287 t.serialize(fsdos); 288 } 289 fsdos.close(); 290 } 291 292 private void checkLoadedTrailer(int version, FixedFileTrailer expected, FixedFileTrailer loaded) 293 throws IOException { 294 assertEquals(version, loaded.getMajorVersion()); 295 assertEquals(expected.getDataIndexCount(), loaded.getDataIndexCount()); 296 297 assertEquals( 298 Math.min(expected.getEntryCount(), version == 1 ? Integer.MAX_VALUE : Long.MAX_VALUE), 299 loaded.getEntryCount()); 300 301 if (version == 1) { 302 assertEquals(expected.getFileInfoOffset(), loaded.getFileInfoOffset()); 303 } 304 305 if (version == 2) { 306 assertEquals(expected.getLastDataBlockOffset(), loaded.getLastDataBlockOffset()); 307 assertEquals(expected.getNumDataIndexLevels(), loaded.getNumDataIndexLevels()); 308 assertEquals(expected.createComparator().getClass().getName(), 309 loaded.createComparator().getClass().getName()); 310 assertEquals(expected.getFirstDataBlockOffset(), loaded.getFirstDataBlockOffset()); 311 assertTrue(expected.createComparator() instanceof CellComparatorImpl); 312 assertEquals(expected.getUncompressedDataIndexSize(), loaded.getUncompressedDataIndexSize()); 313 } 314 315 assertEquals(expected.getLoadOnOpenDataOffset(), loaded.getLoadOnOpenDataOffset()); 316 assertEquals(expected.getMetaIndexCount(), loaded.getMetaIndexCount()); 317 318 assertEquals(expected.getTotalUncompressedBytes(), loaded.getTotalUncompressedBytes()); 319 } 320 321}