001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.io.hfile; 019 020import static org.apache.hadoop.hbase.io.hfile.FixedFileTrailer.createComparator; 021import static org.junit.Assert.assertEquals; 022import static org.junit.Assert.assertNull; 023import static org.junit.Assert.assertThrows; 024import static org.junit.Assert.assertTrue; 025import static org.junit.Assert.fail; 026 027import java.io.ByteArrayInputStream; 028import java.io.ByteArrayOutputStream; 029import java.io.DataInputStream; 030import java.io.DataOutputStream; 031import java.io.IOException; 032import java.util.ArrayList; 033import java.util.Collection; 034import java.util.List; 035import org.apache.hadoop.fs.FSDataInputStream; 036import org.apache.hadoop.fs.FSDataOutputStream; 037import org.apache.hadoop.fs.FileSystem; 038import org.apache.hadoop.fs.Path; 039import org.apache.hadoop.hbase.CellComparator; 040import org.apache.hadoop.hbase.CellComparatorImpl; 041import org.apache.hadoop.hbase.HBaseClassTestRule; 042import org.apache.hadoop.hbase.HBaseTestingUtil; 043import org.apache.hadoop.hbase.InnerStoreCellComparator; 044import org.apache.hadoop.hbase.MetaCellComparator; 045import org.apache.hadoop.hbase.testclassification.IOTests; 046import org.apache.hadoop.hbase.testclassification.SmallTests; 047import org.apache.hadoop.hbase.util.Bytes; 048import org.junit.Before; 049import org.junit.ClassRule; 050import org.junit.Test; 051import org.junit.experimental.categories.Category; 052import org.junit.runner.RunWith; 053import org.junit.runners.Parameterized; 054import org.junit.runners.Parameterized.Parameters; 055import org.slf4j.Logger; 056import org.slf4j.LoggerFactory; 057 058@RunWith(Parameterized.class) 059@Category({ IOTests.class, SmallTests.class }) 060public class TestFixedFileTrailer { 061 062 @ClassRule 063 public static final HBaseClassTestRule CLASS_RULE = 064 HBaseClassTestRule.forClass(TestFixedFileTrailer.class); 065 066 private static final Logger LOG = LoggerFactory.getLogger(TestFixedFileTrailer.class); 067 private static final int MAX_COMPARATOR_NAME_LENGTH = 128; 068 069 /** 070 * The number of used fields by version. Indexed by version minus two. Min version that we support 071 * is V2 072 */ 073 private static final int[] NUM_FIELDS_BY_VERSION = new int[] { 14, 15 }; 074 075 private HBaseTestingUtil util = new HBaseTestingUtil(); 076 private FileSystem fs; 077 private ByteArrayOutputStream baos = new ByteArrayOutputStream(); 078 private int version; 079 080 static { 081 assert NUM_FIELDS_BY_VERSION.length == HFile.MAX_FORMAT_VERSION - HFile.MIN_FORMAT_VERSION + 1; 082 } 083 084 public TestFixedFileTrailer(int version) { 085 this.version = version; 086 } 087 088 @Parameters 089 public static Collection<Object[]> getParameters() { 090 List<Object[]> versionsToTest = new ArrayList<>(); 091 for (int v = HFile.MIN_FORMAT_VERSION; v <= HFile.MAX_FORMAT_VERSION; ++v) 092 versionsToTest.add(new Integer[] { v }); 093 return versionsToTest; 094 } 095 096 @Before 097 public void setUp() throws IOException { 098 fs = FileSystem.get(util.getConfiguration()); 099 } 100 101 @Test 102 public void testCreateComparator() throws IOException { 103 assertEquals(InnerStoreCellComparator.class, 104 createComparator(CellComparator.class.getName()).getClass()); 105 assertEquals(MetaCellComparator.class, 106 createComparator("org.apache.hadoop.hbase.CellComparator$MetaCellComparator").getClass()); 107 assertEquals(MetaCellComparator.class, 108 createComparator("org.apache.hadoop.hbase.CellComparatorImpl$MetaCellComparator").getClass()); 109 assertEquals(MetaCellComparator.class, 110 createComparator(MetaCellComparator.META_COMPARATOR.getClass().getName()).getClass()); 111 assertEquals(MetaCellComparator.META_COMPARATOR.getClass(), 112 createComparator(MetaCellComparator.META_COMPARATOR.getClass().getName()).getClass()); 113 assertEquals(CellComparatorImpl.COMPARATOR.getClass(), 114 createComparator(MetaCellComparator.COMPARATOR.getClass().getName()).getClass()); 115 assertNull(createComparator(Bytes.BYTES_RAWCOMPARATOR.getClass().getName())); 116 assertNull(createComparator("org.apache.hadoop.hbase.KeyValue$RawBytesComparator")); 117 118 // Test an invalid comparatorClassName 119 assertThrows(IOException.class, () -> createComparator("")); 120 } 121 122 @Test 123 public void testTrailer() throws IOException { 124 FixedFileTrailer t = new FixedFileTrailer(version, HFileReaderImpl.PBUF_TRAILER_MINOR_VERSION); 125 t.setDataIndexCount(3); 126 t.setEntryCount(((long) Integer.MAX_VALUE) + 1); 127 128 t.setLastDataBlockOffset(291); 129 t.setNumDataIndexLevels(3); 130 t.setComparatorClass(InnerStoreCellComparator.INNER_STORE_COMPARATOR.getClass()); 131 t.setFirstDataBlockOffset(9081723123L); // Completely unrealistic. 132 t.setUncompressedDataIndexSize(827398717L); // Something random. 133 134 t.setLoadOnOpenOffset(128); 135 t.setMetaIndexCount(7); 136 137 t.setTotalUncompressedBytes(129731987); 138 139 { 140 DataOutputStream dos = new DataOutputStream(baos); // Limited scope. 141 t.serialize(dos); 142 dos.flush(); 143 assertEquals(dos.size(), FixedFileTrailer.getTrailerSize(version)); 144 } 145 146 byte[] bytes = baos.toByteArray(); 147 baos.reset(); 148 149 assertEquals(bytes.length, FixedFileTrailer.getTrailerSize(version)); 150 151 ByteArrayInputStream bais = new ByteArrayInputStream(bytes); 152 153 // Finished writing, trying to read. 154 { 155 DataInputStream dis = new DataInputStream(bais); 156 FixedFileTrailer t2 = 157 new FixedFileTrailer(version, HFileReaderImpl.PBUF_TRAILER_MINOR_VERSION); 158 t2.deserialize(dis); 159 assertEquals(-1, bais.read()); // Ensure we have read everything. 160 checkLoadedTrailer(version, t, t2); 161 } 162 163 // Now check what happens if the trailer is corrupted. 164 Path trailerPath = new Path(util.getDataTestDir(), "trailer_" + version); 165 166 { 167 for (byte invalidVersion : new byte[] { HFile.MIN_FORMAT_VERSION - 1, 168 HFile.MAX_FORMAT_VERSION + 1 }) { 169 bytes[bytes.length - 1] = invalidVersion; 170 writeTrailer(trailerPath, null, bytes); 171 try { 172 readTrailer(trailerPath); 173 fail("Exception expected"); 174 } catch (IllegalArgumentException ex) { 175 // Make it easy to debug this. 176 String msg = ex.getMessage(); 177 String cleanMsg = msg.replaceAll("^(java(\\.[a-zA-Z]+)+:\\s+)?|\\s+\\(.*\\)\\s*$", ""); 178 // will be followed by " expected: ..." 179 assertEquals("Actual exception message is \"" + msg + "\".\nCleaned-up message", 180 "Invalid HFile version: " + invalidVersion, cleanMsg); 181 LOG.info("Got an expected exception: " + msg); 182 } 183 } 184 185 } 186 187 // Now write the trailer into a file and auto-detect the version. 188 writeTrailer(trailerPath, t, null); 189 190 FixedFileTrailer t4 = readTrailer(trailerPath); 191 192 checkLoadedTrailer(version, t, t4); 193 194 String trailerStr = t.toString(); 195 assertEquals( 196 "Invalid number of fields in the string representation " + "of the trailer: " + trailerStr, 197 NUM_FIELDS_BY_VERSION[version - 2], trailerStr.split(", ").length); 198 assertEquals(trailerStr, t4.toString()); 199 } 200 201 @Test 202 public void testTrailerForV2NonPBCompatibility() throws Exception { 203 if (version == 2) { 204 FixedFileTrailer t = new FixedFileTrailer(version, HFileReaderImpl.MINOR_VERSION_NO_CHECKSUM); 205 t.setDataIndexCount(3); 206 t.setEntryCount(((long) Integer.MAX_VALUE) + 1); 207 t.setLastDataBlockOffset(291); 208 t.setNumDataIndexLevels(3); 209 t.setComparatorClass(CellComparatorImpl.COMPARATOR.getClass()); 210 t.setFirstDataBlockOffset(9081723123L); // Completely unrealistic. 211 t.setUncompressedDataIndexSize(827398717L); // Something random. 212 t.setLoadOnOpenOffset(128); 213 t.setMetaIndexCount(7); 214 t.setTotalUncompressedBytes(129731987); 215 216 { 217 DataOutputStream dos = new DataOutputStream(baos); // Limited scope. 218 serializeAsWritable(dos, t); 219 dos.flush(); 220 assertEquals(FixedFileTrailer.getTrailerSize(version), dos.size()); 221 } 222 223 byte[] bytes = baos.toByteArray(); 224 baos.reset(); 225 assertEquals(bytes.length, FixedFileTrailer.getTrailerSize(version)); 226 227 ByteArrayInputStream bais = new ByteArrayInputStream(bytes); 228 { 229 DataInputStream dis = new DataInputStream(bais); 230 FixedFileTrailer t2 = 231 new FixedFileTrailer(version, HFileReaderImpl.MINOR_VERSION_NO_CHECKSUM); 232 t2.deserialize(dis); 233 assertEquals(-1, bais.read()); // Ensure we have read everything. 234 checkLoadedTrailer(version, t, t2); 235 } 236 } 237 } 238 239 // Copied from FixedFileTrailer for testing the reading part of 240 // FixedFileTrailer of non PB 241 // serialized FFTs. 242 private void serializeAsWritable(DataOutputStream output, FixedFileTrailer fft) 243 throws IOException { 244 BlockType.TRAILER.write(output); 245 output.writeLong(fft.getFileInfoOffset()); 246 output.writeLong(fft.getLoadOnOpenDataOffset()); 247 output.writeInt(fft.getDataIndexCount()); 248 output.writeLong(fft.getUncompressedDataIndexSize()); 249 output.writeInt(fft.getMetaIndexCount()); 250 output.writeLong(fft.getTotalUncompressedBytes()); 251 output.writeLong(fft.getEntryCount()); 252 output.writeInt(fft.getCompressionCodec().ordinal()); 253 output.writeInt(fft.getNumDataIndexLevels()); 254 output.writeLong(fft.getFirstDataBlockOffset()); 255 output.writeLong(fft.getLastDataBlockOffset()); 256 Bytes.writeStringFixedSize(output, fft.getComparatorClassName(), MAX_COMPARATOR_NAME_LENGTH); 257 output 258 .writeInt(FixedFileTrailer.materializeVersion(fft.getMajorVersion(), fft.getMinorVersion())); 259 } 260 261 private FixedFileTrailer readTrailer(Path trailerPath) throws IOException { 262 FSDataInputStream fsdis = fs.open(trailerPath); 263 FixedFileTrailer trailerRead = 264 FixedFileTrailer.readFromStream(fsdis, fs.getFileStatus(trailerPath).getLen()); 265 fsdis.close(); 266 return trailerRead; 267 } 268 269 private void writeTrailer(Path trailerPath, FixedFileTrailer t, byte[] useBytesInstead) 270 throws IOException { 271 assert (t == null) != (useBytesInstead == null); // Expect one non-null. 272 273 FSDataOutputStream fsdos = fs.create(trailerPath); 274 fsdos.write(135); // to make deserializer's job less trivial 275 if (useBytesInstead != null) { 276 fsdos.write(useBytesInstead); 277 } else { 278 t.serialize(fsdos); 279 } 280 fsdos.close(); 281 } 282 283 private void checkLoadedTrailer(int version, FixedFileTrailer expected, FixedFileTrailer loaded) 284 throws IOException { 285 assertEquals(version, loaded.getMajorVersion()); 286 assertEquals(expected.getDataIndexCount(), loaded.getDataIndexCount()); 287 288 assertEquals( 289 Math.min(expected.getEntryCount(), version == 1 ? Integer.MAX_VALUE : Long.MAX_VALUE), 290 loaded.getEntryCount()); 291 292 if (version == 1) { 293 assertEquals(expected.getFileInfoOffset(), loaded.getFileInfoOffset()); 294 } 295 296 if (version == 2) { 297 assertEquals(expected.getLastDataBlockOffset(), loaded.getLastDataBlockOffset()); 298 assertEquals(expected.getNumDataIndexLevels(), loaded.getNumDataIndexLevels()); 299 assertEquals(expected.createComparator().getClass().getName(), 300 loaded.createComparator().getClass().getName()); 301 assertEquals(expected.getFirstDataBlockOffset(), loaded.getFirstDataBlockOffset()); 302 assertTrue(expected.createComparator() instanceof CellComparatorImpl); 303 assertEquals(expected.getUncompressedDataIndexSize(), loaded.getUncompressedDataIndexSize()); 304 } 305 306 assertEquals(expected.getLoadOnOpenDataOffset(), loaded.getLoadOnOpenDataOffset()); 307 assertEquals(expected.getMetaIndexCount(), loaded.getMetaIndexCount()); 308 309 assertEquals(expected.getTotalUncompressedBytes(), loaded.getTotalUncompressedBytes()); 310 } 311 312}