001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.io.hfile; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertFalse; 022import static org.junit.Assert.assertNull; 023import static org.junit.Assert.assertTrue; 024import static org.junit.Assert.fail; 025 026import java.io.DataInput; 027import java.io.DataOutput; 028import java.io.IOException; 029import java.nio.ByteBuffer; 030import java.util.Arrays; 031import java.util.Objects; 032import java.util.Random; 033import org.apache.hadoop.conf.Configuration; 034import org.apache.hadoop.fs.FSDataInputStream; 035import org.apache.hadoop.fs.FSDataOutputStream; 036import org.apache.hadoop.fs.FileStatus; 037import org.apache.hadoop.fs.FileSystem; 038import org.apache.hadoop.fs.Path; 039import org.apache.hadoop.hbase.ArrayBackedTag; 040import org.apache.hadoop.hbase.ByteBufferKeyValue; 041import org.apache.hadoop.hbase.Cell; 042import org.apache.hadoop.hbase.CellComparatorImpl; 043import org.apache.hadoop.hbase.CellUtil; 044import org.apache.hadoop.hbase.HBaseClassTestRule; 045import org.apache.hadoop.hbase.HBaseCommonTestingUtility; 046import org.apache.hadoop.hbase.HBaseTestingUtility; 047import org.apache.hadoop.hbase.HConstants; 048import org.apache.hadoop.hbase.KeyValue; 049import org.apache.hadoop.hbase.KeyValue.Type; 050import org.apache.hadoop.hbase.KeyValueUtil; 051import org.apache.hadoop.hbase.PrivateCellUtil; 052import org.apache.hadoop.hbase.Tag; 053import org.apache.hadoop.hbase.io.compress.Compression; 054import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder; 055import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; 056import org.apache.hadoop.hbase.io.hfile.HFile.Reader; 057import org.apache.hadoop.hbase.io.hfile.HFile.Writer; 058import org.apache.hadoop.hbase.nio.ByteBuff; 059import org.apache.hadoop.hbase.regionserver.StoreFileWriter; 060import org.apache.hadoop.hbase.testclassification.IOTests; 061import org.apache.hadoop.hbase.testclassification.SmallTests; 062import org.apache.hadoop.hbase.util.ByteBufferUtils; 063import org.apache.hadoop.hbase.util.Bytes; 064import org.apache.hadoop.io.Writable; 065import org.junit.BeforeClass; 066import org.junit.ClassRule; 067import org.junit.Rule; 068import org.junit.Test; 069import org.junit.experimental.categories.Category; 070import org.junit.rules.TestName; 071import org.slf4j.Logger; 072import org.slf4j.LoggerFactory; 073 074/** 075 * test hfile features. 076 */ 077@Category({IOTests.class, SmallTests.class}) 078public class TestHFile { 079 080 @ClassRule 081 public static final HBaseClassTestRule CLASS_RULE = 082 HBaseClassTestRule.forClass(TestHFile.class); 083 084 @Rule public TestName testName = new TestName(); 085 086 private static final Logger LOG = LoggerFactory.getLogger(TestHFile.class); 087 private static final int NUM_VALID_KEY_TYPES = KeyValue.Type.values().length - 2; 088 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); 089 private static String ROOT_DIR = 090 TEST_UTIL.getDataTestDir("TestHFile").toString(); 091 private final int minBlockSize = 512; 092 private static String localFormatter = "%010d"; 093 private static CacheConfig cacheConf = null; 094 private static Configuration conf ; 095 private static FileSystem fs; 096 097 @BeforeClass 098 public static void setUp() throws Exception { 099 conf = TEST_UTIL.getConfiguration(); 100 fs = TEST_UTIL.getTestFileSystem(); 101 } 102 103 @Test 104 public void testReaderWithoutBlockCache() throws Exception { 105 Path path = writeStoreFile(); 106 try{ 107 readStoreFile(path); 108 } catch (Exception e) { 109 // fail test 110 assertTrue(false); 111 } 112 } 113 114 115 private void readStoreFile(Path storeFilePath) throws Exception { 116 // Open the file reader with block cache disabled. 117 HFile.Reader reader = HFile.createReader(fs, storeFilePath, conf); 118 long offset = 0; 119 while (offset < reader.getTrailer().getLoadOnOpenDataOffset()) { 120 HFileBlock block = reader.readBlock(offset, -1, false, true, false, true, null, null); 121 offset += block.getOnDiskSizeWithHeader(); 122 } 123 } 124 125 private Path writeStoreFile() throws IOException { 126 Path storeFileParentDir = new Path(TEST_UTIL.getDataTestDir(), "TestHFile"); 127 HFileContext meta = new HFileContextBuilder().withBlockSize(64 * 1024).build(); 128 StoreFileWriter sfw = 129 new StoreFileWriter.Builder(conf, fs).withOutputDir(storeFileParentDir) 130 .withComparator(CellComparatorImpl.COMPARATOR).withFileContext(meta).build(); 131 132 final int rowLen = 32; 133 Random RNG = new Random(); 134 for (int i = 0; i < 1000; ++i) { 135 byte[] k = RandomKeyValueUtil.randomOrderedKey(RNG, i); 136 byte[] v = RandomKeyValueUtil.randomValue(RNG); 137 int cfLen = RNG.nextInt(k.length - rowLen + 1); 138 KeyValue kv = 139 new KeyValue(k, 0, rowLen, k, rowLen, cfLen, k, rowLen + cfLen, 140 k.length - rowLen - cfLen, RNG.nextLong(), generateKeyType(RNG), v, 0, v.length); 141 sfw.append(kv); 142 } 143 144 sfw.close(); 145 return sfw.getPath(); 146 } 147 148 public static KeyValue.Type generateKeyType(Random rand) { 149 if (rand.nextBoolean()) { 150 // Let's make half of KVs puts. 151 return KeyValue.Type.Put; 152 } else { 153 KeyValue.Type keyType = KeyValue.Type.values()[1 + rand.nextInt(NUM_VALID_KEY_TYPES)]; 154 if (keyType == KeyValue.Type.Minimum || keyType == KeyValue.Type.Maximum) { 155 throw new RuntimeException("Generated an invalid key type: " + keyType + ". " 156 + "Probably the layout of KeyValue.Type has changed."); 157 } 158 return keyType; 159 } 160 } 161 162 /** 163 * Test empty HFile. 164 * Test all features work reasonably when hfile is empty of entries. 165 * @throws IOException 166 */ 167 @Test 168 public void testEmptyHFile() throws IOException { 169 if (cacheConf == null) cacheConf = new CacheConfig(conf); 170 Path f = new Path(ROOT_DIR, testName.getMethodName()); 171 HFileContext context = new HFileContextBuilder().withIncludesTags(false).build(); 172 Writer w = 173 HFile.getWriterFactory(conf, cacheConf).withPath(fs, f).withFileContext(context).create(); 174 w.close(); 175 Reader r = HFile.createReader(fs, f, cacheConf, true, conf); 176 r.loadFileInfo(); 177 assertFalse(r.getFirstKey().isPresent()); 178 assertFalse(r.getLastKey().isPresent()); 179 } 180 181 /** 182 * Create 0-length hfile and show that it fails 183 */ 184 @Test 185 public void testCorrupt0LengthHFile() throws IOException { 186 if (cacheConf == null) cacheConf = new CacheConfig(conf); 187 Path f = new Path(ROOT_DIR, testName.getMethodName()); 188 FSDataOutputStream fsos = fs.create(f); 189 fsos.close(); 190 191 try { 192 Reader r = HFile.createReader(fs, f, cacheConf, true, conf); 193 } catch (CorruptHFileException che) { 194 // Expected failure 195 return; 196 } 197 fail("Should have thrown exception"); 198 } 199 200 public static void truncateFile(FileSystem fs, Path src, Path dst) throws IOException { 201 FileStatus fst = fs.getFileStatus(src); 202 long len = fst.getLen(); 203 len = len / 2 ; 204 205 // create a truncated hfile 206 FSDataOutputStream fdos = fs.create(dst); 207 byte[] buf = new byte[(int)len]; 208 FSDataInputStream fdis = fs.open(src); 209 fdis.read(buf); 210 fdos.write(buf); 211 fdis.close(); 212 fdos.close(); 213 } 214 215 /** 216 * Create a truncated hfile and verify that exception thrown. 217 */ 218 @Test 219 public void testCorruptTruncatedHFile() throws IOException { 220 if (cacheConf == null) cacheConf = new CacheConfig(conf); 221 Path f = new Path(ROOT_DIR, testName.getMethodName()); 222 HFileContext context = new HFileContextBuilder().build(); 223 Writer w = HFile.getWriterFactory(conf, cacheConf).withPath(this.fs, f) 224 .withFileContext(context).create(); 225 writeSomeRecords(w, 0, 100, false); 226 w.close(); 227 228 Path trunc = new Path(f.getParent(), "trucated"); 229 truncateFile(fs, w.getPath(), trunc); 230 231 try { 232 Reader r = HFile.createReader(fs, trunc, cacheConf, true, conf); 233 } catch (CorruptHFileException che) { 234 // Expected failure 235 return; 236 } 237 fail("Should have thrown exception"); 238 } 239 240 // write some records into the hfile 241 // write them twice 242 private int writeSomeRecords(Writer writer, int start, int n, boolean useTags) 243 throws IOException { 244 String value = "value"; 245 KeyValue kv; 246 for (int i = start; i < (start + n); i++) { 247 String key = String.format(localFormatter, Integer.valueOf(i)); 248 if (useTags) { 249 Tag t = new ArrayBackedTag((byte) 1, "myTag1"); 250 Tag[] tags = new Tag[1]; 251 tags[0] = t; 252 kv = new KeyValue(Bytes.toBytes(key), Bytes.toBytes("family"), Bytes.toBytes("qual"), 253 HConstants.LATEST_TIMESTAMP, Bytes.toBytes(value + key), tags); 254 writer.append(kv); 255 } else { 256 kv = new KeyValue(Bytes.toBytes(key), Bytes.toBytes("family"), Bytes.toBytes("qual"), 257 Bytes.toBytes(value + key)); 258 writer.append(kv); 259 } 260 } 261 return (start + n); 262 } 263 264 private void readAllRecords(HFileScanner scanner) throws IOException { 265 readAndCheckbytes(scanner, 0, 100); 266 } 267 268 // read the records and check 269 private int readAndCheckbytes(HFileScanner scanner, int start, int n) 270 throws IOException { 271 String value = "value"; 272 int i = start; 273 for (; i < (start + n); i++) { 274 ByteBuffer key = ByteBuffer.wrap(((KeyValue)scanner.getKey()).getKey()); 275 ByteBuffer val = scanner.getValue(); 276 String keyStr = String.format(localFormatter, Integer.valueOf(i)); 277 String valStr = value + keyStr; 278 KeyValue kv = new KeyValue(Bytes.toBytes(keyStr), Bytes.toBytes("family"), 279 Bytes.toBytes("qual"), Bytes.toBytes(valStr)); 280 byte[] keyBytes = new KeyValue.KeyOnlyKeyValue(Bytes.toBytes(key), 0, 281 Bytes.toBytes(key).length).getKey(); 282 assertTrue("bytes for keys do not match " + keyStr + " " + 283 Bytes.toString(Bytes.toBytes(key)), 284 Arrays.equals(kv.getKey(), keyBytes)); 285 byte [] valBytes = Bytes.toBytes(val); 286 assertTrue("bytes for vals do not match " + valStr + " " + 287 Bytes.toString(valBytes), 288 Arrays.equals(Bytes.toBytes(valStr), valBytes)); 289 if (!scanner.next()) { 290 break; 291 } 292 } 293 assertEquals(i, start + n - 1); 294 return (start + n); 295 } 296 297 private byte[] getSomeKey(int rowId) { 298 KeyValue kv = new KeyValue(String.format(localFormatter, Integer.valueOf(rowId)).getBytes(), 299 Bytes.toBytes("family"), Bytes.toBytes("qual"), HConstants.LATEST_TIMESTAMP, Type.Put); 300 return kv.getKey(); 301 } 302 303 private void writeRecords(Writer writer, boolean useTags) throws IOException { 304 writeSomeRecords(writer, 0, 100, useTags); 305 writer.close(); 306 } 307 308 private FSDataOutputStream createFSOutput(Path name) throws IOException { 309 //if (fs.exists(name)) fs.delete(name, true); 310 FSDataOutputStream fout = fs.create(name); 311 return fout; 312 } 313 314 /** 315 * test none codecs 316 * @param useTags 317 */ 318 void basicWithSomeCodec(String codec, boolean useTags) throws IOException { 319 if (useTags) { 320 conf.setInt("hfile.format.version", 3); 321 } 322 if (cacheConf == null) cacheConf = new CacheConfig(conf); 323 Path ncHFile = new Path(ROOT_DIR, "basic.hfile." + codec.toString() + useTags); 324 FSDataOutputStream fout = createFSOutput(ncHFile); 325 HFileContext meta = new HFileContextBuilder() 326 .withBlockSize(minBlockSize) 327 .withCompression(HFileWriterImpl.compressionByName(codec)) 328 .build(); 329 Writer writer = HFile.getWriterFactory(conf, cacheConf) 330 .withOutputStream(fout) 331 .withFileContext(meta) 332 .withComparator(CellComparatorImpl.COMPARATOR) 333 .create(); 334 LOG.info(Objects.toString(writer)); 335 writeRecords(writer, useTags); 336 fout.close(); 337 FSDataInputStream fin = fs.open(ncHFile); 338 Reader reader = HFile.createReaderFromStream(ncHFile, fs.open(ncHFile), 339 fs.getFileStatus(ncHFile).getLen(), cacheConf, conf); 340 System.out.println(cacheConf.toString()); 341 // Load up the index. 342 reader.loadFileInfo(); 343 // Get a scanner that caches and that does not use pread. 344 HFileScanner scanner = reader.getScanner(true, false); 345 // Align scanner at start of the file. 346 scanner.seekTo(); 347 readAllRecords(scanner); 348 int seekTo = scanner.seekTo(KeyValueUtil.createKeyValueFromKey(getSomeKey(50))); 349 System.out.println(seekTo); 350 assertTrue("location lookup failed", 351 scanner.seekTo(KeyValueUtil.createKeyValueFromKey(getSomeKey(50))) == 0); 352 // read the key and see if it matches 353 ByteBuffer readKey = ByteBuffer.wrap(((KeyValue)scanner.getKey()).getKey()); 354 assertTrue("seeked key does not match", Arrays.equals(getSomeKey(50), 355 Bytes.toBytes(readKey))); 356 357 scanner.seekTo(KeyValueUtil.createKeyValueFromKey(getSomeKey(0))); 358 ByteBuffer val1 = scanner.getValue(); 359 scanner.seekTo(KeyValueUtil.createKeyValueFromKey(getSomeKey(0))); 360 ByteBuffer val2 = scanner.getValue(); 361 assertTrue(Arrays.equals(Bytes.toBytes(val1), Bytes.toBytes(val2))); 362 363 reader.close(); 364 fin.close(); 365 fs.delete(ncHFile, true); 366 } 367 368 @Test 369 public void testTFileFeatures() throws IOException { 370 testHFilefeaturesInternals(false); 371 testHFilefeaturesInternals(true); 372 } 373 374 protected void testHFilefeaturesInternals(boolean useTags) throws IOException { 375 basicWithSomeCodec("none", useTags); 376 basicWithSomeCodec("gz", useTags); 377 } 378 379 private void writeNumMetablocks(Writer writer, int n) { 380 for (int i = 0; i < n; i++) { 381 writer.appendMetaBlock("HFileMeta" + i, new Writable() { 382 private int val; 383 public Writable setVal(int val) { this.val = val; return this; } 384 385 @Override 386 public void write(DataOutput out) throws IOException { 387 out.write(("something to test" + val).getBytes()); 388 } 389 390 @Override 391 public void readFields(DataInput in) throws IOException { } 392 }.setVal(i)); 393 } 394 } 395 396 private void someTestingWithMetaBlock(Writer writer) { 397 writeNumMetablocks(writer, 10); 398 } 399 400 private void readNumMetablocks(Reader reader, int n) throws IOException { 401 for (int i = 0; i < n; i++) { 402 ByteBuff actual = reader.getMetaBlock("HFileMeta" + i, false).getBufferWithoutHeader(); 403 ByteBuffer expected = 404 ByteBuffer.wrap(("something to test" + i).getBytes()); 405 assertEquals( 406 "failed to match metadata", 407 Bytes.toStringBinary(expected), 408 Bytes.toStringBinary(actual.array(), actual.arrayOffset() + actual.position(), 409 actual.capacity())); 410 } 411 } 412 413 private void someReadingWithMetaBlock(Reader reader) throws IOException { 414 readNumMetablocks(reader, 10); 415 } 416 417 private void metablocks(final String compress) throws Exception { 418 if (cacheConf == null) cacheConf = new CacheConfig(conf); 419 Path mFile = new Path(ROOT_DIR, "meta.hfile"); 420 FSDataOutputStream fout = createFSOutput(mFile); 421 HFileContext meta = new HFileContextBuilder() 422 .withCompression(HFileWriterImpl.compressionByName(compress)) 423 .withBlockSize(minBlockSize).build(); 424 Writer writer = HFile.getWriterFactory(conf, cacheConf) 425 .withOutputStream(fout) 426 .withFileContext(meta) 427 .create(); 428 someTestingWithMetaBlock(writer); 429 writer.close(); 430 fout.close(); 431 FSDataInputStream fin = fs.open(mFile); 432 Reader reader = HFile.createReaderFromStream(mFile, fs.open(mFile), 433 this.fs.getFileStatus(mFile).getLen(), cacheConf, conf); 434 reader.loadFileInfo(); 435 // No data -- this should return false. 436 assertFalse(reader.getScanner(false, false).seekTo()); 437 someReadingWithMetaBlock(reader); 438 fs.delete(mFile, true); 439 reader.close(); 440 fin.close(); 441 } 442 443 // test meta blocks for hfiles 444 @Test 445 public void testMetaBlocks() throws Exception { 446 metablocks("none"); 447 metablocks("gz"); 448 } 449 450 @Test 451 public void testNullMetaBlocks() throws Exception { 452 if (cacheConf == null) cacheConf = new CacheConfig(conf); 453 for (Compression.Algorithm compressAlgo : 454 HBaseCommonTestingUtility.COMPRESSION_ALGORITHMS) { 455 Path mFile = new Path(ROOT_DIR, "nometa_" + compressAlgo + ".hfile"); 456 FSDataOutputStream fout = createFSOutput(mFile); 457 HFileContext meta = new HFileContextBuilder().withCompression(compressAlgo) 458 .withBlockSize(minBlockSize).build(); 459 Writer writer = HFile.getWriterFactory(conf, cacheConf) 460 .withOutputStream(fout) 461 .withFileContext(meta) 462 .create(); 463 KeyValue kv = new KeyValue("foo".getBytes(), "f1".getBytes(), null, "value".getBytes()); 464 writer.append(kv); 465 writer.close(); 466 fout.close(); 467 Reader reader = HFile.createReader(fs, mFile, cacheConf, true, conf); 468 reader.loadFileInfo(); 469 assertNull(reader.getMetaBlock("non-existant", false)); 470 } 471 } 472 473 /** 474 * Make sure the ordinals for our compression algorithms do not change on us. 475 */ 476 @Test 477 public void testCompressionOrdinance() { 478 assertTrue(Compression.Algorithm.LZO.ordinal() == 0); 479 assertTrue(Compression.Algorithm.GZ.ordinal() == 1); 480 assertTrue(Compression.Algorithm.NONE.ordinal() == 2); 481 assertTrue(Compression.Algorithm.SNAPPY.ordinal() == 3); 482 assertTrue(Compression.Algorithm.LZ4.ordinal() == 4); 483 } 484 485 @Test 486 public void testShortMidpointSameQual() { 487 Cell left = CellUtil.createCell(Bytes.toBytes("a"), 488 Bytes.toBytes("a"), 489 Bytes.toBytes("a"), 490 11, 491 KeyValue.Type.Maximum.getCode(), 492 HConstants.EMPTY_BYTE_ARRAY); 493 Cell right = CellUtil.createCell(Bytes.toBytes("a"), 494 Bytes.toBytes("a"), 495 Bytes.toBytes("a"), 496 9, 497 KeyValue.Type.Maximum.getCode(), 498 HConstants.EMPTY_BYTE_ARRAY); 499 Cell mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); 500 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) <= 0); 501 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) == 0); 502 } 503 504 @Test 505 public void testGetShortMidpoint() { 506 Cell left = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); 507 Cell right = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); 508 Cell mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); 509 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) <= 0); 510 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) <= 0); 511 512 left = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); 513 right = CellUtil.createCell(Bytes.toBytes("b"), Bytes.toBytes("a"), Bytes.toBytes("a")); 514 mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); 515 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); 516 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) <= 0); 517 518 left = CellUtil.createCell(Bytes.toBytes("g"), Bytes.toBytes("a"), Bytes.toBytes("a")); 519 right = CellUtil.createCell(Bytes.toBytes("i"), Bytes.toBytes("a"), Bytes.toBytes("a")); 520 mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); 521 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); 522 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) <= 0); 523 524 left = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); 525 right = CellUtil.createCell(Bytes.toBytes("bbbbbbb"), Bytes.toBytes("a"), Bytes.toBytes("a")); 526 mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); 527 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); 528 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) < 0); 529 assertEquals(1, mid.getRowLength()); 530 531 left = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); 532 right = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("a")); 533 mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); 534 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); 535 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) <= 0); 536 537 left = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); 538 right = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("aaaaaaaa"), Bytes.toBytes("b")); 539 mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); 540 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); 541 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) < 0); 542 assertEquals(2, mid.getFamilyLength()); 543 544 left = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); 545 right = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("aaaaaaaaa")); 546 mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); 547 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); 548 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) < 0); 549 assertEquals(2, mid.getQualifierLength()); 550 551 left = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); 552 right = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("b")); 553 mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); 554 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); 555 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) <= 0); 556 assertEquals(1, mid.getQualifierLength()); 557 558 // Assert that if meta comparator, it returns the right cell -- i.e. no 559 // optimization done. 560 left = CellUtil.createCell(Bytes.toBytes("g"), Bytes.toBytes("a"), Bytes.toBytes("a")); 561 right = CellUtil.createCell(Bytes.toBytes("i"), Bytes.toBytes("a"), Bytes.toBytes("a")); 562 mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.META_COMPARATOR, left, right); 563 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); 564 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) == 0); 565 566 /** 567 * See HBASE-7845 568 */ 569 byte[] rowA = Bytes.toBytes("rowA"); 570 byte[] rowB = Bytes.toBytes("rowB"); 571 572 byte[] family = Bytes.toBytes("family"); 573 byte[] qualA = Bytes.toBytes("qfA"); 574 byte[] qualB = Bytes.toBytes("qfB"); 575 final CellComparatorImpl keyComparator = CellComparatorImpl.COMPARATOR; 576 // verify that faked shorter rowkey could be generated 577 long ts = 5; 578 KeyValue kv1 = new KeyValue(Bytes.toBytes("the quick brown fox"), family, qualA, ts, Type.Put); 579 KeyValue kv2 = new KeyValue(Bytes.toBytes("the who test text"), family, qualA, ts, Type.Put); 580 Cell newKey = HFileWriterImpl.getMidpoint(keyComparator, kv1, kv2); 581 assertTrue(keyComparator.compare(kv1, newKey) < 0); 582 assertTrue((keyComparator.compare(kv2, newKey)) > 0); 583 byte[] expectedArray = Bytes.toBytes("the r"); 584 Bytes.equals(newKey.getRowArray(), newKey.getRowOffset(), newKey.getRowLength(), expectedArray, 585 0, expectedArray.length); 586 587 // verify: same with "row + family + qualifier", return rightKey directly 588 kv1 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, 5, Type.Put); 589 kv2 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, 0, Type.Put); 590 assertTrue(keyComparator.compare(kv1, kv2) < 0); 591 newKey = HFileWriterImpl.getMidpoint(keyComparator, kv1, kv2); 592 assertTrue(keyComparator.compare(kv1, newKey) < 0); 593 assertTrue((keyComparator.compare(kv2, newKey)) == 0); 594 kv1 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, -5, Type.Put); 595 kv2 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, -10, Type.Put); 596 assertTrue(keyComparator.compare(kv1, kv2) < 0); 597 newKey = HFileWriterImpl.getMidpoint(keyComparator, kv1, kv2); 598 assertTrue(keyComparator.compare(kv1, newKey) < 0); 599 assertTrue((keyComparator.compare(kv2, newKey)) == 0); 600 601 // verify: same with row, different with qualifier 602 kv1 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, 5, Type.Put); 603 kv2 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualB, 5, Type.Put); 604 assertTrue(keyComparator.compare(kv1, kv2) < 0); 605 newKey = HFileWriterImpl.getMidpoint(keyComparator, kv1, kv2); 606 assertTrue(keyComparator.compare(kv1, newKey) < 0); 607 assertTrue((keyComparator.compare(kv2, newKey)) > 0); 608 assertTrue(Arrays.equals(CellUtil.cloneFamily(newKey), family)); 609 assertTrue(Arrays.equals(CellUtil.cloneQualifier(newKey), qualB)); 610 assertTrue(newKey.getTimestamp() == HConstants.LATEST_TIMESTAMP); 611 assertTrue(newKey.getTypeByte() == Type.Maximum.getCode()); 612 613 // verify metaKeyComparator's getShortMidpointKey output 614 final CellComparatorImpl metaKeyComparator = CellComparatorImpl.META_COMPARATOR; 615 kv1 = new KeyValue(Bytes.toBytes("ilovehbase123"), family, qualA, 5, Type.Put); 616 kv2 = new KeyValue(Bytes.toBytes("ilovehbase234"), family, qualA, 0, Type.Put); 617 newKey = HFileWriterImpl.getMidpoint(metaKeyComparator, kv1, kv2); 618 assertTrue(metaKeyComparator.compare(kv1, newKey) < 0); 619 assertTrue((metaKeyComparator.compare(kv2, newKey) == 0)); 620 621 // verify common fix scenario 622 kv1 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, ts, Type.Put); 623 kv2 = new KeyValue(Bytes.toBytes("ilovehbaseandhdfs"), family, qualA, ts, Type.Put); 624 assertTrue(keyComparator.compare(kv1, kv2) < 0); 625 newKey = HFileWriterImpl.getMidpoint(keyComparator, kv1, kv2); 626 assertTrue(keyComparator.compare(kv1, newKey) < 0); 627 assertTrue((keyComparator.compare(kv2, newKey)) > 0); 628 expectedArray = Bytes.toBytes("ilovehbasea"); 629 Bytes.equals(newKey.getRowArray(), newKey.getRowOffset(), newKey.getRowLength(), expectedArray, 630 0, expectedArray.length); 631 // verify only 1 offset scenario 632 kv1 = new KeyValue(Bytes.toBytes("100abcdefg"), family, qualA, ts, Type.Put); 633 kv2 = new KeyValue(Bytes.toBytes("101abcdefg"), family, qualA, ts, Type.Put); 634 assertTrue(keyComparator.compare(kv1, kv2) < 0); 635 newKey = HFileWriterImpl.getMidpoint(keyComparator, kv1, kv2); 636 assertTrue(keyComparator.compare(kv1, newKey) < 0); 637 assertTrue((keyComparator.compare(kv2, newKey)) > 0); 638 expectedArray = Bytes.toBytes("101"); 639 Bytes.equals(newKey.getRowArray(), newKey.getRowOffset(), newKey.getRowLength(), expectedArray, 640 0, expectedArray.length); 641 } 642 643 @Test 644 public void testDBEShipped() throws IOException { 645 for (DataBlockEncoding encoding : DataBlockEncoding.values()) { 646 DataBlockEncoder encoder = encoding.getEncoder(); 647 if (encoder == null) { 648 continue; 649 } 650 Path f = new Path(ROOT_DIR, testName.getMethodName() + "_" + encoding); 651 HFileContext context = new HFileContextBuilder() 652 .withIncludesTags(false) 653 .withDataBlockEncoding(encoding).build(); 654 HFileWriterImpl writer = (HFileWriterImpl) HFile.getWriterFactory(conf, cacheConf) 655 .withPath(fs, f).withFileContext(context).create(); 656 657 KeyValue kv = new KeyValue(Bytes.toBytes("testkey1"), Bytes.toBytes("family"), 658 Bytes.toBytes("qual"), Bytes.toBytes("testvalue")); 659 KeyValue kv2 = new KeyValue(Bytes.toBytes("testkey2"), Bytes.toBytes("family"), 660 Bytes.toBytes("qual"), Bytes.toBytes("testvalue")); 661 KeyValue kv3 = new KeyValue(Bytes.toBytes("testkey3"), Bytes.toBytes("family"), 662 Bytes.toBytes("qual"), Bytes.toBytes("testvalue")); 663 664 ByteBuffer buffer = ByteBuffer.wrap(kv.getBuffer()); 665 ByteBuffer buffer2 = ByteBuffer.wrap(kv2.getBuffer()); 666 ByteBuffer buffer3 = ByteBuffer.wrap(kv3.getBuffer()); 667 668 writer.append(new ByteBufferKeyValue(buffer, 0, buffer.remaining())); 669 writer.beforeShipped(); 670 671 // pollute first cell's backing ByteBuffer 672 ByteBufferUtils.copyFromBufferToBuffer(buffer3, buffer); 673 674 // write another cell, if DBE not Shipped, test will fail 675 writer.append(new ByteBufferKeyValue(buffer2, 0, buffer2.remaining())); 676 writer.close(); 677 } 678 } 679}