001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.io.hfile; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertFalse; 022import static org.junit.Assert.assertNull; 023import static org.junit.Assert.assertTrue; 024import static org.junit.Assert.fail; 025 026import java.io.DataInput; 027import java.io.DataOutput; 028import java.io.IOException; 029import java.nio.ByteBuffer; 030import java.util.Arrays; 031import java.util.Objects; 032import java.util.Random; 033import org.apache.hadoop.conf.Configuration; 034import org.apache.hadoop.fs.FSDataInputStream; 035import org.apache.hadoop.fs.FSDataOutputStream; 036import org.apache.hadoop.fs.FileStatus; 037import org.apache.hadoop.fs.FileSystem; 038import org.apache.hadoop.fs.Path; 039import org.apache.hadoop.hbase.ArrayBackedTag; 040import org.apache.hadoop.hbase.ByteBufferKeyValue; 041import org.apache.hadoop.hbase.Cell; 042import org.apache.hadoop.hbase.CellComparatorImpl; 043import org.apache.hadoop.hbase.CellUtil; 044import org.apache.hadoop.hbase.HBaseClassTestRule; 045import org.apache.hadoop.hbase.HBaseCommonTestingUtility; 046import org.apache.hadoop.hbase.HBaseTestingUtility; 047import org.apache.hadoop.hbase.HConstants; 048import org.apache.hadoop.hbase.KeyValue; 049import org.apache.hadoop.hbase.KeyValue.Type; 050import org.apache.hadoop.hbase.KeyValueUtil; 051import org.apache.hadoop.hbase.PrivateCellUtil; 052import org.apache.hadoop.hbase.Tag; 053import org.apache.hadoop.hbase.io.compress.Compression; 054import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder; 055import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; 056import org.apache.hadoop.hbase.io.hfile.HFile.Reader; 057import org.apache.hadoop.hbase.io.hfile.HFile.Writer; 058import org.apache.hadoop.hbase.nio.ByteBuff; 059import org.apache.hadoop.hbase.regionserver.StoreFileWriter; 060import org.apache.hadoop.hbase.testclassification.IOTests; 061import org.apache.hadoop.hbase.testclassification.SmallTests; 062import org.apache.hadoop.hbase.util.ByteBufferUtils; 063import org.apache.hadoop.hbase.util.Bytes; 064import org.apache.hadoop.io.Writable; 065import org.junit.BeforeClass; 066import org.junit.ClassRule; 067import org.junit.Rule; 068import org.junit.Test; 069import org.junit.experimental.categories.Category; 070import org.junit.rules.TestName; 071import org.slf4j.Logger; 072import org.slf4j.LoggerFactory; 073 074/** 075 * test hfile features. 076 */ 077@Category({IOTests.class, SmallTests.class}) 078public class TestHFile { 079 080 @ClassRule 081 public static final HBaseClassTestRule CLASS_RULE = 082 HBaseClassTestRule.forClass(TestHFile.class); 083 084 @Rule public TestName testName = new TestName(); 085 086 private static final Logger LOG = LoggerFactory.getLogger(TestHFile.class); 087 private static final int NUM_VALID_KEY_TYPES = KeyValue.Type.values().length - 2; 088 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); 089 private static String ROOT_DIR = 090 TEST_UTIL.getDataTestDir("TestHFile").toString(); 091 private final int minBlockSize = 512; 092 private static String localFormatter = "%010d"; 093 private static CacheConfig cacheConf; 094 private static Configuration conf ; 095 private static FileSystem fs; 096 097 @BeforeClass 098 public static void setUp() throws Exception { 099 conf = TEST_UTIL.getConfiguration(); 100 cacheConf = new CacheConfig(conf); 101 fs = TEST_UTIL.getTestFileSystem(); 102 } 103 104 @Test 105 public void testReaderWithoutBlockCache() throws Exception { 106 Path path = writeStoreFile(); 107 try{ 108 readStoreFile(path); 109 } catch (Exception e) { 110 // fail test 111 assertTrue(false); 112 } 113 } 114 115 116 private void readStoreFile(Path storeFilePath) throws Exception { 117 // Open the file reader with block cache disabled. 118 HFile.Reader reader = HFile.createReader(fs, storeFilePath, conf); 119 long offset = 0; 120 while (offset < reader.getTrailer().getLoadOnOpenDataOffset()) { 121 HFileBlock block = reader.readBlock(offset, -1, false, true, false, true, null, null); 122 offset += block.getOnDiskSizeWithHeader(); 123 } 124 } 125 126 private Path writeStoreFile() throws IOException { 127 Path storeFileParentDir = new Path(TEST_UTIL.getDataTestDir(), "TestHFile"); 128 HFileContext meta = new HFileContextBuilder().withBlockSize(64 * 1024).build(); 129 StoreFileWriter sfw = 130 new StoreFileWriter.Builder(conf, fs).withOutputDir(storeFileParentDir) 131 .withComparator(CellComparatorImpl.COMPARATOR).withFileContext(meta).build(); 132 133 final int rowLen = 32; 134 Random RNG = new Random(); 135 for (int i = 0; i < 1000; ++i) { 136 byte[] k = RandomKeyValueUtil.randomOrderedKey(RNG, i); 137 byte[] v = RandomKeyValueUtil.randomValue(RNG); 138 int cfLen = RNG.nextInt(k.length - rowLen + 1); 139 KeyValue kv = 140 new KeyValue(k, 0, rowLen, k, rowLen, cfLen, k, rowLen + cfLen, 141 k.length - rowLen - cfLen, RNG.nextLong(), generateKeyType(RNG), v, 0, v.length); 142 sfw.append(kv); 143 } 144 145 sfw.close(); 146 return sfw.getPath(); 147 } 148 149 public static KeyValue.Type generateKeyType(Random rand) { 150 if (rand.nextBoolean()) { 151 // Let's make half of KVs puts. 152 return KeyValue.Type.Put; 153 } else { 154 KeyValue.Type keyType = KeyValue.Type.values()[1 + rand.nextInt(NUM_VALID_KEY_TYPES)]; 155 if (keyType == KeyValue.Type.Minimum || keyType == KeyValue.Type.Maximum) { 156 throw new RuntimeException("Generated an invalid key type: " + keyType + ". " 157 + "Probably the layout of KeyValue.Type has changed."); 158 } 159 return keyType; 160 } 161 } 162 163 /** 164 * Test empty HFile. 165 * Test all features work reasonably when hfile is empty of entries. 166 * @throws IOException 167 */ 168 @Test 169 public void testEmptyHFile() throws IOException { 170 Path f = new Path(ROOT_DIR, testName.getMethodName()); 171 HFileContext context = new HFileContextBuilder().withIncludesTags(false).build(); 172 Writer w = 173 HFile.getWriterFactory(conf, cacheConf).withPath(fs, f).withFileContext(context).create(); 174 w.close(); 175 Reader r = HFile.createReader(fs, f, cacheConf, true, conf); 176 r.loadFileInfo(); 177 assertFalse(r.getFirstKey().isPresent()); 178 assertFalse(r.getLastKey().isPresent()); 179 } 180 181 /** 182 * Create 0-length hfile and show that it fails 183 */ 184 @Test 185 public void testCorrupt0LengthHFile() throws IOException { 186 Path f = new Path(ROOT_DIR, testName.getMethodName()); 187 FSDataOutputStream fsos = fs.create(f); 188 fsos.close(); 189 190 try { 191 Reader r = HFile.createReader(fs, f, cacheConf, true, conf); 192 } catch (CorruptHFileException che) { 193 // Expected failure 194 return; 195 } 196 fail("Should have thrown exception"); 197 } 198 199 public static void truncateFile(FileSystem fs, Path src, Path dst) throws IOException { 200 FileStatus fst = fs.getFileStatus(src); 201 long len = fst.getLen(); 202 len = len / 2 ; 203 204 // create a truncated hfile 205 FSDataOutputStream fdos = fs.create(dst); 206 byte[] buf = new byte[(int)len]; 207 FSDataInputStream fdis = fs.open(src); 208 fdis.read(buf); 209 fdos.write(buf); 210 fdis.close(); 211 fdos.close(); 212 } 213 214 /** 215 * Create a truncated hfile and verify that exception thrown. 216 */ 217 @Test 218 public void testCorruptTruncatedHFile() throws IOException { 219 Path f = new Path(ROOT_DIR, testName.getMethodName()); 220 HFileContext context = new HFileContextBuilder().build(); 221 Writer w = HFile.getWriterFactory(conf, cacheConf).withPath(this.fs, f) 222 .withFileContext(context).create(); 223 writeSomeRecords(w, 0, 100, false); 224 w.close(); 225 226 Path trunc = new Path(f.getParent(), "trucated"); 227 truncateFile(fs, w.getPath(), trunc); 228 229 try { 230 Reader r = HFile.createReader(fs, trunc, cacheConf, true, conf); 231 } catch (CorruptHFileException che) { 232 // Expected failure 233 return; 234 } 235 fail("Should have thrown exception"); 236 } 237 238 // write some records into the hfile 239 // write them twice 240 private int writeSomeRecords(Writer writer, int start, int n, boolean useTags) 241 throws IOException { 242 String value = "value"; 243 KeyValue kv; 244 for (int i = start; i < (start + n); i++) { 245 String key = String.format(localFormatter, Integer.valueOf(i)); 246 if (useTags) { 247 Tag t = new ArrayBackedTag((byte) 1, "myTag1"); 248 Tag[] tags = new Tag[1]; 249 tags[0] = t; 250 kv = new KeyValue(Bytes.toBytes(key), Bytes.toBytes("family"), Bytes.toBytes("qual"), 251 HConstants.LATEST_TIMESTAMP, Bytes.toBytes(value + key), tags); 252 writer.append(kv); 253 } else { 254 kv = new KeyValue(Bytes.toBytes(key), Bytes.toBytes("family"), Bytes.toBytes("qual"), 255 Bytes.toBytes(value + key)); 256 writer.append(kv); 257 } 258 } 259 return (start + n); 260 } 261 262 private void readAllRecords(HFileScanner scanner) throws IOException { 263 readAndCheckbytes(scanner, 0, 100); 264 } 265 266 // read the records and check 267 private int readAndCheckbytes(HFileScanner scanner, int start, int n) 268 throws IOException { 269 String value = "value"; 270 int i = start; 271 for (; i < (start + n); i++) { 272 ByteBuffer key = ByteBuffer.wrap(((KeyValue)scanner.getKey()).getKey()); 273 ByteBuffer val = scanner.getValue(); 274 String keyStr = String.format(localFormatter, Integer.valueOf(i)); 275 String valStr = value + keyStr; 276 KeyValue kv = new KeyValue(Bytes.toBytes(keyStr), Bytes.toBytes("family"), 277 Bytes.toBytes("qual"), Bytes.toBytes(valStr)); 278 byte[] keyBytes = new KeyValue.KeyOnlyKeyValue(Bytes.toBytes(key), 0, 279 Bytes.toBytes(key).length).getKey(); 280 assertTrue("bytes for keys do not match " + keyStr + " " + 281 Bytes.toString(Bytes.toBytes(key)), 282 Arrays.equals(kv.getKey(), keyBytes)); 283 byte [] valBytes = Bytes.toBytes(val); 284 assertTrue("bytes for vals do not match " + valStr + " " + 285 Bytes.toString(valBytes), 286 Arrays.equals(Bytes.toBytes(valStr), valBytes)); 287 if (!scanner.next()) { 288 break; 289 } 290 } 291 assertEquals(i, start + n - 1); 292 return (start + n); 293 } 294 295 private byte[] getSomeKey(int rowId) { 296 KeyValue kv = new KeyValue(String.format(localFormatter, Integer.valueOf(rowId)).getBytes(), 297 Bytes.toBytes("family"), Bytes.toBytes("qual"), HConstants.LATEST_TIMESTAMP, Type.Put); 298 return kv.getKey(); 299 } 300 301 private void writeRecords(Writer writer, boolean useTags) throws IOException { 302 writeSomeRecords(writer, 0, 100, useTags); 303 writer.close(); 304 } 305 306 private FSDataOutputStream createFSOutput(Path name) throws IOException { 307 //if (fs.exists(name)) fs.delete(name, true); 308 FSDataOutputStream fout = fs.create(name); 309 return fout; 310 } 311 312 /** 313 * test none codecs 314 * @param useTags 315 */ 316 void basicWithSomeCodec(String codec, boolean useTags) throws IOException { 317 if (useTags) { 318 conf.setInt("hfile.format.version", 3); 319 } 320 Path ncHFile = new Path(ROOT_DIR, "basic.hfile." + codec.toString() + useTags); 321 FSDataOutputStream fout = createFSOutput(ncHFile); 322 HFileContext meta = new HFileContextBuilder() 323 .withBlockSize(minBlockSize) 324 .withCompression(HFileWriterImpl.compressionByName(codec)) 325 .build(); 326 Writer writer = HFile.getWriterFactory(conf, cacheConf) 327 .withOutputStream(fout) 328 .withFileContext(meta) 329 .withComparator(CellComparatorImpl.COMPARATOR) 330 .create(); 331 LOG.info(Objects.toString(writer)); 332 writeRecords(writer, useTags); 333 fout.close(); 334 FSDataInputStream fin = fs.open(ncHFile); 335 Reader reader = HFile.createReaderFromStream(ncHFile, fs.open(ncHFile), 336 fs.getFileStatus(ncHFile).getLen(), cacheConf, conf); 337 System.out.println(cacheConf.toString()); 338 // Load up the index. 339 reader.loadFileInfo(); 340 // Get a scanner that caches and that does not use pread. 341 HFileScanner scanner = reader.getScanner(true, false); 342 // Align scanner at start of the file. 343 scanner.seekTo(); 344 readAllRecords(scanner); 345 int seekTo = scanner.seekTo(KeyValueUtil.createKeyValueFromKey(getSomeKey(50))); 346 System.out.println(seekTo); 347 assertTrue("location lookup failed", 348 scanner.seekTo(KeyValueUtil.createKeyValueFromKey(getSomeKey(50))) == 0); 349 // read the key and see if it matches 350 ByteBuffer readKey = ByteBuffer.wrap(((KeyValue)scanner.getKey()).getKey()); 351 assertTrue("seeked key does not match", Arrays.equals(getSomeKey(50), 352 Bytes.toBytes(readKey))); 353 354 scanner.seekTo(KeyValueUtil.createKeyValueFromKey(getSomeKey(0))); 355 ByteBuffer val1 = scanner.getValue(); 356 scanner.seekTo(KeyValueUtil.createKeyValueFromKey(getSomeKey(0))); 357 ByteBuffer val2 = scanner.getValue(); 358 assertTrue(Arrays.equals(Bytes.toBytes(val1), Bytes.toBytes(val2))); 359 360 reader.close(); 361 fin.close(); 362 fs.delete(ncHFile, true); 363 } 364 365 @Test 366 public void testTFileFeatures() throws IOException { 367 testHFilefeaturesInternals(false); 368 testHFilefeaturesInternals(true); 369 } 370 371 protected void testHFilefeaturesInternals(boolean useTags) throws IOException { 372 basicWithSomeCodec("none", useTags); 373 basicWithSomeCodec("gz", useTags); 374 } 375 376 private void writeNumMetablocks(Writer writer, int n) { 377 for (int i = 0; i < n; i++) { 378 writer.appendMetaBlock("HFileMeta" + i, new Writable() { 379 private int val; 380 public Writable setVal(int val) { this.val = val; return this; } 381 382 @Override 383 public void write(DataOutput out) throws IOException { 384 out.write(("something to test" + val).getBytes()); 385 } 386 387 @Override 388 public void readFields(DataInput in) throws IOException { } 389 }.setVal(i)); 390 } 391 } 392 393 private void someTestingWithMetaBlock(Writer writer) { 394 writeNumMetablocks(writer, 10); 395 } 396 397 private void readNumMetablocks(Reader reader, int n) throws IOException { 398 for (int i = 0; i < n; i++) { 399 ByteBuff actual = reader.getMetaBlock("HFileMeta" + i, false).getBufferWithoutHeader(); 400 ByteBuffer expected = 401 ByteBuffer.wrap(("something to test" + i).getBytes()); 402 assertEquals( 403 "failed to match metadata", 404 Bytes.toStringBinary(expected), 405 Bytes.toStringBinary(actual.array(), actual.arrayOffset() + actual.position(), 406 actual.capacity())); 407 } 408 } 409 410 private void someReadingWithMetaBlock(Reader reader) throws IOException { 411 readNumMetablocks(reader, 10); 412 } 413 414 private void metablocks(final String compress) throws Exception { 415 Path mFile = new Path(ROOT_DIR, "meta.hfile"); 416 FSDataOutputStream fout = createFSOutput(mFile); 417 HFileContext meta = new HFileContextBuilder() 418 .withCompression(HFileWriterImpl.compressionByName(compress)) 419 .withBlockSize(minBlockSize).build(); 420 Writer writer = HFile.getWriterFactory(conf, cacheConf) 421 .withOutputStream(fout) 422 .withFileContext(meta) 423 .create(); 424 someTestingWithMetaBlock(writer); 425 writer.close(); 426 fout.close(); 427 FSDataInputStream fin = fs.open(mFile); 428 Reader reader = HFile.createReaderFromStream(mFile, fs.open(mFile), 429 this.fs.getFileStatus(mFile).getLen(), cacheConf, conf); 430 reader.loadFileInfo(); 431 // No data -- this should return false. 432 assertFalse(reader.getScanner(false, false).seekTo()); 433 someReadingWithMetaBlock(reader); 434 fs.delete(mFile, true); 435 reader.close(); 436 fin.close(); 437 } 438 439 // test meta blocks for hfiles 440 @Test 441 public void testMetaBlocks() throws Exception { 442 metablocks("none"); 443 metablocks("gz"); 444 } 445 446 @Test 447 public void testNullMetaBlocks() throws Exception { 448 for (Compression.Algorithm compressAlgo : 449 HBaseCommonTestingUtility.COMPRESSION_ALGORITHMS) { 450 Path mFile = new Path(ROOT_DIR, "nometa_" + compressAlgo + ".hfile"); 451 FSDataOutputStream fout = createFSOutput(mFile); 452 HFileContext meta = new HFileContextBuilder().withCompression(compressAlgo) 453 .withBlockSize(minBlockSize).build(); 454 Writer writer = HFile.getWriterFactory(conf, cacheConf) 455 .withOutputStream(fout) 456 .withFileContext(meta) 457 .create(); 458 KeyValue kv = new KeyValue("foo".getBytes(), "f1".getBytes(), null, "value".getBytes()); 459 writer.append(kv); 460 writer.close(); 461 fout.close(); 462 Reader reader = HFile.createReader(fs, mFile, cacheConf, true, conf); 463 reader.loadFileInfo(); 464 assertNull(reader.getMetaBlock("non-existant", false)); 465 } 466 } 467 468 /** 469 * Make sure the ordinals for our compression algorithms do not change on us. 470 */ 471 @Test 472 public void testCompressionOrdinance() { 473 assertTrue(Compression.Algorithm.LZO.ordinal() == 0); 474 assertTrue(Compression.Algorithm.GZ.ordinal() == 1); 475 assertTrue(Compression.Algorithm.NONE.ordinal() == 2); 476 assertTrue(Compression.Algorithm.SNAPPY.ordinal() == 3); 477 assertTrue(Compression.Algorithm.LZ4.ordinal() == 4); 478 } 479 480 @Test 481 public void testShortMidpointSameQual() { 482 Cell left = CellUtil.createCell(Bytes.toBytes("a"), 483 Bytes.toBytes("a"), 484 Bytes.toBytes("a"), 485 11, 486 KeyValue.Type.Maximum.getCode(), 487 HConstants.EMPTY_BYTE_ARRAY); 488 Cell right = CellUtil.createCell(Bytes.toBytes("a"), 489 Bytes.toBytes("a"), 490 Bytes.toBytes("a"), 491 9, 492 KeyValue.Type.Maximum.getCode(), 493 HConstants.EMPTY_BYTE_ARRAY); 494 Cell mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); 495 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) <= 0); 496 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) == 0); 497 } 498 499 @Test 500 public void testGetShortMidpoint() { 501 Cell left = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); 502 Cell right = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); 503 Cell mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); 504 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) <= 0); 505 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) <= 0); 506 507 left = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); 508 right = CellUtil.createCell(Bytes.toBytes("b"), Bytes.toBytes("a"), Bytes.toBytes("a")); 509 mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); 510 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); 511 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) <= 0); 512 513 left = CellUtil.createCell(Bytes.toBytes("g"), Bytes.toBytes("a"), Bytes.toBytes("a")); 514 right = CellUtil.createCell(Bytes.toBytes("i"), Bytes.toBytes("a"), Bytes.toBytes("a")); 515 mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); 516 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); 517 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) <= 0); 518 519 left = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); 520 right = CellUtil.createCell(Bytes.toBytes("bbbbbbb"), Bytes.toBytes("a"), Bytes.toBytes("a")); 521 mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); 522 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); 523 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) < 0); 524 assertEquals(1, mid.getRowLength()); 525 526 left = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); 527 right = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("a")); 528 mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); 529 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); 530 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) <= 0); 531 532 left = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); 533 right = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("aaaaaaaa"), Bytes.toBytes("b")); 534 mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); 535 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); 536 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) < 0); 537 assertEquals(2, mid.getFamilyLength()); 538 539 left = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); 540 right = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("aaaaaaaaa")); 541 mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); 542 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); 543 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) < 0); 544 assertEquals(2, mid.getQualifierLength()); 545 546 left = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); 547 right = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("b")); 548 mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); 549 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); 550 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) <= 0); 551 assertEquals(1, mid.getQualifierLength()); 552 553 // Assert that if meta comparator, it returns the right cell -- i.e. no 554 // optimization done. 555 left = CellUtil.createCell(Bytes.toBytes("g"), Bytes.toBytes("a"), Bytes.toBytes("a")); 556 right = CellUtil.createCell(Bytes.toBytes("i"), Bytes.toBytes("a"), Bytes.toBytes("a")); 557 mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.META_COMPARATOR, left, right); 558 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); 559 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) == 0); 560 561 /** 562 * See HBASE-7845 563 */ 564 byte[] rowA = Bytes.toBytes("rowA"); 565 byte[] rowB = Bytes.toBytes("rowB"); 566 567 byte[] family = Bytes.toBytes("family"); 568 byte[] qualA = Bytes.toBytes("qfA"); 569 byte[] qualB = Bytes.toBytes("qfB"); 570 final CellComparatorImpl keyComparator = CellComparatorImpl.COMPARATOR; 571 // verify that faked shorter rowkey could be generated 572 long ts = 5; 573 KeyValue kv1 = new KeyValue(Bytes.toBytes("the quick brown fox"), family, qualA, ts, Type.Put); 574 KeyValue kv2 = new KeyValue(Bytes.toBytes("the who test text"), family, qualA, ts, Type.Put); 575 Cell newKey = HFileWriterImpl.getMidpoint(keyComparator, kv1, kv2); 576 assertTrue(keyComparator.compare(kv1, newKey) < 0); 577 assertTrue((keyComparator.compare(kv2, newKey)) > 0); 578 byte[] expectedArray = Bytes.toBytes("the r"); 579 Bytes.equals(newKey.getRowArray(), newKey.getRowOffset(), newKey.getRowLength(), expectedArray, 580 0, expectedArray.length); 581 582 // verify: same with "row + family + qualifier", return rightKey directly 583 kv1 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, 5, Type.Put); 584 kv2 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, 0, Type.Put); 585 assertTrue(keyComparator.compare(kv1, kv2) < 0); 586 newKey = HFileWriterImpl.getMidpoint(keyComparator, kv1, kv2); 587 assertTrue(keyComparator.compare(kv1, newKey) < 0); 588 assertTrue((keyComparator.compare(kv2, newKey)) == 0); 589 kv1 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, -5, Type.Put); 590 kv2 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, -10, Type.Put); 591 assertTrue(keyComparator.compare(kv1, kv2) < 0); 592 newKey = HFileWriterImpl.getMidpoint(keyComparator, kv1, kv2); 593 assertTrue(keyComparator.compare(kv1, newKey) < 0); 594 assertTrue((keyComparator.compare(kv2, newKey)) == 0); 595 596 // verify: same with row, different with qualifier 597 kv1 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, 5, Type.Put); 598 kv2 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualB, 5, Type.Put); 599 assertTrue(keyComparator.compare(kv1, kv2) < 0); 600 newKey = HFileWriterImpl.getMidpoint(keyComparator, kv1, kv2); 601 assertTrue(keyComparator.compare(kv1, newKey) < 0); 602 assertTrue((keyComparator.compare(kv2, newKey)) > 0); 603 assertTrue(Arrays.equals(CellUtil.cloneFamily(newKey), family)); 604 assertTrue(Arrays.equals(CellUtil.cloneQualifier(newKey), qualB)); 605 assertTrue(newKey.getTimestamp() == HConstants.LATEST_TIMESTAMP); 606 assertTrue(newKey.getTypeByte() == Type.Maximum.getCode()); 607 608 // verify metaKeyComparator's getShortMidpointKey output 609 final CellComparatorImpl metaKeyComparator = CellComparatorImpl.META_COMPARATOR; 610 kv1 = new KeyValue(Bytes.toBytes("ilovehbase123"), family, qualA, 5, Type.Put); 611 kv2 = new KeyValue(Bytes.toBytes("ilovehbase234"), family, qualA, 0, Type.Put); 612 newKey = HFileWriterImpl.getMidpoint(metaKeyComparator, kv1, kv2); 613 assertTrue(metaKeyComparator.compare(kv1, newKey) < 0); 614 assertTrue((metaKeyComparator.compare(kv2, newKey) == 0)); 615 616 // verify common fix scenario 617 kv1 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, ts, Type.Put); 618 kv2 = new KeyValue(Bytes.toBytes("ilovehbaseandhdfs"), family, qualA, ts, Type.Put); 619 assertTrue(keyComparator.compare(kv1, kv2) < 0); 620 newKey = HFileWriterImpl.getMidpoint(keyComparator, kv1, kv2); 621 assertTrue(keyComparator.compare(kv1, newKey) < 0); 622 assertTrue((keyComparator.compare(kv2, newKey)) > 0); 623 expectedArray = Bytes.toBytes("ilovehbasea"); 624 Bytes.equals(newKey.getRowArray(), newKey.getRowOffset(), newKey.getRowLength(), expectedArray, 625 0, expectedArray.length); 626 // verify only 1 offset scenario 627 kv1 = new KeyValue(Bytes.toBytes("100abcdefg"), family, qualA, ts, Type.Put); 628 kv2 = new KeyValue(Bytes.toBytes("101abcdefg"), family, qualA, ts, Type.Put); 629 assertTrue(keyComparator.compare(kv1, kv2) < 0); 630 newKey = HFileWriterImpl.getMidpoint(keyComparator, kv1, kv2); 631 assertTrue(keyComparator.compare(kv1, newKey) < 0); 632 assertTrue((keyComparator.compare(kv2, newKey)) > 0); 633 expectedArray = Bytes.toBytes("101"); 634 Bytes.equals(newKey.getRowArray(), newKey.getRowOffset(), newKey.getRowLength(), expectedArray, 635 0, expectedArray.length); 636 } 637 638 @Test 639 public void testDBEShipped() throws IOException { 640 for (DataBlockEncoding encoding : DataBlockEncoding.values()) { 641 DataBlockEncoder encoder = encoding.getEncoder(); 642 if (encoder == null) { 643 continue; 644 } 645 Path f = new Path(ROOT_DIR, testName.getMethodName() + "_" + encoding); 646 HFileContext context = new HFileContextBuilder() 647 .withIncludesTags(false) 648 .withDataBlockEncoding(encoding).build(); 649 HFileWriterImpl writer = (HFileWriterImpl) HFile.getWriterFactory(conf, cacheConf) 650 .withPath(fs, f).withFileContext(context).create(); 651 652 KeyValue kv = new KeyValue(Bytes.toBytes("testkey1"), Bytes.toBytes("family"), 653 Bytes.toBytes("qual"), Bytes.toBytes("testvalue")); 654 KeyValue kv2 = new KeyValue(Bytes.toBytes("testkey2"), Bytes.toBytes("family"), 655 Bytes.toBytes("qual"), Bytes.toBytes("testvalue")); 656 KeyValue kv3 = new KeyValue(Bytes.toBytes("testkey3"), Bytes.toBytes("family"), 657 Bytes.toBytes("qual"), Bytes.toBytes("testvalue")); 658 659 ByteBuffer buffer = ByteBuffer.wrap(kv.getBuffer()); 660 ByteBuffer buffer2 = ByteBuffer.wrap(kv2.getBuffer()); 661 ByteBuffer buffer3 = ByteBuffer.wrap(kv3.getBuffer()); 662 663 writer.append(new ByteBufferKeyValue(buffer, 0, buffer.remaining())); 664 writer.beforeShipped(); 665 666 // pollute first cell's backing ByteBuffer 667 ByteBufferUtils.copyFromBufferToBuffer(buffer3, buffer); 668 669 // write another cell, if DBE not Shipped, test will fail 670 writer.append(new ByteBufferKeyValue(buffer2, 0, buffer2.remaining())); 671 writer.close(); 672 } 673 } 674}