001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.io.hfile; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertFalse; 022import static org.junit.Assert.assertNull; 023import static org.junit.Assert.assertTrue; 024import static org.junit.Assert.fail; 025 026import java.io.DataInput; 027import java.io.DataOutput; 028import java.io.IOException; 029import java.nio.ByteBuffer; 030import java.util.Arrays; 031import java.util.Objects; 032import java.util.Random; 033import org.apache.hadoop.conf.Configuration; 034import org.apache.hadoop.fs.FSDataInputStream; 035import org.apache.hadoop.fs.FSDataOutputStream; 036import org.apache.hadoop.fs.FileStatus; 037import org.apache.hadoop.fs.FileSystem; 038import org.apache.hadoop.fs.Path; 039import org.apache.hadoop.hbase.ArrayBackedTag; 040import org.apache.hadoop.hbase.Cell; 041import org.apache.hadoop.hbase.CellComparatorImpl; 042import org.apache.hadoop.hbase.CellUtil; 043import org.apache.hadoop.hbase.HBaseClassTestRule; 044import org.apache.hadoop.hbase.HBaseCommonTestingUtility; 045import org.apache.hadoop.hbase.HBaseTestingUtility; 046import org.apache.hadoop.hbase.HConstants; 047import org.apache.hadoop.hbase.KeyValue; 048import org.apache.hadoop.hbase.KeyValue.Type; 049import org.apache.hadoop.hbase.KeyValueUtil; 050import org.apache.hadoop.hbase.PrivateCellUtil; 051import org.apache.hadoop.hbase.Tag; 052import org.apache.hadoop.hbase.io.compress.Compression; 053import org.apache.hadoop.hbase.io.hfile.HFile.Reader; 054import org.apache.hadoop.hbase.io.hfile.HFile.Writer; 055import org.apache.hadoop.hbase.nio.ByteBuff; 056import org.apache.hadoop.hbase.regionserver.StoreFileWriter; 057import org.apache.hadoop.hbase.testclassification.IOTests; 058import org.apache.hadoop.hbase.testclassification.SmallTests; 059import org.apache.hadoop.hbase.util.Bytes; 060import org.apache.hadoop.io.Writable; 061import org.junit.BeforeClass; 062import org.junit.ClassRule; 063import org.junit.Rule; 064import org.junit.Test; 065import org.junit.experimental.categories.Category; 066import org.junit.rules.TestName; 067import org.slf4j.Logger; 068import org.slf4j.LoggerFactory; 069 070/** 071 * test hfile features. 072 */ 073@Category({IOTests.class, SmallTests.class}) 074public class TestHFile { 075 076 @ClassRule 077 public static final HBaseClassTestRule CLASS_RULE = 078 HBaseClassTestRule.forClass(TestHFile.class); 079 080 @Rule public TestName testName = new TestName(); 081 082 private static final Logger LOG = LoggerFactory.getLogger(TestHFile.class); 083 private static final int NUM_VALID_KEY_TYPES = KeyValue.Type.values().length - 2; 084 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); 085 private static String ROOT_DIR = 086 TEST_UTIL.getDataTestDir("TestHFile").toString(); 087 private final int minBlockSize = 512; 088 private static String localFormatter = "%010d"; 089 private static CacheConfig cacheConf = null; 090 private static Configuration conf ; 091 private static FileSystem fs; 092 093 @BeforeClass 094 public static void setUp() throws Exception { 095 conf = TEST_UTIL.getConfiguration(); 096 fs = TEST_UTIL.getTestFileSystem(); 097 } 098 099 @Test 100 public void testReaderWithoutBlockCache() throws Exception { 101 Path path = writeStoreFile(); 102 try{ 103 readStoreFile(path); 104 } catch (Exception e) { 105 // fail test 106 assertTrue(false); 107 } 108 } 109 110 111 private void readStoreFile(Path storeFilePath) throws Exception { 112 // Open the file reader with block cache disabled. 113 HFile.Reader reader = HFile.createReader(fs, storeFilePath, conf); 114 long offset = 0; 115 while (offset < reader.getTrailer().getLoadOnOpenDataOffset()) { 116 HFileBlock block = reader.readBlock(offset, -1, false, true, false, true, null, null); 117 offset += block.getOnDiskSizeWithHeader(); 118 } 119 } 120 121 private Path writeStoreFile() throws IOException { 122 Path storeFileParentDir = new Path(TEST_UTIL.getDataTestDir(), "TestHFile"); 123 HFileContext meta = new HFileContextBuilder().withBlockSize(64 * 1024).build(); 124 StoreFileWriter sfw = 125 new StoreFileWriter.Builder(conf, fs).withOutputDir(storeFileParentDir) 126 .withComparator(CellComparatorImpl.COMPARATOR).withFileContext(meta).build(); 127 128 final int rowLen = 32; 129 Random RNG = new Random(); 130 for (int i = 0; i < 1000; ++i) { 131 byte[] k = RandomKeyValueUtil.randomOrderedKey(RNG, i); 132 byte[] v = RandomKeyValueUtil.randomValue(RNG); 133 int cfLen = RNG.nextInt(k.length - rowLen + 1); 134 KeyValue kv = 135 new KeyValue(k, 0, rowLen, k, rowLen, cfLen, k, rowLen + cfLen, 136 k.length - rowLen - cfLen, RNG.nextLong(), generateKeyType(RNG), v, 0, v.length); 137 sfw.append(kv); 138 } 139 140 sfw.close(); 141 return sfw.getPath(); 142 } 143 144 public static KeyValue.Type generateKeyType(Random rand) { 145 if (rand.nextBoolean()) { 146 // Let's make half of KVs puts. 147 return KeyValue.Type.Put; 148 } else { 149 KeyValue.Type keyType = KeyValue.Type.values()[1 + rand.nextInt(NUM_VALID_KEY_TYPES)]; 150 if (keyType == KeyValue.Type.Minimum || keyType == KeyValue.Type.Maximum) { 151 throw new RuntimeException("Generated an invalid key type: " + keyType + ". " 152 + "Probably the layout of KeyValue.Type has changed."); 153 } 154 return keyType; 155 } 156 } 157 158 /** 159 * Test empty HFile. 160 * Test all features work reasonably when hfile is empty of entries. 161 * @throws IOException 162 */ 163 @Test 164 public void testEmptyHFile() throws IOException { 165 if (cacheConf == null) cacheConf = new CacheConfig(conf); 166 Path f = new Path(ROOT_DIR, testName.getMethodName()); 167 HFileContext context = new HFileContextBuilder().withIncludesTags(false).build(); 168 Writer w = 169 HFile.getWriterFactory(conf, cacheConf).withPath(fs, f).withFileContext(context).create(); 170 w.close(); 171 Reader r = HFile.createReader(fs, f, cacheConf, true, conf); 172 r.loadFileInfo(); 173 assertFalse(r.getFirstKey().isPresent()); 174 assertFalse(r.getLastKey().isPresent()); 175 } 176 177 /** 178 * Create 0-length hfile and show that it fails 179 */ 180 @Test 181 public void testCorrupt0LengthHFile() throws IOException { 182 if (cacheConf == null) cacheConf = new CacheConfig(conf); 183 Path f = new Path(ROOT_DIR, testName.getMethodName()); 184 FSDataOutputStream fsos = fs.create(f); 185 fsos.close(); 186 187 try { 188 Reader r = HFile.createReader(fs, f, cacheConf, true, conf); 189 } catch (CorruptHFileException che) { 190 // Expected failure 191 return; 192 } 193 fail("Should have thrown exception"); 194 } 195 196 public static void truncateFile(FileSystem fs, Path src, Path dst) throws IOException { 197 FileStatus fst = fs.getFileStatus(src); 198 long len = fst.getLen(); 199 len = len / 2 ; 200 201 // create a truncated hfile 202 FSDataOutputStream fdos = fs.create(dst); 203 byte[] buf = new byte[(int)len]; 204 FSDataInputStream fdis = fs.open(src); 205 fdis.read(buf); 206 fdos.write(buf); 207 fdis.close(); 208 fdos.close(); 209 } 210 211 /** 212 * Create a truncated hfile and verify that exception thrown. 213 */ 214 @Test 215 public void testCorruptTruncatedHFile() throws IOException { 216 if (cacheConf == null) cacheConf = new CacheConfig(conf); 217 Path f = new Path(ROOT_DIR, testName.getMethodName()); 218 HFileContext context = new HFileContextBuilder().build(); 219 Writer w = HFile.getWriterFactory(conf, cacheConf).withPath(this.fs, f) 220 .withFileContext(context).create(); 221 writeSomeRecords(w, 0, 100, false); 222 w.close(); 223 224 Path trunc = new Path(f.getParent(), "trucated"); 225 truncateFile(fs, w.getPath(), trunc); 226 227 try { 228 Reader r = HFile.createReader(fs, trunc, cacheConf, true, conf); 229 } catch (CorruptHFileException che) { 230 // Expected failure 231 return; 232 } 233 fail("Should have thrown exception"); 234 } 235 236 // write some records into the hfile 237 // write them twice 238 private int writeSomeRecords(Writer writer, int start, int n, boolean useTags) 239 throws IOException { 240 String value = "value"; 241 KeyValue kv; 242 for (int i = start; i < (start + n); i++) { 243 String key = String.format(localFormatter, Integer.valueOf(i)); 244 if (useTags) { 245 Tag t = new ArrayBackedTag((byte) 1, "myTag1"); 246 Tag[] tags = new Tag[1]; 247 tags[0] = t; 248 kv = new KeyValue(Bytes.toBytes(key), Bytes.toBytes("family"), Bytes.toBytes("qual"), 249 HConstants.LATEST_TIMESTAMP, Bytes.toBytes(value + key), tags); 250 writer.append(kv); 251 } else { 252 kv = new KeyValue(Bytes.toBytes(key), Bytes.toBytes("family"), Bytes.toBytes("qual"), 253 Bytes.toBytes(value + key)); 254 writer.append(kv); 255 } 256 } 257 return (start + n); 258 } 259 260 private void readAllRecords(HFileScanner scanner) throws IOException { 261 readAndCheckbytes(scanner, 0, 100); 262 } 263 264 // read the records and check 265 private int readAndCheckbytes(HFileScanner scanner, int start, int n) 266 throws IOException { 267 String value = "value"; 268 int i = start; 269 for (; i < (start + n); i++) { 270 ByteBuffer key = ByteBuffer.wrap(((KeyValue)scanner.getKey()).getKey()); 271 ByteBuffer val = scanner.getValue(); 272 String keyStr = String.format(localFormatter, Integer.valueOf(i)); 273 String valStr = value + keyStr; 274 KeyValue kv = new KeyValue(Bytes.toBytes(keyStr), Bytes.toBytes("family"), 275 Bytes.toBytes("qual"), Bytes.toBytes(valStr)); 276 byte[] keyBytes = new KeyValue.KeyOnlyKeyValue(Bytes.toBytes(key), 0, 277 Bytes.toBytes(key).length).getKey(); 278 assertTrue("bytes for keys do not match " + keyStr + " " + 279 Bytes.toString(Bytes.toBytes(key)), 280 Arrays.equals(kv.getKey(), keyBytes)); 281 byte [] valBytes = Bytes.toBytes(val); 282 assertTrue("bytes for vals do not match " + valStr + " " + 283 Bytes.toString(valBytes), 284 Arrays.equals(Bytes.toBytes(valStr), valBytes)); 285 if (!scanner.next()) { 286 break; 287 } 288 } 289 assertEquals(i, start + n - 1); 290 return (start + n); 291 } 292 293 private byte[] getSomeKey(int rowId) { 294 KeyValue kv = new KeyValue(String.format(localFormatter, Integer.valueOf(rowId)).getBytes(), 295 Bytes.toBytes("family"), Bytes.toBytes("qual"), HConstants.LATEST_TIMESTAMP, Type.Put); 296 return kv.getKey(); 297 } 298 299 private void writeRecords(Writer writer, boolean useTags) throws IOException { 300 writeSomeRecords(writer, 0, 100, useTags); 301 writer.close(); 302 } 303 304 private FSDataOutputStream createFSOutput(Path name) throws IOException { 305 //if (fs.exists(name)) fs.delete(name, true); 306 FSDataOutputStream fout = fs.create(name); 307 return fout; 308 } 309 310 /** 311 * test none codecs 312 * @param useTags 313 */ 314 void basicWithSomeCodec(String codec, boolean useTags) throws IOException { 315 if (useTags) { 316 conf.setInt("hfile.format.version", 3); 317 } 318 if (cacheConf == null) cacheConf = new CacheConfig(conf); 319 Path ncHFile = new Path(ROOT_DIR, "basic.hfile." + codec.toString() + useTags); 320 FSDataOutputStream fout = createFSOutput(ncHFile); 321 HFileContext meta = new HFileContextBuilder() 322 .withBlockSize(minBlockSize) 323 .withCompression(HFileWriterImpl.compressionByName(codec)) 324 .build(); 325 Writer writer = HFile.getWriterFactory(conf, cacheConf) 326 .withOutputStream(fout) 327 .withFileContext(meta) 328 .withComparator(CellComparatorImpl.COMPARATOR) 329 .create(); 330 LOG.info(Objects.toString(writer)); 331 writeRecords(writer, useTags); 332 fout.close(); 333 FSDataInputStream fin = fs.open(ncHFile); 334 Reader reader = HFile.createReaderFromStream(ncHFile, fs.open(ncHFile), 335 fs.getFileStatus(ncHFile).getLen(), cacheConf, conf); 336 System.out.println(cacheConf.toString()); 337 // Load up the index. 338 reader.loadFileInfo(); 339 // Get a scanner that caches and that does not use pread. 340 HFileScanner scanner = reader.getScanner(true, false); 341 // Align scanner at start of the file. 342 scanner.seekTo(); 343 readAllRecords(scanner); 344 int seekTo = scanner.seekTo(KeyValueUtil.createKeyValueFromKey(getSomeKey(50))); 345 System.out.println(seekTo); 346 assertTrue("location lookup failed", 347 scanner.seekTo(KeyValueUtil.createKeyValueFromKey(getSomeKey(50))) == 0); 348 // read the key and see if it matches 349 ByteBuffer readKey = ByteBuffer.wrap(((KeyValue)scanner.getKey()).getKey()); 350 assertTrue("seeked key does not match", Arrays.equals(getSomeKey(50), 351 Bytes.toBytes(readKey))); 352 353 scanner.seekTo(KeyValueUtil.createKeyValueFromKey(getSomeKey(0))); 354 ByteBuffer val1 = scanner.getValue(); 355 scanner.seekTo(KeyValueUtil.createKeyValueFromKey(getSomeKey(0))); 356 ByteBuffer val2 = scanner.getValue(); 357 assertTrue(Arrays.equals(Bytes.toBytes(val1), Bytes.toBytes(val2))); 358 359 reader.close(); 360 fin.close(); 361 fs.delete(ncHFile, true); 362 } 363 364 @Test 365 public void testTFileFeatures() throws IOException { 366 testHFilefeaturesInternals(false); 367 testHFilefeaturesInternals(true); 368 } 369 370 protected void testHFilefeaturesInternals(boolean useTags) throws IOException { 371 basicWithSomeCodec("none", useTags); 372 basicWithSomeCodec("gz", useTags); 373 } 374 375 private void writeNumMetablocks(Writer writer, int n) { 376 for (int i = 0; i < n; i++) { 377 writer.appendMetaBlock("HFileMeta" + i, new Writable() { 378 private int val; 379 public Writable setVal(int val) { this.val = val; return this; } 380 381 @Override 382 public void write(DataOutput out) throws IOException { 383 out.write(("something to test" + val).getBytes()); 384 } 385 386 @Override 387 public void readFields(DataInput in) throws IOException { } 388 }.setVal(i)); 389 } 390 } 391 392 private void someTestingWithMetaBlock(Writer writer) { 393 writeNumMetablocks(writer, 10); 394 } 395 396 private void readNumMetablocks(Reader reader, int n) throws IOException { 397 for (int i = 0; i < n; i++) { 398 ByteBuff actual = reader.getMetaBlock("HFileMeta" + i, false).getBufferWithoutHeader(); 399 ByteBuffer expected = 400 ByteBuffer.wrap(("something to test" + i).getBytes()); 401 assertEquals( 402 "failed to match metadata", 403 Bytes.toStringBinary(expected), 404 Bytes.toStringBinary(actual.array(), actual.arrayOffset() + actual.position(), 405 actual.capacity())); 406 } 407 } 408 409 private void someReadingWithMetaBlock(Reader reader) throws IOException { 410 readNumMetablocks(reader, 10); 411 } 412 413 private void metablocks(final String compress) throws Exception { 414 if (cacheConf == null) cacheConf = new CacheConfig(conf); 415 Path mFile = new Path(ROOT_DIR, "meta.hfile"); 416 FSDataOutputStream fout = createFSOutput(mFile); 417 HFileContext meta = new HFileContextBuilder() 418 .withCompression(HFileWriterImpl.compressionByName(compress)) 419 .withBlockSize(minBlockSize).build(); 420 Writer writer = HFile.getWriterFactory(conf, cacheConf) 421 .withOutputStream(fout) 422 .withFileContext(meta) 423 .create(); 424 someTestingWithMetaBlock(writer); 425 writer.close(); 426 fout.close(); 427 FSDataInputStream fin = fs.open(mFile); 428 Reader reader = HFile.createReaderFromStream(mFile, fs.open(mFile), 429 this.fs.getFileStatus(mFile).getLen(), cacheConf, conf); 430 reader.loadFileInfo(); 431 // No data -- this should return false. 432 assertFalse(reader.getScanner(false, false).seekTo()); 433 someReadingWithMetaBlock(reader); 434 fs.delete(mFile, true); 435 reader.close(); 436 fin.close(); 437 } 438 439 // test meta blocks for hfiles 440 @Test 441 public void testMetaBlocks() throws Exception { 442 metablocks("none"); 443 metablocks("gz"); 444 } 445 446 @Test 447 public void testNullMetaBlocks() throws Exception { 448 if (cacheConf == null) cacheConf = new CacheConfig(conf); 449 for (Compression.Algorithm compressAlgo : 450 HBaseCommonTestingUtility.COMPRESSION_ALGORITHMS) { 451 Path mFile = new Path(ROOT_DIR, "nometa_" + compressAlgo + ".hfile"); 452 FSDataOutputStream fout = createFSOutput(mFile); 453 HFileContext meta = new HFileContextBuilder().withCompression(compressAlgo) 454 .withBlockSize(minBlockSize).build(); 455 Writer writer = HFile.getWriterFactory(conf, cacheConf) 456 .withOutputStream(fout) 457 .withFileContext(meta) 458 .create(); 459 KeyValue kv = new KeyValue("foo".getBytes(), "f1".getBytes(), null, "value".getBytes()); 460 writer.append(kv); 461 writer.close(); 462 fout.close(); 463 Reader reader = HFile.createReader(fs, mFile, cacheConf, true, conf); 464 reader.loadFileInfo(); 465 assertNull(reader.getMetaBlock("non-existant", false)); 466 } 467 } 468 469 /** 470 * Make sure the ordinals for our compression algorithms do not change on us. 471 */ 472 @Test 473 public void testCompressionOrdinance() { 474 assertTrue(Compression.Algorithm.LZO.ordinal() == 0); 475 assertTrue(Compression.Algorithm.GZ.ordinal() == 1); 476 assertTrue(Compression.Algorithm.NONE.ordinal() == 2); 477 assertTrue(Compression.Algorithm.SNAPPY.ordinal() == 3); 478 assertTrue(Compression.Algorithm.LZ4.ordinal() == 4); 479 } 480 481 @Test 482 public void testShortMidpointSameQual() { 483 Cell left = CellUtil.createCell(Bytes.toBytes("a"), 484 Bytes.toBytes("a"), 485 Bytes.toBytes("a"), 486 11, 487 KeyValue.Type.Maximum.getCode(), 488 HConstants.EMPTY_BYTE_ARRAY); 489 Cell right = CellUtil.createCell(Bytes.toBytes("a"), 490 Bytes.toBytes("a"), 491 Bytes.toBytes("a"), 492 9, 493 KeyValue.Type.Maximum.getCode(), 494 HConstants.EMPTY_BYTE_ARRAY); 495 Cell mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); 496 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) <= 0); 497 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) == 0); 498 } 499 500 @Test 501 public void testGetShortMidpoint() { 502 Cell left = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); 503 Cell right = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); 504 Cell mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); 505 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) <= 0); 506 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) <= 0); 507 508 left = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); 509 right = CellUtil.createCell(Bytes.toBytes("b"), Bytes.toBytes("a"), Bytes.toBytes("a")); 510 mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); 511 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); 512 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) <= 0); 513 514 left = CellUtil.createCell(Bytes.toBytes("g"), Bytes.toBytes("a"), Bytes.toBytes("a")); 515 right = CellUtil.createCell(Bytes.toBytes("i"), Bytes.toBytes("a"), Bytes.toBytes("a")); 516 mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); 517 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); 518 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) <= 0); 519 520 left = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); 521 right = CellUtil.createCell(Bytes.toBytes("bbbbbbb"), Bytes.toBytes("a"), Bytes.toBytes("a")); 522 mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); 523 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); 524 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) < 0); 525 assertEquals(1, mid.getRowLength()); 526 527 left = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); 528 right = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("a")); 529 mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); 530 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); 531 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) <= 0); 532 533 left = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); 534 right = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("aaaaaaaa"), Bytes.toBytes("b")); 535 mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); 536 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); 537 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) < 0); 538 assertEquals(2, mid.getFamilyLength()); 539 540 left = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); 541 right = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("aaaaaaaaa")); 542 mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); 543 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); 544 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) < 0); 545 assertEquals(2, mid.getQualifierLength()); 546 547 left = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); 548 right = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("b")); 549 mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); 550 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); 551 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) <= 0); 552 assertEquals(1, mid.getQualifierLength()); 553 554 // Assert that if meta comparator, it returns the right cell -- i.e. no 555 // optimization done. 556 left = CellUtil.createCell(Bytes.toBytes("g"), Bytes.toBytes("a"), Bytes.toBytes("a")); 557 right = CellUtil.createCell(Bytes.toBytes("i"), Bytes.toBytes("a"), Bytes.toBytes("a")); 558 mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.META_COMPARATOR, left, right); 559 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); 560 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) == 0); 561 562 /** 563 * See HBASE-7845 564 */ 565 byte[] rowA = Bytes.toBytes("rowA"); 566 byte[] rowB = Bytes.toBytes("rowB"); 567 568 byte[] family = Bytes.toBytes("family"); 569 byte[] qualA = Bytes.toBytes("qfA"); 570 byte[] qualB = Bytes.toBytes("qfB"); 571 final CellComparatorImpl keyComparator = CellComparatorImpl.COMPARATOR; 572 // verify that faked shorter rowkey could be generated 573 long ts = 5; 574 KeyValue kv1 = new KeyValue(Bytes.toBytes("the quick brown fox"), family, qualA, ts, Type.Put); 575 KeyValue kv2 = new KeyValue(Bytes.toBytes("the who test text"), family, qualA, ts, Type.Put); 576 Cell newKey = HFileWriterImpl.getMidpoint(keyComparator, kv1, kv2); 577 assertTrue(keyComparator.compare(kv1, newKey) < 0); 578 assertTrue((keyComparator.compare(kv2, newKey)) > 0); 579 byte[] expectedArray = Bytes.toBytes("the r"); 580 Bytes.equals(newKey.getRowArray(), newKey.getRowOffset(), newKey.getRowLength(), expectedArray, 581 0, expectedArray.length); 582 583 // verify: same with "row + family + qualifier", return rightKey directly 584 kv1 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, 5, Type.Put); 585 kv2 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, 0, Type.Put); 586 assertTrue(keyComparator.compare(kv1, kv2) < 0); 587 newKey = HFileWriterImpl.getMidpoint(keyComparator, kv1, kv2); 588 assertTrue(keyComparator.compare(kv1, newKey) < 0); 589 assertTrue((keyComparator.compare(kv2, newKey)) == 0); 590 kv1 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, -5, Type.Put); 591 kv2 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, -10, Type.Put); 592 assertTrue(keyComparator.compare(kv1, kv2) < 0); 593 newKey = HFileWriterImpl.getMidpoint(keyComparator, kv1, kv2); 594 assertTrue(keyComparator.compare(kv1, newKey) < 0); 595 assertTrue((keyComparator.compare(kv2, newKey)) == 0); 596 597 // verify: same with row, different with qualifier 598 kv1 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, 5, Type.Put); 599 kv2 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualB, 5, Type.Put); 600 assertTrue(keyComparator.compare(kv1, kv2) < 0); 601 newKey = HFileWriterImpl.getMidpoint(keyComparator, kv1, kv2); 602 assertTrue(keyComparator.compare(kv1, newKey) < 0); 603 assertTrue((keyComparator.compare(kv2, newKey)) > 0); 604 assertTrue(Arrays.equals(CellUtil.cloneFamily(newKey), family)); 605 assertTrue(Arrays.equals(CellUtil.cloneQualifier(newKey), qualB)); 606 assertTrue(newKey.getTimestamp() == HConstants.LATEST_TIMESTAMP); 607 assertTrue(newKey.getTypeByte() == Type.Maximum.getCode()); 608 609 // verify metaKeyComparator's getShortMidpointKey output 610 final CellComparatorImpl metaKeyComparator = CellComparatorImpl.META_COMPARATOR; 611 kv1 = new KeyValue(Bytes.toBytes("ilovehbase123"), family, qualA, 5, Type.Put); 612 kv2 = new KeyValue(Bytes.toBytes("ilovehbase234"), family, qualA, 0, Type.Put); 613 newKey = HFileWriterImpl.getMidpoint(metaKeyComparator, kv1, kv2); 614 assertTrue(metaKeyComparator.compare(kv1, newKey) < 0); 615 assertTrue((metaKeyComparator.compare(kv2, newKey) == 0)); 616 617 // verify common fix scenario 618 kv1 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, ts, Type.Put); 619 kv2 = new KeyValue(Bytes.toBytes("ilovehbaseandhdfs"), family, qualA, ts, Type.Put); 620 assertTrue(keyComparator.compare(kv1, kv2) < 0); 621 newKey = HFileWriterImpl.getMidpoint(keyComparator, kv1, kv2); 622 assertTrue(keyComparator.compare(kv1, newKey) < 0); 623 assertTrue((keyComparator.compare(kv2, newKey)) > 0); 624 expectedArray = Bytes.toBytes("ilovehbasea"); 625 Bytes.equals(newKey.getRowArray(), newKey.getRowOffset(), newKey.getRowLength(), expectedArray, 626 0, expectedArray.length); 627 // verify only 1 offset scenario 628 kv1 = new KeyValue(Bytes.toBytes("100abcdefg"), family, qualA, ts, Type.Put); 629 kv2 = new KeyValue(Bytes.toBytes("101abcdefg"), family, qualA, ts, Type.Put); 630 assertTrue(keyComparator.compare(kv1, kv2) < 0); 631 newKey = HFileWriterImpl.getMidpoint(keyComparator, kv1, kv2); 632 assertTrue(keyComparator.compare(kv1, newKey) < 0); 633 assertTrue((keyComparator.compare(kv2, newKey)) > 0); 634 expectedArray = Bytes.toBytes("101"); 635 Bytes.equals(newKey.getRowArray(), newKey.getRowOffset(), newKey.getRowLength(), expectedArray, 636 0, expectedArray.length); 637 } 638 639} 640