001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.io.hfile; 019 020import static org.apache.hadoop.hbase.HConstants.BUCKET_CACHE_IOENGINE_KEY; 021import static org.apache.hadoop.hbase.HConstants.BUCKET_CACHE_SIZE_KEY; 022import static org.apache.hadoop.hbase.io.ByteBuffAllocator.BUFFER_SIZE_KEY; 023import static org.apache.hadoop.hbase.io.ByteBuffAllocator.MAX_BUFFER_COUNT_KEY; 024import static org.apache.hadoop.hbase.io.ByteBuffAllocator.MIN_ALLOCATE_SIZE_KEY; 025import static org.apache.hadoop.hbase.io.hfile.BlockCacheFactory.BLOCKCACHE_POLICY_KEY; 026import static org.apache.hadoop.hbase.io.hfile.CacheConfig.EVICT_BLOCKS_ON_CLOSE_KEY; 027import static org.junit.Assert.assertEquals; 028import static org.junit.Assert.assertFalse; 029import static org.junit.Assert.assertNull; 030import static org.junit.Assert.assertTrue; 031import static org.junit.Assert.fail; 032 033import java.io.DataInput; 034import java.io.DataOutput; 035import java.io.IOException; 036import java.nio.ByteBuffer; 037import java.util.ArrayList; 038import java.util.Arrays; 039import java.util.List; 040import java.util.Objects; 041import java.util.Random; 042import java.util.concurrent.ThreadLocalRandom; 043import org.apache.hadoop.conf.Configuration; 044import org.apache.hadoop.fs.FSDataInputStream; 045import org.apache.hadoop.fs.FSDataOutputStream; 046import org.apache.hadoop.fs.FileStatus; 047import org.apache.hadoop.fs.FileSystem; 048import org.apache.hadoop.fs.Path; 049import org.apache.hadoop.hbase.ArrayBackedTag; 050import org.apache.hadoop.hbase.ByteBufferKeyValue; 051import org.apache.hadoop.hbase.Cell; 052import org.apache.hadoop.hbase.CellBuilderType; 053import org.apache.hadoop.hbase.CellComparatorImpl; 054import org.apache.hadoop.hbase.CellUtil; 055import org.apache.hadoop.hbase.ExtendedCell; 056import org.apache.hadoop.hbase.ExtendedCellBuilder; 057import org.apache.hadoop.hbase.ExtendedCellBuilderFactory; 058import org.apache.hadoop.hbase.HBaseClassTestRule; 059import org.apache.hadoop.hbase.HBaseCommonTestingUtil; 060import org.apache.hadoop.hbase.HBaseConfiguration; 061import org.apache.hadoop.hbase.HBaseTestingUtil; 062import org.apache.hadoop.hbase.HConstants; 063import org.apache.hadoop.hbase.KeyValue; 064import org.apache.hadoop.hbase.KeyValue.Type; 065import org.apache.hadoop.hbase.KeyValueUtil; 066import org.apache.hadoop.hbase.MetaCellComparator; 067import org.apache.hadoop.hbase.PrivateCellUtil; 068import org.apache.hadoop.hbase.Tag; 069import org.apache.hadoop.hbase.io.ByteBuffAllocator; 070import org.apache.hadoop.hbase.io.compress.Compression; 071import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder; 072import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; 073import org.apache.hadoop.hbase.io.encoding.IndexBlockEncoding; 074import org.apache.hadoop.hbase.io.hfile.HFile.Reader; 075import org.apache.hadoop.hbase.io.hfile.HFile.Writer; 076import org.apache.hadoop.hbase.io.hfile.ReaderContext.ReaderType; 077import org.apache.hadoop.hbase.monitoring.ThreadLocalServerSideScanMetrics; 078import org.apache.hadoop.hbase.nio.ByteBuff; 079import org.apache.hadoop.hbase.regionserver.StoreFileWriter; 080import org.apache.hadoop.hbase.testclassification.IOTests; 081import org.apache.hadoop.hbase.testclassification.SmallTests; 082import org.apache.hadoop.hbase.util.ByteBufferUtils; 083import org.apache.hadoop.hbase.util.Bytes; 084import org.apache.hadoop.io.Writable; 085import org.junit.Assert; 086import org.junit.BeforeClass; 087import org.junit.ClassRule; 088import org.junit.Rule; 089import org.junit.Test; 090import org.junit.experimental.categories.Category; 091import org.junit.rules.TestName; 092import org.mockito.Mockito; 093import org.slf4j.Logger; 094import org.slf4j.LoggerFactory; 095 096/** 097 * test hfile features. 098 */ 099@Category({ IOTests.class, SmallTests.class }) 100public class TestHFile { 101 102 @ClassRule 103 public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestHFile.class); 104 105 @Rule 106 public TestName testName = new TestName(); 107 108 private static final Logger LOG = LoggerFactory.getLogger(TestHFile.class); 109 private static final int NUM_VALID_KEY_TYPES = KeyValue.Type.values().length - 2; 110 private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil(); 111 private static String ROOT_DIR = TEST_UTIL.getDataTestDir("TestHFile").toString(); 112 private final int minBlockSize = 512; 113 private static String localFormatter = "%010d"; 114 private static CacheConfig cacheConf; 115 private static Configuration conf; 116 private static FileSystem fs; 117 118 @BeforeClass 119 public static void setUp() throws Exception { 120 conf = TEST_UTIL.getConfiguration(); 121 cacheConf = new CacheConfig(conf); 122 fs = TEST_UTIL.getTestFileSystem(); 123 } 124 125 public static Reader createReaderFromStream(ReaderContext context, CacheConfig cacheConf, 126 Configuration conf) throws IOException { 127 HFileInfo fileInfo = new HFileInfo(context, conf); 128 Reader preadReader = HFile.createReader(context, fileInfo, cacheConf, conf); 129 fileInfo.initMetaAndIndex(preadReader); 130 preadReader.close(); 131 context = new ReaderContextBuilder() 132 .withFileSystemAndPath(context.getFileSystem(), context.getFilePath()) 133 .withReaderType(ReaderType.STREAM).build(); 134 Reader streamReader = HFile.createReader(context, fileInfo, cacheConf, conf); 135 return streamReader; 136 } 137 138 private ByteBuffAllocator initAllocator(boolean reservoirEnabled, int bufSize, int bufCount, 139 int minAllocSize) { 140 Configuration that = HBaseConfiguration.create(conf); 141 that.setInt(BUFFER_SIZE_KEY, bufSize); 142 that.setInt(MAX_BUFFER_COUNT_KEY, bufCount); 143 // All ByteBuffers will be allocated from the buffers. 144 that.setInt(MIN_ALLOCATE_SIZE_KEY, minAllocSize); 145 return ByteBuffAllocator.create(that, reservoirEnabled); 146 } 147 148 private void fillByteBuffAllocator(ByteBuffAllocator alloc, int bufCount) { 149 // Fill the allocator with bufCount ByteBuffer 150 List<ByteBuff> buffs = new ArrayList<>(); 151 for (int i = 0; i < bufCount; i++) { 152 buffs.add(alloc.allocateOneBuffer()); 153 Assert.assertEquals(alloc.getFreeBufferCount(), 0); 154 } 155 buffs.forEach(ByteBuff::release); 156 Assert.assertEquals(alloc.getFreeBufferCount(), bufCount); 157 } 158 159 @Test 160 public void testReaderWithoutBlockCache() throws Exception { 161 int bufCount = 32; 162 // AllByteBuffers will be allocated from the buffers. 163 ByteBuffAllocator alloc = initAllocator(true, 64 * 1024, bufCount, 0); 164 fillByteBuffAllocator(alloc, bufCount); 165 // start write to store file. 166 Path path = writeStoreFile(); 167 readStoreFile(path, conf, alloc); 168 Assert.assertEquals(bufCount, alloc.getFreeBufferCount()); 169 alloc.clean(); 170 } 171 172 /** 173 * Test case for HBASE-22127 in LruBlockCache. 174 */ 175 @Test 176 public void testReaderWithLRUBlockCache() throws Exception { 177 int bufCount = 1024, blockSize = 64 * 1024; 178 ByteBuffAllocator alloc = initAllocator(true, bufCount, blockSize, 0); 179 fillByteBuffAllocator(alloc, bufCount); 180 Path storeFilePath = writeStoreFile(); 181 // Open the file reader with LRUBlockCache 182 BlockCache lru = new LruBlockCache(1024 * 1024 * 32, blockSize, true, conf); 183 CacheConfig cacheConfig = new CacheConfig(conf, null, lru, alloc); 184 HFile.Reader reader = HFile.createReader(fs, storeFilePath, cacheConfig, true, conf); 185 long offset = 0; 186 while (offset < reader.getTrailer().getLoadOnOpenDataOffset()) { 187 BlockCacheKey key = new BlockCacheKey(storeFilePath.getName(), offset); 188 HFileBlock block = reader.readBlock(offset, -1, true, true, false, true, null, null); 189 offset += block.getOnDiskSizeWithHeader(); 190 // Ensure the block is an heap one. 191 Cacheable cachedBlock = lru.getBlock(key, false, false, true); 192 Assert.assertNotNull(cachedBlock); 193 Assert.assertTrue(cachedBlock instanceof HFileBlock); 194 Assert.assertFalse(((HFileBlock) cachedBlock).isSharedMem()); 195 // Should never allocate off-heap block from allocator because ensure that it's LRU. 196 Assert.assertEquals(bufCount, alloc.getFreeBufferCount()); 197 block.release(); // return back the ByteBuffer back to allocator. 198 } 199 reader.close(); 200 Assert.assertEquals(bufCount, alloc.getFreeBufferCount()); 201 alloc.clean(); 202 lru.shutdown(); 203 } 204 205 private void assertBytesReadFromCache(boolean isScanMetricsEnabled) throws Exception { 206 assertBytesReadFromCache(isScanMetricsEnabled, DataBlockEncoding.NONE); 207 } 208 209 private void assertBytesReadFromCache(boolean isScanMetricsEnabled, DataBlockEncoding encoding) 210 throws Exception { 211 // Write a store file 212 Path storeFilePath = writeStoreFile(); 213 214 // Initialize the block cache and HFile reader 215 BlockCache lru = BlockCacheFactory.createBlockCache(conf); 216 Assert.assertTrue(lru instanceof LruBlockCache); 217 CacheConfig cacheConfig = new CacheConfig(conf, null, lru, ByteBuffAllocator.HEAP); 218 HFileReaderImpl reader = 219 (HFileReaderImpl) HFile.createReader(fs, storeFilePath, cacheConfig, true, conf); 220 221 // Read the first block in HFile from the block cache. 222 final int offset = 0; 223 BlockCacheKey cacheKey = new BlockCacheKey(storeFilePath.getName(), offset); 224 HFileBlock block = (HFileBlock) lru.getBlock(cacheKey, false, false, true); 225 Assert.assertNull(block); 226 227 // Assert that first block has not been cached in the block cache and no disk I/O happened to 228 // check that. 229 ThreadLocalServerSideScanMetrics.getBytesReadFromBlockCacheAndReset(); 230 ThreadLocalServerSideScanMetrics.getBytesReadFromFsAndReset(); 231 block = reader.getCachedBlock(cacheKey, false, false, true, BlockType.DATA, null); 232 Assert.assertEquals(0, ThreadLocalServerSideScanMetrics.getBytesReadFromBlockCacheAndReset()); 233 Assert.assertEquals(0, ThreadLocalServerSideScanMetrics.getBytesReadFromFsAndReset()); 234 235 // Read the first block from the HFile. 236 block = reader.readBlock(offset, -1, true, true, false, true, BlockType.DATA, null); 237 Assert.assertNotNull(block); 238 int bytesReadFromFs = block.getOnDiskSizeWithHeader(); 239 if (block.getNextBlockOnDiskSize() > 0) { 240 bytesReadFromFs += block.headerSize(); 241 } 242 block.release(); 243 // Assert that disk I/O happened to read the first block. 244 Assert.assertEquals(isScanMetricsEnabled ? bytesReadFromFs : 0, 245 ThreadLocalServerSideScanMetrics.getBytesReadFromFsAndReset()); 246 Assert.assertEquals(0, ThreadLocalServerSideScanMetrics.getBytesReadFromBlockCacheAndReset()); 247 248 // Read the first block again and assert that it has been cached in the block cache. 249 block = reader.getCachedBlock(cacheKey, false, false, true, BlockType.DATA, encoding); 250 long bytesReadFromCache = 0; 251 if (encoding == DataBlockEncoding.NONE) { 252 Assert.assertNotNull(block); 253 bytesReadFromCache = block.getOnDiskSizeWithHeader(); 254 if (block.getNextBlockOnDiskSize() > 0) { 255 bytesReadFromCache += block.headerSize(); 256 } 257 block.release(); 258 // Assert that bytes read from block cache account for same number of bytes that would have 259 // been read from FS if block cache wasn't there. 260 Assert.assertEquals(bytesReadFromFs, bytesReadFromCache); 261 } else { 262 Assert.assertNull(block); 263 } 264 Assert.assertEquals(isScanMetricsEnabled ? bytesReadFromCache : 0, 265 ThreadLocalServerSideScanMetrics.getBytesReadFromBlockCacheAndReset()); 266 Assert.assertEquals(0, ThreadLocalServerSideScanMetrics.getBytesReadFromFsAndReset()); 267 268 reader.close(); 269 } 270 271 @Test 272 public void testBytesReadFromCache() throws Exception { 273 ThreadLocalServerSideScanMetrics.setScanMetricsEnabled(true); 274 assertBytesReadFromCache(true); 275 } 276 277 @Test 278 public void testBytesReadFromCacheWithScanMetricsDisabled() throws Exception { 279 ThreadLocalServerSideScanMetrics.setScanMetricsEnabled(false); 280 assertBytesReadFromCache(false); 281 } 282 283 @Test 284 public void testBytesReadFromCacheWithInvalidDataEncoding() throws Exception { 285 ThreadLocalServerSideScanMetrics.setScanMetricsEnabled(true); 286 assertBytesReadFromCache(true, DataBlockEncoding.FAST_DIFF); 287 } 288 289 private BlockCache initCombinedBlockCache(final String l1CachePolicy) { 290 Configuration that = HBaseConfiguration.create(conf); 291 that.setFloat(BUCKET_CACHE_SIZE_KEY, 32); // 32MB for bucket cache. 292 that.set(BUCKET_CACHE_IOENGINE_KEY, "offheap"); 293 that.set(BLOCKCACHE_POLICY_KEY, l1CachePolicy); 294 BlockCache bc = BlockCacheFactory.createBlockCache(that); 295 Assert.assertNotNull(bc); 296 Assert.assertTrue(bc instanceof CombinedBlockCache); 297 return bc; 298 } 299 300 /** 301 * Test case for HBASE-22127 in CombinedBlockCache 302 */ 303 @Test 304 public void testReaderWithCombinedBlockCache() throws Exception { 305 int bufCount = 1024, blockSize = 64 * 1024; 306 ByteBuffAllocator alloc = initAllocator(true, bufCount, blockSize, 0); 307 fillByteBuffAllocator(alloc, bufCount); 308 Path storeFilePath = writeStoreFile(); 309 // Open the file reader with CombinedBlockCache 310 BlockCache combined = initCombinedBlockCache("LRU"); 311 conf.setBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, true); 312 CacheConfig cacheConfig = new CacheConfig(conf, null, combined, alloc); 313 HFile.Reader reader = HFile.createReader(fs, storeFilePath, cacheConfig, true, conf); 314 long offset = 0; 315 while (offset < reader.getTrailer().getLoadOnOpenDataOffset()) { 316 BlockCacheKey key = new BlockCacheKey(storeFilePath.getName(), offset); 317 HFileBlock block = reader.readBlock(offset, -1, true, true, false, true, null, null); 318 offset += block.getOnDiskSizeWithHeader(); 319 // Read the cached block. 320 Cacheable cachedBlock = combined.getBlock(key, false, false, true); 321 try { 322 Assert.assertNotNull(cachedBlock); 323 Assert.assertTrue(cachedBlock instanceof HFileBlock); 324 HFileBlock hfb = (HFileBlock) cachedBlock; 325 // Data block will be cached in BucketCache, so it should be an off-heap block. 326 if (hfb.getBlockType().isData()) { 327 Assert.assertTrue(hfb.isSharedMem()); 328 } else { 329 // Non-data block will be cached in LRUBlockCache, so it must be an on-heap block. 330 Assert.assertFalse(hfb.isSharedMem()); 331 } 332 } finally { 333 cachedBlock.release(); 334 } 335 block.release(); // return back the ByteBuffer back to allocator. 336 } 337 reader.close(); 338 combined.shutdown(); 339 Assert.assertEquals(bufCount, alloc.getFreeBufferCount()); 340 alloc.clean(); 341 } 342 343 /** 344 * Tests that we properly allocate from the off-heap or on-heap when LRUCache is configured. In 345 * this case, the determining factor is whether we end up caching the block or not. So the below 346 * test cases try different permutations of enabling/disabling via CacheConfig and via user 347 * request (cacheblocks), along with different expected block types. 348 */ 349 @Test 350 public void testReaderBlockAllocationWithLRUCache() throws IOException { 351 // false because caching is fully enabled 352 testReaderBlockAllocationWithLRUCache(true, true, null, false); 353 // false because we only look at cache config when expectedBlockType is non-null 354 testReaderBlockAllocationWithLRUCache(false, true, null, false); 355 // false because cacheBlock is true and even with cache config is disabled, we still cache 356 // important blocks like indexes 357 testReaderBlockAllocationWithLRUCache(false, true, BlockType.INTERMEDIATE_INDEX, false); 358 // true because since it's a DATA block, we honor the cache config 359 testReaderBlockAllocationWithLRUCache(false, true, BlockType.DATA, true); 360 // true for the following 2 because cacheBlock takes precedence over cache config 361 testReaderBlockAllocationWithLRUCache(true, false, null, true); 362 testReaderBlockAllocationWithLRUCache(true, false, BlockType.INTERMEDIATE_INDEX, false); 363 // false for the following 3 because both cache config and cacheBlock are false. 364 // per above, INDEX would supersede cache config, but not cacheBlock 365 testReaderBlockAllocationWithLRUCache(false, false, null, true); 366 testReaderBlockAllocationWithLRUCache(false, false, BlockType.INTERMEDIATE_INDEX, true); 367 testReaderBlockAllocationWithLRUCache(false, false, BlockType.DATA, true); 368 } 369 370 private void testReaderBlockAllocationWithLRUCache(boolean cacheConfigCacheBlockOnRead, 371 boolean cacheBlock, BlockType blockType, boolean expectSharedMem) throws IOException { 372 int bufCount = 1024, blockSize = 64 * 1024; 373 ByteBuffAllocator alloc = initAllocator(true, blockSize, bufCount, 0); 374 fillByteBuffAllocator(alloc, bufCount); 375 Path storeFilePath = writeStoreFile(); 376 Configuration myConf = new Configuration(conf); 377 378 myConf.setBoolean(CacheConfig.CACHE_DATA_ON_READ_KEY, cacheConfigCacheBlockOnRead); 379 // Open the file reader with LRUBlockCache 380 BlockCache lru = new LruBlockCache(1024 * 1024 * 32, blockSize, true, myConf); 381 CacheConfig cacheConfig = new CacheConfig(myConf, null, lru, alloc); 382 HFile.Reader reader = HFile.createReader(fs, storeFilePath, cacheConfig, true, myConf); 383 long offset = 0; 384 while (offset < reader.getTrailer().getLoadOnOpenDataOffset()) { 385 long read = readAtOffsetWithAllocationAsserts(alloc, reader, offset, cacheBlock, blockType, 386 expectSharedMem); 387 if (read < 0) { 388 break; 389 } 390 391 offset += read; 392 } 393 394 reader.close(); 395 Assert.assertEquals(bufCount, alloc.getFreeBufferCount()); 396 alloc.clean(); 397 lru.shutdown(); 398 } 399 400 /** 401 * Tests that we properly allocate from the off-heap or on-heap when CombinedCache is configured. 402 * In this case, we should always use off-heap unless the block is an INDEX (which always goes to 403 * L1 cache which is on-heap) 404 */ 405 @Test 406 public void testReaderBlockAllocationWithCombinedCache() throws IOException { 407 // true because caching is fully enabled and block type null 408 testReaderBlockAllocationWithCombinedCache(true, true, null, true); 409 // false because caching is fully enabled, index block type always goes to on-heap L1 410 testReaderBlockAllocationWithCombinedCache(true, true, BlockType.INTERMEDIATE_INDEX, false); 411 // true because cacheBlocks takes precedence over cache config which block type is null 412 testReaderBlockAllocationWithCombinedCache(false, true, null, true); 413 // false because caching is enabled and block type is index, which always goes to L1 414 testReaderBlockAllocationWithCombinedCache(false, true, BlockType.INTERMEDIATE_INDEX, false); 415 // true because since it's a DATA block, we honor the cache config 416 testReaderBlockAllocationWithCombinedCache(false, true, BlockType.DATA, true); 417 // true for the following 2 because cacheBlock takes precedence over cache config 418 // with caching disabled, we always go to off-heap 419 testReaderBlockAllocationWithCombinedCache(true, false, null, true); 420 testReaderBlockAllocationWithCombinedCache(true, false, BlockType.INTERMEDIATE_INDEX, false); 421 // true for the following 3, because with caching disabled we always go to off-heap 422 testReaderBlockAllocationWithCombinedCache(false, false, null, true); 423 testReaderBlockAllocationWithCombinedCache(false, false, BlockType.INTERMEDIATE_INDEX, true); 424 testReaderBlockAllocationWithCombinedCache(false, false, BlockType.DATA, true); 425 } 426 427 private void testReaderBlockAllocationWithCombinedCache(boolean cacheConfigCacheBlockOnRead, 428 boolean cacheBlock, BlockType blockType, boolean expectSharedMem) throws IOException { 429 int bufCount = 1024, blockSize = 64 * 1024; 430 ByteBuffAllocator alloc = initAllocator(true, blockSize, bufCount, 0); 431 fillByteBuffAllocator(alloc, bufCount); 432 Path storeFilePath = writeStoreFile(); 433 // Open the file reader with CombinedBlockCache 434 BlockCache combined = initCombinedBlockCache("LRU"); 435 Configuration myConf = new Configuration(conf); 436 437 myConf.setBoolean(CacheConfig.CACHE_DATA_ON_READ_KEY, cacheConfigCacheBlockOnRead); 438 myConf.setBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, true); 439 440 CacheConfig cacheConfig = new CacheConfig(myConf, null, combined, alloc); 441 HFile.Reader reader = HFile.createReader(fs, storeFilePath, cacheConfig, true, myConf); 442 long offset = 0; 443 while (offset < reader.getTrailer().getLoadOnOpenDataOffset()) { 444 long read = readAtOffsetWithAllocationAsserts(alloc, reader, offset, cacheBlock, blockType, 445 expectSharedMem); 446 if (read < 0) { 447 break; 448 } 449 450 offset += read; 451 } 452 453 reader.close(); 454 combined.shutdown(); 455 Assert.assertEquals(bufCount, alloc.getFreeBufferCount()); 456 alloc.clean(); 457 } 458 459 private long readAtOffsetWithAllocationAsserts(ByteBuffAllocator alloc, HFile.Reader reader, 460 long offset, boolean cacheBlock, BlockType blockType, boolean expectSharedMem) 461 throws IOException { 462 HFileBlock block; 463 try { 464 block = reader.readBlock(offset, -1, cacheBlock, true, false, true, blockType, null); 465 } catch (IOException e) { 466 if (e.getMessage().contains("Expected block type")) { 467 return -1; 468 } 469 throw e; 470 } 471 472 Assert.assertEquals(expectSharedMem, block.isSharedMem()); 473 474 if (expectSharedMem) { 475 Assert.assertTrue(alloc.getFreeBufferCount() < alloc.getTotalBufferCount()); 476 } else { 477 // Should never allocate off-heap block from allocator because ensure that it's LRU. 478 Assert.assertEquals(alloc.getTotalBufferCount(), alloc.getFreeBufferCount()); 479 } 480 481 try { 482 return block.getOnDiskSizeWithHeader(); 483 } finally { 484 block.release(); // return back the ByteBuffer back to allocator. 485 } 486 } 487 488 private void readStoreFile(Path storeFilePath, Configuration conf, ByteBuffAllocator alloc) 489 throws Exception { 490 // Open the file reader with block cache disabled. 491 CacheConfig cache = new CacheConfig(conf, null, null, alloc); 492 HFile.Reader reader = HFile.createReader(fs, storeFilePath, cache, true, conf); 493 long offset = 0; 494 while (offset < reader.getTrailer().getLoadOnOpenDataOffset()) { 495 HFileBlock block = reader.readBlock(offset, -1, false, true, false, true, null, null); 496 offset += block.getOnDiskSizeWithHeader(); 497 block.release(); // return back the ByteBuffer back to allocator. 498 } 499 reader.close(); 500 } 501 502 private Path writeStoreFile() throws IOException { 503 Path storeFileParentDir = new Path(TEST_UTIL.getDataTestDir(), "TestHFile"); 504 HFileContext meta = new HFileContextBuilder().withBlockSize(64 * 1024).build(); 505 StoreFileWriter sfw = new StoreFileWriter.Builder(conf, fs).withOutputDir(storeFileParentDir) 506 .withFileContext(meta).build(); 507 final int rowLen = 32; 508 Random rand = ThreadLocalRandom.current(); 509 for (int i = 0; i < 1000; ++i) { 510 byte[] k = RandomKeyValueUtil.randomOrderedKey(rand, i); 511 byte[] v = RandomKeyValueUtil.randomValue(rand); 512 int cfLen = rand.nextInt(k.length - rowLen + 1); 513 KeyValue kv = new KeyValue(k, 0, rowLen, k, rowLen, cfLen, k, rowLen + cfLen, 514 k.length - rowLen - cfLen, rand.nextLong(), generateKeyType(rand), v, 0, v.length); 515 sfw.append(kv); 516 } 517 518 sfw.close(); 519 return sfw.getPath(); 520 } 521 522 public static KeyValue.Type generateKeyType(Random rand) { 523 if (rand.nextBoolean()) { 524 // Let's make half of KVs puts. 525 return KeyValue.Type.Put; 526 } else { 527 KeyValue.Type keyType = KeyValue.Type.values()[1 + rand.nextInt(NUM_VALID_KEY_TYPES)]; 528 if (keyType == KeyValue.Type.Minimum || keyType == KeyValue.Type.Maximum) { 529 throw new RuntimeException("Generated an invalid key type: " + keyType + ". " 530 + "Probably the layout of KeyValue.Type has changed."); 531 } 532 return keyType; 533 } 534 } 535 536 /** 537 * Test empty HFile. Test all features work reasonably when hfile is empty of entries. 538 */ 539 @Test 540 public void testEmptyHFile() throws IOException { 541 Path f = new Path(ROOT_DIR, testName.getMethodName()); 542 HFileContext context = new HFileContextBuilder().withIncludesTags(false).build(); 543 Writer w = 544 HFile.getWriterFactory(conf, cacheConf).withPath(fs, f).withFileContext(context).create(); 545 w.close(); 546 Reader r = HFile.createReader(fs, f, cacheConf, true, conf); 547 assertFalse(r.getFirstKey().isPresent()); 548 assertFalse(r.getLastKey().isPresent()); 549 } 550 551 /** 552 * Create 0-length hfile and show that it fails 553 */ 554 @Test 555 public void testCorrupt0LengthHFile() throws IOException { 556 Path f = new Path(ROOT_DIR, testName.getMethodName()); 557 FSDataOutputStream fsos = fs.create(f); 558 fsos.close(); 559 560 try { 561 Reader r = HFile.createReader(fs, f, cacheConf, true, conf); 562 } catch (CorruptHFileException | IllegalArgumentException che) { 563 // Expected failure 564 return; 565 } 566 fail("Should have thrown exception"); 567 } 568 569 @Test 570 public void testCorruptOutOfOrderHFileWrite() throws IOException { 571 Path path = new Path(ROOT_DIR, testName.getMethodName()); 572 FSDataOutputStream mockedOutputStream = Mockito.mock(FSDataOutputStream.class); 573 String columnFamily = "MyColumnFamily"; 574 String tableName = "MyTableName"; 575 HFileContext fileContext = 576 new HFileContextBuilder().withHFileName(testName.getMethodName() + "HFile") 577 .withBlockSize(minBlockSize).withColumnFamily(Bytes.toBytes(columnFamily)) 578 .withTableName(Bytes.toBytes(tableName)).withHBaseCheckSum(false) 579 .withCompression(Compression.Algorithm.NONE).withCompressTags(false).build(); 580 HFileWriterImpl writer = 581 new HFileWriterImpl(conf, cacheConf, path, mockedOutputStream, fileContext); 582 ExtendedCellBuilder cellBuilder = 583 ExtendedCellBuilderFactory.create(CellBuilderType.SHALLOW_COPY); 584 byte[] row = Bytes.toBytes("foo"); 585 byte[] qualifier = Bytes.toBytes("qualifier"); 586 byte[] cf = Bytes.toBytes(columnFamily); 587 byte[] val = Bytes.toBytes("fooVal"); 588 long firstTS = 100L; 589 long secondTS = 101L; 590 ExtendedCell firstCell = cellBuilder.setRow(row).setValue(val).setTimestamp(firstTS) 591 .setQualifier(qualifier).setFamily(cf).setType(Cell.Type.Put).build(); 592 ExtendedCell secondCell = cellBuilder.setRow(row).setValue(val).setTimestamp(secondTS) 593 .setQualifier(qualifier).setFamily(cf).setType(Cell.Type.Put).build(); 594 // second Cell will sort "higher" than the first because later timestamps should come first 595 writer.append(firstCell); 596 try { 597 writer.append(secondCell); 598 } catch (IOException ie) { 599 String message = ie.getMessage(); 600 Assert.assertTrue(message.contains("not lexically larger")); 601 Assert.assertTrue(message.contains(tableName)); 602 Assert.assertTrue(message.contains(columnFamily)); 603 return; 604 } 605 Assert.fail("Exception wasn't thrown even though Cells were appended in the wrong order!"); 606 } 607 608 public static void truncateFile(FileSystem fs, Path src, Path dst) throws IOException { 609 FileStatus fst = fs.getFileStatus(src); 610 long len = fst.getLen(); 611 len = len / 2; 612 613 // create a truncated hfile 614 FSDataOutputStream fdos = fs.create(dst); 615 byte[] buf = new byte[(int) len]; 616 FSDataInputStream fdis = fs.open(src); 617 fdis.read(buf); 618 fdos.write(buf); 619 fdis.close(); 620 fdos.close(); 621 } 622 623 /** 624 * Create a truncated hfile and verify that exception thrown. 625 */ 626 @Test 627 public void testCorruptTruncatedHFile() throws IOException { 628 Path f = new Path(ROOT_DIR, testName.getMethodName()); 629 HFileContext context = new HFileContextBuilder().build(); 630 Writer w = HFile.getWriterFactory(conf, cacheConf).withPath(this.fs, f).withFileContext(context) 631 .create(); 632 writeSomeRecords(w, 0, 100, false); 633 w.close(); 634 635 Path trunc = new Path(f.getParent(), "trucated"); 636 truncateFile(fs, w.getPath(), trunc); 637 638 try { 639 HFile.createReader(fs, trunc, cacheConf, true, conf); 640 } catch (CorruptHFileException | IllegalArgumentException che) { 641 // Expected failure 642 return; 643 } 644 fail("Should have thrown exception"); 645 } 646 647 // write some records into the hfile 648 // write them twice 649 private int writeSomeRecords(Writer writer, int start, int n, boolean useTags) 650 throws IOException { 651 String value = "value"; 652 KeyValue kv; 653 for (int i = start; i < (start + n); i++) { 654 String key = String.format(localFormatter, Integer.valueOf(i)); 655 if (useTags) { 656 Tag t = new ArrayBackedTag((byte) 1, "myTag1"); 657 Tag[] tags = new Tag[1]; 658 tags[0] = t; 659 kv = new KeyValue(Bytes.toBytes(key), Bytes.toBytes("family"), Bytes.toBytes("qual"), 660 HConstants.LATEST_TIMESTAMP, Bytes.toBytes(value + key), tags); 661 writer.append(kv); 662 } else { 663 kv = new KeyValue(Bytes.toBytes(key), Bytes.toBytes("family"), Bytes.toBytes("qual"), 664 Bytes.toBytes(value + key)); 665 writer.append(kv); 666 } 667 } 668 return (start + n); 669 } 670 671 private void readAllRecords(HFileScanner scanner) throws IOException { 672 readAndCheckbytes(scanner, 0, 100); 673 } 674 675 // read the records and check 676 private int readAndCheckbytes(HFileScanner scanner, int start, int n) throws IOException { 677 String value = "value"; 678 int i = start; 679 for (; i < (start + n); i++) { 680 ByteBuffer key = ByteBuffer.wrap(((KeyValue) scanner.getKey()).getKey()); 681 ByteBuffer val = scanner.getValue(); 682 String keyStr = String.format(localFormatter, Integer.valueOf(i)); 683 String valStr = value + keyStr; 684 KeyValue kv = new KeyValue(Bytes.toBytes(keyStr), Bytes.toBytes("family"), 685 Bytes.toBytes("qual"), Bytes.toBytes(valStr)); 686 byte[] keyBytes = 687 new KeyValue.KeyOnlyKeyValue(Bytes.toBytes(key), 0, Bytes.toBytes(key).length).getKey(); 688 assertTrue("bytes for keys do not match " + keyStr + " " + Bytes.toString(Bytes.toBytes(key)), 689 Arrays.equals(kv.getKey(), keyBytes)); 690 byte[] valBytes = Bytes.toBytes(val); 691 assertTrue("bytes for vals do not match " + valStr + " " + Bytes.toString(valBytes), 692 Arrays.equals(Bytes.toBytes(valStr), valBytes)); 693 if (!scanner.next()) { 694 break; 695 } 696 } 697 assertEquals(i, start + n - 1); 698 return (start + n); 699 } 700 701 private byte[] getSomeKey(int rowId) { 702 KeyValue kv = new KeyValue(Bytes.toBytes(String.format(localFormatter, Integer.valueOf(rowId))), 703 Bytes.toBytes("family"), Bytes.toBytes("qual"), HConstants.LATEST_TIMESTAMP, Type.Put); 704 return kv.getKey(); 705 } 706 707 private void writeRecords(Writer writer, boolean useTags) throws IOException { 708 writeSomeRecords(writer, 0, 100, useTags); 709 writer.close(); 710 } 711 712 private FSDataOutputStream createFSOutput(Path name) throws IOException { 713 // if (fs.exists(name)) fs.delete(name, true); 714 FSDataOutputStream fout = fs.create(name); 715 return fout; 716 } 717 718 /** 719 * test none codecs 720 */ 721 void basicWithSomeCodec(String codec, boolean useTags) throws IOException { 722 if (useTags) { 723 conf.setInt("hfile.format.version", 3); 724 } 725 Path ncHFile = new Path(ROOT_DIR, "basic.hfile." + codec.toString() + useTags); 726 FSDataOutputStream fout = createFSOutput(ncHFile); 727 HFileContext meta = new HFileContextBuilder().withBlockSize(minBlockSize) 728 .withCompression(HFileWriterImpl.compressionByName(codec)).build(); 729 Writer writer = 730 HFile.getWriterFactory(conf, cacheConf).withOutputStream(fout).withFileContext(meta).create(); 731 LOG.info(Objects.toString(writer)); 732 writeRecords(writer, useTags); 733 fout.close(); 734 FSDataInputStream fin = fs.open(ncHFile); 735 ReaderContext context = new ReaderContextBuilder().withFileSystemAndPath(fs, ncHFile).build(); 736 Reader reader = createReaderFromStream(context, cacheConf, conf); 737 System.out.println(cacheConf.toString()); 738 // Load up the index. 739 // Get a scanner that caches and that does not use pread. 740 HFileScanner scanner = reader.getScanner(conf, true, false); 741 // Align scanner at start of the file. 742 scanner.seekTo(); 743 readAllRecords(scanner); 744 int seekTo = scanner.seekTo(KeyValueUtil.createKeyValueFromKey(getSomeKey(50))); 745 System.out.println(seekTo); 746 assertTrue("location lookup failed", 747 scanner.seekTo(KeyValueUtil.createKeyValueFromKey(getSomeKey(50))) == 0); 748 // read the key and see if it matches 749 ByteBuffer readKey = ByteBuffer.wrap(((KeyValue) scanner.getKey()).getKey()); 750 assertTrue("seeked key does not match", Arrays.equals(getSomeKey(50), Bytes.toBytes(readKey))); 751 752 scanner.seekTo(KeyValueUtil.createKeyValueFromKey(getSomeKey(0))); 753 ByteBuffer val1 = scanner.getValue(); 754 scanner.seekTo(KeyValueUtil.createKeyValueFromKey(getSomeKey(0))); 755 ByteBuffer val2 = scanner.getValue(); 756 assertTrue(Arrays.equals(Bytes.toBytes(val1), Bytes.toBytes(val2))); 757 758 reader.close(); 759 fin.close(); 760 fs.delete(ncHFile, true); 761 } 762 763 @Test 764 public void testTFileFeatures() throws IOException { 765 testHFilefeaturesInternals(false); 766 testHFilefeaturesInternals(true); 767 } 768 769 protected void testHFilefeaturesInternals(boolean useTags) throws IOException { 770 basicWithSomeCodec("none", useTags); 771 basicWithSomeCodec("gz", useTags); 772 } 773 774 private void writeNumMetablocks(Writer writer, int n) { 775 for (int i = 0; i < n; i++) { 776 writer.appendMetaBlock("HFileMeta" + i, new Writable() { 777 private int val; 778 779 public Writable setVal(int val) { 780 this.val = val; 781 return this; 782 } 783 784 @Override 785 public void write(DataOutput out) throws IOException { 786 out.write(Bytes.toBytes("something to test" + val)); 787 } 788 789 @Override 790 public void readFields(DataInput in) throws IOException { 791 } 792 }.setVal(i)); 793 } 794 } 795 796 private void someTestingWithMetaBlock(Writer writer) { 797 writeNumMetablocks(writer, 10); 798 } 799 800 private void readNumMetablocks(Reader reader, int n) throws IOException { 801 for (int i = 0; i < n; i++) { 802 ByteBuff actual = reader.getMetaBlock("HFileMeta" + i, false).getBufferWithoutHeader(); 803 ByteBuffer expected = ByteBuffer.wrap(Bytes.toBytes("something to test" + i)); 804 assertEquals("failed to match metadata", Bytes.toStringBinary(expected), Bytes.toStringBinary( 805 actual.array(), actual.arrayOffset() + actual.position(), actual.capacity())); 806 } 807 } 808 809 private void someReadingWithMetaBlock(Reader reader) throws IOException { 810 readNumMetablocks(reader, 10); 811 } 812 813 private void metablocks(final String compress) throws Exception { 814 Path mFile = new Path(ROOT_DIR, "meta.hfile"); 815 FSDataOutputStream fout = createFSOutput(mFile); 816 HFileContext meta = 817 new HFileContextBuilder().withCompression(HFileWriterImpl.compressionByName(compress)) 818 .withBlockSize(minBlockSize).build(); 819 Writer writer = 820 HFile.getWriterFactory(conf, cacheConf).withOutputStream(fout).withFileContext(meta).create(); 821 someTestingWithMetaBlock(writer); 822 writer.close(); 823 fout.close(); 824 ReaderContext context = new ReaderContextBuilder().withFileSystemAndPath(fs, mFile).build(); 825 Reader reader = createReaderFromStream(context, cacheConf, conf); 826 // No data -- this should return false. 827 assertFalse(reader.getScanner(conf, false, false).seekTo()); 828 someReadingWithMetaBlock(reader); 829 fs.delete(mFile, true); 830 reader.close(); 831 } 832 833 // test meta blocks for hfiles 834 @Test 835 public void testMetaBlocks() throws Exception { 836 metablocks("none"); 837 metablocks("gz"); 838 } 839 840 @Test 841 public void testNullMetaBlocks() throws Exception { 842 for (Compression.Algorithm compressAlgo : HBaseCommonTestingUtil.COMPRESSION_ALGORITHMS) { 843 Path mFile = new Path(ROOT_DIR, "nometa_" + compressAlgo + ".hfile"); 844 FSDataOutputStream fout = createFSOutput(mFile); 845 HFileContext meta = 846 new HFileContextBuilder().withCompression(compressAlgo).withBlockSize(minBlockSize).build(); 847 Writer writer = HFile.getWriterFactory(conf, cacheConf).withOutputStream(fout) 848 .withFileContext(meta).create(); 849 KeyValue kv = 850 new KeyValue(Bytes.toBytes("foo"), Bytes.toBytes("f1"), null, Bytes.toBytes("value")); 851 writer.append(kv); 852 writer.close(); 853 fout.close(); 854 Reader reader = HFile.createReader(fs, mFile, cacheConf, true, conf); 855 assertNull(reader.getMetaBlock("non-existant", false)); 856 } 857 } 858 859 /** 860 * Make sure the ordinals for our compression algorithms do not change on us. 861 */ 862 @Test 863 public void testCompressionOrdinance() { 864 assertTrue(Compression.Algorithm.LZO.ordinal() == 0); 865 assertTrue(Compression.Algorithm.GZ.ordinal() == 1); 866 assertTrue(Compression.Algorithm.NONE.ordinal() == 2); 867 assertTrue(Compression.Algorithm.SNAPPY.ordinal() == 3); 868 assertTrue(Compression.Algorithm.LZ4.ordinal() == 4); 869 } 870 871 @Test 872 public void testShortMidpointSameQual() { 873 ExtendedCell left = 874 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(Bytes.toBytes("a")) 875 .setFamily(Bytes.toBytes("a")).setQualifier(Bytes.toBytes("a")).setTimestamp(11) 876 .setType(Type.Maximum.getCode()).setValue(HConstants.EMPTY_BYTE_ARRAY).build(); 877 ExtendedCell right = 878 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(Bytes.toBytes("a")) 879 .setFamily(Bytes.toBytes("a")).setQualifier(Bytes.toBytes("a")).setTimestamp(9) 880 .setType(Type.Maximum.getCode()).setValue(HConstants.EMPTY_BYTE_ARRAY).build(); 881 ExtendedCell mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); 882 assertTrue( 883 PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) <= 0); 884 assertTrue( 885 PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) == 0); 886 } 887 888 private ExtendedCell getCell(byte[] row, byte[] family, byte[] qualifier) { 889 return ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(row) 890 .setFamily(family).setQualifier(qualifier).setTimestamp(HConstants.LATEST_TIMESTAMP) 891 .setType(KeyValue.Type.Maximum.getCode()).setValue(HConstants.EMPTY_BYTE_ARRAY).build(); 892 } 893 894 @Test 895 public void testGetShortMidpoint() { 896 ExtendedCell left = getCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); 897 ExtendedCell right = getCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); 898 ExtendedCell mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); 899 assertTrue( 900 PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) <= 0); 901 assertTrue( 902 PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) <= 0); 903 left = getCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); 904 right = getCell(Bytes.toBytes("b"), Bytes.toBytes("a"), Bytes.toBytes("a")); 905 mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); 906 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); 907 assertTrue( 908 PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) <= 0); 909 left = getCell(Bytes.toBytes("g"), Bytes.toBytes("a"), Bytes.toBytes("a")); 910 right = getCell(Bytes.toBytes("i"), Bytes.toBytes("a"), Bytes.toBytes("a")); 911 mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); 912 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); 913 assertTrue( 914 PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) <= 0); 915 left = getCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); 916 right = getCell(Bytes.toBytes("bbbbbbb"), Bytes.toBytes("a"), Bytes.toBytes("a")); 917 mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); 918 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); 919 assertTrue( 920 PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) < 0); 921 assertEquals(1, mid.getRowLength()); 922 left = getCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); 923 right = getCell(Bytes.toBytes("b"), Bytes.toBytes("a"), Bytes.toBytes("a")); 924 mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); 925 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); 926 assertTrue( 927 PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) <= 0); 928 left = getCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); 929 right = getCell(Bytes.toBytes("a"), Bytes.toBytes("aaaaaaaa"), Bytes.toBytes("b")); 930 mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); 931 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); 932 assertTrue( 933 PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) < 0); 934 assertEquals(2, mid.getFamilyLength()); 935 left = getCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); 936 right = getCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("aaaaaaaaa")); 937 mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); 938 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); 939 assertTrue( 940 PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) < 0); 941 assertEquals(2, mid.getQualifierLength()); 942 left = getCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); 943 right = getCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("b")); 944 mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); 945 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); 946 assertTrue( 947 PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) <= 0); 948 assertEquals(1, mid.getQualifierLength()); 949 950 // Verify boundary conditions 951 left = getCell(Bytes.toBytes("a"), Bytes.toBytes("a"), new byte[] { 0x00, (byte) 0xFE }); 952 right = getCell(Bytes.toBytes("a"), Bytes.toBytes("a"), new byte[] { 0x00, (byte) 0xFF }); 953 mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); 954 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); 955 assertTrue( 956 PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) == 0); 957 left = getCell(Bytes.toBytes("a"), Bytes.toBytes("a"), new byte[] { 0x00, 0x12 }); 958 right = getCell(Bytes.toBytes("a"), Bytes.toBytes("a"), new byte[] { 0x00, 0x12, 0x00 }); 959 mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right); 960 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); 961 assertTrue( 962 PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) == 0); 963 964 // Assert that if meta comparator, it returns the right cell -- i.e. no 965 // optimization done. 966 left = getCell(Bytes.toBytes("g"), Bytes.toBytes("a"), Bytes.toBytes("a")); 967 right = getCell(Bytes.toBytes("i"), Bytes.toBytes("a"), Bytes.toBytes("a")); 968 mid = HFileWriterImpl.getMidpoint(MetaCellComparator.META_COMPARATOR, left, right); 969 assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0); 970 assertTrue( 971 PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) == 0); 972 byte[] family = Bytes.toBytes("family"); 973 byte[] qualA = Bytes.toBytes("qfA"); 974 byte[] qualB = Bytes.toBytes("qfB"); 975 final CellComparatorImpl keyComparator = CellComparatorImpl.COMPARATOR; 976 // verify that faked shorter rowkey could be generated 977 long ts = 5; 978 KeyValue kv1 = new KeyValue(Bytes.toBytes("the quick brown fox"), family, qualA, ts, Type.Put); 979 KeyValue kv2 = new KeyValue(Bytes.toBytes("the who test text"), family, qualA, ts, Type.Put); 980 ExtendedCell newKey = HFileWriterImpl.getMidpoint(keyComparator, kv1, kv2); 981 assertTrue(keyComparator.compare(kv1, newKey) < 0); 982 assertTrue((keyComparator.compare(kv2, newKey)) > 0); 983 byte[] expectedArray = Bytes.toBytes("the r"); 984 Bytes.equals(newKey.getRowArray(), newKey.getRowOffset(), newKey.getRowLength(), expectedArray, 985 0, expectedArray.length); 986 987 // verify: same with "row + family + qualifier", return rightKey directly 988 kv1 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, 5, Type.Put); 989 kv2 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, 0, Type.Put); 990 assertTrue(keyComparator.compare(kv1, kv2) < 0); 991 newKey = HFileWriterImpl.getMidpoint(keyComparator, kv1, kv2); 992 assertTrue(keyComparator.compare(kv1, newKey) < 0); 993 assertTrue((keyComparator.compare(kv2, newKey)) == 0); 994 kv1 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, -5, Type.Put); 995 kv2 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, -10, Type.Put); 996 assertTrue(keyComparator.compare(kv1, kv2) < 0); 997 newKey = HFileWriterImpl.getMidpoint(keyComparator, kv1, kv2); 998 assertTrue(keyComparator.compare(kv1, newKey) < 0); 999 assertTrue((keyComparator.compare(kv2, newKey)) == 0); 1000 1001 // verify: same with row, different with qualifier 1002 kv1 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, 5, Type.Put); 1003 kv2 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualB, 5, Type.Put); 1004 assertTrue(keyComparator.compare(kv1, kv2) < 0); 1005 newKey = HFileWriterImpl.getMidpoint(keyComparator, kv1, kv2); 1006 assertTrue(keyComparator.compare(kv1, newKey) < 0); 1007 assertTrue((keyComparator.compare(kv2, newKey)) > 0); 1008 assertTrue(Arrays.equals(CellUtil.cloneFamily(newKey), family)); 1009 assertTrue(Arrays.equals(CellUtil.cloneQualifier(newKey), qualB)); 1010 assertTrue(newKey.getTimestamp() == HConstants.LATEST_TIMESTAMP); 1011 assertTrue(newKey.getTypeByte() == Type.Maximum.getCode()); 1012 1013 // verify metaKeyComparator's getShortMidpointKey output 1014 final CellComparatorImpl metaKeyComparator = MetaCellComparator.META_COMPARATOR; 1015 kv1 = new KeyValue(Bytes.toBytes("ilovehbase123"), family, qualA, 5, Type.Put); 1016 kv2 = new KeyValue(Bytes.toBytes("ilovehbase234"), family, qualA, 0, Type.Put); 1017 newKey = HFileWriterImpl.getMidpoint(metaKeyComparator, kv1, kv2); 1018 assertTrue(metaKeyComparator.compare(kv1, newKey) < 0); 1019 assertTrue((metaKeyComparator.compare(kv2, newKey) == 0)); 1020 1021 // verify common fix scenario 1022 kv1 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, ts, Type.Put); 1023 kv2 = new KeyValue(Bytes.toBytes("ilovehbaseandhdfs"), family, qualA, ts, Type.Put); 1024 assertTrue(keyComparator.compare(kv1, kv2) < 0); 1025 newKey = HFileWriterImpl.getMidpoint(keyComparator, kv1, kv2); 1026 assertTrue(keyComparator.compare(kv1, newKey) < 0); 1027 assertTrue((keyComparator.compare(kv2, newKey)) > 0); 1028 expectedArray = Bytes.toBytes("ilovehbasea"); 1029 Bytes.equals(newKey.getRowArray(), newKey.getRowOffset(), newKey.getRowLength(), expectedArray, 1030 0, expectedArray.length); 1031 // verify only 1 offset scenario 1032 kv1 = new KeyValue(Bytes.toBytes("100abcdefg"), family, qualA, ts, Type.Put); 1033 kv2 = new KeyValue(Bytes.toBytes("101abcdefg"), family, qualA, ts, Type.Put); 1034 assertTrue(keyComparator.compare(kv1, kv2) < 0); 1035 newKey = HFileWriterImpl.getMidpoint(keyComparator, kv1, kv2); 1036 assertTrue(keyComparator.compare(kv1, newKey) < 0); 1037 assertTrue((keyComparator.compare(kv2, newKey)) > 0); 1038 expectedArray = Bytes.toBytes("101"); 1039 Bytes.equals(newKey.getRowArray(), newKey.getRowOffset(), newKey.getRowLength(), expectedArray, 1040 0, expectedArray.length); 1041 } 1042 1043 @Test 1044 public void testDBEShipped() throws IOException { 1045 for (DataBlockEncoding encoding : DataBlockEncoding.values()) { 1046 DataBlockEncoder encoder = encoding.getEncoder(); 1047 if (encoder == null) { 1048 continue; 1049 } 1050 Path f = new Path(ROOT_DIR, testName.getMethodName() + "_" + encoding); 1051 HFileContext context = 1052 new HFileContextBuilder().withIncludesTags(false).withDataBlockEncoding(encoding).build(); 1053 HFileWriterImpl writer = (HFileWriterImpl) HFile.getWriterFactory(conf, cacheConf) 1054 .withPath(fs, f).withFileContext(context).create(); 1055 1056 KeyValue kv = new KeyValue(Bytes.toBytes("testkey1"), Bytes.toBytes("family"), 1057 Bytes.toBytes("qual"), Bytes.toBytes("testvalue")); 1058 KeyValue kv2 = new KeyValue(Bytes.toBytes("testkey2"), Bytes.toBytes("family"), 1059 Bytes.toBytes("qual"), Bytes.toBytes("testvalue")); 1060 KeyValue kv3 = new KeyValue(Bytes.toBytes("testkey3"), Bytes.toBytes("family"), 1061 Bytes.toBytes("qual"), Bytes.toBytes("testvalue")); 1062 1063 ByteBuffer buffer = ByteBuffer.wrap(kv.getBuffer()); 1064 ByteBuffer buffer2 = ByteBuffer.wrap(kv2.getBuffer()); 1065 ByteBuffer buffer3 = ByteBuffer.wrap(kv3.getBuffer()); 1066 1067 writer.append(new ByteBufferKeyValue(buffer, 0, buffer.remaining())); 1068 writer.beforeShipped(); 1069 1070 // pollute first cell's backing ByteBuffer 1071 ByteBufferUtils.copyFromBufferToBuffer(buffer3, buffer); 1072 1073 // write another cell, if DBE not Shipped, test will fail 1074 writer.append(new ByteBufferKeyValue(buffer2, 0, buffer2.remaining())); 1075 writer.close(); 1076 } 1077 } 1078 1079 /** 1080 * Test case for CombinedBlockCache with TinyLfu as L1 cache 1081 */ 1082 @Test 1083 public void testReaderWithTinyLfuCombinedBlockCache() throws Exception { 1084 testReaderCombinedCache("TinyLfu"); 1085 } 1086 1087 /** 1088 * Test case for CombinedBlockCache with AdaptiveLRU as L1 cache 1089 */ 1090 @Test 1091 public void testReaderWithAdaptiveLruCombinedBlockCache() throws Exception { 1092 testReaderCombinedCache("AdaptiveLRU"); 1093 } 1094 1095 /** 1096 * Test case for CombinedBlockCache with AdaptiveLRU as L1 cache 1097 */ 1098 @Test 1099 public void testReaderWithLruCombinedBlockCache() throws Exception { 1100 testReaderCombinedCache("LRU"); 1101 } 1102 1103 private void testReaderCombinedCache(final String l1CachePolicy) throws Exception { 1104 int bufCount = 1024; 1105 int blockSize = 64 * 1024; 1106 ByteBuffAllocator alloc = initAllocator(true, bufCount, blockSize, 0); 1107 fillByteBuffAllocator(alloc, bufCount); 1108 Path storeFilePath = writeStoreFile(); 1109 // Open the file reader with CombinedBlockCache 1110 BlockCache combined = initCombinedBlockCache(l1CachePolicy); 1111 conf.setBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, true); 1112 CacheConfig cacheConfig = new CacheConfig(conf, null, combined, alloc); 1113 HFile.Reader reader = HFile.createReader(fs, storeFilePath, cacheConfig, true, conf); 1114 long offset = 0; 1115 Cacheable cachedBlock = null; 1116 while (offset < reader.getTrailer().getLoadOnOpenDataOffset()) { 1117 BlockCacheKey key = new BlockCacheKey(storeFilePath.getName(), offset); 1118 HFileBlock block = reader.readBlock(offset, -1, true, true, false, true, null, null); 1119 offset += block.getOnDiskSizeWithHeader(); 1120 // Read the cached block. 1121 cachedBlock = combined.getBlock(key, false, false, true); 1122 try { 1123 Assert.assertNotNull(cachedBlock); 1124 Assert.assertTrue(cachedBlock instanceof HFileBlock); 1125 HFileBlock hfb = (HFileBlock) cachedBlock; 1126 // Data block will be cached in BucketCache, so it should be an off-heap block. 1127 if (hfb.getBlockType().isData()) { 1128 Assert.assertTrue(hfb.isSharedMem()); 1129 } else if (!l1CachePolicy.equals("TinyLfu")) { 1130 Assert.assertFalse(hfb.isSharedMem()); 1131 } 1132 } finally { 1133 cachedBlock.release(); 1134 } 1135 block.release(); // return back the ByteBuffer back to allocator. 1136 } 1137 reader.close(); 1138 combined.shutdown(); 1139 if (cachedBlock != null) { 1140 Assert.assertEquals(0, cachedBlock.refCnt()); 1141 } 1142 Assert.assertEquals(bufCount, alloc.getFreeBufferCount()); 1143 alloc.clean(); 1144 } 1145 1146 @Test 1147 public void testHFileContextBuilderWithIndexEncoding() throws IOException { 1148 HFileContext context = 1149 new HFileContextBuilder().withIndexBlockEncoding(IndexBlockEncoding.PREFIX_TREE).build(); 1150 HFileContext newContext = new HFileContextBuilder(context).build(); 1151 assertTrue(newContext.getIndexBlockEncoding() == IndexBlockEncoding.PREFIX_TREE); 1152 } 1153}