001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.io.hfile;
019
020import static org.apache.hadoop.hbase.HConstants.BUCKET_CACHE_IOENGINE_KEY;
021import static org.apache.hadoop.hbase.HConstants.BUCKET_CACHE_SIZE_KEY;
022import static org.apache.hadoop.hbase.io.ByteBuffAllocator.BUFFER_SIZE_KEY;
023import static org.apache.hadoop.hbase.io.ByteBuffAllocator.MAX_BUFFER_COUNT_KEY;
024import static org.apache.hadoop.hbase.io.ByteBuffAllocator.MIN_ALLOCATE_SIZE_KEY;
025import static org.apache.hadoop.hbase.io.hfile.BlockCacheFactory.BLOCKCACHE_POLICY_KEY;
026import static org.apache.hadoop.hbase.io.hfile.CacheConfig.EVICT_BLOCKS_ON_CLOSE_KEY;
027import static org.junit.Assert.assertEquals;
028import static org.junit.Assert.assertFalse;
029import static org.junit.Assert.assertNull;
030import static org.junit.Assert.assertTrue;
031import static org.junit.Assert.fail;
032
033import java.io.DataInput;
034import java.io.DataOutput;
035import java.io.IOException;
036import java.nio.ByteBuffer;
037import java.util.ArrayList;
038import java.util.Arrays;
039import java.util.List;
040import java.util.Objects;
041import java.util.Random;
042import java.util.concurrent.ThreadLocalRandom;
043import org.apache.hadoop.conf.Configuration;
044import org.apache.hadoop.fs.FSDataInputStream;
045import org.apache.hadoop.fs.FSDataOutputStream;
046import org.apache.hadoop.fs.FileStatus;
047import org.apache.hadoop.fs.FileSystem;
048import org.apache.hadoop.fs.Path;
049import org.apache.hadoop.hbase.ArrayBackedTag;
050import org.apache.hadoop.hbase.ByteBufferKeyValue;
051import org.apache.hadoop.hbase.Cell;
052import org.apache.hadoop.hbase.CellBuilderType;
053import org.apache.hadoop.hbase.CellComparatorImpl;
054import org.apache.hadoop.hbase.CellUtil;
055import org.apache.hadoop.hbase.ExtendedCell;
056import org.apache.hadoop.hbase.ExtendedCellBuilder;
057import org.apache.hadoop.hbase.ExtendedCellBuilderFactory;
058import org.apache.hadoop.hbase.HBaseClassTestRule;
059import org.apache.hadoop.hbase.HBaseCommonTestingUtil;
060import org.apache.hadoop.hbase.HBaseConfiguration;
061import org.apache.hadoop.hbase.HBaseTestingUtil;
062import org.apache.hadoop.hbase.HConstants;
063import org.apache.hadoop.hbase.KeyValue;
064import org.apache.hadoop.hbase.KeyValue.Type;
065import org.apache.hadoop.hbase.KeyValueUtil;
066import org.apache.hadoop.hbase.MetaCellComparator;
067import org.apache.hadoop.hbase.PrivateCellUtil;
068import org.apache.hadoop.hbase.Tag;
069import org.apache.hadoop.hbase.io.ByteBuffAllocator;
070import org.apache.hadoop.hbase.io.compress.Compression;
071import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder;
072import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
073import org.apache.hadoop.hbase.io.encoding.IndexBlockEncoding;
074import org.apache.hadoop.hbase.io.hfile.HFile.Reader;
075import org.apache.hadoop.hbase.io.hfile.HFile.Writer;
076import org.apache.hadoop.hbase.io.hfile.ReaderContext.ReaderType;
077import org.apache.hadoop.hbase.monitoring.ThreadLocalServerSideScanMetrics;
078import org.apache.hadoop.hbase.nio.ByteBuff;
079import org.apache.hadoop.hbase.regionserver.StoreFileWriter;
080import org.apache.hadoop.hbase.testclassification.IOTests;
081import org.apache.hadoop.hbase.testclassification.SmallTests;
082import org.apache.hadoop.hbase.util.ByteBufferUtils;
083import org.apache.hadoop.hbase.util.Bytes;
084import org.apache.hadoop.io.Writable;
085import org.junit.Assert;
086import org.junit.BeforeClass;
087import org.junit.ClassRule;
088import org.junit.Rule;
089import org.junit.Test;
090import org.junit.experimental.categories.Category;
091import org.junit.rules.TestName;
092import org.mockito.Mockito;
093import org.slf4j.Logger;
094import org.slf4j.LoggerFactory;
095
096/**
097 * test hfile features.
098 */
099@Category({ IOTests.class, SmallTests.class })
100public class TestHFile {
101
102  @ClassRule
103  public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestHFile.class);
104
105  @Rule
106  public TestName testName = new TestName();
107
108  private static final Logger LOG = LoggerFactory.getLogger(TestHFile.class);
109  private static final int NUM_VALID_KEY_TYPES = KeyValue.Type.values().length - 2;
110  private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
111  private static String ROOT_DIR = TEST_UTIL.getDataTestDir("TestHFile").toString();
112  private final int minBlockSize = 512;
113  private static String localFormatter = "%010d";
114  private static CacheConfig cacheConf;
115  private static Configuration conf;
116  private static FileSystem fs;
117
118  @BeforeClass
119  public static void setUp() throws Exception {
120    conf = TEST_UTIL.getConfiguration();
121    cacheConf = new CacheConfig(conf);
122    fs = TEST_UTIL.getTestFileSystem();
123  }
124
125  public static Reader createReaderFromStream(ReaderContext context, CacheConfig cacheConf,
126    Configuration conf) throws IOException {
127    HFileInfo fileInfo = new HFileInfo(context, conf);
128    Reader preadReader = HFile.createReader(context, fileInfo, cacheConf, conf);
129    fileInfo.initMetaAndIndex(preadReader);
130    preadReader.close();
131    context = new ReaderContextBuilder()
132      .withFileSystemAndPath(context.getFileSystem(), context.getFilePath())
133      .withReaderType(ReaderType.STREAM).build();
134    Reader streamReader = HFile.createReader(context, fileInfo, cacheConf, conf);
135    return streamReader;
136  }
137
138  private ByteBuffAllocator initAllocator(boolean reservoirEnabled, int bufSize, int bufCount,
139    int minAllocSize) {
140    Configuration that = HBaseConfiguration.create(conf);
141    that.setInt(BUFFER_SIZE_KEY, bufSize);
142    that.setInt(MAX_BUFFER_COUNT_KEY, bufCount);
143    // All ByteBuffers will be allocated from the buffers.
144    that.setInt(MIN_ALLOCATE_SIZE_KEY, minAllocSize);
145    return ByteBuffAllocator.create(that, reservoirEnabled);
146  }
147
148  private void fillByteBuffAllocator(ByteBuffAllocator alloc, int bufCount) {
149    // Fill the allocator with bufCount ByteBuffer
150    List<ByteBuff> buffs = new ArrayList<>();
151    for (int i = 0; i < bufCount; i++) {
152      buffs.add(alloc.allocateOneBuffer());
153      Assert.assertEquals(alloc.getFreeBufferCount(), 0);
154    }
155    buffs.forEach(ByteBuff::release);
156    Assert.assertEquals(alloc.getFreeBufferCount(), bufCount);
157  }
158
159  @Test
160  public void testReaderWithoutBlockCache() throws Exception {
161    int bufCount = 32;
162    // AllByteBuffers will be allocated from the buffers.
163    ByteBuffAllocator alloc = initAllocator(true, 64 * 1024, bufCount, 0);
164    fillByteBuffAllocator(alloc, bufCount);
165    // start write to store file.
166    Path path = writeStoreFile();
167    readStoreFile(path, conf, alloc);
168    Assert.assertEquals(bufCount, alloc.getFreeBufferCount());
169    alloc.clean();
170  }
171
172  /**
173   * Test case for HBASE-22127 in LruBlockCache.
174   */
175  @Test
176  public void testReaderWithLRUBlockCache() throws Exception {
177    int bufCount = 1024, blockSize = 64 * 1024;
178    ByteBuffAllocator alloc = initAllocator(true, bufCount, blockSize, 0);
179    fillByteBuffAllocator(alloc, bufCount);
180    Path storeFilePath = writeStoreFile();
181    // Open the file reader with LRUBlockCache
182    BlockCache lru = new LruBlockCache(1024 * 1024 * 32, blockSize, true, conf);
183    CacheConfig cacheConfig = new CacheConfig(conf, null, lru, alloc);
184    HFile.Reader reader = HFile.createReader(fs, storeFilePath, cacheConfig, true, conf);
185    long offset = 0;
186    while (offset < reader.getTrailer().getLoadOnOpenDataOffset()) {
187      BlockCacheKey key = new BlockCacheKey(storeFilePath.getName(), offset);
188      HFileBlock block = reader.readBlock(offset, -1, true, true, false, true, null, null);
189      offset += block.getOnDiskSizeWithHeader();
190      // Ensure the block is an heap one.
191      Cacheable cachedBlock = lru.getBlock(key, false, false, true);
192      Assert.assertNotNull(cachedBlock);
193      Assert.assertTrue(cachedBlock instanceof HFileBlock);
194      Assert.assertFalse(((HFileBlock) cachedBlock).isSharedMem());
195      // Should never allocate off-heap block from allocator because ensure that it's LRU.
196      Assert.assertEquals(bufCount, alloc.getFreeBufferCount());
197      block.release(); // return back the ByteBuffer back to allocator.
198    }
199    reader.close();
200    Assert.assertEquals(bufCount, alloc.getFreeBufferCount());
201    alloc.clean();
202    lru.shutdown();
203  }
204
205  private void assertBytesReadFromCache(boolean isScanMetricsEnabled) throws Exception {
206    assertBytesReadFromCache(isScanMetricsEnabled, DataBlockEncoding.NONE);
207  }
208
209  private void assertBytesReadFromCache(boolean isScanMetricsEnabled, DataBlockEncoding encoding)
210    throws Exception {
211    // Write a store file
212    Path storeFilePath = writeStoreFile();
213
214    // Initialize the block cache and HFile reader
215    BlockCache lru = BlockCacheFactory.createBlockCache(conf);
216    Assert.assertTrue(lru instanceof LruBlockCache);
217    CacheConfig cacheConfig = new CacheConfig(conf, null, lru, ByteBuffAllocator.HEAP);
218    HFileReaderImpl reader =
219      (HFileReaderImpl) HFile.createReader(fs, storeFilePath, cacheConfig, true, conf);
220
221    // Read the first block in HFile from the block cache.
222    final int offset = 0;
223    BlockCacheKey cacheKey = new BlockCacheKey(storeFilePath.getName(), offset);
224    HFileBlock block = (HFileBlock) lru.getBlock(cacheKey, false, false, true);
225    Assert.assertNull(block);
226
227    // Assert that first block has not been cached in the block cache and no disk I/O happened to
228    // check that.
229    ThreadLocalServerSideScanMetrics.getBytesReadFromBlockCacheAndReset();
230    ThreadLocalServerSideScanMetrics.getBytesReadFromFsAndReset();
231    block = reader.getCachedBlock(cacheKey, false, false, true, BlockType.DATA, null);
232    Assert.assertEquals(0, ThreadLocalServerSideScanMetrics.getBytesReadFromBlockCacheAndReset());
233    Assert.assertEquals(0, ThreadLocalServerSideScanMetrics.getBytesReadFromFsAndReset());
234
235    // Read the first block from the HFile.
236    block = reader.readBlock(offset, -1, true, true, false, true, BlockType.DATA, null);
237    Assert.assertNotNull(block);
238    int bytesReadFromFs = block.getOnDiskSizeWithHeader();
239    if (block.getNextBlockOnDiskSize() > 0) {
240      bytesReadFromFs += block.headerSize();
241    }
242    block.release();
243    // Assert that disk I/O happened to read the first block.
244    Assert.assertEquals(isScanMetricsEnabled ? bytesReadFromFs : 0,
245      ThreadLocalServerSideScanMetrics.getBytesReadFromFsAndReset());
246    Assert.assertEquals(0, ThreadLocalServerSideScanMetrics.getBytesReadFromBlockCacheAndReset());
247
248    // Read the first block again and assert that it has been cached in the block cache.
249    block = reader.getCachedBlock(cacheKey, false, false, true, BlockType.DATA, encoding);
250    long bytesReadFromCache = 0;
251    if (encoding == DataBlockEncoding.NONE) {
252      Assert.assertNotNull(block);
253      bytesReadFromCache = block.getOnDiskSizeWithHeader();
254      if (block.getNextBlockOnDiskSize() > 0) {
255        bytesReadFromCache += block.headerSize();
256      }
257      block.release();
258      // Assert that bytes read from block cache account for same number of bytes that would have
259      // been read from FS if block cache wasn't there.
260      Assert.assertEquals(bytesReadFromFs, bytesReadFromCache);
261    } else {
262      Assert.assertNull(block);
263    }
264    Assert.assertEquals(isScanMetricsEnabled ? bytesReadFromCache : 0,
265      ThreadLocalServerSideScanMetrics.getBytesReadFromBlockCacheAndReset());
266    Assert.assertEquals(0, ThreadLocalServerSideScanMetrics.getBytesReadFromFsAndReset());
267
268    reader.close();
269  }
270
271  @Test
272  public void testBytesReadFromCache() throws Exception {
273    ThreadLocalServerSideScanMetrics.setScanMetricsEnabled(true);
274    assertBytesReadFromCache(true);
275  }
276
277  @Test
278  public void testBytesReadFromCacheWithScanMetricsDisabled() throws Exception {
279    ThreadLocalServerSideScanMetrics.setScanMetricsEnabled(false);
280    assertBytesReadFromCache(false);
281  }
282
283  @Test
284  public void testBytesReadFromCacheWithInvalidDataEncoding() throws Exception {
285    ThreadLocalServerSideScanMetrics.setScanMetricsEnabled(true);
286    assertBytesReadFromCache(true, DataBlockEncoding.FAST_DIFF);
287  }
288
289  private BlockCache initCombinedBlockCache(final String l1CachePolicy) {
290    Configuration that = HBaseConfiguration.create(conf);
291    that.setFloat(BUCKET_CACHE_SIZE_KEY, 32); // 32MB for bucket cache.
292    that.set(BUCKET_CACHE_IOENGINE_KEY, "offheap");
293    that.set(BLOCKCACHE_POLICY_KEY, l1CachePolicy);
294    BlockCache bc = BlockCacheFactory.createBlockCache(that);
295    Assert.assertNotNull(bc);
296    Assert.assertTrue(bc instanceof CombinedBlockCache);
297    return bc;
298  }
299
300  /**
301   * Test case for HBASE-22127 in CombinedBlockCache
302   */
303  @Test
304  public void testReaderWithCombinedBlockCache() throws Exception {
305    int bufCount = 1024, blockSize = 64 * 1024;
306    ByteBuffAllocator alloc = initAllocator(true, bufCount, blockSize, 0);
307    fillByteBuffAllocator(alloc, bufCount);
308    Path storeFilePath = writeStoreFile();
309    // Open the file reader with CombinedBlockCache
310    BlockCache combined = initCombinedBlockCache("LRU");
311    conf.setBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, true);
312    CacheConfig cacheConfig = new CacheConfig(conf, null, combined, alloc);
313    HFile.Reader reader = HFile.createReader(fs, storeFilePath, cacheConfig, true, conf);
314    long offset = 0;
315    while (offset < reader.getTrailer().getLoadOnOpenDataOffset()) {
316      BlockCacheKey key = new BlockCacheKey(storeFilePath.getName(), offset);
317      HFileBlock block = reader.readBlock(offset, -1, true, true, false, true, null, null);
318      offset += block.getOnDiskSizeWithHeader();
319      // Read the cached block.
320      Cacheable cachedBlock = combined.getBlock(key, false, false, true);
321      try {
322        Assert.assertNotNull(cachedBlock);
323        Assert.assertTrue(cachedBlock instanceof HFileBlock);
324        HFileBlock hfb = (HFileBlock) cachedBlock;
325        // Data block will be cached in BucketCache, so it should be an off-heap block.
326        if (hfb.getBlockType().isData()) {
327          Assert.assertTrue(hfb.isSharedMem());
328        } else {
329          // Non-data block will be cached in LRUBlockCache, so it must be an on-heap block.
330          Assert.assertFalse(hfb.isSharedMem());
331        }
332      } finally {
333        cachedBlock.release();
334      }
335      block.release(); // return back the ByteBuffer back to allocator.
336    }
337    reader.close();
338    combined.shutdown();
339    Assert.assertEquals(bufCount, alloc.getFreeBufferCount());
340    alloc.clean();
341  }
342
343  /**
344   * Tests that we properly allocate from the off-heap or on-heap when LRUCache is configured. In
345   * this case, the determining factor is whether we end up caching the block or not. So the below
346   * test cases try different permutations of enabling/disabling via CacheConfig and via user
347   * request (cacheblocks), along with different expected block types.
348   */
349  @Test
350  public void testReaderBlockAllocationWithLRUCache() throws IOException {
351    // false because caching is fully enabled
352    testReaderBlockAllocationWithLRUCache(true, true, null, false);
353    // false because we only look at cache config when expectedBlockType is non-null
354    testReaderBlockAllocationWithLRUCache(false, true, null, false);
355    // false because cacheBlock is true and even with cache config is disabled, we still cache
356    // important blocks like indexes
357    testReaderBlockAllocationWithLRUCache(false, true, BlockType.INTERMEDIATE_INDEX, false);
358    // true because since it's a DATA block, we honor the cache config
359    testReaderBlockAllocationWithLRUCache(false, true, BlockType.DATA, true);
360    // true for the following 2 because cacheBlock takes precedence over cache config
361    testReaderBlockAllocationWithLRUCache(true, false, null, true);
362    testReaderBlockAllocationWithLRUCache(true, false, BlockType.INTERMEDIATE_INDEX, false);
363    // false for the following 3 because both cache config and cacheBlock are false.
364    // per above, INDEX would supersede cache config, but not cacheBlock
365    testReaderBlockAllocationWithLRUCache(false, false, null, true);
366    testReaderBlockAllocationWithLRUCache(false, false, BlockType.INTERMEDIATE_INDEX, true);
367    testReaderBlockAllocationWithLRUCache(false, false, BlockType.DATA, true);
368  }
369
370  private void testReaderBlockAllocationWithLRUCache(boolean cacheConfigCacheBlockOnRead,
371    boolean cacheBlock, BlockType blockType, boolean expectSharedMem) throws IOException {
372    int bufCount = 1024, blockSize = 64 * 1024;
373    ByteBuffAllocator alloc = initAllocator(true, blockSize, bufCount, 0);
374    fillByteBuffAllocator(alloc, bufCount);
375    Path storeFilePath = writeStoreFile();
376    Configuration myConf = new Configuration(conf);
377
378    myConf.setBoolean(CacheConfig.CACHE_DATA_ON_READ_KEY, cacheConfigCacheBlockOnRead);
379    // Open the file reader with LRUBlockCache
380    BlockCache lru = new LruBlockCache(1024 * 1024 * 32, blockSize, true, myConf);
381    CacheConfig cacheConfig = new CacheConfig(myConf, null, lru, alloc);
382    HFile.Reader reader = HFile.createReader(fs, storeFilePath, cacheConfig, true, myConf);
383    long offset = 0;
384    while (offset < reader.getTrailer().getLoadOnOpenDataOffset()) {
385      long read = readAtOffsetWithAllocationAsserts(alloc, reader, offset, cacheBlock, blockType,
386        expectSharedMem);
387      if (read < 0) {
388        break;
389      }
390
391      offset += read;
392    }
393
394    reader.close();
395    Assert.assertEquals(bufCount, alloc.getFreeBufferCount());
396    alloc.clean();
397    lru.shutdown();
398  }
399
400  /**
401   * Tests that we properly allocate from the off-heap or on-heap when CombinedCache is configured.
402   * In this case, we should always use off-heap unless the block is an INDEX (which always goes to
403   * L1 cache which is on-heap)
404   */
405  @Test
406  public void testReaderBlockAllocationWithCombinedCache() throws IOException {
407    // true because caching is fully enabled and block type null
408    testReaderBlockAllocationWithCombinedCache(true, true, null, true);
409    // false because caching is fully enabled, index block type always goes to on-heap L1
410    testReaderBlockAllocationWithCombinedCache(true, true, BlockType.INTERMEDIATE_INDEX, false);
411    // true because cacheBlocks takes precedence over cache config which block type is null
412    testReaderBlockAllocationWithCombinedCache(false, true, null, true);
413    // false because caching is enabled and block type is index, which always goes to L1
414    testReaderBlockAllocationWithCombinedCache(false, true, BlockType.INTERMEDIATE_INDEX, false);
415    // true because since it's a DATA block, we honor the cache config
416    testReaderBlockAllocationWithCombinedCache(false, true, BlockType.DATA, true);
417    // true for the following 2 because cacheBlock takes precedence over cache config
418    // with caching disabled, we always go to off-heap
419    testReaderBlockAllocationWithCombinedCache(true, false, null, true);
420    testReaderBlockAllocationWithCombinedCache(true, false, BlockType.INTERMEDIATE_INDEX, false);
421    // true for the following 3, because with caching disabled we always go to off-heap
422    testReaderBlockAllocationWithCombinedCache(false, false, null, true);
423    testReaderBlockAllocationWithCombinedCache(false, false, BlockType.INTERMEDIATE_INDEX, true);
424    testReaderBlockAllocationWithCombinedCache(false, false, BlockType.DATA, true);
425  }
426
427  private void testReaderBlockAllocationWithCombinedCache(boolean cacheConfigCacheBlockOnRead,
428    boolean cacheBlock, BlockType blockType, boolean expectSharedMem) throws IOException {
429    int bufCount = 1024, blockSize = 64 * 1024;
430    ByteBuffAllocator alloc = initAllocator(true, blockSize, bufCount, 0);
431    fillByteBuffAllocator(alloc, bufCount);
432    Path storeFilePath = writeStoreFile();
433    // Open the file reader with CombinedBlockCache
434    BlockCache combined = initCombinedBlockCache("LRU");
435    Configuration myConf = new Configuration(conf);
436
437    myConf.setBoolean(CacheConfig.CACHE_DATA_ON_READ_KEY, cacheConfigCacheBlockOnRead);
438    myConf.setBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, true);
439
440    CacheConfig cacheConfig = new CacheConfig(myConf, null, combined, alloc);
441    HFile.Reader reader = HFile.createReader(fs, storeFilePath, cacheConfig, true, myConf);
442    long offset = 0;
443    while (offset < reader.getTrailer().getLoadOnOpenDataOffset()) {
444      long read = readAtOffsetWithAllocationAsserts(alloc, reader, offset, cacheBlock, blockType,
445        expectSharedMem);
446      if (read < 0) {
447        break;
448      }
449
450      offset += read;
451    }
452
453    reader.close();
454    combined.shutdown();
455    Assert.assertEquals(bufCount, alloc.getFreeBufferCount());
456    alloc.clean();
457  }
458
459  private long readAtOffsetWithAllocationAsserts(ByteBuffAllocator alloc, HFile.Reader reader,
460    long offset, boolean cacheBlock, BlockType blockType, boolean expectSharedMem)
461    throws IOException {
462    HFileBlock block;
463    try {
464      block = reader.readBlock(offset, -1, cacheBlock, true, false, true, blockType, null);
465    } catch (IOException e) {
466      if (e.getMessage().contains("Expected block type")) {
467        return -1;
468      }
469      throw e;
470    }
471
472    Assert.assertEquals(expectSharedMem, block.isSharedMem());
473
474    if (expectSharedMem) {
475      Assert.assertTrue(alloc.getFreeBufferCount() < alloc.getTotalBufferCount());
476    } else {
477      // Should never allocate off-heap block from allocator because ensure that it's LRU.
478      Assert.assertEquals(alloc.getTotalBufferCount(), alloc.getFreeBufferCount());
479    }
480
481    try {
482      return block.getOnDiskSizeWithHeader();
483    } finally {
484      block.release(); // return back the ByteBuffer back to allocator.
485    }
486  }
487
488  private void readStoreFile(Path storeFilePath, Configuration conf, ByteBuffAllocator alloc)
489    throws Exception {
490    // Open the file reader with block cache disabled.
491    CacheConfig cache = new CacheConfig(conf, null, null, alloc);
492    HFile.Reader reader = HFile.createReader(fs, storeFilePath, cache, true, conf);
493    long offset = 0;
494    while (offset < reader.getTrailer().getLoadOnOpenDataOffset()) {
495      HFileBlock block = reader.readBlock(offset, -1, false, true, false, true, null, null);
496      offset += block.getOnDiskSizeWithHeader();
497      block.release(); // return back the ByteBuffer back to allocator.
498    }
499    reader.close();
500  }
501
502  private Path writeStoreFile() throws IOException {
503    Path storeFileParentDir = new Path(TEST_UTIL.getDataTestDir(), "TestHFile");
504    HFileContext meta = new HFileContextBuilder().withBlockSize(64 * 1024).build();
505    StoreFileWriter sfw = new StoreFileWriter.Builder(conf, fs).withOutputDir(storeFileParentDir)
506      .withFileContext(meta).build();
507    final int rowLen = 32;
508    Random rand = ThreadLocalRandom.current();
509    for (int i = 0; i < 1000; ++i) {
510      byte[] k = RandomKeyValueUtil.randomOrderedKey(rand, i);
511      byte[] v = RandomKeyValueUtil.randomValue(rand);
512      int cfLen = rand.nextInt(k.length - rowLen + 1);
513      KeyValue kv = new KeyValue(k, 0, rowLen, k, rowLen, cfLen, k, rowLen + cfLen,
514        k.length - rowLen - cfLen, rand.nextLong(), generateKeyType(rand), v, 0, v.length);
515      sfw.append(kv);
516    }
517
518    sfw.close();
519    return sfw.getPath();
520  }
521
522  public static KeyValue.Type generateKeyType(Random rand) {
523    if (rand.nextBoolean()) {
524      // Let's make half of KVs puts.
525      return KeyValue.Type.Put;
526    } else {
527      KeyValue.Type keyType = KeyValue.Type.values()[1 + rand.nextInt(NUM_VALID_KEY_TYPES)];
528      if (keyType == KeyValue.Type.Minimum || keyType == KeyValue.Type.Maximum) {
529        throw new RuntimeException("Generated an invalid key type: " + keyType + ". "
530          + "Probably the layout of KeyValue.Type has changed.");
531      }
532      return keyType;
533    }
534  }
535
536  /**
537   * Test empty HFile. Test all features work reasonably when hfile is empty of entries.
538   */
539  @Test
540  public void testEmptyHFile() throws IOException {
541    Path f = new Path(ROOT_DIR, testName.getMethodName());
542    HFileContext context = new HFileContextBuilder().withIncludesTags(false).build();
543    Writer w =
544      HFile.getWriterFactory(conf, cacheConf).withPath(fs, f).withFileContext(context).create();
545    w.close();
546    Reader r = HFile.createReader(fs, f, cacheConf, true, conf);
547    assertFalse(r.getFirstKey().isPresent());
548    assertFalse(r.getLastKey().isPresent());
549  }
550
551  /**
552   * Create 0-length hfile and show that it fails
553   */
554  @Test
555  public void testCorrupt0LengthHFile() throws IOException {
556    Path f = new Path(ROOT_DIR, testName.getMethodName());
557    FSDataOutputStream fsos = fs.create(f);
558    fsos.close();
559
560    try {
561      Reader r = HFile.createReader(fs, f, cacheConf, true, conf);
562    } catch (CorruptHFileException | IllegalArgumentException che) {
563      // Expected failure
564      return;
565    }
566    fail("Should have thrown exception");
567  }
568
569  @Test
570  public void testCorruptOutOfOrderHFileWrite() throws IOException {
571    Path path = new Path(ROOT_DIR, testName.getMethodName());
572    FSDataOutputStream mockedOutputStream = Mockito.mock(FSDataOutputStream.class);
573    String columnFamily = "MyColumnFamily";
574    String tableName = "MyTableName";
575    HFileContext fileContext =
576      new HFileContextBuilder().withHFileName(testName.getMethodName() + "HFile")
577        .withBlockSize(minBlockSize).withColumnFamily(Bytes.toBytes(columnFamily))
578        .withTableName(Bytes.toBytes(tableName)).withHBaseCheckSum(false)
579        .withCompression(Compression.Algorithm.NONE).withCompressTags(false).build();
580    HFileWriterImpl writer =
581      new HFileWriterImpl(conf, cacheConf, path, mockedOutputStream, fileContext);
582    ExtendedCellBuilder cellBuilder =
583      ExtendedCellBuilderFactory.create(CellBuilderType.SHALLOW_COPY);
584    byte[] row = Bytes.toBytes("foo");
585    byte[] qualifier = Bytes.toBytes("qualifier");
586    byte[] cf = Bytes.toBytes(columnFamily);
587    byte[] val = Bytes.toBytes("fooVal");
588    long firstTS = 100L;
589    long secondTS = 101L;
590    ExtendedCell firstCell = cellBuilder.setRow(row).setValue(val).setTimestamp(firstTS)
591      .setQualifier(qualifier).setFamily(cf).setType(Cell.Type.Put).build();
592    ExtendedCell secondCell = cellBuilder.setRow(row).setValue(val).setTimestamp(secondTS)
593      .setQualifier(qualifier).setFamily(cf).setType(Cell.Type.Put).build();
594    // second Cell will sort "higher" than the first because later timestamps should come first
595    writer.append(firstCell);
596    try {
597      writer.append(secondCell);
598    } catch (IOException ie) {
599      String message = ie.getMessage();
600      Assert.assertTrue(message.contains("not lexically larger"));
601      Assert.assertTrue(message.contains(tableName));
602      Assert.assertTrue(message.contains(columnFamily));
603      return;
604    }
605    Assert.fail("Exception wasn't thrown even though Cells were appended in the wrong order!");
606  }
607
608  public static void truncateFile(FileSystem fs, Path src, Path dst) throws IOException {
609    FileStatus fst = fs.getFileStatus(src);
610    long len = fst.getLen();
611    len = len / 2;
612
613    // create a truncated hfile
614    FSDataOutputStream fdos = fs.create(dst);
615    byte[] buf = new byte[(int) len];
616    FSDataInputStream fdis = fs.open(src);
617    fdis.read(buf);
618    fdos.write(buf);
619    fdis.close();
620    fdos.close();
621  }
622
623  /**
624   * Create a truncated hfile and verify that exception thrown.
625   */
626  @Test
627  public void testCorruptTruncatedHFile() throws IOException {
628    Path f = new Path(ROOT_DIR, testName.getMethodName());
629    HFileContext context = new HFileContextBuilder().build();
630    Writer w = HFile.getWriterFactory(conf, cacheConf).withPath(this.fs, f).withFileContext(context)
631      .create();
632    writeSomeRecords(w, 0, 100, false);
633    w.close();
634
635    Path trunc = new Path(f.getParent(), "trucated");
636    truncateFile(fs, w.getPath(), trunc);
637
638    try {
639      HFile.createReader(fs, trunc, cacheConf, true, conf);
640    } catch (CorruptHFileException | IllegalArgumentException che) {
641      // Expected failure
642      return;
643    }
644    fail("Should have thrown exception");
645  }
646
647  // write some records into the hfile
648  // write them twice
649  private int writeSomeRecords(Writer writer, int start, int n, boolean useTags)
650    throws IOException {
651    String value = "value";
652    KeyValue kv;
653    for (int i = start; i < (start + n); i++) {
654      String key = String.format(localFormatter, Integer.valueOf(i));
655      if (useTags) {
656        Tag t = new ArrayBackedTag((byte) 1, "myTag1");
657        Tag[] tags = new Tag[1];
658        tags[0] = t;
659        kv = new KeyValue(Bytes.toBytes(key), Bytes.toBytes("family"), Bytes.toBytes("qual"),
660          HConstants.LATEST_TIMESTAMP, Bytes.toBytes(value + key), tags);
661        writer.append(kv);
662      } else {
663        kv = new KeyValue(Bytes.toBytes(key), Bytes.toBytes("family"), Bytes.toBytes("qual"),
664          Bytes.toBytes(value + key));
665        writer.append(kv);
666      }
667    }
668    return (start + n);
669  }
670
671  private void readAllRecords(HFileScanner scanner) throws IOException {
672    readAndCheckbytes(scanner, 0, 100);
673  }
674
675  // read the records and check
676  private int readAndCheckbytes(HFileScanner scanner, int start, int n) throws IOException {
677    String value = "value";
678    int i = start;
679    for (; i < (start + n); i++) {
680      ByteBuffer key = ByteBuffer.wrap(((KeyValue) scanner.getKey()).getKey());
681      ByteBuffer val = scanner.getValue();
682      String keyStr = String.format(localFormatter, Integer.valueOf(i));
683      String valStr = value + keyStr;
684      KeyValue kv = new KeyValue(Bytes.toBytes(keyStr), Bytes.toBytes("family"),
685        Bytes.toBytes("qual"), Bytes.toBytes(valStr));
686      byte[] keyBytes =
687        new KeyValue.KeyOnlyKeyValue(Bytes.toBytes(key), 0, Bytes.toBytes(key).length).getKey();
688      assertTrue("bytes for keys do not match " + keyStr + " " + Bytes.toString(Bytes.toBytes(key)),
689        Arrays.equals(kv.getKey(), keyBytes));
690      byte[] valBytes = Bytes.toBytes(val);
691      assertTrue("bytes for vals do not match " + valStr + " " + Bytes.toString(valBytes),
692        Arrays.equals(Bytes.toBytes(valStr), valBytes));
693      if (!scanner.next()) {
694        break;
695      }
696    }
697    assertEquals(i, start + n - 1);
698    return (start + n);
699  }
700
701  private byte[] getSomeKey(int rowId) {
702    KeyValue kv = new KeyValue(Bytes.toBytes(String.format(localFormatter, Integer.valueOf(rowId))),
703      Bytes.toBytes("family"), Bytes.toBytes("qual"), HConstants.LATEST_TIMESTAMP, Type.Put);
704    return kv.getKey();
705  }
706
707  private void writeRecords(Writer writer, boolean useTags) throws IOException {
708    writeSomeRecords(writer, 0, 100, useTags);
709    writer.close();
710  }
711
712  private FSDataOutputStream createFSOutput(Path name) throws IOException {
713    // if (fs.exists(name)) fs.delete(name, true);
714    FSDataOutputStream fout = fs.create(name);
715    return fout;
716  }
717
718  /**
719   * test none codecs
720   */
721  void basicWithSomeCodec(String codec, boolean useTags) throws IOException {
722    if (useTags) {
723      conf.setInt("hfile.format.version", 3);
724    }
725    Path ncHFile = new Path(ROOT_DIR, "basic.hfile." + codec.toString() + useTags);
726    FSDataOutputStream fout = createFSOutput(ncHFile);
727    HFileContext meta = new HFileContextBuilder().withBlockSize(minBlockSize)
728      .withCompression(HFileWriterImpl.compressionByName(codec)).build();
729    Writer writer =
730      HFile.getWriterFactory(conf, cacheConf).withOutputStream(fout).withFileContext(meta).create();
731    LOG.info(Objects.toString(writer));
732    writeRecords(writer, useTags);
733    fout.close();
734    FSDataInputStream fin = fs.open(ncHFile);
735    ReaderContext context = new ReaderContextBuilder().withFileSystemAndPath(fs, ncHFile).build();
736    Reader reader = createReaderFromStream(context, cacheConf, conf);
737    System.out.println(cacheConf.toString());
738    // Load up the index.
739    // Get a scanner that caches and that does not use pread.
740    HFileScanner scanner = reader.getScanner(conf, true, false);
741    // Align scanner at start of the file.
742    scanner.seekTo();
743    readAllRecords(scanner);
744    int seekTo = scanner.seekTo(KeyValueUtil.createKeyValueFromKey(getSomeKey(50)));
745    System.out.println(seekTo);
746    assertTrue("location lookup failed",
747      scanner.seekTo(KeyValueUtil.createKeyValueFromKey(getSomeKey(50))) == 0);
748    // read the key and see if it matches
749    ByteBuffer readKey = ByteBuffer.wrap(((KeyValue) scanner.getKey()).getKey());
750    assertTrue("seeked key does not match", Arrays.equals(getSomeKey(50), Bytes.toBytes(readKey)));
751
752    scanner.seekTo(KeyValueUtil.createKeyValueFromKey(getSomeKey(0)));
753    ByteBuffer val1 = scanner.getValue();
754    scanner.seekTo(KeyValueUtil.createKeyValueFromKey(getSomeKey(0)));
755    ByteBuffer val2 = scanner.getValue();
756    assertTrue(Arrays.equals(Bytes.toBytes(val1), Bytes.toBytes(val2)));
757
758    reader.close();
759    fin.close();
760    fs.delete(ncHFile, true);
761  }
762
763  @Test
764  public void testTFileFeatures() throws IOException {
765    testHFilefeaturesInternals(false);
766    testHFilefeaturesInternals(true);
767  }
768
769  protected void testHFilefeaturesInternals(boolean useTags) throws IOException {
770    basicWithSomeCodec("none", useTags);
771    basicWithSomeCodec("gz", useTags);
772  }
773
774  private void writeNumMetablocks(Writer writer, int n) {
775    for (int i = 0; i < n; i++) {
776      writer.appendMetaBlock("HFileMeta" + i, new Writable() {
777        private int val;
778
779        public Writable setVal(int val) {
780          this.val = val;
781          return this;
782        }
783
784        @Override
785        public void write(DataOutput out) throws IOException {
786          out.write(Bytes.toBytes("something to test" + val));
787        }
788
789        @Override
790        public void readFields(DataInput in) throws IOException {
791        }
792      }.setVal(i));
793    }
794  }
795
796  private void someTestingWithMetaBlock(Writer writer) {
797    writeNumMetablocks(writer, 10);
798  }
799
800  private void readNumMetablocks(Reader reader, int n) throws IOException {
801    for (int i = 0; i < n; i++) {
802      ByteBuff actual = reader.getMetaBlock("HFileMeta" + i, false).getBufferWithoutHeader();
803      ByteBuffer expected = ByteBuffer.wrap(Bytes.toBytes("something to test" + i));
804      assertEquals("failed to match metadata", Bytes.toStringBinary(expected), Bytes.toStringBinary(
805        actual.array(), actual.arrayOffset() + actual.position(), actual.capacity()));
806    }
807  }
808
809  private void someReadingWithMetaBlock(Reader reader) throws IOException {
810    readNumMetablocks(reader, 10);
811  }
812
813  private void metablocks(final String compress) throws Exception {
814    Path mFile = new Path(ROOT_DIR, "meta.hfile");
815    FSDataOutputStream fout = createFSOutput(mFile);
816    HFileContext meta =
817      new HFileContextBuilder().withCompression(HFileWriterImpl.compressionByName(compress))
818        .withBlockSize(minBlockSize).build();
819    Writer writer =
820      HFile.getWriterFactory(conf, cacheConf).withOutputStream(fout).withFileContext(meta).create();
821    someTestingWithMetaBlock(writer);
822    writer.close();
823    fout.close();
824    ReaderContext context = new ReaderContextBuilder().withFileSystemAndPath(fs, mFile).build();
825    Reader reader = createReaderFromStream(context, cacheConf, conf);
826    // No data -- this should return false.
827    assertFalse(reader.getScanner(conf, false, false).seekTo());
828    someReadingWithMetaBlock(reader);
829    fs.delete(mFile, true);
830    reader.close();
831  }
832
833  // test meta blocks for hfiles
834  @Test
835  public void testMetaBlocks() throws Exception {
836    metablocks("none");
837    metablocks("gz");
838  }
839
840  @Test
841  public void testNullMetaBlocks() throws Exception {
842    for (Compression.Algorithm compressAlgo : HBaseCommonTestingUtil.COMPRESSION_ALGORITHMS) {
843      Path mFile = new Path(ROOT_DIR, "nometa_" + compressAlgo + ".hfile");
844      FSDataOutputStream fout = createFSOutput(mFile);
845      HFileContext meta =
846        new HFileContextBuilder().withCompression(compressAlgo).withBlockSize(minBlockSize).build();
847      Writer writer = HFile.getWriterFactory(conf, cacheConf).withOutputStream(fout)
848        .withFileContext(meta).create();
849      KeyValue kv =
850        new KeyValue(Bytes.toBytes("foo"), Bytes.toBytes("f1"), null, Bytes.toBytes("value"));
851      writer.append(kv);
852      writer.close();
853      fout.close();
854      Reader reader = HFile.createReader(fs, mFile, cacheConf, true, conf);
855      assertNull(reader.getMetaBlock("non-existant", false));
856    }
857  }
858
859  /**
860   * Make sure the ordinals for our compression algorithms do not change on us.
861   */
862  @Test
863  public void testCompressionOrdinance() {
864    assertTrue(Compression.Algorithm.LZO.ordinal() == 0);
865    assertTrue(Compression.Algorithm.GZ.ordinal() == 1);
866    assertTrue(Compression.Algorithm.NONE.ordinal() == 2);
867    assertTrue(Compression.Algorithm.SNAPPY.ordinal() == 3);
868    assertTrue(Compression.Algorithm.LZ4.ordinal() == 4);
869  }
870
871  @Test
872  public void testShortMidpointSameQual() {
873    ExtendedCell left =
874      ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(Bytes.toBytes("a"))
875        .setFamily(Bytes.toBytes("a")).setQualifier(Bytes.toBytes("a")).setTimestamp(11)
876        .setType(Type.Maximum.getCode()).setValue(HConstants.EMPTY_BYTE_ARRAY).build();
877    ExtendedCell right =
878      ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(Bytes.toBytes("a"))
879        .setFamily(Bytes.toBytes("a")).setQualifier(Bytes.toBytes("a")).setTimestamp(9)
880        .setType(Type.Maximum.getCode()).setValue(HConstants.EMPTY_BYTE_ARRAY).build();
881    ExtendedCell mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right);
882    assertTrue(
883      PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) <= 0);
884    assertTrue(
885      PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) == 0);
886  }
887
888  private ExtendedCell getCell(byte[] row, byte[] family, byte[] qualifier) {
889    return ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(row)
890      .setFamily(family).setQualifier(qualifier).setTimestamp(HConstants.LATEST_TIMESTAMP)
891      .setType(KeyValue.Type.Maximum.getCode()).setValue(HConstants.EMPTY_BYTE_ARRAY).build();
892  }
893
894  @Test
895  public void testGetShortMidpoint() {
896    ExtendedCell left = getCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a"));
897    ExtendedCell right = getCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a"));
898    ExtendedCell mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right);
899    assertTrue(
900      PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) <= 0);
901    assertTrue(
902      PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) <= 0);
903    left = getCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a"));
904    right = getCell(Bytes.toBytes("b"), Bytes.toBytes("a"), Bytes.toBytes("a"));
905    mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right);
906    assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0);
907    assertTrue(
908      PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) <= 0);
909    left = getCell(Bytes.toBytes("g"), Bytes.toBytes("a"), Bytes.toBytes("a"));
910    right = getCell(Bytes.toBytes("i"), Bytes.toBytes("a"), Bytes.toBytes("a"));
911    mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right);
912    assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0);
913    assertTrue(
914      PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) <= 0);
915    left = getCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a"));
916    right = getCell(Bytes.toBytes("bbbbbbb"), Bytes.toBytes("a"), Bytes.toBytes("a"));
917    mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right);
918    assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0);
919    assertTrue(
920      PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) < 0);
921    assertEquals(1, mid.getRowLength());
922    left = getCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a"));
923    right = getCell(Bytes.toBytes("b"), Bytes.toBytes("a"), Bytes.toBytes("a"));
924    mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right);
925    assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0);
926    assertTrue(
927      PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) <= 0);
928    left = getCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a"));
929    right = getCell(Bytes.toBytes("a"), Bytes.toBytes("aaaaaaaa"), Bytes.toBytes("b"));
930    mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right);
931    assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0);
932    assertTrue(
933      PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) < 0);
934    assertEquals(2, mid.getFamilyLength());
935    left = getCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a"));
936    right = getCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("aaaaaaaaa"));
937    mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right);
938    assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0);
939    assertTrue(
940      PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) < 0);
941    assertEquals(2, mid.getQualifierLength());
942    left = getCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a"));
943    right = getCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("b"));
944    mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right);
945    assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0);
946    assertTrue(
947      PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) <= 0);
948    assertEquals(1, mid.getQualifierLength());
949
950    // Verify boundary conditions
951    left = getCell(Bytes.toBytes("a"), Bytes.toBytes("a"), new byte[] { 0x00, (byte) 0xFE });
952    right = getCell(Bytes.toBytes("a"), Bytes.toBytes("a"), new byte[] { 0x00, (byte) 0xFF });
953    mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right);
954    assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0);
955    assertTrue(
956      PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) == 0);
957    left = getCell(Bytes.toBytes("a"), Bytes.toBytes("a"), new byte[] { 0x00, 0x12 });
958    right = getCell(Bytes.toBytes("a"), Bytes.toBytes("a"), new byte[] { 0x00, 0x12, 0x00 });
959    mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right);
960    assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0);
961    assertTrue(
962      PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) == 0);
963
964    // Assert that if meta comparator, it returns the right cell -- i.e. no
965    // optimization done.
966    left = getCell(Bytes.toBytes("g"), Bytes.toBytes("a"), Bytes.toBytes("a"));
967    right = getCell(Bytes.toBytes("i"), Bytes.toBytes("a"), Bytes.toBytes("a"));
968    mid = HFileWriterImpl.getMidpoint(MetaCellComparator.META_COMPARATOR, left, right);
969    assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0);
970    assertTrue(
971      PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) == 0);
972    byte[] family = Bytes.toBytes("family");
973    byte[] qualA = Bytes.toBytes("qfA");
974    byte[] qualB = Bytes.toBytes("qfB");
975    final CellComparatorImpl keyComparator = CellComparatorImpl.COMPARATOR;
976    // verify that faked shorter rowkey could be generated
977    long ts = 5;
978    KeyValue kv1 = new KeyValue(Bytes.toBytes("the quick brown fox"), family, qualA, ts, Type.Put);
979    KeyValue kv2 = new KeyValue(Bytes.toBytes("the who test text"), family, qualA, ts, Type.Put);
980    ExtendedCell newKey = HFileWriterImpl.getMidpoint(keyComparator, kv1, kv2);
981    assertTrue(keyComparator.compare(kv1, newKey) < 0);
982    assertTrue((keyComparator.compare(kv2, newKey)) > 0);
983    byte[] expectedArray = Bytes.toBytes("the r");
984    Bytes.equals(newKey.getRowArray(), newKey.getRowOffset(), newKey.getRowLength(), expectedArray,
985      0, expectedArray.length);
986
987    // verify: same with "row + family + qualifier", return rightKey directly
988    kv1 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, 5, Type.Put);
989    kv2 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, 0, Type.Put);
990    assertTrue(keyComparator.compare(kv1, kv2) < 0);
991    newKey = HFileWriterImpl.getMidpoint(keyComparator, kv1, kv2);
992    assertTrue(keyComparator.compare(kv1, newKey) < 0);
993    assertTrue((keyComparator.compare(kv2, newKey)) == 0);
994    kv1 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, -5, Type.Put);
995    kv2 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, -10, Type.Put);
996    assertTrue(keyComparator.compare(kv1, kv2) < 0);
997    newKey = HFileWriterImpl.getMidpoint(keyComparator, kv1, kv2);
998    assertTrue(keyComparator.compare(kv1, newKey) < 0);
999    assertTrue((keyComparator.compare(kv2, newKey)) == 0);
1000
1001    // verify: same with row, different with qualifier
1002    kv1 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, 5, Type.Put);
1003    kv2 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualB, 5, Type.Put);
1004    assertTrue(keyComparator.compare(kv1, kv2) < 0);
1005    newKey = HFileWriterImpl.getMidpoint(keyComparator, kv1, kv2);
1006    assertTrue(keyComparator.compare(kv1, newKey) < 0);
1007    assertTrue((keyComparator.compare(kv2, newKey)) > 0);
1008    assertTrue(Arrays.equals(CellUtil.cloneFamily(newKey), family));
1009    assertTrue(Arrays.equals(CellUtil.cloneQualifier(newKey), qualB));
1010    assertTrue(newKey.getTimestamp() == HConstants.LATEST_TIMESTAMP);
1011    assertTrue(newKey.getTypeByte() == Type.Maximum.getCode());
1012
1013    // verify metaKeyComparator's getShortMidpointKey output
1014    final CellComparatorImpl metaKeyComparator = MetaCellComparator.META_COMPARATOR;
1015    kv1 = new KeyValue(Bytes.toBytes("ilovehbase123"), family, qualA, 5, Type.Put);
1016    kv2 = new KeyValue(Bytes.toBytes("ilovehbase234"), family, qualA, 0, Type.Put);
1017    newKey = HFileWriterImpl.getMidpoint(metaKeyComparator, kv1, kv2);
1018    assertTrue(metaKeyComparator.compare(kv1, newKey) < 0);
1019    assertTrue((metaKeyComparator.compare(kv2, newKey) == 0));
1020
1021    // verify common fix scenario
1022    kv1 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, ts, Type.Put);
1023    kv2 = new KeyValue(Bytes.toBytes("ilovehbaseandhdfs"), family, qualA, ts, Type.Put);
1024    assertTrue(keyComparator.compare(kv1, kv2) < 0);
1025    newKey = HFileWriterImpl.getMidpoint(keyComparator, kv1, kv2);
1026    assertTrue(keyComparator.compare(kv1, newKey) < 0);
1027    assertTrue((keyComparator.compare(kv2, newKey)) > 0);
1028    expectedArray = Bytes.toBytes("ilovehbasea");
1029    Bytes.equals(newKey.getRowArray(), newKey.getRowOffset(), newKey.getRowLength(), expectedArray,
1030      0, expectedArray.length);
1031    // verify only 1 offset scenario
1032    kv1 = new KeyValue(Bytes.toBytes("100abcdefg"), family, qualA, ts, Type.Put);
1033    kv2 = new KeyValue(Bytes.toBytes("101abcdefg"), family, qualA, ts, Type.Put);
1034    assertTrue(keyComparator.compare(kv1, kv2) < 0);
1035    newKey = HFileWriterImpl.getMidpoint(keyComparator, kv1, kv2);
1036    assertTrue(keyComparator.compare(kv1, newKey) < 0);
1037    assertTrue((keyComparator.compare(kv2, newKey)) > 0);
1038    expectedArray = Bytes.toBytes("101");
1039    Bytes.equals(newKey.getRowArray(), newKey.getRowOffset(), newKey.getRowLength(), expectedArray,
1040      0, expectedArray.length);
1041  }
1042
1043  @Test
1044  public void testDBEShipped() throws IOException {
1045    for (DataBlockEncoding encoding : DataBlockEncoding.values()) {
1046      DataBlockEncoder encoder = encoding.getEncoder();
1047      if (encoder == null) {
1048        continue;
1049      }
1050      Path f = new Path(ROOT_DIR, testName.getMethodName() + "_" + encoding);
1051      HFileContext context =
1052        new HFileContextBuilder().withIncludesTags(false).withDataBlockEncoding(encoding).build();
1053      HFileWriterImpl writer = (HFileWriterImpl) HFile.getWriterFactory(conf, cacheConf)
1054        .withPath(fs, f).withFileContext(context).create();
1055
1056      KeyValue kv = new KeyValue(Bytes.toBytes("testkey1"), Bytes.toBytes("family"),
1057        Bytes.toBytes("qual"), Bytes.toBytes("testvalue"));
1058      KeyValue kv2 = new KeyValue(Bytes.toBytes("testkey2"), Bytes.toBytes("family"),
1059        Bytes.toBytes("qual"), Bytes.toBytes("testvalue"));
1060      KeyValue kv3 = new KeyValue(Bytes.toBytes("testkey3"), Bytes.toBytes("family"),
1061        Bytes.toBytes("qual"), Bytes.toBytes("testvalue"));
1062
1063      ByteBuffer buffer = ByteBuffer.wrap(kv.getBuffer());
1064      ByteBuffer buffer2 = ByteBuffer.wrap(kv2.getBuffer());
1065      ByteBuffer buffer3 = ByteBuffer.wrap(kv3.getBuffer());
1066
1067      writer.append(new ByteBufferKeyValue(buffer, 0, buffer.remaining()));
1068      writer.beforeShipped();
1069
1070      // pollute first cell's backing ByteBuffer
1071      ByteBufferUtils.copyFromBufferToBuffer(buffer3, buffer);
1072
1073      // write another cell, if DBE not Shipped, test will fail
1074      writer.append(new ByteBufferKeyValue(buffer2, 0, buffer2.remaining()));
1075      writer.close();
1076    }
1077  }
1078
1079  /**
1080   * Test case for CombinedBlockCache with TinyLfu as L1 cache
1081   */
1082  @Test
1083  public void testReaderWithTinyLfuCombinedBlockCache() throws Exception {
1084    testReaderCombinedCache("TinyLfu");
1085  }
1086
1087  /**
1088   * Test case for CombinedBlockCache with AdaptiveLRU as L1 cache
1089   */
1090  @Test
1091  public void testReaderWithAdaptiveLruCombinedBlockCache() throws Exception {
1092    testReaderCombinedCache("AdaptiveLRU");
1093  }
1094
1095  /**
1096   * Test case for CombinedBlockCache with AdaptiveLRU as L1 cache
1097   */
1098  @Test
1099  public void testReaderWithLruCombinedBlockCache() throws Exception {
1100    testReaderCombinedCache("LRU");
1101  }
1102
1103  private void testReaderCombinedCache(final String l1CachePolicy) throws Exception {
1104    int bufCount = 1024;
1105    int blockSize = 64 * 1024;
1106    ByteBuffAllocator alloc = initAllocator(true, bufCount, blockSize, 0);
1107    fillByteBuffAllocator(alloc, bufCount);
1108    Path storeFilePath = writeStoreFile();
1109    // Open the file reader with CombinedBlockCache
1110    BlockCache combined = initCombinedBlockCache(l1CachePolicy);
1111    conf.setBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, true);
1112    CacheConfig cacheConfig = new CacheConfig(conf, null, combined, alloc);
1113    HFile.Reader reader = HFile.createReader(fs, storeFilePath, cacheConfig, true, conf);
1114    long offset = 0;
1115    Cacheable cachedBlock = null;
1116    while (offset < reader.getTrailer().getLoadOnOpenDataOffset()) {
1117      BlockCacheKey key = new BlockCacheKey(storeFilePath.getName(), offset);
1118      HFileBlock block = reader.readBlock(offset, -1, true, true, false, true, null, null);
1119      offset += block.getOnDiskSizeWithHeader();
1120      // Read the cached block.
1121      cachedBlock = combined.getBlock(key, false, false, true);
1122      try {
1123        Assert.assertNotNull(cachedBlock);
1124        Assert.assertTrue(cachedBlock instanceof HFileBlock);
1125        HFileBlock hfb = (HFileBlock) cachedBlock;
1126        // Data block will be cached in BucketCache, so it should be an off-heap block.
1127        if (hfb.getBlockType().isData()) {
1128          Assert.assertTrue(hfb.isSharedMem());
1129        } else if (!l1CachePolicy.equals("TinyLfu")) {
1130          Assert.assertFalse(hfb.isSharedMem());
1131        }
1132      } finally {
1133        cachedBlock.release();
1134      }
1135      block.release(); // return back the ByteBuffer back to allocator.
1136    }
1137    reader.close();
1138    combined.shutdown();
1139    if (cachedBlock != null) {
1140      Assert.assertEquals(0, cachedBlock.refCnt());
1141    }
1142    Assert.assertEquals(bufCount, alloc.getFreeBufferCount());
1143    alloc.clean();
1144  }
1145
1146  @Test
1147  public void testHFileContextBuilderWithIndexEncoding() throws IOException {
1148    HFileContext context =
1149      new HFileContextBuilder().withIndexBlockEncoding(IndexBlockEncoding.PREFIX_TREE).build();
1150    HFileContext newContext = new HFileContextBuilder(context).build();
1151    assertTrue(newContext.getIndexBlockEncoding() == IndexBlockEncoding.PREFIX_TREE);
1152  }
1153}