001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.io.hfile; 019 020import static org.junit.Assert.assertEquals; 021import java.io.IOException; 022import org.apache.hadoop.conf.Configuration; 023import org.apache.hadoop.fs.FSDataOutputStream; 024import org.apache.hadoop.fs.FileSystem; 025import org.apache.hadoop.fs.Path; 026import org.apache.hadoop.hbase.Cell; 027import org.apache.hadoop.hbase.CellComparatorImpl; 028import org.apache.hadoop.hbase.HBaseClassTestRule; 029import org.apache.hadoop.hbase.HBaseTestingUtility; 030import org.apache.hadoop.hbase.KeyValue; 031import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache; 032import org.apache.hadoop.hbase.testclassification.IOTests; 033import org.apache.hadoop.hbase.testclassification.SmallTests; 034import org.apache.hadoop.hbase.util.Bytes; 035import org.junit.ClassRule; 036import org.junit.Test; 037import org.junit.experimental.categories.Category; 038 039import org.apache.hbase.thirdparty.com.google.common.collect.Lists; 040 041 042/** 043 * Test 044 */ 045@Category({ IOTests.class, SmallTests.class}) 046public class TestHFileReaderImpl { 047 048 @ClassRule 049 public static final HBaseClassTestRule CLASS_RULE = 050 HBaseClassTestRule.forClass(TestHFileReaderImpl.class); 051 052 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); 053 054 static KeyValue toKV(String row) { 055 return new KeyValue(Bytes.toBytes(row), Bytes.toBytes("family"), Bytes.toBytes("qualifier"), 056 Bytes.toBytes("value")); 057 } 058 059 static String toRowStr(Cell c) { 060 return Bytes.toString(c.getRowArray(), c.getRowOffset(), c.getRowLength()); 061 } 062 063 Path makeNewFile() throws IOException { 064 Path ncTFile = new Path(TEST_UTIL.getDataTestDir(), "basic.hfile"); 065 FSDataOutputStream fout = TEST_UTIL.getTestFileSystem().create(ncTFile); 066 int blocksize = toKV("a").getLength() * 3; 067 HFileContext context = 068 new HFileContextBuilder().withBlockSize(blocksize).withIncludesTags(true).build(); 069 Configuration conf = TEST_UTIL.getConfiguration(); 070 HFile.Writer writer = 071 HFile.getWriterFactoryNoCache(conf).withOutputStream(fout).withFileContext(context) 072 .withComparator(CellComparatorImpl.COMPARATOR).create(); 073 // 4 bytes * 3 * 2 for each key/value + 074 // 3 for keys, 15 for values = 42 (woot) 075 writer.append(toKV("c")); 076 writer.append(toKV("e")); 077 writer.append(toKV("g")); 078 // block transition 079 writer.append(toKV("i")); 080 writer.append(toKV("k")); 081 writer.close(); 082 fout.close(); 083 return ncTFile; 084 } 085 086 @Test 087 public void testSeekBefore() throws Exception { 088 Path p = makeNewFile(); 089 FileSystem fs = TEST_UTIL.getTestFileSystem(); 090 Configuration conf = TEST_UTIL.getConfiguration(); 091 int[] bucketSizes = { 512, 2048, 4096, 64 * 1024, 128 * 1024 }; 092 BucketCache bucketcache = 093 new BucketCache("offheap", 128 * 1024 * 1024, 64 * 1024, bucketSizes, 5, 64 * 100, null); 094 095 HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf, bucketcache), true, conf); 096 reader.loadFileInfo(); 097 098 // warm cache 099 HFileScanner scanner = reader.getScanner(true, true); 100 scanner.seekTo(toKV("i")); 101 assertEquals("i", toRowStr(scanner.getCell())); 102 scanner.close(); 103 104 while (bucketcache.getBlockCount() <= 0) { 105 Thread.sleep(10); 106 } 107 108 // reopen again. 109 scanner = reader.getScanner(true, true); 110 scanner.seekTo(toKV("i")); 111 assertEquals("i", toRowStr(scanner.getCell())); 112 scanner.seekBefore(toKV("i")); 113 assertEquals("g", toRowStr(scanner.getCell())); 114 scanner.close(); 115 116 for (CachedBlock cachedBlock : Lists.newArrayList(bucketcache)) { 117 BlockCacheKey cacheKey = 118 new BlockCacheKey(cachedBlock.getFilename(), cachedBlock.getOffset()); 119 int refCount = bucketcache.getRefCount(cacheKey); 120 assertEquals(0, refCount); 121 } 122 123 // case 2 124 scanner = reader.getScanner(true, true); 125 scanner.seekTo(toKV("i")); 126 assertEquals("i", toRowStr(scanner.getCell())); 127 scanner.seekBefore(toKV("c")); 128 scanner.close(); 129 for (CachedBlock cachedBlock : Lists.newArrayList(bucketcache)) { 130 BlockCacheKey cacheKey = 131 new BlockCacheKey(cachedBlock.getFilename(), cachedBlock.getOffset()); 132 int refCount = bucketcache.getRefCount(cacheKey); 133 assertEquals(0, refCount); 134 } 135 136 reader.close(); 137 138 // clear bucketcache 139 for (CachedBlock cachedBlock : Lists.newArrayList(bucketcache)) { 140 BlockCacheKey cacheKey = 141 new BlockCacheKey(cachedBlock.getFilename(), cachedBlock.getOffset()); 142 bucketcache.evictBlock(cacheKey); 143 } 144 bucketcache.shutdown(); 145 146 deleteTestDir(fs); 147 } 148 149 protected void deleteTestDir(FileSystem fs) throws IOException { 150 Path dataTestDir = TEST_UTIL.getDataTestDir(); 151 if(fs.exists(dataTestDir)) { 152 fs.delete(dataTestDir, true); 153 } 154 } 155 156}