001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.io.hfile;
019
020import static org.junit.Assert.assertEquals;
021import java.io.IOException;
022import org.apache.hadoop.conf.Configuration;
023import org.apache.hadoop.fs.FSDataOutputStream;
024import org.apache.hadoop.fs.FileSystem;
025import org.apache.hadoop.fs.Path;
026import org.apache.hadoop.hbase.Cell;
027import org.apache.hadoop.hbase.CellComparatorImpl;
028import org.apache.hadoop.hbase.HBaseClassTestRule;
029import org.apache.hadoop.hbase.HBaseTestingUtility;
030import org.apache.hadoop.hbase.KeyValue;
031import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache;
032import org.apache.hadoop.hbase.testclassification.IOTests;
033import org.apache.hadoop.hbase.testclassification.SmallTests;
034import org.apache.hadoop.hbase.util.Bytes;
035import org.junit.ClassRule;
036import org.junit.Test;
037import org.junit.experimental.categories.Category;
038
039import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
040
041
042/**
043 * Test
044 */
045@Category({ IOTests.class, SmallTests.class})
046public class TestHFileReaderImpl {
047
048  @ClassRule
049  public static final HBaseClassTestRule CLASS_RULE =
050      HBaseClassTestRule.forClass(TestHFileReaderImpl.class);
051
052  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
053
054  static KeyValue toKV(String row) {
055    return new KeyValue(Bytes.toBytes(row), Bytes.toBytes("family"), Bytes.toBytes("qualifier"),
056        Bytes.toBytes("value"));
057  }
058
059  static String toRowStr(Cell c) {
060    return Bytes.toString(c.getRowArray(), c.getRowOffset(), c.getRowLength());
061  }
062
063  Path makeNewFile() throws IOException {
064    Path ncTFile = new Path(TEST_UTIL.getDataTestDir(), "basic.hfile");
065    FSDataOutputStream fout = TEST_UTIL.getTestFileSystem().create(ncTFile);
066    int blocksize = toKV("a").getLength() * 3;
067    HFileContext context =
068        new HFileContextBuilder().withBlockSize(blocksize).withIncludesTags(true).build();
069    Configuration conf = TEST_UTIL.getConfiguration();
070    HFile.Writer writer =
071        HFile.getWriterFactoryNoCache(conf).withOutputStream(fout).withFileContext(context)
072            .withComparator(CellComparatorImpl.COMPARATOR).create();
073    // 4 bytes * 3 * 2 for each key/value +
074    // 3 for keys, 15 for values = 42 (woot)
075    writer.append(toKV("c"));
076    writer.append(toKV("e"));
077    writer.append(toKV("g"));
078    // block transition
079    writer.append(toKV("i"));
080    writer.append(toKV("k"));
081    writer.close();
082    fout.close();
083    return ncTFile;
084  }
085
086  @Test
087  public void testSeekBefore() throws Exception {
088    Path p = makeNewFile();
089    FileSystem fs = TEST_UTIL.getTestFileSystem();
090    Configuration conf = TEST_UTIL.getConfiguration();
091    int[] bucketSizes = { 512, 2048, 4096, 64 * 1024, 128 * 1024 };
092    BucketCache bucketcache =
093        new BucketCache("offheap", 128 * 1024 * 1024, 64 * 1024, bucketSizes, 5, 64 * 100, null);
094
095    HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf, bucketcache), true, conf);
096    reader.loadFileInfo();
097
098    // warm cache
099    HFileScanner scanner = reader.getScanner(true, true);
100    scanner.seekTo(toKV("i"));
101    assertEquals("i", toRowStr(scanner.getCell()));
102    scanner.close();
103
104    while (bucketcache.getBlockCount() <= 0) {
105      Thread.sleep(10);
106    }
107
108    // reopen again.
109    scanner = reader.getScanner(true, true);
110    scanner.seekTo(toKV("i"));
111    assertEquals("i", toRowStr(scanner.getCell()));
112    scanner.seekBefore(toKV("i"));
113    assertEquals("g", toRowStr(scanner.getCell()));
114    scanner.close();
115
116    for (CachedBlock cachedBlock : Lists.newArrayList(bucketcache)) {
117      BlockCacheKey cacheKey =
118          new BlockCacheKey(cachedBlock.getFilename(), cachedBlock.getOffset());
119      int refCount = bucketcache.getRefCount(cacheKey);
120      assertEquals(0, refCount);
121    }
122
123    // case 2
124    scanner = reader.getScanner(true, true);
125    scanner.seekTo(toKV("i"));
126    assertEquals("i", toRowStr(scanner.getCell()));
127    scanner.seekBefore(toKV("c"));
128    scanner.close();
129    for (CachedBlock cachedBlock : Lists.newArrayList(bucketcache)) {
130      BlockCacheKey cacheKey =
131          new BlockCacheKey(cachedBlock.getFilename(), cachedBlock.getOffset());
132      int refCount = bucketcache.getRefCount(cacheKey);
133      assertEquals(0, refCount);
134    }
135
136    reader.close();
137
138    // clear bucketcache
139    for (CachedBlock cachedBlock : Lists.newArrayList(bucketcache)) {
140      BlockCacheKey cacheKey =
141          new BlockCacheKey(cachedBlock.getFilename(), cachedBlock.getOffset());
142      bucketcache.evictBlock(cacheKey);
143    }
144    bucketcache.shutdown();
145
146    deleteTestDir(fs);
147  }
148
149  protected void deleteTestDir(FileSystem fs) throws IOException {
150    Path dataTestDir = TEST_UTIL.getDataTestDir();
151    if(fs.exists(dataTestDir)) {
152      fs.delete(dataTestDir, true);
153    }
154  }
155
156}