001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.io; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertNull; 022import static org.junit.Assert.assertTrue; 023 024import java.io.IOException; 025import java.util.ArrayList; 026import java.util.List; 027import java.util.concurrent.atomic.AtomicInteger; 028import org.apache.hadoop.conf.Configuration; 029import org.apache.hadoop.fs.FileSystem; 030import org.apache.hadoop.fs.Path; 031import org.apache.hadoop.hbase.Cell; 032import org.apache.hadoop.hbase.CellComparatorImpl; 033import org.apache.hadoop.hbase.CellUtil; 034import org.apache.hadoop.hbase.HBaseClassTestRule; 035import org.apache.hadoop.hbase.HBaseTestingUtility; 036import org.apache.hadoop.hbase.KeyValue; 037import org.apache.hadoop.hbase.KeyValueUtil; 038import org.apache.hadoop.hbase.io.hfile.CacheConfig; 039import org.apache.hadoop.hbase.io.hfile.HFile; 040import org.apache.hadoop.hbase.io.hfile.HFileContext; 041import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; 042import org.apache.hadoop.hbase.io.hfile.HFileScanner; 043import org.apache.hadoop.hbase.testclassification.IOTests; 044import org.apache.hadoop.hbase.testclassification.SmallTests; 045import org.apache.hadoop.hbase.util.Bytes; 046import org.junit.AfterClass; 047import org.junit.BeforeClass; 048import org.junit.ClassRule; 049import org.junit.Test; 050import org.junit.experimental.categories.Category; 051 052@Category({ IOTests.class, SmallTests.class }) 053public class TestHalfStoreFileReader { 054 055 @ClassRule 056 public static final HBaseClassTestRule CLASS_RULE = 057 HBaseClassTestRule.forClass(TestHalfStoreFileReader.class); 058 059 private static HBaseTestingUtility TEST_UTIL; 060 061 @BeforeClass 062 public static void setupBeforeClass() throws Exception { 063 TEST_UTIL = new HBaseTestingUtility(); 064 } 065 066 @AfterClass 067 public static void tearDownAfterClass() throws Exception { 068 TEST_UTIL.cleanupTestDir(); 069 } 070 071 /** 072 * Test the scanner and reseek of a half hfile scanner. The scanner API demands that seekTo and 073 * reseekTo() only return < 0 if the key lies before the start of the file (with no position on 074 * the scanner). Returning 0 if perfect match (rare), and return > 1 if we got an imperfect match. 075 * The latter case being the most common, we should generally be returning 1, and if we do, there 076 * may or may not be a 'next' in the scanner/file. A bug in the half file scanner was returning -1 077 * at the end of the bottom half, and that was causing the infrastructure above to go null causing 078 * NPEs and other problems. This test reproduces that failure, and also tests both the bottom and 079 * top of the file while we are at it. 080 * @throws IOException 081 */ 082 @Test 083 public void testHalfScanAndReseek() throws IOException { 084 String root_dir = TEST_UTIL.getDataTestDir().toString(); 085 Path p = new Path(root_dir, "test"); 086 087 Configuration conf = TEST_UTIL.getConfiguration(); 088 FileSystem fs = FileSystem.get(conf); 089 CacheConfig cacheConf = new CacheConfig(conf); 090 HFileContext meta = new HFileContextBuilder().withBlockSize(1024).build(); 091 HFile.Writer w = 092 HFile.getWriterFactory(conf, cacheConf).withPath(fs, p).withFileContext(meta).create(); 093 094 // write some things. 095 List<KeyValue> items = genSomeKeys(); 096 for (KeyValue kv : items) { 097 w.append(kv); 098 } 099 w.close(); 100 101 HFile.Reader r = HFile.createReader(fs, p, cacheConf, true, conf); 102 r.loadFileInfo(); 103 Cell midKV = r.midKey().get(); 104 byte[] midkey = CellUtil.cloneRow(midKV); 105 106 // System.out.println("midkey: " + midKV + " or: " + Bytes.toStringBinary(midkey)); 107 108 Reference bottom = new Reference(midkey, Reference.Range.bottom); 109 doTestOfScanAndReseek(p, fs, bottom, cacheConf); 110 111 Reference top = new Reference(midkey, Reference.Range.top); 112 doTestOfScanAndReseek(p, fs, top, cacheConf); 113 114 r.close(); 115 } 116 117 private void doTestOfScanAndReseek(Path p, FileSystem fs, Reference bottom, CacheConfig cacheConf) 118 throws IOException { 119 final HalfStoreFileReader halfreader = new HalfStoreFileReader(fs, p, cacheConf, bottom, true, 120 new AtomicInteger(0), true, TEST_UTIL.getConfiguration()); 121 halfreader.loadFileInfo(); 122 final HFileScanner scanner = halfreader.getScanner(false, false); 123 124 scanner.seekTo(); 125 Cell curr; 126 do { 127 curr = scanner.getCell(); 128 KeyValue reseekKv = getLastOnCol(curr); 129 int ret = scanner.reseekTo(reseekKv); 130 assertTrue("reseek to returned: " + ret, ret > 0); 131 // System.out.println(curr + ": " + ret); 132 } while (scanner.next()); 133 134 int ret = scanner.reseekTo(getLastOnCol(curr)); 135 // System.out.println("Last reseek: " + ret); 136 assertTrue(ret > 0); 137 138 halfreader.close(true); 139 } 140 141 // Tests the scanner on an HFile that is backed by HalfStoreFiles 142 @Test 143 public void testHalfScanner() throws IOException { 144 String root_dir = TEST_UTIL.getDataTestDir().toString(); 145 Path p = new Path(root_dir, "test"); 146 Configuration conf = TEST_UTIL.getConfiguration(); 147 FileSystem fs = FileSystem.get(conf); 148 CacheConfig cacheConf = new CacheConfig(conf); 149 HFileContext meta = new HFileContextBuilder().withBlockSize(1024).build(); 150 HFile.Writer w = 151 HFile.getWriterFactory(conf, cacheConf).withPath(fs, p).withFileContext(meta).create(); 152 153 // write some things. 154 List<KeyValue> items = genSomeKeys(); 155 for (KeyValue kv : items) { 156 w.append(kv); 157 } 158 w.close(); 159 160 HFile.Reader r = HFile.createReader(fs, p, cacheConf, true, conf); 161 r.loadFileInfo(); 162 Cell midKV = r.midKey().get(); 163 byte[] midkey = CellUtil.cloneRow(midKV); 164 165 Reference bottom = new Reference(midkey, Reference.Range.bottom); 166 Reference top = new Reference(midkey, Reference.Range.top); 167 168 // Ugly code to get the item before the midkey 169 KeyValue beforeMidKey = null; 170 for (KeyValue item : items) { 171 if (CellComparatorImpl.COMPARATOR.compare(item, midKV) >= 0) { 172 break; 173 } 174 beforeMidKey = item; 175 } 176 System.out.println("midkey: " + midKV + " or: " + Bytes.toStringBinary(midkey)); 177 System.out.println("beforeMidKey: " + beforeMidKey); 178 179 // Seek on the splitKey, should be in top, not in bottom 180 Cell foundKeyValue = doTestOfSeekBefore(p, fs, bottom, midKV, cacheConf); 181 assertEquals(beforeMidKey, foundKeyValue); 182 183 // Seek tot the last thing should be the penultimate on the top, the one before the midkey on 184 // the bottom. 185 foundKeyValue = doTestOfSeekBefore(p, fs, top, items.get(items.size() - 1), cacheConf); 186 assertEquals(items.get(items.size() - 2), foundKeyValue); 187 188 foundKeyValue = doTestOfSeekBefore(p, fs, bottom, items.get(items.size() - 1), cacheConf); 189 assertEquals(beforeMidKey, foundKeyValue); 190 191 // Try and seek before something that is in the bottom. 192 foundKeyValue = doTestOfSeekBefore(p, fs, top, items.get(0), cacheConf); 193 assertNull(foundKeyValue); 194 195 // Try and seek before the first thing. 196 foundKeyValue = doTestOfSeekBefore(p, fs, bottom, items.get(0), cacheConf); 197 assertNull(foundKeyValue); 198 199 // Try and seek before the second thing in the top and bottom. 200 foundKeyValue = doTestOfSeekBefore(p, fs, top, items.get(1), cacheConf); 201 assertNull(foundKeyValue); 202 203 foundKeyValue = doTestOfSeekBefore(p, fs, bottom, items.get(1), cacheConf); 204 assertEquals(items.get(0), foundKeyValue); 205 206 // Try to seek before the splitKey in the top file 207 foundKeyValue = doTestOfSeekBefore(p, fs, top, midKV, cacheConf); 208 assertNull(foundKeyValue); 209 } 210 211 private Cell doTestOfSeekBefore(Path p, FileSystem fs, Reference bottom, Cell seekBefore, 212 CacheConfig cacheConfig) throws IOException { 213 final HalfStoreFileReader halfreader = new HalfStoreFileReader(fs, p, cacheConfig, bottom, true, 214 new AtomicInteger(0), true, TEST_UTIL.getConfiguration()); 215 halfreader.loadFileInfo(); 216 final HFileScanner scanner = halfreader.getScanner(false, false); 217 scanner.seekBefore(seekBefore); 218 return scanner.getCell(); 219 } 220 221 private KeyValue getLastOnCol(Cell curr) { 222 return KeyValueUtil.createLastOnRow(curr.getRowArray(), curr.getRowOffset(), 223 curr.getRowLength(), curr.getFamilyArray(), curr.getFamilyOffset(), curr.getFamilyLength(), 224 curr.getQualifierArray(), curr.getQualifierOffset(), curr.getQualifierLength()); 225 } 226 227 static final int SIZE = 1000; 228 229 static byte[] _b(String s) { 230 return Bytes.toBytes(s); 231 } 232 233 List<KeyValue> genSomeKeys() { 234 List<KeyValue> ret = new ArrayList<>(SIZE); 235 for (int i = 0; i < SIZE; i++) { 236 KeyValue kv = 237 new KeyValue(_b(String.format("row_%04d", i)), _b("family"), _b("qualifier"), 1000, // timestamp 238 _b("value")); 239 ret.add(kv); 240 } 241 return ret; 242 } 243}