001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.io.hfile; 019 020import java.util.ArrayList; 021import java.util.List; 022import org.apache.hadoop.conf.Configuration; 023import org.apache.hadoop.fs.FileSystem; 024import org.apache.hadoop.fs.Path; 025import org.apache.hadoop.hbase.CellUtil; 026import org.apache.hadoop.hbase.HBaseClassTestRule; 027import org.apache.hadoop.hbase.HBaseTestingUtility; 028import org.apache.hadoop.hbase.testclassification.IOTests; 029import org.apache.hadoop.hbase.testclassification.SmallTests; 030import org.apache.hadoop.hbase.util.Bytes; 031import org.junit.ClassRule; 032import org.junit.Test; 033import org.junit.experimental.categories.Category; 034 035/** 036 * Test a case when an inline index chunk is converted to a root one. This reproduces the bug in 037 * HBASE-6871. We write a carefully selected number of relatively large keys so that we accumulate 038 * a leaf index chunk that only goes over the configured index chunk size after adding the last 039 * key/value. The bug is in that when we close the file, we convert that inline (leaf-level) chunk 040 * into a root chunk, but then look at the size of that root chunk, find that it is greater than 041 * the configured chunk size, and split it into a number of intermediate index blocks that should 042 * really be leaf-level blocks. If more keys were added, we would flush the leaf-level block, add 043 * another entry to the root-level block, and that would prevent us from upgrading the leaf-level 044 * chunk to the root chunk, thus not triggering the bug. 045 */ 046@Category({IOTests.class, SmallTests.class}) 047public class TestHFileInlineToRootChunkConversion { 048 049 @ClassRule 050 public static final HBaseClassTestRule CLASS_RULE = 051 HBaseClassTestRule.forClass(TestHFileInlineToRootChunkConversion.class); 052 053 private final HBaseTestingUtility testUtil = new HBaseTestingUtility(); 054 private final Configuration conf = testUtil.getConfiguration(); 055 056 @Test 057 public void testWriteHFile() throws Exception { 058 Path hfPath = new Path(testUtil.getDataTestDir(), 059 TestHFileInlineToRootChunkConversion.class.getSimpleName() + ".hfile"); 060 int maxChunkSize = 1024; 061 FileSystem fs = FileSystem.get(conf); 062 CacheConfig cacheConf = new CacheConfig(conf); 063 conf.setInt(HFileBlockIndex.MAX_CHUNK_SIZE_KEY, maxChunkSize); 064 HFileContext context = new HFileContextBuilder().withBlockSize(16).build(); 065 HFile.Writer hfw = new HFile.WriterFactory(conf, cacheConf) 066 .withFileContext(context) 067 .withPath(fs, hfPath).create(); 068 List<byte[]> keys = new ArrayList<>(); 069 StringBuilder sb = new StringBuilder(); 070 071 for (int i = 0; i < 4; ++i) { 072 sb.append("key" + String.format("%05d", i)); 073 sb.append("_"); 074 for (int j = 0; j < 100; ++j) { 075 sb.append('0' + j); 076 } 077 String keyStr = sb.toString(); 078 sb.setLength(0); 079 080 byte[] k = Bytes.toBytes(keyStr); 081 keys.add(k); 082 byte[] v = Bytes.toBytes("value" + i); 083 hfw.append(CellUtil.createCell(k, v)); 084 } 085 hfw.close(); 086 087 HFile.Reader reader = HFile.createReader(fs, hfPath, cacheConf, true, conf); 088 // Scanner doesn't do Cells yet. Fix. 089 HFileScanner scanner = reader.getScanner(true, true); 090 for (int i = 0; i < keys.size(); ++i) { 091 scanner.seekTo(CellUtil.createCell(keys.get(i))); 092 } 093 reader.close(); 094 } 095}