001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.io.hfile; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertTrue; 022import static org.junit.Assert.fail; 023 024import java.io.DataOutputStream; 025import java.io.IOException; 026import java.nio.ByteBuffer; 027import java.util.ArrayList; 028import java.util.Collection; 029import java.util.List; 030import org.apache.hadoop.hbase.Cell; 031import org.apache.hadoop.hbase.HBaseClassTestRule; 032import org.apache.hadoop.hbase.HConstants; 033import org.apache.hadoop.hbase.KeyValue; 034import org.apache.hadoop.hbase.io.ByteArrayOutputStream; 035import org.apache.hadoop.hbase.io.ByteBuffAllocator; 036import org.apache.hadoop.hbase.io.HeapSize; 037import org.apache.hadoop.hbase.io.compress.Compression.Algorithm; 038import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; 039import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultEncodingContext; 040import org.apache.hadoop.hbase.io.encoding.HFileBlockEncodingContext; 041import org.apache.hadoop.hbase.nio.ByteBuff; 042import org.apache.hadoop.hbase.testclassification.IOTests; 043import org.apache.hadoop.hbase.testclassification.MediumTests; 044import org.apache.hadoop.hbase.util.ChecksumType; 045import org.apache.hadoop.hbase.util.RedundantKVGenerator; 046import org.junit.ClassRule; 047import org.junit.Test; 048import org.junit.experimental.categories.Category; 049import org.junit.runner.RunWith; 050import org.junit.runners.Parameterized; 051import org.junit.runners.Parameterized.Parameters; 052 053@RunWith(Parameterized.class) 054@Category({IOTests.class, MediumTests.class}) 055public class TestHFileDataBlockEncoder { 056 057 @ClassRule 058 public static final HBaseClassTestRule CLASS_RULE = 059 HBaseClassTestRule.forClass(TestHFileDataBlockEncoder.class); 060 061 private HFileDataBlockEncoder blockEncoder; 062 private RedundantKVGenerator generator = new RedundantKVGenerator(); 063 private boolean includesMemstoreTS; 064 065 /** 066 * Create test for given data block encoding configuration. 067 * @param blockEncoder What kind of encoding policy will be used. 068 */ 069 public TestHFileDataBlockEncoder(HFileDataBlockEncoder blockEncoder, 070 boolean includesMemstoreTS) { 071 this.blockEncoder = blockEncoder; 072 this.includesMemstoreTS = includesMemstoreTS; 073 System.err.println("Encoding: " + blockEncoder.getDataBlockEncoding() 074 + ", includesMemstoreTS: " + includesMemstoreTS); 075 } 076 077 /** 078 * Test putting and taking out blocks into cache with different 079 * encoding options. 080 */ 081 @Test 082 public void testEncodingWithCache() throws IOException { 083 testEncodingWithCacheInternals(false); 084 testEncodingWithCacheInternals(true); 085 } 086 087 private void testEncodingWithCacheInternals(boolean useTag) throws IOException { 088 List<KeyValue> kvs = generator.generateTestKeyValues(60, useTag); 089 HFileBlock block = getSampleHFileBlock(kvs, useTag); 090 HFileBlock cacheBlock = createBlockOnDisk(kvs, block, useTag); 091 092 LruBlockCache blockCache = 093 new LruBlockCache(8 * 1024 * 1024, 32 * 1024); 094 BlockCacheKey cacheKey = new BlockCacheKey("test", 0); 095 blockCache.cacheBlock(cacheKey, cacheBlock); 096 097 HeapSize heapSize = blockCache.getBlock(cacheKey, false, false, true); 098 assertTrue(heapSize instanceof HFileBlock); 099 100 HFileBlock returnedBlock = (HFileBlock) heapSize;; 101 102 if (blockEncoder.getDataBlockEncoding() == 103 DataBlockEncoding.NONE) { 104 assertEquals(block.getBufferReadOnly(), returnedBlock.getBufferReadOnly()); 105 } else { 106 if (BlockType.ENCODED_DATA != returnedBlock.getBlockType()) { 107 System.out.println(blockEncoder); 108 } 109 assertEquals(BlockType.ENCODED_DATA, returnedBlock.getBlockType()); 110 } 111 } 112 113 /** Test for HBASE-5746. */ 114 @Test 115 public void testHeaderSizeInCacheWithoutChecksum() throws Exception { 116 testHeaderSizeInCacheWithoutChecksumInternals(false); 117 testHeaderSizeInCacheWithoutChecksumInternals(true); 118 } 119 120 private void testHeaderSizeInCacheWithoutChecksumInternals(boolean useTags) throws IOException { 121 int headerSize = HConstants.HFILEBLOCK_HEADER_SIZE_NO_CHECKSUM; 122 // Create some KVs and create the block with old-style header. 123 List<KeyValue> kvs = generator.generateTestKeyValues(60, useTags); 124 ByteBuffer keyValues = RedundantKVGenerator.convertKvToByteBuffer(kvs, includesMemstoreTS); 125 int size = keyValues.limit(); 126 ByteBuffer buf = ByteBuffer.allocate(size + headerSize); 127 buf.position(headerSize); 128 keyValues.rewind(); 129 buf.put(keyValues); 130 HFileContext hfileContext = new HFileContextBuilder().withHBaseCheckSum(false) 131 .withIncludesMvcc(includesMemstoreTS) 132 .withIncludesTags(useTags) 133 .withBlockSize(0) 134 .withChecksumType(ChecksumType.NULL) 135 .build(); 136 HFileBlock block = new HFileBlock(BlockType.DATA, size, size, -1, ByteBuff.wrap(buf), 137 HFileBlock.FILL_HEADER, 0, 0, -1, hfileContext, ByteBuffAllocator.HEAP); 138 HFileBlock cacheBlock = createBlockOnDisk(kvs, block, useTags); 139 assertEquals(headerSize, cacheBlock.getDummyHeaderForVersion().length); 140 } 141 142 /** 143 * Test encoding. 144 * @throws IOException 145 */ 146 @Test 147 public void testEncoding() throws IOException { 148 testEncodingInternals(false); 149 testEncodingInternals(true); 150 } 151 152 /** 153 * Test encoding with offheap keyvalue. This test just verifies if the encoders 154 * work with DBB and does not use the getXXXArray() API 155 * @throws IOException 156 */ 157 @Test 158 public void testEncodingWithOffheapKeyValue() throws IOException { 159 // usually we have just block without headers, but don't complicate that 160 try { 161 List<Cell> kvs = generator.generateTestExtendedOffheapKeyValues(60, true); 162 HFileContext meta = new HFileContextBuilder().withIncludesMvcc(includesMemstoreTS) 163 .withIncludesTags(true).withHBaseCheckSum(true).withCompression(Algorithm.NONE) 164 .withBlockSize(0).withChecksumType(ChecksumType.NULL).build(); 165 writeBlock(kvs, meta, true); 166 } catch (IllegalArgumentException e) { 167 fail("No exception should have been thrown"); 168 } 169 } 170 171 private void testEncodingInternals(boolean useTag) throws IOException { 172 // usually we have just block without headers, but don't complicate that 173 List<KeyValue> kvs = generator.generateTestKeyValues(60, useTag); 174 HFileBlock block = getSampleHFileBlock(kvs, useTag); 175 HFileBlock blockOnDisk = createBlockOnDisk(kvs, block, useTag); 176 177 if (blockEncoder.getDataBlockEncoding() != 178 DataBlockEncoding.NONE) { 179 assertEquals(BlockType.ENCODED_DATA, blockOnDisk.getBlockType()); 180 assertEquals(blockEncoder.getDataBlockEncoding().getId(), 181 blockOnDisk.getDataBlockEncodingId()); 182 } else { 183 assertEquals(BlockType.DATA, blockOnDisk.getBlockType()); 184 } 185 } 186 187 private HFileBlock getSampleHFileBlock(List<KeyValue> kvs, boolean useTag) { 188 ByteBuffer keyValues = RedundantKVGenerator.convertKvToByteBuffer(kvs, includesMemstoreTS); 189 int size = keyValues.limit(); 190 ByteBuffer buf = ByteBuffer.allocate(size + HConstants.HFILEBLOCK_HEADER_SIZE); 191 buf.position(HConstants.HFILEBLOCK_HEADER_SIZE); 192 keyValues.rewind(); 193 buf.put(keyValues); 194 HFileContext meta = new HFileContextBuilder() 195 .withIncludesMvcc(includesMemstoreTS) 196 .withIncludesTags(useTag) 197 .withHBaseCheckSum(true) 198 .withCompression(Algorithm.NONE) 199 .withBlockSize(0) 200 .withChecksumType(ChecksumType.NULL) 201 .build(); 202 HFileBlock b = new HFileBlock(BlockType.DATA, size, size, -1, ByteBuff.wrap(buf), 203 HFileBlock.FILL_HEADER, 0, 0, -1, meta, ByteBuffAllocator.HEAP); 204 return b; 205 } 206 207 private HFileBlock createBlockOnDisk(List<KeyValue> kvs, HFileBlock block, boolean useTags) 208 throws IOException { 209 int size; 210 HFileBlockEncodingContext context = new HFileBlockDefaultEncodingContext( 211 blockEncoder.getDataBlockEncoding(), HConstants.HFILEBLOCK_DUMMY_HEADER, 212 block.getHFileContext()); 213 214 ByteArrayOutputStream baos = new ByteArrayOutputStream(); 215 baos.write(block.getDummyHeaderForVersion()); 216 DataOutputStream dos = new DataOutputStream(baos); 217 blockEncoder.startBlockEncoding(context, dos); 218 for (KeyValue kv : kvs) { 219 blockEncoder.encode(kv, context, dos); 220 } 221 blockEncoder.endBlockEncoding(context, dos, baos.getBuffer(), BlockType.DATA); 222 byte[] encodedBytes = baos.toByteArray(); 223 size = encodedBytes.length - block.getDummyHeaderForVersion().length; 224 return new HFileBlock(context.getBlockType(), size, size, -1, 225 ByteBuff.wrap(ByteBuffer.wrap(encodedBytes)), HFileBlock.FILL_HEADER, 0, 226 block.getOnDiskDataSizeWithHeader(), -1, block.getHFileContext(), ByteBuffAllocator.HEAP); 227 } 228 229 private void writeBlock(List<Cell> kvs, HFileContext fileContext, boolean useTags) 230 throws IOException { 231 HFileBlockEncodingContext context = new HFileBlockDefaultEncodingContext( 232 blockEncoder.getDataBlockEncoding(), HConstants.HFILEBLOCK_DUMMY_HEADER, 233 fileContext); 234 235 ByteArrayOutputStream baos = new ByteArrayOutputStream(); 236 baos.write(HConstants.HFILEBLOCK_DUMMY_HEADER); 237 DataOutputStream dos = new DataOutputStream(baos); 238 blockEncoder.startBlockEncoding(context, dos); 239 for (Cell kv : kvs) { 240 blockEncoder.encode(kv, context, dos); 241 } 242 } 243 244 /** 245 * @return All possible data block encoding configurations 246 */ 247 @Parameters 248 public static Collection<Object[]> getAllConfigurations() { 249 List<Object[]> configurations = new ArrayList<>(); 250 251 for (DataBlockEncoding diskAlgo : DataBlockEncoding.values()) { 252 for (boolean includesMemstoreTS : new boolean[] { false, true }) { 253 HFileDataBlockEncoder dbe = (diskAlgo == DataBlockEncoding.NONE) ? 254 NoOpDataBlockEncoder.INSTANCE : new HFileDataBlockEncoderImpl(diskAlgo); 255 configurations.add(new Object[] { dbe, new Boolean(includesMemstoreTS) }); 256 } 257 } 258 259 return configurations; 260 } 261}