001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.io.hfile;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertTrue;
022import static org.junit.Assert.fail;
023
024import java.io.DataOutputStream;
025import java.io.IOException;
026import java.nio.ByteBuffer;
027import java.util.ArrayList;
028import java.util.Collection;
029import java.util.List;
030
031import org.apache.hadoop.conf.Configuration;
032import org.apache.hadoop.hbase.Cell;
033import org.apache.hadoop.hbase.HBaseClassTestRule;
034import org.apache.hadoop.hbase.HBaseConfiguration;
035import org.apache.hadoop.hbase.HConstants;
036import org.apache.hadoop.hbase.KeyValue;
037import org.apache.hadoop.hbase.io.ByteArrayOutputStream;
038import org.apache.hadoop.hbase.io.ByteBuffAllocator;
039import org.apache.hadoop.hbase.io.HeapSize;
040import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
041import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
042import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultEncodingContext;
043import org.apache.hadoop.hbase.io.encoding.HFileBlockEncodingContext;
044import org.apache.hadoop.hbase.nio.ByteBuff;
045import org.apache.hadoop.hbase.testclassification.IOTests;
046import org.apache.hadoop.hbase.testclassification.MediumTests;
047import org.apache.hadoop.hbase.util.ChecksumType;
048import org.apache.hadoop.hbase.util.RedundantKVGenerator;
049import org.junit.ClassRule;
050import org.junit.Test;
051import org.junit.experimental.categories.Category;
052import org.junit.runner.RunWith;
053import org.junit.runners.Parameterized;
054import org.junit.runners.Parameterized.Parameters;
055
056@RunWith(Parameterized.class)
057@Category({IOTests.class, MediumTests.class})
058public class TestHFileDataBlockEncoder {
059
060  @ClassRule
061  public static final HBaseClassTestRule CLASS_RULE =
062      HBaseClassTestRule.forClass(TestHFileDataBlockEncoder.class);
063
064  private final Configuration conf = HBaseConfiguration.create();
065  private final RedundantKVGenerator generator = new RedundantKVGenerator();
066  private HFileDataBlockEncoder blockEncoder;
067  private boolean includesMemstoreTS;
068
069  /**
070   * Create test for given data block encoding configuration.
071   * @param blockEncoder What kind of encoding policy will be used.
072   */
073  public TestHFileDataBlockEncoder(HFileDataBlockEncoder blockEncoder,
074      boolean includesMemstoreTS) {
075    this.blockEncoder = blockEncoder;
076    this.includesMemstoreTS = includesMemstoreTS;
077    System.err.println("Encoding: " + blockEncoder.getDataBlockEncoding()
078        + ", includesMemstoreTS: " + includesMemstoreTS);
079  }
080
081  /**
082   * Test putting and taking out blocks into cache with different
083   * encoding options.
084   */
085  @Test
086  public void testEncodingWithCache() throws IOException {
087    testEncodingWithCacheInternals(false);
088    testEncodingWithCacheInternals(true);
089  }
090
091  private void testEncodingWithCacheInternals(boolean useTag) throws IOException {
092    List<KeyValue> kvs = generator.generateTestKeyValues(60, useTag);
093    HFileBlock block = getSampleHFileBlock(kvs, useTag);
094    HFileBlock cacheBlock = createBlockOnDisk(conf, kvs, block, useTag);
095
096    LruBlockCache blockCache =
097        new LruBlockCache(8 * 1024 * 1024, 32 * 1024);
098    BlockCacheKey cacheKey = new BlockCacheKey("test", 0);
099    blockCache.cacheBlock(cacheKey, cacheBlock);
100
101    HeapSize heapSize = blockCache.getBlock(cacheKey, false, false, true);
102    assertTrue(heapSize instanceof HFileBlock);
103
104    HFileBlock returnedBlock = (HFileBlock) heapSize;
105
106    if (blockEncoder.getDataBlockEncoding() ==
107        DataBlockEncoding.NONE) {
108      assertEquals(block.getBufferReadOnly(), returnedBlock.getBufferReadOnly());
109    } else {
110      if (BlockType.ENCODED_DATA != returnedBlock.getBlockType()) {
111        System.out.println(blockEncoder);
112      }
113      assertEquals(BlockType.ENCODED_DATA, returnedBlock.getBlockType());
114    }
115  }
116
117  /** Test for HBASE-5746. */
118  @Test
119  public void testHeaderSizeInCacheWithoutChecksum() throws Exception {
120    testHeaderSizeInCacheWithoutChecksumInternals(false);
121    testHeaderSizeInCacheWithoutChecksumInternals(true);
122  }
123
124  private void testHeaderSizeInCacheWithoutChecksumInternals(boolean useTags) throws IOException {
125    int headerSize = HConstants.HFILEBLOCK_HEADER_SIZE_NO_CHECKSUM;
126    // Create some KVs and create the block with old-style header.
127    List<KeyValue> kvs = generator.generateTestKeyValues(60, useTags);
128    ByteBuffer keyValues = RedundantKVGenerator.convertKvToByteBuffer(kvs, includesMemstoreTS);
129    int size = keyValues.limit();
130    ByteBuffer buf = ByteBuffer.allocate(size + headerSize);
131    buf.position(headerSize);
132    keyValues.rewind();
133    buf.put(keyValues);
134    HFileContext hfileContext = new HFileContextBuilder().withHBaseCheckSum(false)
135                        .withIncludesMvcc(includesMemstoreTS)
136                        .withIncludesTags(useTags)
137                        .withBlockSize(0)
138                        .withChecksumType(ChecksumType.NULL)
139                        .build();
140    HFileBlock block = new HFileBlock(BlockType.DATA, size, size, -1, ByteBuff.wrap(buf),
141        HFileBlock.FILL_HEADER, 0, 0, -1, hfileContext, ByteBuffAllocator.HEAP);
142    HFileBlock cacheBlock = createBlockOnDisk(conf, kvs, block, useTags);
143    assertEquals(headerSize, cacheBlock.getDummyHeaderForVersion().length);
144  }
145
146  /**
147   * Test encoding.
148   * @throws IOException
149   */
150  @Test
151  public void testEncoding() throws IOException {
152    testEncodingInternals(false);
153    testEncodingInternals(true);
154  }
155
156  /**
157   * Test encoding with offheap keyvalue. This test just verifies if the encoders
158   * work with DBB and does not use the getXXXArray() API
159   * @throws IOException
160   */
161  @Test
162  public void testEncodingWithOffheapKeyValue() throws IOException {
163    // usually we have just block without headers, but don't complicate that
164    try {
165      List<Cell> kvs = generator.generateTestExtendedOffheapKeyValues(60, true);
166      HFileContext meta = new HFileContextBuilder().withIncludesMvcc(includesMemstoreTS)
167          .withIncludesTags(true).withHBaseCheckSum(true).withCompression(Algorithm.NONE)
168          .withBlockSize(0).withChecksumType(ChecksumType.NULL).build();
169      writeBlock(conf, kvs, meta, true);
170    } catch (IllegalArgumentException e) {
171      fail("No exception should have been thrown");
172    }
173  }
174
175  private void testEncodingInternals(boolean useTag) throws IOException {
176    // usually we have just block without headers, but don't complicate that
177    List<KeyValue> kvs = generator.generateTestKeyValues(60, useTag);
178    HFileBlock block = getSampleHFileBlock(kvs, useTag);
179    HFileBlock blockOnDisk = createBlockOnDisk(conf, kvs, block, useTag);
180
181    if (blockEncoder.getDataBlockEncoding() !=
182        DataBlockEncoding.NONE) {
183      assertEquals(BlockType.ENCODED_DATA, blockOnDisk.getBlockType());
184      assertEquals(blockEncoder.getDataBlockEncoding().getId(),
185          blockOnDisk.getDataBlockEncodingId());
186    } else {
187      assertEquals(BlockType.DATA, blockOnDisk.getBlockType());
188    }
189  }
190
191  private HFileBlock getSampleHFileBlock(List<KeyValue> kvs, boolean useTag) {
192    ByteBuffer keyValues = RedundantKVGenerator.convertKvToByteBuffer(kvs, includesMemstoreTS);
193    int size = keyValues.limit();
194    ByteBuffer buf = ByteBuffer.allocate(size + HConstants.HFILEBLOCK_HEADER_SIZE);
195    buf.position(HConstants.HFILEBLOCK_HEADER_SIZE);
196    keyValues.rewind();
197    buf.put(keyValues);
198    HFileContext meta = new HFileContextBuilder()
199                        .withIncludesMvcc(includesMemstoreTS)
200                        .withIncludesTags(useTag)
201                        .withHBaseCheckSum(true)
202                        .withCompression(Algorithm.NONE)
203                        .withBlockSize(0)
204                        .withChecksumType(ChecksumType.NULL)
205                        .build();
206    HFileBlock b = new HFileBlock(BlockType.DATA, size, size, -1, ByteBuff.wrap(buf),
207        HFileBlock.FILL_HEADER, 0, 0, -1, meta, ByteBuffAllocator.HEAP);
208    return b;
209  }
210
211  private HFileBlock createBlockOnDisk(Configuration conf, List<KeyValue> kvs, HFileBlock block,
212      boolean useTags) throws IOException {
213    int size;
214    HFileBlockEncodingContext context = new HFileBlockDefaultEncodingContext(conf,
215        blockEncoder.getDataBlockEncoding(), HConstants.HFILEBLOCK_DUMMY_HEADER,
216        block.getHFileContext());
217
218    ByteArrayOutputStream baos = new ByteArrayOutputStream();
219    baos.write(block.getDummyHeaderForVersion());
220    DataOutputStream dos = new DataOutputStream(baos);
221    blockEncoder.startBlockEncoding(context, dos);
222    for (KeyValue kv : kvs) {
223      blockEncoder.encode(kv, context, dos);
224    }
225    blockEncoder.endBlockEncoding(context, dos, baos.getBuffer(), BlockType.DATA);
226    byte[] encodedBytes = baos.toByteArray();
227    size = encodedBytes.length - block.getDummyHeaderForVersion().length;
228    return new HFileBlock(context.getBlockType(), size, size, -1,
229        ByteBuff.wrap(ByteBuffer.wrap(encodedBytes)), HFileBlock.FILL_HEADER, 0,
230        block.getOnDiskDataSizeWithHeader(), -1, block.getHFileContext(), ByteBuffAllocator.HEAP);
231  }
232
233  private void writeBlock(Configuration conf, List<Cell> kvs, HFileContext fileContext,
234      boolean useTags) throws IOException {
235    HFileBlockEncodingContext context = new HFileBlockDefaultEncodingContext(conf,
236        blockEncoder.getDataBlockEncoding(), HConstants.HFILEBLOCK_DUMMY_HEADER,
237        fileContext);
238
239    ByteArrayOutputStream baos = new ByteArrayOutputStream();
240    baos.write(HConstants.HFILEBLOCK_DUMMY_HEADER);
241    DataOutputStream dos = new DataOutputStream(baos);
242    blockEncoder.startBlockEncoding(context, dos);
243    for (Cell kv : kvs) {
244      blockEncoder.encode(kv, context, dos);
245    }
246  }
247
248  /**
249   * @return All possible data block encoding configurations
250   */
251  @Parameters
252  public static Collection<Object[]> getAllConfigurations() {
253    List<Object[]> configurations = new ArrayList<>();
254
255    for (DataBlockEncoding diskAlgo : DataBlockEncoding.values()) {
256      for (boolean includesMemstoreTS : new boolean[] { false, true }) {
257        HFileDataBlockEncoder dbe = (diskAlgo == DataBlockEncoding.NONE) ?
258            NoOpDataBlockEncoder.INSTANCE : new HFileDataBlockEncoderImpl(diskAlgo);
259        configurations.add(new Object[] { dbe, new Boolean(includesMemstoreTS) });
260      }
261    }
262
263    return configurations;
264  }
265}