001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.io.hfile;
019
020import static org.apache.hadoop.hbase.io.ByteBuffAllocator.MIN_ALLOCATE_SIZE_KEY;
021import static org.junit.jupiter.api.Assertions.assertArrayEquals;
022import static org.junit.jupiter.api.Assertions.assertEquals;
023import static org.junit.jupiter.api.Assertions.assertTrue;
024
025import java.io.IOException;
026import java.nio.ByteBuffer;
027import java.util.ArrayList;
028import java.util.List;
029import org.apache.hadoop.conf.Configuration;
030import org.apache.hadoop.fs.FileSystem;
031import org.apache.hadoop.fs.Path;
032import org.apache.hadoop.hbase.CellComparatorImpl;
033import org.apache.hadoop.hbase.CellUtil;
034import org.apache.hadoop.hbase.HBaseTestingUtil;
035import org.apache.hadoop.hbase.KeyValue;
036import org.apache.hadoop.hbase.SizeCachedNoTagsByteBufferKeyValue;
037import org.apache.hadoop.hbase.SizeCachedNoTagsKeyValue;
038import org.apache.hadoop.hbase.io.ByteBuffAllocator;
039import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
040import org.apache.hadoop.hbase.testclassification.IOTests;
041import org.apache.hadoop.hbase.testclassification.MediumTests;
042import org.apache.hadoop.hbase.util.Bytes;
043import org.junit.jupiter.api.BeforeEach;
044import org.junit.jupiter.api.Tag;
045import org.junit.jupiter.api.Test;
046
047@Tag(IOTests.TAG)
048@Tag(MediumTests.TAG)
049public class TestRowIndexV1RoundTrip {
050  private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
051  private static final DataBlockEncoding DATA_BLOCK_ENCODING = DataBlockEncoding.ROW_INDEX_V1;
052  private static final int ENTRY_COUNT = 100;
053
054  private Configuration conf;
055  private FileSystem fs;
056
057  @BeforeEach
058  public void setUp() throws IOException {
059    conf = TEST_UTIL.getConfiguration();
060    conf.setLong(MIN_ALLOCATE_SIZE_KEY, 0);
061    fs = FileSystem.get(conf);
062  }
063
064  @Test
065  public void testReadMyWritesOnHeap() throws IOException {
066    Path hfilePath = new Path(TEST_UTIL.getDataTestDir(), "testHFileFormatV3");
067    writeDataToHFile(hfilePath, ENTRY_COUNT);
068    readDataFromHFile(hfilePath, ENTRY_COUNT, true);
069  }
070
071  @Test
072  public void testReadMyWritesOnDirectMem() throws IOException {
073    Path hfilePath = new Path(TEST_UTIL.getDataTestDir(), "testHFileFormatV3");
074    writeDataToHFile(hfilePath, ENTRY_COUNT);
075    readDataFromHFile(hfilePath, ENTRY_COUNT, false);
076  }
077
078  private void writeDataToHFile(Path hfilePath, int entryCount) throws IOException {
079    HFileContext context =
080      new HFileContextBuilder().withBlockSize(1024).withDataBlockEncoding(DATA_BLOCK_ENCODING)
081        .withCellComparator(CellComparatorImpl.COMPARATOR).build();
082    CacheConfig cacheConfig = new CacheConfig(conf);
083    HFile.Writer writer = new HFile.WriterFactory(conf, cacheConfig).withPath(fs, hfilePath)
084      .withFileContext(context).create();
085
086    List<KeyValue> keyValues = new ArrayList<>(entryCount);
087
088    writeKeyValues(entryCount, writer, keyValues);
089  }
090
091  private void writeKeyValues(int entryCount, HFile.Writer writer, List<KeyValue> keyValues)
092    throws IOException {
093    for (int i = 0; i < entryCount; ++i) {
094      byte[] keyBytes = intToBytes(i);
095
096      byte[] valueBytes = Bytes.toBytes(String.format("value %d", i));
097      KeyValue keyValue = new KeyValue(keyBytes, null, null, valueBytes);
098
099      writer.append(keyValue);
100      keyValues.add(keyValue);
101    }
102    writer.close();
103  }
104
105  private void readDataFromHFile(Path hfilePath, int entryCount, boolean onHeap)
106    throws IOException {
107    CacheConfig cacheConfig;
108    if (onHeap) {
109      cacheConfig = new CacheConfig(conf);
110    } else {
111      ByteBuffAllocator allocator = ByteBuffAllocator.create(conf, true);
112      cacheConfig = new CacheConfig(conf, null, null, allocator);
113    }
114    HFile.Reader reader = HFile.createReader(fs, hfilePath, cacheConfig, false, conf);
115    HFileScanner scanner = reader.getScanner(conf, false, false);
116    scanner.seekTo();
117    int i = 1;
118    while (scanner.next()) {
119      byte[] keyBytes = intToBytes(i);
120      // check row key from getKey() and getCell() separately because they use different code paths
121      assertArrayEquals(keyBytes, CellUtil.cloneRow(scanner.getKey()));
122      assertArrayEquals(keyBytes, CellUtil.cloneRow(scanner.getCell()));
123      assertArrayEquals(Bytes.toBytes(String.format("value %d", i)),
124        CellUtil.cloneValue(scanner.getCell()));
125      if (onHeap) {
126        assertTrue(scanner.getCell() instanceof SizeCachedNoTagsKeyValue);
127      } else {
128        assertTrue(scanner.getCell() instanceof SizeCachedNoTagsByteBufferKeyValue);
129      }
130      i += 1;
131    }
132    assertEquals(entryCount, i);
133  }
134
135  private byte[] intToBytes(final int i) {
136    ByteBuffer bb = ByteBuffer.allocate(4);
137    bb.putInt(i);
138    return bb.array();
139  }
140}