001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.io.encoding;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertTrue;
022
023import java.io.IOException;
024import java.util.ArrayList;
025import java.util.List;
026import java.util.Map;
027import java.util.stream.Stream;
028import org.apache.hadoop.hbase.ArrayBackedTag;
029import org.apache.hadoop.hbase.HBaseParameterizedTestTemplate;
030import org.apache.hadoop.hbase.HBaseTestingUtil;
031import org.apache.hadoop.hbase.HConstants;
032import org.apache.hadoop.hbase.KeyValue;
033import org.apache.hadoop.hbase.Tag;
034import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
035import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
036import org.apache.hadoop.hbase.client.Durability;
037import org.apache.hadoop.hbase.client.Get;
038import org.apache.hadoop.hbase.client.Put;
039import org.apache.hadoop.hbase.client.Result;
040import org.apache.hadoop.hbase.io.hfile.BlockCacheFactory;
041import org.apache.hadoop.hbase.io.hfile.HFile;
042import org.apache.hadoop.hbase.io.hfile.LruBlockCache;
043import org.apache.hadoop.hbase.regionserver.BloomType;
044import org.apache.hadoop.hbase.regionserver.HRegion;
045import org.apache.hadoop.hbase.regionserver.Region;
046import org.apache.hadoop.hbase.testclassification.IOTests;
047import org.apache.hadoop.hbase.testclassification.LargeTests;
048import org.apache.hadoop.hbase.util.Bytes;
049import org.apache.hadoop.hbase.util.LoadTestKVGenerator;
050import org.apache.hadoop.hbase.util.Strings;
051import org.junit.jupiter.api.TestTemplate;
052import org.junit.jupiter.params.provider.Arguments;
053
054/**
055 * Tests encoded seekers by loading and reading values.
056 */
057@org.junit.jupiter.api.Tag(IOTests.TAG)
058@org.junit.jupiter.api.Tag(LargeTests.TAG)
059@HBaseParameterizedTestTemplate
060public class TestEncodedSeekers {
061
062  private static final String TABLE_NAME = "encodedSeekersTable";
063  private static final String CF_NAME = "encodedSeekersCF";
064  private static final byte[] CF_BYTES = Bytes.toBytes(CF_NAME);
065  private static final int MAX_VERSIONS = 5;
066
067  private static final int BLOCK_SIZE = 64 * 1024;
068  private static final int MIN_VALUE_SIZE = 30;
069  private static final int MAX_VALUE_SIZE = 60;
070  private static final int NUM_ROWS = 1003;
071  private static final int NUM_COLS_PER_ROW = 20;
072  private static final int NUM_HFILES = 4;
073  private static final int NUM_ROWS_PER_FLUSH = NUM_ROWS / NUM_HFILES;
074
075  private final HBaseTestingUtil testUtil = new HBaseTestingUtil();
076  private final DataBlockEncoding encoding;
077  private final boolean includeTags;
078  private final boolean compressTags;
079
080  /** Enable when debugging */
081  private static final boolean VERBOSE = false;
082
083  public static Stream<Arguments> parameters() {
084    List<Arguments> params = new ArrayList<>();
085    for (DataBlockEncoding encoding : DataBlockEncoding.values()) {
086      for (boolean includeTags : new boolean[] { false, true }) {
087        for (boolean compressTags : new boolean[] { false, true }) {
088          params.add(Arguments.of(encoding, includeTags, compressTags));
089        }
090      }
091    }
092    return params.stream();
093  }
094
095  public TestEncodedSeekers(DataBlockEncoding encoding, boolean includeTags, boolean compressTags) {
096    this.encoding = encoding;
097    this.includeTags = includeTags;
098    this.compressTags = compressTags;
099  }
100
101  @TestTemplate
102  public void testEncodedSeeker() throws IOException {
103    System.err.println("Testing encoded seekers for encoding : " + encoding + ", includeTags : "
104      + includeTags + ", compressTags : " + compressTags);
105    if (includeTags) {
106      testUtil.getConfiguration().setInt(HFile.FORMAT_VERSION_KEY, 3);
107    }
108
109    LruBlockCache cache =
110      (LruBlockCache) BlockCacheFactory.createBlockCache(testUtil.getConfiguration());
111    // Need to disable default row bloom filter for this test to pass.
112    ColumnFamilyDescriptor cfd = ColumnFamilyDescriptorBuilder.newBuilder(CF_BYTES)
113      .setMaxVersions(MAX_VERSIONS).setDataBlockEncoding(encoding).setBlocksize(BLOCK_SIZE)
114      .setBloomFilterType(BloomType.NONE).setCompressTags(compressTags).build();
115    HRegion region = testUtil.createTestRegion(TABLE_NAME, cfd, cache);
116
117    // write the data, but leave some in the memstore
118    doPuts(region);
119
120    // verify correctness when memstore contains data
121    doGets(region);
122
123    // verify correctness again after compacting
124    region.compact(false);
125    doGets(region);
126
127    Map<DataBlockEncoding, Integer> encodingCounts = cache.getEncodingCountsForTest();
128
129    // Ensure that compactions don't pollute the cache with unencoded blocks
130    // in case of in-cache-only encoding.
131    System.err.println("encodingCounts=" + encodingCounts);
132    assertEquals(1, encodingCounts.size());
133    DataBlockEncoding encodingInCache = encodingCounts.keySet().iterator().next();
134    assertEquals(encoding, encodingInCache);
135    assertTrue(encodingCounts.get(encodingInCache) > 0);
136  }
137
138  private void doPuts(HRegion region) throws IOException {
139    LoadTestKVGenerator dataGenerator = new LoadTestKVGenerator(MIN_VALUE_SIZE, MAX_VALUE_SIZE);
140    for (int i = 0; i < NUM_ROWS; ++i) {
141      byte[] key = Bytes.toBytes(LoadTestKVGenerator.md5PrefixedKey(i));
142      for (int j = 0; j < NUM_COLS_PER_ROW; ++j) {
143        Put put = new Put(key);
144        put.setDurability(Durability.ASYNC_WAL);
145        byte[] col = Bytes.toBytes(String.valueOf(j));
146        byte[] value = dataGenerator.generateRandomSizeValue(key, col);
147        if (includeTags) {
148          Tag[] tag = new Tag[1];
149          tag[0] = new ArrayBackedTag((byte) 1, "Visibility");
150          KeyValue kv = new KeyValue(key, CF_BYTES, col, HConstants.LATEST_TIMESTAMP, value, tag);
151          put.add(kv);
152        } else {
153          put.addColumn(CF_BYTES, col, value);
154        }
155        if (VERBOSE) {
156          KeyValue kvPut = new KeyValue(key, CF_BYTES, col, value);
157          System.err.println(Strings.padFront(i + "", ' ', 4) + " " + kvPut);
158        }
159        region.put(put);
160      }
161      if (i % NUM_ROWS_PER_FLUSH == 0) {
162        region.flush(true);
163      }
164    }
165  }
166
167  private void doGets(Region region) throws IOException {
168    for (int i = 0; i < NUM_ROWS; ++i) {
169      final byte[] rowKey = Bytes.toBytes(LoadTestKVGenerator.md5PrefixedKey(i));
170      for (int j = 0; j < NUM_COLS_PER_ROW; ++j) {
171        final String qualStr = String.valueOf(j);
172        if (VERBOSE) {
173          System.err.println(
174            "Reading row " + i + ", column " + j + " " + Bytes.toString(rowKey) + "/" + qualStr);
175        }
176        final byte[] qualBytes = Bytes.toBytes(qualStr);
177        Get get = new Get(rowKey);
178        get.addColumn(CF_BYTES, qualBytes);
179        Result result = region.get(get);
180        assertEquals(1, result.size());
181        byte[] value = result.getValue(CF_BYTES, qualBytes);
182        assertTrue(LoadTestKVGenerator.verify(value, rowKey, qualBytes));
183      }
184    }
185  }
186}