001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.io.encoding;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertTrue;
022
023import java.io.IOException;
024import java.util.ArrayList;
025import java.util.Collection;
026import java.util.List;
027import java.util.Map;
028import org.apache.hadoop.hbase.ArrayBackedTag;
029import org.apache.hadoop.hbase.HBaseClassTestRule;
030import org.apache.hadoop.hbase.HBaseTestingUtility;
031import org.apache.hadoop.hbase.HColumnDescriptor;
032import org.apache.hadoop.hbase.HConstants;
033import org.apache.hadoop.hbase.KeyValue;
034import org.apache.hadoop.hbase.Tag;
035import org.apache.hadoop.hbase.client.Durability;
036import org.apache.hadoop.hbase.client.Get;
037import org.apache.hadoop.hbase.client.Put;
038import org.apache.hadoop.hbase.client.Result;
039import org.apache.hadoop.hbase.io.hfile.CacheConfig;
040import org.apache.hadoop.hbase.io.hfile.HFile;
041import org.apache.hadoop.hbase.io.hfile.LruBlockCache;
042import org.apache.hadoop.hbase.regionserver.BloomType;
043import org.apache.hadoop.hbase.regionserver.HRegion;
044import org.apache.hadoop.hbase.regionserver.Region;
045import org.apache.hadoop.hbase.testclassification.IOTests;
046import org.apache.hadoop.hbase.testclassification.MediumTests;
047import org.apache.hadoop.hbase.util.Bytes;
048import org.apache.hadoop.hbase.util.LoadTestKVGenerator;
049import org.apache.hadoop.hbase.util.Strings;
050import org.junit.ClassRule;
051import org.junit.Test;
052import org.junit.experimental.categories.Category;
053import org.junit.runner.RunWith;
054import org.junit.runners.Parameterized;
055import org.junit.runners.Parameterized.Parameters;
056
057/**
058 * Tests encoded seekers by loading and reading values.
059 */
060@Category({IOTests.class, MediumTests.class})
061@RunWith(Parameterized.class)
062public class TestEncodedSeekers {
063
064  @ClassRule
065  public static final HBaseClassTestRule CLASS_RULE =
066      HBaseClassTestRule.forClass(TestEncodedSeekers.class);
067
068  private static final String TABLE_NAME = "encodedSeekersTable";
069  private static final String CF_NAME = "encodedSeekersCF";
070  private static final byte[] CF_BYTES = Bytes.toBytes(CF_NAME);
071  private static final int MAX_VERSIONS = 5;
072
073  private static final int BLOCK_SIZE = 64 * 1024;
074  private static final int MIN_VALUE_SIZE = 30;
075  private static final int MAX_VALUE_SIZE = 60;
076  private static final int NUM_ROWS = 1003;
077  private static final int NUM_COLS_PER_ROW = 20;
078  private static final int NUM_HFILES = 4;
079  private static final int NUM_ROWS_PER_FLUSH = NUM_ROWS / NUM_HFILES;
080
081  private final HBaseTestingUtility testUtil = HBaseTestingUtility.createLocalHTU();
082  private final DataBlockEncoding encoding;
083  private final boolean includeTags;
084  private final boolean compressTags;
085
086  /** Enable when debugging */
087  private static final boolean VERBOSE = false;
088
089  @Parameters
090  public static Collection<Object[]> parameters() {
091    List<Object[]> paramList = new ArrayList<>();
092    for (DataBlockEncoding encoding : DataBlockEncoding.values()) {
093      for (boolean includeTags : new boolean[] { false, true }) {
094        for (boolean compressTags : new boolean[] { false, true }) {
095          paramList.add(new Object[] { encoding, includeTags, compressTags });
096        }
097      }
098    }
099    return paramList;
100  }
101
102  public TestEncodedSeekers(DataBlockEncoding encoding, boolean includeTags, boolean compressTags) {
103    this.encoding = encoding;
104    this.includeTags = includeTags;
105    this.compressTags = compressTags;
106  }
107
108  @Test
109  public void testEncodedSeeker() throws IOException {
110    System.err.println("Testing encoded seekers for encoding : " + encoding + ", includeTags : "
111        + includeTags + ", compressTags : " + compressTags);
112    if(includeTags) {
113      testUtil.getConfiguration().setInt(HFile.FORMAT_VERSION_KEY, 3);
114    }
115    CacheConfig.instantiateBlockCache(testUtil.getConfiguration());
116    LruBlockCache cache =
117      (LruBlockCache)new CacheConfig(testUtil.getConfiguration()).getBlockCache();
118    cache.clearCache();
119    // Need to disable default row bloom filter for this test to pass.
120    HColumnDescriptor hcd = (new HColumnDescriptor(CF_NAME)).setMaxVersions(MAX_VERSIONS).
121        setDataBlockEncoding(encoding).
122        setBlocksize(BLOCK_SIZE).
123        setBloomFilterType(BloomType.NONE).
124        setCompressTags(compressTags);
125    HRegion region = testUtil.createTestRegion(TABLE_NAME, hcd);
126
127    //write the data, but leave some in the memstore
128    doPuts(region);
129
130    //verify correctness when memstore contains data
131    doGets(region);
132
133    //verify correctness again after compacting
134    region.compact(false);
135    doGets(region);
136
137    Map<DataBlockEncoding, Integer> encodingCounts = cache.getEncodingCountsForTest();
138
139    // Ensure that compactions don't pollute the cache with unencoded blocks
140    // in case of in-cache-only encoding.
141    System.err.println("encodingCounts=" + encodingCounts);
142    assertEquals(1, encodingCounts.size());
143    DataBlockEncoding encodingInCache = encodingCounts.keySet().iterator().next();
144    assertEquals(encoding, encodingInCache);
145    assertTrue(encodingCounts.get(encodingInCache) > 0);
146  }
147
148
149  private void doPuts(HRegion region) throws IOException{
150    LoadTestKVGenerator dataGenerator = new LoadTestKVGenerator(MIN_VALUE_SIZE, MAX_VALUE_SIZE);
151     for (int i = 0; i < NUM_ROWS; ++i) {
152      byte[] key = LoadTestKVGenerator.md5PrefixedKey(i).getBytes();
153      for (int j = 0; j < NUM_COLS_PER_ROW; ++j) {
154        Put put = new Put(key);
155        put.setDurability(Durability.ASYNC_WAL);
156        byte[] col = Bytes.toBytes(String.valueOf(j));
157        byte[] value = dataGenerator.generateRandomSizeValue(key, col);
158        if (includeTags) {
159          Tag[] tag = new Tag[1];
160          tag[0] = new ArrayBackedTag((byte) 1, "Visibility");
161          KeyValue kv = new KeyValue(key, CF_BYTES, col, HConstants.LATEST_TIMESTAMP, value, tag);
162          put.add(kv);
163        } else {
164          put.addColumn(CF_BYTES, col, value);
165        }
166        if(VERBOSE){
167          KeyValue kvPut = new KeyValue(key, CF_BYTES, col, value);
168          System.err.println(Strings.padFront(i+"", ' ', 4)+" "+kvPut);
169        }
170        region.put(put);
171      }
172      if (i % NUM_ROWS_PER_FLUSH == 0) {
173        region.flush(true);
174      }
175    }
176  }
177
178
179  private void doGets(Region region) throws IOException{
180    for (int i = 0; i < NUM_ROWS; ++i) {
181      final byte[] rowKey = LoadTestKVGenerator.md5PrefixedKey(i).getBytes();
182      for (int j = 0; j < NUM_COLS_PER_ROW; ++j) {
183        final String qualStr = String.valueOf(j);
184        if (VERBOSE) {
185          System.err.println("Reading row " + i + ", column " + j + " " + Bytes.toString(rowKey)+"/"
186              +qualStr);
187        }
188        final byte[] qualBytes = Bytes.toBytes(qualStr);
189        Get get = new Get(rowKey);
190        get.addColumn(CF_BYTES, qualBytes);
191        Result result = region.get(get);
192        assertEquals(1, result.size());
193        byte[] value = result.getValue(CF_BYTES, qualBytes);
194        assertTrue(LoadTestKVGenerator.verify(value, rowKey, qualBytes));
195      }
196    }
197  }
198
199}