001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.io.encoding; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertTrue; 022 023import java.io.IOException; 024import java.util.ArrayList; 025import java.util.Collection; 026import java.util.List; 027import java.util.Map; 028import org.apache.hadoop.hbase.ArrayBackedTag; 029import org.apache.hadoop.hbase.HBaseClassTestRule; 030import org.apache.hadoop.hbase.HBaseTestingUtility; 031import org.apache.hadoop.hbase.HColumnDescriptor; 032import org.apache.hadoop.hbase.HConstants; 033import org.apache.hadoop.hbase.KeyValue; 034import org.apache.hadoop.hbase.Tag; 035import org.apache.hadoop.hbase.client.Durability; 036import org.apache.hadoop.hbase.client.Get; 037import org.apache.hadoop.hbase.client.Put; 038import org.apache.hadoop.hbase.client.Result; 039import org.apache.hadoop.hbase.io.hfile.CacheConfig; 040import org.apache.hadoop.hbase.io.hfile.HFile; 041import org.apache.hadoop.hbase.io.hfile.LruBlockCache; 042import org.apache.hadoop.hbase.regionserver.BloomType; 043import org.apache.hadoop.hbase.regionserver.HRegion; 044import org.apache.hadoop.hbase.regionserver.Region; 045import org.apache.hadoop.hbase.testclassification.IOTests; 046import org.apache.hadoop.hbase.testclassification.MediumTests; 047import org.apache.hadoop.hbase.util.Bytes; 048import org.apache.hadoop.hbase.util.LoadTestKVGenerator; 049import org.apache.hadoop.hbase.util.Strings; 050import org.junit.ClassRule; 051import org.junit.Test; 052import org.junit.experimental.categories.Category; 053import org.junit.runner.RunWith; 054import org.junit.runners.Parameterized; 055import org.junit.runners.Parameterized.Parameters; 056 057/** 058 * Tests encoded seekers by loading and reading values. 059 */ 060@Category({IOTests.class, MediumTests.class}) 061@RunWith(Parameterized.class) 062public class TestEncodedSeekers { 063 064 @ClassRule 065 public static final HBaseClassTestRule CLASS_RULE = 066 HBaseClassTestRule.forClass(TestEncodedSeekers.class); 067 068 private static final String TABLE_NAME = "encodedSeekersTable"; 069 private static final String CF_NAME = "encodedSeekersCF"; 070 private static final byte[] CF_BYTES = Bytes.toBytes(CF_NAME); 071 private static final int MAX_VERSIONS = 5; 072 073 private static final int BLOCK_SIZE = 64 * 1024; 074 private static final int MIN_VALUE_SIZE = 30; 075 private static final int MAX_VALUE_SIZE = 60; 076 private static final int NUM_ROWS = 1003; 077 private static final int NUM_COLS_PER_ROW = 20; 078 private static final int NUM_HFILES = 4; 079 private static final int NUM_ROWS_PER_FLUSH = NUM_ROWS / NUM_HFILES; 080 081 private final HBaseTestingUtility testUtil = HBaseTestingUtility.createLocalHTU(); 082 private final DataBlockEncoding encoding; 083 private final boolean includeTags; 084 private final boolean compressTags; 085 086 /** Enable when debugging */ 087 private static final boolean VERBOSE = false; 088 089 @Parameters 090 public static Collection<Object[]> parameters() { 091 List<Object[]> paramList = new ArrayList<>(); 092 for (DataBlockEncoding encoding : DataBlockEncoding.values()) { 093 for (boolean includeTags : new boolean[] { false, true }) { 094 for (boolean compressTags : new boolean[] { false, true }) { 095 paramList.add(new Object[] { encoding, includeTags, compressTags }); 096 } 097 } 098 } 099 return paramList; 100 } 101 102 public TestEncodedSeekers(DataBlockEncoding encoding, boolean includeTags, boolean compressTags) { 103 this.encoding = encoding; 104 this.includeTags = includeTags; 105 this.compressTags = compressTags; 106 } 107 108 @Test 109 public void testEncodedSeeker() throws IOException { 110 System.err.println("Testing encoded seekers for encoding : " + encoding + ", includeTags : " 111 + includeTags + ", compressTags : " + compressTags); 112 if(includeTags) { 113 testUtil.getConfiguration().setInt(HFile.FORMAT_VERSION_KEY, 3); 114 } 115 CacheConfig.instantiateBlockCache(testUtil.getConfiguration()); 116 LruBlockCache cache = 117 (LruBlockCache)new CacheConfig(testUtil.getConfiguration()).getBlockCache(); 118 cache.clearCache(); 119 // Need to disable default row bloom filter for this test to pass. 120 HColumnDescriptor hcd = (new HColumnDescriptor(CF_NAME)).setMaxVersions(MAX_VERSIONS). 121 setDataBlockEncoding(encoding). 122 setBlocksize(BLOCK_SIZE). 123 setBloomFilterType(BloomType.NONE). 124 setCompressTags(compressTags); 125 HRegion region = testUtil.createTestRegion(TABLE_NAME, hcd); 126 127 //write the data, but leave some in the memstore 128 doPuts(region); 129 130 //verify correctness when memstore contains data 131 doGets(region); 132 133 //verify correctness again after compacting 134 region.compact(false); 135 doGets(region); 136 137 Map<DataBlockEncoding, Integer> encodingCounts = cache.getEncodingCountsForTest(); 138 139 // Ensure that compactions don't pollute the cache with unencoded blocks 140 // in case of in-cache-only encoding. 141 System.err.println("encodingCounts=" + encodingCounts); 142 assertEquals(1, encodingCounts.size()); 143 DataBlockEncoding encodingInCache = encodingCounts.keySet().iterator().next(); 144 assertEquals(encoding, encodingInCache); 145 assertTrue(encodingCounts.get(encodingInCache) > 0); 146 } 147 148 149 private void doPuts(HRegion region) throws IOException{ 150 LoadTestKVGenerator dataGenerator = new LoadTestKVGenerator(MIN_VALUE_SIZE, MAX_VALUE_SIZE); 151 for (int i = 0; i < NUM_ROWS; ++i) { 152 byte[] key = LoadTestKVGenerator.md5PrefixedKey(i).getBytes(); 153 for (int j = 0; j < NUM_COLS_PER_ROW; ++j) { 154 Put put = new Put(key); 155 put.setDurability(Durability.ASYNC_WAL); 156 byte[] col = Bytes.toBytes(String.valueOf(j)); 157 byte[] value = dataGenerator.generateRandomSizeValue(key, col); 158 if (includeTags) { 159 Tag[] tag = new Tag[1]; 160 tag[0] = new ArrayBackedTag((byte) 1, "Visibility"); 161 KeyValue kv = new KeyValue(key, CF_BYTES, col, HConstants.LATEST_TIMESTAMP, value, tag); 162 put.add(kv); 163 } else { 164 put.addColumn(CF_BYTES, col, value); 165 } 166 if(VERBOSE){ 167 KeyValue kvPut = new KeyValue(key, CF_BYTES, col, value); 168 System.err.println(Strings.padFront(i+"", ' ', 4)+" "+kvPut); 169 } 170 region.put(put); 171 } 172 if (i % NUM_ROWS_PER_FLUSH == 0) { 173 region.flush(true); 174 } 175 } 176 } 177 178 179 private void doGets(Region region) throws IOException{ 180 for (int i = 0; i < NUM_ROWS; ++i) { 181 final byte[] rowKey = LoadTestKVGenerator.md5PrefixedKey(i).getBytes(); 182 for (int j = 0; j < NUM_COLS_PER_ROW; ++j) { 183 final String qualStr = String.valueOf(j); 184 if (VERBOSE) { 185 System.err.println("Reading row " + i + ", column " + j + " " + Bytes.toString(rowKey)+"/" 186 +qualStr); 187 } 188 final byte[] qualBytes = Bytes.toBytes(qualStr); 189 Get get = new Get(rowKey); 190 get.addColumn(CF_BYTES, qualBytes); 191 Result result = region.get(get); 192 assertEquals(1, result.size()); 193 byte[] value = result.getValue(CF_BYTES, qualBytes); 194 assertTrue(LoadTestKVGenerator.verify(value, rowKey, qualBytes)); 195 } 196 } 197 } 198 199}