001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.filter; 019 020import static org.junit.Assert.assertEquals; 021 022import java.io.IOException; 023import java.nio.ByteBuffer; 024import java.util.ArrayList; 025import java.util.List; 026import org.apache.hadoop.conf.Configuration; 027import org.apache.hadoop.hbase.Cell; 028import org.apache.hadoop.hbase.HBaseClassTestRule; 029import org.apache.hadoop.hbase.HBaseTestingUtil; 030import org.apache.hadoop.hbase.HConstants; 031import org.apache.hadoop.hbase.TableName; 032import org.apache.hadoop.hbase.client.Durability; 033import org.apache.hadoop.hbase.client.Put; 034import org.apache.hadoop.hbase.client.Scan; 035import org.apache.hadoop.hbase.client.Table; 036import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy; 037import org.apache.hadoop.hbase.regionserver.HRegion; 038import org.apache.hadoop.hbase.regionserver.RegionScanner; 039import org.apache.hadoop.hbase.testclassification.FilterTests; 040import org.apache.hadoop.hbase.testclassification.LargeTests; 041import org.apache.hadoop.hbase.util.Bytes; 042import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; 043import org.apache.hadoop.hbase.util.Pair; 044import org.junit.AfterClass; 045import org.junit.BeforeClass; 046import org.junit.ClassRule; 047import org.junit.Test; 048import org.junit.experimental.categories.Category; 049import org.slf4j.Logger; 050import org.slf4j.LoggerFactory; 051 052@Category({ FilterTests.class, LargeTests.class }) 053public class TestFuzzyRowFilterEndToEndLarge { 054 055 @ClassRule 056 public static final HBaseClassTestRule CLASS_RULE = 057 HBaseClassTestRule.forClass(TestFuzzyRowFilterEndToEndLarge.class); 058 059 private static final Logger LOG = LoggerFactory.getLogger(TestFuzzyRowFilterEndToEndLarge.class); 060 061 private final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil(); 062 063 private final static byte fuzzyValue = (byte) 63; 064 065 private static int firstPartCardinality = 30; 066 private static int secondPartCardinality = 30; 067 private static int thirdPartCardinality = 30; 068 private static int colQualifiersTotal = 5; 069 private static int totalFuzzyKeys = thirdPartCardinality / 2; 070 071 private static String table = "TestFuzzyRowFilterEndToEndLarge"; 072 073 @BeforeClass 074 public static void setUpBeforeClass() throws Exception { 075 Configuration conf = TEST_UTIL.getConfiguration(); 076 conf.setInt("hbase.client.scanner.caching", 1000); 077 conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, 078 ConstantSizeRegionSplitPolicy.class.getName()); 079 // set no splits 080 conf.setLong(HConstants.HREGION_MAX_FILESIZE, (1024L) * 1024 * 1024 * 10); 081 082 TEST_UTIL.startMiniCluster(); 083 } 084 085 @AfterClass 086 public static void tearDownAfterClass() throws Exception { 087 TEST_UTIL.shutdownMiniCluster(); 088 } 089 090 @Test 091 public void testEndToEnd() throws Exception { 092 String cf = "f"; 093 094 Table ht = 095 TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(cf), Integer.MAX_VALUE); 096 097 // 10 byte row key - (2 bytes 4 bytes 4 bytes) 098 // 4 byte qualifier 099 // 4 byte value 100 101 for (int i0 = 0; i0 < firstPartCardinality; i0++) { 102 for (int i1 = 0; i1 < secondPartCardinality; i1++) { 103 for (int i2 = 0; i2 < thirdPartCardinality; i2++) { 104 byte[] rk = new byte[10]; 105 106 ByteBuffer buf = ByteBuffer.wrap(rk); 107 buf.clear(); 108 buf.putShort((short) i0); 109 buf.putInt(i1); 110 buf.putInt(i2); 111 for (int c = 0; c < colQualifiersTotal; c++) { 112 byte[] cq = new byte[4]; 113 Bytes.putBytes(cq, 0, Bytes.toBytes(c), 0, 4); 114 115 Put p = new Put(rk); 116 p.setDurability(Durability.SKIP_WAL); 117 p.addColumn(Bytes.toBytes(cf), cq, Bytes.toBytes(c)); 118 ht.put(p); 119 } 120 } 121 } 122 } 123 124 TEST_UTIL.flush(); 125 126 // test passes 127 runTest1(ht); 128 runTest2(ht); 129 130 } 131 132 private void runTest1(Table hTable) throws IOException { 133 // [0, 2, ?, ?, ?, ?, 0, 0, 0, 1] 134 byte[] mask = new byte[] { 0, 0, 1, 1, 1, 1, 0, 0, 0, 0 }; 135 136 List<Pair<byte[], byte[]>> list = new ArrayList<>(); 137 for (int i = 0; i < totalFuzzyKeys; i++) { 138 byte[] fuzzyKey = new byte[10]; 139 ByteBuffer buf = ByteBuffer.wrap(fuzzyKey); 140 buf.clear(); 141 buf.putShort((short) 2); 142 for (int j = 0; j < 4; j++) { 143 buf.put(fuzzyValue); 144 } 145 buf.putInt(i); 146 147 Pair<byte[], byte[]> pair = new Pair<>(fuzzyKey, mask); 148 list.add(pair); 149 } 150 151 int expectedSize = secondPartCardinality * totalFuzzyKeys * colQualifiersTotal; 152 FuzzyRowFilter fuzzyRowFilter0 = new FuzzyRowFilter(list); 153 // Filters are not stateless - we can't reuse them 154 FuzzyRowFilter fuzzyRowFilter1 = new FuzzyRowFilter(list); 155 156 // regular test 157 runScanner(hTable, expectedSize, fuzzyRowFilter0); 158 // optimized from block cache 159 runScanner(hTable, expectedSize, fuzzyRowFilter1); 160 161 } 162 163 private void runTest2(Table hTable) throws IOException { 164 // [0, 0, ?, ?, ?, ?, 0, 0, 0, 0] , [0, 1, ?, ?, ?, ?, 0, 0, 0, 1]... 165 byte[] mask = new byte[] { 0, 0, 1, 1, 1, 1, 0, 0, 0, 0 }; 166 167 List<Pair<byte[], byte[]>> list = new ArrayList<>(); 168 169 for (int i = 0; i < totalFuzzyKeys; i++) { 170 byte[] fuzzyKey = new byte[10]; 171 ByteBuffer buf = ByteBuffer.wrap(fuzzyKey); 172 buf.clear(); 173 buf.putShort((short) (i * 2)); 174 for (int j = 0; j < 4; j++) { 175 buf.put(fuzzyValue); 176 } 177 buf.putInt(i * 2); 178 179 Pair<byte[], byte[]> pair = new Pair<>(fuzzyKey, mask); 180 list.add(pair); 181 } 182 183 int expectedSize = totalFuzzyKeys * secondPartCardinality * colQualifiersTotal; 184 185 FuzzyRowFilter fuzzyRowFilter0 = new FuzzyRowFilter(list); 186 // Filters are not stateless - we can't reuse them 187 FuzzyRowFilter fuzzyRowFilter1 = new FuzzyRowFilter(list); 188 189 // regular test 190 runScanner(hTable, expectedSize, fuzzyRowFilter0); 191 // optimized from block cache 192 runScanner(hTable, expectedSize, fuzzyRowFilter1); 193 194 } 195 196 private void runScanner(Table hTable, int expectedSize, Filter filter) throws IOException { 197 String cf = "f"; 198 Scan scan = new Scan(); 199 scan.addFamily(Bytes.toBytes(cf)); 200 scan.setFilter(filter); 201 List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(TableName.valueOf(table)); 202 HRegion first = regions.get(0); 203 first.getScanner(scan); 204 RegionScanner scanner = first.getScanner(scan); 205 List<Cell> results = new ArrayList<>(); 206 // Result result; 207 long timeBeforeScan = EnvironmentEdgeManager.currentTime(); 208 int found = 0; 209 while (scanner.next(results)) { 210 found += results.size(); 211 results.clear(); 212 } 213 found += results.size(); 214 long scanTime = EnvironmentEdgeManager.currentTime() - timeBeforeScan; 215 scanner.close(); 216 217 LOG.info("\nscan time = " + scanTime + "ms"); 218 LOG.info("found " + found + " results\n"); 219 220 assertEquals(expectedSize, found); 221 } 222}