001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.filter;
019
020import static org.junit.Assert.assertEquals;
021
022import java.io.IOException;
023import java.nio.ByteBuffer;
024import java.util.ArrayList;
025import java.util.List;
026import org.apache.hadoop.conf.Configuration;
027import org.apache.hadoop.hbase.Cell;
028import org.apache.hadoop.hbase.HBaseClassTestRule;
029import org.apache.hadoop.hbase.HBaseTestingUtil;
030import org.apache.hadoop.hbase.HConstants;
031import org.apache.hadoop.hbase.TableName;
032import org.apache.hadoop.hbase.client.Durability;
033import org.apache.hadoop.hbase.client.Put;
034import org.apache.hadoop.hbase.client.Scan;
035import org.apache.hadoop.hbase.client.Table;
036import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
037import org.apache.hadoop.hbase.regionserver.HRegion;
038import org.apache.hadoop.hbase.regionserver.RegionScanner;
039import org.apache.hadoop.hbase.testclassification.FilterTests;
040import org.apache.hadoop.hbase.testclassification.LargeTests;
041import org.apache.hadoop.hbase.util.Bytes;
042import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
043import org.apache.hadoop.hbase.util.Pair;
044import org.junit.AfterClass;
045import org.junit.BeforeClass;
046import org.junit.ClassRule;
047import org.junit.Test;
048import org.junit.experimental.categories.Category;
049import org.slf4j.Logger;
050import org.slf4j.LoggerFactory;
051
052@Category({ FilterTests.class, LargeTests.class })
053public class TestFuzzyRowFilterEndToEndLarge {
054
055  @ClassRule
056  public static final HBaseClassTestRule CLASS_RULE =
057    HBaseClassTestRule.forClass(TestFuzzyRowFilterEndToEndLarge.class);
058
059  private static final Logger LOG = LoggerFactory.getLogger(TestFuzzyRowFilterEndToEndLarge.class);
060
061  private final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
062
063  private final static byte fuzzyValue = (byte) 63;
064
065  private static int firstPartCardinality = 30;
066  private static int secondPartCardinality = 30;
067  private static int thirdPartCardinality = 30;
068  private static int colQualifiersTotal = 5;
069  private static int totalFuzzyKeys = thirdPartCardinality / 2;
070
071  private static String table = "TestFuzzyRowFilterEndToEndLarge";
072
073  @BeforeClass
074  public static void setUpBeforeClass() throws Exception {
075    Configuration conf = TEST_UTIL.getConfiguration();
076    conf.setInt("hbase.client.scanner.caching", 1000);
077    conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
078      ConstantSizeRegionSplitPolicy.class.getName());
079    // set no splits
080    conf.setLong(HConstants.HREGION_MAX_FILESIZE, (1024L) * 1024 * 1024 * 10);
081
082    TEST_UTIL.startMiniCluster();
083  }
084
085  @AfterClass
086  public static void tearDownAfterClass() throws Exception {
087    TEST_UTIL.shutdownMiniCluster();
088  }
089
090  @Test
091  public void testEndToEnd() throws Exception {
092    String cf = "f";
093
094    Table ht =
095      TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(cf), Integer.MAX_VALUE);
096
097    // 10 byte row key - (2 bytes 4 bytes 4 bytes)
098    // 4 byte qualifier
099    // 4 byte value
100
101    for (int i0 = 0; i0 < firstPartCardinality; i0++) {
102      for (int i1 = 0; i1 < secondPartCardinality; i1++) {
103        for (int i2 = 0; i2 < thirdPartCardinality; i2++) {
104          byte[] rk = new byte[10];
105
106          ByteBuffer buf = ByteBuffer.wrap(rk);
107          buf.clear();
108          buf.putShort((short) i0);
109          buf.putInt(i1);
110          buf.putInt(i2);
111          for (int c = 0; c < colQualifiersTotal; c++) {
112            byte[] cq = new byte[4];
113            Bytes.putBytes(cq, 0, Bytes.toBytes(c), 0, 4);
114
115            Put p = new Put(rk);
116            p.setDurability(Durability.SKIP_WAL);
117            p.addColumn(Bytes.toBytes(cf), cq, Bytes.toBytes(c));
118            ht.put(p);
119          }
120        }
121      }
122    }
123
124    TEST_UTIL.flush();
125
126    // test passes
127    runTest1(ht);
128    runTest2(ht);
129
130  }
131
132  private void runTest1(Table hTable) throws IOException {
133    // [0, 2, ?, ?, ?, ?, 0, 0, 0, 1]
134    byte[] mask = new byte[] { 0, 0, 1, 1, 1, 1, 0, 0, 0, 0 };
135
136    List<Pair<byte[], byte[]>> list = new ArrayList<>();
137    for (int i = 0; i < totalFuzzyKeys; i++) {
138      byte[] fuzzyKey = new byte[10];
139      ByteBuffer buf = ByteBuffer.wrap(fuzzyKey);
140      buf.clear();
141      buf.putShort((short) 2);
142      for (int j = 0; j < 4; j++) {
143        buf.put(fuzzyValue);
144      }
145      buf.putInt(i);
146
147      Pair<byte[], byte[]> pair = new Pair<>(fuzzyKey, mask);
148      list.add(pair);
149    }
150
151    int expectedSize = secondPartCardinality * totalFuzzyKeys * colQualifiersTotal;
152    FuzzyRowFilter fuzzyRowFilter0 = new FuzzyRowFilter(list);
153    // Filters are not stateless - we can't reuse them
154    FuzzyRowFilter fuzzyRowFilter1 = new FuzzyRowFilter(list);
155
156    // regular test
157    runScanner(hTable, expectedSize, fuzzyRowFilter0);
158    // optimized from block cache
159    runScanner(hTable, expectedSize, fuzzyRowFilter1);
160
161  }
162
163  private void runTest2(Table hTable) throws IOException {
164    // [0, 0, ?, ?, ?, ?, 0, 0, 0, 0] , [0, 1, ?, ?, ?, ?, 0, 0, 0, 1]...
165    byte[] mask = new byte[] { 0, 0, 1, 1, 1, 1, 0, 0, 0, 0 };
166
167    List<Pair<byte[], byte[]>> list = new ArrayList<>();
168
169    for (int i = 0; i < totalFuzzyKeys; i++) {
170      byte[] fuzzyKey = new byte[10];
171      ByteBuffer buf = ByteBuffer.wrap(fuzzyKey);
172      buf.clear();
173      buf.putShort((short) (i * 2));
174      for (int j = 0; j < 4; j++) {
175        buf.put(fuzzyValue);
176      }
177      buf.putInt(i * 2);
178
179      Pair<byte[], byte[]> pair = new Pair<>(fuzzyKey, mask);
180      list.add(pair);
181    }
182
183    int expectedSize = totalFuzzyKeys * secondPartCardinality * colQualifiersTotal;
184
185    FuzzyRowFilter fuzzyRowFilter0 = new FuzzyRowFilter(list);
186    // Filters are not stateless - we can't reuse them
187    FuzzyRowFilter fuzzyRowFilter1 = new FuzzyRowFilter(list);
188
189    // regular test
190    runScanner(hTable, expectedSize, fuzzyRowFilter0);
191    // optimized from block cache
192    runScanner(hTable, expectedSize, fuzzyRowFilter1);
193
194  }
195
196  private void runScanner(Table hTable, int expectedSize, Filter filter) throws IOException {
197    String cf = "f";
198    Scan scan = new Scan();
199    scan.addFamily(Bytes.toBytes(cf));
200    scan.setFilter(filter);
201    List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(TableName.valueOf(table));
202    HRegion first = regions.get(0);
203    first.getScanner(scan);
204    RegionScanner scanner = first.getScanner(scan);
205    List<Cell> results = new ArrayList<>();
206    // Result result;
207    long timeBeforeScan = EnvironmentEdgeManager.currentTime();
208    int found = 0;
209    while (scanner.next(results)) {
210      found += results.size();
211      results.clear();
212    }
213    found += results.size();
214    long scanTime = EnvironmentEdgeManager.currentTime() - timeBeforeScan;
215    scanner.close();
216
217    LOG.info("\nscan time = " + scanTime + "ms");
218    LOG.info("found " + found + " results\n");
219
220    assertEquals(expectedSize, found);
221  }
222}