001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.filter;
019
020import static org.junit.Assert.assertEquals;
021
022import java.io.IOException;
023import java.nio.ByteBuffer;
024import java.util.ArrayList;
025import java.util.Arrays;
026import java.util.List;
027import org.apache.hadoop.conf.Configuration;
028import org.apache.hadoop.hbase.Cell;
029import org.apache.hadoop.hbase.CellUtil;
030import org.apache.hadoop.hbase.HBaseClassTestRule;
031import org.apache.hadoop.hbase.HBaseTestingUtil;
032import org.apache.hadoop.hbase.HConstants;
033import org.apache.hadoop.hbase.TableName;
034import org.apache.hadoop.hbase.client.Durability;
035import org.apache.hadoop.hbase.client.Put;
036import org.apache.hadoop.hbase.client.Result;
037import org.apache.hadoop.hbase.client.ResultScanner;
038import org.apache.hadoop.hbase.client.Scan;
039import org.apache.hadoop.hbase.client.Table;
040import org.apache.hadoop.hbase.filter.FilterList.Operator;
041import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
042import org.apache.hadoop.hbase.regionserver.HRegion;
043import org.apache.hadoop.hbase.regionserver.RegionScanner;
044import org.apache.hadoop.hbase.testclassification.FilterTests;
045import org.apache.hadoop.hbase.testclassification.LargeTests;
046import org.apache.hadoop.hbase.util.Bytes;
047import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
048import org.apache.hadoop.hbase.util.Pair;
049import org.junit.After;
050import org.junit.AfterClass;
051import org.junit.Before;
052import org.junit.BeforeClass;
053import org.junit.ClassRule;
054import org.junit.Rule;
055import org.junit.Test;
056import org.junit.experimental.categories.Category;
057import org.junit.rules.TestName;
058import org.slf4j.Logger;
059import org.slf4j.LoggerFactory;
060
061import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
062
063@Category({ FilterTests.class, LargeTests.class })
064public class TestFuzzyRowFilterEndToEnd {
065
066  @ClassRule
067  public static final HBaseClassTestRule CLASS_RULE =
068      HBaseClassTestRule.forClass(TestFuzzyRowFilterEndToEnd.class);
069
070  private final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
071  private final static byte fuzzyValue = (byte) 63;
072  private static final Logger LOG = LoggerFactory.getLogger(TestFuzzyRowFilterEndToEnd.class);
073
074  private static int firstPartCardinality = 50;
075  private static int secondPartCardinality = 50;
076  private static int thirdPartCardinality = 50;
077  private static int colQualifiersTotal = 5;
078  private static int totalFuzzyKeys = thirdPartCardinality / 2;
079
080  private static String table = "TestFuzzyRowFilterEndToEnd";
081
082  @Rule
083  public TestName name = new TestName();
084
085  /**
086   * @throws java.lang.Exception
087   */
088  @BeforeClass
089  public static void setUpBeforeClass() throws Exception {
090    Configuration conf = TEST_UTIL.getConfiguration();
091    conf.setInt("hbase.client.scanner.caching", 1000);
092    conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
093      ConstantSizeRegionSplitPolicy.class.getName());
094    // set no splits
095    conf.setLong(HConstants.HREGION_MAX_FILESIZE, (1024L) * 1024 * 1024 * 10);
096
097    TEST_UTIL.startMiniCluster();
098  }
099
100  /**
101   * @throws java.lang.Exception
102   */
103  @AfterClass
104  public static void tearDownAfterClass() throws Exception {
105    TEST_UTIL.shutdownMiniCluster();
106  }
107
108  /**
109   * @throws java.lang.Exception
110   */
111  @Before
112  public void setUp() throws Exception {
113    // Nothing to do.
114  }
115
116  /**
117   * @throws java.lang.Exception
118   */
119  @After
120  public void tearDown() throws Exception {
121    // Nothing to do.
122  }
123
124  // HBASE-15676 Test that fuzzy info of all fixed bits (0s) finds matching row.
125  @Test
126  public void testAllFixedBits() throws IOException {
127    String cf = "f";
128    String cq = "q";
129
130    Table ht =
131        TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), Bytes.toBytes(cf), Integer.MAX_VALUE);
132    // Load data
133    String[] rows = new String[] { "\\x9C\\x00\\x044\\x00\\x00\\x00\\x00",
134        "\\x9C\\x00\\x044\\x01\\x00\\x00\\x00", "\\x9C\\x00\\x044\\x00\\x01\\x00\\x00",
135        "\\x9B\\x00\\x044e\\x9B\\x02\\xBB", "\\x9C\\x00\\x044\\x00\\x00\\x01\\x00",
136        "\\x9C\\x00\\x044\\x00\\x01\\x00\\x01", "\\x9B\\x00\\x044e\\xBB\\xB2\\xBB", };
137
138    for (int i = 0; i < rows.length; i++) {
139      Put p = new Put(Bytes.toBytesBinary(rows[i]));
140      p.addColumn(Bytes.toBytes(cf), Bytes.toBytes(cq), Bytes.toBytes("value"));
141      ht.put(p);
142    }
143
144    TEST_UTIL.flush();
145
146    List<Pair<byte[], byte[]>> data = new ArrayList<>();
147    byte[] fuzzyKey = Bytes.toBytesBinary("\\x9B\\x00\\x044e");
148    byte[] mask = new byte[] { 0, 0, 0, 0, 0 };
149
150    // copy the fuzzy key and mask to test HBASE-18617
151    byte[] copyFuzzyKey = Arrays.copyOf(fuzzyKey, fuzzyKey.length);
152    byte[] copyMask = Arrays.copyOf(mask, mask.length);
153
154    data.add(new Pair<>(fuzzyKey, mask));
155    FuzzyRowFilter filter = new FuzzyRowFilter(data);
156
157    Scan scan = new Scan();
158    scan.setFilter(filter);
159
160    ResultScanner scanner = ht.getScanner(scan);
161    int total = 0;
162    while (scanner.next() != null) {
163      total++;
164    }
165    assertEquals(2, total);
166
167    assertEquals(true, Arrays.equals(copyFuzzyKey, fuzzyKey));
168    assertEquals(true, Arrays.equals(copyMask, mask));
169
170    TEST_UTIL.deleteTable(TableName.valueOf(name.getMethodName()));
171  }
172
173  @Test
174  public void testHBASE14782() throws IOException
175  {
176    String cf = "f";
177    String cq = "q";
178
179    Table ht =
180        TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), Bytes.toBytes(cf), Integer.MAX_VALUE);
181    // Load data
182    String[] rows = new String[] {
183      "\\x9C\\x00\\x044\\x00\\x00\\x00\\x00",
184      "\\x9C\\x00\\x044\\x01\\x00\\x00\\x00",
185      "\\x9C\\x00\\x044\\x00\\x01\\x00\\x00",
186      "\\x9C\\x00\\x044\\x00\\x00\\x01\\x00",
187      "\\x9C\\x00\\x044\\x00\\x01\\x00\\x01",
188      "\\x9B\\x00\\x044e\\xBB\\xB2\\xBB",
189    };
190
191    String badRow = "\\x9C\\x00\\x03\\xE9e\\xBB{X\\x1Fwts\\x1F\\x15vRX";
192
193    for(int i=0; i < rows.length; i++){
194      Put p = new Put(Bytes.toBytesBinary(rows[i]));
195      p.addColumn(Bytes.toBytes(cf), Bytes.toBytes(cq), Bytes.toBytes("value"));
196      ht.put(p);
197    }
198
199    Put p = new Put(Bytes.toBytesBinary(badRow));
200    p.addColumn(Bytes.toBytes(cf), Bytes.toBytes(cq), Bytes.toBytes("value"));
201    ht.put(p);
202
203    TEST_UTIL.flush();
204
205    List<Pair<byte[], byte[]>> data =  new ArrayList<>();
206    byte[] fuzzyKey = Bytes.toBytesBinary("\\x00\\x00\\x044");
207    byte[] mask = new byte[] { 1,0,0,0};
208    data.add(new Pair<>(fuzzyKey, mask));
209    FuzzyRowFilter filter = new FuzzyRowFilter(data);
210
211    Scan scan = new Scan();
212    scan.setFilter(filter);
213
214    ResultScanner scanner = ht.getScanner(scan);
215    int total = 0;
216    while(scanner.next() != null){
217      total++;
218    }
219    assertEquals(rows.length, total);
220    TEST_UTIL.deleteTable(TableName.valueOf(name.getMethodName()));
221  }
222
223  @Test
224  public void testEndToEnd() throws Exception {
225    String cf = "f";
226
227    Table ht =
228        TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(cf), Integer.MAX_VALUE);
229
230    // 10 byte row key - (2 bytes 4 bytes 4 bytes)
231    // 4 byte qualifier
232    // 4 byte value
233
234    for (int i0 = 0; i0 < firstPartCardinality; i0++) {
235
236      for (int i1 = 0; i1 < secondPartCardinality; i1++) {
237
238        for (int i2 = 0; i2 < thirdPartCardinality; i2++) {
239          byte[] rk = new byte[10];
240
241          ByteBuffer buf = ByteBuffer.wrap(rk);
242          buf.clear();
243          buf.putShort((short) i0);
244          buf.putInt(i1);
245          buf.putInt(i2);
246          for (int c = 0; c < colQualifiersTotal; c++) {
247            byte[] cq = new byte[4];
248            Bytes.putBytes(cq, 0, Bytes.toBytes(c), 0, 4);
249
250            Put p = new Put(rk);
251            p.setDurability(Durability.SKIP_WAL);
252            p.addColumn(Bytes.toBytes(cf), cq, Bytes.toBytes(c));
253            ht.put(p);
254          }
255        }
256      }
257    }
258
259    TEST_UTIL.flush();
260
261    // test passes
262    runTest1(ht);
263    runTest2(ht);
264
265  }
266
267  private void runTest1(Table hTable) throws IOException {
268    // [0, 2, ?, ?, ?, ?, 0, 0, 0, 1]
269
270    byte[] mask = new byte[] { 0, 0, 1, 1, 1, 1, 0, 0, 0, 0 };
271
272    List<Pair<byte[], byte[]>> list = new ArrayList<>();
273    for (int i = 0; i < totalFuzzyKeys; i++) {
274      byte[] fuzzyKey = new byte[10];
275      ByteBuffer buf = ByteBuffer.wrap(fuzzyKey);
276      buf.clear();
277      buf.putShort((short) 2);
278      for (int j = 0; j < 4; j++) {
279        buf.put(fuzzyValue);
280      }
281      buf.putInt(i);
282
283      Pair<byte[], byte[]> pair = new Pair<>(fuzzyKey, mask);
284      list.add(pair);
285    }
286
287    int expectedSize = secondPartCardinality * totalFuzzyKeys * colQualifiersTotal;
288    FuzzyRowFilter fuzzyRowFilter0 = new FuzzyRowFilter(list);
289    // Filters are not stateless - we can't reuse them
290    FuzzyRowFilter fuzzyRowFilter1 = new FuzzyRowFilter(list);
291
292    // regular test
293    runScanner(hTable, expectedSize, fuzzyRowFilter0);
294    // optimized from block cache
295    runScanner(hTable, expectedSize, fuzzyRowFilter1);
296
297  }
298
299  private void runTest2(Table hTable) throws IOException {
300    // [0, 0, ?, ?, ?, ?, 0, 0, 0, 0] , [0, 1, ?, ?, ?, ?, 0, 0, 0, 1]...
301
302    byte[] mask = new byte[] { 0, 0, 1, 1, 1, 1, 0, 0, 0, 0 };
303
304    List<Pair<byte[], byte[]>> list = new ArrayList<>();
305
306    for (int i = 0; i < totalFuzzyKeys; i++) {
307      byte[] fuzzyKey = new byte[10];
308      ByteBuffer buf = ByteBuffer.wrap(fuzzyKey);
309      buf.clear();
310      buf.putShort((short) (i * 2));
311      for (int j = 0; j < 4; j++) {
312        buf.put(fuzzyValue);
313      }
314      buf.putInt(i * 2);
315
316      Pair<byte[], byte[]> pair = new Pair<>(fuzzyKey, mask);
317      list.add(pair);
318    }
319
320    int expectedSize = totalFuzzyKeys * secondPartCardinality * colQualifiersTotal;
321
322    FuzzyRowFilter fuzzyRowFilter0 = new FuzzyRowFilter(list);
323    // Filters are not stateless - we can't reuse them
324    FuzzyRowFilter fuzzyRowFilter1 = new FuzzyRowFilter(list);
325
326    // regular test
327    runScanner(hTable, expectedSize, fuzzyRowFilter0);
328    // optimized from block cache
329    runScanner(hTable, expectedSize, fuzzyRowFilter1);
330
331  }
332
333  private void runScanner(Table hTable, int expectedSize, Filter filter) throws IOException {
334
335    String cf = "f";
336    Scan scan = new Scan();
337    scan.addFamily(Bytes.toBytes(cf));
338    scan.setFilter(filter);
339    List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(TableName.valueOf(table));
340    HRegion first = regions.get(0);
341    first.getScanner(scan);
342    RegionScanner scanner = first.getScanner(scan);
343    List<Cell> results = new ArrayList<>();
344    // Result result;
345    long timeBeforeScan = EnvironmentEdgeManager.currentTime();
346    int found = 0;
347    while (scanner.next(results)) {
348      found += results.size();
349      results.clear();
350    }
351    found += results.size();
352    long scanTime = EnvironmentEdgeManager.currentTime() - timeBeforeScan;
353    scanner.close();
354
355    LOG.info("\nscan time = " + scanTime + "ms");
356    LOG.info("found " + found + " results\n");
357
358    assertEquals(expectedSize, found);
359  }
360
361  @SuppressWarnings("deprecation")
362  @Test
363  public void testFilterList() throws Exception {
364    String cf = "f";
365    Table ht =
366        TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), Bytes.toBytes(cf), Integer.MAX_VALUE);
367
368    // 10 byte row key - (2 bytes 4 bytes 4 bytes)
369    // 4 byte qualifier
370    // 4 byte value
371
372    for (int i1 = 0; i1 < 5; i1++) {
373      for (int i2 = 0; i2 < 5; i2++) {
374        byte[] rk = new byte[10];
375
376        ByteBuffer buf = ByteBuffer.wrap(rk);
377        buf.clear();
378        buf.putShort((short) 2);
379        buf.putInt(i1);
380        buf.putInt(i2);
381
382        // Each row contains 5 columns
383        for (int c = 0; c < 5; c++) {
384          byte[] cq = new byte[4];
385          Bytes.putBytes(cq, 0, Bytes.toBytes(c), 0, 4);
386
387          Put p = new Put(rk);
388          p.setDurability(Durability.SKIP_WAL);
389          p.addColumn(Bytes.toBytes(cf), cq, Bytes.toBytes(c));
390          ht.put(p);
391          LOG.info("Inserting: rk: " + Bytes.toStringBinary(rk) + " cq: "
392              + Bytes.toStringBinary(cq));
393        }
394      }
395    }
396
397    TEST_UTIL.flush();
398
399    // test passes if we get back 5 KV's (1 row)
400    runTest(ht, 5);
401
402  }
403
404  @SuppressWarnings("unchecked")
405  private void runTest(Table hTable, int expectedSize) throws IOException {
406    // [0, 2, ?, ?, ?, ?, 0, 0, 0, 1]
407    byte[] fuzzyKey1 = new byte[10];
408    ByteBuffer buf = ByteBuffer.wrap(fuzzyKey1);
409    buf.clear();
410    buf.putShort((short) 2);
411    for (int i = 0; i < 4; i++)
412      buf.put(fuzzyValue);
413    buf.putInt((short) 1);
414    byte[] mask1 = new byte[] { 0, 0, 1, 1, 1, 1, 0, 0, 0, 0 };
415
416    byte[] fuzzyKey2 = new byte[10];
417    buf = ByteBuffer.wrap(fuzzyKey2);
418    buf.clear();
419    buf.putShort((short) 2);
420    buf.putInt((short) 2);
421    for (int i = 0; i < 4; i++)
422      buf.put(fuzzyValue);
423
424    byte[] mask2 = new byte[] { 0, 0, 0, 0, 0, 0, 1, 1, 1, 1 };
425
426    Pair<byte[], byte[]> pair1 = new Pair<>(fuzzyKey1, mask1);
427    Pair<byte[], byte[]> pair2 = new Pair<>(fuzzyKey2, mask2);
428
429    FuzzyRowFilter fuzzyRowFilter1 = new FuzzyRowFilter(Lists.newArrayList(pair1));
430    FuzzyRowFilter fuzzyRowFilter2 = new FuzzyRowFilter(Lists.newArrayList(pair2));
431    // regular test - we expect 1 row back (5 KVs)
432    runScanner(hTable, expectedSize, fuzzyRowFilter1, fuzzyRowFilter2);
433  }
434
435  private void runScanner(Table hTable, int expectedSize, Filter filter1, Filter filter2)
436      throws IOException {
437    String cf = "f";
438    Scan scan = new Scan();
439    scan.addFamily(Bytes.toBytes(cf));
440    FilterList filterList = new FilterList(Operator.MUST_PASS_ALL, filter1, filter2);
441    scan.setFilter(filterList);
442
443    ResultScanner scanner = hTable.getScanner(scan);
444    List<Cell> results = new ArrayList<>();
445    Result result;
446    long timeBeforeScan = EnvironmentEdgeManager.currentTime();
447    while ((result = scanner.next()) != null) {
448      for (Cell kv : result.listCells()) {
449        LOG.info("Got rk: " + Bytes.toStringBinary(CellUtil.cloneRow(kv)) + " cq: "
450            + Bytes.toStringBinary(CellUtil.cloneQualifier(kv)));
451        results.add(kv);
452      }
453    }
454    long scanTime = EnvironmentEdgeManager.currentTime() - timeBeforeScan;
455    scanner.close();
456
457    LOG.info("scan time = " + scanTime + "ms");
458    LOG.info("found " + results.size() + " results");
459
460    assertEquals(expectedSize, results.size());
461  }
462}