Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.filter;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNull;
022
023import java.io.IOException;
024import java.nio.ByteBuffer;
025import java.util.ArrayList;
026import java.util.Arrays;
027import java.util.LinkedList;
028import java.util.List;
029import org.apache.hadoop.conf.Configuration;
030import org.apache.hadoop.hbase.Cell;
031import org.apache.hadoop.hbase.CellUtil;
032import org.apache.hadoop.hbase.CompareOperator;
033import org.apache.hadoop.hbase.HBaseClassTestRule;
034import org.apache.hadoop.hbase.HBaseTestingUtil;
035import org.apache.hadoop.hbase.HConstants;
036import org.apache.hadoop.hbase.TableName;
037import org.apache.hadoop.hbase.client.Durability;
038import org.apache.hadoop.hbase.client.Put;
039import org.apache.hadoop.hbase.client.Result;
040import org.apache.hadoop.hbase.client.ResultScanner;
041import org.apache.hadoop.hbase.client.Scan;
042import org.apache.hadoop.hbase.client.Table;
043import org.apache.hadoop.hbase.filter.FilterList.Operator;
044import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
045import org.apache.hadoop.hbase.testclassification.FilterTests;
046import org.apache.hadoop.hbase.testclassification.MediumTests;
047import org.apache.hadoop.hbase.util.Bytes;
048import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
049import org.apache.hadoop.hbase.util.Pair;
050import org.junit.AfterClass;
051import org.junit.BeforeClass;
052import org.junit.ClassRule;
053import org.junit.Rule;
054import org.junit.Test;
055import org.junit.experimental.categories.Category;
056import org.junit.rules.TestName;
057import org.slf4j.Logger;
058import org.slf4j.LoggerFactory;
059
060import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
061
062@Category({ FilterTests.class, MediumTests.class })
063public class TestFuzzyRowFilterEndToEnd {
064
065  @ClassRule
066  public static final HBaseClassTestRule CLASS_RULE =
067    HBaseClassTestRule.forClass(TestFuzzyRowFilterEndToEnd.class);
068
069  private static final Logger LOG = LoggerFactory.getLogger(TestFuzzyRowFilterEndToEnd.class);
070
071  private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
072
073  private static final byte fuzzyValue = (byte) 63;
074
075  @Rule
076  public TestName name = new TestName();
077
078  @BeforeClass
079  public static void setUpBeforeClass() throws Exception {
080    Configuration conf = TEST_UTIL.getConfiguration();
081    conf.setInt("hbase.client.scanner.caching", 1000);
082    conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
083      ConstantSizeRegionSplitPolicy.class.getName());
084    // set no splits
085    conf.setLong(HConstants.HREGION_MAX_FILESIZE, (1024L) * 1024 * 1024 * 10);
086
087    TEST_UTIL.startMiniCluster();
088  }
089
090  @AfterClass
091  public static void tearDownAfterClass() throws Exception {
092    TEST_UTIL.shutdownMiniCluster();
093  }
094
095  // HBASE-15676 Test that fuzzy info of all fixed bits (0s) finds matching row.
096  @Test
097  public void testAllFixedBits() throws IOException {
098    String cf = "f";
099    String cq = "q";
100
101    Table ht = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), Bytes.toBytes(cf),
102      Integer.MAX_VALUE);
103    // Load data
104    String[] rows = new String[] { "\\x9C\\x00\\x044\\x00\\x00\\x00\\x00",
105      "\\x9C\\x00\\x044\\x01\\x00\\x00\\x00", "\\x9C\\x00\\x044\\x00\\x01\\x00\\x00",
106      "\\x9B\\x00\\x044e\\x9B\\x02\\xBB", "\\x9C\\x00\\x044\\x00\\x00\\x01\\x00",
107      "\\x9C\\x00\\x044\\x00\\x01\\x00\\x01", "\\x9B\\x00\\x044e\\xBB\\xB2\\xBB", };
108
109    for (int i = 0; i < rows.length; i++) {
110      Put p = new Put(Bytes.toBytesBinary(rows[i]));
111      p.addColumn(Bytes.toBytes(cf), Bytes.toBytes(cq), Bytes.toBytes("value"));
112      ht.put(p);
113    }
114
115    TEST_UTIL.flush();
116
117    List<Pair<byte[], byte[]>> data = new ArrayList<>();
118    byte[] fuzzyKey = Bytes.toBytesBinary("\\x9B\\x00\\x044e");
119    byte[] mask = new byte[] { 0, 0, 0, 0, 0 };
120
121    // copy the fuzzy key and mask to test HBASE-18617
122    byte[] copyFuzzyKey = Arrays.copyOf(fuzzyKey, fuzzyKey.length);
123    byte[] copyMask = Arrays.copyOf(mask, mask.length);
124
125    data.add(new Pair<>(fuzzyKey, mask));
126    FuzzyRowFilter filter = new FuzzyRowFilter(data);
127
128    Scan scan = new Scan();
129    scan.setFilter(filter);
130
131    ResultScanner scanner = ht.getScanner(scan);
132    int total = 0;
133    while (scanner.next() != null) {
134      total++;
135    }
136    assertEquals(2, total);
137
138    assertEquals(true, Arrays.equals(copyFuzzyKey, fuzzyKey));
139    assertEquals(true, Arrays.equals(copyMask, mask));
140
141    TEST_UTIL.deleteTable(TableName.valueOf(name.getMethodName()));
142  }
143
144  @Test
145  public void testHBASE14782() throws IOException {
146    String cf = "f";
147    String cq = "q";
148
149    Table ht = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), Bytes.toBytes(cf),
150      Integer.MAX_VALUE);
151    // Load data
152    String[] rows =
153      new String[] { "\\x9C\\x00\\x044\\x00\\x00\\x00\\x00", "\\x9C\\x00\\x044\\x01\\x00\\x00\\x00",
154        "\\x9C\\x00\\x044\\x00\\x01\\x00\\x00", "\\x9C\\x00\\x044\\x00\\x00\\x01\\x00",
155        "\\x9C\\x00\\x044\\x00\\x01\\x00\\x01", "\\x9B\\x00\\x044e\\xBB\\xB2\\xBB", };
156
157    String badRow = "\\x9C\\x00\\x03\\xE9e\\xBB{X\\x1Fwts\\x1F\\x15vRX";
158
159    for (int i = 0; i < rows.length; i++) {
160      Put p = new Put(Bytes.toBytesBinary(rows[i]));
161      p.addColumn(Bytes.toBytes(cf), Bytes.toBytes(cq), Bytes.toBytes("value"));
162      ht.put(p);
163    }
164
165    Put p = new Put(Bytes.toBytesBinary(badRow));
166    p.addColumn(Bytes.toBytes(cf), Bytes.toBytes(cq), Bytes.toBytes("value"));
167    ht.put(p);
168
169    TEST_UTIL.flush();
170
171    List<Pair<byte[], byte[]>> data = new ArrayList<>();
172    byte[] fuzzyKey = Bytes.toBytesBinary("\\x00\\x00\\x044");
173    byte[] mask = new byte[] { 1, 0, 0, 0 };
174    data.add(new Pair<>(fuzzyKey, mask));
175    FuzzyRowFilter filter = new FuzzyRowFilter(data);
176
177    Scan scan = new Scan();
178    scan.setFilter(filter);
179
180    ResultScanner scanner = ht.getScanner(scan);
181    int total = 0;
182    while (scanner.next() != null) {
183      total++;
184    }
185    assertEquals(rows.length, total);
186    TEST_UTIL.deleteTable(TableName.valueOf(name.getMethodName()));
187  }
188
189  @Test
190  public void testFilterList() throws Exception {
191    String cf = "f";
192    Table ht = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), Bytes.toBytes(cf),
193      Integer.MAX_VALUE);
194
195    // 10 byte row key - (2 bytes 4 bytes 4 bytes)
196    // 4 byte qualifier
197    // 4 byte value
198
199    for (int i1 = 0; i1 < 5; i1++) {
200      for (int i2 = 0; i2 < 5; i2++) {
201        byte[] rk = new byte[10];
202
203        ByteBuffer buf = ByteBuffer.wrap(rk);
204        buf.clear();
205        buf.putShort((short) 2);
206        buf.putInt(i1);
207        buf.putInt(i2);
208
209        // Each row contains 5 columns
210        for (int c = 0; c < 5; c++) {
211          byte[] cq = new byte[4];
212          Bytes.putBytes(cq, 0, Bytes.toBytes(c), 0, 4);
213
214          Put p = new Put(rk);
215          p.setDurability(Durability.SKIP_WAL);
216          p.addColumn(Bytes.toBytes(cf), cq, Bytes.toBytes(c));
217          ht.put(p);
218          LOG.info(
219            "Inserting: rk: " + Bytes.toStringBinary(rk) + " cq: " + Bytes.toStringBinary(cq));
220        }
221      }
222    }
223
224    TEST_UTIL.flush();
225
226    // test passes if we get back 5 KV's (1 row)
227    runTest(ht, 5);
228
229  }
230
231  private void runTest(Table hTable, int expectedSize) throws IOException {
232    // [0, 2, ?, ?, ?, ?, 0, 0, 0, 1]
233    byte[] fuzzyKey1 = new byte[10];
234    ByteBuffer buf = ByteBuffer.wrap(fuzzyKey1);
235    buf.clear();
236    buf.putShort((short) 2);
237    for (int i = 0; i < 4; i++)
238      buf.put(fuzzyValue);
239    buf.putInt((short) 1);
240    byte[] mask1 = new byte[] { 0, 0, 1, 1, 1, 1, 0, 0, 0, 0 };
241
242    byte[] fuzzyKey2 = new byte[10];
243    buf = ByteBuffer.wrap(fuzzyKey2);
244    buf.clear();
245    buf.putShort((short) 2);
246    buf.putInt((short) 2);
247    for (int i = 0; i < 4; i++)
248      buf.put(fuzzyValue);
249
250    byte[] mask2 = new byte[] { 0, 0, 0, 0, 0, 0, 1, 1, 1, 1 };
251
252    Pair<byte[], byte[]> pair1 = new Pair<>(fuzzyKey1, mask1);
253    Pair<byte[], byte[]> pair2 = new Pair<>(fuzzyKey2, mask2);
254
255    FuzzyRowFilter fuzzyRowFilter1 = new FuzzyRowFilter(Lists.newArrayList(pair1));
256    FuzzyRowFilter fuzzyRowFilter2 = new FuzzyRowFilter(Lists.newArrayList(pair2));
257    // regular test - we expect 1 row back (5 KVs)
258    runScanner(hTable, expectedSize, fuzzyRowFilter1, fuzzyRowFilter2);
259  }
260
261  private void runScanner(Table hTable, int expectedSize, Filter filter1, Filter filter2)
262    throws IOException {
263    String cf = "f";
264    Scan scan = new Scan();
265    scan.addFamily(Bytes.toBytes(cf));
266    FilterList filterList = new FilterList(Operator.MUST_PASS_ALL, filter1, filter2);
267    scan.setFilter(filterList);
268
269    ResultScanner scanner = hTable.getScanner(scan);
270    List<Cell> results = new ArrayList<>();
271    Result result;
272    long timeBeforeScan = EnvironmentEdgeManager.currentTime();
273    while ((result = scanner.next()) != null) {
274      for (Cell kv : result.listCells()) {
275        LOG.info("Got rk: " + Bytes.toStringBinary(CellUtil.cloneRow(kv)) + " cq: "
276          + Bytes.toStringBinary(CellUtil.cloneQualifier(kv)));
277        results.add(kv);
278      }
279    }
280    long scanTime = EnvironmentEdgeManager.currentTime() - timeBeforeScan;
281    scanner.close();
282
283    LOG.info("scan time = " + scanTime + "ms");
284    LOG.info("found " + results.size() + " results");
285
286    assertEquals(expectedSize, results.size());
287  }
288
289  @Test
290  public void testHBASE26967() throws IOException {
291    byte[] row1 = Bytes.toBytes("1");
292    byte[] row2 = Bytes.toBytes("2");
293    String cf1 = "f1";
294    String cf2 = "f2";
295    String cq1 = "col1";
296    String cq2 = "col2";
297
298    Table ht =
299      TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), new String[] { cf1, cf2 });
300
301    // Put data
302    List<Put> puts = Lists.newArrayList();
303    puts.add(new Put(row1).addColumn(Bytes.toBytes(cf1), Bytes.toBytes(cq1), Bytes.toBytes("a1")));
304    puts.add(new Put(row1).addColumn(Bytes.toBytes(cf2), Bytes.toBytes(cq2), Bytes.toBytes("a2")));
305    puts.add(new Put(row2).addColumn(Bytes.toBytes(cf1), Bytes.toBytes(cq1), Bytes.toBytes("b1")));
306    puts.add(new Put(row2).addColumn(Bytes.toBytes(cf2), Bytes.toBytes(cq2), Bytes.toBytes("b2")));
307    ht.put(puts);
308
309    TEST_UTIL.flush();
310
311    // FuzzyRowFilter
312    List<Pair<byte[], byte[]>> data = Lists.newArrayList();
313    byte[] fuzzyKey = Bytes.toBytes("1");
314    byte[] mask = new byte[] { 0 };
315    data.add(new Pair<>(fuzzyKey, mask));
316    FuzzyRowFilter fuzzyRowFilter = new FuzzyRowFilter(data);
317
318    // SingleColumnValueFilter
319    Filter singleColumnValueFilter = new SingleColumnValueFilter(Bytes.toBytes(cf2),
320      Bytes.toBytes(cq2), CompareOperator.EQUAL, Bytes.toBytes("x"));
321
322    // FilterList
323    FilterList filterList = new FilterList(Operator.MUST_PASS_ONE);
324    filterList.addFilter(Lists.newArrayList(fuzzyRowFilter, singleColumnValueFilter));
325
326    Scan scan = new Scan();
327    scan.setFilter(filterList);
328
329    ResultScanner scanner = ht.getScanner(scan);
330    Result rs = scanner.next();
331    assertEquals(0, Bytes.compareTo(row1, rs.getRow()));
332
333    // The two cells (1,f1,col1,a1) (1,f2,col2,a2)
334    assertEquals(2, rs.listCells().size());
335
336    // Only one row who's rowKey=1
337    assertNull(scanner.next());
338
339    TEST_UTIL.deleteTable(TableName.valueOf(name.getMethodName()));
340  }
341
342  @Test
343  public void testHBASE28634() throws IOException {
344    final String CF = "f";
345    final String CQ = "name";
346
347    Table ht = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), Bytes.toBytes(CF));
348
349    // Put data
350    List<Put> puts = Lists.newArrayList();
351    puts.add(new Put(Bytes.toBytes("111311")).addColumn(Bytes.toBytes(CF), Bytes.toBytes(CQ),
352      Bytes.toBytes("a1")));
353    puts.add(new Put(Bytes.toBytes("111444")).addColumn(Bytes.toBytes(CF), Bytes.toBytes(CQ),
354      Bytes.toBytes("a2")));
355    puts.add(new Put(Bytes.toBytes("111511")).addColumn(Bytes.toBytes(CF), Bytes.toBytes(CQ),
356      Bytes.toBytes("a3")));
357    puts.add(new Put(Bytes.toBytes("111611")).addColumn(Bytes.toBytes(CF), Bytes.toBytes(CQ),
358      Bytes.toBytes("a4")));
359    puts.add(new Put(Bytes.toBytes("111446")).addColumn(Bytes.toBytes(CF), Bytes.toBytes(CQ),
360      Bytes.toBytes("a5")));
361    puts.add(new Put(Bytes.toBytes("111777")).addColumn(Bytes.toBytes(CF), Bytes.toBytes(CQ),
362      Bytes.toBytes("a6")));
363    puts.add(new Put(Bytes.toBytes("111777")).addColumn(Bytes.toBytes(CF), Bytes.toBytes(CQ),
364      Bytes.toBytes("a")));
365    ht.put(puts);
366
367    TEST_UTIL.flush();
368
369    // Forward scan
370    LinkedList<Pair<byte[], byte[]>> fuzzyList = new LinkedList<>();
371    byte[] fuzzyRowKey = Bytes.toBytes("111433");
372    byte[] mask = Bytes.toBytesBinary("\\xFF\\xFF\\xFF\\xFF\\x02\\x02");
373    fuzzyList.add(new Pair<>(fuzzyRowKey, mask));
374    FuzzyRowFilter fuzzyRowFilter = new FuzzyRowFilter(fuzzyList);
375
376    Scan scan = new Scan();
377    scan.setFilter(fuzzyRowFilter);
378
379    ResultScanner scanner = ht.getScanner(scan);
380    List<byte[]> actualRowsList = new ArrayList<>();
381    for (Result result : scanner) {
382      byte[] row = result.getRow();
383      actualRowsList.add(row);
384    }
385
386    assertEquals(2, actualRowsList.size());
387
388    // Reverse scan
389    scan = new Scan();
390    scan.setFilter(fuzzyRowFilter);
391    scan.setReversed(true);
392
393    scanner = ht.getScanner(scan);
394    actualRowsList = new ArrayList<>();
395    for (Result result : scanner) {
396      byte[] row = result.getRow();
397      actualRowsList.add(row);
398    }
399
400    assertEquals(2, actualRowsList.size());
401
402    TEST_UTIL.deleteTable(TableName.valueOf(name.getMethodName()));
403  }
404}