Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.filter;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNull;
022
023import java.io.IOException;
024import java.nio.ByteBuffer;
025import java.util.ArrayList;
026import java.util.Arrays;
027import java.util.List;
028import org.apache.hadoop.conf.Configuration;
029import org.apache.hadoop.hbase.Cell;
030import org.apache.hadoop.hbase.CellUtil;
031import org.apache.hadoop.hbase.CompareOperator;
032import org.apache.hadoop.hbase.HBaseClassTestRule;
033import org.apache.hadoop.hbase.HBaseTestingUtil;
034import org.apache.hadoop.hbase.HConstants;
035import org.apache.hadoop.hbase.TableName;
036import org.apache.hadoop.hbase.client.Durability;
037import org.apache.hadoop.hbase.client.Put;
038import org.apache.hadoop.hbase.client.Result;
039import org.apache.hadoop.hbase.client.ResultScanner;
040import org.apache.hadoop.hbase.client.Scan;
041import org.apache.hadoop.hbase.client.Table;
042import org.apache.hadoop.hbase.filter.FilterList.Operator;
043import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
044import org.apache.hadoop.hbase.testclassification.FilterTests;
045import org.apache.hadoop.hbase.testclassification.MediumTests;
046import org.apache.hadoop.hbase.util.Bytes;
047import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
048import org.apache.hadoop.hbase.util.Pair;
049import org.junit.AfterClass;
050import org.junit.BeforeClass;
051import org.junit.ClassRule;
052import org.junit.Rule;
053import org.junit.Test;
054import org.junit.experimental.categories.Category;
055import org.junit.rules.TestName;
056import org.slf4j.Logger;
057import org.slf4j.LoggerFactory;
058
059import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
060
061@Category({ FilterTests.class, MediumTests.class })
062public class TestFuzzyRowFilterEndToEnd {
063
064  @ClassRule
065  public static final HBaseClassTestRule CLASS_RULE =
066    HBaseClassTestRule.forClass(TestFuzzyRowFilterEndToEnd.class);
067
068  private static final Logger LOG = LoggerFactory.getLogger(TestFuzzyRowFilterEndToEnd.class);
069
070  private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
071
072  private static final byte fuzzyValue = (byte) 63;
073
074  @Rule
075  public TestName name = new TestName();
076
077  @BeforeClass
078  public static void setUpBeforeClass() throws Exception {
079    Configuration conf = TEST_UTIL.getConfiguration();
080    conf.setInt("hbase.client.scanner.caching", 1000);
081    conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
082      ConstantSizeRegionSplitPolicy.class.getName());
083    // set no splits
084    conf.setLong(HConstants.HREGION_MAX_FILESIZE, (1024L) * 1024 * 1024 * 10);
085
086    TEST_UTIL.startMiniCluster();
087  }
088
089  @AfterClass
090  public static void tearDownAfterClass() throws Exception {
091    TEST_UTIL.shutdownMiniCluster();
092  }
093
094  // HBASE-15676 Test that fuzzy info of all fixed bits (0s) finds matching row.
095  @Test
096  public void testAllFixedBits() throws IOException {
097    String cf = "f";
098    String cq = "q";
099
100    Table ht = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), Bytes.toBytes(cf),
101      Integer.MAX_VALUE);
102    // Load data
103    String[] rows = new String[] { "\\x9C\\x00\\x044\\x00\\x00\\x00\\x00",
104      "\\x9C\\x00\\x044\\x01\\x00\\x00\\x00", "\\x9C\\x00\\x044\\x00\\x01\\x00\\x00",
105      "\\x9B\\x00\\x044e\\x9B\\x02\\xBB", "\\x9C\\x00\\x044\\x00\\x00\\x01\\x00",
106      "\\x9C\\x00\\x044\\x00\\x01\\x00\\x01", "\\x9B\\x00\\x044e\\xBB\\xB2\\xBB", };
107
108    for (int i = 0; i < rows.length; i++) {
109      Put p = new Put(Bytes.toBytesBinary(rows[i]));
110      p.addColumn(Bytes.toBytes(cf), Bytes.toBytes(cq), Bytes.toBytes("value"));
111      ht.put(p);
112    }
113
114    TEST_UTIL.flush();
115
116    List<Pair<byte[], byte[]>> data = new ArrayList<>();
117    byte[] fuzzyKey = Bytes.toBytesBinary("\\x9B\\x00\\x044e");
118    byte[] mask = new byte[] { 0, 0, 0, 0, 0 };
119
120    // copy the fuzzy key and mask to test HBASE-18617
121    byte[] copyFuzzyKey = Arrays.copyOf(fuzzyKey, fuzzyKey.length);
122    byte[] copyMask = Arrays.copyOf(mask, mask.length);
123
124    data.add(new Pair<>(fuzzyKey, mask));
125    FuzzyRowFilter filter = new FuzzyRowFilter(data);
126
127    Scan scan = new Scan();
128    scan.setFilter(filter);
129
130    ResultScanner scanner = ht.getScanner(scan);
131    int total = 0;
132    while (scanner.next() != null) {
133      total++;
134    }
135    assertEquals(2, total);
136
137    assertEquals(true, Arrays.equals(copyFuzzyKey, fuzzyKey));
138    assertEquals(true, Arrays.equals(copyMask, mask));
139
140    TEST_UTIL.deleteTable(TableName.valueOf(name.getMethodName()));
141  }
142
143  @Test
144  public void testHBASE14782() throws IOException {
145    String cf = "f";
146    String cq = "q";
147
148    Table ht = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), Bytes.toBytes(cf),
149      Integer.MAX_VALUE);
150    // Load data
151    String[] rows =
152      new String[] { "\\x9C\\x00\\x044\\x00\\x00\\x00\\x00", "\\x9C\\x00\\x044\\x01\\x00\\x00\\x00",
153        "\\x9C\\x00\\x044\\x00\\x01\\x00\\x00", "\\x9C\\x00\\x044\\x00\\x00\\x01\\x00",
154        "\\x9C\\x00\\x044\\x00\\x01\\x00\\x01", "\\x9B\\x00\\x044e\\xBB\\xB2\\xBB", };
155
156    String badRow = "\\x9C\\x00\\x03\\xE9e\\xBB{X\\x1Fwts\\x1F\\x15vRX";
157
158    for (int i = 0; i < rows.length; i++) {
159      Put p = new Put(Bytes.toBytesBinary(rows[i]));
160      p.addColumn(Bytes.toBytes(cf), Bytes.toBytes(cq), Bytes.toBytes("value"));
161      ht.put(p);
162    }
163
164    Put p = new Put(Bytes.toBytesBinary(badRow));
165    p.addColumn(Bytes.toBytes(cf), Bytes.toBytes(cq), Bytes.toBytes("value"));
166    ht.put(p);
167
168    TEST_UTIL.flush();
169
170    List<Pair<byte[], byte[]>> data = new ArrayList<>();
171    byte[] fuzzyKey = Bytes.toBytesBinary("\\x00\\x00\\x044");
172    byte[] mask = new byte[] { 1, 0, 0, 0 };
173    data.add(new Pair<>(fuzzyKey, mask));
174    FuzzyRowFilter filter = new FuzzyRowFilter(data);
175
176    Scan scan = new Scan();
177    scan.setFilter(filter);
178
179    ResultScanner scanner = ht.getScanner(scan);
180    int total = 0;
181    while (scanner.next() != null) {
182      total++;
183    }
184    assertEquals(rows.length, total);
185    TEST_UTIL.deleteTable(TableName.valueOf(name.getMethodName()));
186  }
187
188  @Test
189  public void testFilterList() throws Exception {
190    String cf = "f";
191    Table ht = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), Bytes.toBytes(cf),
192      Integer.MAX_VALUE);
193
194    // 10 byte row key - (2 bytes 4 bytes 4 bytes)
195    // 4 byte qualifier
196    // 4 byte value
197
198    for (int i1 = 0; i1 < 5; i1++) {
199      for (int i2 = 0; i2 < 5; i2++) {
200        byte[] rk = new byte[10];
201
202        ByteBuffer buf = ByteBuffer.wrap(rk);
203        buf.clear();
204        buf.putShort((short) 2);
205        buf.putInt(i1);
206        buf.putInt(i2);
207
208        // Each row contains 5 columns
209        for (int c = 0; c < 5; c++) {
210          byte[] cq = new byte[4];
211          Bytes.putBytes(cq, 0, Bytes.toBytes(c), 0, 4);
212
213          Put p = new Put(rk);
214          p.setDurability(Durability.SKIP_WAL);
215          p.addColumn(Bytes.toBytes(cf), cq, Bytes.toBytes(c));
216          ht.put(p);
217          LOG.info(
218            "Inserting: rk: " + Bytes.toStringBinary(rk) + " cq: " + Bytes.toStringBinary(cq));
219        }
220      }
221    }
222
223    TEST_UTIL.flush();
224
225    // test passes if we get back 5 KV's (1 row)
226    runTest(ht, 5);
227
228  }
229
230  private void runTest(Table hTable, int expectedSize) throws IOException {
231    // [0, 2, ?, ?, ?, ?, 0, 0, 0, 1]
232    byte[] fuzzyKey1 = new byte[10];
233    ByteBuffer buf = ByteBuffer.wrap(fuzzyKey1);
234    buf.clear();
235    buf.putShort((short) 2);
236    for (int i = 0; i < 4; i++)
237      buf.put(fuzzyValue);
238    buf.putInt((short) 1);
239    byte[] mask1 = new byte[] { 0, 0, 1, 1, 1, 1, 0, 0, 0, 0 };
240
241    byte[] fuzzyKey2 = new byte[10];
242    buf = ByteBuffer.wrap(fuzzyKey2);
243    buf.clear();
244    buf.putShort((short) 2);
245    buf.putInt((short) 2);
246    for (int i = 0; i < 4; i++)
247      buf.put(fuzzyValue);
248
249    byte[] mask2 = new byte[] { 0, 0, 0, 0, 0, 0, 1, 1, 1, 1 };
250
251    Pair<byte[], byte[]> pair1 = new Pair<>(fuzzyKey1, mask1);
252    Pair<byte[], byte[]> pair2 = new Pair<>(fuzzyKey2, mask2);
253
254    FuzzyRowFilter fuzzyRowFilter1 = new FuzzyRowFilter(Lists.newArrayList(pair1));
255    FuzzyRowFilter fuzzyRowFilter2 = new FuzzyRowFilter(Lists.newArrayList(pair2));
256    // regular test - we expect 1 row back (5 KVs)
257    runScanner(hTable, expectedSize, fuzzyRowFilter1, fuzzyRowFilter2);
258  }
259
260  private void runScanner(Table hTable, int expectedSize, Filter filter1, Filter filter2)
261    throws IOException {
262    String cf = "f";
263    Scan scan = new Scan();
264    scan.addFamily(Bytes.toBytes(cf));
265    FilterList filterList = new FilterList(Operator.MUST_PASS_ALL, filter1, filter2);
266    scan.setFilter(filterList);
267
268    ResultScanner scanner = hTable.getScanner(scan);
269    List<Cell> results = new ArrayList<>();
270    Result result;
271    long timeBeforeScan = EnvironmentEdgeManager.currentTime();
272    while ((result = scanner.next()) != null) {
273      for (Cell kv : result.listCells()) {
274        LOG.info("Got rk: " + Bytes.toStringBinary(CellUtil.cloneRow(kv)) + " cq: "
275          + Bytes.toStringBinary(CellUtil.cloneQualifier(kv)));
276        results.add(kv);
277      }
278    }
279    long scanTime = EnvironmentEdgeManager.currentTime() - timeBeforeScan;
280    scanner.close();
281
282    LOG.info("scan time = " + scanTime + "ms");
283    LOG.info("found " + results.size() + " results");
284
285    assertEquals(expectedSize, results.size());
286  }
287
288  @Test
289  public void testHBASE26967() throws IOException {
290    byte[] row1 = Bytes.toBytes("1");
291    byte[] row2 = Bytes.toBytes("2");
292    String cf1 = "f1";
293    String cf2 = "f2";
294    String cq1 = "col1";
295    String cq2 = "col2";
296
297    Table ht =
298      TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), new String[] { cf1, cf2 });
299
300    // Put data
301    List<Put> puts = Lists.newArrayList();
302    puts.add(new Put(row1).addColumn(Bytes.toBytes(cf1), Bytes.toBytes(cq1), Bytes.toBytes("a1")));
303    puts.add(new Put(row1).addColumn(Bytes.toBytes(cf2), Bytes.toBytes(cq2), Bytes.toBytes("a2")));
304    puts.add(new Put(row2).addColumn(Bytes.toBytes(cf1), Bytes.toBytes(cq1), Bytes.toBytes("b1")));
305    puts.add(new Put(row2).addColumn(Bytes.toBytes(cf2), Bytes.toBytes(cq2), Bytes.toBytes("b2")));
306    ht.put(puts);
307
308    TEST_UTIL.flush();
309
310    // FuzzyRowFilter
311    List<Pair<byte[], byte[]>> data = Lists.newArrayList();
312    byte[] fuzzyKey = Bytes.toBytes("1");
313    byte[] mask = new byte[] { 0 };
314    data.add(new Pair<>(fuzzyKey, mask));
315    FuzzyRowFilter fuzzyRowFilter = new FuzzyRowFilter(data);
316
317    // SingleColumnValueFilter
318    Filter singleColumnValueFilter = new SingleColumnValueFilter(Bytes.toBytes(cf2),
319      Bytes.toBytes(cq2), CompareOperator.EQUAL, Bytes.toBytes("x"));
320
321    // FilterList
322    FilterList filterList = new FilterList(Operator.MUST_PASS_ONE);
323    filterList.addFilter(Lists.newArrayList(fuzzyRowFilter, singleColumnValueFilter));
324
325    Scan scan = new Scan();
326    scan.setFilter(filterList);
327
328    ResultScanner scanner = ht.getScanner(scan);
329    Result rs = scanner.next();
330    assertEquals(0, Bytes.compareTo(row1, rs.getRow()));
331
332    // The two cells (1,f1,col1,a1) (1,f2,col2,a2)
333    assertEquals(2, rs.listCells().size());
334
335    // Only one row who's rowKey=1
336    assertNull(scanner.next());
337
338    TEST_UTIL.deleteTable(TableName.valueOf(name.getMethodName()));
339  }
340}