001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.filter;
019
020import static org.junit.Assert.assertTrue;
021
022import java.io.IOException;
023import java.util.ArrayList;
024import java.util.Arrays;
025import java.util.List;
026
027import org.apache.commons.codec.binary.Hex;
028import org.apache.hadoop.hbase.Cell;
029import org.apache.hadoop.hbase.CellUtil;
030import org.apache.hadoop.hbase.CompareOperator;
031import org.apache.hadoop.hbase.HBaseClassTestRule;
032import org.apache.hadoop.hbase.HBaseTestingUtility;
033import org.apache.hadoop.hbase.TableName;
034import org.apache.hadoop.hbase.client.Put;
035import org.apache.hadoop.hbase.client.Result;
036import org.apache.hadoop.hbase.client.ResultScanner;
037import org.apache.hadoop.hbase.client.Scan;
038import org.apache.hadoop.hbase.client.Table;
039import org.apache.hadoop.hbase.testclassification.MediumTests;
040import org.apache.hadoop.hbase.util.Bytes;
041import org.junit.AfterClass;
042import org.junit.BeforeClass;
043import org.junit.ClassRule;
044import org.junit.Rule;
045import org.junit.Test;
046import org.junit.experimental.categories.Category;
047import org.junit.rules.TestName;
048import org.slf4j.Logger;
049import org.slf4j.LoggerFactory;
050
051@Category(MediumTests.class)
052public class TestFiltersWithBinaryComponentComparator {
053
054  /**
055   * See https://issues.apache.org/jira/browse/HBASE-22969 - for need of BinaryComponentComparator
056   * The descrption on jira should also help you in understanding tests implemented in this class
057   */
058
059  @ClassRule
060  public static final HBaseClassTestRule CLASS_RULE =
061      HBaseClassTestRule.forClass(TestFiltersWithBinaryComponentComparator.class);
062
063  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
064  private static final Logger LOG =
065      LoggerFactory.getLogger(TestFiltersWithBinaryComponentComparator.class);
066  private byte[] family = Bytes.toBytes("family");
067  private byte[] qf = Bytes.toBytes("qf");
068  private TableName tableName;
069  private int aOffset = 0;
070  private int bOffset = 4;
071  private int cOffset = 8;
072  private int dOffset = 12;
073
074  @Rule
075  public TestName name = new TestName();
076
077  @BeforeClass
078  public static void setUpBeforeClass() throws Exception {
079    TEST_UTIL.startMiniCluster();
080  }
081
082  @AfterClass
083  public static void tearDownAfterClass() throws Exception {
084    TEST_UTIL.shutdownMiniCluster();
085  }
086
087  @Test
088  public void testRowFilterWithBinaryComponentComparator() throws IOException {
089    //SELECT * from table where a=1 and b > 10 and b < 20 and c > 90 and c < 100 and d=1
090    tableName = TableName.valueOf(name.getMethodName());
091    Table ht = TEST_UTIL.createTable(tableName, family, Integer.MAX_VALUE);
092    generateRows(ht, family, qf);
093    FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL);
094    setRowFilters(filterList);
095    Scan scan = createScan(filterList);
096    List<Cell> result = getResults(ht,scan);
097    for(Cell cell: result){
098      byte[] key = CellUtil.cloneRow(cell);
099      int a = Bytes.readAsInt(key,aOffset,4);
100      int b = Bytes.readAsInt(key,bOffset,4);
101      int c = Bytes.readAsInt(key,cOffset,4);
102      int d = Bytes.readAsInt(key,dOffset,4);
103      assertTrue(a == 1 &&
104                 b > 10 &&
105                 b < 20 &&
106                 c > 90 &&
107                 c < 100 &&
108                 d == 1);
109    }
110    ht.close();
111  }
112
113  @Test
114  public void testValueFilterWithBinaryComponentComparator() throws IOException {
115    //SELECT * from table where value has 'y' at position 1
116    tableName = TableName.valueOf(name.getMethodName());
117    Table ht = TEST_UTIL.createTable(tableName, family, Integer.MAX_VALUE);
118    generateRows(ht, family, qf);
119    FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL);
120    setValueFilters(filterList);
121    Scan scan = new Scan();
122    scan.setFilter(filterList);
123    List<Cell> result = getResults(ht,scan);
124    for(Cell cell: result){
125      byte[] value = CellUtil.cloneValue(cell);
126      assertTrue(Bytes.toString(value).charAt(1) == 'y');
127    }
128    ht.close();
129  }
130
131  @Test
132  public void testRowAndValueFilterWithBinaryComponentComparator() throws IOException {
133    //SELECT * from table where a=1 and b > 10 and b < 20 and c > 90 and c < 100 and d=1
134    //and value has 'y' at position 1"
135    tableName = TableName.valueOf(name.getMethodName());
136    Table ht = TEST_UTIL.createTable(tableName, family, Integer.MAX_VALUE);
137    generateRows(ht, family, qf);
138    FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL);
139    setRowFilters(filterList);
140    setValueFilters(filterList);
141    Scan scan = new Scan();
142    scan.setFilter(filterList);
143    List<Cell> result = getResults(ht,scan);
144    for(Cell cell: result){
145      byte[] key = CellUtil.cloneRow(cell);
146      int a = Bytes.readAsInt(key,aOffset,4);
147      int b = Bytes.readAsInt(key,bOffset,4);
148      int c = Bytes.readAsInt(key,cOffset,4);
149      int d = Bytes.readAsInt(key,dOffset,4);
150      assertTrue(a == 1 &&
151                 b > 10 &&
152                 b < 20 &&
153                 c > 90 &&
154                 c < 100 &&
155                 d == 1);
156      byte[] value = CellUtil.cloneValue(cell);
157      assertTrue(Bytes.toString(value).charAt(1) == 'y');
158    }
159    ht.close();
160  }
161
162  /**
163   * Since we are trying to emulate
164   * SQL: SELECT * from table where a = 1 and b > 10 and b < 20 and
165   * c > 90 and c < 100 and d = 1
166   * We are generating rows with:
167   * a = 1, b >=9 and b < 22, c >= 89 and c < 102, and d = 1
168   * At the end the table will look something like this:
169   * ------------
170   *  a| b|  c|d|
171   * ------------
172   *  1| 9| 89|1|family:qf|xyz|
173   *  -----------
174   *  1| 9| 90|1|family:qf|abc|
175   *  -----------
176   *  1| 9| 91|1|family:qf|xyz|
177   *  -------------------------
178   *  .
179   *  -------------------------
180   *  .
181   *  -------------------------
182   *  1|21|101|1|family:qf|xyz|
183   */
184  private void generateRows(Table ht, byte[] family, byte[] qf)
185      throws IOException {
186    for(int a = 1; a < 2; ++a) {
187      for(int b = 9; b < 22; ++b) {
188        for(int c = 89; c < 102; ++c) {
189          for(int d = 1; d < 2 ; ++d) {
190            byte[] key = new byte[16];
191            Bytes.putInt(key,0,a);
192            Bytes.putInt(key,4,b);
193            Bytes.putInt(key,8,c);
194            Bytes.putInt(key,12,d);
195            Put row = new Put(key);
196            if (c%2==0) {
197              row.addColumn(family, qf, Bytes.toBytes("abc"));
198              if (LOG.isInfoEnabled()) {
199                LOG.info("added row: {} with value 'abc'", Arrays.toString(Hex.encodeHex(key)));
200              }
201            } else {
202              row.addColumn(family, qf, Bytes.toBytes("xyz"));
203              if (LOG.isInfoEnabled()) {
204                LOG.info("added row: {} with value 'xyz'", Arrays.toString(Hex.encodeHex(key)));
205              }
206            }
207          }
208        }
209      }
210    }
211    TEST_UTIL.flush();
212  }
213
214  private void setRowFilters(FilterList filterList) {
215    //offset for b as it is second component of "a+b+c+d"
216    //'a' is at offset 0
217    int bOffset = 4;
218    byte[] b10 = Bytes.toBytes(10); //tests b > 10
219    Filter b10Filter = new RowFilter(CompareOperator.GREATER,
220            new BinaryComponentComparator(b10,bOffset));
221    filterList.addFilter(b10Filter);
222
223    byte[] b20  = Bytes.toBytes(20); //tests b < 20
224    Filter b20Filter = new RowFilter(CompareOperator.LESS,
225            new BinaryComponentComparator(b20,bOffset));
226    filterList.addFilter(b20Filter);
227
228    //offset for c as it is third component of "a+b+c+d"
229    int cOffset = 8;
230    byte[] c90  = Bytes.toBytes(90); //tests c > 90
231    Filter c90Filter = new RowFilter(CompareOperator.GREATER,
232            new BinaryComponentComparator(c90,cOffset));
233    filterList.addFilter(c90Filter);
234
235    byte[] c100  = Bytes.toBytes(100); //tests c < 100
236    Filter c100Filter = new RowFilter(CompareOperator.LESS,
237            new BinaryComponentComparator(c100,cOffset));
238    filterList.addFilter(c100Filter);
239
240    //offset for d as it is fourth component of "a+b+c+d"
241    int dOffset = 12;
242    byte[] d1   = Bytes.toBytes(1); //tests d == 1
243    Filter dFilter  = new RowFilter(CompareOperator.EQUAL,
244            new BinaryComponentComparator(d1,dOffset));
245
246    filterList.addFilter(dFilter);
247
248  }
249
250  /**
251   * We have rows with either "abc" or "xyz".
252   * We want values which have 'y' at second position
253   * of the string.
254   * As a result only values with "xyz" shall be returned
255  */
256  private void setValueFilters(FilterList filterList) {
257    int offset = 1;
258    byte[] y = Bytes.toBytes("y");
259    Filter yFilter  = new ValueFilter(CompareOperator.EQUAL,
260            new BinaryComponentComparator(y,offset));
261    filterList.addFilter(yFilter);
262  }
263
264  private Scan createScan(FilterList list) {
265    //build start and end key for scan
266    byte[] startKey = new byte[16]; //key size with four ints
267    Bytes.putInt(startKey,aOffset,1); //a=1, takes care of a = 1
268    Bytes.putInt(startKey,bOffset,11); //b=11, takes care of b > 10
269    Bytes.putInt(startKey,cOffset,91); //c=91,
270    Bytes.putInt(startKey,dOffset,1); //d=1,
271
272    byte[] endKey = new byte[16];
273    Bytes.putInt(endKey,aOffset,1); //a=1, takes care of a = 1
274    Bytes.putInt(endKey,bOffset,20); //b=20, takes care of b < 20
275    Bytes.putInt(endKey,cOffset,100); //c=100,
276    Bytes.putInt(endKey,dOffset,1); //d=1,
277
278    //setup scan
279    Scan scan = new Scan().withStartRow(startKey).withStopRow(endKey);
280    scan.setFilter(list);
281    return scan;
282  }
283
284  private List<Cell> getResults(Table ht, Scan scan) throws IOException {
285    ResultScanner scanner = ht.getScanner(scan);
286    List<Cell> results = new ArrayList<>();
287    Result r;
288    while ((r = scanner.next()) != null) {
289      for (Cell kv : r.listCells()) {
290        results.add(kv);
291      }
292    }
293    scanner.close();
294    return results;
295  }
296
297}