001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.filter; 019 020import static org.junit.Assert.assertTrue; 021 022import java.io.IOException; 023import java.util.ArrayList; 024import java.util.Arrays; 025import java.util.List; 026 027import org.apache.commons.codec.binary.Hex; 028import org.apache.hadoop.hbase.Cell; 029import org.apache.hadoop.hbase.CellUtil; 030import org.apache.hadoop.hbase.CompareOperator; 031import org.apache.hadoop.hbase.HBaseClassTestRule; 032import org.apache.hadoop.hbase.HBaseTestingUtility; 033import org.apache.hadoop.hbase.TableName; 034import org.apache.hadoop.hbase.client.Put; 035import org.apache.hadoop.hbase.client.Result; 036import org.apache.hadoop.hbase.client.ResultScanner; 037import org.apache.hadoop.hbase.client.Scan; 038import org.apache.hadoop.hbase.client.Table; 039import org.apache.hadoop.hbase.testclassification.MediumTests; 040import org.apache.hadoop.hbase.util.Bytes; 041import org.junit.AfterClass; 042import org.junit.BeforeClass; 043import org.junit.ClassRule; 044import org.junit.Rule; 045import org.junit.Test; 046import org.junit.experimental.categories.Category; 047import org.junit.rules.TestName; 048import org.slf4j.Logger; 049import org.slf4j.LoggerFactory; 050 051@Category(MediumTests.class) 052public class TestFiltersWithBinaryComponentComparator { 053 054 /** 055 * See https://issues.apache.org/jira/browse/HBASE-22969 - for need of BinaryComponentComparator 056 * The descrption on jira should also help you in understanding tests implemented in this class 057 */ 058 059 @ClassRule 060 public static final HBaseClassTestRule CLASS_RULE = 061 HBaseClassTestRule.forClass(TestFiltersWithBinaryComponentComparator.class); 062 063 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); 064 private static final Logger LOG = 065 LoggerFactory.getLogger(TestFiltersWithBinaryComponentComparator.class); 066 private byte[] family = Bytes.toBytes("family"); 067 private byte[] qf = Bytes.toBytes("qf"); 068 private TableName tableName; 069 private int aOffset = 0; 070 private int bOffset = 4; 071 private int cOffset = 8; 072 private int dOffset = 12; 073 074 @Rule 075 public TestName name = new TestName(); 076 077 @BeforeClass 078 public static void setUpBeforeClass() throws Exception { 079 TEST_UTIL.startMiniCluster(); 080 } 081 082 @AfterClass 083 public static void tearDownAfterClass() throws Exception { 084 TEST_UTIL.shutdownMiniCluster(); 085 } 086 087 @Test 088 public void testRowFilterWithBinaryComponentComparator() throws IOException { 089 //SELECT * from table where a=1 and b > 10 and b < 20 and c > 90 and c < 100 and d=1 090 tableName = TableName.valueOf(name.getMethodName()); 091 Table ht = TEST_UTIL.createTable(tableName, family, Integer.MAX_VALUE); 092 generateRows(ht, family, qf); 093 FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL); 094 setRowFilters(filterList); 095 Scan scan = createScan(filterList); 096 List<Cell> result = getResults(ht,scan); 097 for(Cell cell: result){ 098 byte[] key = CellUtil.cloneRow(cell); 099 int a = Bytes.readAsInt(key,aOffset,4); 100 int b = Bytes.readAsInt(key,bOffset,4); 101 int c = Bytes.readAsInt(key,cOffset,4); 102 int d = Bytes.readAsInt(key,dOffset,4); 103 assertTrue(a == 1 && 104 b > 10 && 105 b < 20 && 106 c > 90 && 107 c < 100 && 108 d == 1); 109 } 110 ht.close(); 111 } 112 113 @Test 114 public void testValueFilterWithBinaryComponentComparator() throws IOException { 115 //SELECT * from table where value has 'y' at position 1 116 tableName = TableName.valueOf(name.getMethodName()); 117 Table ht = TEST_UTIL.createTable(tableName, family, Integer.MAX_VALUE); 118 generateRows(ht, family, qf); 119 FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL); 120 setValueFilters(filterList); 121 Scan scan = new Scan(); 122 scan.setFilter(filterList); 123 List<Cell> result = getResults(ht,scan); 124 for(Cell cell: result){ 125 byte[] value = CellUtil.cloneValue(cell); 126 assertTrue(Bytes.toString(value).charAt(1) == 'y'); 127 } 128 ht.close(); 129 } 130 131 @Test 132 public void testRowAndValueFilterWithBinaryComponentComparator() throws IOException { 133 //SELECT * from table where a=1 and b > 10 and b < 20 and c > 90 and c < 100 and d=1 134 //and value has 'y' at position 1" 135 tableName = TableName.valueOf(name.getMethodName()); 136 Table ht = TEST_UTIL.createTable(tableName, family, Integer.MAX_VALUE); 137 generateRows(ht, family, qf); 138 FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL); 139 setRowFilters(filterList); 140 setValueFilters(filterList); 141 Scan scan = new Scan(); 142 scan.setFilter(filterList); 143 List<Cell> result = getResults(ht,scan); 144 for(Cell cell: result){ 145 byte[] key = CellUtil.cloneRow(cell); 146 int a = Bytes.readAsInt(key,aOffset,4); 147 int b = Bytes.readAsInt(key,bOffset,4); 148 int c = Bytes.readAsInt(key,cOffset,4); 149 int d = Bytes.readAsInt(key,dOffset,4); 150 assertTrue(a == 1 && 151 b > 10 && 152 b < 20 && 153 c > 90 && 154 c < 100 && 155 d == 1); 156 byte[] value = CellUtil.cloneValue(cell); 157 assertTrue(Bytes.toString(value).charAt(1) == 'y'); 158 } 159 ht.close(); 160 } 161 162 /** 163 * Since we are trying to emulate 164 * SQL: SELECT * from table where a = 1 and b > 10 and b < 20 and 165 * c > 90 and c < 100 and d = 1 166 * We are generating rows with: 167 * a = 1, b >=9 and b < 22, c >= 89 and c < 102, and d = 1 168 * At the end the table will look something like this: 169 * ------------ 170 * a| b| c|d| 171 * ------------ 172 * 1| 9| 89|1|family:qf|xyz| 173 * ----------- 174 * 1| 9| 90|1|family:qf|abc| 175 * ----------- 176 * 1| 9| 91|1|family:qf|xyz| 177 * ------------------------- 178 * . 179 * ------------------------- 180 * . 181 * ------------------------- 182 * 1|21|101|1|family:qf|xyz| 183 */ 184 private void generateRows(Table ht, byte[] family, byte[] qf) 185 throws IOException { 186 for(int a = 1; a < 2; ++a) { 187 for(int b = 9; b < 22; ++b) { 188 for(int c = 89; c < 102; ++c) { 189 for(int d = 1; d < 2 ; ++d) { 190 byte[] key = new byte[16]; 191 Bytes.putInt(key,0,a); 192 Bytes.putInt(key,4,b); 193 Bytes.putInt(key,8,c); 194 Bytes.putInt(key,12,d); 195 Put row = new Put(key); 196 if (c%2==0) { 197 row.addColumn(family, qf, Bytes.toBytes("abc")); 198 if (LOG.isInfoEnabled()) { 199 LOG.info("added row: {} with value 'abc'", Arrays.toString(Hex.encodeHex(key))); 200 } 201 } else { 202 row.addColumn(family, qf, Bytes.toBytes("xyz")); 203 if (LOG.isInfoEnabled()) { 204 LOG.info("added row: {} with value 'xyz'", Arrays.toString(Hex.encodeHex(key))); 205 } 206 } 207 } 208 } 209 } 210 } 211 TEST_UTIL.flush(); 212 } 213 214 private void setRowFilters(FilterList filterList) { 215 //offset for b as it is second component of "a+b+c+d" 216 //'a' is at offset 0 217 int bOffset = 4; 218 byte[] b10 = Bytes.toBytes(10); //tests b > 10 219 Filter b10Filter = new RowFilter(CompareOperator.GREATER, 220 new BinaryComponentComparator(b10,bOffset)); 221 filterList.addFilter(b10Filter); 222 223 byte[] b20 = Bytes.toBytes(20); //tests b < 20 224 Filter b20Filter = new RowFilter(CompareOperator.LESS, 225 new BinaryComponentComparator(b20,bOffset)); 226 filterList.addFilter(b20Filter); 227 228 //offset for c as it is third component of "a+b+c+d" 229 int cOffset = 8; 230 byte[] c90 = Bytes.toBytes(90); //tests c > 90 231 Filter c90Filter = new RowFilter(CompareOperator.GREATER, 232 new BinaryComponentComparator(c90,cOffset)); 233 filterList.addFilter(c90Filter); 234 235 byte[] c100 = Bytes.toBytes(100); //tests c < 100 236 Filter c100Filter = new RowFilter(CompareOperator.LESS, 237 new BinaryComponentComparator(c100,cOffset)); 238 filterList.addFilter(c100Filter); 239 240 //offset for d as it is fourth component of "a+b+c+d" 241 int dOffset = 12; 242 byte[] d1 = Bytes.toBytes(1); //tests d == 1 243 Filter dFilter = new RowFilter(CompareOperator.EQUAL, 244 new BinaryComponentComparator(d1,dOffset)); 245 246 filterList.addFilter(dFilter); 247 248 } 249 250 /** 251 * We have rows with either "abc" or "xyz". 252 * We want values which have 'y' at second position 253 * of the string. 254 * As a result only values with "xyz" shall be returned 255 */ 256 private void setValueFilters(FilterList filterList) { 257 int offset = 1; 258 byte[] y = Bytes.toBytes("y"); 259 Filter yFilter = new ValueFilter(CompareOperator.EQUAL, 260 new BinaryComponentComparator(y,offset)); 261 filterList.addFilter(yFilter); 262 } 263 264 private Scan createScan(FilterList list) { 265 //build start and end key for scan 266 byte[] startKey = new byte[16]; //key size with four ints 267 Bytes.putInt(startKey,aOffset,1); //a=1, takes care of a = 1 268 Bytes.putInt(startKey,bOffset,11); //b=11, takes care of b > 10 269 Bytes.putInt(startKey,cOffset,91); //c=91, 270 Bytes.putInt(startKey,dOffset,1); //d=1, 271 272 byte[] endKey = new byte[16]; 273 Bytes.putInt(endKey,aOffset,1); //a=1, takes care of a = 1 274 Bytes.putInt(endKey,bOffset,20); //b=20, takes care of b < 20 275 Bytes.putInt(endKey,cOffset,100); //c=100, 276 Bytes.putInt(endKey,dOffset,1); //d=1, 277 278 //setup scan 279 Scan scan = new Scan().withStartRow(startKey).withStopRow(endKey); 280 scan.setFilter(list); 281 return scan; 282 } 283 284 private List<Cell> getResults(Table ht, Scan scan) throws IOException { 285 ResultScanner scanner = ht.getScanner(scan); 286 List<Cell> results = new ArrayList<>(); 287 Result r; 288 while ((r = scanner.next()) != null) { 289 for (Cell kv : r.listCells()) { 290 results.add(kv); 291 } 292 } 293 scanner.close(); 294 return results; 295 } 296 297}