001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.filter; 019 020import static org.junit.Assert.assertTrue; 021 022import java.io.IOException; 023import java.util.ArrayList; 024import java.util.Arrays; 025import java.util.List; 026import org.apache.commons.codec.binary.Hex; 027import org.apache.hadoop.hbase.Cell; 028import org.apache.hadoop.hbase.CellUtil; 029import org.apache.hadoop.hbase.CompareOperator; 030import org.apache.hadoop.hbase.HBaseClassTestRule; 031import org.apache.hadoop.hbase.HBaseTestingUtility; 032import org.apache.hadoop.hbase.TableName; 033import org.apache.hadoop.hbase.client.Put; 034import org.apache.hadoop.hbase.client.Result; 035import org.apache.hadoop.hbase.client.ResultScanner; 036import org.apache.hadoop.hbase.client.Scan; 037import org.apache.hadoop.hbase.client.Table; 038import org.apache.hadoop.hbase.testclassification.MediumTests; 039import org.apache.hadoop.hbase.util.Bytes; 040import org.junit.AfterClass; 041import org.junit.BeforeClass; 042import org.junit.ClassRule; 043import org.junit.Rule; 044import org.junit.Test; 045import org.junit.experimental.categories.Category; 046import org.junit.rules.TestName; 047import org.slf4j.Logger; 048import org.slf4j.LoggerFactory; 049 050@Category(MediumTests.class) 051public class TestFiltersWithBinaryComponentComparator { 052 053 /** 054 * See https://issues.apache.org/jira/browse/HBASE-22969 - for need of BinaryComponentComparator 055 * The descrption on jira should also help you in understanding tests implemented in this class 056 */ 057 058 @ClassRule 059 public static final HBaseClassTestRule CLASS_RULE = 060 HBaseClassTestRule.forClass(TestFiltersWithBinaryComponentComparator.class); 061 062 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); 063 private static final Logger LOG = 064 LoggerFactory.getLogger(TestFiltersWithBinaryComponentComparator.class); 065 private byte[] family = Bytes.toBytes("family"); 066 private byte[] qf = Bytes.toBytes("qf"); 067 private TableName tableName; 068 private int aOffset = 0; 069 private int bOffset = 4; 070 private int cOffset = 8; 071 private int dOffset = 12; 072 073 @Rule 074 public TestName name = new TestName(); 075 076 @BeforeClass 077 public static void setUpBeforeClass() throws Exception { 078 TEST_UTIL.startMiniCluster(); 079 } 080 081 @AfterClass 082 public static void tearDownAfterClass() throws Exception { 083 TEST_UTIL.shutdownMiniCluster(); 084 } 085 086 @Test 087 public void testRowFilterWithBinaryComponentComparator() throws IOException { 088 // SELECT * from table where a=1 and b > 10 and b < 20 and c > 90 and c < 100 and d=1 089 tableName = TableName.valueOf(name.getMethodName()); 090 Table ht = TEST_UTIL.createTable(tableName, family, Integer.MAX_VALUE); 091 generateRows(ht, family, qf); 092 FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL); 093 setRowFilters(filterList); 094 Scan scan = createScan(filterList); 095 List<Cell> result = getResults(ht, scan); 096 for (Cell cell : result) { 097 byte[] key = CellUtil.cloneRow(cell); 098 int a = Bytes.readAsInt(key, aOffset, 4); 099 int b = Bytes.readAsInt(key, bOffset, 4); 100 int c = Bytes.readAsInt(key, cOffset, 4); 101 int d = Bytes.readAsInt(key, dOffset, 4); 102 assertTrue(a == 1 && b > 10 && b < 20 && c > 90 && c < 100 && d == 1); 103 } 104 ht.close(); 105 } 106 107 @Test 108 public void testValueFilterWithBinaryComponentComparator() throws IOException { 109 // SELECT * from table where value has 'y' at position 1 110 tableName = TableName.valueOf(name.getMethodName()); 111 Table ht = TEST_UTIL.createTable(tableName, family, Integer.MAX_VALUE); 112 generateRows(ht, family, qf); 113 FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL); 114 setValueFilters(filterList); 115 Scan scan = new Scan(); 116 scan.setFilter(filterList); 117 List<Cell> result = getResults(ht, scan); 118 for (Cell cell : result) { 119 byte[] value = CellUtil.cloneValue(cell); 120 assertTrue(Bytes.toString(value).charAt(1) == 'y'); 121 } 122 ht.close(); 123 } 124 125 @Test 126 public void testRowAndValueFilterWithBinaryComponentComparator() throws IOException { 127 // SELECT * from table where a=1 and b > 10 and b < 20 and c > 90 and c < 100 and d=1 128 // and value has 'y' at position 1" 129 tableName = TableName.valueOf(name.getMethodName()); 130 Table ht = TEST_UTIL.createTable(tableName, family, Integer.MAX_VALUE); 131 generateRows(ht, family, qf); 132 FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL); 133 setRowFilters(filterList); 134 setValueFilters(filterList); 135 Scan scan = new Scan(); 136 scan.setFilter(filterList); 137 List<Cell> result = getResults(ht, scan); 138 for (Cell cell : result) { 139 byte[] key = CellUtil.cloneRow(cell); 140 int a = Bytes.readAsInt(key, aOffset, 4); 141 int b = Bytes.readAsInt(key, bOffset, 4); 142 int c = Bytes.readAsInt(key, cOffset, 4); 143 int d = Bytes.readAsInt(key, dOffset, 4); 144 assertTrue(a == 1 && b > 10 && b < 20 && c > 90 && c < 100 && d == 1); 145 byte[] value = CellUtil.cloneValue(cell); 146 assertTrue(Bytes.toString(value).charAt(1) == 'y'); 147 } 148 ht.close(); 149 } 150 151 /** 152 * Since we are trying to emulate SQL: SELECT * from table where a = 1 and b > 10 and b < 20 and c 153 * > 90 and c < 100 and d = 1 We are generating rows with: a = 1, b >=9 and b < 22, c >= 89 and c 154 * < 102, and d = 1 At the end the table will look something like this: ------------ a| b| c|d| 155 * ------------ 1| 9| 89|1|family:qf|xyz| ----------- 1| 9| 90|1|family:qf|abc| ----------- 1| 9| 156 * 91|1|family:qf|xyz| ------------------------- . ------------------------- . 157 * ------------------------- 1|21|101|1|family:qf|xyz| 158 */ 159 private void generateRows(Table ht, byte[] family, byte[] qf) throws IOException { 160 for (int a = 1; a < 2; ++a) { 161 for (int b = 9; b < 22; ++b) { 162 for (int c = 89; c < 102; ++c) { 163 for (int d = 1; d < 2; ++d) { 164 byte[] key = new byte[16]; 165 Bytes.putInt(key, 0, a); 166 Bytes.putInt(key, 4, b); 167 Bytes.putInt(key, 8, c); 168 Bytes.putInt(key, 12, d); 169 Put row = new Put(key); 170 if (c % 2 == 0) { 171 row.addColumn(family, qf, Bytes.toBytes("abc")); 172 if (LOG.isInfoEnabled()) { 173 LOG.info("added row: {} with value 'abc'", Arrays.toString(Hex.encodeHex(key))); 174 } 175 } else { 176 row.addColumn(family, qf, Bytes.toBytes("xyz")); 177 if (LOG.isInfoEnabled()) { 178 LOG.info("added row: {} with value 'xyz'", Arrays.toString(Hex.encodeHex(key))); 179 } 180 } 181 } 182 } 183 } 184 } 185 TEST_UTIL.flush(); 186 } 187 188 private void setRowFilters(FilterList filterList) { 189 // offset for b as it is second component of "a+b+c+d" 190 // 'a' is at offset 0 191 int bOffset = 4; 192 byte[] b10 = Bytes.toBytes(10); // tests b > 10 193 Filter b10Filter = 194 new RowFilter(CompareOperator.GREATER, new BinaryComponentComparator(b10, bOffset)); 195 filterList.addFilter(b10Filter); 196 197 byte[] b20 = Bytes.toBytes(20); // tests b < 20 198 Filter b20Filter = 199 new RowFilter(CompareOperator.LESS, new BinaryComponentComparator(b20, bOffset)); 200 filterList.addFilter(b20Filter); 201 202 // offset for c as it is third component of "a+b+c+d" 203 int cOffset = 8; 204 byte[] c90 = Bytes.toBytes(90); // tests c > 90 205 Filter c90Filter = 206 new RowFilter(CompareOperator.GREATER, new BinaryComponentComparator(c90, cOffset)); 207 filterList.addFilter(c90Filter); 208 209 byte[] c100 = Bytes.toBytes(100); // tests c < 100 210 Filter c100Filter = 211 new RowFilter(CompareOperator.LESS, new BinaryComponentComparator(c100, cOffset)); 212 filterList.addFilter(c100Filter); 213 214 // offset for d as it is fourth component of "a+b+c+d" 215 int dOffset = 12; 216 byte[] d1 = Bytes.toBytes(1); // tests d == 1 217 Filter dFilter = 218 new RowFilter(CompareOperator.EQUAL, new BinaryComponentComparator(d1, dOffset)); 219 220 filterList.addFilter(dFilter); 221 222 } 223 224 /** 225 * We have rows with either "abc" or "xyz". We want values which have 'y' at second position of 226 * the string. As a result only values with "xyz" shall be returned 227 */ 228 private void setValueFilters(FilterList filterList) { 229 int offset = 1; 230 byte[] y = Bytes.toBytes("y"); 231 Filter yFilter = 232 new ValueFilter(CompareOperator.EQUAL, new BinaryComponentComparator(y, offset)); 233 filterList.addFilter(yFilter); 234 } 235 236 private Scan createScan(FilterList list) { 237 // build start and end key for scan 238 byte[] startKey = new byte[16]; // key size with four ints 239 Bytes.putInt(startKey, aOffset, 1); // a=1, takes care of a = 1 240 Bytes.putInt(startKey, bOffset, 11); // b=11, takes care of b > 10 241 Bytes.putInt(startKey, cOffset, 91); // c=91, 242 Bytes.putInt(startKey, dOffset, 1); // d=1, 243 244 byte[] endKey = new byte[16]; 245 Bytes.putInt(endKey, aOffset, 1); // a=1, takes care of a = 1 246 Bytes.putInt(endKey, bOffset, 20); // b=20, takes care of b < 20 247 Bytes.putInt(endKey, cOffset, 100); // c=100, 248 Bytes.putInt(endKey, dOffset, 1); // d=1, 249 250 // setup scan 251 Scan scan = new Scan().withStartRow(startKey).withStopRow(endKey); 252 scan.setFilter(list); 253 return scan; 254 } 255 256 private List<Cell> getResults(Table ht, Scan scan) throws IOException { 257 ResultScanner scanner = ht.getScanner(scan); 258 List<Cell> results = new ArrayList<>(); 259 Result r; 260 while ((r = scanner.next()) != null) { 261 for (Cell kv : r.listCells()) { 262 results.add(kv); 263 } 264 } 265 scanner.close(); 266 return results; 267 } 268 269}