001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.filter; 019 020import static org.junit.Assert.*; 021 022import java.io.IOException; 023import java.util.ArrayList; 024import java.util.HashMap; 025import java.util.HashSet; 026import java.util.List; 027import java.util.Map; 028import java.util.Objects; 029import java.util.Set; 030 031import org.apache.hadoop.hbase.Cell; 032import org.apache.hadoop.hbase.HBaseClassTestRule; 033import org.apache.hadoop.hbase.HBaseTestingUtility; 034import org.apache.hadoop.hbase.KeyValue; 035import org.apache.hadoop.hbase.KeyValueTestUtil; 036import org.apache.hadoop.hbase.TableName; 037import org.apache.hadoop.hbase.client.Durability; 038import org.apache.hadoop.hbase.client.Put; 039import org.apache.hadoop.hbase.client.Result; 040import org.apache.hadoop.hbase.client.ResultScanner; 041import org.apache.hadoop.hbase.client.Scan; 042import org.apache.hadoop.hbase.client.Table; 043import org.apache.hadoop.hbase.filter.FilterList.Operator; 044import org.apache.hadoop.hbase.testclassification.FilterTests; 045import org.apache.hadoop.hbase.testclassification.MediumTests; 046import org.apache.hadoop.hbase.util.Bytes; 047import org.junit.After; 048import org.junit.AfterClass; 049import org.junit.Before; 050import org.junit.BeforeClass; 051import org.junit.ClassRule; 052import org.junit.Rule; 053import org.junit.Test; 054import org.junit.experimental.categories.Category; 055import org.junit.rules.TestName; 056import org.slf4j.Logger; 057import org.slf4j.LoggerFactory; 058 059class StringRange { 060 private String start = null; 061 private String end = null; 062 private boolean startInclusive = true; 063 private boolean endInclusive = false; 064 065 public StringRange(String start, boolean startInclusive, String end, 066 boolean endInclusive) { 067 this.start = start; 068 this.startInclusive = startInclusive; 069 this.end = end; 070 this.endInclusive = endInclusive; 071 } 072 073 public String getStart() { 074 return this.start; 075 } 076 077 public String getEnd() { 078 return this.end; 079 } 080 081 public boolean isStartInclusive() { 082 return this.startInclusive; 083 } 084 085 public boolean isEndInclusive() { 086 return this.endInclusive; 087 } 088 089 @Override 090 public int hashCode() { 091 int hashCode = 0; 092 if (this.start != null) { 093 hashCode ^= this.start.hashCode(); 094 } 095 096 if (this.end != null) { 097 hashCode ^= this.end.hashCode(); 098 } 099 return hashCode; 100 } 101 102 @Override 103 public boolean equals(Object obj) { 104 if (this == obj) { 105 return true; 106 } 107 if (obj == null) { 108 return false; 109 } 110 if (!(obj instanceof StringRange)) { 111 return false; 112 } 113 StringRange oth = (StringRange) obj; 114 return this.startInclusive == oth.startInclusive && 115 this.endInclusive == oth.endInclusive && 116 Objects.equals(this.start, oth.start) && 117 Objects.equals(this.end, oth.end); 118 } 119 120 @Override 121 public String toString() { 122 String result = (this.startInclusive ? "[" : "(") 123 + (this.start == null ? null : this.start) + ", " 124 + (this.end == null ? null : this.end) 125 + (this.endInclusive ? "]" : ")"); 126 return result; 127 } 128 129 public boolean inRange(String value) { 130 boolean afterStart = true; 131 if (this.start != null) { 132 int startCmp = value.compareTo(this.start); 133 afterStart = this.startInclusive ? startCmp >= 0 : startCmp > 0; 134 } 135 136 boolean beforeEnd = true; 137 if (this.end != null) { 138 int endCmp = value.compareTo(this.end); 139 beforeEnd = this.endInclusive ? endCmp <= 0 : endCmp < 0; 140 } 141 142 return afterStart && beforeEnd; 143 } 144 145} 146 147 148@Category({FilterTests.class, MediumTests.class}) 149public class TestColumnRangeFilter { 150 151 @ClassRule 152 public static final HBaseClassTestRule CLASS_RULE = 153 HBaseClassTestRule.forClass(TestColumnRangeFilter.class); 154 155 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); 156 157 private static final Logger LOG = LoggerFactory.getLogger(TestColumnRangeFilter.class); 158 159 @Rule 160 public TestName name = new TestName(); 161 162 @BeforeClass 163 public static void setUpBeforeClass() throws Exception { 164 TEST_UTIL.startMiniCluster(); 165 } 166 167 @AfterClass 168 public static void tearDownAfterClass() throws Exception { 169 TEST_UTIL.shutdownMiniCluster(); 170 } 171 172 @Before 173 public void setUp() throws Exception { 174 // Nothing to do. 175 } 176 177 @After 178 public void tearDown() throws Exception { 179 // Nothing to do. 180 } 181 182 @Test 183 public void TestColumnRangeFilterClient() throws Exception { 184 String family = "Family"; 185 Table ht = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), 186 Bytes.toBytes(family), Integer.MAX_VALUE); 187 188 List<String> rows = generateRandomWords(10, 8); 189 long maxTimestamp = 2; 190 List<String> columns = generateRandomWords(20000, 8); 191 192 List<KeyValue> kvList = new ArrayList<>(); 193 194 Map<StringRange, List<KeyValue>> rangeMap = new HashMap<>(); 195 196 rangeMap.put(new StringRange(null, true, "b", false), 197 new ArrayList<>()); 198 rangeMap.put(new StringRange("p", true, "q", false), 199 new ArrayList<>()); 200 rangeMap.put(new StringRange("r", false, "s", true), 201 new ArrayList<>()); 202 rangeMap.put(new StringRange("z", false, null, false), 203 new ArrayList<>()); 204 String valueString = "ValueString"; 205 206 for (String row : rows) { 207 Put p = new Put(Bytes.toBytes(row)); 208 p.setDurability(Durability.SKIP_WAL); 209 for (String column : columns) { 210 for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) { 211 KeyValue kv = KeyValueTestUtil.create(row, family, column, timestamp, 212 valueString); 213 p.add(kv); 214 kvList.add(kv); 215 for (StringRange s : rangeMap.keySet()) { 216 if (s.inRange(column)) { 217 rangeMap.get(s).add(kv); 218 } 219 } 220 } 221 } 222 ht.put(p); 223 } 224 225 TEST_UTIL.flush(); 226 227 ColumnRangeFilter filter; 228 Scan scan = new Scan(); 229 scan.setMaxVersions(); 230 for (StringRange s : rangeMap.keySet()) { 231 filter = new ColumnRangeFilter(s.getStart() == null ? null : Bytes.toBytes(s.getStart()), 232 s.isStartInclusive(), s.getEnd() == null ? null : Bytes.toBytes(s.getEnd()), 233 s.isEndInclusive()); 234 assertEquals(rangeMap.get(s).size(), cellsCount(ht, filter)); 235 } 236 ht.close(); 237 } 238 239 @Test 240 public void TestColumnRangeFilterWithColumnPaginationFilter() throws Exception { 241 String family = "Family"; 242 String table = "TestColumnRangeFilterWithColumnPaginationFilter"; 243 try (Table ht = 244 TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(family), Integer.MAX_VALUE)) { 245 // one row. 246 String row = "row"; 247 // One version 248 long timestamp = 100; 249 // 10 columns 250 int[] columns = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; 251 String valueString = "ValueString"; 252 253 Put p = new Put(Bytes.toBytes(row)); 254 p.setDurability(Durability.SKIP_WAL); 255 for (int column : columns) { 256 KeyValue kv = 257 KeyValueTestUtil.create(row, family, Integer.toString(column), timestamp, valueString); 258 p.add(kv); 259 } 260 ht.put(p); 261 262 TEST_UTIL.flush(); 263 264 // Column range from 1 to 9. 265 StringRange stringRange = new StringRange("1", true, "9", false); 266 ColumnRangeFilter filter1 = new ColumnRangeFilter(Bytes.toBytes(stringRange.getStart()), 267 stringRange.isStartInclusive(), Bytes.toBytes(stringRange.getEnd()), 268 stringRange.isEndInclusive()); 269 270 ColumnPaginationFilter filter2 = new ColumnPaginationFilter(5, 0); 271 ColumnPaginationFilter filter3 = new ColumnPaginationFilter(5, 1); 272 ColumnPaginationFilter filter4 = new ColumnPaginationFilter(5, 2); 273 ColumnPaginationFilter filter5 = new ColumnPaginationFilter(5, 6); 274 ColumnPaginationFilter filter6 = new ColumnPaginationFilter(5, 9); 275 assertEquals(5, cellsCount(ht, new FilterList(Operator.MUST_PASS_ALL, filter1, filter2))); 276 assertEquals(5, cellsCount(ht, new FilterList(Operator.MUST_PASS_ALL, filter1, filter3))); 277 assertEquals(5, cellsCount(ht, new FilterList(Operator.MUST_PASS_ALL, filter1, filter4))); 278 assertEquals(2, cellsCount(ht, new FilterList(Operator.MUST_PASS_ALL, filter1, filter5))); 279 assertEquals(0, cellsCount(ht, new FilterList(Operator.MUST_PASS_ALL, filter1, filter6))); 280 } 281 } 282 283 private int cellsCount(Table table, Filter filter) throws IOException { 284 Scan scan = new Scan().setFilter(filter).readAllVersions(); 285 try (ResultScanner scanner = table.getScanner(scan)) { 286 List<Cell> results = new ArrayList<>(); 287 Result result; 288 while ((result = scanner.next()) != null) { 289 result.listCells().forEach(results::add); 290 } 291 return results.size(); 292 } 293 } 294 295 List<String> generateRandomWords(int numberOfWords, int maxLengthOfWords) { 296 Set<String> wordSet = new HashSet<>(); 297 for (int i = 0; i < numberOfWords; i++) { 298 int lengthOfWords = (int) (Math.random() * maxLengthOfWords) + 1; 299 char[] wordChar = new char[lengthOfWords]; 300 for (int j = 0; j < wordChar.length; j++) { 301 wordChar[j] = (char) (Math.random() * 26 + 97); 302 } 303 String word = new String(wordChar); 304 wordSet.add(word); 305 } 306 List<String> wordList = new ArrayList<>(wordSet); 307 return wordList; 308 } 309 310} 311