001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.filter; 019 020import static org.junit.Assert.assertEquals; 021 022import java.io.IOException; 023import java.util.ArrayList; 024import java.util.HashMap; 025import java.util.HashSet; 026import java.util.List; 027import java.util.Map; 028import java.util.Set; 029import org.apache.hadoop.hbase.Cell; 030import org.apache.hadoop.hbase.HBaseClassTestRule; 031import org.apache.hadoop.hbase.HBaseTestingUtility; 032import org.apache.hadoop.hbase.KeyValue; 033import org.apache.hadoop.hbase.KeyValueTestUtil; 034import org.apache.hadoop.hbase.TableName; 035import org.apache.hadoop.hbase.client.Durability; 036import org.apache.hadoop.hbase.client.Put; 037import org.apache.hadoop.hbase.client.Result; 038import org.apache.hadoop.hbase.client.ResultScanner; 039import org.apache.hadoop.hbase.client.Scan; 040import org.apache.hadoop.hbase.client.Table; 041import org.apache.hadoop.hbase.filter.FilterList.Operator; 042import org.apache.hadoop.hbase.testclassification.FilterTests; 043import org.apache.hadoop.hbase.testclassification.MediumTests; 044import org.apache.hadoop.hbase.util.Bytes; 045import org.junit.After; 046import org.junit.AfterClass; 047import org.junit.Before; 048import org.junit.BeforeClass; 049import org.junit.ClassRule; 050import org.junit.Rule; 051import org.junit.Test; 052import org.junit.experimental.categories.Category; 053import org.junit.rules.TestName; 054import org.slf4j.Logger; 055import org.slf4j.LoggerFactory; 056 057class StringRange { 058 private String start = null; 059 private String end = null; 060 private boolean startInclusive = true; 061 private boolean endInclusive = false; 062 063 public StringRange(String start, boolean startInclusive, String end, 064 boolean endInclusive) { 065 this.start = start; 066 this.startInclusive = startInclusive; 067 this.end = end; 068 this.endInclusive = endInclusive; 069 } 070 071 public String getStart() { 072 return this.start; 073 } 074 075 public String getEnd() { 076 return this.end; 077 } 078 079 public boolean isStartInclusive() { 080 return this.startInclusive; 081 } 082 083 public boolean isEndInclusive() { 084 return this.endInclusive; 085 } 086 087 @Override 088 public int hashCode() { 089 int hashCode = 0; 090 if (this.start != null) { 091 hashCode ^= this.start.hashCode(); 092 } 093 094 if (this.end != null) { 095 hashCode ^= this.end.hashCode(); 096 } 097 return hashCode; 098 } 099 100 @Override 101 public String toString() { 102 String result = (this.startInclusive ? "[" : "(") 103 + (this.start == null ? null : this.start) + ", " 104 + (this.end == null ? null : this.end) 105 + (this.endInclusive ? "]" : ")"); 106 return result; 107 } 108 109 public boolean inRange(String value) { 110 boolean afterStart = true; 111 if (this.start != null) { 112 int startCmp = value.compareTo(this.start); 113 afterStart = this.startInclusive ? startCmp >= 0 : startCmp > 0; 114 } 115 116 boolean beforeEnd = true; 117 if (this.end != null) { 118 int endCmp = value.compareTo(this.end); 119 beforeEnd = this.endInclusive ? endCmp <= 0 : endCmp < 0; 120 } 121 122 return afterStart && beforeEnd; 123 } 124 125} 126 127 128@Category({FilterTests.class, MediumTests.class}) 129public class TestColumnRangeFilter { 130 131 @ClassRule 132 public static final HBaseClassTestRule CLASS_RULE = 133 HBaseClassTestRule.forClass(TestColumnRangeFilter.class); 134 135 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); 136 137 private static final Logger LOG = LoggerFactory.getLogger(TestColumnRangeFilter.class); 138 139 @Rule 140 public TestName name = new TestName(); 141 142 /** 143 * @throws java.lang.Exception 144 */ 145 @BeforeClass 146 public static void setUpBeforeClass() throws Exception { 147 TEST_UTIL.startMiniCluster(); 148 } 149 150 /** 151 * @throws java.lang.Exception 152 */ 153 @AfterClass 154 public static void tearDownAfterClass() throws Exception { 155 TEST_UTIL.shutdownMiniCluster(); 156 } 157 158 /** 159 * @throws java.lang.Exception 160 */ 161 @Before 162 public void setUp() throws Exception { 163 // Nothing to do. 164 } 165 166 /** 167 * @throws java.lang.Exception 168 */ 169 @After 170 public void tearDown() throws Exception { 171 // Nothing to do. 172 } 173 174 @Test 175 public void TestColumnRangeFilterClient() throws Exception { 176 String family = "Family"; 177 Table ht = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), 178 Bytes.toBytes(family), Integer.MAX_VALUE); 179 180 List<String> rows = generateRandomWords(10, 8); 181 long maxTimestamp = 2; 182 List<String> columns = generateRandomWords(20000, 8); 183 184 List<KeyValue> kvList = new ArrayList<>(); 185 186 Map<StringRange, List<KeyValue>> rangeMap = new HashMap<>(); 187 188 rangeMap.put(new StringRange(null, true, "b", false), 189 new ArrayList<>()); 190 rangeMap.put(new StringRange("p", true, "q", false), 191 new ArrayList<>()); 192 rangeMap.put(new StringRange("r", false, "s", true), 193 new ArrayList<>()); 194 rangeMap.put(new StringRange("z", false, null, false), 195 new ArrayList<>()); 196 String valueString = "ValueString"; 197 198 for (String row : rows) { 199 Put p = new Put(Bytes.toBytes(row)); 200 p.setDurability(Durability.SKIP_WAL); 201 for (String column : columns) { 202 for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) { 203 KeyValue kv = KeyValueTestUtil.create(row, family, column, timestamp, 204 valueString); 205 p.add(kv); 206 kvList.add(kv); 207 for (StringRange s : rangeMap.keySet()) { 208 if (s.inRange(column)) { 209 rangeMap.get(s).add(kv); 210 } 211 } 212 } 213 } 214 ht.put(p); 215 } 216 217 TEST_UTIL.flush(); 218 219 ColumnRangeFilter filter; 220 Scan scan = new Scan(); 221 scan.setMaxVersions(); 222 for (StringRange s : rangeMap.keySet()) { 223 filter = new ColumnRangeFilter(s.getStart() == null ? null : Bytes.toBytes(s.getStart()), 224 s.isStartInclusive(), s.getEnd() == null ? null : Bytes.toBytes(s.getEnd()), 225 s.isEndInclusive()); 226 assertEquals(rangeMap.get(s).size(), cellsCount(ht, filter)); 227 } 228 ht.close(); 229 } 230 231 @Test 232 public void TestColumnRangeFilterWithColumnPaginationFilter() throws Exception { 233 String family = "Family"; 234 String table = "TestColumnRangeFilterWithColumnPaginationFilter"; 235 try (Table ht = 236 TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(family), Integer.MAX_VALUE)) { 237 // one row. 238 String row = "row"; 239 // One version 240 long timestamp = 100; 241 // 10 columns 242 int[] columns = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; 243 String valueString = "ValueString"; 244 245 Put p = new Put(Bytes.toBytes(row)); 246 p.setDurability(Durability.SKIP_WAL); 247 for (int column : columns) { 248 KeyValue kv = 249 KeyValueTestUtil.create(row, family, Integer.toString(column), timestamp, valueString); 250 p.add(kv); 251 } 252 ht.put(p); 253 254 TEST_UTIL.flush(); 255 256 // Column range from 1 to 9. 257 StringRange stringRange = new StringRange("1", true, "9", false); 258 ColumnRangeFilter filter1 = new ColumnRangeFilter(Bytes.toBytes(stringRange.getStart()), 259 stringRange.isStartInclusive(), Bytes.toBytes(stringRange.getEnd()), 260 stringRange.isEndInclusive()); 261 262 ColumnPaginationFilter filter2 = new ColumnPaginationFilter(5, 0); 263 ColumnPaginationFilter filter3 = new ColumnPaginationFilter(5, 1); 264 ColumnPaginationFilter filter4 = new ColumnPaginationFilter(5, 2); 265 ColumnPaginationFilter filter5 = new ColumnPaginationFilter(5, 6); 266 ColumnPaginationFilter filter6 = new ColumnPaginationFilter(5, 9); 267 assertEquals(5, cellsCount(ht, new FilterList(Operator.MUST_PASS_ALL, filter1, filter2))); 268 assertEquals(5, cellsCount(ht, new FilterList(Operator.MUST_PASS_ALL, filter1, filter3))); 269 assertEquals(5, cellsCount(ht, new FilterList(Operator.MUST_PASS_ALL, filter1, filter4))); 270 assertEquals(2, cellsCount(ht, new FilterList(Operator.MUST_PASS_ALL, filter1, filter5))); 271 assertEquals(0, cellsCount(ht, new FilterList(Operator.MUST_PASS_ALL, filter1, filter6))); 272 } 273 } 274 275 private int cellsCount(Table table, Filter filter) throws IOException { 276 Scan scan = new Scan().setFilter(filter).readAllVersions(); 277 try (ResultScanner scanner = table.getScanner(scan)) { 278 List<Cell> results = new ArrayList<>(); 279 Result result; 280 while ((result = scanner.next()) != null) { 281 result.listCells().forEach(results::add); 282 } 283 return results.size(); 284 } 285 } 286 287 List<String> generateRandomWords(int numberOfWords, int maxLengthOfWords) { 288 Set<String> wordSet = new HashSet<>(); 289 for (int i = 0; i < numberOfWords; i++) { 290 int lengthOfWords = (int) (Math.random() * maxLengthOfWords) + 1; 291 char[] wordChar = new char[lengthOfWords]; 292 for (int j = 0; j < wordChar.length; j++) { 293 wordChar[j] = (char) (Math.random() * 26 + 97); 294 } 295 String word = new String(wordChar); 296 wordSet.add(word); 297 } 298 List<String> wordList = new ArrayList<>(wordSet); 299 return wordList; 300 } 301 302} 303