Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.filter;
019
020import static org.junit.Assert.*;
021
022import java.io.IOException;
023import java.util.ArrayList;
024import java.util.HashMap;
025import java.util.HashSet;
026import java.util.List;
027import java.util.Map;
028import java.util.Objects;
029import java.util.Set;
030import org.apache.hadoop.hbase.Cell;
031import org.apache.hadoop.hbase.HBaseClassTestRule;
032import org.apache.hadoop.hbase.HBaseTestingUtil;
033import org.apache.hadoop.hbase.KeyValue;
034import org.apache.hadoop.hbase.KeyValueTestUtil;
035import org.apache.hadoop.hbase.TableName;
036import org.apache.hadoop.hbase.client.Durability;
037import org.apache.hadoop.hbase.client.Put;
038import org.apache.hadoop.hbase.client.Result;
039import org.apache.hadoop.hbase.client.ResultScanner;
040import org.apache.hadoop.hbase.client.Scan;
041import org.apache.hadoop.hbase.client.Table;
042import org.apache.hadoop.hbase.filter.FilterList.Operator;
043import org.apache.hadoop.hbase.testclassification.FilterTests;
044import org.apache.hadoop.hbase.testclassification.MediumTests;
045import org.apache.hadoop.hbase.util.Bytes;
046import org.junit.After;
047import org.junit.AfterClass;
048import org.junit.Before;
049import org.junit.BeforeClass;
050import org.junit.ClassRule;
051import org.junit.Rule;
052import org.junit.Test;
053import org.junit.experimental.categories.Category;
054import org.junit.rules.TestName;
055import org.slf4j.Logger;
056import org.slf4j.LoggerFactory;
057
058class StringRange {
059  private String start = null;
060  private String end = null;
061  private boolean startInclusive = true;
062  private boolean endInclusive = false;
063
064  public StringRange(String start, boolean startInclusive, String end, boolean endInclusive) {
065    this.start = start;
066    this.startInclusive = startInclusive;
067    this.end = end;
068    this.endInclusive = endInclusive;
069  }
070
071  public String getStart() {
072    return this.start;
073  }
074
075  public String getEnd() {
076    return this.end;
077  }
078
079  public boolean isStartInclusive() {
080    return this.startInclusive;
081  }
082
083  public boolean isEndInclusive() {
084    return this.endInclusive;
085  }
086
087  @Override
088  public int hashCode() {
089    int hashCode = 0;
090    if (this.start != null) {
091      hashCode ^= this.start.hashCode();
092    }
093
094    if (this.end != null) {
095      hashCode ^= this.end.hashCode();
096    }
097    return hashCode;
098  }
099
100  @Override
101  public boolean equals(Object obj) {
102    if (this == obj) {
103      return true;
104    }
105    if (obj == null) {
106      return false;
107    }
108    if (!(obj instanceof StringRange)) {
109      return false;
110    }
111    StringRange oth = (StringRange) obj;
112    return this.startInclusive == oth.startInclusive && this.endInclusive == oth.endInclusive
113      && Objects.equals(this.start, oth.start) && Objects.equals(this.end, oth.end);
114  }
115
116  @Override
117  public String toString() {
118    String result = (this.startInclusive ? "[" : "(") + (this.start == null ? null : this.start)
119      + ", " + (this.end == null ? null : this.end) + (this.endInclusive ? "]" : ")");
120    return result;
121  }
122
123  public boolean inRange(String value) {
124    boolean afterStart = true;
125    if (this.start != null) {
126      int startCmp = value.compareTo(this.start);
127      afterStart = this.startInclusive ? startCmp >= 0 : startCmp > 0;
128    }
129
130    boolean beforeEnd = true;
131    if (this.end != null) {
132      int endCmp = value.compareTo(this.end);
133      beforeEnd = this.endInclusive ? endCmp <= 0 : endCmp < 0;
134    }
135
136    return afterStart && beforeEnd;
137  }
138
139}
140
141@Category({ FilterTests.class, MediumTests.class })
142public class TestColumnRangeFilter {
143
144  @ClassRule
145  public static final HBaseClassTestRule CLASS_RULE =
146    HBaseClassTestRule.forClass(TestColumnRangeFilter.class);
147
148  private final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
149
150  private static final Logger LOG = LoggerFactory.getLogger(TestColumnRangeFilter.class);
151
152  @Rule
153  public TestName name = new TestName();
154
155  @BeforeClass
156  public static void setUpBeforeClass() throws Exception {
157    TEST_UTIL.startMiniCluster();
158  }
159
160  @AfterClass
161  public static void tearDownAfterClass() throws Exception {
162    TEST_UTIL.shutdownMiniCluster();
163  }
164
165  @Before
166  public void setUp() throws Exception {
167    // Nothing to do.
168  }
169
170  @After
171  public void tearDown() throws Exception {
172    // Nothing to do.
173  }
174
175  @Test
176  public void TestColumnRangeFilterClient() throws Exception {
177    String family = "Family";
178    Table ht = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), Bytes.toBytes(family),
179      Integer.MAX_VALUE);
180
181    List<String> rows = generateRandomWords(10, 8);
182    long maxTimestamp = 2;
183    List<String> columns = generateRandomWords(20000, 8);
184
185    List<KeyValue> kvList = new ArrayList<>();
186
187    Map<StringRange, List<KeyValue>> rangeMap = new HashMap<>();
188
189    rangeMap.put(new StringRange(null, true, "b", false), new ArrayList<>());
190    rangeMap.put(new StringRange("p", true, "q", false), new ArrayList<>());
191    rangeMap.put(new StringRange("r", false, "s", true), new ArrayList<>());
192    rangeMap.put(new StringRange("z", false, null, false), new ArrayList<>());
193    String valueString = "ValueString";
194
195    for (String row : rows) {
196      Put p = new Put(Bytes.toBytes(row));
197      p.setDurability(Durability.SKIP_WAL);
198      for (String column : columns) {
199        for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
200          KeyValue kv = KeyValueTestUtil.create(row, family, column, timestamp, valueString);
201          p.add(kv);
202          kvList.add(kv);
203          for (StringRange s : rangeMap.keySet()) {
204            if (s.inRange(column)) {
205              rangeMap.get(s).add(kv);
206            }
207          }
208        }
209      }
210      ht.put(p);
211    }
212
213    TEST_UTIL.flush();
214
215    ColumnRangeFilter filter;
216    Scan scan = new Scan();
217    scan.readAllVersions();
218    for (StringRange s : rangeMap.keySet()) {
219      filter = new ColumnRangeFilter(s.getStart() == null ? null : Bytes.toBytes(s.getStart()),
220        s.isStartInclusive(), s.getEnd() == null ? null : Bytes.toBytes(s.getEnd()),
221        s.isEndInclusive());
222      assertEquals(rangeMap.get(s).size(), cellsCount(ht, filter));
223    }
224    ht.close();
225  }
226
227  @Test
228  public void TestColumnRangeFilterWithColumnPaginationFilter() throws Exception {
229    String family = "Family";
230    String table = "TestColumnRangeFilterWithColumnPaginationFilter";
231    try (Table ht =
232      TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(family), Integer.MAX_VALUE)) {
233      // one row.
234      String row = "row";
235      // One version
236      long timestamp = 100;
237      // 10 columns
238      int[] columns = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 };
239      String valueString = "ValueString";
240
241      Put p = new Put(Bytes.toBytes(row));
242      p.setDurability(Durability.SKIP_WAL);
243      for (int column : columns) {
244        KeyValue kv =
245          KeyValueTestUtil.create(row, family, Integer.toString(column), timestamp, valueString);
246        p.add(kv);
247      }
248      ht.put(p);
249
250      TEST_UTIL.flush();
251
252      // Column range from 1 to 9.
253      StringRange stringRange = new StringRange("1", true, "9", false);
254      ColumnRangeFilter filter1 =
255        new ColumnRangeFilter(Bytes.toBytes(stringRange.getStart()), stringRange.isStartInclusive(),
256          Bytes.toBytes(stringRange.getEnd()), stringRange.isEndInclusive());
257
258      ColumnPaginationFilter filter2 = new ColumnPaginationFilter(5, 0);
259      ColumnPaginationFilter filter3 = new ColumnPaginationFilter(5, 1);
260      ColumnPaginationFilter filter4 = new ColumnPaginationFilter(5, 2);
261      ColumnPaginationFilter filter5 = new ColumnPaginationFilter(5, 6);
262      ColumnPaginationFilter filter6 = new ColumnPaginationFilter(5, 9);
263      assertEquals(5, cellsCount(ht, new FilterList(Operator.MUST_PASS_ALL, filter1, filter2)));
264      assertEquals(5, cellsCount(ht, new FilterList(Operator.MUST_PASS_ALL, filter1, filter3)));
265      assertEquals(5, cellsCount(ht, new FilterList(Operator.MUST_PASS_ALL, filter1, filter4)));
266      assertEquals(2, cellsCount(ht, new FilterList(Operator.MUST_PASS_ALL, filter1, filter5)));
267      assertEquals(0, cellsCount(ht, new FilterList(Operator.MUST_PASS_ALL, filter1, filter6)));
268    }
269  }
270
271  private int cellsCount(Table table, Filter filter) throws IOException {
272    Scan scan = new Scan().setFilter(filter).readAllVersions();
273    try (ResultScanner scanner = table.getScanner(scan)) {
274      List<Cell> results = new ArrayList<>();
275      Result result;
276      while ((result = scanner.next()) != null) {
277        result.listCells().forEach(results::add);
278      }
279      return results.size();
280    }
281  }
282
283  List<String> generateRandomWords(int numberOfWords, int maxLengthOfWords) {
284    Set<String> wordSet = new HashSet<>();
285    for (int i = 0; i < numberOfWords; i++) {
286      int lengthOfWords = (int) (Math.random() * maxLengthOfWords) + 1;
287      char[] wordChar = new char[lengthOfWords];
288      for (int j = 0; j < wordChar.length; j++) {
289        wordChar[j] = (char) (Math.random() * 26 + 97);
290      }
291      String word = new String(wordChar);
292      wordSet.add(word);
293    }
294    List<String> wordList = new ArrayList<>(wordSet);
295    return wordList;
296  }
297
298}