Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.filter;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021
022import java.io.IOException;
023import java.util.ArrayList;
024import java.util.HashMap;
025import java.util.HashSet;
026import java.util.List;
027import java.util.Map;
028import java.util.Objects;
029import java.util.Set;
030import org.apache.hadoop.hbase.Cell;
031import org.apache.hadoop.hbase.HBaseTestingUtil;
032import org.apache.hadoop.hbase.KeyValue;
033import org.apache.hadoop.hbase.KeyValueTestUtil;
034import org.apache.hadoop.hbase.TableName;
035import org.apache.hadoop.hbase.client.Durability;
036import org.apache.hadoop.hbase.client.Put;
037import org.apache.hadoop.hbase.client.Result;
038import org.apache.hadoop.hbase.client.ResultScanner;
039import org.apache.hadoop.hbase.client.Scan;
040import org.apache.hadoop.hbase.client.Table;
041import org.apache.hadoop.hbase.filter.FilterList.Operator;
042import org.apache.hadoop.hbase.testclassification.FilterTests;
043import org.apache.hadoop.hbase.testclassification.MediumTests;
044import org.apache.hadoop.hbase.util.Bytes;
045import org.junit.jupiter.api.AfterAll;
046import org.junit.jupiter.api.AfterEach;
047import org.junit.jupiter.api.BeforeAll;
048import org.junit.jupiter.api.BeforeEach;
049import org.junit.jupiter.api.Tag;
050import org.junit.jupiter.api.Test;
051import org.junit.jupiter.api.TestInfo;
052import org.slf4j.Logger;
053import org.slf4j.LoggerFactory;
054
055@Tag(FilterTests.TAG)
056@Tag(MediumTests.TAG)
057public class TestColumnRangeFilter {
058
059  private final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
060
061  private static final Logger LOG = LoggerFactory.getLogger(TestColumnRangeFilter.class);
062
063  @BeforeAll
064  public static void setUpBeforeClass() throws Exception {
065    TEST_UTIL.startMiniCluster();
066  }
067
068  @AfterAll
069  public static void tearDownAfterClass() throws Exception {
070    TEST_UTIL.shutdownMiniCluster();
071  }
072
073  @BeforeEach
074  public void setUp() throws Exception {
075    // Nothing to do.
076  }
077
078  @AfterEach
079  public void tearDown() throws Exception {
080    // Nothing to do.
081  }
082
083  @Test
084  public void TestColumnRangeFilterClient(TestInfo testInfo) throws Exception {
085    String family = "Family";
086    Table ht = TEST_UTIL.createTable(TableName.valueOf(testInfo.getTestMethod().get().getName()),
087      Bytes.toBytes(family), Integer.MAX_VALUE);
088
089    List<String> rows = generateRandomWords(10, 8);
090    long maxTimestamp = 2;
091    List<String> columns = generateRandomWords(20000, 8);
092
093    List<KeyValue> kvList = new ArrayList<>();
094
095    Map<StringRange, List<KeyValue>> rangeMap = new HashMap<>();
096
097    rangeMap.put(new StringRange(null, true, "b", false), new ArrayList<>());
098    rangeMap.put(new StringRange("p", true, "q", false), new ArrayList<>());
099    rangeMap.put(new StringRange("r", false, "s", true), new ArrayList<>());
100    rangeMap.put(new StringRange("z", false, null, false), new ArrayList<>());
101    String valueString = "ValueString";
102
103    for (String row : rows) {
104      Put p = new Put(Bytes.toBytes(row));
105      p.setDurability(Durability.SKIP_WAL);
106      for (String column : columns) {
107        for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
108          KeyValue kv = KeyValueTestUtil.create(row, family, column, timestamp, valueString);
109          p.add(kv);
110          kvList.add(kv);
111          for (StringRange s : rangeMap.keySet()) {
112            if (s.inRange(column)) {
113              rangeMap.get(s).add(kv);
114            }
115          }
116        }
117      }
118      ht.put(p);
119    }
120
121    TEST_UTIL.flush();
122
123    ColumnRangeFilter filter;
124    Scan scan = new Scan();
125    scan.readAllVersions();
126    for (StringRange s : rangeMap.keySet()) {
127      filter = new ColumnRangeFilter(s.getStart() == null ? null : Bytes.toBytes(s.getStart()),
128        s.isStartInclusive(), s.getEnd() == null ? null : Bytes.toBytes(s.getEnd()),
129        s.isEndInclusive());
130      assertEquals(rangeMap.get(s).size(), cellsCount(ht, filter));
131    }
132    ht.close();
133  }
134
135  @Test
136  public void TestColumnRangeFilterWithColumnPaginationFilter() throws Exception {
137    String family = "Family";
138    String table = "TestColumnRangeFilterWithColumnPaginationFilter";
139    try (Table ht =
140      TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(family), Integer.MAX_VALUE)) {
141      // one row.
142      String row = "row";
143      // One version
144      long timestamp = 100;
145      // 10 columns
146      int[] columns = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 };
147      String valueString = "ValueString";
148
149      Put p = new Put(Bytes.toBytes(row));
150      p.setDurability(Durability.SKIP_WAL);
151      for (int column : columns) {
152        KeyValue kv =
153          KeyValueTestUtil.create(row, family, Integer.toString(column), timestamp, valueString);
154        p.add(kv);
155      }
156      ht.put(p);
157
158      TEST_UTIL.flush();
159
160      // Column range from 1 to 9.
161      StringRange stringRange = new StringRange("1", true, "9", false);
162      ColumnRangeFilter filter1 =
163        new ColumnRangeFilter(Bytes.toBytes(stringRange.getStart()), stringRange.isStartInclusive(),
164          Bytes.toBytes(stringRange.getEnd()), stringRange.isEndInclusive());
165
166      ColumnPaginationFilter filter2 = new ColumnPaginationFilter(5, 0);
167      ColumnPaginationFilter filter3 = new ColumnPaginationFilter(5, 1);
168      ColumnPaginationFilter filter4 = new ColumnPaginationFilter(5, 2);
169      ColumnPaginationFilter filter5 = new ColumnPaginationFilter(5, 6);
170      ColumnPaginationFilter filter6 = new ColumnPaginationFilter(5, 9);
171      assertEquals(5, cellsCount(ht, new FilterList(Operator.MUST_PASS_ALL, filter1, filter2)));
172      assertEquals(5, cellsCount(ht, new FilterList(Operator.MUST_PASS_ALL, filter1, filter3)));
173      assertEquals(5, cellsCount(ht, new FilterList(Operator.MUST_PASS_ALL, filter1, filter4)));
174      assertEquals(2, cellsCount(ht, new FilterList(Operator.MUST_PASS_ALL, filter1, filter5)));
175      assertEquals(0, cellsCount(ht, new FilterList(Operator.MUST_PASS_ALL, filter1, filter6)));
176    }
177  }
178
179  private int cellsCount(Table table, Filter filter) throws IOException {
180    Scan scan = new Scan().setFilter(filter).readAllVersions();
181    try (ResultScanner scanner = table.getScanner(scan)) {
182      List<Cell> results = new ArrayList<>();
183      Result result;
184      while ((result = scanner.next()) != null) {
185        result.listCells().forEach(results::add);
186      }
187      return results.size();
188    }
189  }
190
191  List<String> generateRandomWords(int numberOfWords, int maxLengthOfWords) {
192    Set<String> wordSet = new HashSet<>();
193    for (int i = 0; i < numberOfWords; i++) {
194      int lengthOfWords = (int) (Math.random() * maxLengthOfWords) + 1;
195      char[] wordChar = new char[lengthOfWords];
196      for (int j = 0; j < wordChar.length; j++) {
197        wordChar[j] = (char) (Math.random() * 26 + 97);
198      }
199      String word = new String(wordChar);
200      wordSet.add(word);
201    }
202    List<String> wordList = new ArrayList<>(wordSet);
203    return wordList;
204  }
205
206}
207
208class StringRange {
209  private String start = null;
210  private String end = null;
211  private boolean startInclusive = true;
212  private boolean endInclusive = false;
213
214  public StringRange(String start, boolean startInclusive, String end, boolean endInclusive) {
215    this.start = start;
216    this.startInclusive = startInclusive;
217    this.end = end;
218    this.endInclusive = endInclusive;
219  }
220
221  public String getStart() {
222    return this.start;
223  }
224
225  public String getEnd() {
226    return this.end;
227  }
228
229  public boolean isStartInclusive() {
230    return this.startInclusive;
231  }
232
233  public boolean isEndInclusive() {
234    return this.endInclusive;
235  }
236
237  @Override
238  public int hashCode() {
239    int hashCode = 0;
240    if (this.start != null) {
241      hashCode ^= this.start.hashCode();
242    }
243
244    if (this.end != null) {
245      hashCode ^= this.end.hashCode();
246    }
247    return hashCode;
248  }
249
250  @Override
251  public boolean equals(Object obj) {
252    if (this == obj) {
253      return true;
254    }
255    if (obj == null) {
256      return false;
257    }
258    if (!(obj instanceof StringRange)) {
259      return false;
260    }
261    StringRange oth = (StringRange) obj;
262    return this.startInclusive == oth.startInclusive && this.endInclusive == oth.endInclusive
263      && Objects.equals(this.start, oth.start) && Objects.equals(this.end, oth.end);
264  }
265
266  @Override
267  public String toString() {
268    String result = (this.startInclusive ? "[" : "(") + (this.start == null ? null : this.start)
269      + ", " + (this.end == null ? null : this.end) + (this.endInclusive ? "]" : ")");
270    return result;
271  }
272
273  public boolean inRange(String value) {
274    boolean afterStart = true;
275    if (this.start != null) {
276      int startCmp = value.compareTo(this.start);
277      afterStart = this.startInclusive ? startCmp >= 0 : startCmp > 0;
278    }
279
280    boolean beforeEnd = true;
281    if (this.end != null) {
282      int endCmp = value.compareTo(this.end);
283      beforeEnd = this.endInclusive ? endCmp <= 0 : endCmp < 0;
284    }
285
286    return afterStart && beforeEnd;
287  }
288}