001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.filter;
019
020import static org.junit.Assert.*;
021
022import java.io.IOException;
023import java.util.ArrayList;
024import java.util.HashMap;
025import java.util.HashSet;
026import java.util.List;
027import java.util.Map;
028import java.util.Objects;
029import java.util.Set;
030
031import org.apache.hadoop.hbase.Cell;
032import org.apache.hadoop.hbase.HBaseClassTestRule;
033import org.apache.hadoop.hbase.HBaseTestingUtility;
034import org.apache.hadoop.hbase.KeyValue;
035import org.apache.hadoop.hbase.KeyValueTestUtil;
036import org.apache.hadoop.hbase.TableName;
037import org.apache.hadoop.hbase.client.Durability;
038import org.apache.hadoop.hbase.client.Put;
039import org.apache.hadoop.hbase.client.Result;
040import org.apache.hadoop.hbase.client.ResultScanner;
041import org.apache.hadoop.hbase.client.Scan;
042import org.apache.hadoop.hbase.client.Table;
043import org.apache.hadoop.hbase.filter.FilterList.Operator;
044import org.apache.hadoop.hbase.testclassification.FilterTests;
045import org.apache.hadoop.hbase.testclassification.MediumTests;
046import org.apache.hadoop.hbase.util.Bytes;
047import org.junit.After;
048import org.junit.AfterClass;
049import org.junit.Before;
050import org.junit.BeforeClass;
051import org.junit.ClassRule;
052import org.junit.Rule;
053import org.junit.Test;
054import org.junit.experimental.categories.Category;
055import org.junit.rules.TestName;
056import org.slf4j.Logger;
057import org.slf4j.LoggerFactory;
058
059class StringRange {
060  private String start = null;
061  private String end = null;
062  private boolean startInclusive = true;
063  private boolean endInclusive = false;
064
065  public StringRange(String start, boolean startInclusive, String end,
066      boolean endInclusive) {
067    this.start = start;
068    this.startInclusive = startInclusive;
069    this.end = end;
070    this.endInclusive = endInclusive;
071  }
072
073  public String getStart() {
074    return this.start;
075  }
076
077  public String getEnd() {
078    return this.end;
079  }
080
081  public boolean isStartInclusive() {
082    return this.startInclusive;
083  }
084
085  public boolean isEndInclusive() {
086    return this.endInclusive;
087  }
088
089  @Override
090  public int hashCode() {
091    int hashCode = 0;
092    if (this.start != null) {
093      hashCode ^= this.start.hashCode();
094    }
095
096    if (this.end != null) {
097      hashCode ^= this.end.hashCode();
098    }
099    return hashCode;
100  }
101
102  @Override
103  public boolean equals(Object obj) {
104    if (this == obj) {
105      return true;
106    }
107    if (obj == null) {
108      return false;
109    }
110    if (!(obj instanceof StringRange)) {
111      return false;
112    }
113    StringRange oth = (StringRange) obj;
114    return this.startInclusive == oth.startInclusive &&
115        this.endInclusive == oth.endInclusive &&
116        Objects.equals(this.start, oth.start) &&
117        Objects.equals(this.end, oth.end);
118  }
119
120  @Override
121  public String toString() {
122    String result = (this.startInclusive ? "[" : "(")
123          + (this.start == null ? null : this.start) + ", "
124          + (this.end == null ? null : this.end)
125          + (this.endInclusive ? "]" : ")");
126    return result;
127  }
128
129   public boolean inRange(String value) {
130    boolean afterStart = true;
131    if (this.start != null) {
132      int startCmp = value.compareTo(this.start);
133      afterStart = this.startInclusive ? startCmp >= 0 : startCmp > 0;
134    }
135
136    boolean beforeEnd = true;
137    if (this.end != null) {
138      int endCmp = value.compareTo(this.end);
139      beforeEnd = this.endInclusive ? endCmp <= 0 : endCmp < 0;
140    }
141
142    return afterStart && beforeEnd;
143  }
144
145}
146
147
148@Category({FilterTests.class, MediumTests.class})
149public class TestColumnRangeFilter {
150
151  @ClassRule
152  public static final HBaseClassTestRule CLASS_RULE =
153      HBaseClassTestRule.forClass(TestColumnRangeFilter.class);
154
155  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
156
157  private static final Logger LOG = LoggerFactory.getLogger(TestColumnRangeFilter.class);
158
159  @Rule
160  public TestName name = new TestName();
161
162  @BeforeClass
163  public static void setUpBeforeClass() throws Exception {
164    TEST_UTIL.startMiniCluster();
165  }
166
167  @AfterClass
168  public static void tearDownAfterClass() throws Exception {
169    TEST_UTIL.shutdownMiniCluster();
170  }
171
172  @Before
173  public void setUp() throws Exception {
174    // Nothing to do.
175  }
176
177  @After
178  public void tearDown() throws Exception {
179    // Nothing to do.
180  }
181
182  @Test
183  public void TestColumnRangeFilterClient() throws Exception {
184    String family = "Family";
185    Table ht = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()),
186        Bytes.toBytes(family), Integer.MAX_VALUE);
187
188    List<String> rows = generateRandomWords(10, 8);
189    long maxTimestamp = 2;
190    List<String> columns = generateRandomWords(20000, 8);
191
192    List<KeyValue> kvList = new ArrayList<>();
193
194    Map<StringRange, List<KeyValue>> rangeMap = new HashMap<>();
195
196    rangeMap.put(new StringRange(null, true, "b", false),
197        new ArrayList<>());
198    rangeMap.put(new StringRange("p", true, "q", false),
199        new ArrayList<>());
200    rangeMap.put(new StringRange("r", false, "s", true),
201        new ArrayList<>());
202    rangeMap.put(new StringRange("z", false, null, false),
203        new ArrayList<>());
204    String valueString = "ValueString";
205
206    for (String row : rows) {
207      Put p = new Put(Bytes.toBytes(row));
208      p.setDurability(Durability.SKIP_WAL);
209      for (String column : columns) {
210        for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
211          KeyValue kv = KeyValueTestUtil.create(row, family, column, timestamp,
212              valueString);
213          p.add(kv);
214          kvList.add(kv);
215          for (StringRange s : rangeMap.keySet()) {
216            if (s.inRange(column)) {
217              rangeMap.get(s).add(kv);
218            }
219          }
220        }
221      }
222      ht.put(p);
223    }
224
225    TEST_UTIL.flush();
226
227    ColumnRangeFilter filter;
228    Scan scan = new Scan();
229    scan.setMaxVersions();
230    for (StringRange s : rangeMap.keySet()) {
231      filter = new ColumnRangeFilter(s.getStart() == null ? null : Bytes.toBytes(s.getStart()),
232          s.isStartInclusive(), s.getEnd() == null ? null : Bytes.toBytes(s.getEnd()),
233          s.isEndInclusive());
234      assertEquals(rangeMap.get(s).size(), cellsCount(ht, filter));
235    }
236    ht.close();
237  }
238
239  @Test
240  public void TestColumnRangeFilterWithColumnPaginationFilter() throws Exception {
241    String family = "Family";
242    String table = "TestColumnRangeFilterWithColumnPaginationFilter";
243    try (Table ht =
244        TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(family), Integer.MAX_VALUE)) {
245      // one row.
246      String row = "row";
247      // One version
248      long timestamp = 100;
249      // 10 columns
250      int[] columns = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 };
251      String valueString = "ValueString";
252
253      Put p = new Put(Bytes.toBytes(row));
254      p.setDurability(Durability.SKIP_WAL);
255      for (int column : columns) {
256        KeyValue kv =
257            KeyValueTestUtil.create(row, family, Integer.toString(column), timestamp, valueString);
258        p.add(kv);
259      }
260      ht.put(p);
261
262      TEST_UTIL.flush();
263
264      // Column range from 1 to 9.
265      StringRange stringRange = new StringRange("1", true, "9", false);
266      ColumnRangeFilter filter1 = new ColumnRangeFilter(Bytes.toBytes(stringRange.getStart()),
267          stringRange.isStartInclusive(), Bytes.toBytes(stringRange.getEnd()),
268          stringRange.isEndInclusive());
269
270      ColumnPaginationFilter filter2 = new ColumnPaginationFilter(5, 0);
271      ColumnPaginationFilter filter3 = new ColumnPaginationFilter(5, 1);
272      ColumnPaginationFilter filter4 = new ColumnPaginationFilter(5, 2);
273      ColumnPaginationFilter filter5 = new ColumnPaginationFilter(5, 6);
274      ColumnPaginationFilter filter6 = new ColumnPaginationFilter(5, 9);
275      assertEquals(5, cellsCount(ht, new FilterList(Operator.MUST_PASS_ALL, filter1, filter2)));
276      assertEquals(5, cellsCount(ht, new FilterList(Operator.MUST_PASS_ALL, filter1, filter3)));
277      assertEquals(5, cellsCount(ht, new FilterList(Operator.MUST_PASS_ALL, filter1, filter4)));
278      assertEquals(2, cellsCount(ht, new FilterList(Operator.MUST_PASS_ALL, filter1, filter5)));
279      assertEquals(0, cellsCount(ht, new FilterList(Operator.MUST_PASS_ALL, filter1, filter6)));
280    }
281  }
282
283  private int cellsCount(Table table, Filter filter) throws IOException {
284    Scan scan = new Scan().setFilter(filter).readAllVersions();
285    try (ResultScanner scanner = table.getScanner(scan)) {
286      List<Cell> results = new ArrayList<>();
287      Result result;
288      while ((result = scanner.next()) != null) {
289        result.listCells().forEach(results::add);
290      }
291      return results.size();
292    }
293  }
294
295  List<String> generateRandomWords(int numberOfWords, int maxLengthOfWords) {
296    Set<String> wordSet = new HashSet<>();
297    for (int i = 0; i < numberOfWords; i++) {
298      int lengthOfWords = (int) (Math.random() * maxLengthOfWords) + 1;
299      char[] wordChar = new char[lengthOfWords];
300      for (int j = 0; j < wordChar.length; j++) {
301        wordChar[j] = (char) (Math.random() * 26 + 97);
302      }
303      String word = new String(wordChar);
304      wordSet.add(word);
305    }
306    List<String> wordList = new ArrayList<>(wordSet);
307    return wordList;
308  }
309
310}
311