001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.filter;
019
020import static org.junit.Assert.assertEquals;
021
022import java.io.IOException;
023import java.util.ArrayList;
024import java.util.HashMap;
025import java.util.HashSet;
026import java.util.List;
027import java.util.Map;
028import java.util.Set;
029import org.apache.hadoop.hbase.Cell;
030import org.apache.hadoop.hbase.HBaseClassTestRule;
031import org.apache.hadoop.hbase.HBaseTestingUtility;
032import org.apache.hadoop.hbase.KeyValue;
033import org.apache.hadoop.hbase.KeyValueTestUtil;
034import org.apache.hadoop.hbase.TableName;
035import org.apache.hadoop.hbase.client.Durability;
036import org.apache.hadoop.hbase.client.Put;
037import org.apache.hadoop.hbase.client.Result;
038import org.apache.hadoop.hbase.client.ResultScanner;
039import org.apache.hadoop.hbase.client.Scan;
040import org.apache.hadoop.hbase.client.Table;
041import org.apache.hadoop.hbase.filter.FilterList.Operator;
042import org.apache.hadoop.hbase.testclassification.FilterTests;
043import org.apache.hadoop.hbase.testclassification.MediumTests;
044import org.apache.hadoop.hbase.util.Bytes;
045import org.junit.After;
046import org.junit.AfterClass;
047import org.junit.Before;
048import org.junit.BeforeClass;
049import org.junit.ClassRule;
050import org.junit.Rule;
051import org.junit.Test;
052import org.junit.experimental.categories.Category;
053import org.junit.rules.TestName;
054import org.slf4j.Logger;
055import org.slf4j.LoggerFactory;
056
057class StringRange {
058  private String start = null;
059  private String end = null;
060  private boolean startInclusive = true;
061  private boolean endInclusive = false;
062
063  public StringRange(String start, boolean startInclusive, String end,
064      boolean endInclusive) {
065    this.start = start;
066    this.startInclusive = startInclusive;
067    this.end = end;
068    this.endInclusive = endInclusive;
069  }
070
071  public String getStart() {
072    return this.start;
073  }
074
075  public String getEnd() {
076    return this.end;
077  }
078
079  public boolean isStartInclusive() {
080    return this.startInclusive;
081  }
082
083  public boolean isEndInclusive() {
084    return this.endInclusive;
085  }
086
087  @Override
088  public int hashCode() {
089    int hashCode = 0;
090    if (this.start != null) {
091      hashCode ^= this.start.hashCode();
092    }
093
094    if (this.end != null) {
095      hashCode ^= this.end.hashCode();
096    }
097    return hashCode;
098  }
099
100  @Override
101  public String toString() {
102    String result = (this.startInclusive ? "[" : "(")
103          + (this.start == null ? null : this.start) + ", "
104          + (this.end == null ? null : this.end)
105          + (this.endInclusive ? "]" : ")");
106    return result;
107  }
108
109   public boolean inRange(String value) {
110    boolean afterStart = true;
111    if (this.start != null) {
112      int startCmp = value.compareTo(this.start);
113      afterStart = this.startInclusive ? startCmp >= 0 : startCmp > 0;
114    }
115
116    boolean beforeEnd = true;
117    if (this.end != null) {
118      int endCmp = value.compareTo(this.end);
119      beforeEnd = this.endInclusive ? endCmp <= 0 : endCmp < 0;
120    }
121
122    return afterStart && beforeEnd;
123  }
124
125}
126
127
128@Category({FilterTests.class, MediumTests.class})
129public class TestColumnRangeFilter {
130
131  @ClassRule
132  public static final HBaseClassTestRule CLASS_RULE =
133      HBaseClassTestRule.forClass(TestColumnRangeFilter.class);
134
135  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
136
137  private static final Logger LOG = LoggerFactory.getLogger(TestColumnRangeFilter.class);
138
139  @Rule
140  public TestName name = new TestName();
141
142  /**
143   * @throws java.lang.Exception
144   */
145  @BeforeClass
146  public static void setUpBeforeClass() throws Exception {
147    TEST_UTIL.startMiniCluster();
148  }
149
150  /**
151   * @throws java.lang.Exception
152   */
153  @AfterClass
154  public static void tearDownAfterClass() throws Exception {
155    TEST_UTIL.shutdownMiniCluster();
156  }
157
158  /**
159   * @throws java.lang.Exception
160   */
161  @Before
162  public void setUp() throws Exception {
163    // Nothing to do.
164  }
165
166  /**
167   * @throws java.lang.Exception
168   */
169  @After
170  public void tearDown() throws Exception {
171    // Nothing to do.
172  }
173
174  @Test
175  public void TestColumnRangeFilterClient() throws Exception {
176    String family = "Family";
177    Table ht = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()),
178        Bytes.toBytes(family), Integer.MAX_VALUE);
179
180    List<String> rows = generateRandomWords(10, 8);
181    long maxTimestamp = 2;
182    List<String> columns = generateRandomWords(20000, 8);
183
184    List<KeyValue> kvList = new ArrayList<>();
185
186    Map<StringRange, List<KeyValue>> rangeMap = new HashMap<>();
187
188    rangeMap.put(new StringRange(null, true, "b", false),
189        new ArrayList<>());
190    rangeMap.put(new StringRange("p", true, "q", false),
191        new ArrayList<>());
192    rangeMap.put(new StringRange("r", false, "s", true),
193        new ArrayList<>());
194    rangeMap.put(new StringRange("z", false, null, false),
195        new ArrayList<>());
196    String valueString = "ValueString";
197
198    for (String row : rows) {
199      Put p = new Put(Bytes.toBytes(row));
200      p.setDurability(Durability.SKIP_WAL);
201      for (String column : columns) {
202        for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
203          KeyValue kv = KeyValueTestUtil.create(row, family, column, timestamp,
204              valueString);
205          p.add(kv);
206          kvList.add(kv);
207          for (StringRange s : rangeMap.keySet()) {
208            if (s.inRange(column)) {
209              rangeMap.get(s).add(kv);
210            }
211          }
212        }
213      }
214      ht.put(p);
215    }
216
217    TEST_UTIL.flush();
218
219    ColumnRangeFilter filter;
220    Scan scan = new Scan();
221    scan.setMaxVersions();
222    for (StringRange s : rangeMap.keySet()) {
223      filter = new ColumnRangeFilter(s.getStart() == null ? null : Bytes.toBytes(s.getStart()),
224          s.isStartInclusive(), s.getEnd() == null ? null : Bytes.toBytes(s.getEnd()),
225          s.isEndInclusive());
226      assertEquals(rangeMap.get(s).size(), cellsCount(ht, filter));
227    }
228    ht.close();
229  }
230
231  @Test
232  public void TestColumnRangeFilterWithColumnPaginationFilter() throws Exception {
233    String family = "Family";
234    String table = "TestColumnRangeFilterWithColumnPaginationFilter";
235    try (Table ht =
236        TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(family), Integer.MAX_VALUE)) {
237      // one row.
238      String row = "row";
239      // One version
240      long timestamp = 100;
241      // 10 columns
242      int[] columns = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 };
243      String valueString = "ValueString";
244
245      Put p = new Put(Bytes.toBytes(row));
246      p.setDurability(Durability.SKIP_WAL);
247      for (int column : columns) {
248        KeyValue kv =
249            KeyValueTestUtil.create(row, family, Integer.toString(column), timestamp, valueString);
250        p.add(kv);
251      }
252      ht.put(p);
253
254      TEST_UTIL.flush();
255
256      // Column range from 1 to 9.
257      StringRange stringRange = new StringRange("1", true, "9", false);
258      ColumnRangeFilter filter1 = new ColumnRangeFilter(Bytes.toBytes(stringRange.getStart()),
259          stringRange.isStartInclusive(), Bytes.toBytes(stringRange.getEnd()),
260          stringRange.isEndInclusive());
261
262      ColumnPaginationFilter filter2 = new ColumnPaginationFilter(5, 0);
263      ColumnPaginationFilter filter3 = new ColumnPaginationFilter(5, 1);
264      ColumnPaginationFilter filter4 = new ColumnPaginationFilter(5, 2);
265      ColumnPaginationFilter filter5 = new ColumnPaginationFilter(5, 6);
266      ColumnPaginationFilter filter6 = new ColumnPaginationFilter(5, 9);
267      assertEquals(5, cellsCount(ht, new FilterList(Operator.MUST_PASS_ALL, filter1, filter2)));
268      assertEquals(5, cellsCount(ht, new FilterList(Operator.MUST_PASS_ALL, filter1, filter3)));
269      assertEquals(5, cellsCount(ht, new FilterList(Operator.MUST_PASS_ALL, filter1, filter4)));
270      assertEquals(2, cellsCount(ht, new FilterList(Operator.MUST_PASS_ALL, filter1, filter5)));
271      assertEquals(0, cellsCount(ht, new FilterList(Operator.MUST_PASS_ALL, filter1, filter6)));
272    }
273  }
274
275  private int cellsCount(Table table, Filter filter) throws IOException {
276    Scan scan = new Scan().setFilter(filter).readAllVersions();
277    try (ResultScanner scanner = table.getScanner(scan)) {
278      List<Cell> results = new ArrayList<>();
279      Result result;
280      while ((result = scanner.next()) != null) {
281        result.listCells().forEach(results::add);
282      }
283      return results.size();
284    }
285  }
286
287  List<String> generateRandomWords(int numberOfWords, int maxLengthOfWords) {
288    Set<String> wordSet = new HashSet<>();
289    for (int i = 0; i < numberOfWords; i++) {
290      int lengthOfWords = (int) (Math.random() * maxLengthOfWords) + 1;
291      char[] wordChar = new char[lengthOfWords];
292      for (int j = 0; j < wordChar.length; j++) {
293        wordChar[j] = (char) (Math.random() * 26 + 97);
294      }
295      String word = new String(wordChar);
296      wordSet.add(word);
297    }
298    List<String> wordList = new ArrayList<>(wordSet);
299    return wordList;
300  }
301
302}
303