001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.filter;
019
020import static org.junit.Assert.assertEquals;
021
022import java.io.IOException;
023import java.util.ArrayList;
024import java.util.HashMap;
025import java.util.HashSet;
026import java.util.List;
027import java.util.Map;
028import java.util.Set;
029import org.apache.hadoop.hbase.Cell;
030import org.apache.hadoop.hbase.HBaseClassTestRule;
031import org.apache.hadoop.hbase.HBaseTestingUtility;
032import org.apache.hadoop.hbase.HColumnDescriptor;
033import org.apache.hadoop.hbase.HRegionInfo;
034import org.apache.hadoop.hbase.HTableDescriptor;
035import org.apache.hadoop.hbase.KeyValue;
036import org.apache.hadoop.hbase.KeyValueTestUtil;
037import org.apache.hadoop.hbase.TableName;
038import org.apache.hadoop.hbase.client.Durability;
039import org.apache.hadoop.hbase.client.Put;
040import org.apache.hadoop.hbase.client.Scan;
041import org.apache.hadoop.hbase.regionserver.HRegion;
042import org.apache.hadoop.hbase.regionserver.InternalScanner;
043import org.apache.hadoop.hbase.testclassification.FilterTests;
044import org.apache.hadoop.hbase.testclassification.SmallTests;
045import org.apache.hadoop.hbase.util.Bytes;
046import org.junit.ClassRule;
047import org.junit.Rule;
048import org.junit.Test;
049import org.junit.experimental.categories.Category;
050import org.junit.rules.TestName;
051
052@Category({FilterTests.class, SmallTests.class})
053public class TestMultipleColumnPrefixFilter {
054
055  @ClassRule
056  public static final HBaseClassTestRule CLASS_RULE =
057      HBaseClassTestRule.forClass(TestMultipleColumnPrefixFilter.class);
058
059  private final static HBaseTestingUtility TEST_UTIL = new
060      HBaseTestingUtility();
061
062  @Rule
063  public TestName name = new TestName();
064
065  @Test
066  public void testMultipleColumnPrefixFilter() throws IOException {
067    String family = "Family";
068    HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName()));
069    HColumnDescriptor hcd = new HColumnDescriptor(family);
070    hcd.setMaxVersions(3);
071    htd.addFamily(hcd);
072    // HRegionInfo info = new HRegionInfo(htd, null, null, false);
073    HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
074    HRegion region = HBaseTestingUtility.createRegionAndWAL(info, TEST_UTIL.
075        getDataTestDir(), TEST_UTIL.getConfiguration(), htd);
076
077    List<String> rows = generateRandomWords(100, "row");
078    List<String> columns = generateRandomWords(10000, "column");
079    long maxTimestamp = 2;
080
081    List<Cell> kvList = new ArrayList<>();
082
083    Map<String, List<Cell>> prefixMap = new HashMap<>();
084
085    prefixMap.put("p", new ArrayList<>());
086    prefixMap.put("q", new ArrayList<>());
087    prefixMap.put("s", new ArrayList<>());
088
089    String valueString = "ValueString";
090
091    for (String row: rows) {
092      Put p = new Put(Bytes.toBytes(row));
093      p.setDurability(Durability.SKIP_WAL);
094      for (String column: columns) {
095        for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
096          KeyValue kv = KeyValueTestUtil.create(row, family, column, timestamp,
097              valueString);
098          p.add(kv);
099          kvList.add(kv);
100          for (String s: prefixMap.keySet()) {
101            if (column.startsWith(s)) {
102              prefixMap.get(s).add(kv);
103            }
104          }
105        }
106      }
107      region.put(p);
108    }
109
110    MultipleColumnPrefixFilter filter;
111    Scan scan = new Scan();
112    scan.setMaxVersions();
113    byte [][] filter_prefix = new byte [2][];
114    filter_prefix[0] = new byte [] {'p'};
115    filter_prefix[1] = new byte [] {'q'};
116
117    filter = new MultipleColumnPrefixFilter(filter_prefix);
118    scan.setFilter(filter);
119    List<Cell> results = new ArrayList<>();
120    InternalScanner scanner = region.getScanner(scan);
121    while (scanner.next(results))
122      ;
123    assertEquals(prefixMap.get("p").size() + prefixMap.get("q").size(), results.size());
124
125    HBaseTestingUtility.closeRegionAndWAL(region);
126  }
127
128  @Test
129  public void testMultipleColumnPrefixFilterWithManyFamilies() throws IOException {
130    String family1 = "Family1";
131    String family2 = "Family2";
132    HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName()));
133    HColumnDescriptor hcd1 = new HColumnDescriptor(family1);
134    hcd1.setMaxVersions(3);
135    htd.addFamily(hcd1);
136    HColumnDescriptor hcd2 = new HColumnDescriptor(family2);
137    hcd2.setMaxVersions(3);
138    htd.addFamily(hcd2);
139    HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
140    HRegion region = HBaseTestingUtility.createRegionAndWAL(info, TEST_UTIL.
141        getDataTestDir(), TEST_UTIL.getConfiguration(), htd);
142
143    List<String> rows = generateRandomWords(100, "row");
144    List<String> columns = generateRandomWords(10000, "column");
145    long maxTimestamp = 3;
146
147    List<Cell> kvList = new ArrayList<>();
148
149    Map<String, List<Cell>> prefixMap = new HashMap<>();
150
151    prefixMap.put("p", new ArrayList<>());
152    prefixMap.put("q", new ArrayList<>());
153    prefixMap.put("s", new ArrayList<>());
154
155    String valueString = "ValueString";
156
157    for (String row: rows) {
158      Put p = new Put(Bytes.toBytes(row));
159      p.setDurability(Durability.SKIP_WAL);
160      for (String column: columns) {
161        for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
162          double rand = Math.random();
163          Cell kv;
164          if (rand < 0.5) {
165            kv = KeyValueTestUtil.create(row, family1, column, timestamp, valueString);
166          } else {
167            kv = KeyValueTestUtil.create(row, family2, column, timestamp, valueString);
168          }
169          p.add(kv);
170          kvList.add(kv);
171          for (String s: prefixMap.keySet()) {
172            if (column.startsWith(s)) {
173              prefixMap.get(s).add(kv);
174            }
175          }
176        }
177      }
178      region.put(p);
179    }
180
181    MultipleColumnPrefixFilter filter;
182    Scan scan = new Scan();
183    scan.setMaxVersions();
184    byte [][] filter_prefix = new byte [2][];
185    filter_prefix[0] = new byte [] {'p'};
186    filter_prefix[1] = new byte [] {'q'};
187
188    filter = new MultipleColumnPrefixFilter(filter_prefix);
189    scan.setFilter(filter);
190    List<Cell> results = new ArrayList<>();
191    InternalScanner scanner = region.getScanner(scan);
192    while (scanner.next(results))
193      ;
194    assertEquals(prefixMap.get("p").size() + prefixMap.get("q").size(), results.size());
195
196    HBaseTestingUtility.closeRegionAndWAL(region);
197  }
198
199  @Test
200  public void testMultipleColumnPrefixFilterWithColumnPrefixFilter() throws IOException {
201    String family = "Family";
202    HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName()));
203    htd.addFamily(new HColumnDescriptor(family));
204    HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
205    HRegion region = HBaseTestingUtility.createRegionAndWAL(info, TEST_UTIL.
206        getDataTestDir(), TEST_UTIL.getConfiguration(), htd);
207
208    List<String> rows = generateRandomWords(100, "row");
209    List<String> columns = generateRandomWords(10000, "column");
210    long maxTimestamp = 2;
211
212    String valueString = "ValueString";
213
214    for (String row: rows) {
215      Put p = new Put(Bytes.toBytes(row));
216      p.setDurability(Durability.SKIP_WAL);
217      for (String column: columns) {
218        for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
219          KeyValue kv = KeyValueTestUtil.create(row, family, column, timestamp,
220              valueString);
221          p.add(kv);
222        }
223      }
224      region.put(p);
225    }
226
227    MultipleColumnPrefixFilter multiplePrefixFilter;
228    Scan scan1 = new Scan();
229    scan1.setMaxVersions();
230    byte [][] filter_prefix = new byte [1][];
231    filter_prefix[0] = new byte [] {'p'};
232
233    multiplePrefixFilter = new MultipleColumnPrefixFilter(filter_prefix);
234    scan1.setFilter(multiplePrefixFilter);
235    List<Cell> results1 = new ArrayList<>();
236    InternalScanner scanner1 = region.getScanner(scan1);
237    while (scanner1.next(results1))
238      ;
239
240    ColumnPrefixFilter singlePrefixFilter;
241    Scan scan2 = new Scan();
242    scan2.setMaxVersions();
243    singlePrefixFilter = new ColumnPrefixFilter(Bytes.toBytes("p"));
244
245    scan2.setFilter(singlePrefixFilter);
246    List<Cell> results2 = new ArrayList<>();
247    InternalScanner scanner2 = region.getScanner(scan1);
248    while (scanner2.next(results2))
249      ;
250
251    assertEquals(results1.size(), results2.size());
252
253    HBaseTestingUtility.closeRegionAndWAL(region);
254  }
255
256  List<String> generateRandomWords(int numberOfWords, String suffix) {
257    Set<String> wordSet = new HashSet<>();
258    for (int i = 0; i < numberOfWords; i++) {
259      int lengthOfWords = (int) (Math.random()*2) + 1;
260      char[] wordChar = new char[lengthOfWords];
261      for (int j = 0; j < wordChar.length; j++) {
262        wordChar[j] = (char) (Math.random() * 26 + 97);
263      }
264      String word;
265      if (suffix == null) {
266        word = new String(wordChar);
267      } else {
268        word = new String(wordChar) + suffix;
269      }
270      wordSet.add(word);
271    }
272    List<String> wordList = new ArrayList<>(wordSet);
273    return wordList;
274  }
275
276}
277
278