001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.filter;
019
020import static org.junit.Assert.assertEquals;
021
022import java.io.IOException;
023import java.util.ArrayList;
024import java.util.HashMap;
025import java.util.HashSet;
026import java.util.List;
027import java.util.Map;
028import java.util.Set;
029import org.apache.hadoop.hbase.Cell;
030import org.apache.hadoop.hbase.HBaseClassTestRule;
031import org.apache.hadoop.hbase.HBaseTestingUtility;
032import org.apache.hadoop.hbase.HColumnDescriptor;
033import org.apache.hadoop.hbase.HRegionInfo;
034import org.apache.hadoop.hbase.HTableDescriptor;
035import org.apache.hadoop.hbase.KeyValue;
036import org.apache.hadoop.hbase.KeyValueTestUtil;
037import org.apache.hadoop.hbase.TableName;
038import org.apache.hadoop.hbase.client.Durability;
039import org.apache.hadoop.hbase.client.Put;
040import org.apache.hadoop.hbase.client.Scan;
041import org.apache.hadoop.hbase.regionserver.HRegion;
042import org.apache.hadoop.hbase.regionserver.InternalScanner;
043import org.apache.hadoop.hbase.testclassification.FilterTests;
044import org.apache.hadoop.hbase.testclassification.SmallTests;
045import org.apache.hadoop.hbase.util.Bytes;
046import org.junit.ClassRule;
047import org.junit.Rule;
048import org.junit.Test;
049import org.junit.experimental.categories.Category;
050import org.junit.rules.TestName;
051
052@Category({FilterTests.class, SmallTests.class})
053public class TestColumnPrefixFilter {
054
055  @ClassRule
056  public static final HBaseClassTestRule CLASS_RULE =
057      HBaseClassTestRule.forClass(TestColumnPrefixFilter.class);
058
059  private final static HBaseTestingUtility TEST_UTIL = new
060      HBaseTestingUtility();
061
062  @Rule
063  public TestName name = new TestName();
064
065  @Test
066  public void testColumnPrefixFilter() throws IOException {
067    String family = "Family";
068    HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName()));
069    htd.addFamily((new HColumnDescriptor(family)).setMaxVersions(3));
070    HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
071    HRegion region = HBaseTestingUtility.createRegionAndWAL(info, TEST_UTIL.getDataTestDir(),
072        TEST_UTIL.getConfiguration(), htd);
073    try {
074      List<String> rows = generateRandomWords(100, "row");
075      List<String> columns = generateRandomWords(10000, "column");
076      long maxTimestamp = 2;
077
078      List<Cell> kvList = new ArrayList<>();
079
080      Map<String, List<Cell>> prefixMap = new HashMap<>();
081
082      prefixMap.put("p", new ArrayList<>());
083      prefixMap.put("s", new ArrayList<>());
084
085      String valueString = "ValueString";
086
087      for (String row: rows) {
088        Put p = new Put(Bytes.toBytes(row));
089        p.setDurability(Durability.SKIP_WAL);
090        for (String column: columns) {
091          for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
092            KeyValue kv = KeyValueTestUtil.create(row, family, column, timestamp,
093                valueString);
094            p.add(kv);
095            kvList.add(kv);
096            for (String s: prefixMap.keySet()) {
097              if (column.startsWith(s)) {
098                prefixMap.get(s).add(kv);
099              }
100            }
101          }
102        }
103        region.put(p);
104      }
105
106      ColumnPrefixFilter filter;
107      Scan scan = new Scan();
108      scan.setMaxVersions();
109      for (String s: prefixMap.keySet()) {
110        filter = new ColumnPrefixFilter(Bytes.toBytes(s));
111
112        scan.setFilter(filter);
113
114        InternalScanner scanner = region.getScanner(scan);
115        List<Cell> results = new ArrayList<>();
116        while (scanner.next(results))
117          ;
118        assertEquals(prefixMap.get(s).size(), results.size());
119      }
120    } finally {
121      HBaseTestingUtility.closeRegionAndWAL(region);
122    }
123
124    HBaseTestingUtility.closeRegionAndWAL(region);
125  }
126
127  @Test
128  public void testColumnPrefixFilterWithFilterList() throws IOException {
129    String family = "Family";
130    HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName()));
131    htd.addFamily((new HColumnDescriptor(family)).setMaxVersions(3));
132    HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
133    HRegion region = HBaseTestingUtility.createRegionAndWAL(info, TEST_UTIL.getDataTestDir(),
134        TEST_UTIL.getConfiguration(), htd);
135    try {
136      List<String> rows = generateRandomWords(100, "row");
137      List<String> columns = generateRandomWords(10000, "column");
138      long maxTimestamp = 2;
139
140      List<Cell> kvList = new ArrayList<>();
141
142      Map<String, List<Cell>> prefixMap = new HashMap<>();
143
144      prefixMap.put("p", new ArrayList<>());
145      prefixMap.put("s", new ArrayList<>());
146
147      String valueString = "ValueString";
148
149      for (String row: rows) {
150        Put p = new Put(Bytes.toBytes(row));
151        p.setDurability(Durability.SKIP_WAL);
152        for (String column: columns) {
153          for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
154            KeyValue kv = KeyValueTestUtil.create(row, family, column, timestamp,
155                valueString);
156            p.add(kv);
157            kvList.add(kv);
158            for (String s: prefixMap.keySet()) {
159              if (column.startsWith(s)) {
160                prefixMap.get(s).add(kv);
161              }
162            }
163          }
164        }
165        region.put(p);
166      }
167
168      ColumnPrefixFilter filter;
169      Scan scan = new Scan();
170      scan.setMaxVersions();
171      for (String s: prefixMap.keySet()) {
172        filter = new ColumnPrefixFilter(Bytes.toBytes(s));
173
174        //this is how this test differs from the one above
175        FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL);
176        filterList.addFilter(filter);
177        scan.setFilter(filterList);
178
179        InternalScanner scanner = region.getScanner(scan);
180        List<Cell> results = new ArrayList<>();
181        while (scanner.next(results))
182          ;
183        assertEquals(prefixMap.get(s).size(), results.size());
184      }
185    } finally {
186      HBaseTestingUtility.closeRegionAndWAL(region);
187    }
188
189    HBaseTestingUtility.closeRegionAndWAL(region);
190  }
191
192  List<String> generateRandomWords(int numberOfWords, String suffix) {
193    Set<String> wordSet = new HashSet<>();
194    for (int i = 0; i < numberOfWords; i++) {
195      int lengthOfWords = (int) (Math.random()*2) + 1;
196      char[] wordChar = new char[lengthOfWords];
197      for (int j = 0; j < wordChar.length; j++) {
198        wordChar[j] = (char) (Math.random() * 26 + 97);
199      }
200      String word;
201      if (suffix == null) {
202        word = new String(wordChar);
203      } else {
204        word = new String(wordChar) + suffix;
205      }
206      wordSet.add(word);
207    }
208    List<String> wordList = new ArrayList<>(wordSet);
209    return wordList;
210  }
211
212}
213