001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.filter;
019
020import static org.junit.Assert.assertEquals;
021
022import java.io.IOException;
023import java.util.ArrayList;
024import java.util.HashMap;
025import java.util.HashSet;
026import java.util.List;
027import java.util.Map;
028import java.util.Set;
029import org.apache.hadoop.hbase.Cell;
030import org.apache.hadoop.hbase.HBaseClassTestRule;
031import org.apache.hadoop.hbase.HBaseTestingUtility;
032import org.apache.hadoop.hbase.HColumnDescriptor;
033import org.apache.hadoop.hbase.HRegionInfo;
034import org.apache.hadoop.hbase.HTableDescriptor;
035import org.apache.hadoop.hbase.KeyValue;
036import org.apache.hadoop.hbase.KeyValueTestUtil;
037import org.apache.hadoop.hbase.TableName;
038import org.apache.hadoop.hbase.client.Durability;
039import org.apache.hadoop.hbase.client.Put;
040import org.apache.hadoop.hbase.client.Scan;
041import org.apache.hadoop.hbase.regionserver.HRegion;
042import org.apache.hadoop.hbase.regionserver.InternalScanner;
043import org.apache.hadoop.hbase.testclassification.FilterTests;
044import org.apache.hadoop.hbase.testclassification.SmallTests;
045import org.apache.hadoop.hbase.util.Bytes;
046import org.junit.ClassRule;
047import org.junit.Rule;
048import org.junit.Test;
049import org.junit.experimental.categories.Category;
050import org.junit.rules.TestName;
051
052@Category({ FilterTests.class, SmallTests.class })
053public class TestColumnPrefixFilter {
054
055  @ClassRule
056  public static final HBaseClassTestRule CLASS_RULE =
057    HBaseClassTestRule.forClass(TestColumnPrefixFilter.class);
058
059  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
060
061  @Rule
062  public TestName name = new TestName();
063
064  @Test
065  public void testColumnPrefixFilter() throws IOException {
066    String family = "Family";
067    HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName()));
068    htd.addFamily((new HColumnDescriptor(family)).setMaxVersions(3));
069    HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
070    HRegion region = HBaseTestingUtility.createRegionAndWAL(info, TEST_UTIL.getDataTestDir(),
071      TEST_UTIL.getConfiguration(), htd);
072    try {
073      List<String> rows = generateRandomWords(100, "row");
074      List<String> columns = generateRandomWords(10000, "column");
075      long maxTimestamp = 2;
076
077      List<Cell> kvList = new ArrayList<>();
078
079      Map<String, List<Cell>> prefixMap = new HashMap<>();
080
081      prefixMap.put("p", new ArrayList<>());
082      prefixMap.put("s", new ArrayList<>());
083
084      String valueString = "ValueString";
085
086      for (String row : rows) {
087        Put p = new Put(Bytes.toBytes(row));
088        p.setDurability(Durability.SKIP_WAL);
089        for (String column : columns) {
090          for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
091            KeyValue kv = KeyValueTestUtil.create(row, family, column, timestamp, valueString);
092            p.add(kv);
093            kvList.add(kv);
094            for (String s : prefixMap.keySet()) {
095              if (column.startsWith(s)) {
096                prefixMap.get(s).add(kv);
097              }
098            }
099          }
100        }
101        region.put(p);
102      }
103
104      ColumnPrefixFilter filter;
105      Scan scan = new Scan();
106      scan.setMaxVersions();
107      for (String s : prefixMap.keySet()) {
108        filter = new ColumnPrefixFilter(Bytes.toBytes(s));
109
110        scan.setFilter(filter);
111
112        InternalScanner scanner = region.getScanner(scan);
113        List<Cell> results = new ArrayList<>();
114        while (scanner.next(results))
115          ;
116        assertEquals(prefixMap.get(s).size(), results.size());
117      }
118    } finally {
119      HBaseTestingUtility.closeRegionAndWAL(region);
120    }
121
122    HBaseTestingUtility.closeRegionAndWAL(region);
123  }
124
125  @Test
126  public void testColumnPrefixFilterWithFilterList() throws IOException {
127    String family = "Family";
128    HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName()));
129    htd.addFamily((new HColumnDescriptor(family)).setMaxVersions(3));
130    HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
131    HRegion region = HBaseTestingUtility.createRegionAndWAL(info, TEST_UTIL.getDataTestDir(),
132      TEST_UTIL.getConfiguration(), htd);
133    try {
134      List<String> rows = generateRandomWords(100, "row");
135      List<String> columns = generateRandomWords(10000, "column");
136      long maxTimestamp = 2;
137
138      List<Cell> kvList = new ArrayList<>();
139
140      Map<String, List<Cell>> prefixMap = new HashMap<>();
141
142      prefixMap.put("p", new ArrayList<>());
143      prefixMap.put("s", new ArrayList<>());
144
145      String valueString = "ValueString";
146
147      for (String row : rows) {
148        Put p = new Put(Bytes.toBytes(row));
149        p.setDurability(Durability.SKIP_WAL);
150        for (String column : columns) {
151          for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
152            KeyValue kv = KeyValueTestUtil.create(row, family, column, timestamp, valueString);
153            p.add(kv);
154            kvList.add(kv);
155            for (String s : prefixMap.keySet()) {
156              if (column.startsWith(s)) {
157                prefixMap.get(s).add(kv);
158              }
159            }
160          }
161        }
162        region.put(p);
163      }
164
165      ColumnPrefixFilter filter;
166      Scan scan = new Scan();
167      scan.setMaxVersions();
168      for (String s : prefixMap.keySet()) {
169        filter = new ColumnPrefixFilter(Bytes.toBytes(s));
170
171        // this is how this test differs from the one above
172        FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL);
173        filterList.addFilter(filter);
174        scan.setFilter(filterList);
175
176        InternalScanner scanner = region.getScanner(scan);
177        List<Cell> results = new ArrayList<>();
178        while (scanner.next(results))
179          ;
180        assertEquals(prefixMap.get(s).size(), results.size());
181      }
182    } finally {
183      HBaseTestingUtility.closeRegionAndWAL(region);
184    }
185
186    HBaseTestingUtility.closeRegionAndWAL(region);
187  }
188
189  List<String> generateRandomWords(int numberOfWords, String suffix) {
190    Set<String> wordSet = new HashSet<>();
191    for (int i = 0; i < numberOfWords; i++) {
192      int lengthOfWords = (int) (Math.random() * 2) + 1;
193      char[] wordChar = new char[lengthOfWords];
194      for (int j = 0; j < wordChar.length; j++) {
195        wordChar[j] = (char) (Math.random() * 26 + 97);
196      }
197      String word;
198      if (suffix == null) {
199        word = new String(wordChar);
200      } else {
201        word = new String(wordChar) + suffix;
202      }
203      wordSet.add(word);
204    }
205    List<String> wordList = new ArrayList<>(wordSet);
206    return wordList;
207  }
208
209}