001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.filter;
019
020import static org.junit.Assert.assertEquals;
021
022import java.io.IOException;
023import java.util.ArrayList;
024import java.util.HashMap;
025import java.util.HashSet;
026import java.util.List;
027import java.util.Map;
028import java.util.Set;
029import org.apache.hadoop.hbase.Cell;
030import org.apache.hadoop.hbase.HBaseClassTestRule;
031import org.apache.hadoop.hbase.HBaseTestingUtil;
032import org.apache.hadoop.hbase.KeyValue;
033import org.apache.hadoop.hbase.KeyValueTestUtil;
034import org.apache.hadoop.hbase.TableName;
035import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
036import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
037import org.apache.hadoop.hbase.client.Durability;
038import org.apache.hadoop.hbase.client.Put;
039import org.apache.hadoop.hbase.client.RegionInfo;
040import org.apache.hadoop.hbase.client.RegionInfoBuilder;
041import org.apache.hadoop.hbase.client.Scan;
042import org.apache.hadoop.hbase.client.TableDescriptor;
043import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
044import org.apache.hadoop.hbase.regionserver.HRegion;
045import org.apache.hadoop.hbase.regionserver.InternalScanner;
046import org.apache.hadoop.hbase.testclassification.FilterTests;
047import org.apache.hadoop.hbase.testclassification.SmallTests;
048import org.apache.hadoop.hbase.util.Bytes;
049import org.junit.ClassRule;
050import org.junit.Rule;
051import org.junit.Test;
052import org.junit.experimental.categories.Category;
053import org.junit.rules.TestName;
054
055@Category({ FilterTests.class, SmallTests.class })
056public class TestColumnPrefixFilter {
057
058  @ClassRule
059  public static final HBaseClassTestRule CLASS_RULE =
060    HBaseClassTestRule.forClass(TestColumnPrefixFilter.class);
061
062  private final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
063
064  @Rule
065  public TestName name = new TestName();
066
067  @Test
068  public void testColumnPrefixFilter() throws IOException {
069    String family = "Family";
070    TableDescriptorBuilder tableDescriptorBuilder =
071      TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName()));
072    ColumnFamilyDescriptor columnFamilyDescriptor =
073      ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(family)).setMaxVersions(3).build();
074    tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
075    TableDescriptor tableDescriptor = tableDescriptorBuilder.build();
076    RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
077    HRegion region = HBaseTestingUtil.createRegionAndWAL(info, TEST_UTIL.getDataTestDir(),
078      TEST_UTIL.getConfiguration(), tableDescriptor);
079    try {
080      List<String> rows = generateRandomWords(100, "row");
081      List<String> columns = generateRandomWords(10000, "column");
082      long maxTimestamp = 2;
083
084      List<Cell> kvList = new ArrayList<>();
085
086      Map<String, List<Cell>> prefixMap = new HashMap<>();
087
088      prefixMap.put("p", new ArrayList<>());
089      prefixMap.put("s", new ArrayList<>());
090
091      String valueString = "ValueString";
092
093      for (String row : rows) {
094        Put p = new Put(Bytes.toBytes(row));
095        p.setDurability(Durability.SKIP_WAL);
096        for (String column : columns) {
097          for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
098            KeyValue kv = KeyValueTestUtil.create(row, family, column, timestamp, valueString);
099            p.add(kv);
100            kvList.add(kv);
101            for (String s : prefixMap.keySet()) {
102              if (column.startsWith(s)) {
103                prefixMap.get(s).add(kv);
104              }
105            }
106          }
107        }
108        region.put(p);
109      }
110
111      ColumnPrefixFilter filter;
112      Scan scan = new Scan();
113      scan.readAllVersions();
114      for (String s : prefixMap.keySet()) {
115        filter = new ColumnPrefixFilter(Bytes.toBytes(s));
116
117        scan.setFilter(filter);
118
119        InternalScanner scanner = region.getScanner(scan);
120        List<Cell> results = new ArrayList<>();
121        while (scanner.next(results))
122          ;
123        assertEquals(prefixMap.get(s).size(), results.size());
124      }
125    } finally {
126      HBaseTestingUtil.closeRegionAndWAL(region);
127    }
128
129    HBaseTestingUtil.closeRegionAndWAL(region);
130  }
131
132  @Test
133  public void testColumnPrefixFilterWithFilterList() throws IOException {
134    String family = "Family";
135    TableDescriptorBuilder tableDescriptorBuilder =
136      TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName()));
137    ColumnFamilyDescriptor columnFamilyDescriptor =
138      ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(family)).setMaxVersions(3).build();
139    tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
140    TableDescriptor tableDescriptor = tableDescriptorBuilder.build();
141    RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
142    HRegion region = HBaseTestingUtil.createRegionAndWAL(info, TEST_UTIL.getDataTestDir(),
143      TEST_UTIL.getConfiguration(), tableDescriptor);
144    try {
145      List<String> rows = generateRandomWords(100, "row");
146      List<String> columns = generateRandomWords(10000, "column");
147      long maxTimestamp = 2;
148
149      List<Cell> kvList = new ArrayList<>();
150
151      Map<String, List<Cell>> prefixMap = new HashMap<>();
152
153      prefixMap.put("p", new ArrayList<>());
154      prefixMap.put("s", new ArrayList<>());
155
156      String valueString = "ValueString";
157
158      for (String row : rows) {
159        Put p = new Put(Bytes.toBytes(row));
160        p.setDurability(Durability.SKIP_WAL);
161        for (String column : columns) {
162          for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
163            KeyValue kv = KeyValueTestUtil.create(row, family, column, timestamp, valueString);
164            p.add(kv);
165            kvList.add(kv);
166            for (String s : prefixMap.keySet()) {
167              if (column.startsWith(s)) {
168                prefixMap.get(s).add(kv);
169              }
170            }
171          }
172        }
173        region.put(p);
174      }
175
176      ColumnPrefixFilter filter;
177      Scan scan = new Scan();
178      scan.readAllVersions();
179      for (String s : prefixMap.keySet()) {
180        filter = new ColumnPrefixFilter(Bytes.toBytes(s));
181
182        // this is how this test differs from the one above
183        FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL);
184        filterList.addFilter(filter);
185        scan.setFilter(filterList);
186
187        InternalScanner scanner = region.getScanner(scan);
188        List<Cell> results = new ArrayList<>();
189        while (scanner.next(results))
190          ;
191        assertEquals(prefixMap.get(s).size(), results.size());
192      }
193    } finally {
194      HBaseTestingUtil.closeRegionAndWAL(region);
195    }
196
197    HBaseTestingUtil.closeRegionAndWAL(region);
198  }
199
200  List<String> generateRandomWords(int numberOfWords, String suffix) {
201    Set<String> wordSet = new HashSet<>();
202    for (int i = 0; i < numberOfWords; i++) {
203      int lengthOfWords = (int) (Math.random() * 2) + 1;
204      char[] wordChar = new char[lengthOfWords];
205      for (int j = 0; j < wordChar.length; j++) {
206        wordChar[j] = (char) (Math.random() * 26 + 97);
207      }
208      String word;
209      if (suffix == null) {
210        word = new String(wordChar);
211      } else {
212        word = new String(wordChar) + suffix;
213      }
214      wordSet.add(word);
215    }
216    List<String> wordList = new ArrayList<>(wordSet);
217    return wordList;
218  }
219
220}