001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.filter;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021
022import java.io.IOException;
023import java.util.ArrayList;
024import java.util.HashMap;
025import java.util.HashSet;
026import java.util.List;
027import java.util.Map;
028import java.util.Set;
029import org.apache.hadoop.hbase.Cell;
030import org.apache.hadoop.hbase.HBaseTestingUtil;
031import org.apache.hadoop.hbase.KeyValue;
032import org.apache.hadoop.hbase.KeyValueTestUtil;
033import org.apache.hadoop.hbase.TableName;
034import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
035import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
036import org.apache.hadoop.hbase.client.Durability;
037import org.apache.hadoop.hbase.client.Put;
038import org.apache.hadoop.hbase.client.RegionInfo;
039import org.apache.hadoop.hbase.client.RegionInfoBuilder;
040import org.apache.hadoop.hbase.client.Scan;
041import org.apache.hadoop.hbase.client.TableDescriptor;
042import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
043import org.apache.hadoop.hbase.regionserver.HRegion;
044import org.apache.hadoop.hbase.regionserver.InternalScanner;
045import org.apache.hadoop.hbase.testclassification.FilterTests;
046import org.apache.hadoop.hbase.testclassification.SmallTests;
047import org.apache.hadoop.hbase.util.Bytes;
048import org.junit.jupiter.api.Tag;
049import org.junit.jupiter.api.Test;
050import org.junit.jupiter.api.TestInfo;
051
052@Tag(FilterTests.TAG)
053@Tag(SmallTests.TAG)
054public class TestColumnPrefixFilter {
055
056  private final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
057
058  @Test
059  public void testColumnPrefixFilter(TestInfo testInfo) throws IOException {
060    String family = "Family";
061    TableDescriptorBuilder tableDescriptorBuilder = TableDescriptorBuilder
062      .newBuilder(TableName.valueOf(testInfo.getTestMethod().get().getName()));
063    ColumnFamilyDescriptor columnFamilyDescriptor =
064      ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(family)).setMaxVersions(3).build();
065    tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
066    TableDescriptor tableDescriptor = tableDescriptorBuilder.build();
067    RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
068    HRegion region = HBaseTestingUtil.createRegionAndWAL(info, TEST_UTIL.getDataTestDir(),
069      TEST_UTIL.getConfiguration(), tableDescriptor);
070    try {
071      List<String> rows = generateRandomWords(100, "row");
072      List<String> columns = generateRandomWords(10000, "column");
073      long maxTimestamp = 2;
074
075      List<Cell> kvList = new ArrayList<>();
076
077      Map<String, List<Cell>> prefixMap = new HashMap<>();
078
079      prefixMap.put("p", new ArrayList<>());
080      prefixMap.put("s", new ArrayList<>());
081
082      String valueString = "ValueString";
083
084      for (String row : rows) {
085        Put p = new Put(Bytes.toBytes(row));
086        p.setDurability(Durability.SKIP_WAL);
087        for (String column : columns) {
088          for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
089            KeyValue kv = KeyValueTestUtil.create(row, family, column, timestamp, valueString);
090            p.add(kv);
091            kvList.add(kv);
092            for (String s : prefixMap.keySet()) {
093              if (column.startsWith(s)) {
094                prefixMap.get(s).add(kv);
095              }
096            }
097          }
098        }
099        region.put(p);
100      }
101
102      ColumnPrefixFilter filter;
103      Scan scan = new Scan();
104      scan.readAllVersions();
105      for (String s : prefixMap.keySet()) {
106        filter = new ColumnPrefixFilter(Bytes.toBytes(s));
107
108        scan.setFilter(filter);
109
110        InternalScanner scanner = region.getScanner(scan);
111        List<Cell> results = new ArrayList<>();
112        while (scanner.next(results))
113          ;
114        assertEquals(prefixMap.get(s).size(), results.size());
115      }
116    } finally {
117      HBaseTestingUtil.closeRegionAndWAL(region);
118    }
119
120    HBaseTestingUtil.closeRegionAndWAL(region);
121  }
122
123  @Test
124  public void testColumnPrefixFilterWithFilterList(TestInfo testInfo) throws IOException {
125    String family = "Family";
126    TableDescriptorBuilder tableDescriptorBuilder = TableDescriptorBuilder
127      .newBuilder(TableName.valueOf(testInfo.getTestMethod().get().getName()));
128    ColumnFamilyDescriptor columnFamilyDescriptor =
129      ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(family)).setMaxVersions(3).build();
130    tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
131    TableDescriptor tableDescriptor = tableDescriptorBuilder.build();
132    RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
133    HRegion region = HBaseTestingUtil.createRegionAndWAL(info, TEST_UTIL.getDataTestDir(),
134      TEST_UTIL.getConfiguration(), tableDescriptor);
135    try {
136      List<String> rows = generateRandomWords(100, "row");
137      List<String> columns = generateRandomWords(10000, "column");
138      long maxTimestamp = 2;
139
140      List<Cell> kvList = new ArrayList<>();
141
142      Map<String, List<Cell>> prefixMap = new HashMap<>();
143
144      prefixMap.put("p", new ArrayList<>());
145      prefixMap.put("s", new ArrayList<>());
146
147      String valueString = "ValueString";
148
149      for (String row : rows) {
150        Put p = new Put(Bytes.toBytes(row));
151        p.setDurability(Durability.SKIP_WAL);
152        for (String column : columns) {
153          for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
154            KeyValue kv = KeyValueTestUtil.create(row, family, column, timestamp, valueString);
155            p.add(kv);
156            kvList.add(kv);
157            for (String s : prefixMap.keySet()) {
158              if (column.startsWith(s)) {
159                prefixMap.get(s).add(kv);
160              }
161            }
162          }
163        }
164        region.put(p);
165      }
166
167      ColumnPrefixFilter filter;
168      Scan scan = new Scan();
169      scan.readAllVersions();
170      for (String s : prefixMap.keySet()) {
171        filter = new ColumnPrefixFilter(Bytes.toBytes(s));
172
173        // this is how this test differs from the one above
174        FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL);
175        filterList.addFilter(filter);
176        scan.setFilter(filterList);
177
178        InternalScanner scanner = region.getScanner(scan);
179        List<Cell> results = new ArrayList<>();
180        while (scanner.next(results))
181          ;
182        assertEquals(prefixMap.get(s).size(), results.size());
183      }
184    } finally {
185      HBaseTestingUtil.closeRegionAndWAL(region);
186    }
187
188    HBaseTestingUtil.closeRegionAndWAL(region);
189  }
190
191  List<String> generateRandomWords(int numberOfWords, String suffix) {
192    Set<String> wordSet = new HashSet<>();
193    for (int i = 0; i < numberOfWords; i++) {
194      int lengthOfWords = (int) (Math.random() * 2) + 1;
195      char[] wordChar = new char[lengthOfWords];
196      for (int j = 0; j < wordChar.length; j++) {
197        wordChar[j] = (char) (Math.random() * 26 + 97);
198      }
199      String word;
200      if (suffix == null) {
201        word = new String(wordChar);
202      } else {
203        word = new String(wordChar) + suffix;
204      }
205      wordSet.add(word);
206    }
207    List<String> wordList = new ArrayList<>(wordSet);
208    return wordList;
209  }
210
211}