001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.filter; 019 020import static org.junit.Assert.assertEquals; 021 022import java.io.IOException; 023import java.util.ArrayList; 024import java.util.HashMap; 025import java.util.HashSet; 026import java.util.List; 027import java.util.Map; 028import java.util.Set; 029import org.apache.hadoop.hbase.Cell; 030import org.apache.hadoop.hbase.HBaseClassTestRule; 031import org.apache.hadoop.hbase.HBaseTestingUtil; 032import org.apache.hadoop.hbase.KeyValue; 033import org.apache.hadoop.hbase.KeyValueTestUtil; 034import org.apache.hadoop.hbase.TableName; 035import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; 036import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; 037import org.apache.hadoop.hbase.client.Durability; 038import org.apache.hadoop.hbase.client.Put; 039import org.apache.hadoop.hbase.client.RegionInfo; 040import org.apache.hadoop.hbase.client.RegionInfoBuilder; 041import org.apache.hadoop.hbase.client.Scan; 042import org.apache.hadoop.hbase.client.TableDescriptor; 043import org.apache.hadoop.hbase.client.TableDescriptorBuilder; 044import org.apache.hadoop.hbase.regionserver.HRegion; 045import org.apache.hadoop.hbase.regionserver.InternalScanner; 046import org.apache.hadoop.hbase.testclassification.FilterTests; 047import org.apache.hadoop.hbase.testclassification.SmallTests; 048import org.apache.hadoop.hbase.util.Bytes; 049import org.junit.ClassRule; 050import org.junit.Rule; 051import org.junit.Test; 052import org.junit.experimental.categories.Category; 053import org.junit.rules.TestName; 054 055@Category({ FilterTests.class, SmallTests.class }) 056public class TestColumnPrefixFilter { 057 058 @ClassRule 059 public static final HBaseClassTestRule CLASS_RULE = 060 HBaseClassTestRule.forClass(TestColumnPrefixFilter.class); 061 062 private final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil(); 063 064 @Rule 065 public TestName name = new TestName(); 066 067 @Test 068 public void testColumnPrefixFilter() throws IOException { 069 String family = "Family"; 070 TableDescriptorBuilder tableDescriptorBuilder = 071 TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())); 072 ColumnFamilyDescriptor columnFamilyDescriptor = 073 ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(family)).setMaxVersions(3).build(); 074 tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor); 075 TableDescriptor tableDescriptor = tableDescriptorBuilder.build(); 076 RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build(); 077 HRegion region = HBaseTestingUtil.createRegionAndWAL(info, TEST_UTIL.getDataTestDir(), 078 TEST_UTIL.getConfiguration(), tableDescriptor); 079 try { 080 List<String> rows = generateRandomWords(100, "row"); 081 List<String> columns = generateRandomWords(10000, "column"); 082 long maxTimestamp = 2; 083 084 List<Cell> kvList = new ArrayList<>(); 085 086 Map<String, List<Cell>> prefixMap = new HashMap<>(); 087 088 prefixMap.put("p", new ArrayList<>()); 089 prefixMap.put("s", new ArrayList<>()); 090 091 String valueString = "ValueString"; 092 093 for (String row : rows) { 094 Put p = new Put(Bytes.toBytes(row)); 095 p.setDurability(Durability.SKIP_WAL); 096 for (String column : columns) { 097 for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) { 098 KeyValue kv = KeyValueTestUtil.create(row, family, column, timestamp, valueString); 099 p.add(kv); 100 kvList.add(kv); 101 for (String s : prefixMap.keySet()) { 102 if (column.startsWith(s)) { 103 prefixMap.get(s).add(kv); 104 } 105 } 106 } 107 } 108 region.put(p); 109 } 110 111 ColumnPrefixFilter filter; 112 Scan scan = new Scan(); 113 scan.readAllVersions(); 114 for (String s : prefixMap.keySet()) { 115 filter = new ColumnPrefixFilter(Bytes.toBytes(s)); 116 117 scan.setFilter(filter); 118 119 InternalScanner scanner = region.getScanner(scan); 120 List<Cell> results = new ArrayList<>(); 121 while (scanner.next(results)) 122 ; 123 assertEquals(prefixMap.get(s).size(), results.size()); 124 } 125 } finally { 126 HBaseTestingUtil.closeRegionAndWAL(region); 127 } 128 129 HBaseTestingUtil.closeRegionAndWAL(region); 130 } 131 132 @Test 133 public void testColumnPrefixFilterWithFilterList() throws IOException { 134 String family = "Family"; 135 TableDescriptorBuilder tableDescriptorBuilder = 136 TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())); 137 ColumnFamilyDescriptor columnFamilyDescriptor = 138 ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(family)).setMaxVersions(3).build(); 139 tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor); 140 TableDescriptor tableDescriptor = tableDescriptorBuilder.build(); 141 RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build(); 142 HRegion region = HBaseTestingUtil.createRegionAndWAL(info, TEST_UTIL.getDataTestDir(), 143 TEST_UTIL.getConfiguration(), tableDescriptor); 144 try { 145 List<String> rows = generateRandomWords(100, "row"); 146 List<String> columns = generateRandomWords(10000, "column"); 147 long maxTimestamp = 2; 148 149 List<Cell> kvList = new ArrayList<>(); 150 151 Map<String, List<Cell>> prefixMap = new HashMap<>(); 152 153 prefixMap.put("p", new ArrayList<>()); 154 prefixMap.put("s", new ArrayList<>()); 155 156 String valueString = "ValueString"; 157 158 for (String row : rows) { 159 Put p = new Put(Bytes.toBytes(row)); 160 p.setDurability(Durability.SKIP_WAL); 161 for (String column : columns) { 162 for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) { 163 KeyValue kv = KeyValueTestUtil.create(row, family, column, timestamp, valueString); 164 p.add(kv); 165 kvList.add(kv); 166 for (String s : prefixMap.keySet()) { 167 if (column.startsWith(s)) { 168 prefixMap.get(s).add(kv); 169 } 170 } 171 } 172 } 173 region.put(p); 174 } 175 176 ColumnPrefixFilter filter; 177 Scan scan = new Scan(); 178 scan.readAllVersions(); 179 for (String s : prefixMap.keySet()) { 180 filter = new ColumnPrefixFilter(Bytes.toBytes(s)); 181 182 // this is how this test differs from the one above 183 FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL); 184 filterList.addFilter(filter); 185 scan.setFilter(filterList); 186 187 InternalScanner scanner = region.getScanner(scan); 188 List<Cell> results = new ArrayList<>(); 189 while (scanner.next(results)) 190 ; 191 assertEquals(prefixMap.get(s).size(), results.size()); 192 } 193 } finally { 194 HBaseTestingUtil.closeRegionAndWAL(region); 195 } 196 197 HBaseTestingUtil.closeRegionAndWAL(region); 198 } 199 200 List<String> generateRandomWords(int numberOfWords, String suffix) { 201 Set<String> wordSet = new HashSet<>(); 202 for (int i = 0; i < numberOfWords; i++) { 203 int lengthOfWords = (int) (Math.random() * 2) + 1; 204 char[] wordChar = new char[lengthOfWords]; 205 for (int j = 0; j < wordChar.length; j++) { 206 wordChar[j] = (char) (Math.random() * 26 + 97); 207 } 208 String word; 209 if (suffix == null) { 210 word = new String(wordChar); 211 } else { 212 word = new String(wordChar) + suffix; 213 } 214 wordSet.add(word); 215 } 216 List<String> wordList = new ArrayList<>(wordSet); 217 return wordList; 218 } 219 220}