001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.filter; 019 020import static org.junit.Assert.assertEquals; 021 022import java.io.IOException; 023import java.util.ArrayList; 024import java.util.HashMap; 025import java.util.HashSet; 026import java.util.List; 027import java.util.Map; 028import java.util.Set; 029import org.apache.hadoop.hbase.Cell; 030import org.apache.hadoop.hbase.HBaseClassTestRule; 031import org.apache.hadoop.hbase.HBaseTestingUtility; 032import org.apache.hadoop.hbase.HColumnDescriptor; 033import org.apache.hadoop.hbase.HRegionInfo; 034import org.apache.hadoop.hbase.HTableDescriptor; 035import org.apache.hadoop.hbase.KeyValue; 036import org.apache.hadoop.hbase.KeyValueTestUtil; 037import org.apache.hadoop.hbase.TableName; 038import org.apache.hadoop.hbase.client.Durability; 039import org.apache.hadoop.hbase.client.Put; 040import org.apache.hadoop.hbase.client.Scan; 041import org.apache.hadoop.hbase.regionserver.HRegion; 042import org.apache.hadoop.hbase.regionserver.InternalScanner; 043import org.apache.hadoop.hbase.testclassification.FilterTests; 044import org.apache.hadoop.hbase.testclassification.SmallTests; 045import org.apache.hadoop.hbase.util.Bytes; 046import org.junit.ClassRule; 047import org.junit.Rule; 048import org.junit.Test; 049import org.junit.experimental.categories.Category; 050import org.junit.rules.TestName; 051 052@Category({FilterTests.class, SmallTests.class}) 053public class TestColumnPrefixFilter { 054 055 @ClassRule 056 public static final HBaseClassTestRule CLASS_RULE = 057 HBaseClassTestRule.forClass(TestColumnPrefixFilter.class); 058 059 private final static HBaseTestingUtility TEST_UTIL = new 060 HBaseTestingUtility(); 061 062 @Rule 063 public TestName name = new TestName(); 064 065 @Test 066 public void testColumnPrefixFilter() throws IOException { 067 String family = "Family"; 068 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName())); 069 htd.addFamily((new HColumnDescriptor(family)).setMaxVersions(3)); 070 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false); 071 HRegion region = HBaseTestingUtility.createRegionAndWAL(info, TEST_UTIL.getDataTestDir(), 072 TEST_UTIL.getConfiguration(), htd); 073 try { 074 List<String> rows = generateRandomWords(100, "row"); 075 List<String> columns = generateRandomWords(10000, "column"); 076 long maxTimestamp = 2; 077 078 List<Cell> kvList = new ArrayList<>(); 079 080 Map<String, List<Cell>> prefixMap = new HashMap<>(); 081 082 prefixMap.put("p", new ArrayList<>()); 083 prefixMap.put("s", new ArrayList<>()); 084 085 String valueString = "ValueString"; 086 087 for (String row: rows) { 088 Put p = new Put(Bytes.toBytes(row)); 089 p.setDurability(Durability.SKIP_WAL); 090 for (String column: columns) { 091 for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) { 092 KeyValue kv = KeyValueTestUtil.create(row, family, column, timestamp, 093 valueString); 094 p.add(kv); 095 kvList.add(kv); 096 for (String s: prefixMap.keySet()) { 097 if (column.startsWith(s)) { 098 prefixMap.get(s).add(kv); 099 } 100 } 101 } 102 } 103 region.put(p); 104 } 105 106 ColumnPrefixFilter filter; 107 Scan scan = new Scan(); 108 scan.setMaxVersions(); 109 for (String s: prefixMap.keySet()) { 110 filter = new ColumnPrefixFilter(Bytes.toBytes(s)); 111 112 scan.setFilter(filter); 113 114 InternalScanner scanner = region.getScanner(scan); 115 List<Cell> results = new ArrayList<>(); 116 while (scanner.next(results)) 117 ; 118 assertEquals(prefixMap.get(s).size(), results.size()); 119 } 120 } finally { 121 HBaseTestingUtility.closeRegionAndWAL(region); 122 } 123 124 HBaseTestingUtility.closeRegionAndWAL(region); 125 } 126 127 @Test 128 public void testColumnPrefixFilterWithFilterList() throws IOException { 129 String family = "Family"; 130 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName())); 131 htd.addFamily((new HColumnDescriptor(family)).setMaxVersions(3)); 132 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false); 133 HRegion region = HBaseTestingUtility.createRegionAndWAL(info, TEST_UTIL.getDataTestDir(), 134 TEST_UTIL.getConfiguration(), htd); 135 try { 136 List<String> rows = generateRandomWords(100, "row"); 137 List<String> columns = generateRandomWords(10000, "column"); 138 long maxTimestamp = 2; 139 140 List<Cell> kvList = new ArrayList<>(); 141 142 Map<String, List<Cell>> prefixMap = new HashMap<>(); 143 144 prefixMap.put("p", new ArrayList<>()); 145 prefixMap.put("s", new ArrayList<>()); 146 147 String valueString = "ValueString"; 148 149 for (String row: rows) { 150 Put p = new Put(Bytes.toBytes(row)); 151 p.setDurability(Durability.SKIP_WAL); 152 for (String column: columns) { 153 for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) { 154 KeyValue kv = KeyValueTestUtil.create(row, family, column, timestamp, 155 valueString); 156 p.add(kv); 157 kvList.add(kv); 158 for (String s: prefixMap.keySet()) { 159 if (column.startsWith(s)) { 160 prefixMap.get(s).add(kv); 161 } 162 } 163 } 164 } 165 region.put(p); 166 } 167 168 ColumnPrefixFilter filter; 169 Scan scan = new Scan(); 170 scan.setMaxVersions(); 171 for (String s: prefixMap.keySet()) { 172 filter = new ColumnPrefixFilter(Bytes.toBytes(s)); 173 174 //this is how this test differs from the one above 175 FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL); 176 filterList.addFilter(filter); 177 scan.setFilter(filterList); 178 179 InternalScanner scanner = region.getScanner(scan); 180 List<Cell> results = new ArrayList<>(); 181 while (scanner.next(results)) 182 ; 183 assertEquals(prefixMap.get(s).size(), results.size()); 184 } 185 } finally { 186 HBaseTestingUtility.closeRegionAndWAL(region); 187 } 188 189 HBaseTestingUtility.closeRegionAndWAL(region); 190 } 191 192 List<String> generateRandomWords(int numberOfWords, String suffix) { 193 Set<String> wordSet = new HashSet<>(); 194 for (int i = 0; i < numberOfWords; i++) { 195 int lengthOfWords = (int) (Math.random()*2) + 1; 196 char[] wordChar = new char[lengthOfWords]; 197 for (int j = 0; j < wordChar.length; j++) { 198 wordChar[j] = (char) (Math.random() * 26 + 97); 199 } 200 String word; 201 if (suffix == null) { 202 word = new String(wordChar); 203 } else { 204 word = new String(wordChar) + suffix; 205 } 206 wordSet.add(word); 207 } 208 List<String> wordList = new ArrayList<>(wordSet); 209 return wordList; 210 } 211 212} 213