001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.filter; 019 020import static org.junit.Assert.assertEquals; 021 022import java.io.IOException; 023import java.util.ArrayList; 024import java.util.HashMap; 025import java.util.HashSet; 026import java.util.List; 027import java.util.Map; 028import java.util.Set; 029import org.apache.hadoop.hbase.Cell; 030import org.apache.hadoop.hbase.HBaseClassTestRule; 031import org.apache.hadoop.hbase.HBaseTestingUtility; 032import org.apache.hadoop.hbase.HColumnDescriptor; 033import org.apache.hadoop.hbase.HRegionInfo; 034import org.apache.hadoop.hbase.HTableDescriptor; 035import org.apache.hadoop.hbase.KeyValue; 036import org.apache.hadoop.hbase.KeyValueTestUtil; 037import org.apache.hadoop.hbase.TableName; 038import org.apache.hadoop.hbase.client.Durability; 039import org.apache.hadoop.hbase.client.Put; 040import org.apache.hadoop.hbase.client.Scan; 041import org.apache.hadoop.hbase.regionserver.HRegion; 042import org.apache.hadoop.hbase.regionserver.InternalScanner; 043import org.apache.hadoop.hbase.testclassification.FilterTests; 044import org.apache.hadoop.hbase.testclassification.SmallTests; 045import org.apache.hadoop.hbase.util.Bytes; 046import org.junit.ClassRule; 047import org.junit.Rule; 048import org.junit.Test; 049import org.junit.experimental.categories.Category; 050import org.junit.rules.TestName; 051 052@Category({ FilterTests.class, SmallTests.class }) 053public class TestColumnPrefixFilter { 054 055 @ClassRule 056 public static final HBaseClassTestRule CLASS_RULE = 057 HBaseClassTestRule.forClass(TestColumnPrefixFilter.class); 058 059 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); 060 061 @Rule 062 public TestName name = new TestName(); 063 064 @Test 065 public void testColumnPrefixFilter() throws IOException { 066 String family = "Family"; 067 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName())); 068 htd.addFamily((new HColumnDescriptor(family)).setMaxVersions(3)); 069 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false); 070 HRegion region = HBaseTestingUtility.createRegionAndWAL(info, TEST_UTIL.getDataTestDir(), 071 TEST_UTIL.getConfiguration(), htd); 072 try { 073 List<String> rows = generateRandomWords(100, "row"); 074 List<String> columns = generateRandomWords(10000, "column"); 075 long maxTimestamp = 2; 076 077 List<Cell> kvList = new ArrayList<>(); 078 079 Map<String, List<Cell>> prefixMap = new HashMap<>(); 080 081 prefixMap.put("p", new ArrayList<>()); 082 prefixMap.put("s", new ArrayList<>()); 083 084 String valueString = "ValueString"; 085 086 for (String row : rows) { 087 Put p = new Put(Bytes.toBytes(row)); 088 p.setDurability(Durability.SKIP_WAL); 089 for (String column : columns) { 090 for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) { 091 KeyValue kv = KeyValueTestUtil.create(row, family, column, timestamp, valueString); 092 p.add(kv); 093 kvList.add(kv); 094 for (String s : prefixMap.keySet()) { 095 if (column.startsWith(s)) { 096 prefixMap.get(s).add(kv); 097 } 098 } 099 } 100 } 101 region.put(p); 102 } 103 104 ColumnPrefixFilter filter; 105 Scan scan = new Scan(); 106 scan.setMaxVersions(); 107 for (String s : prefixMap.keySet()) { 108 filter = new ColumnPrefixFilter(Bytes.toBytes(s)); 109 110 scan.setFilter(filter); 111 112 InternalScanner scanner = region.getScanner(scan); 113 List<Cell> results = new ArrayList<>(); 114 while (scanner.next(results)) 115 ; 116 assertEquals(prefixMap.get(s).size(), results.size()); 117 } 118 } finally { 119 HBaseTestingUtility.closeRegionAndWAL(region); 120 } 121 122 HBaseTestingUtility.closeRegionAndWAL(region); 123 } 124 125 @Test 126 public void testColumnPrefixFilterWithFilterList() throws IOException { 127 String family = "Family"; 128 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName())); 129 htd.addFamily((new HColumnDescriptor(family)).setMaxVersions(3)); 130 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false); 131 HRegion region = HBaseTestingUtility.createRegionAndWAL(info, TEST_UTIL.getDataTestDir(), 132 TEST_UTIL.getConfiguration(), htd); 133 try { 134 List<String> rows = generateRandomWords(100, "row"); 135 List<String> columns = generateRandomWords(10000, "column"); 136 long maxTimestamp = 2; 137 138 List<Cell> kvList = new ArrayList<>(); 139 140 Map<String, List<Cell>> prefixMap = new HashMap<>(); 141 142 prefixMap.put("p", new ArrayList<>()); 143 prefixMap.put("s", new ArrayList<>()); 144 145 String valueString = "ValueString"; 146 147 for (String row : rows) { 148 Put p = new Put(Bytes.toBytes(row)); 149 p.setDurability(Durability.SKIP_WAL); 150 for (String column : columns) { 151 for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) { 152 KeyValue kv = KeyValueTestUtil.create(row, family, column, timestamp, valueString); 153 p.add(kv); 154 kvList.add(kv); 155 for (String s : prefixMap.keySet()) { 156 if (column.startsWith(s)) { 157 prefixMap.get(s).add(kv); 158 } 159 } 160 } 161 } 162 region.put(p); 163 } 164 165 ColumnPrefixFilter filter; 166 Scan scan = new Scan(); 167 scan.setMaxVersions(); 168 for (String s : prefixMap.keySet()) { 169 filter = new ColumnPrefixFilter(Bytes.toBytes(s)); 170 171 // this is how this test differs from the one above 172 FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL); 173 filterList.addFilter(filter); 174 scan.setFilter(filterList); 175 176 InternalScanner scanner = region.getScanner(scan); 177 List<Cell> results = new ArrayList<>(); 178 while (scanner.next(results)) 179 ; 180 assertEquals(prefixMap.get(s).size(), results.size()); 181 } 182 } finally { 183 HBaseTestingUtility.closeRegionAndWAL(region); 184 } 185 186 HBaseTestingUtility.closeRegionAndWAL(region); 187 } 188 189 List<String> generateRandomWords(int numberOfWords, String suffix) { 190 Set<String> wordSet = new HashSet<>(); 191 for (int i = 0; i < numberOfWords; i++) { 192 int lengthOfWords = (int) (Math.random() * 2) + 1; 193 char[] wordChar = new char[lengthOfWords]; 194 for (int j = 0; j < wordChar.length; j++) { 195 wordChar[j] = (char) (Math.random() * 26 + 97); 196 } 197 String word; 198 if (suffix == null) { 199 word = new String(wordChar); 200 } else { 201 word = new String(wordChar) + suffix; 202 } 203 wordSet.add(word); 204 } 205 List<String> wordList = new ArrayList<>(wordSet); 206 return wordList; 207 } 208 209}