001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.filter; 019 020import static org.junit.Assert.assertEquals; 021 022import java.io.IOException; 023import java.util.ArrayList; 024import java.util.HashMap; 025import java.util.HashSet; 026import java.util.List; 027import java.util.Map; 028import java.util.Set; 029import org.apache.hadoop.hbase.Cell; 030import org.apache.hadoop.hbase.HBaseClassTestRule; 031import org.apache.hadoop.hbase.HBaseTestingUtility; 032import org.apache.hadoop.hbase.HColumnDescriptor; 033import org.apache.hadoop.hbase.HRegionInfo; 034import org.apache.hadoop.hbase.HTableDescriptor; 035import org.apache.hadoop.hbase.KeyValue; 036import org.apache.hadoop.hbase.KeyValueTestUtil; 037import org.apache.hadoop.hbase.TableName; 038import org.apache.hadoop.hbase.client.Durability; 039import org.apache.hadoop.hbase.client.Put; 040import org.apache.hadoop.hbase.client.Scan; 041import org.apache.hadoop.hbase.regionserver.HRegion; 042import org.apache.hadoop.hbase.regionserver.InternalScanner; 043import org.apache.hadoop.hbase.testclassification.FilterTests; 044import org.apache.hadoop.hbase.testclassification.SmallTests; 045import org.apache.hadoop.hbase.util.Bytes; 046import org.junit.ClassRule; 047import org.junit.Rule; 048import org.junit.Test; 049import org.junit.experimental.categories.Category; 050import org.junit.rules.TestName; 051 052@Category({FilterTests.class, SmallTests.class}) 053public class TestMultipleColumnPrefixFilter { 054 055 @ClassRule 056 public static final HBaseClassTestRule CLASS_RULE = 057 HBaseClassTestRule.forClass(TestMultipleColumnPrefixFilter.class); 058 059 private final static HBaseTestingUtility TEST_UTIL = new 060 HBaseTestingUtility(); 061 062 @Rule 063 public TestName name = new TestName(); 064 065 @Test 066 public void testMultipleColumnPrefixFilter() throws IOException { 067 String family = "Family"; 068 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName())); 069 HColumnDescriptor hcd = new HColumnDescriptor(family); 070 hcd.setMaxVersions(3); 071 htd.addFamily(hcd); 072 // HRegionInfo info = new HRegionInfo(htd, null, null, false); 073 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false); 074 HRegion region = HBaseTestingUtility.createRegionAndWAL(info, TEST_UTIL. 075 getDataTestDir(), TEST_UTIL.getConfiguration(), htd); 076 077 List<String> rows = generateRandomWords(100, "row"); 078 List<String> columns = generateRandomWords(10000, "column"); 079 long maxTimestamp = 2; 080 081 List<Cell> kvList = new ArrayList<>(); 082 083 Map<String, List<Cell>> prefixMap = new HashMap<>(); 084 085 prefixMap.put("p", new ArrayList<>()); 086 prefixMap.put("q", new ArrayList<>()); 087 prefixMap.put("s", new ArrayList<>()); 088 089 String valueString = "ValueString"; 090 091 for (String row: rows) { 092 Put p = new Put(Bytes.toBytes(row)); 093 p.setDurability(Durability.SKIP_WAL); 094 for (String column: columns) { 095 for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) { 096 KeyValue kv = KeyValueTestUtil.create(row, family, column, timestamp, 097 valueString); 098 p.add(kv); 099 kvList.add(kv); 100 for (String s: prefixMap.keySet()) { 101 if (column.startsWith(s)) { 102 prefixMap.get(s).add(kv); 103 } 104 } 105 } 106 } 107 region.put(p); 108 } 109 110 MultipleColumnPrefixFilter filter; 111 Scan scan = new Scan(); 112 scan.setMaxVersions(); 113 byte [][] filter_prefix = new byte [2][]; 114 filter_prefix[0] = new byte [] {'p'}; 115 filter_prefix[1] = new byte [] {'q'}; 116 117 filter = new MultipleColumnPrefixFilter(filter_prefix); 118 scan.setFilter(filter); 119 List<Cell> results = new ArrayList<>(); 120 InternalScanner scanner = region.getScanner(scan); 121 while (scanner.next(results)) 122 ; 123 assertEquals(prefixMap.get("p").size() + prefixMap.get("q").size(), results.size()); 124 125 HBaseTestingUtility.closeRegionAndWAL(region); 126 } 127 128 @Test 129 public void testMultipleColumnPrefixFilterWithManyFamilies() throws IOException { 130 String family1 = "Family1"; 131 String family2 = "Family2"; 132 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName())); 133 HColumnDescriptor hcd1 = new HColumnDescriptor(family1); 134 hcd1.setMaxVersions(3); 135 htd.addFamily(hcd1); 136 HColumnDescriptor hcd2 = new HColumnDescriptor(family2); 137 hcd2.setMaxVersions(3); 138 htd.addFamily(hcd2); 139 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false); 140 HRegion region = HBaseTestingUtility.createRegionAndWAL(info, TEST_UTIL. 141 getDataTestDir(), TEST_UTIL.getConfiguration(), htd); 142 143 List<String> rows = generateRandomWords(100, "row"); 144 List<String> columns = generateRandomWords(10000, "column"); 145 long maxTimestamp = 3; 146 147 List<Cell> kvList = new ArrayList<>(); 148 149 Map<String, List<Cell>> prefixMap = new HashMap<>(); 150 151 prefixMap.put("p", new ArrayList<>()); 152 prefixMap.put("q", new ArrayList<>()); 153 prefixMap.put("s", new ArrayList<>()); 154 155 String valueString = "ValueString"; 156 157 for (String row: rows) { 158 Put p = new Put(Bytes.toBytes(row)); 159 p.setDurability(Durability.SKIP_WAL); 160 for (String column: columns) { 161 for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) { 162 double rand = Math.random(); 163 Cell kv; 164 if (rand < 0.5) { 165 kv = KeyValueTestUtil.create(row, family1, column, timestamp, valueString); 166 } else { 167 kv = KeyValueTestUtil.create(row, family2, column, timestamp, valueString); 168 } 169 p.add(kv); 170 kvList.add(kv); 171 for (String s: prefixMap.keySet()) { 172 if (column.startsWith(s)) { 173 prefixMap.get(s).add(kv); 174 } 175 } 176 } 177 } 178 region.put(p); 179 } 180 181 MultipleColumnPrefixFilter filter; 182 Scan scan = new Scan(); 183 scan.setMaxVersions(); 184 byte [][] filter_prefix = new byte [2][]; 185 filter_prefix[0] = new byte [] {'p'}; 186 filter_prefix[1] = new byte [] {'q'}; 187 188 filter = new MultipleColumnPrefixFilter(filter_prefix); 189 scan.setFilter(filter); 190 List<Cell> results = new ArrayList<>(); 191 InternalScanner scanner = region.getScanner(scan); 192 while (scanner.next(results)) 193 ; 194 assertEquals(prefixMap.get("p").size() + prefixMap.get("q").size(), results.size()); 195 196 HBaseTestingUtility.closeRegionAndWAL(region); 197 } 198 199 @Test 200 public void testMultipleColumnPrefixFilterWithColumnPrefixFilter() throws IOException { 201 String family = "Family"; 202 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName())); 203 htd.addFamily(new HColumnDescriptor(family)); 204 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false); 205 HRegion region = HBaseTestingUtility.createRegionAndWAL(info, TEST_UTIL. 206 getDataTestDir(), TEST_UTIL.getConfiguration(), htd); 207 208 List<String> rows = generateRandomWords(100, "row"); 209 List<String> columns = generateRandomWords(10000, "column"); 210 long maxTimestamp = 2; 211 212 String valueString = "ValueString"; 213 214 for (String row: rows) { 215 Put p = new Put(Bytes.toBytes(row)); 216 p.setDurability(Durability.SKIP_WAL); 217 for (String column: columns) { 218 for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) { 219 KeyValue kv = KeyValueTestUtil.create(row, family, column, timestamp, 220 valueString); 221 p.add(kv); 222 } 223 } 224 region.put(p); 225 } 226 227 MultipleColumnPrefixFilter multiplePrefixFilter; 228 Scan scan1 = new Scan(); 229 scan1.setMaxVersions(); 230 byte [][] filter_prefix = new byte [1][]; 231 filter_prefix[0] = new byte [] {'p'}; 232 233 multiplePrefixFilter = new MultipleColumnPrefixFilter(filter_prefix); 234 scan1.setFilter(multiplePrefixFilter); 235 List<Cell> results1 = new ArrayList<>(); 236 InternalScanner scanner1 = region.getScanner(scan1); 237 while (scanner1.next(results1)) 238 ; 239 240 ColumnPrefixFilter singlePrefixFilter; 241 Scan scan2 = new Scan(); 242 scan2.setMaxVersions(); 243 singlePrefixFilter = new ColumnPrefixFilter(Bytes.toBytes("p")); 244 245 scan2.setFilter(singlePrefixFilter); 246 List<Cell> results2 = new ArrayList<>(); 247 InternalScanner scanner2 = region.getScanner(scan1); 248 while (scanner2.next(results2)) 249 ; 250 251 assertEquals(results1.size(), results2.size()); 252 253 HBaseTestingUtility.closeRegionAndWAL(region); 254 } 255 256 List<String> generateRandomWords(int numberOfWords, String suffix) { 257 Set<String> wordSet = new HashSet<>(); 258 for (int i = 0; i < numberOfWords; i++) { 259 int lengthOfWords = (int) (Math.random()*2) + 1; 260 char[] wordChar = new char[lengthOfWords]; 261 for (int j = 0; j < wordChar.length; j++) { 262 wordChar[j] = (char) (Math.random() * 26 + 97); 263 } 264 String word; 265 if (suffix == null) { 266 word = new String(wordChar); 267 } else { 268 word = new String(wordChar) + suffix; 269 } 270 wordSet.add(word); 271 } 272 List<String> wordList = new ArrayList<>(wordSet); 273 return wordList; 274 } 275 276} 277 278