001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.filter; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertNotNull; 022import static org.junit.Assert.assertTrue; 023 024import java.io.IOException; 025import java.util.ArrayList; 026import java.util.Arrays; 027import java.util.List; 028import org.apache.hadoop.hbase.Cell; 029import org.apache.hadoop.hbase.CellComparatorImpl; 030import org.apache.hadoop.hbase.CompareOperator; 031import org.apache.hadoop.hbase.HBaseClassTestRule; 032import org.apache.hadoop.hbase.HBaseTestingUtility; 033import org.apache.hadoop.hbase.HColumnDescriptor; 034import org.apache.hadoop.hbase.HRegionInfo; 035import org.apache.hadoop.hbase.HTableDescriptor; 036import org.apache.hadoop.hbase.KeyValue; 037import org.apache.hadoop.hbase.TableName; 038import org.apache.hadoop.hbase.client.Put; 039import org.apache.hadoop.hbase.client.Scan; 040import org.apache.hadoop.hbase.filter.Filter.ReturnCode; 041import org.apache.hadoop.hbase.regionserver.HRegion; 042import org.apache.hadoop.hbase.regionserver.InternalScanner; 043import org.apache.hadoop.hbase.testclassification.FilterTests; 044import org.apache.hadoop.hbase.testclassification.SmallTests; 045import org.apache.hadoop.hbase.util.Bytes; 046import org.junit.After; 047import org.junit.Before; 048import org.junit.ClassRule; 049import org.junit.Test; 050import org.junit.experimental.categories.Category; 051import org.slf4j.Logger; 052import org.slf4j.LoggerFactory; 053 054@Category({FilterTests.class, SmallTests.class}) 055public class TestDependentColumnFilter { 056 057 @ClassRule 058 public static final HBaseClassTestRule CLASS_RULE = 059 HBaseClassTestRule.forClass(TestDependentColumnFilter.class); 060 061 private static final Logger LOG = LoggerFactory.getLogger(TestDependentColumnFilter.class); 062 private static final byte[][] ROWS = { 063 Bytes.toBytes("test1"),Bytes.toBytes("test2") 064 }; 065 private static final byte[][] FAMILIES = { 066 Bytes.toBytes("familyOne"),Bytes.toBytes("familyTwo") 067 }; 068 private static final long STAMP_BASE = System.currentTimeMillis(); 069 private static final long[] STAMPS = { 070 STAMP_BASE-100, STAMP_BASE-200, STAMP_BASE-300 071 }; 072 private static final byte[] QUALIFIER = Bytes.toBytes("qualifier"); 073 private static final byte[][] BAD_VALS = { 074 Bytes.toBytes("bad1"), Bytes.toBytes("bad2"), Bytes.toBytes("bad3") 075 }; 076 private static final byte[] MATCH_VAL = Bytes.toBytes("match"); 077 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); 078 079 List<KeyValue> testVals; 080 private HRegion region; 081 082 @Before 083 public void setUp() throws Exception { 084 testVals = makeTestVals(); 085 086 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(this.getClass().getSimpleName())); 087 HColumnDescriptor hcd0 = new HColumnDescriptor(FAMILIES[0]); 088 hcd0.setMaxVersions(3); 089 htd.addFamily(hcd0); 090 HColumnDescriptor hcd1 = new HColumnDescriptor(FAMILIES[1]); 091 hcd1.setMaxVersions(3); 092 htd.addFamily(hcd1); 093 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false); 094 this.region = HBaseTestingUtility.createRegionAndWAL(info, TEST_UTIL.getDataTestDir(), 095 TEST_UTIL.getConfiguration(), htd); 096 addData(); 097 } 098 099 @After 100 public void tearDown() throws Exception { 101 HBaseTestingUtility.closeRegionAndWAL(this.region); 102 } 103 104 private void addData() throws IOException { 105 Put put = new Put(ROWS[0]); 106 // add in an entry for each stamp, with 2 as a "good" value 107 put.addColumn(FAMILIES[0], QUALIFIER, STAMPS[0], BAD_VALS[0]); 108 put.addColumn(FAMILIES[0], QUALIFIER, STAMPS[1], BAD_VALS[1]); 109 put.addColumn(FAMILIES[0], QUALIFIER, STAMPS[2], MATCH_VAL); 110 // add in entries for stamps 0 and 2. 111 // without a value check both will be "accepted" 112 // with one 2 will be accepted(since the corresponding ts entry 113 // has a matching value 114 put.addColumn(FAMILIES[1], QUALIFIER, STAMPS[0], BAD_VALS[0]); 115 put.addColumn(FAMILIES[1], QUALIFIER, STAMPS[2], BAD_VALS[2]); 116 117 this.region.put(put); 118 119 put = new Put(ROWS[1]); 120 put.addColumn(FAMILIES[0], QUALIFIER, STAMPS[0], BAD_VALS[0]); 121 // there is no corresponding timestamp for this so it should never pass 122 put.addColumn(FAMILIES[0], QUALIFIER, STAMPS[2], MATCH_VAL); 123 // if we reverse the qualifiers this one should pass 124 put.addColumn(FAMILIES[1], QUALIFIER, STAMPS[0], MATCH_VAL); 125 // should pass 126 put.addColumn(FAMILIES[1], QUALIFIER, STAMPS[1], BAD_VALS[2]); 127 128 this.region.put(put); 129 } 130 131 private List<KeyValue> makeTestVals() { 132 List<KeyValue> testVals = new ArrayList<>(); 133 testVals.add(new KeyValue(ROWS[0], FAMILIES[0], QUALIFIER, STAMPS[0], BAD_VALS[0])); 134 testVals.add(new KeyValue(ROWS[0], FAMILIES[0], QUALIFIER, STAMPS[1], BAD_VALS[1])); 135 testVals.add(new KeyValue(ROWS[0], FAMILIES[1], QUALIFIER, STAMPS[1], BAD_VALS[2])); 136 testVals.add(new KeyValue(ROWS[0], FAMILIES[1], QUALIFIER, STAMPS[0], MATCH_VAL)); 137 testVals.add(new KeyValue(ROWS[0], FAMILIES[1], QUALIFIER, STAMPS[2], BAD_VALS[2])); 138 139 return testVals; 140 } 141 142 /** 143 * This shouldn't be confused with TestFilter#verifyScan 144 * as expectedKeys is not the per row total, but the scan total 145 * 146 * @param s 147 * @param expectedRows 148 * @param expectedCells 149 * @throws IOException 150 */ 151 private void verifyScan(Scan s, long expectedRows, long expectedCells) 152 throws IOException { 153 InternalScanner scanner = this.region.getScanner(s); 154 List<Cell> results = new ArrayList<>(); 155 int i = 0; 156 int cells = 0; 157 for (boolean done = true; done; i++) { 158 done = scanner.next(results); 159 Arrays.sort(results.toArray(new Cell[results.size()]), 160 CellComparatorImpl.COMPARATOR); 161 LOG.info("counter=" + i + ", " + results); 162 if (results.isEmpty()) break; 163 cells += results.size(); 164 assertTrue("Scanned too many rows! Only expected " + expectedRows + 165 " total but already scanned " + (i+1), expectedRows > i); 166 assertTrue("Expected " + expectedCells + " cells total but " + 167 "already scanned " + cells, expectedCells >= cells); 168 results.clear(); 169 } 170 assertEquals("Expected " + expectedRows + " rows but scanned " + i + 171 " rows", expectedRows, i); 172 assertEquals("Expected " + expectedCells + " cells but scanned " + cells + 173 " cells", expectedCells, cells); 174 } 175 176 /** 177 * Test scans using a DependentColumnFilter 178 */ 179 @Test 180 public void testScans() throws Exception { 181 Filter filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER); 182 183 Scan scan = new Scan(); 184 scan.setFilter(filter); 185 scan.setMaxVersions(Integer.MAX_VALUE); 186 187 verifyScan(scan, 2, 8); 188 189 // drop the filtering cells 190 filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER, true); 191 scan = new Scan(); 192 scan.setFilter(filter); 193 scan.setMaxVersions(Integer.MAX_VALUE); 194 195 verifyScan(scan, 2, 3); 196 197 // include a comparator operation 198 filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER, false, 199 CompareOperator.EQUAL, new BinaryComparator(MATCH_VAL)); 200 scan = new Scan(); 201 scan.setFilter(filter); 202 scan.setMaxVersions(Integer.MAX_VALUE); 203 204 /* 205 * expecting to get the following 3 cells 206 * row 0 207 * put.add(FAMILIES[0], QUALIFIER, STAMPS[2], MATCH_VAL); 208 * put.add(FAMILIES[1], QUALIFIER, STAMPS[2], BAD_VALS[2]); 209 * row 1 210 * put.add(FAMILIES[0], QUALIFIER, STAMPS[2], MATCH_VAL); 211 */ 212 verifyScan(scan, 2, 3); 213 214 // include a comparator operation and drop comparator 215 filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER, true, 216 CompareOperator.EQUAL, new BinaryComparator(MATCH_VAL)); 217 scan = new Scan(); 218 scan.setFilter(filter); 219 scan.setMaxVersions(Integer.MAX_VALUE); 220 221 /* 222 * expecting to get the following 1 cell 223 * row 0 224 * put.add(FAMILIES[1], QUALIFIER, STAMPS[2], BAD_VALS[2]); 225 */ 226 verifyScan(scan, 1, 1); 227 228 } 229 230 /** 231 * Test that the filter correctly drops rows without a corresponding timestamp 232 * 233 * @throws Exception 234 */ 235 @Test 236 public void testFilterDropping() throws Exception { 237 Filter filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER); 238 List<Cell> accepted = new ArrayList<>(); 239 for(Cell val : testVals) { 240 if(filter.filterCell(val) == ReturnCode.INCLUDE) { 241 accepted.add(val); 242 } 243 } 244 assertEquals("check all values accepted from filterCell", 5, accepted.size()); 245 246 filter.filterRowCells(accepted); 247 assertEquals("check filterRow(List<KeyValue>) dropped cell without corresponding column entry", 4, accepted.size()); 248 249 // start do it again with dependent column dropping on 250 filter = new DependentColumnFilter(FAMILIES[1], QUALIFIER, true); 251 accepted.clear(); 252 for(KeyValue val : testVals) { 253 if(filter.filterCell(val) == ReturnCode.INCLUDE) { 254 accepted.add(val); 255 } 256 } 257 assertEquals("check the filtering column cells got dropped", 2, accepted.size()); 258 259 filter.filterRowCells(accepted); 260 assertEquals("check cell retention", 2, accepted.size()); 261 } 262 263 /** 264 * Test for HBASE-8794. Avoid NullPointerException in DependentColumnFilter.toString(). 265 */ 266 @Test 267 public void testToStringWithNullComparator() { 268 // Test constructor that implicitly sets a null comparator 269 Filter filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER); 270 assertNotNull(filter.toString()); 271 assertTrue("check string contains 'null' as compatator is null", 272 filter.toString().contains("null")); 273 274 // Test constructor with explicit null comparator 275 filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER, true, CompareOperator.EQUAL, null); 276 assertNotNull(filter.toString()); 277 assertTrue("check string contains 'null' as compatator is null", 278 filter.toString().contains("null")); 279 } 280 281 @Test 282 public void testToStringWithNonNullComparator() { 283 Filter filter = 284 new DependentColumnFilter(FAMILIES[0], QUALIFIER, true, CompareOperator.EQUAL, 285 new BinaryComparator(MATCH_VAL)); 286 assertNotNull(filter.toString()); 287 assertTrue("check string contains comparator value", filter.toString().contains("match")); 288 } 289 290} 291