001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.filter; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertNotNull; 022import static org.junit.Assert.assertTrue; 023 024import java.io.IOException; 025import java.util.ArrayList; 026import java.util.Arrays; 027import java.util.List; 028import org.apache.hadoop.hbase.Cell; 029import org.apache.hadoop.hbase.CellComparatorImpl; 030import org.apache.hadoop.hbase.CompareOperator; 031import org.apache.hadoop.hbase.HBaseClassTestRule; 032import org.apache.hadoop.hbase.HBaseTestingUtil; 033import org.apache.hadoop.hbase.KeyValue; 034import org.apache.hadoop.hbase.TableName; 035import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; 036import org.apache.hadoop.hbase.client.Put; 037import org.apache.hadoop.hbase.client.RegionInfo; 038import org.apache.hadoop.hbase.client.RegionInfoBuilder; 039import org.apache.hadoop.hbase.client.Scan; 040import org.apache.hadoop.hbase.client.TableDescriptor; 041import org.apache.hadoop.hbase.client.TableDescriptorBuilder; 042import org.apache.hadoop.hbase.filter.Filter.ReturnCode; 043import org.apache.hadoop.hbase.regionserver.HRegion; 044import org.apache.hadoop.hbase.regionserver.InternalScanner; 045import org.apache.hadoop.hbase.testclassification.FilterTests; 046import org.apache.hadoop.hbase.testclassification.SmallTests; 047import org.apache.hadoop.hbase.util.Bytes; 048import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; 049import org.junit.After; 050import org.junit.Before; 051import org.junit.ClassRule; 052import org.junit.Test; 053import org.junit.experimental.categories.Category; 054import org.slf4j.Logger; 055import org.slf4j.LoggerFactory; 056 057@Category({ FilterTests.class, SmallTests.class }) 058public class TestDependentColumnFilter { 059 060 @ClassRule 061 public static final HBaseClassTestRule CLASS_RULE = 062 HBaseClassTestRule.forClass(TestDependentColumnFilter.class); 063 064 private static final Logger LOG = LoggerFactory.getLogger(TestDependentColumnFilter.class); 065 private static final byte[][] ROWS = { Bytes.toBytes("test1"), Bytes.toBytes("test2") }; 066 private static final byte[][] FAMILIES = 067 { Bytes.toBytes("familyOne"), Bytes.toBytes("familyTwo") }; 068 private static final long STAMP_BASE = EnvironmentEdgeManager.currentTime(); 069 private static final long[] STAMPS = { STAMP_BASE - 100, STAMP_BASE - 200, STAMP_BASE - 300 }; 070 private static final byte[] QUALIFIER = Bytes.toBytes("qualifier"); 071 private static final byte[][] BAD_VALS = 072 { Bytes.toBytes("bad1"), Bytes.toBytes("bad2"), Bytes.toBytes("bad3") }; 073 private static final byte[] MATCH_VAL = Bytes.toBytes("match"); 074 private final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil(); 075 076 List<KeyValue> testVals; 077 private HRegion region; 078 079 @Before 080 public void setUp() throws Exception { 081 testVals = makeTestVals(); 082 083 TableDescriptor tableDescriptor = 084 TableDescriptorBuilder.newBuilder(TableName.valueOf(this.getClass().getSimpleName())) 085 .setColumnFamily( 086 ColumnFamilyDescriptorBuilder.newBuilder(FAMILIES[0]).setMaxVersions(3).build()) 087 .setColumnFamily( 088 ColumnFamilyDescriptorBuilder.newBuilder(FAMILIES[1]).setMaxVersions(3).build()) 089 .build(); 090 RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build(); 091 this.region = HBaseTestingUtil.createRegionAndWAL(info, TEST_UTIL.getDataTestDir(), 092 TEST_UTIL.getConfiguration(), tableDescriptor); 093 addData(); 094 } 095 096 @After 097 public void tearDown() throws Exception { 098 HBaseTestingUtil.closeRegionAndWAL(this.region); 099 } 100 101 private void addData() throws IOException { 102 Put put = new Put(ROWS[0]); 103 // add in an entry for each stamp, with 2 as a "good" value 104 put.addColumn(FAMILIES[0], QUALIFIER, STAMPS[0], BAD_VALS[0]); 105 put.addColumn(FAMILIES[0], QUALIFIER, STAMPS[1], BAD_VALS[1]); 106 put.addColumn(FAMILIES[0], QUALIFIER, STAMPS[2], MATCH_VAL); 107 // add in entries for stamps 0 and 2. 108 // without a value check both will be "accepted" 109 // with one 2 will be accepted(since the corresponding ts entry 110 // has a matching value 111 put.addColumn(FAMILIES[1], QUALIFIER, STAMPS[0], BAD_VALS[0]); 112 put.addColumn(FAMILIES[1], QUALIFIER, STAMPS[2], BAD_VALS[2]); 113 114 this.region.put(put); 115 116 put = new Put(ROWS[1]); 117 put.addColumn(FAMILIES[0], QUALIFIER, STAMPS[0], BAD_VALS[0]); 118 // there is no corresponding timestamp for this so it should never pass 119 put.addColumn(FAMILIES[0], QUALIFIER, STAMPS[2], MATCH_VAL); 120 // if we reverse the qualifiers this one should pass 121 put.addColumn(FAMILIES[1], QUALIFIER, STAMPS[0], MATCH_VAL); 122 // should pass 123 put.addColumn(FAMILIES[1], QUALIFIER, STAMPS[1], BAD_VALS[2]); 124 125 this.region.put(put); 126 } 127 128 private List<KeyValue> makeTestVals() { 129 List<KeyValue> testVals = new ArrayList<>(); 130 testVals.add(new KeyValue(ROWS[0], FAMILIES[0], QUALIFIER, STAMPS[0], BAD_VALS[0])); 131 testVals.add(new KeyValue(ROWS[0], FAMILIES[0], QUALIFIER, STAMPS[1], BAD_VALS[1])); 132 testVals.add(new KeyValue(ROWS[0], FAMILIES[1], QUALIFIER, STAMPS[1], BAD_VALS[2])); 133 testVals.add(new KeyValue(ROWS[0], FAMILIES[1], QUALIFIER, STAMPS[0], MATCH_VAL)); 134 testVals.add(new KeyValue(ROWS[0], FAMILIES[1], QUALIFIER, STAMPS[2], BAD_VALS[2])); 135 136 return testVals; 137 } 138 139 /** 140 * This shouldn't be confused with TestFilter#verifyScan as expectedKeys is not the per row total, 141 * but the scan total 142 */ 143 private void verifyScan(Scan s, long expectedRows, long expectedCells) throws IOException { 144 InternalScanner scanner = this.region.getScanner(s); 145 List<Cell> results = new ArrayList<>(); 146 int i = 0; 147 int cells = 0; 148 for (boolean done = true; done; i++) { 149 done = scanner.next(results); 150 Arrays.sort(results.toArray(new Cell[results.size()]), CellComparatorImpl.COMPARATOR); 151 LOG.info("counter=" + i + ", " + results); 152 if (results.isEmpty()) break; 153 cells += results.size(); 154 assertTrue("Scanned too many rows! Only expected " + expectedRows 155 + " total but already scanned " + (i + 1), expectedRows > i); 156 assertTrue("Expected " + expectedCells + " cells total but " + "already scanned " + cells, 157 expectedCells >= cells); 158 results.clear(); 159 } 160 assertEquals("Expected " + expectedRows + " rows but scanned " + i + " rows", expectedRows, i); 161 assertEquals("Expected " + expectedCells + " cells but scanned " + cells + " cells", 162 expectedCells, cells); 163 } 164 165 /** 166 * Test scans using a DependentColumnFilter 167 */ 168 @Test 169 public void testScans() throws Exception { 170 Filter filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER); 171 172 Scan scan = new Scan(); 173 scan.setFilter(filter); 174 scan.readVersions(Integer.MAX_VALUE); 175 176 verifyScan(scan, 2, 8); 177 178 // drop the filtering cells 179 filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER, true); 180 scan = new Scan(); 181 scan.setFilter(filter); 182 scan.readVersions(Integer.MAX_VALUE); 183 184 verifyScan(scan, 2, 3); 185 186 // include a comparator operation 187 filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER, false, CompareOperator.EQUAL, 188 new BinaryComparator(MATCH_VAL)); 189 scan = new Scan(); 190 scan.setFilter(filter); 191 scan.readVersions(Integer.MAX_VALUE); 192 193 /* 194 * expecting to get the following 3 cells row 0 put.add(FAMILIES[0], QUALIFIER, STAMPS[2], 195 * MATCH_VAL); put.add(FAMILIES[1], QUALIFIER, STAMPS[2], BAD_VALS[2]); row 1 196 * put.add(FAMILIES[0], QUALIFIER, STAMPS[2], MATCH_VAL); 197 */ 198 verifyScan(scan, 2, 3); 199 200 // include a comparator operation and drop comparator 201 filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER, true, CompareOperator.EQUAL, 202 new BinaryComparator(MATCH_VAL)); 203 scan = new Scan(); 204 scan.setFilter(filter); 205 scan.readVersions(Integer.MAX_VALUE); 206 207 /* 208 * expecting to get the following 1 cell row 0 put.add(FAMILIES[1], QUALIFIER, STAMPS[2], 209 * BAD_VALS[2]); 210 */ 211 verifyScan(scan, 1, 1); 212 213 } 214 215 /** 216 * Test that the filter correctly drops rows without a corresponding timestamp 217 */ 218 @Test 219 public void testFilterDropping() throws Exception { 220 Filter filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER); 221 List<Cell> accepted = new ArrayList<>(); 222 for (Cell val : testVals) { 223 if (filter.filterCell(val) == ReturnCode.INCLUDE) { 224 accepted.add(val); 225 } 226 } 227 assertEquals("check all values accepted from filterCell", 5, accepted.size()); 228 229 filter.filterRowCells(accepted); 230 assertEquals("check filterRow(List<KeyValue>) dropped cell without corresponding column entry", 231 4, accepted.size()); 232 233 // start do it again with dependent column dropping on 234 filter = new DependentColumnFilter(FAMILIES[1], QUALIFIER, true); 235 accepted.clear(); 236 for (KeyValue val : testVals) { 237 if (filter.filterCell(val) == ReturnCode.INCLUDE) { 238 accepted.add(val); 239 } 240 } 241 assertEquals("check the filtering column cells got dropped", 2, accepted.size()); 242 243 filter.filterRowCells(accepted); 244 assertEquals("check cell retention", 2, accepted.size()); 245 } 246 247 /** 248 * Test for HBASE-8794. Avoid NullPointerException in DependentColumnFilter.toString(). 249 */ 250 @Test 251 public void testToStringWithNullComparator() { 252 // Test constructor that implicitly sets a null comparator 253 Filter filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER); 254 assertNotNull(filter.toString()); 255 assertTrue("check string contains 'null' as compatator is null", 256 filter.toString().contains("null")); 257 258 // Test constructor with explicit null comparator 259 filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER, true, CompareOperator.EQUAL, null); 260 assertNotNull(filter.toString()); 261 assertTrue("check string contains 'null' as compatator is null", 262 filter.toString().contains("null")); 263 } 264 265 @Test 266 public void testToStringWithNonNullComparator() { 267 Filter filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER, true, CompareOperator.EQUAL, 268 new BinaryComparator(MATCH_VAL)); 269 assertNotNull(filter.toString()); 270 assertTrue("check string contains comparator value", filter.toString().contains("match")); 271 } 272 273}