001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.filter;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNotNull;
022import static org.junit.Assert.assertTrue;
023
024import java.io.IOException;
025import java.util.ArrayList;
026import java.util.Arrays;
027import java.util.List;
028import org.apache.hadoop.hbase.Cell;
029import org.apache.hadoop.hbase.CellComparatorImpl;
030import org.apache.hadoop.hbase.CompareOperator;
031import org.apache.hadoop.hbase.HBaseClassTestRule;
032import org.apache.hadoop.hbase.HBaseTestingUtil;
033import org.apache.hadoop.hbase.KeyValue;
034import org.apache.hadoop.hbase.TableName;
035import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
036import org.apache.hadoop.hbase.client.Put;
037import org.apache.hadoop.hbase.client.RegionInfo;
038import org.apache.hadoop.hbase.client.RegionInfoBuilder;
039import org.apache.hadoop.hbase.client.Scan;
040import org.apache.hadoop.hbase.client.TableDescriptor;
041import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
042import org.apache.hadoop.hbase.filter.Filter.ReturnCode;
043import org.apache.hadoop.hbase.regionserver.HRegion;
044import org.apache.hadoop.hbase.regionserver.InternalScanner;
045import org.apache.hadoop.hbase.testclassification.FilterTests;
046import org.apache.hadoop.hbase.testclassification.SmallTests;
047import org.apache.hadoop.hbase.util.Bytes;
048import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
049import org.junit.After;
050import org.junit.Before;
051import org.junit.ClassRule;
052import org.junit.Test;
053import org.junit.experimental.categories.Category;
054import org.slf4j.Logger;
055import org.slf4j.LoggerFactory;
056
057@Category({ FilterTests.class, SmallTests.class })
058public class TestDependentColumnFilter {
059
060  @ClassRule
061  public static final HBaseClassTestRule CLASS_RULE =
062    HBaseClassTestRule.forClass(TestDependentColumnFilter.class);
063
064  private static final Logger LOG = LoggerFactory.getLogger(TestDependentColumnFilter.class);
065  private static final byte[][] ROWS = { Bytes.toBytes("test1"), Bytes.toBytes("test2") };
066  private static final byte[][] FAMILIES =
067    { Bytes.toBytes("familyOne"), Bytes.toBytes("familyTwo") };
068  private static final long STAMP_BASE = EnvironmentEdgeManager.currentTime();
069  private static final long[] STAMPS = { STAMP_BASE - 100, STAMP_BASE - 200, STAMP_BASE - 300 };
070  private static final byte[] QUALIFIER = Bytes.toBytes("qualifier");
071  private static final byte[][] BAD_VALS =
072    { Bytes.toBytes("bad1"), Bytes.toBytes("bad2"), Bytes.toBytes("bad3") };
073  private static final byte[] MATCH_VAL = Bytes.toBytes("match");
074  private final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
075
076  List<KeyValue> testVals;
077  private HRegion region;
078
079  @Before
080  public void setUp() throws Exception {
081    testVals = makeTestVals();
082
083    TableDescriptor tableDescriptor =
084      TableDescriptorBuilder.newBuilder(TableName.valueOf(this.getClass().getSimpleName()))
085        .setColumnFamily(
086          ColumnFamilyDescriptorBuilder.newBuilder(FAMILIES[0]).setMaxVersions(3).build())
087        .setColumnFamily(
088          ColumnFamilyDescriptorBuilder.newBuilder(FAMILIES[1]).setMaxVersions(3).build())
089        .build();
090    RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
091    this.region = HBaseTestingUtil.createRegionAndWAL(info, TEST_UTIL.getDataTestDir(),
092      TEST_UTIL.getConfiguration(), tableDescriptor);
093    addData();
094  }
095
096  @After
097  public void tearDown() throws Exception {
098    HBaseTestingUtil.closeRegionAndWAL(this.region);
099  }
100
101  private void addData() throws IOException {
102    Put put = new Put(ROWS[0]);
103    // add in an entry for each stamp, with 2 as a "good" value
104    put.addColumn(FAMILIES[0], QUALIFIER, STAMPS[0], BAD_VALS[0]);
105    put.addColumn(FAMILIES[0], QUALIFIER, STAMPS[1], BAD_VALS[1]);
106    put.addColumn(FAMILIES[0], QUALIFIER, STAMPS[2], MATCH_VAL);
107    // add in entries for stamps 0 and 2.
108    // without a value check both will be "accepted"
109    // with one 2 will be accepted(since the corresponding ts entry
110    // has a matching value
111    put.addColumn(FAMILIES[1], QUALIFIER, STAMPS[0], BAD_VALS[0]);
112    put.addColumn(FAMILIES[1], QUALIFIER, STAMPS[2], BAD_VALS[2]);
113
114    this.region.put(put);
115
116    put = new Put(ROWS[1]);
117    put.addColumn(FAMILIES[0], QUALIFIER, STAMPS[0], BAD_VALS[0]);
118    // there is no corresponding timestamp for this so it should never pass
119    put.addColumn(FAMILIES[0], QUALIFIER, STAMPS[2], MATCH_VAL);
120    // if we reverse the qualifiers this one should pass
121    put.addColumn(FAMILIES[1], QUALIFIER, STAMPS[0], MATCH_VAL);
122    // should pass
123    put.addColumn(FAMILIES[1], QUALIFIER, STAMPS[1], BAD_VALS[2]);
124
125    this.region.put(put);
126  }
127
128  private List<KeyValue> makeTestVals() {
129    List<KeyValue> testVals = new ArrayList<>();
130    testVals.add(new KeyValue(ROWS[0], FAMILIES[0], QUALIFIER, STAMPS[0], BAD_VALS[0]));
131    testVals.add(new KeyValue(ROWS[0], FAMILIES[0], QUALIFIER, STAMPS[1], BAD_VALS[1]));
132    testVals.add(new KeyValue(ROWS[0], FAMILIES[1], QUALIFIER, STAMPS[1], BAD_VALS[2]));
133    testVals.add(new KeyValue(ROWS[0], FAMILIES[1], QUALIFIER, STAMPS[0], MATCH_VAL));
134    testVals.add(new KeyValue(ROWS[0], FAMILIES[1], QUALIFIER, STAMPS[2], BAD_VALS[2]));
135
136    return testVals;
137  }
138
139  /**
140   * This shouldn't be confused with TestFilter#verifyScan as expectedKeys is not the per row total,
141   * but the scan total
142   */
143  private void verifyScan(Scan s, long expectedRows, long expectedCells) throws IOException {
144    InternalScanner scanner = this.region.getScanner(s);
145    List<Cell> results = new ArrayList<>();
146    int i = 0;
147    int cells = 0;
148    for (boolean done = true; done; i++) {
149      done = scanner.next(results);
150      Arrays.sort(results.toArray(new Cell[results.size()]), CellComparatorImpl.COMPARATOR);
151      LOG.info("counter=" + i + ", " + results);
152      if (results.isEmpty()) break;
153      cells += results.size();
154      assertTrue("Scanned too many rows! Only expected " + expectedRows
155        + " total but already scanned " + (i + 1), expectedRows > i);
156      assertTrue("Expected " + expectedCells + " cells total but " + "already scanned " + cells,
157        expectedCells >= cells);
158      results.clear();
159    }
160    assertEquals("Expected " + expectedRows + " rows but scanned " + i + " rows", expectedRows, i);
161    assertEquals("Expected " + expectedCells + " cells but scanned " + cells + " cells",
162      expectedCells, cells);
163  }
164
165  /**
166   * Test scans using a DependentColumnFilter
167   */
168  @Test
169  public void testScans() throws Exception {
170    Filter filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER);
171
172    Scan scan = new Scan();
173    scan.setFilter(filter);
174    scan.readVersions(Integer.MAX_VALUE);
175
176    verifyScan(scan, 2, 8);
177
178    // drop the filtering cells
179    filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER, true);
180    scan = new Scan();
181    scan.setFilter(filter);
182    scan.readVersions(Integer.MAX_VALUE);
183
184    verifyScan(scan, 2, 3);
185
186    // include a comparator operation
187    filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER, false, CompareOperator.EQUAL,
188      new BinaryComparator(MATCH_VAL));
189    scan = new Scan();
190    scan.setFilter(filter);
191    scan.readVersions(Integer.MAX_VALUE);
192
193    /*
194     * expecting to get the following 3 cells row 0 put.add(FAMILIES[0], QUALIFIER, STAMPS[2],
195     * MATCH_VAL); put.add(FAMILIES[1], QUALIFIER, STAMPS[2], BAD_VALS[2]); row 1
196     * put.add(FAMILIES[0], QUALIFIER, STAMPS[2], MATCH_VAL);
197     */
198    verifyScan(scan, 2, 3);
199
200    // include a comparator operation and drop comparator
201    filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER, true, CompareOperator.EQUAL,
202      new BinaryComparator(MATCH_VAL));
203    scan = new Scan();
204    scan.setFilter(filter);
205    scan.readVersions(Integer.MAX_VALUE);
206
207    /*
208     * expecting to get the following 1 cell row 0 put.add(FAMILIES[1], QUALIFIER, STAMPS[2],
209     * BAD_VALS[2]);
210     */
211    verifyScan(scan, 1, 1);
212
213  }
214
215  /**
216   * Test that the filter correctly drops rows without a corresponding timestamp
217   */
218  @Test
219  public void testFilterDropping() throws Exception {
220    Filter filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER);
221    List<Cell> accepted = new ArrayList<>();
222    for (Cell val : testVals) {
223      if (filter.filterCell(val) == ReturnCode.INCLUDE) {
224        accepted.add(val);
225      }
226    }
227    assertEquals("check all values accepted from filterCell", 5, accepted.size());
228
229    filter.filterRowCells(accepted);
230    assertEquals("check filterRow(List<KeyValue>) dropped cell without corresponding column entry",
231      4, accepted.size());
232
233    // start do it again with dependent column dropping on
234    filter = new DependentColumnFilter(FAMILIES[1], QUALIFIER, true);
235    accepted.clear();
236    for (KeyValue val : testVals) {
237      if (filter.filterCell(val) == ReturnCode.INCLUDE) {
238        accepted.add(val);
239      }
240    }
241    assertEquals("check the filtering column cells got dropped", 2, accepted.size());
242
243    filter.filterRowCells(accepted);
244    assertEquals("check cell retention", 2, accepted.size());
245  }
246
247  /**
248   * Test for HBASE-8794. Avoid NullPointerException in DependentColumnFilter.toString().
249   */
250  @Test
251  public void testToStringWithNullComparator() {
252    // Test constructor that implicitly sets a null comparator
253    Filter filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER);
254    assertNotNull(filter.toString());
255    assertTrue("check string contains 'null' as compatator is null",
256      filter.toString().contains("null"));
257
258    // Test constructor with explicit null comparator
259    filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER, true, CompareOperator.EQUAL, null);
260    assertNotNull(filter.toString());
261    assertTrue("check string contains 'null' as compatator is null",
262      filter.toString().contains("null"));
263  }
264
265  @Test
266  public void testToStringWithNonNullComparator() {
267    Filter filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER, true, CompareOperator.EQUAL,
268      new BinaryComparator(MATCH_VAL));
269    assertNotNull(filter.toString());
270    assertTrue("check string contains comparator value", filter.toString().contains("match"));
271  }
272
273}