Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.filter;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNotNull;
022import static org.junit.Assert.assertTrue;
023
024import java.io.IOException;
025import java.util.ArrayList;
026import java.util.Arrays;
027import java.util.List;
028import org.apache.hadoop.hbase.Cell;
029import org.apache.hadoop.hbase.CellComparatorImpl;
030import org.apache.hadoop.hbase.CompareOperator;
031import org.apache.hadoop.hbase.HBaseClassTestRule;
032import org.apache.hadoop.hbase.HBaseTestingUtility;
033import org.apache.hadoop.hbase.HColumnDescriptor;
034import org.apache.hadoop.hbase.HRegionInfo;
035import org.apache.hadoop.hbase.HTableDescriptor;
036import org.apache.hadoop.hbase.KeyValue;
037import org.apache.hadoop.hbase.TableName;
038import org.apache.hadoop.hbase.client.Put;
039import org.apache.hadoop.hbase.client.Scan;
040import org.apache.hadoop.hbase.filter.Filter.ReturnCode;
041import org.apache.hadoop.hbase.regionserver.HRegion;
042import org.apache.hadoop.hbase.regionserver.InternalScanner;
043import org.apache.hadoop.hbase.testclassification.FilterTests;
044import org.apache.hadoop.hbase.testclassification.SmallTests;
045import org.apache.hadoop.hbase.util.Bytes;
046import org.junit.After;
047import org.junit.Before;
048import org.junit.ClassRule;
049import org.junit.Test;
050import org.junit.experimental.categories.Category;
051import org.slf4j.Logger;
052import org.slf4j.LoggerFactory;
053
054@Category({FilterTests.class, SmallTests.class})
055public class TestDependentColumnFilter {
056
057  @ClassRule
058  public static final HBaseClassTestRule CLASS_RULE =
059      HBaseClassTestRule.forClass(TestDependentColumnFilter.class);
060
061  private static final Logger LOG = LoggerFactory.getLogger(TestDependentColumnFilter.class);
062  private static final byte[][] ROWS = {
063    Bytes.toBytes("test1"),Bytes.toBytes("test2")
064  };
065  private static final byte[][] FAMILIES = {
066    Bytes.toBytes("familyOne"),Bytes.toBytes("familyTwo")
067  };
068  private static final long STAMP_BASE = System.currentTimeMillis();
069  private static final long[] STAMPS = {
070    STAMP_BASE-100, STAMP_BASE-200, STAMP_BASE-300
071  };
072  private static final byte[] QUALIFIER = Bytes.toBytes("qualifier");
073  private static final byte[][] BAD_VALS = {
074    Bytes.toBytes("bad1"), Bytes.toBytes("bad2"), Bytes.toBytes("bad3")
075  };
076  private static final byte[] MATCH_VAL = Bytes.toBytes("match");
077  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
078
079  List<KeyValue> testVals;
080  private HRegion region;
081
082  @Before
083  public void setUp() throws Exception {
084    testVals = makeTestVals();
085
086    HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(this.getClass().getSimpleName()));
087    HColumnDescriptor hcd0 = new HColumnDescriptor(FAMILIES[0]);
088    hcd0.setMaxVersions(3);
089    htd.addFamily(hcd0);
090    HColumnDescriptor hcd1 = new HColumnDescriptor(FAMILIES[1]);
091    hcd1.setMaxVersions(3);
092    htd.addFamily(hcd1);
093    HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
094    this.region = HBaseTestingUtility.createRegionAndWAL(info, TEST_UTIL.getDataTestDir(),
095        TEST_UTIL.getConfiguration(), htd);
096    addData();
097  }
098
099  @After
100  public void tearDown() throws Exception {
101    HBaseTestingUtility.closeRegionAndWAL(this.region);
102  }
103
104  private void addData() throws IOException {
105    Put put = new Put(ROWS[0]);
106    // add in an entry for each stamp, with 2 as a "good" value
107    put.addColumn(FAMILIES[0], QUALIFIER, STAMPS[0], BAD_VALS[0]);
108    put.addColumn(FAMILIES[0], QUALIFIER, STAMPS[1], BAD_VALS[1]);
109    put.addColumn(FAMILIES[0], QUALIFIER, STAMPS[2], MATCH_VAL);
110    // add in entries for stamps 0 and 2.
111    // without a value check both will be "accepted"
112    // with one 2 will be accepted(since the corresponding ts entry
113    // has a matching value
114    put.addColumn(FAMILIES[1], QUALIFIER, STAMPS[0], BAD_VALS[0]);
115    put.addColumn(FAMILIES[1], QUALIFIER, STAMPS[2], BAD_VALS[2]);
116
117    this.region.put(put);
118
119    put = new Put(ROWS[1]);
120    put.addColumn(FAMILIES[0], QUALIFIER, STAMPS[0], BAD_VALS[0]);
121    // there is no corresponding timestamp for this so it should never pass
122    put.addColumn(FAMILIES[0], QUALIFIER, STAMPS[2], MATCH_VAL);
123    // if we reverse the qualifiers this one should pass
124    put.addColumn(FAMILIES[1], QUALIFIER, STAMPS[0], MATCH_VAL);
125    // should pass
126    put.addColumn(FAMILIES[1], QUALIFIER, STAMPS[1], BAD_VALS[2]);
127
128    this.region.put(put);
129  }
130
131  private List<KeyValue> makeTestVals() {
132    List<KeyValue> testVals = new ArrayList<>();
133    testVals.add(new KeyValue(ROWS[0], FAMILIES[0], QUALIFIER, STAMPS[0], BAD_VALS[0]));
134    testVals.add(new KeyValue(ROWS[0], FAMILIES[0], QUALIFIER, STAMPS[1], BAD_VALS[1]));
135    testVals.add(new KeyValue(ROWS[0], FAMILIES[1], QUALIFIER, STAMPS[1], BAD_VALS[2]));
136    testVals.add(new KeyValue(ROWS[0], FAMILIES[1], QUALIFIER, STAMPS[0], MATCH_VAL));
137    testVals.add(new KeyValue(ROWS[0], FAMILIES[1], QUALIFIER, STAMPS[2], BAD_VALS[2]));
138
139    return testVals;
140  }
141
142  /**
143   * This shouldn't be confused with TestFilter#verifyScan
144   * as expectedKeys is not the per row total, but the scan total
145   *
146   * @param s
147   * @param expectedRows
148   * @param expectedCells
149   * @throws IOException
150   */
151  private void verifyScan(Scan s, long expectedRows, long expectedCells)
152  throws IOException {
153    InternalScanner scanner = this.region.getScanner(s);
154    List<Cell> results = new ArrayList<>();
155    int i = 0;
156    int cells = 0;
157    for (boolean done = true; done; i++) {
158      done = scanner.next(results);
159      Arrays.sort(results.toArray(new Cell[results.size()]),
160          CellComparatorImpl.COMPARATOR);
161      LOG.info("counter=" + i + ", " + results);
162      if (results.isEmpty()) break;
163      cells += results.size();
164      assertTrue("Scanned too many rows! Only expected " + expectedRows +
165          " total but already scanned " + (i+1), expectedRows > i);
166      assertTrue("Expected " + expectedCells + " cells total but " +
167          "already scanned " + cells, expectedCells >= cells);
168      results.clear();
169    }
170    assertEquals("Expected " + expectedRows + " rows but scanned " + i +
171        " rows", expectedRows, i);
172    assertEquals("Expected " + expectedCells + " cells but scanned " + cells +
173            " cells", expectedCells, cells);
174  }
175
176  /**
177   * Test scans using a DependentColumnFilter
178   */
179  @Test
180  public void testScans() throws Exception {
181    Filter filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER);
182
183    Scan scan = new Scan();
184    scan.setFilter(filter);
185    scan.setMaxVersions(Integer.MAX_VALUE);
186
187    verifyScan(scan, 2, 8);
188
189    // drop the filtering cells
190    filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER, true);
191    scan = new Scan();
192    scan.setFilter(filter);
193    scan.setMaxVersions(Integer.MAX_VALUE);
194
195    verifyScan(scan, 2, 3);
196
197    // include a comparator operation
198    filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER, false,
199    CompareOperator.EQUAL, new BinaryComparator(MATCH_VAL));
200    scan = new Scan();
201    scan.setFilter(filter);
202    scan.setMaxVersions(Integer.MAX_VALUE);
203
204    /*
205     * expecting to get the following 3 cells
206     * row 0
207     *   put.add(FAMILIES[0], QUALIFIER, STAMPS[2], MATCH_VAL);
208     *   put.add(FAMILIES[1], QUALIFIER, STAMPS[2], BAD_VALS[2]);
209     * row 1
210     *   put.add(FAMILIES[0], QUALIFIER, STAMPS[2], MATCH_VAL);
211     */
212    verifyScan(scan, 2, 3);
213
214    // include a comparator operation and drop comparator
215    filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER, true,
216    CompareOperator.EQUAL, new BinaryComparator(MATCH_VAL));
217    scan = new Scan();
218    scan.setFilter(filter);
219    scan.setMaxVersions(Integer.MAX_VALUE);
220
221    /*
222     * expecting to get the following 1 cell
223     * row 0
224     *   put.add(FAMILIES[1], QUALIFIER, STAMPS[2], BAD_VALS[2]);
225     */
226    verifyScan(scan, 1, 1);
227
228  }
229
230  /**
231   * Test that the filter correctly drops rows without a corresponding timestamp
232   *
233   * @throws Exception
234   */
235  @Test
236  public void testFilterDropping() throws Exception {
237    Filter filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER);
238    List<Cell> accepted = new ArrayList<>();
239    for(Cell val : testVals) {
240      if(filter.filterCell(val) == ReturnCode.INCLUDE) {
241        accepted.add(val);
242      }
243    }
244    assertEquals("check all values accepted from filterCell", 5, accepted.size());
245
246    filter.filterRowCells(accepted);
247    assertEquals("check filterRow(List<KeyValue>) dropped cell without corresponding column entry", 4, accepted.size());
248
249    // start do it again with dependent column dropping on
250    filter = new DependentColumnFilter(FAMILIES[1], QUALIFIER, true);
251    accepted.clear();
252    for(KeyValue val : testVals) {
253        if(filter.filterCell(val) == ReturnCode.INCLUDE) {
254          accepted.add(val);
255        }
256      }
257      assertEquals("check the filtering column cells got dropped", 2, accepted.size());
258
259      filter.filterRowCells(accepted);
260      assertEquals("check cell retention", 2, accepted.size());
261  }
262
263  /**
264   * Test for HBASE-8794. Avoid NullPointerException in DependentColumnFilter.toString().
265   */
266  @Test
267  public void testToStringWithNullComparator() {
268    // Test constructor that implicitly sets a null comparator
269    Filter filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER);
270    assertNotNull(filter.toString());
271    assertTrue("check string contains 'null' as compatator is null",
272      filter.toString().contains("null"));
273
274    // Test constructor with explicit null comparator
275    filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER, true, CompareOperator.EQUAL, null);
276    assertNotNull(filter.toString());
277    assertTrue("check string contains 'null' as compatator is null",
278      filter.toString().contains("null"));
279  }
280
281  @Test
282  public void testToStringWithNonNullComparator() {
283    Filter filter =
284        new DependentColumnFilter(FAMILIES[0], QUALIFIER, true, CompareOperator.EQUAL,
285            new BinaryComparator(MATCH_VAL));
286    assertNotNull(filter.toString());
287    assertTrue("check string contains comparator value", filter.toString().contains("match"));
288  }
289
290}
291