001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.rest;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNotNull;
022import static org.junit.Assert.assertTrue;
023
024import java.io.ByteArrayInputStream;
025import java.io.StringWriter;
026import java.util.ArrayList;
027import java.util.Arrays;
028import java.util.Iterator;
029import java.util.List;
030import javax.xml.bind.JAXBContext;
031import javax.xml.bind.Marshaller;
032import javax.xml.bind.Unmarshaller;
033import org.apache.hadoop.hbase.CellUtil;
034import org.apache.hadoop.hbase.CompareOperator;
035import org.apache.hadoop.hbase.HBaseClassTestRule;
036import org.apache.hadoop.hbase.HBaseTestingUtil;
037import org.apache.hadoop.hbase.HConstants;
038import org.apache.hadoop.hbase.KeyValue;
039import org.apache.hadoop.hbase.TableName;
040import org.apache.hadoop.hbase.client.Admin;
041import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
042import org.apache.hadoop.hbase.client.Delete;
043import org.apache.hadoop.hbase.client.Durability;
044import org.apache.hadoop.hbase.client.Put;
045import org.apache.hadoop.hbase.client.Scan;
046import org.apache.hadoop.hbase.client.Table;
047import org.apache.hadoop.hbase.client.TableDescriptor;
048import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
049import org.apache.hadoop.hbase.filter.BinaryComparator;
050import org.apache.hadoop.hbase.filter.Filter;
051import org.apache.hadoop.hbase.filter.FilterList;
052import org.apache.hadoop.hbase.filter.FilterList.Operator;
053import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter;
054import org.apache.hadoop.hbase.filter.FuzzyRowFilter;
055import org.apache.hadoop.hbase.filter.InclusiveStopFilter;
056import org.apache.hadoop.hbase.filter.MultiRowRangeFilter;
057import org.apache.hadoop.hbase.filter.PageFilter;
058import org.apache.hadoop.hbase.filter.PrefixFilter;
059import org.apache.hadoop.hbase.filter.QualifierFilter;
060import org.apache.hadoop.hbase.filter.RegexStringComparator;
061import org.apache.hadoop.hbase.filter.RowFilter;
062import org.apache.hadoop.hbase.filter.SkipFilter;
063import org.apache.hadoop.hbase.filter.SubstringComparator;
064import org.apache.hadoop.hbase.filter.ValueFilter;
065import org.apache.hadoop.hbase.rest.client.Client;
066import org.apache.hadoop.hbase.rest.client.Cluster;
067import org.apache.hadoop.hbase.rest.client.Response;
068import org.apache.hadoop.hbase.rest.model.CellModel;
069import org.apache.hadoop.hbase.rest.model.CellSetModel;
070import org.apache.hadoop.hbase.rest.model.RowModel;
071import org.apache.hadoop.hbase.rest.model.ScannerModel;
072import org.apache.hadoop.hbase.testclassification.MediumTests;
073import org.apache.hadoop.hbase.testclassification.RestTests;
074import org.apache.hadoop.hbase.util.Bytes;
075import org.apache.hadoop.hbase.util.Pair;
076import org.junit.AfterClass;
077import org.junit.BeforeClass;
078import org.junit.ClassRule;
079import org.junit.Test;
080import org.junit.experimental.categories.Category;
081import org.slf4j.Logger;
082import org.slf4j.LoggerFactory;
083
084@Category({ RestTests.class, MediumTests.class })
085public class TestScannersWithFilters {
086  @ClassRule
087  public static final HBaseClassTestRule CLASS_RULE =
088    HBaseClassTestRule.forClass(TestScannersWithFilters.class);
089
090  private static final Logger LOG = LoggerFactory.getLogger(TestScannersWithFilters.class);
091
092  private static final TableName TABLE = TableName.valueOf("TestScannersWithFilters");
093
094  private static final byte[][] ROWS_ONE = { Bytes.toBytes("testRowOne-0"),
095    Bytes.toBytes("testRowOne-1"), Bytes.toBytes("testRowOne-2"), Bytes.toBytes("testRowOne-3") };
096
097  private static final byte[][] ROWS_TWO = { Bytes.toBytes("testRowTwo-0"),
098    Bytes.toBytes("testRowTwo-1"), Bytes.toBytes("testRowTwo-2"), Bytes.toBytes("testRowTwo-3") };
099
100  private static final byte[][] FAMILIES =
101    { Bytes.toBytes("testFamilyOne"), Bytes.toBytes("testFamilyTwo") };
102
103  private static final byte[][] QUALIFIERS_ONE =
104    { Bytes.toBytes("testQualifierOne-0"), Bytes.toBytes("testQualifierOne-1"),
105      Bytes.toBytes("testQualifierOne-2"), Bytes.toBytes("testQualifierOne-3") };
106
107  private static final byte[][] QUALIFIERS_TWO =
108    { Bytes.toBytes("testQualifierTwo-0"), Bytes.toBytes("testQualifierTwo-1"),
109      Bytes.toBytes("testQualifierTwo-2"), Bytes.toBytes("testQualifierTwo-3") };
110
111  private static final byte[][] VALUES =
112    { Bytes.toBytes("testValueOne"), Bytes.toBytes("testValueTwo") };
113
114  private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
115  private static final HBaseRESTTestingUtility REST_TEST_UTIL = new HBaseRESTTestingUtility();
116  private static Client client;
117  private static JAXBContext context;
118  private static Marshaller marshaller;
119  private static Unmarshaller unmarshaller;
120  private static long numRows = (long) ROWS_ONE.length + ROWS_TWO.length;
121  private static long colsPerRow = (long) FAMILIES.length * QUALIFIERS_ONE.length;
122
123  @BeforeClass
124  public static void setUpBeforeClass() throws Exception {
125    TEST_UTIL.startMiniCluster(3);
126    REST_TEST_UTIL.startServletContainer(TEST_UTIL.getConfiguration());
127    context = JAXBContext.newInstance(CellModel.class, CellSetModel.class, RowModel.class,
128      ScannerModel.class);
129    marshaller = context.createMarshaller();
130    unmarshaller = context.createUnmarshaller();
131    client = new Client(new Cluster().add("localhost", REST_TEST_UTIL.getServletPort()));
132    Admin admin = TEST_UTIL.getAdmin();
133    if (!admin.tableExists(TABLE)) {
134      TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(TABLE)
135        .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILIES[0]))
136        .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILIES[1])).build();
137      admin.createTable(tableDescriptor);
138      Table table = TEST_UTIL.getConnection().getTable(TABLE);
139      // Insert first half
140      for (byte[] ROW : ROWS_ONE) {
141        Put p = new Put(ROW);
142        p.setDurability(Durability.SKIP_WAL);
143        for (byte[] QUALIFIER : QUALIFIERS_ONE) {
144          p.addColumn(FAMILIES[0], QUALIFIER, VALUES[0]);
145        }
146        table.put(p);
147      }
148      for (byte[] ROW : ROWS_TWO) {
149        Put p = new Put(ROW);
150        p.setDurability(Durability.SKIP_WAL);
151        for (byte[] QUALIFIER : QUALIFIERS_TWO) {
152          p.addColumn(FAMILIES[1], QUALIFIER, VALUES[1]);
153        }
154        table.put(p);
155      }
156
157      // Insert second half (reverse families)
158      for (byte[] ROW : ROWS_ONE) {
159        Put p = new Put(ROW);
160        p.setDurability(Durability.SKIP_WAL);
161        for (byte[] QUALIFIER : QUALIFIERS_ONE) {
162          p.addColumn(FAMILIES[1], QUALIFIER, VALUES[0]);
163        }
164        table.put(p);
165      }
166      for (byte[] ROW : ROWS_TWO) {
167        Put p = new Put(ROW);
168        p.setDurability(Durability.SKIP_WAL);
169        for (byte[] QUALIFIER : QUALIFIERS_TWO) {
170          p.addColumn(FAMILIES[0], QUALIFIER, VALUES[1]);
171        }
172        table.put(p);
173      }
174
175      // Delete the second qualifier from all rows and families
176      for (byte[] ROW : ROWS_ONE) {
177        Delete d = new Delete(ROW);
178        d.addColumns(FAMILIES[0], QUALIFIERS_ONE[1]);
179        d.addColumns(FAMILIES[1], QUALIFIERS_ONE[1]);
180        table.delete(d);
181      }
182      for (byte[] ROW : ROWS_TWO) {
183        Delete d = new Delete(ROW);
184        d.addColumns(FAMILIES[0], QUALIFIERS_TWO[1]);
185        d.addColumns(FAMILIES[1], QUALIFIERS_TWO[1]);
186        table.delete(d);
187      }
188      colsPerRow -= 2;
189
190      // Delete the second rows from both groups, one column at a time
191      for (byte[] QUALIFIER : QUALIFIERS_ONE) {
192        Delete d = new Delete(ROWS_ONE[1]);
193        d.addColumns(FAMILIES[0], QUALIFIER);
194        d.addColumns(FAMILIES[1], QUALIFIER);
195        table.delete(d);
196      }
197      for (byte[] QUALIFIER : QUALIFIERS_TWO) {
198        Delete d = new Delete(ROWS_TWO[1]);
199        d.addColumns(FAMILIES[0], QUALIFIER);
200        d.addColumns(FAMILIES[1], QUALIFIER);
201        table.delete(d);
202      }
203      numRows -= 2;
204      table.close();
205    }
206  }
207
208  @AfterClass
209  public static void tearDownAfterClass() throws Exception {
210    REST_TEST_UTIL.shutdownServletContainer();
211    TEST_UTIL.shutdownMiniCluster();
212  }
213
214  private static void verifyScan(Scan s, long expectedRows, long expectedKeys) throws Exception {
215    ScannerModel model = ScannerModel.fromScan(s);
216    model.setBatch(Integer.MAX_VALUE); // fetch it all at once
217    StringWriter writer = new StringWriter();
218    marshaller.marshal(model, writer);
219    LOG.debug(writer.toString());
220    byte[] body = Bytes.toBytes(writer.toString());
221    Response response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML, body);
222    assertEquals(201, response.getCode());
223    String scannerURI = response.getLocation();
224    assertNotNull(scannerURI);
225
226    // get a cell set
227    response = client.get(scannerURI, Constants.MIMETYPE_XML);
228    assertEquals(200, response.getCode());
229    assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
230    CellSetModel cells =
231      (CellSetModel) unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
232
233    int rows = cells.getRows().size();
234    assertEquals(
235      "Unexpected number of rows! Expected " + expectedRows + " total but scanned " + rows,
236      expectedRows, rows);
237    for (RowModel row : cells.getRows()) {
238      int count = row.getCells().size();
239      assertEquals("Expected " + expectedKeys + " keys per row but " + "returned " + count,
240        expectedKeys, count);
241    }
242
243    // delete the scanner
244    response = client.delete(scannerURI);
245    assertEquals(200, response.getCode());
246  }
247
248  private static void verifyScanFull(Scan s, KeyValue[] kvs) throws Exception {
249    ScannerModel model = ScannerModel.fromScan(s);
250    model.setBatch(Integer.MAX_VALUE); // fetch it all at once
251    StringWriter writer = new StringWriter();
252    marshaller.marshal(model, writer);
253    LOG.debug(writer.toString());
254    byte[] body = Bytes.toBytes(writer.toString());
255    Response response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML, body);
256    assertEquals(201, response.getCode());
257    String scannerURI = response.getLocation();
258    assertNotNull(scannerURI);
259
260    // get a cell set
261    response = client.get(scannerURI, Constants.MIMETYPE_XML);
262    assertEquals(200, response.getCode());
263    assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
264    CellSetModel cellSet =
265      (CellSetModel) unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
266
267    // delete the scanner
268    response = client.delete(scannerURI);
269    assertEquals(200, response.getCode());
270
271    int row = 0;
272    int idx = 0;
273    Iterator<RowModel> i = cellSet.getRows().iterator();
274    for (boolean done = true; done; row++) {
275      done = i.hasNext();
276      if (!done) {
277        break;
278      }
279
280      RowModel rowModel = i.next();
281      List<CellModel> cells = rowModel.getCells();
282      if (cells.isEmpty()) {
283        break;
284      }
285
286      assertTrue("Scanned too many keys! Only expected " + kvs.length
287        + " total but already scanned " + (cells.size() + idx), kvs.length >= idx + cells.size());
288      for (CellModel cell : cells) {
289        assertTrue("Row mismatch", Bytes.equals(rowModel.getKey(), CellUtil.cloneRow(kvs[idx])));
290        byte[][] split = CellUtil.parseColumn(cell.getColumn());
291        assertTrue("Family mismatch", Bytes.equals(split[0], CellUtil.cloneFamily(kvs[idx])));
292        assertTrue("Qualifier mismatch", Bytes.equals(split[1], CellUtil.cloneQualifier(kvs[idx])));
293        assertTrue("Value mismatch", Bytes.equals(cell.getValue(), CellUtil.cloneValue(kvs[idx])));
294        idx++;
295      }
296    }
297    assertEquals("Expected " + kvs.length + " total keys but scanned " + idx, kvs.length, idx);
298  }
299
300  private static void verifyScanNoEarlyOut(Scan s, long expectedRows, long expectedKeys)
301    throws Exception {
302    ScannerModel model = ScannerModel.fromScan(s);
303    model.setBatch(Integer.MAX_VALUE); // fetch it all at once
304    StringWriter writer = new StringWriter();
305    marshaller.marshal(model, writer);
306    LOG.debug(writer.toString());
307    byte[] body = Bytes.toBytes(writer.toString());
308    Response response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML, body);
309    assertEquals(201, response.getCode());
310    String scannerURI = response.getLocation();
311    assertNotNull(scannerURI);
312
313    // get a cell set
314    response = client.get(scannerURI, Constants.MIMETYPE_XML);
315    assertEquals(200, response.getCode());
316    assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
317    CellSetModel cellSet =
318      (CellSetModel) unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
319
320    // delete the scanner
321    response = client.delete(scannerURI);
322    assertEquals(200, response.getCode());
323
324    Iterator<RowModel> i = cellSet.getRows().iterator();
325    int j = 0;
326    for (boolean done = true; done; j++) {
327      done = i.hasNext();
328      if (!done) {
329        break;
330      }
331
332      RowModel rowModel = i.next();
333      List<CellModel> cells = rowModel.getCells();
334      if (cells.isEmpty()) {
335        break;
336      }
337
338      assertTrue("Scanned too many rows! Only expected " + expectedRows
339        + " total but already scanned " + (j + 1), expectedRows > j);
340      assertEquals("Expected " + expectedKeys + " keys per row but " + "returned " + cells.size(),
341        expectedKeys, cells.size());
342    }
343    assertEquals("Expected " + expectedRows + " rows but scanned " + j + " rows", expectedRows, j);
344  }
345
346  @Test
347  public void testNoFilter() throws Exception {
348    // No filter
349    long expectedRows = numRows;
350    long expectedKeys = colsPerRow;
351
352    // Both families
353    Scan s = new Scan();
354    verifyScan(s, expectedRows, expectedKeys);
355
356    // One family
357    s = new Scan();
358    s.addFamily(FAMILIES[0]);
359    verifyScan(s, expectedRows, expectedKeys / 2);
360  }
361
362  @Test
363  public void testPrefixFilter() throws Exception {
364    // Grab rows from group one (half of total)
365    long expectedRows = numRows / 2;
366    long expectedKeys = colsPerRow;
367    Scan s = new Scan();
368    s.setFilter(new PrefixFilter(Bytes.toBytes("testRowOne")));
369    verifyScan(s, expectedRows, expectedKeys);
370  }
371
372  @Test
373  public void testPageFilter() throws Exception {
374    // KVs in first 6 rows
375    KeyValue[] expectedKVs = {
376      // testRowOne-0
377      new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
378      new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]),
379      new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
380      new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]),
381      new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]),
382      new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]),
383      // testRowOne-2
384      new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
385      new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]),
386      new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
387      new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]),
388      new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]),
389      new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]),
390      // testRowOne-3
391      new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
392      new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]),
393      new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
394      new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]),
395      new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]),
396      new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]),
397      // testRowTwo-0
398      new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
399      new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
400      new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
401      new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
402      new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]),
403      new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]),
404      // testRowTwo-2
405      new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
406      new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
407      new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
408      new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
409      new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]),
410      new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]),
411      // testRowTwo-3
412      new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
413      new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
414      new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
415      new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
416      new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]),
417      new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]) };
418
419    // Grab all 6 rows
420    long expectedRows = 6;
421    long expectedKeys = colsPerRow;
422    Scan s = new Scan();
423    s.setFilter(new PageFilter(expectedRows));
424    verifyScan(s, expectedRows, expectedKeys);
425    s.setFilter(new PageFilter(expectedRows));
426    verifyScanFull(s, expectedKVs);
427
428    // Grab first 4 rows (6 cols per row)
429    expectedRows = 4;
430    expectedKeys = colsPerRow;
431    s = new Scan();
432    s.setFilter(new PageFilter(expectedRows));
433    verifyScan(s, expectedRows, expectedKeys);
434    s.setFilter(new PageFilter(expectedRows));
435    verifyScanFull(s, Arrays.copyOf(expectedKVs, 24));
436
437    // Grab first 2 rows
438    expectedRows = 2;
439    expectedKeys = colsPerRow;
440    s = new Scan();
441    s.setFilter(new PageFilter(expectedRows));
442    verifyScan(s, expectedRows, expectedKeys);
443    s.setFilter(new PageFilter(expectedRows));
444    verifyScanFull(s, Arrays.copyOf(expectedKVs, 12));
445
446    // Grab first row
447    expectedRows = 1;
448    expectedKeys = colsPerRow;
449    s = new Scan();
450    s.setFilter(new PageFilter(expectedRows));
451    verifyScan(s, expectedRows, expectedKeys);
452    s.setFilter(new PageFilter(expectedRows));
453    verifyScanFull(s, Arrays.copyOf(expectedKVs, 6));
454  }
455
456  @Test
457  public void testInclusiveStopFilter() throws Exception {
458    // Grab rows from group one
459
460    // If we just use start/stop row, we get total/2 - 1 rows
461    long expectedRows = (numRows / 2) - 1;
462    long expectedKeys = colsPerRow;
463    Scan s = new Scan().withStartRow(Bytes.toBytes("testRowOne-0"))
464      .withStopRow(Bytes.toBytes("testRowOne-3"));
465    verifyScan(s, expectedRows, expectedKeys);
466
467    // Now use start row with inclusive stop filter
468    expectedRows = numRows / 2;
469    s = new Scan().withStartRow(Bytes.toBytes("testRowOne-0"));
470    s.setFilter(new InclusiveStopFilter(Bytes.toBytes("testRowOne-3")));
471    verifyScan(s, expectedRows, expectedKeys);
472
473    // Grab rows from group two
474
475    // If we just use start/stop row, we get total/2 - 1 rows
476    expectedRows = (numRows / 2) - 1;
477    expectedKeys = colsPerRow;
478    s = new Scan().withStartRow(Bytes.toBytes("testRowTwo-0"))
479      .withStopRow(Bytes.toBytes("testRowTwo-3"));
480    verifyScan(s, expectedRows, expectedKeys);
481
482    // Now use start row with inclusive stop filter
483    expectedRows = numRows / 2;
484    s = new Scan().withStartRow(Bytes.toBytes("testRowTwo-0"));
485    s.setFilter(new InclusiveStopFilter(Bytes.toBytes("testRowTwo-3")));
486    verifyScan(s, expectedRows, expectedKeys);
487  }
488
489  @Test
490  public void testQualifierFilter() throws Exception {
491    // Match two keys (one from each family) in half the rows
492    long expectedRows = numRows / 2;
493    long expectedKeys = 2;
494    Filter f = new QualifierFilter(CompareOperator.EQUAL,
495      new BinaryComparator(Bytes.toBytes("testQualifierOne-2")));
496    Scan s = new Scan();
497    s.setFilter(f);
498    verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
499
500    // Match keys less than same qualifier
501    // Expect only two keys (one from each family) in half the rows
502    expectedRows = numRows / 2;
503    expectedKeys = 2;
504    f = new QualifierFilter(CompareOperator.LESS,
505      new BinaryComparator(Bytes.toBytes("testQualifierOne-2")));
506    s = new Scan();
507    s.setFilter(f);
508    verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
509
510    // Match keys less than or equal. Expect four keys (two from each family) in half the rows
511    expectedRows = numRows / 2;
512    expectedKeys = 4;
513    f = new QualifierFilter(CompareOperator.LESS_OR_EQUAL,
514      new BinaryComparator(Bytes.toBytes("testQualifierOne-2")));
515    s = new Scan();
516    s.setFilter(f);
517    verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
518
519    // Match keys not equal. Expect four keys (two from each family)
520    // Only look in first group of rows
521    expectedRows = numRows / 2;
522    expectedKeys = 4;
523    f = new QualifierFilter(CompareOperator.NOT_EQUAL,
524      new BinaryComparator(Bytes.toBytes("testQualifierOne-2")));
525    s =
526      new Scan().withStartRow(HConstants.EMPTY_START_ROW).withStopRow(Bytes.toBytes("testRowTwo"));
527    s.setFilter(f);
528    verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
529
530    // Match keys greater or equal. Expect four keys (two from each family)
531    // Only look in first group of rows
532    expectedRows = numRows / 2;
533    expectedKeys = 4;
534    f = new QualifierFilter(CompareOperator.GREATER_OR_EQUAL,
535      new BinaryComparator(Bytes.toBytes("testQualifierOne-2")));
536    s =
537      new Scan().withStartRow(HConstants.EMPTY_START_ROW).withStopRow(Bytes.toBytes("testRowTwo"));
538    s.setFilter(f);
539    verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
540
541    // Match keys greater. Expect two keys (one from each family)
542    // Only look in first group of rows
543    expectedRows = numRows / 2;
544    expectedKeys = 2;
545    f = new QualifierFilter(CompareOperator.GREATER,
546      new BinaryComparator(Bytes.toBytes("testQualifierOne-2")));
547    s =
548      new Scan().withStartRow(HConstants.EMPTY_START_ROW).withStopRow(Bytes.toBytes("testRowTwo"));
549    s.setFilter(f);
550    verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
551
552    // Match keys not equal to. Look across rows and fully validate the keys and ordering
553    // Expect varied numbers of keys, 4 per row in group one, 6 per row in group two
554    f = new QualifierFilter(CompareOperator.NOT_EQUAL, new BinaryComparator(QUALIFIERS_ONE[2]));
555    s = new Scan();
556    s.setFilter(f);
557
558    KeyValue[] kvs = {
559      // testRowOne-0
560      new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
561      new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
562      new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]),
563      new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]),
564      // testRowOne-2
565      new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
566      new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
567      new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]),
568      new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]),
569      // testRowOne-3
570      new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
571      new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
572      new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]),
573      new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]),
574      // testRowTwo-0
575      new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
576      new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
577      new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
578      new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
579      new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]),
580      new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]),
581      // testRowTwo-2
582      new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
583      new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
584      new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
585      new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
586      new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]),
587      new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]),
588      // testRowTwo-3
589      new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
590      new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
591      new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
592      new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
593      new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]),
594      new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), };
595    verifyScanFull(s, kvs);
596
597    // Test across rows and groups with a regex. Filter out "test*-2"
598    // Expect 4 keys per row across both groups
599    f = new QualifierFilter(CompareOperator.NOT_EQUAL, new RegexStringComparator("test.+-2"));
600    s = new Scan();
601    s.setFilter(f);
602
603    kvs = new KeyValue[] {
604      // testRowOne-0
605      new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
606      new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
607      new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]),
608      new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]),
609      // testRowOne-2
610      new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
611      new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
612      new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]),
613      new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]),
614      // testRowOne-3
615      new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
616      new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
617      new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]),
618      new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]),
619      // testRowTwo-0
620      new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
621      new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
622      new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
623      new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]),
624      // testRowTwo-2
625      new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
626      new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
627      new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
628      new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]),
629      // testRowTwo-3
630      new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
631      new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
632      new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
633      new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), };
634    verifyScanFull(s, kvs);
635  }
636
637  @Test
638  public void testRowFilter() throws Exception {
639    // Match a single row, all keys
640    long expectedRows = 1;
641    long expectedKeys = colsPerRow;
642    Filter f =
643      new RowFilter(CompareOperator.EQUAL, new BinaryComparator(Bytes.toBytes("testRowOne-2")));
644    Scan s = new Scan();
645    s.setFilter(f);
646    verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
647
648    // Match a two rows, one from each group, using regex
649    expectedRows = 2;
650    expectedKeys = colsPerRow;
651    f = new RowFilter(CompareOperator.EQUAL, new RegexStringComparator("testRow.+-2"));
652    s = new Scan();
653    s.setFilter(f);
654    verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
655
656    // Match rows less than
657    // Expect all keys in one row
658    expectedRows = 1;
659    expectedKeys = colsPerRow;
660    f = new RowFilter(CompareOperator.LESS, new BinaryComparator(Bytes.toBytes("testRowOne-2")));
661    s = new Scan();
662    s.setFilter(f);
663    verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
664
665    // Match rows less than or equal
666    // Expect all keys in two rows
667    expectedRows = 2;
668    expectedKeys = colsPerRow;
669    f = new RowFilter(CompareOperator.LESS_OR_EQUAL,
670      new BinaryComparator(Bytes.toBytes("testRowOne-2")));
671    s = new Scan();
672    s.setFilter(f);
673    verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
674
675    // Match rows not equal
676    // Expect all keys in all but one row
677    expectedRows = numRows - 1;
678    expectedKeys = colsPerRow;
679    f =
680      new RowFilter(CompareOperator.NOT_EQUAL, new BinaryComparator(Bytes.toBytes("testRowOne-2")));
681    s = new Scan();
682    s.setFilter(f);
683    verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
684
685    // Match keys greater or equal
686    // Expect all keys in all but one row
687    expectedRows = numRows - 1;
688    expectedKeys = colsPerRow;
689    f = new RowFilter(CompareOperator.GREATER_OR_EQUAL,
690      new BinaryComparator(Bytes.toBytes("testRowOne-2")));
691    s = new Scan();
692    s.setFilter(f);
693    verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
694
695    // Match keys greater
696    // Expect all keys in all but two rows
697    expectedRows = numRows - 2;
698    expectedKeys = colsPerRow;
699    f = new RowFilter(CompareOperator.GREATER, new BinaryComparator(Bytes.toBytes("testRowOne-2")));
700    s = new Scan();
701    s.setFilter(f);
702    verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
703
704    // Match rows not equal to testRowTwo-2
705    // Look across rows and fully validate the keys and ordering
706    // Should see all keys in all rows but testRowTwo-2
707    f =
708      new RowFilter(CompareOperator.NOT_EQUAL, new BinaryComparator(Bytes.toBytes("testRowOne-2")));
709    s = new Scan();
710    s.setFilter(f);
711
712    KeyValue[] kvs = {
713      // testRowOne-0
714      new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
715      new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]),
716      new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
717      new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]),
718      new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]),
719      new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]),
720      // testRowOne-3
721      new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
722      new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]),
723      new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
724      new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]),
725      new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]),
726      new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]),
727      // testRowTwo-0
728      new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
729      new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
730      new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
731      new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
732      new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]),
733      new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]),
734      // testRowTwo-2
735      new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
736      new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
737      new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
738      new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
739      new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]),
740      new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]),
741      // testRowTwo-3
742      new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
743      new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
744      new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
745      new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
746      new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]),
747      new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), };
748    verifyScanFull(s, kvs);
749
750    // Test across rows and groups with a regex
751    // Filter out everything that doesn't match "*-2"
752    // Expect all keys in two rows
753    f = new RowFilter(CompareOperator.EQUAL, new RegexStringComparator(".+-2"));
754    s = new Scan();
755    s.setFilter(f);
756
757    kvs = new KeyValue[] {
758      // testRowOne-2
759      new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
760      new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]),
761      new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
762      new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]),
763      new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]),
764      new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]),
765      // testRowTwo-2
766      new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
767      new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
768      new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
769      new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
770      new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]),
771      new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]) };
772    verifyScanFull(s, kvs);
773  }
774
775  @Test
776  public void testValueFilter() throws Exception {
777    // Match group one rows
778    long expectedRows = numRows / 2;
779    long expectedKeys = colsPerRow;
780    Filter f =
781      new ValueFilter(CompareOperator.EQUAL, new BinaryComparator(Bytes.toBytes("testValueOne")));
782    Scan s = new Scan();
783    s.setFilter(f);
784    verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
785
786    // Match group two rows
787    expectedRows = numRows / 2;
788    expectedKeys = colsPerRow;
789    f = new ValueFilter(CompareOperator.EQUAL, new BinaryComparator(Bytes.toBytes("testValueTwo")));
790    s = new Scan();
791    s.setFilter(f);
792    verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
793
794    // Match all values using regex
795    expectedRows = numRows;
796    expectedKeys = colsPerRow;
797    f = new ValueFilter(CompareOperator.EQUAL, new RegexStringComparator("testValue((One)|(Two))"));
798    s = new Scan();
799    s.setFilter(f);
800    verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
801
802    // Match values less than
803    // Expect group one rows
804    expectedRows = numRows / 2;
805    expectedKeys = colsPerRow;
806    f = new ValueFilter(CompareOperator.LESS, new BinaryComparator(Bytes.toBytes("testValueTwo")));
807    s = new Scan();
808    s.setFilter(f);
809    verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
810
811    // Match values less than or equal
812    // Expect all rows
813    expectedRows = numRows;
814    expectedKeys = colsPerRow;
815    f = new ValueFilter(CompareOperator.LESS_OR_EQUAL,
816      new BinaryComparator(Bytes.toBytes("testValueTwo")));
817    s = new Scan();
818    s.setFilter(f);
819    verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
820
821    // Match values less than or equal
822    // Expect group one rows
823    expectedRows = numRows / 2;
824    expectedKeys = colsPerRow;
825    f = new ValueFilter(CompareOperator.LESS_OR_EQUAL,
826      new BinaryComparator(Bytes.toBytes("testValueOne")));
827    s = new Scan();
828    s.setFilter(f);
829    verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
830
831    // Match values not equal
832    // Expect half the rows
833    expectedRows = numRows / 2;
834    expectedKeys = colsPerRow;
835    f = new ValueFilter(CompareOperator.NOT_EQUAL,
836      new BinaryComparator(Bytes.toBytes("testValueOne")));
837    s = new Scan();
838    s.setFilter(f);
839    verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
840
841    // Match values greater or equal
842    // Expect all rows
843    expectedRows = numRows;
844    expectedKeys = colsPerRow;
845    f = new ValueFilter(CompareOperator.GREATER_OR_EQUAL,
846      new BinaryComparator(Bytes.toBytes("testValueOne")));
847    s = new Scan();
848    s.setFilter(f);
849    verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
850
851    // Match values greater
852    // Expect half rows
853    expectedRows = numRows / 2;
854    expectedKeys = colsPerRow;
855    f =
856      new ValueFilter(CompareOperator.GREATER, new BinaryComparator(Bytes.toBytes("testValueOne")));
857    s = new Scan();
858    s.setFilter(f);
859    verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
860
861    // Match values not equal to testValueOne
862    // Look across rows and fully validate the keys and ordering
863    // Should see all keys in all group two rows
864    f = new ValueFilter(CompareOperator.NOT_EQUAL,
865      new BinaryComparator(Bytes.toBytes("testValueOne")));
866    s = new Scan();
867    s.setFilter(f);
868
869    KeyValue[] kvs = {
870      // testRowTwo-0
871      new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
872      new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
873      new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
874      new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
875      new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]),
876      new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]),
877      // testRowTwo-2
878      new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
879      new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
880      new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
881      new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
882      new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]),
883      new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]),
884      // testRowTwo-3
885      new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
886      new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
887      new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
888      new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
889      new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]),
890      new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), };
891    verifyScanFull(s, kvs);
892  }
893
894  @Test
895  public void testSkipFilter() throws Exception {
896    // Test for qualifier regex: "testQualifierOne-2"
897    // Should only get rows from second group, and all keys
898    Filter f = new SkipFilter(new QualifierFilter(CompareOperator.NOT_EQUAL,
899      new BinaryComparator(Bytes.toBytes("testQualifierOne-2"))));
900    Scan s = new Scan();
901    s.setFilter(f);
902
903    KeyValue[] kvs = {
904      // testRowTwo-0
905      new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
906      new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
907      new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
908      new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
909      new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]),
910      new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]),
911      // testRowTwo-2
912      new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
913      new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
914      new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
915      new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
916      new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]),
917      new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]),
918      // testRowTwo-3
919      new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
920      new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
921      new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
922      new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
923      new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]),
924      new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), };
925    verifyScanFull(s, kvs);
926  }
927
928  @Test
929  public void testFilterList() throws Exception {
930    // Test getting a single row, single key using Row, Qualifier, and Value
931    // regular expression and substring filters
932    // Use must pass all
933    List<Filter> filters = new ArrayList<>(3);
934    filters.add(new RowFilter(CompareOperator.EQUAL, new RegexStringComparator(".+-2")));
935    filters.add(new QualifierFilter(CompareOperator.EQUAL, new RegexStringComparator(".+-2")));
936    filters.add(new ValueFilter(CompareOperator.EQUAL, new SubstringComparator("One")));
937    Filter f = new FilterList(Operator.MUST_PASS_ALL, filters);
938    Scan s = new Scan();
939    s.addFamily(FAMILIES[0]);
940    s.setFilter(f);
941    KeyValue[] kvs = { new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]) };
942    verifyScanFull(s, kvs);
943
944    // Test getting everything with a MUST_PASS_ONE filter including row, qf,
945    // val, regular expression and substring filters
946    filters.clear();
947    filters.add(new RowFilter(CompareOperator.EQUAL, new RegexStringComparator(".+Two.+")));
948    filters.add(new QualifierFilter(CompareOperator.EQUAL, new RegexStringComparator(".+-2")));
949    filters.add(new ValueFilter(CompareOperator.EQUAL, new SubstringComparator("One")));
950    f = new FilterList(Operator.MUST_PASS_ONE, filters);
951    s = new Scan();
952    s.setFilter(f);
953    verifyScanNoEarlyOut(s, numRows, colsPerRow);
954  }
955
956  @Test
957  public void testFirstKeyOnlyFilter() throws Exception {
958    Scan s = new Scan();
959    s.setFilter(new FirstKeyOnlyFilter());
960    // Expected KVs, the first KV from each of the remaining 6 rows
961    KeyValue[] kvs = { new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
962      new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
963      new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
964      new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
965      new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
966      new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]) };
967    verifyScanFull(s, kvs);
968  }
969
970  @Test
971  public void testMultiRowRangeFilter() throws Exception {
972    long expectedRows = 2;
973    long expectedKeys = colsPerRow;
974    List<MultiRowRangeFilter.RowRange> ranges = new ArrayList<>();
975    // Both return only the third element, as the second one is deleted during initialization.
976    ranges.add(new MultiRowRangeFilter.RowRange(ROWS_ONE[1], true, ROWS_ONE[2], true));
977    ranges.add(new MultiRowRangeFilter.RowRange(ROWS_TWO[0], false, ROWS_TWO[3], false));
978
979    Scan s = new Scan();
980    s.setFilter(new MultiRowRangeFilter(ranges));
981    verifyScan(s, expectedRows, expectedKeys);
982  }
983
984  @Test
985  public void testFuzzyRowFilter() throws Exception {
986    long expectedRows = 4;
987    long expectedKeys = colsPerRow;
988    List<Pair<byte[], byte[]>> fuzzyKeys = new ArrayList<>();
989
990    // Exact match for ROWS_ONE[0] (one row)
991    byte[] rowOneMask = new byte[ROWS_ONE[0].length];
992    Arrays.fill(rowOneMask, (byte) 0);
993    fuzzyKeys.add(new Pair<>(ROWS_ONE[0], rowOneMask));
994    // All ROW_TWO keys (three rows)
995    byte[] rowTwoMask = new byte[ROWS_TWO[0].length];
996    Arrays.fill(rowTwoMask, (byte) 0);
997    rowTwoMask[rowTwoMask.length - 1] = (byte) 1;
998    fuzzyKeys.add(new Pair<>(ROWS_TWO[2], rowTwoMask));
999
1000    Scan s = new Scan();
1001    s.setFilter(new FuzzyRowFilter(fuzzyKeys));
1002    verifyScan(s, expectedRows, expectedKeys);
1003  }
1004}