001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.apache.hadoop.hbase.KeyValueTestUtil.create;
021import static org.apache.hadoop.hbase.regionserver.KeyValueScanFixture.scanFixture;
022import static org.junit.Assert.assertEquals;
023import static org.junit.Assert.assertFalse;
024import static org.junit.Assert.assertNull;
025import static org.junit.Assert.assertTrue;
026
027import java.io.IOException;
028import java.util.ArrayList;
029import java.util.Arrays;
030import java.util.Collections;
031import java.util.List;
032import java.util.NavigableSet;
033import java.util.TreeSet;
034import java.util.concurrent.atomic.AtomicInteger;
035
036import org.apache.hadoop.conf.Configuration;
037import org.apache.hadoop.hbase.Cell;
038import org.apache.hadoop.hbase.CellBuilderType;
039import org.apache.hadoop.hbase.CellComparator;
040import org.apache.hadoop.hbase.CellUtil;
041import org.apache.hadoop.hbase.CompareOperator;
042import org.apache.hadoop.hbase.ExtendedCellBuilderFactory;
043import org.apache.hadoop.hbase.HBaseClassTestRule;
044import org.apache.hadoop.hbase.HBaseConfiguration;
045import org.apache.hadoop.hbase.HBaseTestingUtility;
046import org.apache.hadoop.hbase.HConstants;
047import org.apache.hadoop.hbase.KeepDeletedCells;
048import org.apache.hadoop.hbase.KeyValue;
049import org.apache.hadoop.hbase.PrivateCellUtil;
050import org.apache.hadoop.hbase.client.Get;
051import org.apache.hadoop.hbase.client.Scan;
052import org.apache.hadoop.hbase.filter.BinaryComparator;
053import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
054import org.apache.hadoop.hbase.filter.Filter;
055import org.apache.hadoop.hbase.filter.QualifierFilter;
056import org.apache.hadoop.hbase.testclassification.RegionServerTests;
057import org.apache.hadoop.hbase.testclassification.SmallTests;
058import org.apache.hadoop.hbase.util.Bytes;
059import org.apache.hadoop.hbase.util.EnvironmentEdge;
060import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
061import org.junit.ClassRule;
062import org.junit.Ignore;
063import org.junit.Rule;
064import org.junit.Test;
065import org.junit.experimental.categories.Category;
066import org.junit.rules.TestName;
067import org.slf4j.Logger;
068import org.slf4j.LoggerFactory;
069
070// Can't be small as it plays with EnvironmentEdgeManager
071@Category({RegionServerTests.class, SmallTests.class})
072public class TestStoreScanner {
073
074  @ClassRule
075  public static final HBaseClassTestRule CLASS_RULE =
076      HBaseClassTestRule.forClass(TestStoreScanner.class);
077
078  private static final Logger LOG = LoggerFactory.getLogger(TestStoreScanner.class);
079  @Rule public TestName name = new TestName();
080  private static final String CF_STR = "cf";
081  private static final byte[] CF = Bytes.toBytes(CF_STR);
082  static Configuration CONF = HBaseConfiguration.create();
083  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
084  private ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, Integer.MAX_VALUE, Long.MAX_VALUE,
085      KeepDeletedCells.FALSE, HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false);
086
087  /**
088   * From here on down, we have a bunch of defines and specific CELL_GRID of Cells. The
089   * CELL_GRID then has a Scanner that can fake out 'block' transitions. All this elaborate
090   * setup is for tests that ensure we don't overread, and that the {@link StoreScanner} is not
091   * overly enthusiastic.
092   */
093  private static final byte[] ZERO = new byte[] {'0'};
094  private static final byte[] ZERO_POINT_ZERO = new byte[] {'0', '.', '0'};
095  private static final byte[] ONE = new byte[] {'1'};
096  private static final byte[] TWO = new byte[] {'2'};
097  private static final byte[] TWO_POINT_TWO = new byte[] {'2', '.', '2'};
098  private static final byte[] THREE = new byte[] {'3'};
099  private static final byte[] FOUR = new byte[] {'4'};
100  private static final byte[] FIVE = new byte[] {'5'};
101  private static final byte[] VALUE = new byte[] {'v'};
102  private static final int CELL_GRID_BLOCK2_BOUNDARY = 4;
103  private static final int CELL_GRID_BLOCK3_BOUNDARY = 11;
104  private static final int CELL_GRID_BLOCK4_BOUNDARY = 15;
105  private static final int CELL_GRID_BLOCK5_BOUNDARY = 19;
106
107  /**
108   * Five rows by four columns distinguished by column qualifier (column qualifier is one of the
109   * four rows... ONE, TWO, etc.). Exceptions are a weird row after TWO; it is TWO_POINT_TWO.
110   * And then row FOUR has five columns finishing w/ row FIVE having a single column.
111   * We will use this to test scan does the right thing as it
112   * we do Gets, StoreScanner#optimize, and what we do on (faked) block boundaries.
113   */
114  private static final Cell[] CELL_GRID = new Cell [] {
115    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(ONE)
116      .setFamily(CF).setQualifier(ONE).setTimestamp(1L)
117      .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(),
118    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(ONE)
119      .setFamily(CF).setQualifier(TWO).setTimestamp(1L)
120      .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(),
121    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(ONE)
122      .setFamily(CF).setQualifier(THREE).setTimestamp(1L)
123      .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(),
124    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(ONE)
125      .setFamily(CF).setQualifier(FOUR).setTimestamp(1L)
126      .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(),
127    // Offset 4 CELL_GRID_BLOCK2_BOUNDARY
128    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(TWO)
129      .setFamily(CF).setQualifier(ONE).setTimestamp(1L)
130      .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(),
131    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(TWO)
132      .setFamily(CF).setQualifier(TWO).setTimestamp(1L)
133      .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(),
134    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(TWO)
135      .setFamily(CF).setQualifier(THREE).setTimestamp(1L)
136      .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(),
137    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(TWO)
138      .setFamily(CF).setQualifier(FOUR).setTimestamp(1L)
139      .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(),
140    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(TWO_POINT_TWO)
141      .setFamily(CF).setQualifier(ZERO).setTimestamp(1L)
142      .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(),
143    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(TWO_POINT_TWO)
144      .setFamily(CF).setQualifier(ZERO_POINT_ZERO).setTimestamp(1L)
145      .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(),
146    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(TWO_POINT_TWO)
147      .setFamily(CF).setQualifier(FIVE).setTimestamp(1L)
148      .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(),
149    // Offset 11! CELL_GRID_BLOCK3_BOUNDARY
150    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(THREE)
151      .setFamily(CF).setQualifier(ONE).setTimestamp(1L)
152      .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(),
153    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(THREE)
154      .setFamily(CF).setQualifier(TWO).setTimestamp(1L)
155      .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(),
156    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(THREE)
157      .setFamily(CF).setQualifier(THREE).setTimestamp(1L)
158      .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(),
159    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(THREE)
160      .setFamily(CF).setQualifier(FOUR).setTimestamp(1L)
161      .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(),
162    // Offset 15 CELL_GRID_BLOCK4_BOUNDARY
163    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(FOUR)
164      .setFamily(CF).setQualifier(ONE).setTimestamp(1L)
165      .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(),
166    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(FOUR)
167      .setFamily(CF).setQualifier(TWO).setTimestamp(1L)
168      .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(),
169    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(FOUR)
170      .setFamily(CF).setQualifier(THREE).setTimestamp(1L)
171      .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(),
172    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(FOUR)
173      .setFamily(CF).setQualifier(FOUR).setTimestamp(1L)
174      .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(),
175    // Offset 19 CELL_GRID_BLOCK5_BOUNDARY
176    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(FOUR)
177      .setFamily(CF).setQualifier(FIVE).setTimestamp(1L)
178      .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(),
179    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(FIVE)
180      .setFamily(CF).setQualifier(ZERO).setTimestamp(1L)
181      .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(),
182  };
183
184  private static class KeyValueHeapWithCount extends KeyValueHeap {
185
186    final AtomicInteger count;
187
188    public KeyValueHeapWithCount(List<? extends KeyValueScanner> scanners,
189        CellComparator comparator, AtomicInteger count) throws IOException {
190      super(scanners, comparator);
191      this.count = count;
192    }
193
194    @Override
195    public Cell peek() {
196      this.count.incrementAndGet();
197      return super.peek();
198    }
199  }
200
201  /**
202   * A StoreScanner for our CELL_GRID above. Fakes the block transitions. Does counts of
203   * calls to optimize and counts of when optimize actually did an optimize.
204   */
205  private static class CellGridStoreScanner extends StoreScanner {
206    // Count of how often optimize is called and of how often it does an optimize.
207    AtomicInteger count;
208    final AtomicInteger optimization = new AtomicInteger(0);
209
210    CellGridStoreScanner(final Scan scan, ScanInfo scanInfo) throws IOException {
211      super(scan, scanInfo, scan.getFamilyMap().get(CF), Arrays.<KeyValueScanner> asList(
212        new KeyValueScanner[] {new KeyValueScanFixture(CellComparator.getInstance(), CELL_GRID)}));
213    }
214
215    @Override
216    protected void resetKVHeap(List<? extends KeyValueScanner> scanners,
217        CellComparator comparator) throws IOException {
218      if (count == null) {
219        count = new AtomicInteger(0);
220      }
221      heap = newKVHeap(scanners, comparator);
222    }
223
224    @Override
225    protected KeyValueHeap newKVHeap(List<? extends KeyValueScanner> scanners,
226        CellComparator comparator) throws IOException {
227      return new KeyValueHeapWithCount(scanners, comparator, count);
228    }
229
230    @Override
231    protected boolean trySkipToNextRow(Cell cell) throws IOException {
232      boolean optimized = super.trySkipToNextRow(cell);
233      LOG.info("Cell=" + cell + ", nextIndex=" + CellUtil.toString(getNextIndexedKey(), false)
234          + ", optimized=" + optimized);
235      if (optimized) {
236        optimization.incrementAndGet();
237      }
238      return optimized;
239    }
240
241    @Override
242    protected boolean trySkipToNextColumn(Cell cell) throws IOException {
243      boolean optimized = super.trySkipToNextColumn(cell);
244      LOG.info("Cell=" + cell + ", nextIndex=" + CellUtil.toString(getNextIndexedKey(), false)
245          + ", optimized=" + optimized);
246      if (optimized) {
247        optimization.incrementAndGet();
248      }
249      return optimized;
250    }
251
252    @Override
253    public Cell getNextIndexedKey() {
254      // Fake block boundaries by having index of next block change as we go through scan.
255      return count.get() > CELL_GRID_BLOCK4_BOUNDARY?
256          PrivateCellUtil.createFirstOnRow(CELL_GRID[CELL_GRID_BLOCK5_BOUNDARY]):
257            count.get() > CELL_GRID_BLOCK3_BOUNDARY?
258                PrivateCellUtil.createFirstOnRow(CELL_GRID[CELL_GRID_BLOCK4_BOUNDARY]):
259                  count.get() > CELL_GRID_BLOCK2_BOUNDARY?
260                      PrivateCellUtil.createFirstOnRow(CELL_GRID[CELL_GRID_BLOCK3_BOUNDARY]):
261                        PrivateCellUtil.createFirstOnRow(CELL_GRID[CELL_GRID_BLOCK2_BOUNDARY]);
262    }
263  }
264
265  private static final int CELL_WITH_VERSIONS_BLOCK2_BOUNDARY = 4;
266
267  private static final Cell[] CELL_WITH_VERSIONS = new Cell [] {
268    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(ONE)
269      .setFamily(CF).setQualifier(ONE).setTimestamp(2L)
270      .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(),
271    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(ONE)
272      .setFamily(CF).setQualifier(ONE).setTimestamp(1L)
273      .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(),
274    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(ONE)
275      .setFamily(CF).setQualifier(TWO).setTimestamp(2L)
276      .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(),
277    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(ONE)
278      .setFamily(CF).setQualifier(TWO).setTimestamp(1L)
279      .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(),
280    // Offset 4 CELL_WITH_VERSIONS_BLOCK2_BOUNDARY
281    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(TWO)
282      .setFamily(CF).setQualifier(ONE).setTimestamp(1L)
283      .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(),
284    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(TWO)
285      .setFamily(CF).setQualifier(TWO).setTimestamp(1L)
286      .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(),
287  };
288
289  private static class CellWithVersionsStoreScanner extends StoreScanner {
290    // Count of how often optimize is called and of how often it does an optimize.
291    final AtomicInteger optimization = new AtomicInteger(0);
292
293    CellWithVersionsStoreScanner(final Scan scan, ScanInfo scanInfo) throws IOException {
294      super(scan, scanInfo, scan.getFamilyMap().get(CF),
295          Arrays.<KeyValueScanner> asList(new KeyValueScanner[] {
296            new KeyValueScanFixture(CellComparator.getInstance(), CELL_WITH_VERSIONS) }));
297    }
298
299    @Override
300    protected boolean trySkipToNextColumn(Cell cell) throws IOException {
301      boolean optimized = super.trySkipToNextColumn(cell);
302      LOG.info("Cell=" + cell + ", nextIndex=" + CellUtil.toString(getNextIndexedKey(), false)
303          + ", optimized=" + optimized);
304      if (optimized) {
305        optimization.incrementAndGet();
306      }
307      return optimized;
308    }
309
310    @Override
311    public Cell getNextIndexedKey() {
312      // Fake block boundaries by having index of next block change as we go through scan.
313      return PrivateCellUtil
314          .createFirstOnRow(CELL_WITH_VERSIONS[CELL_WITH_VERSIONS_BLOCK2_BOUNDARY]);
315    }
316  }
317
318  private static class CellWithVersionsNoOptimizeStoreScanner extends StoreScanner {
319    // Count of how often optimize is called and of how often it does an optimize.
320    final AtomicInteger optimization = new AtomicInteger(0);
321
322    CellWithVersionsNoOptimizeStoreScanner(Scan scan, ScanInfo scanInfo) throws IOException {
323      super(scan, scanInfo, scan.getFamilyMap().get(CF),
324        Arrays.<KeyValueScanner> asList(new KeyValueScanner[] {
325          new KeyValueScanFixture(CellComparator.getInstance(), CELL_WITH_VERSIONS) }));
326    }
327
328    @Override
329    protected boolean trySkipToNextColumn(Cell cell) throws IOException {
330      boolean optimized = super.trySkipToNextColumn(cell);
331      LOG.info("Cell=" + cell + ", nextIndex=" + CellUtil.toString(getNextIndexedKey(), false)
332          + ", optimized=" + optimized);
333      if (optimized) {
334        optimization.incrementAndGet();
335      }
336      return optimized;
337    }
338
339    @Override
340    public Cell getNextIndexedKey() {
341      return null;
342    }
343  }
344
345  @Test
346  public void testWithColumnCountGetFilter() throws Exception {
347    Get get = new Get(ONE);
348    get.readAllVersions();
349    get.addFamily(CF);
350    get.setFilter(new ColumnCountGetFilter(2));
351
352    try (CellWithVersionsNoOptimizeStoreScanner scannerNoOptimize =
353        new CellWithVersionsNoOptimizeStoreScanner(new Scan(get), this.scanInfo)) {
354      List<Cell> results = new ArrayList<>();
355      while (scannerNoOptimize.next(results)) {
356        continue;
357      }
358      assertEquals(2, results.size());
359      assertTrue(CellUtil.matchingColumn(results.get(0), CELL_WITH_VERSIONS[0]));
360      assertTrue(CellUtil.matchingColumn(results.get(1), CELL_WITH_VERSIONS[2]));
361      assertTrue("Optimize should do some optimizations",
362        scannerNoOptimize.optimization.get() == 0);
363    }
364
365    get.setFilter(new ColumnCountGetFilter(2));
366    try (CellWithVersionsStoreScanner scanner =
367        new CellWithVersionsStoreScanner(new Scan(get), this.scanInfo)) {
368      List<Cell> results = new ArrayList<>();
369      while (scanner.next(results)) {
370        continue;
371      }
372      assertEquals(2, results.size());
373      assertTrue(CellUtil.matchingColumn(results.get(0), CELL_WITH_VERSIONS[0]));
374      assertTrue(CellUtil.matchingColumn(results.get(1), CELL_WITH_VERSIONS[2]));
375      assertTrue("Optimize should do some optimizations", scanner.optimization.get() > 0);
376    }
377  }
378
379  /*
380   * Test utility for building a NavigableSet for scanners.
381   * @param strCols
382   * @return
383   */
384  NavigableSet<byte[]> getCols(String ...strCols) {
385    NavigableSet<byte[]> cols = new TreeSet<>(Bytes.BYTES_COMPARATOR);
386    for (String col : strCols) {
387      byte[] bytes = Bytes.toBytes(col);
388      cols.add(bytes);
389    }
390    return cols;
391  }
392
393  @Test
394  public void testFullRowGetDoesNotOverreadWhenRowInsideOneBlock() throws IOException {
395    // Do a Get against row two. Row two is inside a block that starts with row TWO but ends with
396    // row TWO_POINT_TWO. We should read one block only.
397    Get get = new Get(TWO);
398    Scan scan = new Scan(get);
399    try (CellGridStoreScanner scanner = new CellGridStoreScanner(scan, this.scanInfo)) {
400      List<Cell> results = new ArrayList<>();
401      while (scanner.next(results)) {
402        continue;
403      }
404      // Should be four results of column 1 (though there are 5 rows in the CELL_GRID -- the
405      // TWO_POINT_TWO row does not have a a column ONE.
406      assertEquals(4, results.size());
407      // We should have gone the optimize route 5 times totally... an INCLUDE for the four cells
408      // in the row plus the DONE on the end.
409      assertEquals(5, scanner.count.get());
410      assertEquals(1, scanner.memstoreOnlyReads);
411      // For a full row Get, there should be no opportunity for scanner optimization.
412      assertEquals(0, scanner.optimization.get());
413    }
414  }
415
416  @Test
417  public void testFullRowSpansBlocks() throws IOException {
418    // Do a Get against row FOUR. It spans two blocks.
419    Get get = new Get(FOUR);
420    Scan scan = new Scan(get);
421    try (CellGridStoreScanner scanner = new CellGridStoreScanner(scan, this.scanInfo)) {
422      List<Cell> results = new ArrayList<>();
423      while (scanner.next(results)) {
424        continue;
425      }
426      // Should be four results of column 1 (though there are 5 rows in the CELL_GRID -- the
427      // TWO_POINT_TWO row does not have a a column ONE.
428      assertEquals(5, results.size());
429      // We should have gone the optimize route 6 times totally... an INCLUDE for the five cells
430      // in the row plus the DONE on the end.
431      assertEquals(6, scanner.count.get());
432      // For a full row Get, there should be no opportunity for scanner optimization.
433      assertEquals(0, scanner.optimization.get());
434    }
435  }
436
437  /**
438   * Test optimize in StoreScanner. Test that we skip to the next 'block' when we it makes sense
439   * reading the block 'index'.
440   */
441  @Test
442  public void testOptimize() throws IOException {
443    Scan scan = new Scan();
444    // A scan that just gets the first qualifier on each row of the CELL_GRID
445    scan.addColumn(CF, ONE);
446    try (CellGridStoreScanner scanner = new CellGridStoreScanner(scan, this.scanInfo)) {
447      List<Cell> results = new ArrayList<>();
448      while (scanner.next(results)) {
449        continue;
450      }
451      // Should be four results of column 1 (though there are 5 rows in the CELL_GRID -- the
452      // TWO_POINT_TWO row does not have a a column ONE.
453      assertEquals(4, results.size());
454      for (Cell cell: results) {
455        assertTrue(Bytes.equals(ONE, 0, ONE.length,
456            cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()));
457      }
458      assertTrue("Optimize should do some optimizations", scanner.optimization.get() > 0);
459    }
460  }
461
462  /**
463   * Ensure the optimize Scan method in StoreScanner does not get in the way of a Get doing minimum
464   * work... seeking to start of block and then SKIPPING until we find the wanted Cell.
465   * This 'simple' scenario mimics case of all Cells fitting inside a single HFileBlock.
466   * See HBASE-15392. This test is a little cryptic. Takes a bit of staring to figure what it up to.
467   */
468  @Test
469  public void testOptimizeAndGet() throws IOException {
470    // First test a Get of two columns in the row R2. Every Get is a Scan. Get columns named
471    // R2 and R3.
472    Get get = new Get(TWO);
473    get.addColumn(CF, TWO);
474    get.addColumn(CF, THREE);
475    Scan scan = new Scan(get);
476    try (CellGridStoreScanner scanner = new CellGridStoreScanner(scan, this.scanInfo)) {
477      List<Cell> results = new ArrayList<>();
478      // For a Get there should be no more next's after the first call.
479      assertEquals(false, scanner.next(results));
480      // Should be one result only.
481      assertEquals(2, results.size());
482      // And we should have gone through optimize twice only.
483      assertEquals("First qcode is SEEK_NEXT_COL and second INCLUDE_AND_SEEK_NEXT_ROW", 3,
484        scanner.count.get());
485      assertEquals("Memstore Read count should be", 1,
486        scanner.memstoreOnlyReads);
487    }
488  }
489
490  /**
491   * Ensure that optimize does not cause the Get to do more seeking than required. Optimize
492   * (see HBASE-15392) was causing us to seek all Cells in a block when a Get Scan if the next block
493   * index/start key was a different row to the current one. A bug. We'd call next too often
494   * because we had to exhaust all Cells in the current row making us load the next block just to
495   * discard what we read there. This test is a little cryptic. Takes a bit of staring to figure
496   * what it up to.
497   */
498  @Test
499  public void testOptimizeAndGetWithFakedNextBlockIndexStart() throws IOException {
500    // First test a Get of second column in the row R2. Every Get is a Scan. Second column has a
501    // qualifier of R2.
502    Get get = new Get(THREE);
503    get.addColumn(CF, TWO);
504    Scan scan = new Scan(get);
505    try (CellGridStoreScanner scanner = new CellGridStoreScanner(scan, this.scanInfo)) {
506      List<Cell> results = new ArrayList<>();
507      // For a Get there should be no more next's after the first call.
508      assertEquals(false, scanner.next(results));
509      // Should be one result only.
510      assertEquals(1, results.size());
511      // And we should have gone through optimize twice only.
512      assertEquals("First qcode is SEEK_NEXT_COL and second INCLUDE_AND_SEEK_NEXT_ROW", 2,
513        scanner.count.get());
514    }
515  }
516
517  @Test
518  public void testScanTimeRange() throws IOException {
519    String r1 = "R1";
520    // returns only 1 of these 2 even though same timestamp
521    KeyValue [] kvs = new KeyValue[] {
522        create(r1, CF_STR, "a", 1, KeyValue.Type.Put, "dont-care"),
523        create(r1, CF_STR, "a", 2, KeyValue.Type.Put, "dont-care"),
524        create(r1, CF_STR, "a", 3, KeyValue.Type.Put, "dont-care"),
525        create(r1, CF_STR, "a", 4, KeyValue.Type.Put, "dont-care"),
526        create(r1, CF_STR, "a", 5, KeyValue.Type.Put, "dont-care"),
527    };
528    List<KeyValueScanner> scanners = Arrays.<KeyValueScanner>asList(
529      new KeyValueScanner[] {new KeyValueScanFixture(CellComparator.getInstance(), kvs)});
530    Scan scanSpec = new Scan().withStartRow(Bytes.toBytes(r1));
531    scanSpec.setTimeRange(0, 6);
532    scanSpec.readAllVersions();
533    List<Cell> results = null;
534    try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) {
535      results = new ArrayList<>();
536      assertEquals(true, scan.next(results));
537      assertEquals(5, results.size());
538      assertEquals(kvs[kvs.length - 1], results.get(0));
539    }
540    // Scan limited TimeRange
541    scanSpec = new Scan().withStartRow(Bytes.toBytes(r1));
542    scanSpec.setTimeRange(1, 3);
543    scanSpec.readAllVersions();
544    try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) {
545      results = new ArrayList<>();
546      assertEquals(true, scan.next(results));
547      assertEquals(2, results.size());
548    }
549    // Another range.
550    scanSpec = new Scan().withStartRow(Bytes.toBytes(r1));
551    scanSpec.setTimeRange(5, 10);
552    scanSpec.readAllVersions();
553    try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) {
554      results = new ArrayList<>();
555      assertEquals(true, scan.next(results));
556      assertEquals(1, results.size());
557    }
558    // See how TimeRange and Versions interact.
559    // Another range.
560    scanSpec = new Scan().withStartRow(Bytes.toBytes(r1));
561    scanSpec.setTimeRange(0, 10);
562    scanSpec.readVersions(3);
563    try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) {
564      results = new ArrayList<>();
565      assertEquals(true, scan.next(results));
566      assertEquals(3, results.size());
567    }
568  }
569
570  @Test
571  public void testScanSameTimestamp() throws IOException {
572    // returns only 1 of these 2 even though same timestamp
573    KeyValue [] kvs = new KeyValue[] {
574        create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"),
575        create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"),
576    };
577    List<KeyValueScanner> scanners = Arrays.asList(
578      new KeyValueScanner[] {new KeyValueScanFixture(CellComparator.getInstance(), kvs)});
579
580    Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1"));
581    // this only uses maxVersions (default=1) and TimeRange (default=all)
582    try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, null, scanners)) {
583      List<Cell> results = new ArrayList<>();
584      assertEquals(true, scan.next(results));
585      assertEquals(1, results.size());
586      assertEquals(1, scan.memstoreOnlyReads);
587      assertEquals(kvs[0], results.get(0));
588    }
589  }
590
591  @Test
592  public void testNonUserScan() throws IOException {
593    // returns only 1 of these 2 even though same timestamp
594    KeyValue[] kvs = new KeyValue[] { create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"),
595        create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"), };
596    List<KeyValueScanner> scanners = Arrays.asList(
597      new KeyValueScanner[] { new KeyValueScanFixture(CellComparator.getInstance(), kvs) });
598
599    Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1"));
600    // this only uses maxVersions (default=1) and TimeRange (default=all)
601    try (StoreScanner scan =
602        new StoreScanner(scanSpec, scanInfo, null, scanners, ScanType.COMPACT_RETAIN_DELETES)) {
603      List<Cell> results = new ArrayList<>();
604      assertEquals(true, scan.next(results));
605      assertEquals(1, results.size());
606      // the type is not a user scan. so it won't account for the memstore reads
607      assertEquals(0, scan.memstoreOnlyReads);
608      assertEquals(kvs[0], results.get(0));
609    }
610  }
611
612  /*
613   * Test test shows exactly how the matcher's return codes confuses the StoreScanner
614   * and prevent it from doing the right thing.  Seeking once, then nexting twice
615   * should return R1, then R2, but in this case it doesnt.
616   * TODO this comment makes no sense above. Appears to do the right thing.
617   * @throws IOException
618   */
619  @Test
620  public void testWontNextToNext() throws IOException {
621    // build the scan file:
622    KeyValue [] kvs = new KeyValue[] {
623        create("R1", "cf", "a", 2, KeyValue.Type.Put, "dont-care"),
624        create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"),
625        create("R2", "cf", "a", 1, KeyValue.Type.Put, "dont-care")
626    };
627    List<KeyValueScanner> scanners = scanFixture(kvs);
628
629    Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1"));
630    // this only uses maxVersions (default=1) and TimeRange (default=all)
631    try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) {
632      List<Cell> results = new ArrayList<>();
633      scan.next(results);
634      assertEquals(1, results.size());
635      assertEquals(kvs[0], results.get(0));
636      // should be ok...
637      // now scan _next_ again.
638      results.clear();
639      scan.next(results);
640      assertEquals(1, results.size());
641      assertEquals(kvs[2], results.get(0));
642
643      results.clear();
644      scan.next(results);
645      assertEquals(0, results.size());
646    }
647  }
648
649
650  @Test
651  public void testDeleteVersionSameTimestamp() throws IOException {
652    KeyValue [] kvs = new KeyValue [] {
653        create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"),
654        create("R1", "cf", "a", 1, KeyValue.Type.Delete, "dont-care"),
655    };
656    List<KeyValueScanner> scanners = scanFixture(kvs);
657    Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1"));
658    try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) {
659      List<Cell> results = new ArrayList<>();
660      assertFalse(scan.next(results));
661      assertEquals(0, results.size());
662    }
663  }
664
665  /*
666   * Test the case where there is a delete row 'in front of' the next row, the scanner
667   * will move to the next row.
668   */
669  @Test
670  public void testDeletedRowThenGoodRow() throws IOException {
671    KeyValue [] kvs = new KeyValue [] {
672        create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"),
673        create("R1", "cf", "a", 1, KeyValue.Type.Delete, "dont-care"),
674        create("R2", "cf", "a", 20, KeyValue.Type.Put, "dont-care")
675    };
676    List<KeyValueScanner> scanners = scanFixture(kvs);
677    Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1"));
678    try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) {
679      List<Cell> results = new ArrayList<>();
680      assertEquals(true, scan.next(results));
681      assertEquals(0, results.size());
682
683      assertEquals(true, scan.next(results));
684      assertEquals(1, results.size());
685      assertEquals(kvs[2], results.get(0));
686
687      assertEquals(false, scan.next(results));
688    }
689  }
690
691  @Test
692  public void testDeleteVersionMaskingMultiplePuts() throws IOException {
693    long now = System.currentTimeMillis();
694    KeyValue [] kvs1 = new KeyValue[] {
695        create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care"),
696        create("R1", "cf", "a", now, KeyValue.Type.Delete, "dont-care")
697    };
698    KeyValue [] kvs2 = new KeyValue[] {
699        create("R1", "cf", "a", now-500, KeyValue.Type.Put, "dont-care"),
700        create("R1", "cf", "a", now-100, KeyValue.Type.Put, "dont-care"),
701        create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care")
702    };
703    List<KeyValueScanner> scanners = scanFixture(kvs1, kvs2);
704
705    try (StoreScanner scan = new StoreScanner(new Scan().withStartRow(Bytes.toBytes("R1")),
706        scanInfo, getCols("a"), scanners)) {
707      List<Cell> results = new ArrayList<>();
708      // the two put at ts=now will be masked by the 1 delete, and
709      // since the scan default returns 1 version we'll return the newest
710      // key, which is kvs[2], now-100.
711      assertEquals(true, scan.next(results));
712      assertEquals(1, results.size());
713      assertEquals(kvs2[1], results.get(0));
714    }
715  }
716
717  @Test
718  public void testDeleteVersionsMixedAndMultipleVersionReturn() throws IOException {
719    long now = System.currentTimeMillis();
720    KeyValue [] kvs1 = new KeyValue[] {
721        create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care"),
722        create("R1", "cf", "a", now, KeyValue.Type.Delete, "dont-care")
723    };
724    KeyValue [] kvs2 = new KeyValue[] {
725        create("R1", "cf", "a", now-500, KeyValue.Type.Put, "dont-care"),
726        create("R1", "cf", "a", now+500, KeyValue.Type.Put, "dont-care"),
727        create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care"),
728        create("R2", "cf", "z", now, KeyValue.Type.Put, "dont-care")
729    };
730    List<KeyValueScanner> scanners = scanFixture(kvs1, kvs2);
731
732    Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1")).readVersions(2);
733    try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) {
734      List<Cell> results = new ArrayList<>();
735      assertEquals(true, scan.next(results));
736      assertEquals(2, results.size());
737      assertEquals(kvs2[1], results.get(0));
738      assertEquals(kvs2[0], results.get(1));
739    }
740  }
741
742  @Test
743  public void testWildCardOneVersionScan() throws IOException {
744    KeyValue [] kvs = new KeyValue [] {
745        create("R1", "cf", "a", 2, KeyValue.Type.Put, "dont-care"),
746        create("R1", "cf", "b", 1, KeyValue.Type.Put, "dont-care"),
747        create("R1", "cf", "a", 1, KeyValue.Type.DeleteColumn, "dont-care"),
748    };
749    List<KeyValueScanner> scanners = scanFixture(kvs);
750    try (StoreScanner scan =
751        new StoreScanner(new Scan().withStartRow(Bytes.toBytes("R1")), scanInfo, null, scanners)) {
752      List<Cell> results = new ArrayList<>();
753      assertEquals(true, scan.next(results));
754      assertEquals(2, results.size());
755      assertEquals(kvs[0], results.get(0));
756      assertEquals(kvs[1], results.get(1));
757    }
758  }
759
760  @Test
761  public void testWildCardScannerUnderDeletes() throws IOException {
762    KeyValue [] kvs = new KeyValue [] {
763        create("R1", "cf", "a", 2, KeyValue.Type.Put, "dont-care"), // inc
764        // orphaned delete column.
765        create("R1", "cf", "a", 1, KeyValue.Type.DeleteColumn, "dont-care"),
766        // column b
767        create("R1", "cf", "b", 2, KeyValue.Type.Put, "dont-care"), // inc
768        create("R1", "cf", "b", 1, KeyValue.Type.Put, "dont-care"), // inc
769        // column c
770        create("R1", "cf", "c", 10, KeyValue.Type.Delete, "dont-care"),
771        create("R1", "cf", "c", 10, KeyValue.Type.Put, "dont-care"), // no
772        create("R1", "cf", "c", 9, KeyValue.Type.Put, "dont-care"),  // inc
773        // column d
774        create("R1", "cf", "d", 11, KeyValue.Type.Put, "dont-care"), // inc
775        create("R1", "cf", "d", 10, KeyValue.Type.DeleteColumn, "dont-care"),
776        create("R1", "cf", "d", 9, KeyValue.Type.Put, "dont-care"),  // no
777        create("R1", "cf", "d", 8, KeyValue.Type.Put, "dont-care"),  // no
778
779    };
780    List<KeyValueScanner> scanners = scanFixture(kvs);
781    try (StoreScanner scan =
782        new StoreScanner(new Scan().readVersions(2), scanInfo, null, scanners)) {
783      List<Cell> results = new ArrayList<>();
784      assertEquals(true, scan.next(results));
785      assertEquals(5, results.size());
786      assertEquals(kvs[0], results.get(0));
787      assertEquals(kvs[2], results.get(1));
788      assertEquals(kvs[3], results.get(2));
789      assertEquals(kvs[6], results.get(3));
790      assertEquals(kvs[7], results.get(4));
791    }
792  }
793
794  @Test
795  public void testDeleteFamily() throws IOException {
796    KeyValue[] kvs = new KeyValue[] {
797        create("R1", "cf", "a", 100, KeyValue.Type.DeleteFamily, "dont-care"),
798        create("R1", "cf", "b", 11, KeyValue.Type.Put, "dont-care"),
799        create("R1", "cf", "c", 11, KeyValue.Type.Put, "dont-care"),
800        create("R1", "cf", "d", 11, KeyValue.Type.Put, "dont-care"),
801        create("R1", "cf", "e", 11, KeyValue.Type.Put, "dont-care"),
802        create("R1", "cf", "e", 11, KeyValue.Type.DeleteColumn, "dont-care"),
803        create("R1", "cf", "f", 11, KeyValue.Type.Put, "dont-care"),
804        create("R1", "cf", "g", 11, KeyValue.Type.Put, "dont-care"),
805        create("R1", "cf", "g", 11, KeyValue.Type.Delete, "dont-care"),
806        create("R1", "cf", "h", 11, KeyValue.Type.Put, "dont-care"),
807        create("R1", "cf", "i", 11, KeyValue.Type.Put, "dont-care"),
808        create("R2", "cf", "a", 11, KeyValue.Type.Put, "dont-care"),
809    };
810    List<KeyValueScanner> scanners = scanFixture(kvs);
811    try (StoreScanner scan =
812        new StoreScanner(new Scan().readAllVersions(), scanInfo, null, scanners)) {
813      List<Cell> results = new ArrayList<>();
814      assertEquals(true, scan.next(results));
815      assertEquals(0, results.size());
816      assertEquals(true, scan.next(results));
817      assertEquals(1, results.size());
818      assertEquals(kvs[kvs.length - 1], results.get(0));
819
820      assertEquals(false, scan.next(results));
821    }
822  }
823
824  @Test
825  public void testDeleteColumn() throws IOException {
826    KeyValue [] kvs = new KeyValue[] {
827        create("R1", "cf", "a", 10, KeyValue.Type.DeleteColumn, "dont-care"),
828        create("R1", "cf", "a", 9, KeyValue.Type.Delete, "dont-care"),
829        create("R1", "cf", "a", 8, KeyValue.Type.Put, "dont-care"),
830        create("R1", "cf", "b", 5, KeyValue.Type.Put, "dont-care")
831    };
832    List<KeyValueScanner> scanners = scanFixture(kvs);
833    try (StoreScanner scan = new StoreScanner(new Scan(), scanInfo, null, scanners)) {
834      List<Cell> results = new ArrayList<>();
835      assertEquals(true, scan.next(results));
836      assertEquals(1, results.size());
837      assertEquals(kvs[3], results.get(0));
838    }
839  }
840
841  private static final KeyValue[] kvs = new KeyValue[] {
842        create("R1", "cf", "a", 11, KeyValue.Type.Put, "dont-care"),
843        create("R1", "cf", "b", 11, KeyValue.Type.Put, "dont-care"),
844        create("R1", "cf", "c", 11, KeyValue.Type.Put, "dont-care"),
845        create("R1", "cf", "d", 11, KeyValue.Type.Put, "dont-care"),
846        create("R1", "cf", "e", 11, KeyValue.Type.Put, "dont-care"),
847        create("R1", "cf", "f", 11, KeyValue.Type.Put, "dont-care"),
848        create("R1", "cf", "g", 11, KeyValue.Type.Put, "dont-care"),
849        create("R1", "cf", "h", 11, KeyValue.Type.Put, "dont-care"),
850        create("R1", "cf", "i", 11, KeyValue.Type.Put, "dont-care"),
851        create("R2", "cf", "a", 11, KeyValue.Type.Put, "dont-care"),
852    };
853
854  @Test
855  public void testSkipColumn() throws IOException {
856    List<KeyValueScanner> scanners = scanFixture(kvs);
857    try (StoreScanner scan = new StoreScanner(new Scan(), scanInfo, getCols("a", "d"), scanners)) {
858      List<Cell> results = new ArrayList<>();
859      assertEquals(true, scan.next(results));
860      assertEquals(2, results.size());
861      assertEquals(kvs[0], results.get(0));
862      assertEquals(kvs[3], results.get(1));
863      results.clear();
864
865      assertEquals(true, scan.next(results));
866      assertEquals(1, results.size());
867      assertEquals(kvs[kvs.length - 1], results.get(0));
868
869      results.clear();
870      assertEquals(false, scan.next(results));
871    }
872  }
873
874  /*
875   * Test expiration of KeyValues in combination with a configured TTL for
876   * a column family (as should be triggered in a major compaction).
877   */
878  @Test
879  public void testWildCardTtlScan() throws IOException {
880    long now = System.currentTimeMillis();
881    KeyValue [] kvs = new KeyValue[] {
882        create("R1", "cf", "a", now-1000, KeyValue.Type.Put, "dont-care"),
883        create("R1", "cf", "b", now-10, KeyValue.Type.Put, "dont-care"),
884        create("R1", "cf", "c", now-200, KeyValue.Type.Put, "dont-care"),
885        create("R1", "cf", "d", now-10000, KeyValue.Type.Put, "dont-care"),
886        create("R2", "cf", "a", now, KeyValue.Type.Put, "dont-care"),
887        create("R2", "cf", "b", now-10, KeyValue.Type.Put, "dont-care"),
888        create("R2", "cf", "c", now-200, KeyValue.Type.Put, "dont-care"),
889        create("R2", "cf", "c", now-1000, KeyValue.Type.Put, "dont-care")
890    };
891    List<KeyValueScanner> scanners = scanFixture(kvs);
892    Scan scan = new Scan();
893    scan.readVersions(1);
894    ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, 500, KeepDeletedCells.FALSE,
895        HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false);
896    try (StoreScanner scanner = new StoreScanner(scan, scanInfo, null, scanners)) {
897      List<Cell> results = new ArrayList<>();
898      assertEquals(true, scanner.next(results));
899      assertEquals(2, results.size());
900      assertEquals(kvs[1], results.get(0));
901      assertEquals(kvs[2], results.get(1));
902      results.clear();
903
904      assertEquals(true, scanner.next(results));
905      assertEquals(3, results.size());
906      assertEquals(kvs[4], results.get(0));
907      assertEquals(kvs[5], results.get(1));
908      assertEquals(kvs[6], results.get(2));
909      results.clear();
910
911      assertEquals(false, scanner.next(results));
912    }
913  }
914
915  @Test
916  public void testScannerReseekDoesntNPE() throws Exception {
917    List<KeyValueScanner> scanners = scanFixture(kvs);
918    try (StoreScanner scan = new StoreScanner(new Scan(), scanInfo, getCols("a", "d"), scanners)) {
919      // Previously a updateReaders twice in a row would cause an NPE. In test this would also
920      // normally cause an NPE because scan.store is null. So as long as we get through these
921      // two calls we are good and the bug was quashed.
922      scan.updateReaders(Collections.emptyList(), Collections.emptyList());
923      scan.updateReaders(Collections.emptyList(), Collections.emptyList());
924      scan.peek();
925    }
926  }
927
928  @Test @Ignore("this fails, since we don't handle deletions, etc, in peek")
929  public void testPeek() throws Exception {
930    KeyValue[] kvs = new KeyValue [] {
931        create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"),
932        create("R1", "cf", "a", 1, KeyValue.Type.Delete, "dont-care"),
933    };
934    List<KeyValueScanner> scanners = scanFixture(kvs);
935    Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1"));
936    try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) {
937      assertNull(scan.peek());
938    }
939  }
940
941  /**
942   * Ensure that expired delete family markers don't override valid puts
943   */
944  @Test
945  public void testExpiredDeleteFamily() throws Exception {
946    long now = System.currentTimeMillis();
947    KeyValue[] kvs = new KeyValue[] {
948      new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null, now-1000,
949        KeyValue.Type.DeleteFamily),
950      create("R1", "cf", "a", now-10, KeyValue.Type.Put,
951        "dont-care"),
952    };
953    List<KeyValueScanner> scanners = scanFixture(kvs);
954    Scan scan = new Scan();
955    scan.readVersions(1);
956    // scanner with ttl equal to 500
957    ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, 500, KeepDeletedCells.FALSE,
958        HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false);
959    try (StoreScanner scanner = new StoreScanner(scan, scanInfo, null, scanners)) {
960      List<Cell> results = new ArrayList<>();
961      assertEquals(true, scanner.next(results));
962      assertEquals(1, results.size());
963      assertEquals(kvs[1], results.get(0));
964      results.clear();
965
966      assertEquals(false, scanner.next(results));
967    }
968  }
969
970  @Test
971  public void testDeleteMarkerLongevity() throws Exception {
972    try {
973      final long now = System.currentTimeMillis();
974      EnvironmentEdgeManagerTestHelper.injectEdge(new EnvironmentEdge() {
975        @Override
976        public long currentTime() {
977          return now;
978        }
979      });
980      KeyValue[] kvs = new KeyValue[]{
981        /*0*/ new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null,
982        now - 100, KeyValue.Type.DeleteFamily), // live
983        /*1*/ new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null,
984        now - 1000, KeyValue.Type.DeleteFamily), // expired
985        /*2*/ create("R1", "cf", "a", now - 50,
986        KeyValue.Type.Put, "v3"), // live
987        /*3*/ create("R1", "cf", "a", now - 55,
988        KeyValue.Type.Delete, "dontcare"), // live
989        /*4*/ create("R1", "cf", "a", now - 55,
990        KeyValue.Type.Put, "deleted-version v2"), // deleted
991        /*5*/ create("R1", "cf", "a", now - 60,
992        KeyValue.Type.Put, "v1"), // live
993        /*6*/ create("R1", "cf", "a", now - 65,
994        KeyValue.Type.Put, "v0"), // max-version reached
995        /*7*/ create("R1", "cf", "a",
996        now - 100, KeyValue.Type.DeleteColumn, "dont-care"), // max-version
997        /*8*/ create("R1", "cf", "b", now - 600,
998        KeyValue.Type.DeleteColumn, "dont-care"), //expired
999        /*9*/ create("R1", "cf", "b", now - 70,
1000        KeyValue.Type.Put, "v2"), //live
1001        /*10*/ create("R1", "cf", "b", now - 750,
1002        KeyValue.Type.Put, "v1"), //expired
1003        /*11*/ create("R1", "cf", "c", now - 500,
1004        KeyValue.Type.Delete, "dontcare"), //expired
1005        /*12*/ create("R1", "cf", "c", now - 600,
1006        KeyValue.Type.Put, "v1"), //expired
1007        /*13*/ create("R1", "cf", "c", now - 1000,
1008        KeyValue.Type.Delete, "dontcare"), //expired
1009        /*14*/ create("R1", "cf", "d", now - 60,
1010        KeyValue.Type.Put, "expired put"), //live
1011        /*15*/ create("R1", "cf", "d", now - 100,
1012        KeyValue.Type.Delete, "not-expired delete"), //live
1013      };
1014      List<KeyValueScanner> scanners = scanFixture(kvs);
1015      ScanInfo scanInfo = new ScanInfo(CONF, Bytes.toBytes("cf"),
1016        0 /* minVersions */,
1017        2 /* maxVersions */, 500 /* ttl */,
1018        KeepDeletedCells.FALSE /* keepDeletedCells */,
1019        HConstants.DEFAULT_BLOCKSIZE /* block size */,
1020        200, /* timeToPurgeDeletes */
1021        CellComparator.getInstance(), false);
1022      try (StoreScanner scanner =
1023          new StoreScanner(scanInfo, 2, ScanType.COMPACT_DROP_DELETES, scanners)) {
1024        List<Cell> results = new ArrayList<>();
1025        results = new ArrayList<>();
1026        assertEquals(true, scanner.next(results));
1027        assertEquals(kvs[0], results.get(0));
1028        assertEquals(kvs[2], results.get(1));
1029        assertEquals(kvs[3], results.get(2));
1030        assertEquals(kvs[5], results.get(3));
1031        assertEquals(kvs[9], results.get(4));
1032        assertEquals(kvs[14], results.get(5));
1033        assertEquals(kvs[15], results.get(6));
1034        assertEquals(7, results.size());
1035      }
1036    } finally {
1037      EnvironmentEdgeManagerTestHelper.reset();
1038    }
1039  }
1040
1041  @Test
1042  public void testPreadNotEnabledForCompactionStoreScanners() throws Exception {
1043    long now = System.currentTimeMillis();
1044    KeyValue[] kvs = new KeyValue[] {
1045      new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null, now - 1000,
1046        KeyValue.Type.DeleteFamily),
1047      create("R1", "cf", "a", now - 10, KeyValue.Type.Put, "dont-care"), };
1048    List<KeyValueScanner> scanners = scanFixture(kvs);
1049    ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, 500, KeepDeletedCells.FALSE,
1050        HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false);
1051    try (StoreScanner storeScanner = new StoreScanner(scanInfo, -1,
1052      ScanType.COMPACT_RETAIN_DELETES, scanners)) {
1053      assertFalse(storeScanner.isScanUsePread());
1054    }
1055  }
1056
1057  @Test
1058  public void testReadVersionWithRawAndFilter() throws IOException {
1059    ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, Long.MAX_VALUE,
1060            KeepDeletedCells.FALSE, HConstants.DEFAULT_BLOCKSIZE, 0
1061            , CellComparator.getInstance(), false);
1062    KeyValue [] kvs = new KeyValue[] {
1063            create("R1", "cf", "a", 3, KeyValue.Type.Put, "dont-care"),
1064            create("R1", "cf", "a", 2, KeyValue.Type.Put, "dont-care"),
1065            create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care")
1066    };
1067    List<KeyValueScanner> scanners = Arrays.asList(
1068      new KeyValueScanner[]{
1069        new KeyValueScanFixture(CellComparator.getInstance(), kvs)
1070      });
1071
1072    BinaryComparator comp = new BinaryComparator(Bytes.toBytes("a"));
1073    Filter filter = new QualifierFilter(CompareOperator.EQUAL, comp);
1074    Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1")).readVersions(2).setRaw(true);
1075    scanSpec.setFilter(filter);
1076    try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, null, scanners)) {
1077      List<Cell> results = new ArrayList<>();
1078      assertEquals(true, scan.next(results));
1079      assertEquals(2, results.size());
1080    }
1081  }
1082}