001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.apache.hadoop.hbase.KeyValueTestUtil.create;
021import static org.apache.hadoop.hbase.regionserver.KeyValueScanFixture.scanFixture;
022import static org.junit.Assert.assertEquals;
023import static org.junit.Assert.assertFalse;
024import static org.junit.Assert.assertNull;
025import static org.junit.Assert.assertSame;
026import static org.junit.Assert.assertTrue;
027
028import java.io.IOException;
029import java.util.ArrayList;
030import java.util.Arrays;
031import java.util.Collections;
032import java.util.List;
033import java.util.NavigableSet;
034import java.util.TreeSet;
035import java.util.concurrent.atomic.AtomicInteger;
036import org.apache.hadoop.conf.Configuration;
037import org.apache.hadoop.hbase.Cell;
038import org.apache.hadoop.hbase.CellBuilderType;
039import org.apache.hadoop.hbase.CellComparator;
040import org.apache.hadoop.hbase.CellUtil;
041import org.apache.hadoop.hbase.CompareOperator;
042import org.apache.hadoop.hbase.ExtendedCellBuilderFactory;
043import org.apache.hadoop.hbase.HBaseClassTestRule;
044import org.apache.hadoop.hbase.HBaseConfiguration;
045import org.apache.hadoop.hbase.HBaseTestingUtil;
046import org.apache.hadoop.hbase.HConstants;
047import org.apache.hadoop.hbase.KeepDeletedCells;
048import org.apache.hadoop.hbase.KeyValue;
049import org.apache.hadoop.hbase.PrivateCellUtil;
050import org.apache.hadoop.hbase.client.Get;
051import org.apache.hadoop.hbase.client.Scan;
052import org.apache.hadoop.hbase.filter.BinaryComparator;
053import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
054import org.apache.hadoop.hbase.filter.Filter;
055import org.apache.hadoop.hbase.filter.QualifierFilter;
056import org.apache.hadoop.hbase.testclassification.RegionServerTests;
057import org.apache.hadoop.hbase.testclassification.SmallTests;
058import org.apache.hadoop.hbase.util.Bytes;
059import org.apache.hadoop.hbase.util.CollectionBackedScanner;
060import org.apache.hadoop.hbase.util.EnvironmentEdge;
061import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
062import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
063import org.junit.ClassRule;
064import org.junit.Ignore;
065import org.junit.Rule;
066import org.junit.Test;
067import org.junit.experimental.categories.Category;
068import org.junit.rules.TestName;
069import org.slf4j.Logger;
070import org.slf4j.LoggerFactory;
071
072// Can't be small as it plays with EnvironmentEdgeManager
073@Category({ RegionServerTests.class, SmallTests.class })
074public class TestStoreScanner {
075
076  @ClassRule
077  public static final HBaseClassTestRule CLASS_RULE =
078    HBaseClassTestRule.forClass(TestStoreScanner.class);
079
080  private static final Logger LOG = LoggerFactory.getLogger(TestStoreScanner.class);
081  @Rule
082  public TestName name = new TestName();
083  private static final String CF_STR = "cf";
084  private static final byte[] CF = Bytes.toBytes(CF_STR);
085  static Configuration CONF = HBaseConfiguration.create();
086  private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
087  private ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, Integer.MAX_VALUE, Long.MAX_VALUE,
088    KeepDeletedCells.FALSE, HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false);
089
090  /**
091   * From here on down, we have a bunch of defines and specific CELL_GRID of Cells. The CELL_GRID
092   * then has a Scanner that can fake out 'block' transitions. All this elaborate setup is for tests
093   * that ensure we don't overread, and that the {@link StoreScanner} is not overly enthusiastic.
094   */
095  private static final byte[] ZERO = new byte[] { '0' };
096  private static final byte[] ZERO_POINT_ZERO = new byte[] { '0', '.', '0' };
097  private static final byte[] ONE = new byte[] { '1' };
098  private static final byte[] TWO = new byte[] { '2' };
099  private static final byte[] TWO_POINT_TWO = new byte[] { '2', '.', '2' };
100  private static final byte[] THREE = new byte[] { '3' };
101  private static final byte[] FOUR = new byte[] { '4' };
102  private static final byte[] FIVE = new byte[] { '5' };
103  private static final byte[] VALUE = new byte[] { 'v' };
104  private static final int CELL_GRID_BLOCK2_BOUNDARY = 4;
105  private static final int CELL_GRID_BLOCK3_BOUNDARY = 11;
106  private static final int CELL_GRID_BLOCK4_BOUNDARY = 15;
107  private static final int CELL_GRID_BLOCK5_BOUNDARY = 19;
108
109  /**
110   * Five rows by four columns distinguished by column qualifier (column qualifier is one of the
111   * four rows... ONE, TWO, etc.). Exceptions are a weird row after TWO; it is TWO_POINT_TWO. And
112   * then row FOUR has five columns finishing w/ row FIVE having a single column. We will use this
113   * to test scan does the right thing as it we do Gets, StoreScanner#optimize, and what we do on
114   * (faked) block boundaries.
115   */
116  private static final Cell[] CELL_GRID = new Cell[] {
117    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(ONE).setFamily(CF)
118      .setQualifier(ONE).setTimestamp(1L).setType(KeyValue.Type.Put.getCode()).setValue(VALUE)
119      .build(),
120    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(ONE).setFamily(CF)
121      .setQualifier(TWO).setTimestamp(1L).setType(KeyValue.Type.Put.getCode()).setValue(VALUE)
122      .build(),
123    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(ONE).setFamily(CF)
124      .setQualifier(THREE).setTimestamp(1L).setType(KeyValue.Type.Put.getCode()).setValue(VALUE)
125      .build(),
126    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(ONE).setFamily(CF)
127      .setQualifier(FOUR).setTimestamp(1L).setType(KeyValue.Type.Put.getCode()).setValue(VALUE)
128      .build(),
129    // Offset 4 CELL_GRID_BLOCK2_BOUNDARY
130    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(TWO).setFamily(CF)
131      .setQualifier(ONE).setTimestamp(1L).setType(KeyValue.Type.Put.getCode()).setValue(VALUE)
132      .build(),
133    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(TWO).setFamily(CF)
134      .setQualifier(TWO).setTimestamp(1L).setType(KeyValue.Type.Put.getCode()).setValue(VALUE)
135      .build(),
136    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(TWO).setFamily(CF)
137      .setQualifier(THREE).setTimestamp(1L).setType(KeyValue.Type.Put.getCode()).setValue(VALUE)
138      .build(),
139    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(TWO).setFamily(CF)
140      .setQualifier(FOUR).setTimestamp(1L).setType(KeyValue.Type.Put.getCode()).setValue(VALUE)
141      .build(),
142    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(TWO_POINT_TWO).setFamily(CF)
143      .setQualifier(ZERO).setTimestamp(1L).setType(KeyValue.Type.Put.getCode()).setValue(VALUE)
144      .build(),
145    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(TWO_POINT_TWO).setFamily(CF)
146      .setQualifier(ZERO_POINT_ZERO).setTimestamp(1L).setType(KeyValue.Type.Put.getCode())
147      .setValue(VALUE).build(),
148    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(TWO_POINT_TWO).setFamily(CF)
149      .setQualifier(FIVE).setTimestamp(1L).setType(KeyValue.Type.Put.getCode()).setValue(VALUE)
150      .build(),
151    // Offset 11! CELL_GRID_BLOCK3_BOUNDARY
152    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(THREE).setFamily(CF)
153      .setQualifier(ONE).setTimestamp(1L).setType(KeyValue.Type.Put.getCode()).setValue(VALUE)
154      .build(),
155    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(THREE).setFamily(CF)
156      .setQualifier(TWO).setTimestamp(1L).setType(KeyValue.Type.Put.getCode()).setValue(VALUE)
157      .build(),
158    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(THREE).setFamily(CF)
159      .setQualifier(THREE).setTimestamp(1L).setType(KeyValue.Type.Put.getCode()).setValue(VALUE)
160      .build(),
161    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(THREE).setFamily(CF)
162      .setQualifier(FOUR).setTimestamp(1L).setType(KeyValue.Type.Put.getCode()).setValue(VALUE)
163      .build(),
164    // Offset 15 CELL_GRID_BLOCK4_BOUNDARY
165    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(FOUR).setFamily(CF)
166      .setQualifier(ONE).setTimestamp(1L).setType(KeyValue.Type.Put.getCode()).setValue(VALUE)
167      .build(),
168    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(FOUR).setFamily(CF)
169      .setQualifier(TWO).setTimestamp(1L).setType(KeyValue.Type.Put.getCode()).setValue(VALUE)
170      .build(),
171    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(FOUR).setFamily(CF)
172      .setQualifier(THREE).setTimestamp(1L).setType(KeyValue.Type.Put.getCode()).setValue(VALUE)
173      .build(),
174    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(FOUR).setFamily(CF)
175      .setQualifier(FOUR).setTimestamp(1L).setType(KeyValue.Type.Put.getCode()).setValue(VALUE)
176      .build(),
177    // Offset 19 CELL_GRID_BLOCK5_BOUNDARY
178    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(FOUR).setFamily(CF)
179      .setQualifier(FIVE).setTimestamp(1L).setType(KeyValue.Type.Put.getCode()).setValue(VALUE)
180      .build(),
181    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(FIVE).setFamily(CF)
182      .setQualifier(ZERO).setTimestamp(1L).setType(KeyValue.Type.Put.getCode()).setValue(VALUE)
183      .build(), };
184
185  private static class KeyValueHeapWithCount extends KeyValueHeap {
186
187    final AtomicInteger count;
188
189    public KeyValueHeapWithCount(List<? extends KeyValueScanner> scanners,
190      CellComparator comparator, AtomicInteger count) throws IOException {
191      super(scanners, comparator);
192      this.count = count;
193    }
194
195    @Override
196    public Cell peek() {
197      this.count.incrementAndGet();
198      return super.peek();
199    }
200  }
201
202  /**
203   * A StoreScanner for our CELL_GRID above. Fakes the block transitions. Does counts of calls to
204   * optimize and counts of when optimize actually did an optimize.
205   */
206  private static class CellGridStoreScanner extends StoreScanner {
207    // Count of how often optimize is called and of how often it does an optimize.
208    AtomicInteger count;
209    final AtomicInteger optimization = new AtomicInteger(0);
210
211    CellGridStoreScanner(final Scan scan, ScanInfo scanInfo) throws IOException {
212      super(scan, scanInfo, scan.getFamilyMap().get(CF),
213        Arrays.<KeyValueScanner> asList(new KeyValueScanner[] {
214          new KeyValueScanFixture(CellComparator.getInstance(), CELL_GRID) }));
215    }
216
217    @Override
218    protected void resetKVHeap(List<? extends KeyValueScanner> scanners, CellComparator comparator)
219      throws IOException {
220      if (count == null) {
221        count = new AtomicInteger(0);
222      }
223      heap = newKVHeap(scanners, comparator);
224    }
225
226    @Override
227    protected KeyValueHeap newKVHeap(List<? extends KeyValueScanner> scanners,
228      CellComparator comparator) throws IOException {
229      return new KeyValueHeapWithCount(scanners, comparator, count);
230    }
231
232    @Override
233    protected boolean trySkipToNextRow(Cell cell) throws IOException {
234      boolean optimized = super.trySkipToNextRow(cell);
235      LOG.info("Cell=" + cell + ", nextIndex=" + CellUtil.toString(getNextIndexedKey(), false)
236        + ", optimized=" + optimized);
237      if (optimized) {
238        optimization.incrementAndGet();
239      }
240      return optimized;
241    }
242
243    @Override
244    protected boolean trySkipToNextColumn(Cell cell) throws IOException {
245      boolean optimized = super.trySkipToNextColumn(cell);
246      LOG.info("Cell=" + cell + ", nextIndex=" + CellUtil.toString(getNextIndexedKey(), false)
247        + ", optimized=" + optimized);
248      if (optimized) {
249        optimization.incrementAndGet();
250      }
251      return optimized;
252    }
253
254    @Override
255    public Cell getNextIndexedKey() {
256      // Fake block boundaries by having index of next block change as we go through scan.
257      return count.get() > CELL_GRID_BLOCK4_BOUNDARY
258        ? PrivateCellUtil.createFirstOnRow(CELL_GRID[CELL_GRID_BLOCK5_BOUNDARY])
259        : count.get() > CELL_GRID_BLOCK3_BOUNDARY
260          ? PrivateCellUtil.createFirstOnRow(CELL_GRID[CELL_GRID_BLOCK4_BOUNDARY])
261        : count.get() > CELL_GRID_BLOCK2_BOUNDARY
262          ? PrivateCellUtil.createFirstOnRow(CELL_GRID[CELL_GRID_BLOCK3_BOUNDARY])
263        : PrivateCellUtil.createFirstOnRow(CELL_GRID[CELL_GRID_BLOCK2_BOUNDARY]);
264    }
265  }
266
267  private static final int CELL_WITH_VERSIONS_BLOCK2_BOUNDARY = 4;
268
269  private static final Cell[] CELL_WITH_VERSIONS = new Cell[] {
270    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(ONE).setFamily(CF)
271      .setQualifier(ONE).setTimestamp(2L).setType(KeyValue.Type.Put.getCode()).setValue(VALUE)
272      .build(),
273    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(ONE).setFamily(CF)
274      .setQualifier(ONE).setTimestamp(1L).setType(KeyValue.Type.Put.getCode()).setValue(VALUE)
275      .build(),
276    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(ONE).setFamily(CF)
277      .setQualifier(TWO).setTimestamp(2L).setType(KeyValue.Type.Put.getCode()).setValue(VALUE)
278      .build(),
279    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(ONE).setFamily(CF)
280      .setQualifier(TWO).setTimestamp(1L).setType(KeyValue.Type.Put.getCode()).setValue(VALUE)
281      .build(),
282    // Offset 4 CELL_WITH_VERSIONS_BLOCK2_BOUNDARY
283    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(TWO).setFamily(CF)
284      .setQualifier(ONE).setTimestamp(1L).setType(KeyValue.Type.Put.getCode()).setValue(VALUE)
285      .build(),
286    ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(TWO).setFamily(CF)
287      .setQualifier(TWO).setTimestamp(1L).setType(KeyValue.Type.Put.getCode()).setValue(VALUE)
288      .build(), };
289
290  private static class CellWithVersionsStoreScanner extends StoreScanner {
291    // Count of how often optimize is called and of how often it does an optimize.
292    final AtomicInteger optimization = new AtomicInteger(0);
293
294    CellWithVersionsStoreScanner(final Scan scan, ScanInfo scanInfo) throws IOException {
295      super(scan, scanInfo, scan.getFamilyMap().get(CF),
296        Arrays.<KeyValueScanner> asList(new KeyValueScanner[] {
297          new KeyValueScanFixture(CellComparator.getInstance(), CELL_WITH_VERSIONS) }));
298    }
299
300    @Override
301    protected boolean trySkipToNextColumn(Cell cell) throws IOException {
302      boolean optimized = super.trySkipToNextColumn(cell);
303      LOG.info("Cell=" + cell + ", nextIndex=" + CellUtil.toString(getNextIndexedKey(), false)
304        + ", optimized=" + optimized);
305      if (optimized) {
306        optimization.incrementAndGet();
307      }
308      return optimized;
309    }
310
311    @Override
312    public Cell getNextIndexedKey() {
313      // Fake block boundaries by having index of next block change as we go through scan.
314      return PrivateCellUtil
315        .createFirstOnRow(CELL_WITH_VERSIONS[CELL_WITH_VERSIONS_BLOCK2_BOUNDARY]);
316    }
317  }
318
319  private static class CellWithVersionsNoOptimizeStoreScanner extends StoreScanner {
320    // Count of how often optimize is called and of how often it does an optimize.
321    final AtomicInteger optimization = new AtomicInteger(0);
322
323    CellWithVersionsNoOptimizeStoreScanner(Scan scan, ScanInfo scanInfo) throws IOException {
324      super(scan, scanInfo, scan.getFamilyMap().get(CF),
325        Arrays.<KeyValueScanner> asList(new KeyValueScanner[] {
326          new KeyValueScanFixture(CellComparator.getInstance(), CELL_WITH_VERSIONS) }));
327    }
328
329    @Override
330    protected boolean trySkipToNextColumn(Cell cell) throws IOException {
331      boolean optimized = super.trySkipToNextColumn(cell);
332      LOG.info("Cell=" + cell + ", nextIndex=" + CellUtil.toString(getNextIndexedKey(), false)
333        + ", optimized=" + optimized);
334      if (optimized) {
335        optimization.incrementAndGet();
336      }
337      return optimized;
338    }
339
340    @Override
341    public Cell getNextIndexedKey() {
342      return null;
343    }
344  }
345
346  @Test
347  public void testWithColumnCountGetFilter() throws Exception {
348    Get get = new Get(ONE);
349    get.readAllVersions();
350    get.addFamily(CF);
351    get.setFilter(new ColumnCountGetFilter(2));
352
353    try (CellWithVersionsNoOptimizeStoreScanner scannerNoOptimize =
354      new CellWithVersionsNoOptimizeStoreScanner(new Scan(get), this.scanInfo)) {
355      List<Cell> results = new ArrayList<>();
356      while (scannerNoOptimize.next(results)) {
357        continue;
358      }
359      assertEquals(2, results.size());
360      assertTrue(CellUtil.matchingColumn(results.get(0), CELL_WITH_VERSIONS[0]));
361      assertTrue(CellUtil.matchingColumn(results.get(1), CELL_WITH_VERSIONS[2]));
362      assertTrue("Optimize should do some optimizations",
363        scannerNoOptimize.optimization.get() == 0);
364    }
365
366    get.setFilter(new ColumnCountGetFilter(2));
367    try (CellWithVersionsStoreScanner scanner =
368      new CellWithVersionsStoreScanner(new Scan(get), this.scanInfo)) {
369      List<Cell> results = new ArrayList<>();
370      while (scanner.next(results)) {
371        continue;
372      }
373      assertEquals(2, results.size());
374      assertTrue(CellUtil.matchingColumn(results.get(0), CELL_WITH_VERSIONS[0]));
375      assertTrue(CellUtil.matchingColumn(results.get(1), CELL_WITH_VERSIONS[2]));
376      assertTrue("Optimize should do some optimizations", scanner.optimization.get() > 0);
377    }
378  }
379
380  /*
381   * Test utility for building a NavigableSet for scanners. nn
382   */
383  NavigableSet<byte[]> getCols(String... strCols) {
384    NavigableSet<byte[]> cols = new TreeSet<>(Bytes.BYTES_COMPARATOR);
385    for (String col : strCols) {
386      byte[] bytes = Bytes.toBytes(col);
387      cols.add(bytes);
388    }
389    return cols;
390  }
391
392  @Test
393  public void testFullRowGetDoesNotOverreadWhenRowInsideOneBlock() throws IOException {
394    // Do a Get against row two. Row two is inside a block that starts with row TWO but ends with
395    // row TWO_POINT_TWO. We should read one block only.
396    Get get = new Get(TWO);
397    Scan scan = new Scan(get);
398    try (CellGridStoreScanner scanner = new CellGridStoreScanner(scan, this.scanInfo)) {
399      List<Cell> results = new ArrayList<>();
400      while (scanner.next(results)) {
401        continue;
402      }
403      // Should be four results of column 1 (though there are 5 rows in the CELL_GRID -- the
404      // TWO_POINT_TWO row does not have a a column ONE.
405      assertEquals(4, results.size());
406      // We should have gone the optimize route 5 times totally... an INCLUDE for the four cells
407      // in the row plus the DONE on the end.
408      assertEquals(5, scanner.count.get());
409      assertEquals(1, scanner.memstoreOnlyReads);
410      // For a full row Get, there should be no opportunity for scanner optimization.
411      assertEquals(0, scanner.optimization.get());
412    }
413  }
414
415  @Test
416  public void testFullRowSpansBlocks() throws IOException {
417    // Do a Get against row FOUR. It spans two blocks.
418    Get get = new Get(FOUR);
419    Scan scan = new Scan(get);
420    try (CellGridStoreScanner scanner = new CellGridStoreScanner(scan, this.scanInfo)) {
421      List<Cell> results = new ArrayList<>();
422      while (scanner.next(results)) {
423        continue;
424      }
425      // Should be four results of column 1 (though there are 5 rows in the CELL_GRID -- the
426      // TWO_POINT_TWO row does not have a a column ONE.
427      assertEquals(5, results.size());
428      // We should have gone the optimize route 6 times totally... an INCLUDE for the five cells
429      // in the row plus the DONE on the end.
430      assertEquals(6, scanner.count.get());
431      // For a full row Get, there should be no opportunity for scanner optimization.
432      assertEquals(0, scanner.optimization.get());
433    }
434  }
435
436  /**
437   * Test optimize in StoreScanner. Test that we skip to the next 'block' when we it makes sense
438   * reading the block 'index'.
439   */
440  @Test
441  public void testOptimize() throws IOException {
442    Scan scan = new Scan();
443    // A scan that just gets the first qualifier on each row of the CELL_GRID
444    scan.addColumn(CF, ONE);
445    try (CellGridStoreScanner scanner = new CellGridStoreScanner(scan, this.scanInfo)) {
446      List<Cell> results = new ArrayList<>();
447      while (scanner.next(results)) {
448        continue;
449      }
450      // Should be four results of column 1 (though there are 5 rows in the CELL_GRID -- the
451      // TWO_POINT_TWO row does not have a a column ONE.
452      assertEquals(4, results.size());
453      for (Cell cell : results) {
454        assertTrue(Bytes.equals(ONE, 0, ONE.length, cell.getQualifierArray(),
455          cell.getQualifierOffset(), cell.getQualifierLength()));
456      }
457      assertTrue("Optimize should do some optimizations", scanner.optimization.get() > 0);
458    }
459  }
460
461  /**
462   * Ensure the optimize Scan method in StoreScanner does not get in the way of a Get doing minimum
463   * work... seeking to start of block and then SKIPPING until we find the wanted Cell. This
464   * 'simple' scenario mimics case of all Cells fitting inside a single HFileBlock. See HBASE-15392.
465   * This test is a little cryptic. Takes a bit of staring to figure what it up to.
466   */
467  @Test
468  public void testOptimizeAndGet() throws IOException {
469    // First test a Get of two columns in the row R2. Every Get is a Scan. Get columns named
470    // R2 and R3.
471    Get get = new Get(TWO);
472    get.addColumn(CF, TWO);
473    get.addColumn(CF, THREE);
474    Scan scan = new Scan(get);
475    try (CellGridStoreScanner scanner = new CellGridStoreScanner(scan, this.scanInfo)) {
476      List<Cell> results = new ArrayList<>();
477      // For a Get there should be no more next's after the first call.
478      assertEquals(false, scanner.next(results));
479      // Should be one result only.
480      assertEquals(2, results.size());
481      // And we should have gone through optimize twice only.
482      assertEquals("First qcode is SEEK_NEXT_COL and second INCLUDE_AND_SEEK_NEXT_ROW", 3,
483        scanner.count.get());
484      assertEquals("Memstore Read count should be", 1, scanner.memstoreOnlyReads);
485    }
486  }
487
488  /**
489   * Ensure that optimize does not cause the Get to do more seeking than required. Optimize (see
490   * HBASE-15392) was causing us to seek all Cells in a block when a Get Scan if the next block
491   * index/start key was a different row to the current one. A bug. We'd call next too often because
492   * we had to exhaust all Cells in the current row making us load the next block just to discard
493   * what we read there. This test is a little cryptic. Takes a bit of staring to figure what it up
494   * to.
495   */
496  @Test
497  public void testOptimizeAndGetWithFakedNextBlockIndexStart() throws IOException {
498    // First test a Get of second column in the row R2. Every Get is a Scan. Second column has a
499    // qualifier of R2.
500    Get get = new Get(THREE);
501    get.addColumn(CF, TWO);
502    Scan scan = new Scan(get);
503    try (CellGridStoreScanner scanner = new CellGridStoreScanner(scan, this.scanInfo)) {
504      List<Cell> results = new ArrayList<>();
505      // For a Get there should be no more next's after the first call.
506      assertEquals(false, scanner.next(results));
507      // Should be one result only.
508      assertEquals(1, results.size());
509      // And we should have gone through optimize twice only.
510      assertEquals("First qcode is SEEK_NEXT_COL and second INCLUDE_AND_SEEK_NEXT_ROW", 2,
511        scanner.count.get());
512    }
513  }
514
515  @Test
516  public void testScanTimeRange() throws IOException {
517    String r1 = "R1";
518    // returns only 1 of these 2 even though same timestamp
519    KeyValue[] kvs = new KeyValue[] { create(r1, CF_STR, "a", 1, KeyValue.Type.Put, "dont-care"),
520      create(r1, CF_STR, "a", 2, KeyValue.Type.Put, "dont-care"),
521      create(r1, CF_STR, "a", 3, KeyValue.Type.Put, "dont-care"),
522      create(r1, CF_STR, "a", 4, KeyValue.Type.Put, "dont-care"),
523      create(r1, CF_STR, "a", 5, KeyValue.Type.Put, "dont-care"), };
524    List<KeyValueScanner> scanners = Arrays.<KeyValueScanner> asList(
525      new KeyValueScanner[] { new KeyValueScanFixture(CellComparator.getInstance(), kvs) });
526    Scan scanSpec = new Scan().withStartRow(Bytes.toBytes(r1));
527    scanSpec.setTimeRange(0, 6);
528    scanSpec.readAllVersions();
529    List<Cell> results = null;
530    try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) {
531      results = new ArrayList<>();
532      assertEquals(true, scan.next(results));
533      assertEquals(5, results.size());
534      assertEquals(kvs[kvs.length - 1], results.get(0));
535    }
536    // Scan limited TimeRange
537    scanSpec = new Scan().withStartRow(Bytes.toBytes(r1));
538    scanSpec.setTimeRange(1, 3);
539    scanSpec.readAllVersions();
540    try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) {
541      results = new ArrayList<>();
542      assertEquals(true, scan.next(results));
543      assertEquals(2, results.size());
544    }
545    // Another range.
546    scanSpec = new Scan().withStartRow(Bytes.toBytes(r1));
547    scanSpec.setTimeRange(5, 10);
548    scanSpec.readAllVersions();
549    try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) {
550      results = new ArrayList<>();
551      assertEquals(true, scan.next(results));
552      assertEquals(1, results.size());
553    }
554    // See how TimeRange and Versions interact.
555    // Another range.
556    scanSpec = new Scan().withStartRow(Bytes.toBytes(r1));
557    scanSpec.setTimeRange(0, 10);
558    scanSpec.readVersions(3);
559    try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) {
560      results = new ArrayList<>();
561      assertEquals(true, scan.next(results));
562      assertEquals(3, results.size());
563    }
564  }
565
566  @Test
567  public void testScanSameTimestamp() throws IOException {
568    // returns only 1 of these 2 even though same timestamp
569    KeyValue[] kvs = new KeyValue[] { create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"),
570      create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"), };
571    List<KeyValueScanner> scanners = Arrays
572      .asList(new KeyValueScanner[] { new KeyValueScanFixture(CellComparator.getInstance(), kvs) });
573
574    Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1"));
575    // this only uses maxVersions (default=1) and TimeRange (default=all)
576    try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, null, scanners)) {
577      List<Cell> results = new ArrayList<>();
578      assertEquals(true, scan.next(results));
579      assertEquals(1, results.size());
580      assertEquals(1, scan.memstoreOnlyReads);
581      assertEquals(kvs[0], results.get(0));
582    }
583  }
584
585  @Test
586  public void testNonUserScan() throws IOException {
587    // returns only 1 of these 2 even though same timestamp
588    KeyValue[] kvs = new KeyValue[] { create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"),
589      create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"), };
590    List<KeyValueScanner> scanners = Arrays
591      .asList(new KeyValueScanner[] { new KeyValueScanFixture(CellComparator.getInstance(), kvs) });
592
593    Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1"));
594    // this only uses maxVersions (default=1) and TimeRange (default=all)
595    try (StoreScanner scan =
596      new StoreScanner(scanSpec, scanInfo, null, scanners, ScanType.COMPACT_RETAIN_DELETES)) {
597      List<Cell> results = new ArrayList<>();
598      assertEquals(true, scan.next(results));
599      assertEquals(1, results.size());
600      // the type is not a user scan. so it won't account for the memstore reads
601      assertEquals(0, scan.memstoreOnlyReads);
602      assertEquals(kvs[0], results.get(0));
603    }
604  }
605
606  /*
607   * Test test shows exactly how the matcher's return codes confuses the StoreScanner and prevent it
608   * from doing the right thing. Seeking once, then nexting twice should return R1, then R2, but in
609   * this case it doesnt. TODO this comment makes no sense above. Appears to do the right thing. n
610   */
611  @Test
612  public void testWontNextToNext() throws IOException {
613    // build the scan file:
614    KeyValue[] kvs = new KeyValue[] { create("R1", "cf", "a", 2, KeyValue.Type.Put, "dont-care"),
615      create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"),
616      create("R2", "cf", "a", 1, KeyValue.Type.Put, "dont-care") };
617    List<KeyValueScanner> scanners = scanFixture(kvs);
618
619    Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1"));
620    // this only uses maxVersions (default=1) and TimeRange (default=all)
621    try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) {
622      List<Cell> results = new ArrayList<>();
623      scan.next(results);
624      assertEquals(1, results.size());
625      assertEquals(kvs[0], results.get(0));
626      // should be ok...
627      // now scan _next_ again.
628      results.clear();
629      scan.next(results);
630      assertEquals(1, results.size());
631      assertEquals(kvs[2], results.get(0));
632
633      results.clear();
634      scan.next(results);
635      assertEquals(0, results.size());
636    }
637  }
638
639  @Test
640  public void testDeleteVersionSameTimestamp() throws IOException {
641    KeyValue[] kvs = new KeyValue[] { create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"),
642      create("R1", "cf", "a", 1, KeyValue.Type.Delete, "dont-care"), };
643    List<KeyValueScanner> scanners = scanFixture(kvs);
644    Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1"));
645    try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) {
646      List<Cell> results = new ArrayList<>();
647      assertFalse(scan.next(results));
648      assertEquals(0, results.size());
649    }
650  }
651
652  /*
653   * Test the case where there is a delete row 'in front of' the next row, the scanner will move to
654   * the next row.
655   */
656  @Test
657  public void testDeletedRowThenGoodRow() throws IOException {
658    KeyValue[] kvs = new KeyValue[] { create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"),
659      create("R1", "cf", "a", 1, KeyValue.Type.Delete, "dont-care"),
660      create("R2", "cf", "a", 20, KeyValue.Type.Put, "dont-care") };
661    List<KeyValueScanner> scanners = scanFixture(kvs);
662    Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1"));
663    try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) {
664      List<Cell> results = new ArrayList<>();
665      assertEquals(true, scan.next(results));
666      assertEquals(0, results.size());
667
668      assertEquals(true, scan.next(results));
669      assertEquals(1, results.size());
670      assertEquals(kvs[2], results.get(0));
671
672      assertEquals(false, scan.next(results));
673    }
674  }
675
676  @Test
677  public void testDeleteVersionMaskingMultiplePuts() throws IOException {
678    long now = EnvironmentEdgeManager.currentTime();
679    KeyValue[] kvs1 = new KeyValue[] { create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care"),
680      create("R1", "cf", "a", now, KeyValue.Type.Delete, "dont-care") };
681    KeyValue[] kvs2 =
682      new KeyValue[] { create("R1", "cf", "a", now - 500, KeyValue.Type.Put, "dont-care"),
683        create("R1", "cf", "a", now - 100, KeyValue.Type.Put, "dont-care"),
684        create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care") };
685    List<KeyValueScanner> scanners = scanFixture(kvs1, kvs2);
686
687    try (StoreScanner scan = new StoreScanner(new Scan().withStartRow(Bytes.toBytes("R1")),
688      scanInfo, getCols("a"), scanners)) {
689      List<Cell> results = new ArrayList<>();
690      // the two put at ts=now will be masked by the 1 delete, and
691      // since the scan default returns 1 version we'll return the newest
692      // key, which is kvs[2], now-100.
693      assertEquals(true, scan.next(results));
694      assertEquals(1, results.size());
695      assertEquals(kvs2[1], results.get(0));
696    }
697  }
698
699  @Test
700  public void testDeleteVersionsMixedAndMultipleVersionReturn() throws IOException {
701    long now = EnvironmentEdgeManager.currentTime();
702    KeyValue[] kvs1 = new KeyValue[] { create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care"),
703      create("R1", "cf", "a", now, KeyValue.Type.Delete, "dont-care") };
704    KeyValue[] kvs2 =
705      new KeyValue[] { create("R1", "cf", "a", now - 500, KeyValue.Type.Put, "dont-care"),
706        create("R1", "cf", "a", now + 500, KeyValue.Type.Put, "dont-care"),
707        create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care"),
708        create("R2", "cf", "z", now, KeyValue.Type.Put, "dont-care") };
709    List<KeyValueScanner> scanners = scanFixture(kvs1, kvs2);
710
711    Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1")).readVersions(2);
712    try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) {
713      List<Cell> results = new ArrayList<>();
714      assertEquals(true, scan.next(results));
715      assertEquals(2, results.size());
716      assertEquals(kvs2[1], results.get(0));
717      assertEquals(kvs2[0], results.get(1));
718    }
719  }
720
721  @Test
722  public void testWildCardOneVersionScan() throws IOException {
723    KeyValue[] kvs = new KeyValue[] { create("R1", "cf", "a", 2, KeyValue.Type.Put, "dont-care"),
724      create("R1", "cf", "b", 1, KeyValue.Type.Put, "dont-care"),
725      create("R1", "cf", "a", 1, KeyValue.Type.DeleteColumn, "dont-care"), };
726    List<KeyValueScanner> scanners = scanFixture(kvs);
727    try (StoreScanner scan =
728      new StoreScanner(new Scan().withStartRow(Bytes.toBytes("R1")), scanInfo, null, scanners)) {
729      List<Cell> results = new ArrayList<>();
730      assertEquals(true, scan.next(results));
731      assertEquals(2, results.size());
732      assertEquals(kvs[0], results.get(0));
733      assertEquals(kvs[1], results.get(1));
734    }
735  }
736
737  @Test
738  public void testWildCardScannerUnderDeletes() throws IOException {
739    KeyValue[] kvs = new KeyValue[] {
740      // inc
741      create("R1", "cf", "a", 2, KeyValue.Type.Put, "dont-care"),
742      // orphaned delete column.
743      create("R1", "cf", "a", 1, KeyValue.Type.DeleteColumn, "dont-care"),
744      // column b
745      // inc
746      create("R1", "cf", "b", 2, KeyValue.Type.Put, "dont-care"),
747      // inc
748      create("R1", "cf", "b", 1, KeyValue.Type.Put, "dont-care"),
749      // column c
750      create("R1", "cf", "c", 10, KeyValue.Type.Delete, "dont-care"),
751      // no
752      create("R1", "cf", "c", 10, KeyValue.Type.Put, "dont-care"),
753      // inc
754      create("R1", "cf", "c", 9, KeyValue.Type.Put, "dont-care"),
755      // column d
756      // inc
757      create("R1", "cf", "d", 11, KeyValue.Type.Put, "dont-care"),
758      create("R1", "cf", "d", 10, KeyValue.Type.DeleteColumn, "dont-care"),
759      // no
760      create("R1", "cf", "d", 9, KeyValue.Type.Put, "dont-care"),
761      // no
762      create("R1", "cf", "d", 8, KeyValue.Type.Put, "dont-care"),
763
764    };
765    List<KeyValueScanner> scanners = scanFixture(kvs);
766    try (
767      StoreScanner scan = new StoreScanner(new Scan().readVersions(2), scanInfo, null, scanners)) {
768      List<Cell> results = new ArrayList<>();
769      assertEquals(true, scan.next(results));
770      assertEquals(5, results.size());
771      assertEquals(kvs[0], results.get(0));
772      assertEquals(kvs[2], results.get(1));
773      assertEquals(kvs[3], results.get(2));
774      assertEquals(kvs[6], results.get(3));
775      assertEquals(kvs[7], results.get(4));
776    }
777  }
778
779  @Test
780  public void testDeleteFamily() throws IOException {
781    KeyValue[] kvs =
782      new KeyValue[] { create("R1", "cf", "a", 100, KeyValue.Type.DeleteFamily, "dont-care"),
783        create("R1", "cf", "b", 11, KeyValue.Type.Put, "dont-care"),
784        create("R1", "cf", "c", 11, KeyValue.Type.Put, "dont-care"),
785        create("R1", "cf", "d", 11, KeyValue.Type.Put, "dont-care"),
786        create("R1", "cf", "e", 11, KeyValue.Type.Put, "dont-care"),
787        create("R1", "cf", "e", 11, KeyValue.Type.DeleteColumn, "dont-care"),
788        create("R1", "cf", "f", 11, KeyValue.Type.Put, "dont-care"),
789        create("R1", "cf", "g", 11, KeyValue.Type.Put, "dont-care"),
790        create("R1", "cf", "g", 11, KeyValue.Type.Delete, "dont-care"),
791        create("R1", "cf", "h", 11, KeyValue.Type.Put, "dont-care"),
792        create("R1", "cf", "i", 11, KeyValue.Type.Put, "dont-care"),
793        create("R2", "cf", "a", 11, KeyValue.Type.Put, "dont-care"), };
794    List<KeyValueScanner> scanners = scanFixture(kvs);
795    try (StoreScanner scan =
796      new StoreScanner(new Scan().readAllVersions(), scanInfo, null, scanners)) {
797      List<Cell> results = new ArrayList<>();
798      assertEquals(true, scan.next(results));
799      assertEquals(0, results.size());
800      assertEquals(true, scan.next(results));
801      assertEquals(1, results.size());
802      assertEquals(kvs[kvs.length - 1], results.get(0));
803
804      assertEquals(false, scan.next(results));
805    }
806  }
807
808  @Test
809  public void testDeleteColumn() throws IOException {
810    KeyValue[] kvs =
811      new KeyValue[] { create("R1", "cf", "a", 10, KeyValue.Type.DeleteColumn, "dont-care"),
812        create("R1", "cf", "a", 9, KeyValue.Type.Delete, "dont-care"),
813        create("R1", "cf", "a", 8, KeyValue.Type.Put, "dont-care"),
814        create("R1", "cf", "b", 5, KeyValue.Type.Put, "dont-care") };
815    List<KeyValueScanner> scanners = scanFixture(kvs);
816    try (StoreScanner scan = new StoreScanner(new Scan(), scanInfo, null, scanners)) {
817      List<Cell> results = new ArrayList<>();
818      assertEquals(true, scan.next(results));
819      assertEquals(1, results.size());
820      assertEquals(kvs[3], results.get(0));
821    }
822  }
823
824  private static final KeyValue[] kvs =
825    new KeyValue[] { create("R1", "cf", "a", 11, KeyValue.Type.Put, "dont-care"),
826      create("R1", "cf", "b", 11, KeyValue.Type.Put, "dont-care"),
827      create("R1", "cf", "c", 11, KeyValue.Type.Put, "dont-care"),
828      create("R1", "cf", "d", 11, KeyValue.Type.Put, "dont-care"),
829      create("R1", "cf", "e", 11, KeyValue.Type.Put, "dont-care"),
830      create("R1", "cf", "f", 11, KeyValue.Type.Put, "dont-care"),
831      create("R1", "cf", "g", 11, KeyValue.Type.Put, "dont-care"),
832      create("R1", "cf", "h", 11, KeyValue.Type.Put, "dont-care"),
833      create("R1", "cf", "i", 11, KeyValue.Type.Put, "dont-care"),
834      create("R2", "cf", "a", 11, KeyValue.Type.Put, "dont-care"), };
835
836  @Test
837  public void testSkipColumn() throws IOException {
838    List<KeyValueScanner> scanners = scanFixture(kvs);
839    try (StoreScanner scan = new StoreScanner(new Scan(), scanInfo, getCols("a", "d"), scanners)) {
840      List<Cell> results = new ArrayList<>();
841      assertEquals(true, scan.next(results));
842      assertEquals(2, results.size());
843      assertEquals(kvs[0], results.get(0));
844      assertEquals(kvs[3], results.get(1));
845      results.clear();
846
847      assertEquals(true, scan.next(results));
848      assertEquals(1, results.size());
849      assertEquals(kvs[kvs.length - 1], results.get(0));
850
851      results.clear();
852      assertEquals(false, scan.next(results));
853    }
854  }
855
856  /*
857   * Test expiration of KeyValues in combination with a configured TTL for a column family (as
858   * should be triggered in a major compaction).
859   */
860  @Test
861  public void testWildCardTtlScan() throws IOException {
862    long now = EnvironmentEdgeManager.currentTime();
863    KeyValue[] kvs =
864      new KeyValue[] { create("R1", "cf", "a", now - 1000, KeyValue.Type.Put, "dont-care"),
865        create("R1", "cf", "b", now - 10, KeyValue.Type.Put, "dont-care"),
866        create("R1", "cf", "c", now - 200, KeyValue.Type.Put, "dont-care"),
867        create("R1", "cf", "d", now - 10000, KeyValue.Type.Put, "dont-care"),
868        create("R2", "cf", "a", now, KeyValue.Type.Put, "dont-care"),
869        create("R2", "cf", "b", now - 10, KeyValue.Type.Put, "dont-care"),
870        create("R2", "cf", "c", now - 200, KeyValue.Type.Put, "dont-care"),
871        create("R2", "cf", "c", now - 1000, KeyValue.Type.Put, "dont-care") };
872    List<KeyValueScanner> scanners = scanFixture(kvs);
873    Scan scan = new Scan();
874    scan.readVersions(1);
875    ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, 500, KeepDeletedCells.FALSE,
876      HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false);
877    try (StoreScanner scanner = new StoreScanner(scan, scanInfo, null, scanners)) {
878      List<Cell> results = new ArrayList<>();
879      assertEquals(true, scanner.next(results));
880      assertEquals(2, results.size());
881      assertEquals(kvs[1], results.get(0));
882      assertEquals(kvs[2], results.get(1));
883      results.clear();
884
885      assertEquals(true, scanner.next(results));
886      assertEquals(3, results.size());
887      assertEquals(kvs[4], results.get(0));
888      assertEquals(kvs[5], results.get(1));
889      assertEquals(kvs[6], results.get(2));
890      results.clear();
891
892      assertEquals(false, scanner.next(results));
893    }
894  }
895
896  @Test
897  public void testScannerReseekDoesntNPE() throws Exception {
898    List<KeyValueScanner> scanners = scanFixture(kvs);
899    try (StoreScanner scan = new StoreScanner(new Scan(), scanInfo, getCols("a", "d"), scanners)) {
900      // Previously a updateReaders twice in a row would cause an NPE. In test this would also
901      // normally cause an NPE because scan.store is null. So as long as we get through these
902      // two calls we are good and the bug was quashed.
903      scan.updateReaders(Collections.emptyList(), Collections.emptyList());
904      scan.updateReaders(Collections.emptyList(), Collections.emptyList());
905      scan.peek();
906    }
907  }
908
909  @Test
910  @Ignore("this fails, since we don't handle deletions, etc, in peek")
911  public void testPeek() throws Exception {
912    KeyValue[] kvs = new KeyValue[] { create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"),
913      create("R1", "cf", "a", 1, KeyValue.Type.Delete, "dont-care"), };
914    List<KeyValueScanner> scanners = scanFixture(kvs);
915    Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1"));
916    try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) {
917      assertNull(scan.peek());
918    }
919  }
920
921  /**
922   * Ensure that expired delete family markers don't override valid puts
923   */
924  @Test
925  public void testExpiredDeleteFamily() throws Exception {
926    long now = EnvironmentEdgeManager.currentTime();
927    KeyValue[] kvs = new KeyValue[] {
928      new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null, now - 1000,
929        KeyValue.Type.DeleteFamily),
930      create("R1", "cf", "a", now - 10, KeyValue.Type.Put, "dont-care"), };
931    List<KeyValueScanner> scanners = scanFixture(kvs);
932    Scan scan = new Scan();
933    scan.readVersions(1);
934    // scanner with ttl equal to 500
935    ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, 500, KeepDeletedCells.FALSE,
936      HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false);
937    try (StoreScanner scanner = new StoreScanner(scan, scanInfo, null, scanners)) {
938      List<Cell> results = new ArrayList<>();
939      assertEquals(true, scanner.next(results));
940      assertEquals(1, results.size());
941      assertEquals(kvs[1], results.get(0));
942      results.clear();
943
944      assertEquals(false, scanner.next(results));
945    }
946  }
947
948  @Test
949  public void testDeleteMarkerLongevity() throws Exception {
950    try {
951      final long now = EnvironmentEdgeManager.currentTime();
952      EnvironmentEdgeManagerTestHelper.injectEdge(new EnvironmentEdge() {
953        @Override
954        public long currentTime() {
955          return now;
956        }
957      });
958      // @formatter:off
959      KeyValue[] kvs = new KeyValue[]{
960        /*0*/ new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null,
961        now - 100, KeyValue.Type.DeleteFamily), // live
962        /*1*/ new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null,
963        now - 1000, KeyValue.Type.DeleteFamily), // expired
964        /*2*/ create("R1", "cf", "a", now - 50,
965        KeyValue.Type.Put, "v3"), // live
966        /*3*/ create("R1", "cf", "a", now - 55,
967        KeyValue.Type.Delete, "dontcare"), // live
968        /*4*/ create("R1", "cf", "a", now - 55,
969        KeyValue.Type.Put, "deleted-version v2"), // deleted
970        /*5*/ create("R1", "cf", "a", now - 60,
971        KeyValue.Type.Put, "v1"), // live
972        /*6*/ create("R1", "cf", "a", now - 65,
973        KeyValue.Type.Put, "v0"), // max-version reached
974        /*7*/ create("R1", "cf", "a",
975        now - 100, KeyValue.Type.DeleteColumn, "dont-care"), // max-version
976        /*8*/ create("R1", "cf", "b", now - 600,
977        KeyValue.Type.DeleteColumn, "dont-care"), // expired
978        /*9*/ create("R1", "cf", "b", now - 70,
979        KeyValue.Type.Put, "v2"), // live
980        /*10*/ create("R1", "cf", "b", now - 750,
981        KeyValue.Type.Put, "v1"), // expired
982        /*11*/ create("R1", "cf", "c", now - 500,
983        KeyValue.Type.Delete, "dontcare"), // expired
984        /*12*/ create("R1", "cf", "c", now - 600,
985        KeyValue.Type.Put, "v1"), // expired
986        /*13*/ create("R1", "cf", "c", now - 1000,
987        KeyValue.Type.Delete, "dontcare"), // expired
988        /*14*/ create("R1", "cf", "d", now - 60,
989        KeyValue.Type.Put, "expired put"), // live
990        /*15*/ create("R1", "cf", "d", now - 100,
991        KeyValue.Type.Delete, "not-expired delete"), // live
992      };
993      // @formatter:on
994      List<KeyValueScanner> scanners = scanFixture(kvs);
995      ScanInfo scanInfo = new ScanInfo(CONF, Bytes.toBytes("cf"), 0 /* minVersions */,
996        2 /* maxVersions */, 500 /* ttl */, KeepDeletedCells.FALSE /* keepDeletedCells */,
997        HConstants.DEFAULT_BLOCKSIZE /* block size */, 200, /* timeToPurgeDeletes */
998        CellComparator.getInstance(), false);
999      try (StoreScanner scanner =
1000        new StoreScanner(scanInfo, 2, ScanType.COMPACT_DROP_DELETES, scanners)) {
1001        List<Cell> results = new ArrayList<>();
1002        results = new ArrayList<>();
1003        assertEquals(true, scanner.next(results));
1004        assertEquals(kvs[0], results.get(0));
1005        assertEquals(kvs[2], results.get(1));
1006        assertEquals(kvs[3], results.get(2));
1007        assertEquals(kvs[5], results.get(3));
1008        assertEquals(kvs[9], results.get(4));
1009        assertEquals(kvs[14], results.get(5));
1010        assertEquals(kvs[15], results.get(6));
1011        assertEquals(7, results.size());
1012      }
1013    } finally {
1014      EnvironmentEdgeManagerTestHelper.reset();
1015    }
1016  }
1017
1018  @Test
1019  public void testPreadNotEnabledForCompactionStoreScanners() throws Exception {
1020    long now = EnvironmentEdgeManager.currentTime();
1021    KeyValue[] kvs = new KeyValue[] {
1022      new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null, now - 1000,
1023        KeyValue.Type.DeleteFamily),
1024      create("R1", "cf", "a", now - 10, KeyValue.Type.Put, "dont-care"), };
1025    List<KeyValueScanner> scanners = scanFixture(kvs);
1026    ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, 500, KeepDeletedCells.FALSE,
1027      HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false);
1028    try (StoreScanner storeScanner =
1029      new StoreScanner(scanInfo, -1, ScanType.COMPACT_RETAIN_DELETES, scanners)) {
1030      assertFalse(storeScanner.isScanUsePread());
1031    }
1032  }
1033
1034  @Test
1035  public void testReadVersionWithRawAndFilter() throws IOException {
1036    ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, Long.MAX_VALUE, KeepDeletedCells.FALSE,
1037      HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false);
1038    KeyValue[] kvs = new KeyValue[] { create("R1", "cf", "a", 3, KeyValue.Type.Put, "dont-care"),
1039      create("R1", "cf", "a", 2, KeyValue.Type.Put, "dont-care"),
1040      create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care") };
1041    List<KeyValueScanner> scanners = Arrays
1042      .asList(new KeyValueScanner[] { new KeyValueScanFixture(CellComparator.getInstance(), kvs) });
1043
1044    BinaryComparator comp = new BinaryComparator(Bytes.toBytes("a"));
1045    Filter filter = new QualifierFilter(CompareOperator.EQUAL, comp);
1046    Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1")).readVersions(2).setRaw(true);
1047    scanSpec.setFilter(filter);
1048    try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, null, scanners)) {
1049      List<Cell> results = new ArrayList<>();
1050      assertEquals(true, scan.next(results));
1051      assertEquals(2, results.size());
1052    }
1053  }
1054
1055  @Test
1056  public void testScannersClosedWhenCheckingOnlyMemStore() throws IOException {
1057    class MyCollectionBackedScanner extends CollectionBackedScanner {
1058      final boolean fileScanner;
1059      boolean closed;
1060
1061      MyCollectionBackedScanner(boolean fileScanner) {
1062        super(Collections.emptySortedSet());
1063        this.fileScanner = fileScanner;
1064      }
1065
1066      @Override
1067      public boolean isFileScanner() {
1068        return fileScanner;
1069      }
1070
1071      @Override
1072      public void close() {
1073        super.close();
1074        closed = true;
1075      }
1076    }
1077
1078    ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, Long.MAX_VALUE, KeepDeletedCells.FALSE,
1079      HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false);
1080    InternalScan scan = new InternalScan(new Scan());
1081    scan.checkOnlyMemStore();
1082    MyCollectionBackedScanner fileScanner = new MyCollectionBackedScanner(true);
1083    MyCollectionBackedScanner memStoreScanner = new MyCollectionBackedScanner(false);
1084    List<? extends KeyValueScanner> allScanners = Arrays.asList(fileScanner, memStoreScanner);
1085
1086    try (StoreScanner scanner = new StoreScanner(scan, scanInfo, null, allScanners)) {
1087      List<KeyValueScanner> remaining = scanner.selectScannersFrom(null, allScanners);
1088
1089      assertEquals(1, remaining.size());
1090      assertSame(memStoreScanner, remaining.get(0));
1091      assertTrue(fileScanner.closed);
1092      assertFalse(memStoreScanner.closed);
1093    }
1094  }
1095}