001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.apache.hadoop.hbase.CellUtil.createCell;
021import static org.apache.hadoop.hbase.KeyValueTestUtil.create;
022import static org.apache.hadoop.hbase.regionserver.KeyValueScanFixture.scanFixture;
023import static org.junit.Assert.assertEquals;
024import static org.junit.Assert.assertFalse;
025import static org.junit.Assert.assertNull;
026import static org.junit.Assert.assertTrue;
027
028import java.io.IOException;
029import java.util.ArrayList;
030import java.util.Arrays;
031import java.util.Collections;
032import java.util.List;
033import java.util.NavigableSet;
034import java.util.TreeSet;
035import java.util.concurrent.atomic.AtomicInteger;
036
037import org.apache.hadoop.conf.Configuration;
038import org.apache.hadoop.hbase.Cell;
039import org.apache.hadoop.hbase.CellComparator;
040import org.apache.hadoop.hbase.CellUtil;
041import org.apache.hadoop.hbase.HBaseClassTestRule;
042import org.apache.hadoop.hbase.HBaseConfiguration;
043import org.apache.hadoop.hbase.HBaseTestingUtility;
044import org.apache.hadoop.hbase.HConstants;
045import org.apache.hadoop.hbase.KeepDeletedCells;
046import org.apache.hadoop.hbase.KeyValue;
047import org.apache.hadoop.hbase.PrivateCellUtil;
048import org.apache.hadoop.hbase.client.Get;
049import org.apache.hadoop.hbase.client.Scan;
050import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
051import org.apache.hadoop.hbase.testclassification.MediumTests;
052import org.apache.hadoop.hbase.testclassification.RegionServerTests;
053import org.apache.hadoop.hbase.util.Bytes;
054import org.apache.hadoop.hbase.util.EnvironmentEdge;
055import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
056import org.junit.ClassRule;
057import org.junit.Ignore;
058import org.junit.Rule;
059import org.junit.Test;
060import org.junit.experimental.categories.Category;
061import org.junit.rules.TestName;
062import org.slf4j.Logger;
063import org.slf4j.LoggerFactory;
064
065// Can't be small as it plays with EnvironmentEdgeManager
066@Category({RegionServerTests.class, MediumTests.class})
067public class TestStoreScanner {
068
069  @ClassRule
070  public static final HBaseClassTestRule CLASS_RULE =
071      HBaseClassTestRule.forClass(TestStoreScanner.class);
072
073  private static final Logger LOG = LoggerFactory.getLogger(TestStoreScanner.class);
074  @Rule public TestName name = new TestName();
075  private static final String CF_STR = "cf";
076  private static final byte[] CF = Bytes.toBytes(CF_STR);
077  static Configuration CONF = HBaseConfiguration.create();
078  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
079  private ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, Integer.MAX_VALUE, Long.MAX_VALUE,
080      KeepDeletedCells.FALSE, HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false);
081
082  /**
083   * From here on down, we have a bunch of defines and specific CELL_GRID of Cells. The
084   * CELL_GRID then has a Scanner that can fake out 'block' transitions. All this elaborate
085   * setup is for tests that ensure we don't overread, and that the {@link StoreScanner} is not
086   * overly enthusiastic.
087   */
088  private static final byte[] ZERO = new byte[] {'0'};
089  private static final byte[] ZERO_POINT_ZERO = new byte[] {'0', '.', '0'};
090  private static final byte[] ONE = new byte[] {'1'};
091  private static final byte[] TWO = new byte[] {'2'};
092  private static final byte[] TWO_POINT_TWO = new byte[] {'2', '.', '2'};
093  private static final byte[] THREE = new byte[] {'3'};
094  private static final byte[] FOUR = new byte[] {'4'};
095  private static final byte[] FIVE = new byte[] {'5'};
096  private static final byte[] VALUE = new byte[] {'v'};
097  private static final int CELL_GRID_BLOCK2_BOUNDARY = 4;
098  private static final int CELL_GRID_BLOCK3_BOUNDARY = 11;
099  private static final int CELL_GRID_BLOCK4_BOUNDARY = 15;
100  private static final int CELL_GRID_BLOCK5_BOUNDARY = 19;
101
102  /**
103   * Five rows by four columns distinguished by column qualifier (column qualifier is one of the
104   * four rows... ONE, TWO, etc.). Exceptions are a weird row after TWO; it is TWO_POINT_TWO.
105   * And then row FOUR has five columns finishing w/ row FIVE having a single column.
106   * We will use this to test scan does the right thing as it
107   * we do Gets, StoreScanner#optimize, and what we do on (faked) block boundaries.
108   */
109  private static final Cell[] CELL_GRID = new Cell [] {
110    createCell(ONE, CF, ONE, 1L, KeyValue.Type.Put.getCode(), VALUE),
111    createCell(ONE, CF, TWO, 1L, KeyValue.Type.Put.getCode(), VALUE),
112    createCell(ONE, CF, THREE, 1L, KeyValue.Type.Put.getCode(), VALUE),
113    createCell(ONE, CF, FOUR, 1L, KeyValue.Type.Put.getCode(), VALUE),
114    // Offset 4 CELL_GRID_BLOCK2_BOUNDARY
115    createCell(TWO, CF, ONE, 1L, KeyValue.Type.Put.getCode(), VALUE),
116    createCell(TWO, CF, TWO, 1L, KeyValue.Type.Put.getCode(), VALUE),
117    createCell(TWO, CF, THREE, 1L, KeyValue.Type.Put.getCode(), VALUE),
118    createCell(TWO, CF, FOUR, 1L, KeyValue.Type.Put.getCode(), VALUE),
119    createCell(TWO_POINT_TWO, CF, ZERO, 1L, KeyValue.Type.Put.getCode(), VALUE),
120    createCell(TWO_POINT_TWO, CF, ZERO_POINT_ZERO, 1L, KeyValue.Type.Put.getCode(), VALUE),
121    createCell(TWO_POINT_TWO, CF, FIVE, 1L, KeyValue.Type.Put.getCode(), VALUE),
122    // Offset 11! CELL_GRID_BLOCK3_BOUNDARY
123    createCell(THREE, CF, ONE, 1L, KeyValue.Type.Put.getCode(), VALUE),
124    createCell(THREE, CF, TWO, 1L, KeyValue.Type.Put.getCode(), VALUE),
125    createCell(THREE, CF, THREE, 1L, KeyValue.Type.Put.getCode(), VALUE),
126    createCell(THREE, CF, FOUR, 1L, KeyValue.Type.Put.getCode(), VALUE),
127    // Offset 15 CELL_GRID_BLOCK4_BOUNDARY
128    createCell(FOUR, CF, ONE, 1L, KeyValue.Type.Put.getCode(), VALUE),
129    createCell(FOUR, CF, TWO, 1L, KeyValue.Type.Put.getCode(), VALUE),
130    createCell(FOUR, CF, THREE, 1L, KeyValue.Type.Put.getCode(), VALUE),
131    createCell(FOUR, CF, FOUR, 1L, KeyValue.Type.Put.getCode(), VALUE),
132    // Offset 19 CELL_GRID_BLOCK5_BOUNDARY
133    createCell(FOUR, CF, FIVE, 1L, KeyValue.Type.Put.getCode(), VALUE),
134    createCell(FIVE, CF, ZERO, 1L, KeyValue.Type.Put.getCode(), VALUE),
135  };
136
137  private static class KeyValueHeapWithCount extends KeyValueHeap {
138
139    final AtomicInteger count;
140
141    public KeyValueHeapWithCount(List<? extends KeyValueScanner> scanners,
142        CellComparator comparator, AtomicInteger count) throws IOException {
143      super(scanners, comparator);
144      this.count = count;
145    }
146
147    @Override
148    public Cell peek() {
149      this.count.incrementAndGet();
150      return super.peek();
151    }
152  }
153
154  /**
155   * A StoreScanner for our CELL_GRID above. Fakes the block transitions. Does counts of
156   * calls to optimize and counts of when optimize actually did an optimize.
157   */
158  private static class CellGridStoreScanner extends StoreScanner {
159    // Count of how often optimize is called and of how often it does an optimize.
160    AtomicInteger count;
161    final AtomicInteger optimization = new AtomicInteger(0);
162
163    CellGridStoreScanner(final Scan scan, ScanInfo scanInfo) throws IOException {
164      super(scan, scanInfo, scan.getFamilyMap().get(CF), Arrays.<KeyValueScanner> asList(
165        new KeyValueScanner[] { new KeyValueScanFixture(CellComparator.getInstance(), CELL_GRID) }));
166    }
167
168    @Override
169    protected void resetKVHeap(List<? extends KeyValueScanner> scanners,
170        CellComparator comparator) throws IOException {
171      if (count == null) {
172        count = new AtomicInteger(0);
173      }
174      heap = newKVHeap(scanners, comparator);
175    }
176
177    @Override
178    protected KeyValueHeap newKVHeap(List<? extends KeyValueScanner> scanners,
179        CellComparator comparator) throws IOException {
180      return new KeyValueHeapWithCount(scanners, comparator, count);
181    }
182
183    @Override
184    protected boolean trySkipToNextRow(Cell cell) throws IOException {
185      boolean optimized = super.trySkipToNextRow(cell);
186      LOG.info("Cell=" + cell + ", nextIndex=" + CellUtil.toString(getNextIndexedKey(), false)
187          + ", optimized=" + optimized);
188      if (optimized) {
189        optimization.incrementAndGet();
190      }
191      return optimized;
192    }
193
194    @Override
195    protected boolean trySkipToNextColumn(Cell cell) throws IOException {
196      boolean optimized = super.trySkipToNextColumn(cell);
197      LOG.info("Cell=" + cell + ", nextIndex=" + CellUtil.toString(getNextIndexedKey(), false)
198          + ", optimized=" + optimized);
199      if (optimized) {
200        optimization.incrementAndGet();
201      }
202      return optimized;
203    }
204
205    @Override
206    public Cell getNextIndexedKey() {
207      // Fake block boundaries by having index of next block change as we go through scan.
208      return count.get() > CELL_GRID_BLOCK4_BOUNDARY?
209          PrivateCellUtil.createFirstOnRow(CELL_GRID[CELL_GRID_BLOCK5_BOUNDARY]):
210            count.get() > CELL_GRID_BLOCK3_BOUNDARY?
211                PrivateCellUtil.createFirstOnRow(CELL_GRID[CELL_GRID_BLOCK4_BOUNDARY]):
212                  count.get() > CELL_GRID_BLOCK2_BOUNDARY?
213                      PrivateCellUtil.createFirstOnRow(CELL_GRID[CELL_GRID_BLOCK3_BOUNDARY]):
214                        PrivateCellUtil.createFirstOnRow(CELL_GRID[CELL_GRID_BLOCK2_BOUNDARY]);
215    }
216  };
217
218  private static final int CELL_WITH_VERSIONS_BLOCK2_BOUNDARY = 4;
219
220  private static final Cell[] CELL_WITH_VERSIONS = new Cell [] {
221    createCell(ONE, CF, ONE, 2L, KeyValue.Type.Put.getCode(), VALUE),
222    createCell(ONE, CF, ONE, 1L, KeyValue.Type.Put.getCode(), VALUE),
223    createCell(ONE, CF, TWO, 2L, KeyValue.Type.Put.getCode(), VALUE),
224    createCell(ONE, CF, TWO, 1L, KeyValue.Type.Put.getCode(), VALUE),
225    // Offset 4 CELL_WITH_VERSIONS_BLOCK2_BOUNDARY
226    createCell(TWO, CF, ONE, 1L, KeyValue.Type.Put.getCode(), VALUE),
227    createCell(TWO, CF, TWO, 1L, KeyValue.Type.Put.getCode(), VALUE),
228  };
229
230  private static class CellWithVersionsStoreScanner extends StoreScanner {
231    // Count of how often optimize is called and of how often it does an optimize.
232    final AtomicInteger optimization = new AtomicInteger(0);
233
234    CellWithVersionsStoreScanner(final Scan scan, ScanInfo scanInfo) throws IOException {
235      super(scan, scanInfo, scan.getFamilyMap().get(CF),
236          Arrays.<KeyValueScanner> asList(new KeyValueScanner[] {
237              new KeyValueScanFixture(CellComparator.getInstance(), CELL_WITH_VERSIONS) }));
238    }
239
240    @Override
241    protected boolean trySkipToNextColumn(Cell cell) throws IOException {
242      boolean optimized = super.trySkipToNextColumn(cell);
243      LOG.info("Cell=" + cell + ", nextIndex=" + CellUtil.toString(getNextIndexedKey(), false)
244          + ", optimized=" + optimized);
245      if (optimized) {
246        optimization.incrementAndGet();
247      }
248      return optimized;
249    }
250
251    @Override
252    public Cell getNextIndexedKey() {
253      // Fake block boundaries by having index of next block change as we go through scan.
254      return PrivateCellUtil
255          .createFirstOnRow(CELL_WITH_VERSIONS[CELL_WITH_VERSIONS_BLOCK2_BOUNDARY]);
256    }
257  };
258
259  private static class CellWithVersionsNoOptimizeStoreScanner extends StoreScanner {
260    // Count of how often optimize is called and of how often it does an optimize.
261    final AtomicInteger optimization = new AtomicInteger(0);
262
263    CellWithVersionsNoOptimizeStoreScanner(Scan scan, ScanInfo scanInfo) throws IOException {
264      super(scan, scanInfo, scan.getFamilyMap().get(CF),
265          Arrays.<KeyValueScanner> asList(new KeyValueScanner[] {
266              new KeyValueScanFixture(CellComparator.getInstance(), CELL_WITH_VERSIONS) }));
267    }
268
269    @Override
270    protected boolean trySkipToNextColumn(Cell cell) throws IOException {
271      boolean optimized = super.trySkipToNextColumn(cell);
272      LOG.info("Cell=" + cell + ", nextIndex=" + CellUtil.toString(getNextIndexedKey(), false)
273          + ", optimized=" + optimized);
274      if (optimized) {
275        optimization.incrementAndGet();
276      }
277      return optimized;
278    }
279
280    @Override
281    public Cell getNextIndexedKey() {
282      return null;
283    }
284  };
285
286  @Test
287  public void testWithColumnCountGetFilter() throws Exception {
288    Get get = new Get(ONE);
289    get.readAllVersions();
290    get.addFamily(CF);
291    get.setFilter(new ColumnCountGetFilter(2));
292
293    try (CellWithVersionsNoOptimizeStoreScanner scannerNoOptimize =
294        new CellWithVersionsNoOptimizeStoreScanner(new Scan(get), this.scanInfo)) {
295      List<Cell> results = new ArrayList<>();
296      while (scannerNoOptimize.next(results)) {
297        continue;
298      }
299      assertEquals(2, results.size());
300      assertTrue(CellUtil.matchingColumn(results.get(0), CELL_WITH_VERSIONS[0]));
301      assertTrue(CellUtil.matchingColumn(results.get(1), CELL_WITH_VERSIONS[2]));
302      assertTrue("Optimize should do some optimizations",
303        scannerNoOptimize.optimization.get() == 0);
304    }
305
306    get.setFilter(new ColumnCountGetFilter(2));
307    try (CellWithVersionsStoreScanner scanner =
308        new CellWithVersionsStoreScanner(new Scan(get), this.scanInfo)) {
309      List<Cell> results = new ArrayList<>();
310      while (scanner.next(results)) {
311        continue;
312      }
313      assertEquals(2, results.size());
314      assertTrue(CellUtil.matchingColumn(results.get(0), CELL_WITH_VERSIONS[0]));
315      assertTrue(CellUtil.matchingColumn(results.get(1), CELL_WITH_VERSIONS[2]));
316      assertTrue("Optimize should do some optimizations", scanner.optimization.get() > 0);
317    }
318  }
319
320  /*
321   * Test utility for building a NavigableSet for scanners.
322   * @param strCols
323   * @return
324   */
325  NavigableSet<byte[]> getCols(String ...strCols) {
326    NavigableSet<byte[]> cols = new TreeSet<>(Bytes.BYTES_COMPARATOR);
327    for (String col : strCols) {
328      byte[] bytes = Bytes.toBytes(col);
329      cols.add(bytes);
330    }
331    return cols;
332  }
333
334  @Test
335  public void testFullRowGetDoesNotOverreadWhenRowInsideOneBlock() throws IOException {
336    // Do a Get against row two. Row two is inside a block that starts with row TWO but ends with
337    // row TWO_POINT_TWO. We should read one block only.
338    Get get = new Get(TWO);
339    Scan scan = new Scan(get);
340    try (CellGridStoreScanner scanner = new CellGridStoreScanner(scan, this.scanInfo)) {
341      List<Cell> results = new ArrayList<>();
342      while (scanner.next(results)) {
343        continue;
344      }
345      // Should be four results of column 1 (though there are 5 rows in the CELL_GRID -- the
346      // TWO_POINT_TWO row does not have a a column ONE.
347      assertEquals(4, results.size());
348      // We should have gone the optimize route 5 times totally... an INCLUDE for the four cells
349      // in the row plus the DONE on the end.
350      assertEquals(5, scanner.count.get());
351      // For a full row Get, there should be no opportunity for scanner optimization.
352      assertEquals(0, scanner.optimization.get());
353    }
354  }
355
356  @Test
357  public void testFullRowSpansBlocks() throws IOException {
358    // Do a Get against row FOUR. It spans two blocks.
359    Get get = new Get(FOUR);
360    Scan scan = new Scan(get);
361    try (CellGridStoreScanner scanner = new CellGridStoreScanner(scan, this.scanInfo)) {
362      List<Cell> results = new ArrayList<>();
363      while (scanner.next(results)) {
364        continue;
365      }
366      // Should be four results of column 1 (though there are 5 rows in the CELL_GRID -- the
367      // TWO_POINT_TWO row does not have a a column ONE.
368      assertEquals(5, results.size());
369      // We should have gone the optimize route 6 times totally... an INCLUDE for the five cells
370      // in the row plus the DONE on the end.
371      assertEquals(6, scanner.count.get());
372      // For a full row Get, there should be no opportunity for scanner optimization.
373      assertEquals(0, scanner.optimization.get());
374    }
375  }
376
377  /**
378   * Test optimize in StoreScanner. Test that we skip to the next 'block' when we it makes sense
379   * reading the block 'index'.
380   * @throws IOException
381   */
382  @Test
383  public void testOptimize() throws IOException {
384    Scan scan = new Scan();
385    // A scan that just gets the first qualifier on each row of the CELL_GRID
386    scan.addColumn(CF, ONE);
387    try (CellGridStoreScanner scanner = new CellGridStoreScanner(scan, this.scanInfo)) {
388      List<Cell> results = new ArrayList<>();
389      while (scanner.next(results)) {
390        continue;
391      }
392      // Should be four results of column 1 (though there are 5 rows in the CELL_GRID -- the
393      // TWO_POINT_TWO row does not have a a column ONE.
394      assertEquals(4, results.size());
395      for (Cell cell: results) {
396        assertTrue(Bytes.equals(ONE, 0, ONE.length,
397            cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()));
398      }
399      assertTrue("Optimize should do some optimizations", scanner.optimization.get() > 0);
400    }
401  }
402
403  /**
404   * Ensure the optimize Scan method in StoreScanner does not get in the way of a Get doing minimum
405   * work... seeking to start of block and then SKIPPING until we find the wanted Cell.
406   * This 'simple' scenario mimics case of all Cells fitting inside a single HFileBlock.
407   * See HBASE-15392. This test is a little cryptic. Takes a bit of staring to figure what it up to.
408   * @throws IOException
409   */
410  @Test
411  public void testOptimizeAndGet() throws IOException {
412    // First test a Get of two columns in the row R2. Every Get is a Scan. Get columns named
413    // R2 and R3.
414    Get get = new Get(TWO);
415    get.addColumn(CF, TWO);
416    get.addColumn(CF, THREE);
417    Scan scan = new Scan(get);
418    try (CellGridStoreScanner scanner = new CellGridStoreScanner(scan, this.scanInfo)) {
419      List<Cell> results = new ArrayList<>();
420      // For a Get there should be no more next's after the first call.
421      assertEquals(false, scanner.next(results));
422      // Should be one result only.
423      assertEquals(2, results.size());
424      // And we should have gone through optimize twice only.
425      assertEquals("First qcode is SEEK_NEXT_COL and second INCLUDE_AND_SEEK_NEXT_ROW", 3,
426        scanner.count.get());
427    }
428  }
429
430  /**
431   * Ensure that optimize does not cause the Get to do more seeking than required. Optimize
432   * (see HBASE-15392) was causing us to seek all Cells in a block when a Get Scan if the next block
433   * index/start key was a different row to the current one. A bug. We'd call next too often
434   * because we had to exhaust all Cells in the current row making us load the next block just to
435   * discard what we read there. This test is a little cryptic. Takes a bit of staring to figure
436   * what it up to.
437   * @throws IOException
438   */
439  @Test
440  public void testOptimizeAndGetWithFakedNextBlockIndexStart() throws IOException {
441    // First test a Get of second column in the row R2. Every Get is a Scan. Second column has a
442    // qualifier of R2.
443    Get get = new Get(THREE);
444    get.addColumn(CF, TWO);
445    Scan scan = new Scan(get);
446    try (CellGridStoreScanner scanner = new CellGridStoreScanner(scan, this.scanInfo)) {
447      List<Cell> results = new ArrayList<>();
448      // For a Get there should be no more next's after the first call.
449      assertEquals(false, scanner.next(results));
450      // Should be one result only.
451      assertEquals(1, results.size());
452      // And we should have gone through optimize twice only.
453      assertEquals("First qcode is SEEK_NEXT_COL and second INCLUDE_AND_SEEK_NEXT_ROW", 2,
454        scanner.count.get());
455    }
456  }
457
458  @Test
459  public void testScanTimeRange() throws IOException {
460    String r1 = "R1";
461    // returns only 1 of these 2 even though same timestamp
462    KeyValue [] kvs = new KeyValue[] {
463        create(r1, CF_STR, "a", 1, KeyValue.Type.Put, "dont-care"),
464        create(r1, CF_STR, "a", 2, KeyValue.Type.Put, "dont-care"),
465        create(r1, CF_STR, "a", 3, KeyValue.Type.Put, "dont-care"),
466        create(r1, CF_STR, "a", 4, KeyValue.Type.Put, "dont-care"),
467        create(r1, CF_STR, "a", 5, KeyValue.Type.Put, "dont-care"),
468    };
469    List<KeyValueScanner> scanners = Arrays.<KeyValueScanner>asList(
470        new KeyValueScanner[] {
471            new KeyValueScanFixture(CellComparator.getInstance(), kvs)
472    });
473    Scan scanSpec = new Scan().withStartRow(Bytes.toBytes(r1));
474    scanSpec.setTimeRange(0, 6);
475    scanSpec.readAllVersions();
476    List<Cell> results = null;
477    try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) {
478      results = new ArrayList<>();
479      assertEquals(true, scan.next(results));
480      assertEquals(5, results.size());
481      assertEquals(kvs[kvs.length - 1], results.get(0));
482    }
483    // Scan limited TimeRange
484    scanSpec = new Scan().withStartRow(Bytes.toBytes(r1));
485    scanSpec.setTimeRange(1, 3);
486    scanSpec.readAllVersions();
487    try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) {
488      results = new ArrayList<>();
489      assertEquals(true, scan.next(results));
490      assertEquals(2, results.size());
491    }
492    // Another range.
493    scanSpec = new Scan().withStartRow(Bytes.toBytes(r1));
494    scanSpec.setTimeRange(5, 10);
495    scanSpec.readAllVersions();
496    try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) {
497      results = new ArrayList<>();
498      assertEquals(true, scan.next(results));
499      assertEquals(1, results.size());
500    }
501    // See how TimeRange and Versions interact.
502    // Another range.
503    scanSpec = new Scan().withStartRow(Bytes.toBytes(r1));
504    scanSpec.setTimeRange(0, 10);
505    scanSpec.readVersions(3);
506    try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) {
507      results = new ArrayList<>();
508      assertEquals(true, scan.next(results));
509      assertEquals(3, results.size());
510    }
511  }
512
513  @Test
514  public void testScanSameTimestamp() throws IOException {
515    // returns only 1 of these 2 even though same timestamp
516    KeyValue [] kvs = new KeyValue[] {
517        create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"),
518        create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"),
519    };
520    List<KeyValueScanner> scanners = Arrays.asList(
521        new KeyValueScanner[] {
522            new KeyValueScanFixture(CellComparator.getInstance(), kvs)
523        });
524
525    Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1"));
526    // this only uses maxVersions (default=1) and TimeRange (default=all)
527    try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) {
528      List<Cell> results = new ArrayList<>();
529      assertEquals(true, scan.next(results));
530      assertEquals(1, results.size());
531      assertEquals(kvs[0], results.get(0));
532    }
533  }
534
535  /*
536   * Test test shows exactly how the matcher's return codes confuses the StoreScanner
537   * and prevent it from doing the right thing.  Seeking once, then nexting twice
538   * should return R1, then R2, but in this case it doesnt.
539   * TODO this comment makes no sense above. Appears to do the right thing.
540   * @throws IOException
541   */
542  @Test
543  public void testWontNextToNext() throws IOException {
544    // build the scan file:
545    KeyValue [] kvs = new KeyValue[] {
546        create("R1", "cf", "a", 2, KeyValue.Type.Put, "dont-care"),
547        create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"),
548        create("R2", "cf", "a", 1, KeyValue.Type.Put, "dont-care")
549    };
550    List<KeyValueScanner> scanners = scanFixture(kvs);
551
552    Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1"));
553    // this only uses maxVersions (default=1) and TimeRange (default=all)
554    try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) {
555      List<Cell> results = new ArrayList<>();
556      scan.next(results);
557      assertEquals(1, results.size());
558      assertEquals(kvs[0], results.get(0));
559      // should be ok...
560      // now scan _next_ again.
561      results.clear();
562      scan.next(results);
563      assertEquals(1, results.size());
564      assertEquals(kvs[2], results.get(0));
565
566      results.clear();
567      scan.next(results);
568      assertEquals(0, results.size());
569    }
570  }
571
572
573  @Test
574  public void testDeleteVersionSameTimestamp() throws IOException {
575    KeyValue [] kvs = new KeyValue [] {
576        create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"),
577        create("R1", "cf", "a", 1, KeyValue.Type.Delete, "dont-care"),
578    };
579    List<KeyValueScanner> scanners = scanFixture(kvs);
580    Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1"));
581    try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) {
582      List<Cell> results = new ArrayList<>();
583      assertFalse(scan.next(results));
584      assertEquals(0, results.size());
585    }
586  }
587
588  /*
589   * Test the case where there is a delete row 'in front of' the next row, the scanner
590   * will move to the next row.
591   */
592  @Test
593  public void testDeletedRowThenGoodRow() throws IOException {
594    KeyValue [] kvs = new KeyValue [] {
595        create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"),
596        create("R1", "cf", "a", 1, KeyValue.Type.Delete, "dont-care"),
597        create("R2", "cf", "a", 20, KeyValue.Type.Put, "dont-care")
598    };
599    List<KeyValueScanner> scanners = scanFixture(kvs);
600    Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1"));
601    try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) {
602      List<Cell> results = new ArrayList<>();
603      assertEquals(true, scan.next(results));
604      assertEquals(0, results.size());
605
606      assertEquals(true, scan.next(results));
607      assertEquals(1, results.size());
608      assertEquals(kvs[2], results.get(0));
609
610      assertEquals(false, scan.next(results));
611    }
612  }
613
614  @Test
615  public void testDeleteVersionMaskingMultiplePuts() throws IOException {
616    long now = System.currentTimeMillis();
617    KeyValue [] kvs1 = new KeyValue[] {
618        create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care"),
619        create("R1", "cf", "a", now, KeyValue.Type.Delete, "dont-care")
620    };
621    KeyValue [] kvs2 = new KeyValue[] {
622        create("R1", "cf", "a", now-500, KeyValue.Type.Put, "dont-care"),
623        create("R1", "cf", "a", now-100, KeyValue.Type.Put, "dont-care"),
624        create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care")
625    };
626    List<KeyValueScanner> scanners = scanFixture(kvs1, kvs2);
627
628    try (StoreScanner scan = new StoreScanner(new Scan().withStartRow(Bytes.toBytes("R1")),
629        scanInfo, getCols("a"), scanners)) {
630      List<Cell> results = new ArrayList<>();
631      // the two put at ts=now will be masked by the 1 delete, and
632      // since the scan default returns 1 version we'll return the newest
633      // key, which is kvs[2], now-100.
634      assertEquals(true, scan.next(results));
635      assertEquals(1, results.size());
636      assertEquals(kvs2[1], results.get(0));
637    }
638  }
639
640  @Test
641  public void testDeleteVersionsMixedAndMultipleVersionReturn() throws IOException {
642    long now = System.currentTimeMillis();
643    KeyValue [] kvs1 = new KeyValue[] {
644        create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care"),
645        create("R1", "cf", "a", now, KeyValue.Type.Delete, "dont-care")
646    };
647    KeyValue [] kvs2 = new KeyValue[] {
648        create("R1", "cf", "a", now-500, KeyValue.Type.Put, "dont-care"),
649        create("R1", "cf", "a", now+500, KeyValue.Type.Put, "dont-care"),
650        create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care"),
651        create("R2", "cf", "z", now, KeyValue.Type.Put, "dont-care")
652    };
653    List<KeyValueScanner> scanners = scanFixture(kvs1, kvs2);
654
655    Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1")).readVersions(2);
656    try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) {
657      List<Cell> results = new ArrayList<>();
658      assertEquals(true, scan.next(results));
659      assertEquals(2, results.size());
660      assertEquals(kvs2[1], results.get(0));
661      assertEquals(kvs2[0], results.get(1));
662    }
663  }
664
665  @Test
666  public void testWildCardOneVersionScan() throws IOException {
667    KeyValue [] kvs = new KeyValue [] {
668        create("R1", "cf", "a", 2, KeyValue.Type.Put, "dont-care"),
669        create("R1", "cf", "b", 1, KeyValue.Type.Put, "dont-care"),
670        create("R1", "cf", "a", 1, KeyValue.Type.DeleteColumn, "dont-care"),
671    };
672    List<KeyValueScanner> scanners = scanFixture(kvs);
673    try (StoreScanner scan =
674        new StoreScanner(new Scan().withStartRow(Bytes.toBytes("R1")), scanInfo, null, scanners)) {
675      List<Cell> results = new ArrayList<>();
676      assertEquals(true, scan.next(results));
677      assertEquals(2, results.size());
678      assertEquals(kvs[0], results.get(0));
679      assertEquals(kvs[1], results.get(1));
680    }
681  }
682
683  @Test
684  public void testWildCardScannerUnderDeletes() throws IOException {
685    KeyValue [] kvs = new KeyValue [] {
686        create("R1", "cf", "a", 2, KeyValue.Type.Put, "dont-care"), // inc
687        // orphaned delete column.
688        create("R1", "cf", "a", 1, KeyValue.Type.DeleteColumn, "dont-care"),
689        // column b
690        create("R1", "cf", "b", 2, KeyValue.Type.Put, "dont-care"), // inc
691        create("R1", "cf", "b", 1, KeyValue.Type.Put, "dont-care"), // inc
692        // column c
693        create("R1", "cf", "c", 10, KeyValue.Type.Delete, "dont-care"),
694        create("R1", "cf", "c", 10, KeyValue.Type.Put, "dont-care"), // no
695        create("R1", "cf", "c", 9, KeyValue.Type.Put, "dont-care"),  // inc
696        // column d
697        create("R1", "cf", "d", 11, KeyValue.Type.Put, "dont-care"), // inc
698        create("R1", "cf", "d", 10, KeyValue.Type.DeleteColumn, "dont-care"),
699        create("R1", "cf", "d", 9, KeyValue.Type.Put, "dont-care"),  // no
700        create("R1", "cf", "d", 8, KeyValue.Type.Put, "dont-care"),  // no
701
702    };
703    List<KeyValueScanner> scanners = scanFixture(kvs);
704    try (StoreScanner scan =
705        new StoreScanner(new Scan().readVersions(2), scanInfo, null, scanners)) {
706      List<Cell> results = new ArrayList<>();
707      assertEquals(true, scan.next(results));
708      assertEquals(5, results.size());
709      assertEquals(kvs[0], results.get(0));
710      assertEquals(kvs[2], results.get(1));
711      assertEquals(kvs[3], results.get(2));
712      assertEquals(kvs[6], results.get(3));
713      assertEquals(kvs[7], results.get(4));
714    }
715  }
716
717  @Test
718  public void testDeleteFamily() throws IOException {
719    KeyValue[] kvs = new KeyValue[] {
720        create("R1", "cf", "a", 100, KeyValue.Type.DeleteFamily, "dont-care"),
721        create("R1", "cf", "b", 11, KeyValue.Type.Put, "dont-care"),
722        create("R1", "cf", "c", 11, KeyValue.Type.Put, "dont-care"),
723        create("R1", "cf", "d", 11, KeyValue.Type.Put, "dont-care"),
724        create("R1", "cf", "e", 11, KeyValue.Type.Put, "dont-care"),
725        create("R1", "cf", "e", 11, KeyValue.Type.DeleteColumn, "dont-care"),
726        create("R1", "cf", "f", 11, KeyValue.Type.Put, "dont-care"),
727        create("R1", "cf", "g", 11, KeyValue.Type.Put, "dont-care"),
728        create("R1", "cf", "g", 11, KeyValue.Type.Delete, "dont-care"),
729        create("R1", "cf", "h", 11, KeyValue.Type.Put, "dont-care"),
730        create("R1", "cf", "i", 11, KeyValue.Type.Put, "dont-care"),
731        create("R2", "cf", "a", 11, KeyValue.Type.Put, "dont-care"),
732    };
733    List<KeyValueScanner> scanners = scanFixture(kvs);
734    try (StoreScanner scan =
735        new StoreScanner(new Scan().readAllVersions(), scanInfo, null, scanners)) {
736      List<Cell> results = new ArrayList<>();
737      assertEquals(true, scan.next(results));
738      assertEquals(0, results.size());
739      assertEquals(true, scan.next(results));
740      assertEquals(1, results.size());
741      assertEquals(kvs[kvs.length - 1], results.get(0));
742
743      assertEquals(false, scan.next(results));
744    }
745  }
746
747  @Test
748  public void testDeleteColumn() throws IOException {
749    KeyValue [] kvs = new KeyValue[] {
750        create("R1", "cf", "a", 10, KeyValue.Type.DeleteColumn, "dont-care"),
751        create("R1", "cf", "a", 9, KeyValue.Type.Delete, "dont-care"),
752        create("R1", "cf", "a", 8, KeyValue.Type.Put, "dont-care"),
753        create("R1", "cf", "b", 5, KeyValue.Type.Put, "dont-care")
754    };
755    List<KeyValueScanner> scanners = scanFixture(kvs);
756    try (StoreScanner scan = new StoreScanner(new Scan(), scanInfo, null, scanners)) {
757      List<Cell> results = new ArrayList<>();
758      assertEquals(true, scan.next(results));
759      assertEquals(1, results.size());
760      assertEquals(kvs[3], results.get(0));
761    }
762  }
763
764  private static final KeyValue[] kvs = new KeyValue[] {
765        create("R1", "cf", "a", 11, KeyValue.Type.Put, "dont-care"),
766        create("R1", "cf", "b", 11, KeyValue.Type.Put, "dont-care"),
767        create("R1", "cf", "c", 11, KeyValue.Type.Put, "dont-care"),
768        create("R1", "cf", "d", 11, KeyValue.Type.Put, "dont-care"),
769        create("R1", "cf", "e", 11, KeyValue.Type.Put, "dont-care"),
770        create("R1", "cf", "f", 11, KeyValue.Type.Put, "dont-care"),
771        create("R1", "cf", "g", 11, KeyValue.Type.Put, "dont-care"),
772        create("R1", "cf", "h", 11, KeyValue.Type.Put, "dont-care"),
773        create("R1", "cf", "i", 11, KeyValue.Type.Put, "dont-care"),
774        create("R2", "cf", "a", 11, KeyValue.Type.Put, "dont-care"),
775    };
776
777  @Test
778  public void testSkipColumn() throws IOException {
779    List<KeyValueScanner> scanners = scanFixture(kvs);
780    try (StoreScanner scan = new StoreScanner(new Scan(), scanInfo, getCols("a", "d"), scanners)) {
781      List<Cell> results = new ArrayList<>();
782      assertEquals(true, scan.next(results));
783      assertEquals(2, results.size());
784      assertEquals(kvs[0], results.get(0));
785      assertEquals(kvs[3], results.get(1));
786      results.clear();
787
788      assertEquals(true, scan.next(results));
789      assertEquals(1, results.size());
790      assertEquals(kvs[kvs.length - 1], results.get(0));
791
792      results.clear();
793      assertEquals(false, scan.next(results));
794    }
795  }
796
797  /*
798   * Test expiration of KeyValues in combination with a configured TTL for
799   * a column family (as should be triggered in a major compaction).
800   */
801  @Test
802  public void testWildCardTtlScan() throws IOException {
803    long now = System.currentTimeMillis();
804    KeyValue [] kvs = new KeyValue[] {
805        create("R1", "cf", "a", now-1000, KeyValue.Type.Put, "dont-care"),
806        create("R1", "cf", "b", now-10, KeyValue.Type.Put, "dont-care"),
807        create("R1", "cf", "c", now-200, KeyValue.Type.Put, "dont-care"),
808        create("R1", "cf", "d", now-10000, KeyValue.Type.Put, "dont-care"),
809        create("R2", "cf", "a", now, KeyValue.Type.Put, "dont-care"),
810        create("R2", "cf", "b", now-10, KeyValue.Type.Put, "dont-care"),
811        create("R2", "cf", "c", now-200, KeyValue.Type.Put, "dont-care"),
812        create("R2", "cf", "c", now-1000, KeyValue.Type.Put, "dont-care")
813    };
814    List<KeyValueScanner> scanners = scanFixture(kvs);
815    Scan scan = new Scan();
816    scan.readVersions(1);
817    ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, 500, KeepDeletedCells.FALSE,
818        HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false);
819    try (StoreScanner scanner = new StoreScanner(scan, scanInfo, null, scanners)) {
820      List<Cell> results = new ArrayList<>();
821      assertEquals(true, scanner.next(results));
822      assertEquals(2, results.size());
823      assertEquals(kvs[1], results.get(0));
824      assertEquals(kvs[2], results.get(1));
825      results.clear();
826
827      assertEquals(true, scanner.next(results));
828      assertEquals(3, results.size());
829      assertEquals(kvs[4], results.get(0));
830      assertEquals(kvs[5], results.get(1));
831      assertEquals(kvs[6], results.get(2));
832      results.clear();
833
834      assertEquals(false, scanner.next(results));
835    }
836  }
837
838  @Test
839  public void testScannerReseekDoesntNPE() throws Exception {
840    List<KeyValueScanner> scanners = scanFixture(kvs);
841    try (StoreScanner scan = new StoreScanner(new Scan(), scanInfo, getCols("a", "d"), scanners)) {
842      // Previously a updateReaders twice in a row would cause an NPE. In test this would also
843      // normally cause an NPE because scan.store is null. So as long as we get through these
844      // two calls we are good and the bug was quashed.
845      scan.updateReaders(Collections.emptyList(), Collections.emptyList());
846      scan.updateReaders(Collections.emptyList(), Collections.emptyList());
847      scan.peek();
848    }
849  }
850
851  @Test @Ignore("this fails, since we don't handle deletions, etc, in peek")
852  public void testPeek() throws Exception {
853    KeyValue[] kvs = new KeyValue [] {
854        create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"),
855        create("R1", "cf", "a", 1, KeyValue.Type.Delete, "dont-care"),
856    };
857    List<KeyValueScanner> scanners = scanFixture(kvs);
858    Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1"));
859    try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) {
860      assertNull(scan.peek());
861    }
862  }
863
864  /**
865   * Ensure that expired delete family markers don't override valid puts
866   */
867  @Test
868  public void testExpiredDeleteFamily() throws Exception {
869    long now = System.currentTimeMillis();
870    KeyValue[] kvs = new KeyValue[] {
871        new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null, now-1000,
872            KeyValue.Type.DeleteFamily),
873        create("R1", "cf", "a", now-10, KeyValue.Type.Put,
874            "dont-care"),
875    };
876    List<KeyValueScanner> scanners = scanFixture(kvs);
877    Scan scan = new Scan();
878    scan.readVersions(1);
879    // scanner with ttl equal to 500
880    ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, 500, KeepDeletedCells.FALSE,
881        HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false);
882    try (StoreScanner scanner = new StoreScanner(scan, scanInfo, null, scanners)) {
883      List<Cell> results = new ArrayList<>();
884      assertEquals(true, scanner.next(results));
885      assertEquals(1, results.size());
886      assertEquals(kvs[1], results.get(0));
887      results.clear();
888
889      assertEquals(false, scanner.next(results));
890    }
891  }
892
893  @Test
894  public void testDeleteMarkerLongevity() throws Exception {
895    try {
896      final long now = System.currentTimeMillis();
897      EnvironmentEdgeManagerTestHelper.injectEdge(new EnvironmentEdge() {
898        @Override
899        public long currentTime() {
900          return now;
901        }
902      });
903      KeyValue[] kvs = new KeyValue[]{
904        /*0*/ new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null,
905        now - 100, KeyValue.Type.DeleteFamily), // live
906        /*1*/ new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null,
907        now - 1000, KeyValue.Type.DeleteFamily), // expired
908        /*2*/ create("R1", "cf", "a", now - 50,
909        KeyValue.Type.Put, "v3"), // live
910        /*3*/ create("R1", "cf", "a", now - 55,
911        KeyValue.Type.Delete, "dontcare"), // live
912        /*4*/ create("R1", "cf", "a", now - 55,
913        KeyValue.Type.Put, "deleted-version v2"), // deleted
914        /*5*/ create("R1", "cf", "a", now - 60,
915        KeyValue.Type.Put, "v1"), // live
916        /*6*/ create("R1", "cf", "a", now - 65,
917        KeyValue.Type.Put, "v0"), // max-version reached
918        /*7*/ create("R1", "cf", "a",
919        now - 100, KeyValue.Type.DeleteColumn, "dont-care"), // max-version
920        /*8*/ create("R1", "cf", "b", now - 600,
921        KeyValue.Type.DeleteColumn, "dont-care"), //expired
922        /*9*/ create("R1", "cf", "b", now - 70,
923        KeyValue.Type.Put, "v2"), //live
924        /*10*/ create("R1", "cf", "b", now - 750,
925        KeyValue.Type.Put, "v1"), //expired
926        /*11*/ create("R1", "cf", "c", now - 500,
927        KeyValue.Type.Delete, "dontcare"), //expired
928        /*12*/ create("R1", "cf", "c", now - 600,
929        KeyValue.Type.Put, "v1"), //expired
930        /*13*/ create("R1", "cf", "c", now - 1000,
931        KeyValue.Type.Delete, "dontcare"), //expired
932        /*14*/ create("R1", "cf", "d", now - 60,
933        KeyValue.Type.Put, "expired put"), //live
934        /*15*/ create("R1", "cf", "d", now - 100,
935        KeyValue.Type.Delete, "not-expired delete"), //live
936      };
937      List<KeyValueScanner> scanners = scanFixture(kvs);
938      ScanInfo scanInfo = new ScanInfo(CONF, Bytes.toBytes("cf"),
939        0 /* minVersions */,
940        2 /* maxVersions */, 500 /* ttl */,
941        KeepDeletedCells.FALSE /* keepDeletedCells */,
942        HConstants.DEFAULT_BLOCKSIZE /* block size */,
943        200, /* timeToPurgeDeletes */
944        CellComparator.getInstance(), false);
945      try (StoreScanner scanner =
946          new StoreScanner(scanInfo, 2, ScanType.COMPACT_DROP_DELETES, scanners)) {
947        List<Cell> results = new ArrayList<>();
948        results = new ArrayList<>();
949        assertEquals(true, scanner.next(results));
950        assertEquals(kvs[0], results.get(0));
951        assertEquals(kvs[2], results.get(1));
952        assertEquals(kvs[3], results.get(2));
953        assertEquals(kvs[5], results.get(3));
954        assertEquals(kvs[9], results.get(4));
955        assertEquals(kvs[14], results.get(5));
956        assertEquals(kvs[15], results.get(6));
957        assertEquals(7, results.size());
958      }
959    } finally {
960      EnvironmentEdgeManagerTestHelper.reset();
961    }
962  }
963
964  @Test
965  public void testPreadNotEnabledForCompactionStoreScanners() throws Exception {
966    long now = System.currentTimeMillis();
967    KeyValue[] kvs = new KeyValue[] {
968        new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null, now - 1000,
969            KeyValue.Type.DeleteFamily),
970        create("R1", "cf", "a", now - 10, KeyValue.Type.Put, "dont-care"), };
971    List<KeyValueScanner> scanners = scanFixture(kvs);
972    ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, 500, KeepDeletedCells.FALSE,
973        HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false);
974    try (StoreScanner storeScanner = new StoreScanner(scanInfo, -1,
975      ScanType.COMPACT_RETAIN_DELETES, scanners)) {
976      assertFalse(storeScanner.isScanUsePread());
977    }
978  }
979}