001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.regionserver; 019 020import static org.apache.hadoop.hbase.KeyValueTestUtil.create; 021import static org.apache.hadoop.hbase.regionserver.KeyValueScanFixture.scanFixture; 022import static org.junit.Assert.assertEquals; 023import static org.junit.Assert.assertFalse; 024import static org.junit.Assert.assertNull; 025import static org.junit.Assert.assertSame; 026import static org.junit.Assert.assertTrue; 027 028import java.io.IOException; 029import java.util.ArrayList; 030import java.util.Arrays; 031import java.util.Collections; 032import java.util.List; 033import java.util.NavigableSet; 034import java.util.TreeSet; 035import java.util.concurrent.atomic.AtomicInteger; 036import org.apache.hadoop.conf.Configuration; 037import org.apache.hadoop.hbase.Cell; 038import org.apache.hadoop.hbase.CellBuilderType; 039import org.apache.hadoop.hbase.CellComparator; 040import org.apache.hadoop.hbase.CellUtil; 041import org.apache.hadoop.hbase.CompareOperator; 042import org.apache.hadoop.hbase.ExtendedCellBuilderFactory; 043import org.apache.hadoop.hbase.HBaseClassTestRule; 044import org.apache.hadoop.hbase.HBaseConfiguration; 045import org.apache.hadoop.hbase.HBaseTestingUtil; 046import org.apache.hadoop.hbase.HConstants; 047import org.apache.hadoop.hbase.KeepDeletedCells; 048import org.apache.hadoop.hbase.KeyValue; 049import org.apache.hadoop.hbase.PrivateCellUtil; 050import org.apache.hadoop.hbase.client.Get; 051import org.apache.hadoop.hbase.client.Scan; 052import org.apache.hadoop.hbase.filter.BinaryComparator; 053import org.apache.hadoop.hbase.filter.ColumnCountGetFilter; 054import org.apache.hadoop.hbase.filter.Filter; 055import org.apache.hadoop.hbase.filter.QualifierFilter; 056import org.apache.hadoop.hbase.testclassification.RegionServerTests; 057import org.apache.hadoop.hbase.testclassification.SmallTests; 058import org.apache.hadoop.hbase.util.Bytes; 059import org.apache.hadoop.hbase.util.CollectionBackedScanner; 060import org.apache.hadoop.hbase.util.EnvironmentEdge; 061import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; 062import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper; 063import org.junit.ClassRule; 064import org.junit.Ignore; 065import org.junit.Rule; 066import org.junit.Test; 067import org.junit.experimental.categories.Category; 068import org.junit.rules.TestName; 069import org.slf4j.Logger; 070import org.slf4j.LoggerFactory; 071 072// Can't be small as it plays with EnvironmentEdgeManager 073@Category({ RegionServerTests.class, SmallTests.class }) 074public class TestStoreScanner { 075 076 @ClassRule 077 public static final HBaseClassTestRule CLASS_RULE = 078 HBaseClassTestRule.forClass(TestStoreScanner.class); 079 080 private static final Logger LOG = LoggerFactory.getLogger(TestStoreScanner.class); 081 @Rule 082 public TestName name = new TestName(); 083 private static final String CF_STR = "cf"; 084 private static final byte[] CF = Bytes.toBytes(CF_STR); 085 static Configuration CONF = HBaseConfiguration.create(); 086 private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil(); 087 private ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, Integer.MAX_VALUE, Long.MAX_VALUE, 088 KeepDeletedCells.FALSE, HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false); 089 090 /** 091 * From here on down, we have a bunch of defines and specific CELL_GRID of Cells. The CELL_GRID 092 * then has a Scanner that can fake out 'block' transitions. All this elaborate setup is for tests 093 * that ensure we don't overread, and that the {@link StoreScanner} is not overly enthusiastic. 094 */ 095 private static final byte[] ZERO = new byte[] { '0' }; 096 private static final byte[] ZERO_POINT_ZERO = new byte[] { '0', '.', '0' }; 097 private static final byte[] ONE = new byte[] { '1' }; 098 private static final byte[] TWO = new byte[] { '2' }; 099 private static final byte[] TWO_POINT_TWO = new byte[] { '2', '.', '2' }; 100 private static final byte[] THREE = new byte[] { '3' }; 101 private static final byte[] FOUR = new byte[] { '4' }; 102 private static final byte[] FIVE = new byte[] { '5' }; 103 private static final byte[] VALUE = new byte[] { 'v' }; 104 private static final int CELL_GRID_BLOCK2_BOUNDARY = 4; 105 private static final int CELL_GRID_BLOCK3_BOUNDARY = 11; 106 private static final int CELL_GRID_BLOCK4_BOUNDARY = 15; 107 private static final int CELL_GRID_BLOCK5_BOUNDARY = 19; 108 109 /** 110 * Five rows by four columns distinguished by column qualifier (column qualifier is one of the 111 * four rows... ONE, TWO, etc.). Exceptions are a weird row after TWO; it is TWO_POINT_TWO. And 112 * then row FOUR has five columns finishing w/ row FIVE having a single column. We will use this 113 * to test scan does the right thing as it we do Gets, StoreScanner#optimize, and what we do on 114 * (faked) block boundaries. 115 */ 116 private static final Cell[] CELL_GRID = new Cell[] { 117 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(ONE).setFamily(CF) 118 .setQualifier(ONE).setTimestamp(1L).setType(KeyValue.Type.Put.getCode()).setValue(VALUE) 119 .build(), 120 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(ONE).setFamily(CF) 121 .setQualifier(TWO).setTimestamp(1L).setType(KeyValue.Type.Put.getCode()).setValue(VALUE) 122 .build(), 123 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(ONE).setFamily(CF) 124 .setQualifier(THREE).setTimestamp(1L).setType(KeyValue.Type.Put.getCode()).setValue(VALUE) 125 .build(), 126 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(ONE).setFamily(CF) 127 .setQualifier(FOUR).setTimestamp(1L).setType(KeyValue.Type.Put.getCode()).setValue(VALUE) 128 .build(), 129 // Offset 4 CELL_GRID_BLOCK2_BOUNDARY 130 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(TWO).setFamily(CF) 131 .setQualifier(ONE).setTimestamp(1L).setType(KeyValue.Type.Put.getCode()).setValue(VALUE) 132 .build(), 133 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(TWO).setFamily(CF) 134 .setQualifier(TWO).setTimestamp(1L).setType(KeyValue.Type.Put.getCode()).setValue(VALUE) 135 .build(), 136 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(TWO).setFamily(CF) 137 .setQualifier(THREE).setTimestamp(1L).setType(KeyValue.Type.Put.getCode()).setValue(VALUE) 138 .build(), 139 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(TWO).setFamily(CF) 140 .setQualifier(FOUR).setTimestamp(1L).setType(KeyValue.Type.Put.getCode()).setValue(VALUE) 141 .build(), 142 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(TWO_POINT_TWO).setFamily(CF) 143 .setQualifier(ZERO).setTimestamp(1L).setType(KeyValue.Type.Put.getCode()).setValue(VALUE) 144 .build(), 145 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(TWO_POINT_TWO).setFamily(CF) 146 .setQualifier(ZERO_POINT_ZERO).setTimestamp(1L).setType(KeyValue.Type.Put.getCode()) 147 .setValue(VALUE).build(), 148 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(TWO_POINT_TWO).setFamily(CF) 149 .setQualifier(FIVE).setTimestamp(1L).setType(KeyValue.Type.Put.getCode()).setValue(VALUE) 150 .build(), 151 // Offset 11! CELL_GRID_BLOCK3_BOUNDARY 152 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(THREE).setFamily(CF) 153 .setQualifier(ONE).setTimestamp(1L).setType(KeyValue.Type.Put.getCode()).setValue(VALUE) 154 .build(), 155 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(THREE).setFamily(CF) 156 .setQualifier(TWO).setTimestamp(1L).setType(KeyValue.Type.Put.getCode()).setValue(VALUE) 157 .build(), 158 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(THREE).setFamily(CF) 159 .setQualifier(THREE).setTimestamp(1L).setType(KeyValue.Type.Put.getCode()).setValue(VALUE) 160 .build(), 161 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(THREE).setFamily(CF) 162 .setQualifier(FOUR).setTimestamp(1L).setType(KeyValue.Type.Put.getCode()).setValue(VALUE) 163 .build(), 164 // Offset 15 CELL_GRID_BLOCK4_BOUNDARY 165 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(FOUR).setFamily(CF) 166 .setQualifier(ONE).setTimestamp(1L).setType(KeyValue.Type.Put.getCode()).setValue(VALUE) 167 .build(), 168 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(FOUR).setFamily(CF) 169 .setQualifier(TWO).setTimestamp(1L).setType(KeyValue.Type.Put.getCode()).setValue(VALUE) 170 .build(), 171 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(FOUR).setFamily(CF) 172 .setQualifier(THREE).setTimestamp(1L).setType(KeyValue.Type.Put.getCode()).setValue(VALUE) 173 .build(), 174 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(FOUR).setFamily(CF) 175 .setQualifier(FOUR).setTimestamp(1L).setType(KeyValue.Type.Put.getCode()).setValue(VALUE) 176 .build(), 177 // Offset 19 CELL_GRID_BLOCK5_BOUNDARY 178 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(FOUR).setFamily(CF) 179 .setQualifier(FIVE).setTimestamp(1L).setType(KeyValue.Type.Put.getCode()).setValue(VALUE) 180 .build(), 181 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(FIVE).setFamily(CF) 182 .setQualifier(ZERO).setTimestamp(1L).setType(KeyValue.Type.Put.getCode()).setValue(VALUE) 183 .build(), }; 184 185 private static class KeyValueHeapWithCount extends KeyValueHeap { 186 187 final AtomicInteger count; 188 189 public KeyValueHeapWithCount(List<? extends KeyValueScanner> scanners, 190 CellComparator comparator, AtomicInteger count) throws IOException { 191 super(scanners, comparator); 192 this.count = count; 193 } 194 195 @Override 196 public Cell peek() { 197 this.count.incrementAndGet(); 198 return super.peek(); 199 } 200 } 201 202 /** 203 * A StoreScanner for our CELL_GRID above. Fakes the block transitions. Does counts of calls to 204 * optimize and counts of when optimize actually did an optimize. 205 */ 206 private static class CellGridStoreScanner extends StoreScanner { 207 // Count of how often optimize is called and of how often it does an optimize. 208 AtomicInteger count; 209 final AtomicInteger optimization = new AtomicInteger(0); 210 211 CellGridStoreScanner(final Scan scan, ScanInfo scanInfo) throws IOException { 212 super(scan, scanInfo, scan.getFamilyMap().get(CF), 213 Arrays.<KeyValueScanner> asList(new KeyValueScanner[] { 214 new KeyValueScanFixture(CellComparator.getInstance(), CELL_GRID) })); 215 } 216 217 @Override 218 protected void resetKVHeap(List<? extends KeyValueScanner> scanners, CellComparator comparator) 219 throws IOException { 220 if (count == null) { 221 count = new AtomicInteger(0); 222 } 223 heap = newKVHeap(scanners, comparator); 224 } 225 226 @Override 227 protected KeyValueHeap newKVHeap(List<? extends KeyValueScanner> scanners, 228 CellComparator comparator) throws IOException { 229 return new KeyValueHeapWithCount(scanners, comparator, count); 230 } 231 232 @Override 233 protected boolean trySkipToNextRow(Cell cell) throws IOException { 234 boolean optimized = super.trySkipToNextRow(cell); 235 LOG.info("Cell=" + cell + ", nextIndex=" + CellUtil.toString(getNextIndexedKey(), false) 236 + ", optimized=" + optimized); 237 if (optimized) { 238 optimization.incrementAndGet(); 239 } 240 return optimized; 241 } 242 243 @Override 244 protected boolean trySkipToNextColumn(Cell cell) throws IOException { 245 boolean optimized = super.trySkipToNextColumn(cell); 246 LOG.info("Cell=" + cell + ", nextIndex=" + CellUtil.toString(getNextIndexedKey(), false) 247 + ", optimized=" + optimized); 248 if (optimized) { 249 optimization.incrementAndGet(); 250 } 251 return optimized; 252 } 253 254 @Override 255 public Cell getNextIndexedKey() { 256 // Fake block boundaries by having index of next block change as we go through scan. 257 return count.get() > CELL_GRID_BLOCK4_BOUNDARY 258 ? PrivateCellUtil.createFirstOnRow(CELL_GRID[CELL_GRID_BLOCK5_BOUNDARY]) 259 : count.get() > CELL_GRID_BLOCK3_BOUNDARY 260 ? PrivateCellUtil.createFirstOnRow(CELL_GRID[CELL_GRID_BLOCK4_BOUNDARY]) 261 : count.get() > CELL_GRID_BLOCK2_BOUNDARY 262 ? PrivateCellUtil.createFirstOnRow(CELL_GRID[CELL_GRID_BLOCK3_BOUNDARY]) 263 : PrivateCellUtil.createFirstOnRow(CELL_GRID[CELL_GRID_BLOCK2_BOUNDARY]); 264 } 265 } 266 267 private static final int CELL_WITH_VERSIONS_BLOCK2_BOUNDARY = 4; 268 269 private static final Cell[] CELL_WITH_VERSIONS = new Cell[] { 270 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(ONE).setFamily(CF) 271 .setQualifier(ONE).setTimestamp(2L).setType(KeyValue.Type.Put.getCode()).setValue(VALUE) 272 .build(), 273 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(ONE).setFamily(CF) 274 .setQualifier(ONE).setTimestamp(1L).setType(KeyValue.Type.Put.getCode()).setValue(VALUE) 275 .build(), 276 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(ONE).setFamily(CF) 277 .setQualifier(TWO).setTimestamp(2L).setType(KeyValue.Type.Put.getCode()).setValue(VALUE) 278 .build(), 279 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(ONE).setFamily(CF) 280 .setQualifier(TWO).setTimestamp(1L).setType(KeyValue.Type.Put.getCode()).setValue(VALUE) 281 .build(), 282 // Offset 4 CELL_WITH_VERSIONS_BLOCK2_BOUNDARY 283 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(TWO).setFamily(CF) 284 .setQualifier(ONE).setTimestamp(1L).setType(KeyValue.Type.Put.getCode()).setValue(VALUE) 285 .build(), 286 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(TWO).setFamily(CF) 287 .setQualifier(TWO).setTimestamp(1L).setType(KeyValue.Type.Put.getCode()).setValue(VALUE) 288 .build(), }; 289 290 private static class CellWithVersionsStoreScanner extends StoreScanner { 291 // Count of how often optimize is called and of how often it does an optimize. 292 final AtomicInteger optimization = new AtomicInteger(0); 293 294 CellWithVersionsStoreScanner(final Scan scan, ScanInfo scanInfo) throws IOException { 295 super(scan, scanInfo, scan.getFamilyMap().get(CF), 296 Arrays.<KeyValueScanner> asList(new KeyValueScanner[] { 297 new KeyValueScanFixture(CellComparator.getInstance(), CELL_WITH_VERSIONS) })); 298 } 299 300 @Override 301 protected boolean trySkipToNextColumn(Cell cell) throws IOException { 302 boolean optimized = super.trySkipToNextColumn(cell); 303 LOG.info("Cell=" + cell + ", nextIndex=" + CellUtil.toString(getNextIndexedKey(), false) 304 + ", optimized=" + optimized); 305 if (optimized) { 306 optimization.incrementAndGet(); 307 } 308 return optimized; 309 } 310 311 @Override 312 public Cell getNextIndexedKey() { 313 // Fake block boundaries by having index of next block change as we go through scan. 314 return PrivateCellUtil 315 .createFirstOnRow(CELL_WITH_VERSIONS[CELL_WITH_VERSIONS_BLOCK2_BOUNDARY]); 316 } 317 } 318 319 private static class CellWithVersionsNoOptimizeStoreScanner extends StoreScanner { 320 // Count of how often optimize is called and of how often it does an optimize. 321 final AtomicInteger optimization = new AtomicInteger(0); 322 323 CellWithVersionsNoOptimizeStoreScanner(Scan scan, ScanInfo scanInfo) throws IOException { 324 super(scan, scanInfo, scan.getFamilyMap().get(CF), 325 Arrays.<KeyValueScanner> asList(new KeyValueScanner[] { 326 new KeyValueScanFixture(CellComparator.getInstance(), CELL_WITH_VERSIONS) })); 327 } 328 329 @Override 330 protected boolean trySkipToNextColumn(Cell cell) throws IOException { 331 boolean optimized = super.trySkipToNextColumn(cell); 332 LOG.info("Cell=" + cell + ", nextIndex=" + CellUtil.toString(getNextIndexedKey(), false) 333 + ", optimized=" + optimized); 334 if (optimized) { 335 optimization.incrementAndGet(); 336 } 337 return optimized; 338 } 339 340 @Override 341 public Cell getNextIndexedKey() { 342 return null; 343 } 344 } 345 346 @Test 347 public void testWithColumnCountGetFilter() throws Exception { 348 Get get = new Get(ONE); 349 get.readAllVersions(); 350 get.addFamily(CF); 351 get.setFilter(new ColumnCountGetFilter(2)); 352 353 try (CellWithVersionsNoOptimizeStoreScanner scannerNoOptimize = 354 new CellWithVersionsNoOptimizeStoreScanner(new Scan(get), this.scanInfo)) { 355 List<Cell> results = new ArrayList<>(); 356 while (scannerNoOptimize.next(results)) { 357 continue; 358 } 359 assertEquals(2, results.size()); 360 assertTrue(CellUtil.matchingColumn(results.get(0), CELL_WITH_VERSIONS[0])); 361 assertTrue(CellUtil.matchingColumn(results.get(1), CELL_WITH_VERSIONS[2])); 362 assertTrue("Optimize should do some optimizations", 363 scannerNoOptimize.optimization.get() == 0); 364 } 365 366 get.setFilter(new ColumnCountGetFilter(2)); 367 try (CellWithVersionsStoreScanner scanner = 368 new CellWithVersionsStoreScanner(new Scan(get), this.scanInfo)) { 369 List<Cell> results = new ArrayList<>(); 370 while (scanner.next(results)) { 371 continue; 372 } 373 assertEquals(2, results.size()); 374 assertTrue(CellUtil.matchingColumn(results.get(0), CELL_WITH_VERSIONS[0])); 375 assertTrue(CellUtil.matchingColumn(results.get(1), CELL_WITH_VERSIONS[2])); 376 assertTrue("Optimize should do some optimizations", scanner.optimization.get() > 0); 377 } 378 } 379 380 /* 381 * Test utility for building a NavigableSet for scanners. 382 */ 383 NavigableSet<byte[]> getCols(String... strCols) { 384 NavigableSet<byte[]> cols = new TreeSet<>(Bytes.BYTES_COMPARATOR); 385 for (String col : strCols) { 386 byte[] bytes = Bytes.toBytes(col); 387 cols.add(bytes); 388 } 389 return cols; 390 } 391 392 @Test 393 public void testFullRowGetDoesNotOverreadWhenRowInsideOneBlock() throws IOException { 394 // Do a Get against row two. Row two is inside a block that starts with row TWO but ends with 395 // row TWO_POINT_TWO. We should read one block only. 396 Get get = new Get(TWO); 397 Scan scan = new Scan(get); 398 try (CellGridStoreScanner scanner = new CellGridStoreScanner(scan, this.scanInfo)) { 399 List<Cell> results = new ArrayList<>(); 400 while (scanner.next(results)) { 401 continue; 402 } 403 // Should be four results of column 1 (though there are 5 rows in the CELL_GRID -- the 404 // TWO_POINT_TWO row does not have a a column ONE. 405 assertEquals(4, results.size()); 406 // We should have gone the optimize route 5 times totally... an INCLUDE for the four cells 407 // in the row plus the DONE on the end. 408 assertEquals(5, scanner.count.get()); 409 assertEquals(1, scanner.memstoreOnlyReads); 410 // For a full row Get, there should be no opportunity for scanner optimization. 411 assertEquals(0, scanner.optimization.get()); 412 } 413 } 414 415 @Test 416 public void testFullRowSpansBlocks() throws IOException { 417 // Do a Get against row FOUR. It spans two blocks. 418 Get get = new Get(FOUR); 419 Scan scan = new Scan(get); 420 try (CellGridStoreScanner scanner = new CellGridStoreScanner(scan, this.scanInfo)) { 421 List<Cell> results = new ArrayList<>(); 422 while (scanner.next(results)) { 423 continue; 424 } 425 // Should be four results of column 1 (though there are 5 rows in the CELL_GRID -- the 426 // TWO_POINT_TWO row does not have a a column ONE. 427 assertEquals(5, results.size()); 428 // We should have gone the optimize route 6 times totally... an INCLUDE for the five cells 429 // in the row plus the DONE on the end. 430 assertEquals(6, scanner.count.get()); 431 // For a full row Get, there should be no opportunity for scanner optimization. 432 assertEquals(0, scanner.optimization.get()); 433 } 434 } 435 436 /** 437 * Test optimize in StoreScanner. Test that we skip to the next 'block' when we it makes sense 438 * reading the block 'index'. 439 */ 440 @Test 441 public void testOptimize() throws IOException { 442 Scan scan = new Scan(); 443 // A scan that just gets the first qualifier on each row of the CELL_GRID 444 scan.addColumn(CF, ONE); 445 try (CellGridStoreScanner scanner = new CellGridStoreScanner(scan, this.scanInfo)) { 446 List<Cell> results = new ArrayList<>(); 447 while (scanner.next(results)) { 448 continue; 449 } 450 // Should be four results of column 1 (though there are 5 rows in the CELL_GRID -- the 451 // TWO_POINT_TWO row does not have a a column ONE. 452 assertEquals(4, results.size()); 453 for (Cell cell : results) { 454 assertTrue(Bytes.equals(ONE, 0, ONE.length, cell.getQualifierArray(), 455 cell.getQualifierOffset(), cell.getQualifierLength())); 456 } 457 assertTrue("Optimize should do some optimizations", scanner.optimization.get() > 0); 458 } 459 } 460 461 /** 462 * Ensure the optimize Scan method in StoreScanner does not get in the way of a Get doing minimum 463 * work... seeking to start of block and then SKIPPING until we find the wanted Cell. This 464 * 'simple' scenario mimics case of all Cells fitting inside a single HFileBlock. See HBASE-15392. 465 * This test is a little cryptic. Takes a bit of staring to figure what it up to. 466 */ 467 @Test 468 public void testOptimizeAndGet() throws IOException { 469 // First test a Get of two columns in the row R2. Every Get is a Scan. Get columns named 470 // R2 and R3. 471 Get get = new Get(TWO); 472 get.addColumn(CF, TWO); 473 get.addColumn(CF, THREE); 474 Scan scan = new Scan(get); 475 try (CellGridStoreScanner scanner = new CellGridStoreScanner(scan, this.scanInfo)) { 476 List<Cell> results = new ArrayList<>(); 477 // For a Get there should be no more next's after the first call. 478 assertEquals(false, scanner.next(results)); 479 // Should be one result only. 480 assertEquals(2, results.size()); 481 // And we should have gone through optimize twice only. 482 assertEquals("First qcode is SEEK_NEXT_COL and second INCLUDE_AND_SEEK_NEXT_ROW", 3, 483 scanner.count.get()); 484 assertEquals("Memstore Read count should be", 1, scanner.memstoreOnlyReads); 485 } 486 } 487 488 /** 489 * Ensure that optimize does not cause the Get to do more seeking than required. Optimize (see 490 * HBASE-15392) was causing us to seek all Cells in a block when a Get Scan if the next block 491 * index/start key was a different row to the current one. A bug. We'd call next too often because 492 * we had to exhaust all Cells in the current row making us load the next block just to discard 493 * what we read there. This test is a little cryptic. Takes a bit of staring to figure what it up 494 * to. 495 */ 496 @Test 497 public void testOptimizeAndGetWithFakedNextBlockIndexStart() throws IOException { 498 // First test a Get of second column in the row R2. Every Get is a Scan. Second column has a 499 // qualifier of R2. 500 Get get = new Get(THREE); 501 get.addColumn(CF, TWO); 502 Scan scan = new Scan(get); 503 try (CellGridStoreScanner scanner = new CellGridStoreScanner(scan, this.scanInfo)) { 504 List<Cell> results = new ArrayList<>(); 505 // For a Get there should be no more next's after the first call. 506 assertEquals(false, scanner.next(results)); 507 // Should be one result only. 508 assertEquals(1, results.size()); 509 // And we should have gone through optimize twice only. 510 assertEquals("First qcode is SEEK_NEXT_COL and second INCLUDE_AND_SEEK_NEXT_ROW", 2, 511 scanner.count.get()); 512 } 513 } 514 515 @Test 516 public void testScanTimeRange() throws IOException { 517 String r1 = "R1"; 518 // returns only 1 of these 2 even though same timestamp 519 KeyValue[] kvs = new KeyValue[] { create(r1, CF_STR, "a", 1, KeyValue.Type.Put, "dont-care"), 520 create(r1, CF_STR, "a", 2, KeyValue.Type.Put, "dont-care"), 521 create(r1, CF_STR, "a", 3, KeyValue.Type.Put, "dont-care"), 522 create(r1, CF_STR, "a", 4, KeyValue.Type.Put, "dont-care"), 523 create(r1, CF_STR, "a", 5, KeyValue.Type.Put, "dont-care"), }; 524 List<KeyValueScanner> scanners = Arrays.<KeyValueScanner> asList( 525 new KeyValueScanner[] { new KeyValueScanFixture(CellComparator.getInstance(), kvs) }); 526 Scan scanSpec = new Scan().withStartRow(Bytes.toBytes(r1)); 527 scanSpec.setTimeRange(0, 6); 528 scanSpec.readAllVersions(); 529 List<Cell> results = null; 530 try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) { 531 results = new ArrayList<>(); 532 assertEquals(true, scan.next(results)); 533 assertEquals(5, results.size()); 534 assertEquals(kvs[kvs.length - 1], results.get(0)); 535 } 536 // Scan limited TimeRange 537 scanSpec = new Scan().withStartRow(Bytes.toBytes(r1)); 538 scanSpec.setTimeRange(1, 3); 539 scanSpec.readAllVersions(); 540 try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) { 541 results = new ArrayList<>(); 542 assertEquals(true, scan.next(results)); 543 assertEquals(2, results.size()); 544 } 545 // Another range. 546 scanSpec = new Scan().withStartRow(Bytes.toBytes(r1)); 547 scanSpec.setTimeRange(5, 10); 548 scanSpec.readAllVersions(); 549 try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) { 550 results = new ArrayList<>(); 551 assertEquals(true, scan.next(results)); 552 assertEquals(1, results.size()); 553 } 554 // See how TimeRange and Versions interact. 555 // Another range. 556 scanSpec = new Scan().withStartRow(Bytes.toBytes(r1)); 557 scanSpec.setTimeRange(0, 10); 558 scanSpec.readVersions(3); 559 try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) { 560 results = new ArrayList<>(); 561 assertEquals(true, scan.next(results)); 562 assertEquals(3, results.size()); 563 } 564 } 565 566 @Test 567 public void testScanSameTimestamp() throws IOException { 568 // returns only 1 of these 2 even though same timestamp 569 KeyValue[] kvs = new KeyValue[] { create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"), 570 create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"), }; 571 List<KeyValueScanner> scanners = Arrays 572 .asList(new KeyValueScanner[] { new KeyValueScanFixture(CellComparator.getInstance(), kvs) }); 573 574 Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1")); 575 // this only uses maxVersions (default=1) and TimeRange (default=all) 576 try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, null, scanners)) { 577 List<Cell> results = new ArrayList<>(); 578 assertEquals(true, scan.next(results)); 579 assertEquals(1, results.size()); 580 assertEquals(1, scan.memstoreOnlyReads); 581 assertEquals(kvs[0], results.get(0)); 582 } 583 } 584 585 @Test 586 public void testNonUserScan() throws IOException { 587 // returns only 1 of these 2 even though same timestamp 588 KeyValue[] kvs = new KeyValue[] { create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"), 589 create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"), }; 590 List<KeyValueScanner> scanners = Arrays 591 .asList(new KeyValueScanner[] { new KeyValueScanFixture(CellComparator.getInstance(), kvs) }); 592 593 Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1")); 594 // this only uses maxVersions (default=1) and TimeRange (default=all) 595 try (StoreScanner scan = 596 new StoreScanner(scanSpec, scanInfo, null, scanners, ScanType.COMPACT_RETAIN_DELETES)) { 597 List<Cell> results = new ArrayList<>(); 598 assertEquals(true, scan.next(results)); 599 assertEquals(1, results.size()); 600 // the type is not a user scan. so it won't account for the memstore reads 601 assertEquals(0, scan.memstoreOnlyReads); 602 assertEquals(kvs[0], results.get(0)); 603 } 604 } 605 606 /* 607 * Test test shows exactly how the matcher's return codes confuses the StoreScanner and prevent it 608 * from doing the right thing. Seeking once, then nexting twice should return R1, then R2, but in 609 * this case it doesnt. TODO this comment makes no sense above. Appears to do the right thing. 610 */ 611 @Test 612 public void testWontNextToNext() throws IOException { 613 // build the scan file: 614 KeyValue[] kvs = new KeyValue[] { create("R1", "cf", "a", 2, KeyValue.Type.Put, "dont-care"), 615 create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"), 616 create("R2", "cf", "a", 1, KeyValue.Type.Put, "dont-care") }; 617 List<KeyValueScanner> scanners = scanFixture(kvs); 618 619 Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1")); 620 // this only uses maxVersions (default=1) and TimeRange (default=all) 621 try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) { 622 List<Cell> results = new ArrayList<>(); 623 scan.next(results); 624 assertEquals(1, results.size()); 625 assertEquals(kvs[0], results.get(0)); 626 // should be ok... 627 // now scan _next_ again. 628 results.clear(); 629 scan.next(results); 630 assertEquals(1, results.size()); 631 assertEquals(kvs[2], results.get(0)); 632 633 results.clear(); 634 scan.next(results); 635 assertEquals(0, results.size()); 636 } 637 } 638 639 @Test 640 public void testDeleteVersionSameTimestamp() throws IOException { 641 KeyValue[] kvs = new KeyValue[] { create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"), 642 create("R1", "cf", "a", 1, KeyValue.Type.Delete, "dont-care"), }; 643 List<KeyValueScanner> scanners = scanFixture(kvs); 644 Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1")); 645 try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) { 646 List<Cell> results = new ArrayList<>(); 647 assertFalse(scan.next(results)); 648 assertEquals(0, results.size()); 649 } 650 } 651 652 /* 653 * Test the case where there is a delete row 'in front of' the next row, the scanner will move to 654 * the next row. 655 */ 656 @Test 657 public void testDeletedRowThenGoodRow() throws IOException { 658 KeyValue[] kvs = new KeyValue[] { create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"), 659 create("R1", "cf", "a", 1, KeyValue.Type.Delete, "dont-care"), 660 create("R2", "cf", "a", 20, KeyValue.Type.Put, "dont-care") }; 661 List<KeyValueScanner> scanners = scanFixture(kvs); 662 Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1")); 663 try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) { 664 List<Cell> results = new ArrayList<>(); 665 assertEquals(true, scan.next(results)); 666 assertEquals(0, results.size()); 667 668 assertEquals(true, scan.next(results)); 669 assertEquals(1, results.size()); 670 assertEquals(kvs[2], results.get(0)); 671 672 assertEquals(false, scan.next(results)); 673 } 674 } 675 676 @Test 677 public void testDeleteVersionMaskingMultiplePuts() throws IOException { 678 long now = EnvironmentEdgeManager.currentTime(); 679 KeyValue[] kvs1 = new KeyValue[] { create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care"), 680 create("R1", "cf", "a", now, KeyValue.Type.Delete, "dont-care") }; 681 KeyValue[] kvs2 = 682 new KeyValue[] { create("R1", "cf", "a", now - 500, KeyValue.Type.Put, "dont-care"), 683 create("R1", "cf", "a", now - 100, KeyValue.Type.Put, "dont-care"), 684 create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care") }; 685 List<KeyValueScanner> scanners = scanFixture(kvs1, kvs2); 686 687 try (StoreScanner scan = new StoreScanner(new Scan().withStartRow(Bytes.toBytes("R1")), 688 scanInfo, getCols("a"), scanners)) { 689 List<Cell> results = new ArrayList<>(); 690 // the two put at ts=now will be masked by the 1 delete, and 691 // since the scan default returns 1 version we'll return the newest 692 // key, which is kvs[2], now-100. 693 assertEquals(true, scan.next(results)); 694 assertEquals(1, results.size()); 695 assertEquals(kvs2[1], results.get(0)); 696 } 697 } 698 699 @Test 700 public void testDeleteVersionsMixedAndMultipleVersionReturn() throws IOException { 701 long now = EnvironmentEdgeManager.currentTime(); 702 KeyValue[] kvs1 = new KeyValue[] { create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care"), 703 create("R1", "cf", "a", now, KeyValue.Type.Delete, "dont-care") }; 704 KeyValue[] kvs2 = 705 new KeyValue[] { create("R1", "cf", "a", now - 500, KeyValue.Type.Put, "dont-care"), 706 create("R1", "cf", "a", now + 500, KeyValue.Type.Put, "dont-care"), 707 create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care"), 708 create("R2", "cf", "z", now, KeyValue.Type.Put, "dont-care") }; 709 List<KeyValueScanner> scanners = scanFixture(kvs1, kvs2); 710 711 Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1")).readVersions(2); 712 try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) { 713 List<Cell> results = new ArrayList<>(); 714 assertEquals(true, scan.next(results)); 715 assertEquals(2, results.size()); 716 assertEquals(kvs2[1], results.get(0)); 717 assertEquals(kvs2[0], results.get(1)); 718 } 719 } 720 721 @Test 722 public void testWildCardOneVersionScan() throws IOException { 723 KeyValue[] kvs = new KeyValue[] { create("R1", "cf", "a", 2, KeyValue.Type.Put, "dont-care"), 724 create("R1", "cf", "b", 1, KeyValue.Type.Put, "dont-care"), 725 create("R1", "cf", "a", 1, KeyValue.Type.DeleteColumn, "dont-care"), }; 726 List<KeyValueScanner> scanners = scanFixture(kvs); 727 try (StoreScanner scan = 728 new StoreScanner(new Scan().withStartRow(Bytes.toBytes("R1")), scanInfo, null, scanners)) { 729 List<Cell> results = new ArrayList<>(); 730 assertEquals(true, scan.next(results)); 731 assertEquals(2, results.size()); 732 assertEquals(kvs[0], results.get(0)); 733 assertEquals(kvs[1], results.get(1)); 734 } 735 } 736 737 @Test 738 public void testWildCardScannerUnderDeletes() throws IOException { 739 KeyValue[] kvs = new KeyValue[] { 740 // inc 741 create("R1", "cf", "a", 2, KeyValue.Type.Put, "dont-care"), 742 // orphaned delete column. 743 create("R1", "cf", "a", 1, KeyValue.Type.DeleteColumn, "dont-care"), 744 // column b 745 // inc 746 create("R1", "cf", "b", 2, KeyValue.Type.Put, "dont-care"), 747 // inc 748 create("R1", "cf", "b", 1, KeyValue.Type.Put, "dont-care"), 749 // column c 750 create("R1", "cf", "c", 10, KeyValue.Type.Delete, "dont-care"), 751 // no 752 create("R1", "cf", "c", 10, KeyValue.Type.Put, "dont-care"), 753 // inc 754 create("R1", "cf", "c", 9, KeyValue.Type.Put, "dont-care"), 755 // column d 756 // inc 757 create("R1", "cf", "d", 11, KeyValue.Type.Put, "dont-care"), 758 create("R1", "cf", "d", 10, KeyValue.Type.DeleteColumn, "dont-care"), 759 // no 760 create("R1", "cf", "d", 9, KeyValue.Type.Put, "dont-care"), 761 // no 762 create("R1", "cf", "d", 8, KeyValue.Type.Put, "dont-care"), 763 764 }; 765 List<KeyValueScanner> scanners = scanFixture(kvs); 766 try ( 767 StoreScanner scan = new StoreScanner(new Scan().readVersions(2), scanInfo, null, scanners)) { 768 List<Cell> results = new ArrayList<>(); 769 assertEquals(true, scan.next(results)); 770 assertEquals(5, results.size()); 771 assertEquals(kvs[0], results.get(0)); 772 assertEquals(kvs[2], results.get(1)); 773 assertEquals(kvs[3], results.get(2)); 774 assertEquals(kvs[6], results.get(3)); 775 assertEquals(kvs[7], results.get(4)); 776 } 777 } 778 779 @Test 780 public void testDeleteFamily() throws IOException { 781 KeyValue[] kvs = 782 new KeyValue[] { create("R1", "cf", "a", 100, KeyValue.Type.DeleteFamily, "dont-care"), 783 create("R1", "cf", "b", 11, KeyValue.Type.Put, "dont-care"), 784 create("R1", "cf", "c", 11, KeyValue.Type.Put, "dont-care"), 785 create("R1", "cf", "d", 11, KeyValue.Type.Put, "dont-care"), 786 create("R1", "cf", "e", 11, KeyValue.Type.Put, "dont-care"), 787 create("R1", "cf", "e", 11, KeyValue.Type.DeleteColumn, "dont-care"), 788 create("R1", "cf", "f", 11, KeyValue.Type.Put, "dont-care"), 789 create("R1", "cf", "g", 11, KeyValue.Type.Put, "dont-care"), 790 create("R1", "cf", "g", 11, KeyValue.Type.Delete, "dont-care"), 791 create("R1", "cf", "h", 11, KeyValue.Type.Put, "dont-care"), 792 create("R1", "cf", "i", 11, KeyValue.Type.Put, "dont-care"), 793 create("R2", "cf", "a", 11, KeyValue.Type.Put, "dont-care"), }; 794 List<KeyValueScanner> scanners = scanFixture(kvs); 795 try (StoreScanner scan = 796 new StoreScanner(new Scan().readAllVersions(), scanInfo, null, scanners)) { 797 List<Cell> results = new ArrayList<>(); 798 assertEquals(true, scan.next(results)); 799 assertEquals(0, results.size()); 800 assertEquals(true, scan.next(results)); 801 assertEquals(1, results.size()); 802 assertEquals(kvs[kvs.length - 1], results.get(0)); 803 804 assertEquals(false, scan.next(results)); 805 } 806 } 807 808 @Test 809 public void testDeleteColumn() throws IOException { 810 KeyValue[] kvs = 811 new KeyValue[] { create("R1", "cf", "a", 10, KeyValue.Type.DeleteColumn, "dont-care"), 812 create("R1", "cf", "a", 9, KeyValue.Type.Delete, "dont-care"), 813 create("R1", "cf", "a", 8, KeyValue.Type.Put, "dont-care"), 814 create("R1", "cf", "b", 5, KeyValue.Type.Put, "dont-care") }; 815 List<KeyValueScanner> scanners = scanFixture(kvs); 816 try (StoreScanner scan = new StoreScanner(new Scan(), scanInfo, null, scanners)) { 817 List<Cell> results = new ArrayList<>(); 818 assertEquals(true, scan.next(results)); 819 assertEquals(1, results.size()); 820 assertEquals(kvs[3], results.get(0)); 821 } 822 } 823 824 private static final KeyValue[] kvs = 825 new KeyValue[] { create("R1", "cf", "a", 11, KeyValue.Type.Put, "dont-care"), 826 create("R1", "cf", "b", 11, KeyValue.Type.Put, "dont-care"), 827 create("R1", "cf", "c", 11, KeyValue.Type.Put, "dont-care"), 828 create("R1", "cf", "d", 11, KeyValue.Type.Put, "dont-care"), 829 create("R1", "cf", "e", 11, KeyValue.Type.Put, "dont-care"), 830 create("R1", "cf", "f", 11, KeyValue.Type.Put, "dont-care"), 831 create("R1", "cf", "g", 11, KeyValue.Type.Put, "dont-care"), 832 create("R1", "cf", "h", 11, KeyValue.Type.Put, "dont-care"), 833 create("R1", "cf", "i", 11, KeyValue.Type.Put, "dont-care"), 834 create("R2", "cf", "a", 11, KeyValue.Type.Put, "dont-care"), }; 835 836 @Test 837 public void testSkipColumn() throws IOException { 838 List<KeyValueScanner> scanners = scanFixture(kvs); 839 try (StoreScanner scan = new StoreScanner(new Scan(), scanInfo, getCols("a", "d"), scanners)) { 840 List<Cell> results = new ArrayList<>(); 841 assertEquals(true, scan.next(results)); 842 assertEquals(2, results.size()); 843 assertEquals(kvs[0], results.get(0)); 844 assertEquals(kvs[3], results.get(1)); 845 results.clear(); 846 847 assertEquals(true, scan.next(results)); 848 assertEquals(1, results.size()); 849 assertEquals(kvs[kvs.length - 1], results.get(0)); 850 851 results.clear(); 852 assertEquals(false, scan.next(results)); 853 } 854 } 855 856 /* 857 * Test expiration of KeyValues in combination with a configured TTL for a column family (as 858 * should be triggered in a major compaction). 859 */ 860 @Test 861 public void testWildCardTtlScan() throws IOException { 862 long now = EnvironmentEdgeManager.currentTime(); 863 KeyValue[] kvs = 864 new KeyValue[] { create("R1", "cf", "a", now - 1000, KeyValue.Type.Put, "dont-care"), 865 create("R1", "cf", "b", now - 10, KeyValue.Type.Put, "dont-care"), 866 create("R1", "cf", "c", now - 200, KeyValue.Type.Put, "dont-care"), 867 create("R1", "cf", "d", now - 10000, KeyValue.Type.Put, "dont-care"), 868 create("R2", "cf", "a", now, KeyValue.Type.Put, "dont-care"), 869 create("R2", "cf", "b", now - 10, KeyValue.Type.Put, "dont-care"), 870 create("R2", "cf", "c", now - 200, KeyValue.Type.Put, "dont-care"), 871 create("R2", "cf", "c", now - 1000, KeyValue.Type.Put, "dont-care") }; 872 List<KeyValueScanner> scanners = scanFixture(kvs); 873 Scan scan = new Scan(); 874 scan.readVersions(1); 875 ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, 500, KeepDeletedCells.FALSE, 876 HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false); 877 try (StoreScanner scanner = new StoreScanner(scan, scanInfo, null, scanners)) { 878 List<Cell> results = new ArrayList<>(); 879 assertEquals(true, scanner.next(results)); 880 assertEquals(2, results.size()); 881 assertEquals(kvs[1], results.get(0)); 882 assertEquals(kvs[2], results.get(1)); 883 results.clear(); 884 885 assertEquals(true, scanner.next(results)); 886 assertEquals(3, results.size()); 887 assertEquals(kvs[4], results.get(0)); 888 assertEquals(kvs[5], results.get(1)); 889 assertEquals(kvs[6], results.get(2)); 890 results.clear(); 891 892 assertEquals(false, scanner.next(results)); 893 } 894 } 895 896 @Test 897 public void testScannerReseekDoesntNPE() throws Exception { 898 List<KeyValueScanner> scanners = scanFixture(kvs); 899 try (StoreScanner scan = new StoreScanner(new Scan(), scanInfo, getCols("a", "d"), scanners)) { 900 // Previously a updateReaders twice in a row would cause an NPE. In test this would also 901 // normally cause an NPE because scan.store is null. So as long as we get through these 902 // two calls we are good and the bug was quashed. 903 scan.updateReaders(Collections.emptyList(), Collections.emptyList()); 904 scan.updateReaders(Collections.emptyList(), Collections.emptyList()); 905 scan.peek(); 906 } 907 } 908 909 @Test 910 @Ignore("this fails, since we don't handle deletions, etc, in peek") 911 public void testPeek() throws Exception { 912 KeyValue[] kvs = new KeyValue[] { create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"), 913 create("R1", "cf", "a", 1, KeyValue.Type.Delete, "dont-care"), }; 914 List<KeyValueScanner> scanners = scanFixture(kvs); 915 Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1")); 916 try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) { 917 assertNull(scan.peek()); 918 } 919 } 920 921 /** 922 * Ensure that expired delete family markers don't override valid puts 923 */ 924 @Test 925 public void testExpiredDeleteFamily() throws Exception { 926 long now = EnvironmentEdgeManager.currentTime(); 927 KeyValue[] kvs = new KeyValue[] { 928 new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null, now - 1000, 929 KeyValue.Type.DeleteFamily), 930 create("R1", "cf", "a", now - 10, KeyValue.Type.Put, "dont-care"), }; 931 List<KeyValueScanner> scanners = scanFixture(kvs); 932 Scan scan = new Scan(); 933 scan.readVersions(1); 934 // scanner with ttl equal to 500 935 ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, 500, KeepDeletedCells.FALSE, 936 HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false); 937 try (StoreScanner scanner = new StoreScanner(scan, scanInfo, null, scanners)) { 938 List<Cell> results = new ArrayList<>(); 939 assertEquals(true, scanner.next(results)); 940 assertEquals(1, results.size()); 941 assertEquals(kvs[1], results.get(0)); 942 results.clear(); 943 944 assertEquals(false, scanner.next(results)); 945 } 946 } 947 948 @Test 949 public void testDeleteMarkerLongevity() throws Exception { 950 try { 951 final long now = EnvironmentEdgeManager.currentTime(); 952 EnvironmentEdgeManagerTestHelper.injectEdge(new EnvironmentEdge() { 953 @Override 954 public long currentTime() { 955 return now; 956 } 957 }); 958 // @formatter:off 959 KeyValue[] kvs = new KeyValue[]{ 960 /*0*/ new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null, 961 now - 100, KeyValue.Type.DeleteFamily), // live 962 /*1*/ new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null, 963 now - 1000, KeyValue.Type.DeleteFamily), // expired 964 /*2*/ create("R1", "cf", "a", now - 50, 965 KeyValue.Type.Put, "v3"), // live 966 /*3*/ create("R1", "cf", "a", now - 55, 967 KeyValue.Type.Delete, "dontcare"), // live 968 /*4*/ create("R1", "cf", "a", now - 55, 969 KeyValue.Type.Put, "deleted-version v2"), // deleted 970 /*5*/ create("R1", "cf", "a", now - 60, 971 KeyValue.Type.Put, "v1"), // live 972 /*6*/ create("R1", "cf", "a", now - 65, 973 KeyValue.Type.Put, "v0"), // max-version reached 974 /*7*/ create("R1", "cf", "a", 975 now - 100, KeyValue.Type.DeleteColumn, "dont-care"), // max-version 976 /*8*/ create("R1", "cf", "b", now - 600, 977 KeyValue.Type.DeleteColumn, "dont-care"), // expired 978 /*9*/ create("R1", "cf", "b", now - 70, 979 KeyValue.Type.Put, "v2"), // live 980 /*10*/ create("R1", "cf", "b", now - 750, 981 KeyValue.Type.Put, "v1"), // expired 982 /*11*/ create("R1", "cf", "c", now - 500, 983 KeyValue.Type.Delete, "dontcare"), // expired 984 /*12*/ create("R1", "cf", "c", now - 600, 985 KeyValue.Type.Put, "v1"), // expired 986 /*13*/ create("R1", "cf", "c", now - 1000, 987 KeyValue.Type.Delete, "dontcare"), // expired 988 /*14*/ create("R1", "cf", "d", now - 60, 989 KeyValue.Type.Put, "expired put"), // live 990 /*15*/ create("R1", "cf", "d", now - 100, 991 KeyValue.Type.Delete, "not-expired delete"), // live 992 }; 993 // @formatter:on 994 List<KeyValueScanner> scanners = scanFixture(kvs); 995 ScanInfo scanInfo = new ScanInfo(CONF, Bytes.toBytes("cf"), 0 /* minVersions */, 996 2 /* maxVersions */, 500 /* ttl */, KeepDeletedCells.FALSE /* keepDeletedCells */, 997 HConstants.DEFAULT_BLOCKSIZE /* block size */, 200, /* timeToPurgeDeletes */ 998 CellComparator.getInstance(), false); 999 try (StoreScanner scanner = 1000 new StoreScanner(scanInfo, 2, ScanType.COMPACT_DROP_DELETES, scanners)) { 1001 List<Cell> results = new ArrayList<>(); 1002 results = new ArrayList<>(); 1003 assertEquals(true, scanner.next(results)); 1004 assertEquals(kvs[0], results.get(0)); 1005 assertEquals(kvs[2], results.get(1)); 1006 assertEquals(kvs[3], results.get(2)); 1007 assertEquals(kvs[5], results.get(3)); 1008 assertEquals(kvs[9], results.get(4)); 1009 assertEquals(kvs[14], results.get(5)); 1010 assertEquals(kvs[15], results.get(6)); 1011 assertEquals(7, results.size()); 1012 } 1013 } finally { 1014 EnvironmentEdgeManagerTestHelper.reset(); 1015 } 1016 } 1017 1018 @Test 1019 public void testPreadNotEnabledForCompactionStoreScanners() throws Exception { 1020 long now = EnvironmentEdgeManager.currentTime(); 1021 KeyValue[] kvs = new KeyValue[] { 1022 new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null, now - 1000, 1023 KeyValue.Type.DeleteFamily), 1024 create("R1", "cf", "a", now - 10, KeyValue.Type.Put, "dont-care"), }; 1025 List<KeyValueScanner> scanners = scanFixture(kvs); 1026 ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, 500, KeepDeletedCells.FALSE, 1027 HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false); 1028 try (StoreScanner storeScanner = 1029 new StoreScanner(scanInfo, -1, ScanType.COMPACT_RETAIN_DELETES, scanners)) { 1030 assertFalse(storeScanner.isScanUsePread()); 1031 } 1032 } 1033 1034 @Test 1035 public void testReadVersionWithRawAndFilter() throws IOException { 1036 ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, Long.MAX_VALUE, KeepDeletedCells.FALSE, 1037 HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false); 1038 KeyValue[] kvs = new KeyValue[] { create("R1", "cf", "a", 3, KeyValue.Type.Put, "dont-care"), 1039 create("R1", "cf", "a", 2, KeyValue.Type.Put, "dont-care"), 1040 create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care") }; 1041 List<KeyValueScanner> scanners = Arrays 1042 .asList(new KeyValueScanner[] { new KeyValueScanFixture(CellComparator.getInstance(), kvs) }); 1043 1044 BinaryComparator comp = new BinaryComparator(Bytes.toBytes("a")); 1045 Filter filter = new QualifierFilter(CompareOperator.EQUAL, comp); 1046 Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1")).readVersions(2).setRaw(true); 1047 scanSpec.setFilter(filter); 1048 try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, null, scanners)) { 1049 List<Cell> results = new ArrayList<>(); 1050 assertEquals(true, scan.next(results)); 1051 assertEquals(2, results.size()); 1052 } 1053 } 1054 1055 @Test 1056 public void testScannersClosedWhenCheckingOnlyMemStore() throws IOException { 1057 class MyCollectionBackedScanner extends CollectionBackedScanner { 1058 final boolean fileScanner; 1059 boolean closed; 1060 1061 MyCollectionBackedScanner(boolean fileScanner) { 1062 super(Collections.emptySortedSet()); 1063 this.fileScanner = fileScanner; 1064 } 1065 1066 @Override 1067 public boolean isFileScanner() { 1068 return fileScanner; 1069 } 1070 1071 @Override 1072 public void close() { 1073 super.close(); 1074 closed = true; 1075 } 1076 } 1077 1078 ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, Long.MAX_VALUE, KeepDeletedCells.FALSE, 1079 HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false); 1080 InternalScan scan = new InternalScan(new Scan()); 1081 scan.checkOnlyMemStore(); 1082 MyCollectionBackedScanner fileScanner = new MyCollectionBackedScanner(true); 1083 MyCollectionBackedScanner memStoreScanner = new MyCollectionBackedScanner(false); 1084 List<? extends KeyValueScanner> allScanners = Arrays.asList(fileScanner, memStoreScanner); 1085 1086 try (StoreScanner scanner = new StoreScanner(scan, scanInfo, null, allScanners)) { 1087 List<KeyValueScanner> remaining = scanner.selectScannersFrom(null, allScanners); 1088 1089 assertEquals(1, remaining.size()); 1090 assertSame(memStoreScanner, remaining.get(0)); 1091 assertTrue(fileScanner.closed); 1092 assertFalse(memStoreScanner.closed); 1093 } 1094 } 1095}