001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.regionserver; 019 020import static org.apache.hadoop.hbase.KeyValueTestUtil.create; 021import static org.apache.hadoop.hbase.regionserver.KeyValueScanFixture.scanFixture; 022import static org.junit.Assert.assertEquals; 023import static org.junit.Assert.assertFalse; 024import static org.junit.Assert.assertNull; 025import static org.junit.Assert.assertTrue; 026 027import java.io.IOException; 028import java.util.ArrayList; 029import java.util.Arrays; 030import java.util.Collections; 031import java.util.List; 032import java.util.NavigableSet; 033import java.util.TreeSet; 034import java.util.concurrent.atomic.AtomicInteger; 035 036import org.apache.hadoop.conf.Configuration; 037import org.apache.hadoop.hbase.Cell; 038import org.apache.hadoop.hbase.CellBuilderType; 039import org.apache.hadoop.hbase.CellComparator; 040import org.apache.hadoop.hbase.CellUtil; 041import org.apache.hadoop.hbase.ExtendedCellBuilderFactory; 042import org.apache.hadoop.hbase.HBaseClassTestRule; 043import org.apache.hadoop.hbase.HBaseConfiguration; 044import org.apache.hadoop.hbase.HBaseTestingUtility; 045import org.apache.hadoop.hbase.HConstants; 046import org.apache.hadoop.hbase.KeepDeletedCells; 047import org.apache.hadoop.hbase.KeyValue; 048import org.apache.hadoop.hbase.PrivateCellUtil; 049import org.apache.hadoop.hbase.client.Get; 050import org.apache.hadoop.hbase.client.Scan; 051import org.apache.hadoop.hbase.filter.ColumnCountGetFilter; 052import org.apache.hadoop.hbase.testclassification.MediumTests; 053import org.apache.hadoop.hbase.testclassification.RegionServerTests; 054import org.apache.hadoop.hbase.util.Bytes; 055import org.apache.hadoop.hbase.util.EnvironmentEdge; 056import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper; 057import org.junit.ClassRule; 058import org.junit.Ignore; 059import org.junit.Rule; 060import org.junit.Test; 061import org.junit.experimental.categories.Category; 062import org.junit.rules.TestName; 063import org.slf4j.Logger; 064import org.slf4j.LoggerFactory; 065 066// Can't be small as it plays with EnvironmentEdgeManager 067@Category({RegionServerTests.class, MediumTests.class}) 068public class TestStoreScanner { 069 070 @ClassRule 071 public static final HBaseClassTestRule CLASS_RULE = 072 HBaseClassTestRule.forClass(TestStoreScanner.class); 073 074 private static final Logger LOG = LoggerFactory.getLogger(TestStoreScanner.class); 075 @Rule public TestName name = new TestName(); 076 private static final String CF_STR = "cf"; 077 private static final byte[] CF = Bytes.toBytes(CF_STR); 078 static Configuration CONF = HBaseConfiguration.create(); 079 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); 080 private ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, Integer.MAX_VALUE, Long.MAX_VALUE, 081 KeepDeletedCells.FALSE, HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false); 082 083 /** 084 * From here on down, we have a bunch of defines and specific CELL_GRID of Cells. The 085 * CELL_GRID then has a Scanner that can fake out 'block' transitions. All this elaborate 086 * setup is for tests that ensure we don't overread, and that the {@link StoreScanner} is not 087 * overly enthusiastic. 088 */ 089 private static final byte[] ZERO = new byte[] {'0'}; 090 private static final byte[] ZERO_POINT_ZERO = new byte[] {'0', '.', '0'}; 091 private static final byte[] ONE = new byte[] {'1'}; 092 private static final byte[] TWO = new byte[] {'2'}; 093 private static final byte[] TWO_POINT_TWO = new byte[] {'2', '.', '2'}; 094 private static final byte[] THREE = new byte[] {'3'}; 095 private static final byte[] FOUR = new byte[] {'4'}; 096 private static final byte[] FIVE = new byte[] {'5'}; 097 private static final byte[] VALUE = new byte[] {'v'}; 098 private static final int CELL_GRID_BLOCK2_BOUNDARY = 4; 099 private static final int CELL_GRID_BLOCK3_BOUNDARY = 11; 100 private static final int CELL_GRID_BLOCK4_BOUNDARY = 15; 101 private static final int CELL_GRID_BLOCK5_BOUNDARY = 19; 102 103 /** 104 * Five rows by four columns distinguished by column qualifier (column qualifier is one of the 105 * four rows... ONE, TWO, etc.). Exceptions are a weird row after TWO; it is TWO_POINT_TWO. 106 * And then row FOUR has five columns finishing w/ row FIVE having a single column. 107 * We will use this to test scan does the right thing as it 108 * we do Gets, StoreScanner#optimize, and what we do on (faked) block boundaries. 109 */ 110 private static final Cell[] CELL_GRID = new Cell [] { 111 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(ONE) 112 .setFamily(CF).setQualifier(ONE).setTimestamp(1L) 113 .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), 114 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(ONE) 115 .setFamily(CF).setQualifier(TWO).setTimestamp(1L) 116 .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), 117 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(ONE) 118 .setFamily(CF).setQualifier(THREE).setTimestamp(1L) 119 .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), 120 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(ONE) 121 .setFamily(CF).setQualifier(FOUR).setTimestamp(1L) 122 .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), 123 // Offset 4 CELL_GRID_BLOCK2_BOUNDARY 124 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(TWO) 125 .setFamily(CF).setQualifier(ONE).setTimestamp(1L) 126 .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), 127 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(TWO) 128 .setFamily(CF).setQualifier(TWO).setTimestamp(1L) 129 .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), 130 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(TWO) 131 .setFamily(CF).setQualifier(THREE).setTimestamp(1L) 132 .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), 133 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(TWO) 134 .setFamily(CF).setQualifier(FOUR).setTimestamp(1L) 135 .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), 136 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(TWO_POINT_TWO) 137 .setFamily(CF).setQualifier(ZERO).setTimestamp(1L) 138 .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), 139 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(TWO_POINT_TWO) 140 .setFamily(CF).setQualifier(ZERO_POINT_ZERO).setTimestamp(1L) 141 .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), 142 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(TWO_POINT_TWO) 143 .setFamily(CF).setQualifier(FIVE).setTimestamp(1L) 144 .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), 145 // Offset 11! CELL_GRID_BLOCK3_BOUNDARY 146 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(THREE) 147 .setFamily(CF).setQualifier(ONE).setTimestamp(1L) 148 .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), 149 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(THREE) 150 .setFamily(CF).setQualifier(TWO).setTimestamp(1L) 151 .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), 152 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(THREE) 153 .setFamily(CF).setQualifier(THREE).setTimestamp(1L) 154 .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), 155 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(THREE) 156 .setFamily(CF).setQualifier(FOUR).setTimestamp(1L) 157 .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), 158 // Offset 15 CELL_GRID_BLOCK4_BOUNDARY 159 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(FOUR) 160 .setFamily(CF).setQualifier(ONE).setTimestamp(1L) 161 .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), 162 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(FOUR) 163 .setFamily(CF).setQualifier(TWO).setTimestamp(1L) 164 .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), 165 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(FOUR) 166 .setFamily(CF).setQualifier(THREE).setTimestamp(1L) 167 .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), 168 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(FOUR) 169 .setFamily(CF).setQualifier(FOUR).setTimestamp(1L) 170 .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), 171 // Offset 19 CELL_GRID_BLOCK5_BOUNDARY 172 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(FOUR) 173 .setFamily(CF).setQualifier(FIVE).setTimestamp(1L) 174 .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), 175 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(FIVE) 176 .setFamily(CF).setQualifier(ZERO).setTimestamp(1L) 177 .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), 178 }; 179 180 private static class KeyValueHeapWithCount extends KeyValueHeap { 181 182 final AtomicInteger count; 183 184 public KeyValueHeapWithCount(List<? extends KeyValueScanner> scanners, 185 CellComparator comparator, AtomicInteger count) throws IOException { 186 super(scanners, comparator); 187 this.count = count; 188 } 189 190 @Override 191 public Cell peek() { 192 this.count.incrementAndGet(); 193 return super.peek(); 194 } 195 } 196 197 /** 198 * A StoreScanner for our CELL_GRID above. Fakes the block transitions. Does counts of 199 * calls to optimize and counts of when optimize actually did an optimize. 200 */ 201 private static class CellGridStoreScanner extends StoreScanner { 202 // Count of how often optimize is called and of how often it does an optimize. 203 AtomicInteger count; 204 final AtomicInteger optimization = new AtomicInteger(0); 205 206 CellGridStoreScanner(final Scan scan, ScanInfo scanInfo) throws IOException { 207 super(scan, scanInfo, scan.getFamilyMap().get(CF), Arrays.<KeyValueScanner> asList( 208 new KeyValueScanner[] { new KeyValueScanFixture(CellComparator.getInstance(), CELL_GRID) })); 209 } 210 211 @Override 212 protected void resetKVHeap(List<? extends KeyValueScanner> scanners, 213 CellComparator comparator) throws IOException { 214 if (count == null) { 215 count = new AtomicInteger(0); 216 } 217 heap = newKVHeap(scanners, comparator); 218 } 219 220 @Override 221 protected KeyValueHeap newKVHeap(List<? extends KeyValueScanner> scanners, 222 CellComparator comparator) throws IOException { 223 return new KeyValueHeapWithCount(scanners, comparator, count); 224 } 225 226 @Override 227 protected boolean trySkipToNextRow(Cell cell) throws IOException { 228 boolean optimized = super.trySkipToNextRow(cell); 229 LOG.info("Cell=" + cell + ", nextIndex=" + CellUtil.toString(getNextIndexedKey(), false) 230 + ", optimized=" + optimized); 231 if (optimized) { 232 optimization.incrementAndGet(); 233 } 234 return optimized; 235 } 236 237 @Override 238 protected boolean trySkipToNextColumn(Cell cell) throws IOException { 239 boolean optimized = super.trySkipToNextColumn(cell); 240 LOG.info("Cell=" + cell + ", nextIndex=" + CellUtil.toString(getNextIndexedKey(), false) 241 + ", optimized=" + optimized); 242 if (optimized) { 243 optimization.incrementAndGet(); 244 } 245 return optimized; 246 } 247 248 @Override 249 public Cell getNextIndexedKey() { 250 // Fake block boundaries by having index of next block change as we go through scan. 251 return count.get() > CELL_GRID_BLOCK4_BOUNDARY? 252 PrivateCellUtil.createFirstOnRow(CELL_GRID[CELL_GRID_BLOCK5_BOUNDARY]): 253 count.get() > CELL_GRID_BLOCK3_BOUNDARY? 254 PrivateCellUtil.createFirstOnRow(CELL_GRID[CELL_GRID_BLOCK4_BOUNDARY]): 255 count.get() > CELL_GRID_BLOCK2_BOUNDARY? 256 PrivateCellUtil.createFirstOnRow(CELL_GRID[CELL_GRID_BLOCK3_BOUNDARY]): 257 PrivateCellUtil.createFirstOnRow(CELL_GRID[CELL_GRID_BLOCK2_BOUNDARY]); 258 } 259 } 260 261 private static final int CELL_WITH_VERSIONS_BLOCK2_BOUNDARY = 4; 262 263 private static final Cell[] CELL_WITH_VERSIONS = new Cell [] { 264 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(ONE) 265 .setFamily(CF).setQualifier(ONE).setTimestamp(2L) 266 .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), 267 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(ONE) 268 .setFamily(CF).setQualifier(ONE).setTimestamp(1L) 269 .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), 270 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(ONE) 271 .setFamily(CF).setQualifier(TWO).setTimestamp(2L) 272 .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), 273 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(ONE) 274 .setFamily(CF).setQualifier(TWO).setTimestamp(1L) 275 .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), 276 // Offset 4 CELL_WITH_VERSIONS_BLOCK2_BOUNDARY 277 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(TWO) 278 .setFamily(CF).setQualifier(ONE).setTimestamp(1L) 279 .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), 280 ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(TWO) 281 .setFamily(CF).setQualifier(TWO).setTimestamp(1L) 282 .setType(KeyValue.Type.Put.getCode()).setValue(VALUE).build(), 283 }; 284 285 private static class CellWithVersionsStoreScanner extends StoreScanner { 286 // Count of how often optimize is called and of how often it does an optimize. 287 final AtomicInteger optimization = new AtomicInteger(0); 288 289 CellWithVersionsStoreScanner(final Scan scan, ScanInfo scanInfo) throws IOException { 290 super(scan, scanInfo, scan.getFamilyMap().get(CF), 291 Arrays.<KeyValueScanner> asList(new KeyValueScanner[] { 292 new KeyValueScanFixture(CellComparator.getInstance(), CELL_WITH_VERSIONS) })); 293 } 294 295 @Override 296 protected boolean trySkipToNextColumn(Cell cell) throws IOException { 297 boolean optimized = super.trySkipToNextColumn(cell); 298 LOG.info("Cell=" + cell + ", nextIndex=" + CellUtil.toString(getNextIndexedKey(), false) 299 + ", optimized=" + optimized); 300 if (optimized) { 301 optimization.incrementAndGet(); 302 } 303 return optimized; 304 } 305 306 @Override 307 public Cell getNextIndexedKey() { 308 // Fake block boundaries by having index of next block change as we go through scan. 309 return PrivateCellUtil 310 .createFirstOnRow(CELL_WITH_VERSIONS[CELL_WITH_VERSIONS_BLOCK2_BOUNDARY]); 311 } 312 } 313 314 private static class CellWithVersionsNoOptimizeStoreScanner extends StoreScanner { 315 // Count of how often optimize is called and of how often it does an optimize. 316 final AtomicInteger optimization = new AtomicInteger(0); 317 318 CellWithVersionsNoOptimizeStoreScanner(Scan scan, ScanInfo scanInfo) throws IOException { 319 super(scan, scanInfo, scan.getFamilyMap().get(CF), 320 Arrays.<KeyValueScanner> asList(new KeyValueScanner[] { 321 new KeyValueScanFixture(CellComparator.getInstance(), CELL_WITH_VERSIONS) })); 322 } 323 324 @Override 325 protected boolean trySkipToNextColumn(Cell cell) throws IOException { 326 boolean optimized = super.trySkipToNextColumn(cell); 327 LOG.info("Cell=" + cell + ", nextIndex=" + CellUtil.toString(getNextIndexedKey(), false) 328 + ", optimized=" + optimized); 329 if (optimized) { 330 optimization.incrementAndGet(); 331 } 332 return optimized; 333 } 334 335 @Override 336 public Cell getNextIndexedKey() { 337 return null; 338 } 339 } 340 341 @Test 342 public void testWithColumnCountGetFilter() throws Exception { 343 Get get = new Get(ONE); 344 get.readAllVersions(); 345 get.addFamily(CF); 346 get.setFilter(new ColumnCountGetFilter(2)); 347 348 try (CellWithVersionsNoOptimizeStoreScanner scannerNoOptimize = 349 new CellWithVersionsNoOptimizeStoreScanner(new Scan(get), this.scanInfo)) { 350 List<Cell> results = new ArrayList<>(); 351 while (scannerNoOptimize.next(results)) { 352 continue; 353 } 354 assertEquals(2, results.size()); 355 assertTrue(CellUtil.matchingColumn(results.get(0), CELL_WITH_VERSIONS[0])); 356 assertTrue(CellUtil.matchingColumn(results.get(1), CELL_WITH_VERSIONS[2])); 357 assertTrue("Optimize should do some optimizations", 358 scannerNoOptimize.optimization.get() == 0); 359 } 360 361 get.setFilter(new ColumnCountGetFilter(2)); 362 try (CellWithVersionsStoreScanner scanner = 363 new CellWithVersionsStoreScanner(new Scan(get), this.scanInfo)) { 364 List<Cell> results = new ArrayList<>(); 365 while (scanner.next(results)) { 366 continue; 367 } 368 assertEquals(2, results.size()); 369 assertTrue(CellUtil.matchingColumn(results.get(0), CELL_WITH_VERSIONS[0])); 370 assertTrue(CellUtil.matchingColumn(results.get(1), CELL_WITH_VERSIONS[2])); 371 assertTrue("Optimize should do some optimizations", scanner.optimization.get() > 0); 372 } 373 } 374 375 /* 376 * Test utility for building a NavigableSet for scanners. 377 * @param strCols 378 * @return 379 */ 380 NavigableSet<byte[]> getCols(String ...strCols) { 381 NavigableSet<byte[]> cols = new TreeSet<>(Bytes.BYTES_COMPARATOR); 382 for (String col : strCols) { 383 byte[] bytes = Bytes.toBytes(col); 384 cols.add(bytes); 385 } 386 return cols; 387 } 388 389 @Test 390 public void testFullRowGetDoesNotOverreadWhenRowInsideOneBlock() throws IOException { 391 // Do a Get against row two. Row two is inside a block that starts with row TWO but ends with 392 // row TWO_POINT_TWO. We should read one block only. 393 Get get = new Get(TWO); 394 Scan scan = new Scan(get); 395 try (CellGridStoreScanner scanner = new CellGridStoreScanner(scan, this.scanInfo)) { 396 List<Cell> results = new ArrayList<>(); 397 while (scanner.next(results)) { 398 continue; 399 } 400 // Should be four results of column 1 (though there are 5 rows in the CELL_GRID -- the 401 // TWO_POINT_TWO row does not have a a column ONE. 402 assertEquals(4, results.size()); 403 // We should have gone the optimize route 5 times totally... an INCLUDE for the four cells 404 // in the row plus the DONE on the end. 405 assertEquals(5, scanner.count.get()); 406 // For a full row Get, there should be no opportunity for scanner optimization. 407 assertEquals(0, scanner.optimization.get()); 408 } 409 } 410 411 @Test 412 public void testFullRowSpansBlocks() throws IOException { 413 // Do a Get against row FOUR. It spans two blocks. 414 Get get = new Get(FOUR); 415 Scan scan = new Scan(get); 416 try (CellGridStoreScanner scanner = new CellGridStoreScanner(scan, this.scanInfo)) { 417 List<Cell> results = new ArrayList<>(); 418 while (scanner.next(results)) { 419 continue; 420 } 421 // Should be four results of column 1 (though there are 5 rows in the CELL_GRID -- the 422 // TWO_POINT_TWO row does not have a a column ONE. 423 assertEquals(5, results.size()); 424 // We should have gone the optimize route 6 times totally... an INCLUDE for the five cells 425 // in the row plus the DONE on the end. 426 assertEquals(6, scanner.count.get()); 427 // For a full row Get, there should be no opportunity for scanner optimization. 428 assertEquals(0, scanner.optimization.get()); 429 } 430 } 431 432 /** 433 * Test optimize in StoreScanner. Test that we skip to the next 'block' when we it makes sense 434 * reading the block 'index'. 435 * @throws IOException 436 */ 437 @Test 438 public void testOptimize() throws IOException { 439 Scan scan = new Scan(); 440 // A scan that just gets the first qualifier on each row of the CELL_GRID 441 scan.addColumn(CF, ONE); 442 try (CellGridStoreScanner scanner = new CellGridStoreScanner(scan, this.scanInfo)) { 443 List<Cell> results = new ArrayList<>(); 444 while (scanner.next(results)) { 445 continue; 446 } 447 // Should be four results of column 1 (though there are 5 rows in the CELL_GRID -- the 448 // TWO_POINT_TWO row does not have a a column ONE. 449 assertEquals(4, results.size()); 450 for (Cell cell: results) { 451 assertTrue(Bytes.equals(ONE, 0, ONE.length, 452 cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength())); 453 } 454 assertTrue("Optimize should do some optimizations", scanner.optimization.get() > 0); 455 } 456 } 457 458 /** 459 * Ensure the optimize Scan method in StoreScanner does not get in the way of a Get doing minimum 460 * work... seeking to start of block and then SKIPPING until we find the wanted Cell. 461 * This 'simple' scenario mimics case of all Cells fitting inside a single HFileBlock. 462 * See HBASE-15392. This test is a little cryptic. Takes a bit of staring to figure what it up to. 463 * @throws IOException 464 */ 465 @Test 466 public void testOptimizeAndGet() throws IOException { 467 // First test a Get of two columns in the row R2. Every Get is a Scan. Get columns named 468 // R2 and R3. 469 Get get = new Get(TWO); 470 get.addColumn(CF, TWO); 471 get.addColumn(CF, THREE); 472 Scan scan = new Scan(get); 473 try (CellGridStoreScanner scanner = new CellGridStoreScanner(scan, this.scanInfo)) { 474 List<Cell> results = new ArrayList<>(); 475 // For a Get there should be no more next's after the first call. 476 assertEquals(false, scanner.next(results)); 477 // Should be one result only. 478 assertEquals(2, results.size()); 479 // And we should have gone through optimize twice only. 480 assertEquals("First qcode is SEEK_NEXT_COL and second INCLUDE_AND_SEEK_NEXT_ROW", 3, 481 scanner.count.get()); 482 } 483 } 484 485 /** 486 * Ensure that optimize does not cause the Get to do more seeking than required. Optimize 487 * (see HBASE-15392) was causing us to seek all Cells in a block when a Get Scan if the next block 488 * index/start key was a different row to the current one. A bug. We'd call next too often 489 * because we had to exhaust all Cells in the current row making us load the next block just to 490 * discard what we read there. This test is a little cryptic. Takes a bit of staring to figure 491 * what it up to. 492 * @throws IOException 493 */ 494 @Test 495 public void testOptimizeAndGetWithFakedNextBlockIndexStart() throws IOException { 496 // First test a Get of second column in the row R2. Every Get is a Scan. Second column has a 497 // qualifier of R2. 498 Get get = new Get(THREE); 499 get.addColumn(CF, TWO); 500 Scan scan = new Scan(get); 501 try (CellGridStoreScanner scanner = new CellGridStoreScanner(scan, this.scanInfo)) { 502 List<Cell> results = new ArrayList<>(); 503 // For a Get there should be no more next's after the first call. 504 assertEquals(false, scanner.next(results)); 505 // Should be one result only. 506 assertEquals(1, results.size()); 507 // And we should have gone through optimize twice only. 508 assertEquals("First qcode is SEEK_NEXT_COL and second INCLUDE_AND_SEEK_NEXT_ROW", 2, 509 scanner.count.get()); 510 } 511 } 512 513 @Test 514 public void testScanTimeRange() throws IOException { 515 String r1 = "R1"; 516 // returns only 1 of these 2 even though same timestamp 517 KeyValue [] kvs = new KeyValue[] { 518 create(r1, CF_STR, "a", 1, KeyValue.Type.Put, "dont-care"), 519 create(r1, CF_STR, "a", 2, KeyValue.Type.Put, "dont-care"), 520 create(r1, CF_STR, "a", 3, KeyValue.Type.Put, "dont-care"), 521 create(r1, CF_STR, "a", 4, KeyValue.Type.Put, "dont-care"), 522 create(r1, CF_STR, "a", 5, KeyValue.Type.Put, "dont-care"), 523 }; 524 List<KeyValueScanner> scanners = Arrays.<KeyValueScanner>asList( 525 new KeyValueScanner[] { 526 new KeyValueScanFixture(CellComparator.getInstance(), kvs) 527 }); 528 Scan scanSpec = new Scan().withStartRow(Bytes.toBytes(r1)); 529 scanSpec.setTimeRange(0, 6); 530 scanSpec.readAllVersions(); 531 List<Cell> results = null; 532 try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) { 533 results = new ArrayList<>(); 534 assertEquals(true, scan.next(results)); 535 assertEquals(5, results.size()); 536 assertEquals(kvs[kvs.length - 1], results.get(0)); 537 } 538 // Scan limited TimeRange 539 scanSpec = new Scan().withStartRow(Bytes.toBytes(r1)); 540 scanSpec.setTimeRange(1, 3); 541 scanSpec.readAllVersions(); 542 try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) { 543 results = new ArrayList<>(); 544 assertEquals(true, scan.next(results)); 545 assertEquals(2, results.size()); 546 } 547 // Another range. 548 scanSpec = new Scan().withStartRow(Bytes.toBytes(r1)); 549 scanSpec.setTimeRange(5, 10); 550 scanSpec.readAllVersions(); 551 try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) { 552 results = new ArrayList<>(); 553 assertEquals(true, scan.next(results)); 554 assertEquals(1, results.size()); 555 } 556 // See how TimeRange and Versions interact. 557 // Another range. 558 scanSpec = new Scan().withStartRow(Bytes.toBytes(r1)); 559 scanSpec.setTimeRange(0, 10); 560 scanSpec.readVersions(3); 561 try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) { 562 results = new ArrayList<>(); 563 assertEquals(true, scan.next(results)); 564 assertEquals(3, results.size()); 565 } 566 } 567 568 @Test 569 public void testScanSameTimestamp() throws IOException { 570 // returns only 1 of these 2 even though same timestamp 571 KeyValue [] kvs = new KeyValue[] { 572 create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"), 573 create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"), 574 }; 575 List<KeyValueScanner> scanners = Arrays.asList( 576 new KeyValueScanner[] { 577 new KeyValueScanFixture(CellComparator.getInstance(), kvs) 578 }); 579 580 Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1")); 581 // this only uses maxVersions (default=1) and TimeRange (default=all) 582 try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, null, scanners)) { 583 List<Cell> results = new ArrayList<>(); 584 assertEquals(true, scan.next(results)); 585 assertEquals(1, results.size()); 586 assertEquals(kvs[0], results.get(0)); 587 } 588 } 589 590 /* 591 * Test test shows exactly how the matcher's return codes confuses the StoreScanner 592 * and prevent it from doing the right thing. Seeking once, then nexting twice 593 * should return R1, then R2, but in this case it doesnt. 594 * TODO this comment makes no sense above. Appears to do the right thing. 595 * @throws IOException 596 */ 597 @Test 598 public void testWontNextToNext() throws IOException { 599 // build the scan file: 600 KeyValue [] kvs = new KeyValue[] { 601 create("R1", "cf", "a", 2, KeyValue.Type.Put, "dont-care"), 602 create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"), 603 create("R2", "cf", "a", 1, KeyValue.Type.Put, "dont-care") 604 }; 605 List<KeyValueScanner> scanners = scanFixture(kvs); 606 607 Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1")); 608 // this only uses maxVersions (default=1) and TimeRange (default=all) 609 try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) { 610 List<Cell> results = new ArrayList<>(); 611 scan.next(results); 612 assertEquals(1, results.size()); 613 assertEquals(kvs[0], results.get(0)); 614 // should be ok... 615 // now scan _next_ again. 616 results.clear(); 617 scan.next(results); 618 assertEquals(1, results.size()); 619 assertEquals(kvs[2], results.get(0)); 620 621 results.clear(); 622 scan.next(results); 623 assertEquals(0, results.size()); 624 } 625 } 626 627 628 @Test 629 public void testDeleteVersionSameTimestamp() throws IOException { 630 KeyValue [] kvs = new KeyValue [] { 631 create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"), 632 create("R1", "cf", "a", 1, KeyValue.Type.Delete, "dont-care"), 633 }; 634 List<KeyValueScanner> scanners = scanFixture(kvs); 635 Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1")); 636 try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) { 637 List<Cell> results = new ArrayList<>(); 638 assertFalse(scan.next(results)); 639 assertEquals(0, results.size()); 640 } 641 } 642 643 /* 644 * Test the case where there is a delete row 'in front of' the next row, the scanner 645 * will move to the next row. 646 */ 647 @Test 648 public void testDeletedRowThenGoodRow() throws IOException { 649 KeyValue [] kvs = new KeyValue [] { 650 create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"), 651 create("R1", "cf", "a", 1, KeyValue.Type.Delete, "dont-care"), 652 create("R2", "cf", "a", 20, KeyValue.Type.Put, "dont-care") 653 }; 654 List<KeyValueScanner> scanners = scanFixture(kvs); 655 Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1")); 656 try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) { 657 List<Cell> results = new ArrayList<>(); 658 assertEquals(true, scan.next(results)); 659 assertEquals(0, results.size()); 660 661 assertEquals(true, scan.next(results)); 662 assertEquals(1, results.size()); 663 assertEquals(kvs[2], results.get(0)); 664 665 assertEquals(false, scan.next(results)); 666 } 667 } 668 669 @Test 670 public void testDeleteVersionMaskingMultiplePuts() throws IOException { 671 long now = System.currentTimeMillis(); 672 KeyValue [] kvs1 = new KeyValue[] { 673 create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care"), 674 create("R1", "cf", "a", now, KeyValue.Type.Delete, "dont-care") 675 }; 676 KeyValue [] kvs2 = new KeyValue[] { 677 create("R1", "cf", "a", now-500, KeyValue.Type.Put, "dont-care"), 678 create("R1", "cf", "a", now-100, KeyValue.Type.Put, "dont-care"), 679 create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care") 680 }; 681 List<KeyValueScanner> scanners = scanFixture(kvs1, kvs2); 682 683 try (StoreScanner scan = new StoreScanner(new Scan().withStartRow(Bytes.toBytes("R1")), 684 scanInfo, getCols("a"), scanners)) { 685 List<Cell> results = new ArrayList<>(); 686 // the two put at ts=now will be masked by the 1 delete, and 687 // since the scan default returns 1 version we'll return the newest 688 // key, which is kvs[2], now-100. 689 assertEquals(true, scan.next(results)); 690 assertEquals(1, results.size()); 691 assertEquals(kvs2[1], results.get(0)); 692 } 693 } 694 695 @Test 696 public void testDeleteVersionsMixedAndMultipleVersionReturn() throws IOException { 697 long now = System.currentTimeMillis(); 698 KeyValue [] kvs1 = new KeyValue[] { 699 create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care"), 700 create("R1", "cf", "a", now, KeyValue.Type.Delete, "dont-care") 701 }; 702 KeyValue [] kvs2 = new KeyValue[] { 703 create("R1", "cf", "a", now-500, KeyValue.Type.Put, "dont-care"), 704 create("R1", "cf", "a", now+500, KeyValue.Type.Put, "dont-care"), 705 create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care"), 706 create("R2", "cf", "z", now, KeyValue.Type.Put, "dont-care") 707 }; 708 List<KeyValueScanner> scanners = scanFixture(kvs1, kvs2); 709 710 Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1")).readVersions(2); 711 try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) { 712 List<Cell> results = new ArrayList<>(); 713 assertEquals(true, scan.next(results)); 714 assertEquals(2, results.size()); 715 assertEquals(kvs2[1], results.get(0)); 716 assertEquals(kvs2[0], results.get(1)); 717 } 718 } 719 720 @Test 721 public void testWildCardOneVersionScan() throws IOException { 722 KeyValue [] kvs = new KeyValue [] { 723 create("R1", "cf", "a", 2, KeyValue.Type.Put, "dont-care"), 724 create("R1", "cf", "b", 1, KeyValue.Type.Put, "dont-care"), 725 create("R1", "cf", "a", 1, KeyValue.Type.DeleteColumn, "dont-care"), 726 }; 727 List<KeyValueScanner> scanners = scanFixture(kvs); 728 try (StoreScanner scan = 729 new StoreScanner(new Scan().withStartRow(Bytes.toBytes("R1")), scanInfo, null, scanners)) { 730 List<Cell> results = new ArrayList<>(); 731 assertEquals(true, scan.next(results)); 732 assertEquals(2, results.size()); 733 assertEquals(kvs[0], results.get(0)); 734 assertEquals(kvs[1], results.get(1)); 735 } 736 } 737 738 @Test 739 public void testWildCardScannerUnderDeletes() throws IOException { 740 KeyValue [] kvs = new KeyValue [] { 741 create("R1", "cf", "a", 2, KeyValue.Type.Put, "dont-care"), // inc 742 // orphaned delete column. 743 create("R1", "cf", "a", 1, KeyValue.Type.DeleteColumn, "dont-care"), 744 // column b 745 create("R1", "cf", "b", 2, KeyValue.Type.Put, "dont-care"), // inc 746 create("R1", "cf", "b", 1, KeyValue.Type.Put, "dont-care"), // inc 747 // column c 748 create("R1", "cf", "c", 10, KeyValue.Type.Delete, "dont-care"), 749 create("R1", "cf", "c", 10, KeyValue.Type.Put, "dont-care"), // no 750 create("R1", "cf", "c", 9, KeyValue.Type.Put, "dont-care"), // inc 751 // column d 752 create("R1", "cf", "d", 11, KeyValue.Type.Put, "dont-care"), // inc 753 create("R1", "cf", "d", 10, KeyValue.Type.DeleteColumn, "dont-care"), 754 create("R1", "cf", "d", 9, KeyValue.Type.Put, "dont-care"), // no 755 create("R1", "cf", "d", 8, KeyValue.Type.Put, "dont-care"), // no 756 757 }; 758 List<KeyValueScanner> scanners = scanFixture(kvs); 759 try (StoreScanner scan = 760 new StoreScanner(new Scan().readVersions(2), scanInfo, null, scanners)) { 761 List<Cell> results = new ArrayList<>(); 762 assertEquals(true, scan.next(results)); 763 assertEquals(5, results.size()); 764 assertEquals(kvs[0], results.get(0)); 765 assertEquals(kvs[2], results.get(1)); 766 assertEquals(kvs[3], results.get(2)); 767 assertEquals(kvs[6], results.get(3)); 768 assertEquals(kvs[7], results.get(4)); 769 } 770 } 771 772 @Test 773 public void testDeleteFamily() throws IOException { 774 KeyValue[] kvs = new KeyValue[] { 775 create("R1", "cf", "a", 100, KeyValue.Type.DeleteFamily, "dont-care"), 776 create("R1", "cf", "b", 11, KeyValue.Type.Put, "dont-care"), 777 create("R1", "cf", "c", 11, KeyValue.Type.Put, "dont-care"), 778 create("R1", "cf", "d", 11, KeyValue.Type.Put, "dont-care"), 779 create("R1", "cf", "e", 11, KeyValue.Type.Put, "dont-care"), 780 create("R1", "cf", "e", 11, KeyValue.Type.DeleteColumn, "dont-care"), 781 create("R1", "cf", "f", 11, KeyValue.Type.Put, "dont-care"), 782 create("R1", "cf", "g", 11, KeyValue.Type.Put, "dont-care"), 783 create("R1", "cf", "g", 11, KeyValue.Type.Delete, "dont-care"), 784 create("R1", "cf", "h", 11, KeyValue.Type.Put, "dont-care"), 785 create("R1", "cf", "i", 11, KeyValue.Type.Put, "dont-care"), 786 create("R2", "cf", "a", 11, KeyValue.Type.Put, "dont-care"), 787 }; 788 List<KeyValueScanner> scanners = scanFixture(kvs); 789 try (StoreScanner scan = 790 new StoreScanner(new Scan().readAllVersions(), scanInfo, null, scanners)) { 791 List<Cell> results = new ArrayList<>(); 792 assertEquals(true, scan.next(results)); 793 assertEquals(0, results.size()); 794 assertEquals(true, scan.next(results)); 795 assertEquals(1, results.size()); 796 assertEquals(kvs[kvs.length - 1], results.get(0)); 797 798 assertEquals(false, scan.next(results)); 799 } 800 } 801 802 @Test 803 public void testDeleteColumn() throws IOException { 804 KeyValue [] kvs = new KeyValue[] { 805 create("R1", "cf", "a", 10, KeyValue.Type.DeleteColumn, "dont-care"), 806 create("R1", "cf", "a", 9, KeyValue.Type.Delete, "dont-care"), 807 create("R1", "cf", "a", 8, KeyValue.Type.Put, "dont-care"), 808 create("R1", "cf", "b", 5, KeyValue.Type.Put, "dont-care") 809 }; 810 List<KeyValueScanner> scanners = scanFixture(kvs); 811 try (StoreScanner scan = new StoreScanner(new Scan(), scanInfo, null, scanners)) { 812 List<Cell> results = new ArrayList<>(); 813 assertEquals(true, scan.next(results)); 814 assertEquals(1, results.size()); 815 assertEquals(kvs[3], results.get(0)); 816 } 817 } 818 819 private static final KeyValue[] kvs = new KeyValue[] { 820 create("R1", "cf", "a", 11, KeyValue.Type.Put, "dont-care"), 821 create("R1", "cf", "b", 11, KeyValue.Type.Put, "dont-care"), 822 create("R1", "cf", "c", 11, KeyValue.Type.Put, "dont-care"), 823 create("R1", "cf", "d", 11, KeyValue.Type.Put, "dont-care"), 824 create("R1", "cf", "e", 11, KeyValue.Type.Put, "dont-care"), 825 create("R1", "cf", "f", 11, KeyValue.Type.Put, "dont-care"), 826 create("R1", "cf", "g", 11, KeyValue.Type.Put, "dont-care"), 827 create("R1", "cf", "h", 11, KeyValue.Type.Put, "dont-care"), 828 create("R1", "cf", "i", 11, KeyValue.Type.Put, "dont-care"), 829 create("R2", "cf", "a", 11, KeyValue.Type.Put, "dont-care"), 830 }; 831 832 @Test 833 public void testSkipColumn() throws IOException { 834 List<KeyValueScanner> scanners = scanFixture(kvs); 835 try (StoreScanner scan = new StoreScanner(new Scan(), scanInfo, getCols("a", "d"), scanners)) { 836 List<Cell> results = new ArrayList<>(); 837 assertEquals(true, scan.next(results)); 838 assertEquals(2, results.size()); 839 assertEquals(kvs[0], results.get(0)); 840 assertEquals(kvs[3], results.get(1)); 841 results.clear(); 842 843 assertEquals(true, scan.next(results)); 844 assertEquals(1, results.size()); 845 assertEquals(kvs[kvs.length - 1], results.get(0)); 846 847 results.clear(); 848 assertEquals(false, scan.next(results)); 849 } 850 } 851 852 /* 853 * Test expiration of KeyValues in combination with a configured TTL for 854 * a column family (as should be triggered in a major compaction). 855 */ 856 @Test 857 public void testWildCardTtlScan() throws IOException { 858 long now = System.currentTimeMillis(); 859 KeyValue [] kvs = new KeyValue[] { 860 create("R1", "cf", "a", now-1000, KeyValue.Type.Put, "dont-care"), 861 create("R1", "cf", "b", now-10, KeyValue.Type.Put, "dont-care"), 862 create("R1", "cf", "c", now-200, KeyValue.Type.Put, "dont-care"), 863 create("R1", "cf", "d", now-10000, KeyValue.Type.Put, "dont-care"), 864 create("R2", "cf", "a", now, KeyValue.Type.Put, "dont-care"), 865 create("R2", "cf", "b", now-10, KeyValue.Type.Put, "dont-care"), 866 create("R2", "cf", "c", now-200, KeyValue.Type.Put, "dont-care"), 867 create("R2", "cf", "c", now-1000, KeyValue.Type.Put, "dont-care") 868 }; 869 List<KeyValueScanner> scanners = scanFixture(kvs); 870 Scan scan = new Scan(); 871 scan.readVersions(1); 872 ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, 500, KeepDeletedCells.FALSE, 873 HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false); 874 try (StoreScanner scanner = new StoreScanner(scan, scanInfo, null, scanners)) { 875 List<Cell> results = new ArrayList<>(); 876 assertEquals(true, scanner.next(results)); 877 assertEquals(2, results.size()); 878 assertEquals(kvs[1], results.get(0)); 879 assertEquals(kvs[2], results.get(1)); 880 results.clear(); 881 882 assertEquals(true, scanner.next(results)); 883 assertEquals(3, results.size()); 884 assertEquals(kvs[4], results.get(0)); 885 assertEquals(kvs[5], results.get(1)); 886 assertEquals(kvs[6], results.get(2)); 887 results.clear(); 888 889 assertEquals(false, scanner.next(results)); 890 } 891 } 892 893 @Test 894 public void testScannerReseekDoesntNPE() throws Exception { 895 List<KeyValueScanner> scanners = scanFixture(kvs); 896 try (StoreScanner scan = new StoreScanner(new Scan(), scanInfo, getCols("a", "d"), scanners)) { 897 // Previously a updateReaders twice in a row would cause an NPE. In test this would also 898 // normally cause an NPE because scan.store is null. So as long as we get through these 899 // two calls we are good and the bug was quashed. 900 scan.updateReaders(Collections.emptyList(), Collections.emptyList()); 901 scan.updateReaders(Collections.emptyList(), Collections.emptyList()); 902 scan.peek(); 903 } 904 } 905 906 @Test @Ignore("this fails, since we don't handle deletions, etc, in peek") 907 public void testPeek() throws Exception { 908 KeyValue[] kvs = new KeyValue [] { 909 create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"), 910 create("R1", "cf", "a", 1, KeyValue.Type.Delete, "dont-care"), 911 }; 912 List<KeyValueScanner> scanners = scanFixture(kvs); 913 Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1")); 914 try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) { 915 assertNull(scan.peek()); 916 } 917 } 918 919 /** 920 * Ensure that expired delete family markers don't override valid puts 921 */ 922 @Test 923 public void testExpiredDeleteFamily() throws Exception { 924 long now = System.currentTimeMillis(); 925 KeyValue[] kvs = new KeyValue[] { 926 new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null, now-1000, 927 KeyValue.Type.DeleteFamily), 928 create("R1", "cf", "a", now-10, KeyValue.Type.Put, 929 "dont-care"), 930 }; 931 List<KeyValueScanner> scanners = scanFixture(kvs); 932 Scan scan = new Scan(); 933 scan.readVersions(1); 934 // scanner with ttl equal to 500 935 ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, 500, KeepDeletedCells.FALSE, 936 HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false); 937 try (StoreScanner scanner = new StoreScanner(scan, scanInfo, null, scanners)) { 938 List<Cell> results = new ArrayList<>(); 939 assertEquals(true, scanner.next(results)); 940 assertEquals(1, results.size()); 941 assertEquals(kvs[1], results.get(0)); 942 results.clear(); 943 944 assertEquals(false, scanner.next(results)); 945 } 946 } 947 948 @Test 949 public void testDeleteMarkerLongevity() throws Exception { 950 try { 951 final long now = System.currentTimeMillis(); 952 EnvironmentEdgeManagerTestHelper.injectEdge(new EnvironmentEdge() { 953 @Override 954 public long currentTime() { 955 return now; 956 } 957 }); 958 KeyValue[] kvs = new KeyValue[]{ 959 /*0*/ new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null, 960 now - 100, KeyValue.Type.DeleteFamily), // live 961 /*1*/ new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null, 962 now - 1000, KeyValue.Type.DeleteFamily), // expired 963 /*2*/ create("R1", "cf", "a", now - 50, 964 KeyValue.Type.Put, "v3"), // live 965 /*3*/ create("R1", "cf", "a", now - 55, 966 KeyValue.Type.Delete, "dontcare"), // live 967 /*4*/ create("R1", "cf", "a", now - 55, 968 KeyValue.Type.Put, "deleted-version v2"), // deleted 969 /*5*/ create("R1", "cf", "a", now - 60, 970 KeyValue.Type.Put, "v1"), // live 971 /*6*/ create("R1", "cf", "a", now - 65, 972 KeyValue.Type.Put, "v0"), // max-version reached 973 /*7*/ create("R1", "cf", "a", 974 now - 100, KeyValue.Type.DeleteColumn, "dont-care"), // max-version 975 /*8*/ create("R1", "cf", "b", now - 600, 976 KeyValue.Type.DeleteColumn, "dont-care"), //expired 977 /*9*/ create("R1", "cf", "b", now - 70, 978 KeyValue.Type.Put, "v2"), //live 979 /*10*/ create("R1", "cf", "b", now - 750, 980 KeyValue.Type.Put, "v1"), //expired 981 /*11*/ create("R1", "cf", "c", now - 500, 982 KeyValue.Type.Delete, "dontcare"), //expired 983 /*12*/ create("R1", "cf", "c", now - 600, 984 KeyValue.Type.Put, "v1"), //expired 985 /*13*/ create("R1", "cf", "c", now - 1000, 986 KeyValue.Type.Delete, "dontcare"), //expired 987 /*14*/ create("R1", "cf", "d", now - 60, 988 KeyValue.Type.Put, "expired put"), //live 989 /*15*/ create("R1", "cf", "d", now - 100, 990 KeyValue.Type.Delete, "not-expired delete"), //live 991 }; 992 List<KeyValueScanner> scanners = scanFixture(kvs); 993 ScanInfo scanInfo = new ScanInfo(CONF, Bytes.toBytes("cf"), 994 0 /* minVersions */, 995 2 /* maxVersions */, 500 /* ttl */, 996 KeepDeletedCells.FALSE /* keepDeletedCells */, 997 HConstants.DEFAULT_BLOCKSIZE /* block size */, 998 200, /* timeToPurgeDeletes */ 999 CellComparator.getInstance(), false); 1000 try (StoreScanner scanner = 1001 new StoreScanner(scanInfo, 2, ScanType.COMPACT_DROP_DELETES, scanners)) { 1002 List<Cell> results = new ArrayList<>(); 1003 results = new ArrayList<>(); 1004 assertEquals(true, scanner.next(results)); 1005 assertEquals(kvs[0], results.get(0)); 1006 assertEquals(kvs[2], results.get(1)); 1007 assertEquals(kvs[3], results.get(2)); 1008 assertEquals(kvs[5], results.get(3)); 1009 assertEquals(kvs[9], results.get(4)); 1010 assertEquals(kvs[14], results.get(5)); 1011 assertEquals(kvs[15], results.get(6)); 1012 assertEquals(7, results.size()); 1013 } 1014 } finally { 1015 EnvironmentEdgeManagerTestHelper.reset(); 1016 } 1017 } 1018 1019 @Test 1020 public void testPreadNotEnabledForCompactionStoreScanners() throws Exception { 1021 long now = System.currentTimeMillis(); 1022 KeyValue[] kvs = new KeyValue[] { 1023 new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null, now - 1000, 1024 KeyValue.Type.DeleteFamily), 1025 create("R1", "cf", "a", now - 10, KeyValue.Type.Put, "dont-care"), }; 1026 List<KeyValueScanner> scanners = scanFixture(kvs); 1027 ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, 500, KeepDeletedCells.FALSE, 1028 HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false); 1029 try (StoreScanner storeScanner = new StoreScanner(scanInfo, -1, 1030 ScanType.COMPACT_RETAIN_DELETES, scanners)) { 1031 assertFalse(storeScanner.isScanUsePread()); 1032 } 1033 } 1034}