001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.regionserver; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertNotNull; 022import static org.junit.Assert.assertNull; 023import static org.junit.Assert.assertTrue; 024 025import java.io.IOException; 026import java.util.ArrayList; 027import java.util.Arrays; 028import java.util.List; 029import java.util.NavigableMap; 030import java.util.Objects; 031import java.util.TreeMap; 032import java.util.concurrent.atomic.AtomicLong; 033import java.util.concurrent.atomic.AtomicReference; 034import org.apache.hadoop.conf.Configuration; 035import org.apache.hadoop.fs.Path; 036import org.apache.hadoop.hbase.Cell; 037import org.apache.hadoop.hbase.CellComparatorImpl; 038import org.apache.hadoop.hbase.CellUtil; 039import org.apache.hadoop.hbase.ExtendedCell; 040import org.apache.hadoop.hbase.HBaseClassTestRule; 041import org.apache.hadoop.hbase.HBaseConfiguration; 042import org.apache.hadoop.hbase.HBaseTestingUtil; 043import org.apache.hadoop.hbase.HConstants; 044import org.apache.hadoop.hbase.KeepDeletedCells; 045import org.apache.hadoop.hbase.KeyValue; 046import org.apache.hadoop.hbase.KeyValueTestUtil; 047import org.apache.hadoop.hbase.KeyValueUtil; 048import org.apache.hadoop.hbase.TableDescriptors; 049import org.apache.hadoop.hbase.TableName; 050import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; 051import org.apache.hadoop.hbase.client.Put; 052import org.apache.hadoop.hbase.client.RegionInfo; 053import org.apache.hadoop.hbase.client.RegionInfoBuilder; 054import org.apache.hadoop.hbase.client.Scan; 055import org.apache.hadoop.hbase.client.TableDescriptor; 056import org.apache.hadoop.hbase.client.TableDescriptorBuilder; 057import org.apache.hadoop.hbase.exceptions.UnexpectedStateException; 058import org.apache.hadoop.hbase.monitoring.ThreadLocalServerSideScanMetrics; 059import org.apache.hadoop.hbase.testclassification.MediumTests; 060import org.apache.hadoop.hbase.testclassification.RegionServerTests; 061import org.apache.hadoop.hbase.util.Bytes; 062import org.apache.hadoop.hbase.util.EnvironmentEdge; 063import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; 064import org.apache.hadoop.hbase.util.FSTableDescriptors; 065import org.apache.hadoop.hbase.wal.WALFactory; 066import org.junit.AfterClass; 067import org.junit.Assert; 068import org.junit.Before; 069import org.junit.ClassRule; 070import org.junit.Rule; 071import org.junit.Test; 072import org.junit.experimental.categories.Category; 073import org.junit.rules.TestName; 074import org.slf4j.Logger; 075import org.slf4j.LoggerFactory; 076 077import org.apache.hbase.thirdparty.com.google.common.base.Joiner; 078import org.apache.hbase.thirdparty.com.google.common.collect.Iterables; 079import org.apache.hbase.thirdparty.com.google.common.collect.Lists; 080 081/** memstore test case */ 082@Category({ RegionServerTests.class, MediumTests.class }) 083public class TestDefaultMemStore { 084 085 @ClassRule 086 public static final HBaseClassTestRule CLASS_RULE = 087 HBaseClassTestRule.forClass(TestDefaultMemStore.class); 088 089 private static final Logger LOG = LoggerFactory.getLogger(TestDefaultMemStore.class); 090 @Rule 091 public TestName name = new TestName(); 092 protected AbstractMemStore memstore; 093 protected static final int ROW_COUNT = 10; 094 protected static final int QUALIFIER_COUNT = ROW_COUNT; 095 protected static final byte[] FAMILY = Bytes.toBytes("column"); 096 protected MultiVersionConcurrencyControl mvcc; 097 protected AtomicLong startSeqNum = new AtomicLong(0); 098 protected ChunkCreator chunkCreator; 099 100 private String getName() { 101 return this.name.getMethodName(); 102 } 103 104 @Before 105 public void setUp() throws Exception { 106 internalSetUp(); 107 // no pool 108 this.chunkCreator = ChunkCreator.initialize(MemStoreLAB.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, 109 null, MemStoreLAB.INDEX_CHUNK_SIZE_PERCENTAGE_DEFAULT); 110 this.memstore = new DefaultMemStore(); 111 } 112 113 @AfterClass 114 public static void tearDownClass() throws Exception { 115 ChunkCreator.getInstance().clearChunkIds(); 116 } 117 118 protected void internalSetUp() throws Exception { 119 this.mvcc = new MultiVersionConcurrencyControl(); 120 } 121 122 @Test 123 public void testPutSameKey() { 124 byte[] bytes = Bytes.toBytes(getName()); 125 KeyValue kv = new KeyValue(bytes, bytes, bytes, bytes); 126 this.memstore.add(kv, null); 127 byte[] other = Bytes.toBytes("somethingelse"); 128 KeyValue samekey = new KeyValue(bytes, bytes, bytes, other); 129 this.memstore.add(samekey, null); 130 Cell found = this.memstore.getActive().first(); 131 assertEquals(1, this.memstore.getActive().getCellsCount()); 132 assertTrue(Bytes.toString(found.getValueArray()), CellUtil.matchingValue(samekey, found)); 133 } 134 135 @Test 136 public void testPutSameCell() { 137 byte[] bytes = Bytes.toBytes(getName()); 138 KeyValue kv = new KeyValue(bytes, bytes, bytes, bytes); 139 MemStoreSizing sizeChangeForFirstCell = new NonThreadSafeMemStoreSizing(); 140 this.memstore.add(kv, sizeChangeForFirstCell); 141 MemStoreSizing sizeChangeForSecondCell = new NonThreadSafeMemStoreSizing(); 142 this.memstore.add(kv, sizeChangeForSecondCell); 143 // make sure memstore size increase won't double-count MSLAB chunk size 144 assertEquals(Segment.getCellLength(kv), sizeChangeForFirstCell.getMemStoreSize().getDataSize()); 145 Segment segment = this.memstore.getActive(); 146 MemStoreLAB msLab = segment.getMemStoreLAB(); 147 if (msLab != null) { 148 // make sure memstore size increased even when writing the same cell, if using MSLAB 149 assertEquals(Segment.getCellLength(kv), 150 sizeChangeForSecondCell.getMemStoreSize().getDataSize()); 151 // make sure chunk size increased even when writing the same cell, if using MSLAB 152 if (msLab instanceof MemStoreLABImpl) { 153 // since we add the chunkID at the 0th offset of the chunk and the 154 // chunkid is an int we need to account for those 4 bytes 155 assertEquals(2 * Segment.getCellLength(kv) + Bytes.SIZEOF_INT, 156 ((MemStoreLABImpl) msLab).getCurrentChunk().getNextFreeOffset()); 157 } 158 } else { 159 // make sure no memstore size change w/o MSLAB 160 assertEquals(0, sizeChangeForSecondCell.getMemStoreSize().getDataSize()); 161 assertEquals(0, sizeChangeForSecondCell.getMemStoreSize().getHeapSize()); 162 } 163 } 164 165 /** 166 * Test memstore snapshot happening while scanning. 167 */ 168 @Test 169 public void testScanAcrossSnapshot() throws IOException { 170 int rowCount = addRows(this.memstore); 171 List<KeyValueScanner> memstorescanners = this.memstore.getScanners(0); 172 Scan scan = new Scan(); 173 List<Cell> result = new ArrayList<>(); 174 Configuration conf = HBaseConfiguration.create(); 175 ScanInfo scanInfo = 176 new ScanInfo(conf, null, 0, 1, HConstants.LATEST_TIMESTAMP, KeepDeletedCells.FALSE, 177 HConstants.DEFAULT_BLOCKSIZE, 0, this.memstore.getComparator(), false); 178 int count = 0; 179 try (StoreScanner s = new StoreScanner(scan, scanInfo, null, memstorescanners)) { 180 while (s.next(result)) { 181 LOG.info(Objects.toString(result)); 182 count++; 183 // Row count is same as column count. 184 assertEquals(rowCount, result.size()); 185 result.clear(); 186 } 187 } 188 assertEquals(rowCount, count); 189 for (KeyValueScanner scanner : memstorescanners) { 190 scanner.close(); 191 } 192 193 memstorescanners = this.memstore.getScanners(mvcc.getReadPoint()); 194 // Now assert can count same number even if a snapshot mid-scan. 195 count = 0; 196 try (StoreScanner s = new StoreScanner(scan, scanInfo, null, memstorescanners)) { 197 while (s.next(result)) { 198 LOG.info(Objects.toString(result)); 199 // Assert the stuff is coming out in right order. 200 assertTrue(CellUtil.matchingRows(result.get(0), Bytes.toBytes(count))); 201 count++; 202 // Row count is same as column count. 203 assertEquals(rowCount, result.size()); 204 if (count == 2) { 205 this.memstore.snapshot(); 206 LOG.info("Snapshotted"); 207 } 208 result.clear(); 209 } 210 } 211 assertEquals(rowCount, count); 212 for (KeyValueScanner scanner : memstorescanners) { 213 scanner.close(); 214 } 215 memstorescanners = this.memstore.getScanners(mvcc.getReadPoint()); 216 // Assert that new values are seen in kvset as we scan. 217 long ts = EnvironmentEdgeManager.currentTime(); 218 count = 0; 219 int snapshotIndex = 5; 220 try (StoreScanner s = new StoreScanner(scan, scanInfo, null, memstorescanners)) { 221 while (s.next(result)) { 222 LOG.info(Objects.toString(result)); 223 // Assert the stuff is coming out in right order. 224 assertTrue(CellUtil.matchingRows(result.get(0), Bytes.toBytes(count))); 225 // Row count is same as column count. 226 assertEquals("count=" + count + ", result=" + result, rowCount, result.size()); 227 count++; 228 if (count == snapshotIndex) { 229 MemStoreSnapshot snapshot = this.memstore.snapshot(); 230 this.memstore.clearSnapshot(snapshot.getId()); 231 // Added more rows into kvset. But the scanner wont see these rows. 232 addRows(this.memstore, ts); 233 LOG.info("Snapshotted, cleared it and then added values (which wont be seen)"); 234 } 235 result.clear(); 236 } 237 } 238 assertEquals(rowCount, count); 239 } 240 241 /** 242 * A simple test which verifies the 3 possible states when scanning across snapshot. 243 */ 244 @Test 245 public void testScanAcrossSnapshot2() throws IOException, CloneNotSupportedException { 246 // we are going to the scanning across snapshot with two kvs 247 // kv1 should always be returned before kv2 248 final byte[] one = Bytes.toBytes(1); 249 final byte[] two = Bytes.toBytes(2); 250 final byte[] f = Bytes.toBytes("f"); 251 final byte[] q = Bytes.toBytes("q"); 252 final byte[] v = Bytes.toBytes(3); 253 254 final KeyValue kv1 = new KeyValue(one, f, q, v); 255 final KeyValue kv2 = new KeyValue(two, f, q, v); 256 257 // use case 1: both kvs in kvset 258 this.memstore.add(kv1.clone(), null); 259 this.memstore.add(kv2.clone(), null); 260 // snapshot is empty,active segment is not empty, 261 // empty segment is skipped. 262 verifyOneScanAcrossSnapshot2(kv1, kv2); 263 264 // use case 2: both kvs in snapshot 265 // active segment is empty,snapshot is not empty, 266 // empty segment is skipped. 267 this.memstore.snapshot(); 268 // 269 verifyOneScanAcrossSnapshot2(kv1, kv2); 270 271 // use case 3: first in snapshot second in kvset 272 this.memstore = new DefaultMemStore(); 273 this.memstore.add(kv1.clone(), null); 274 this.memstore.snapshot(); 275 this.memstore.add(kv2.clone(), null); 276 verifyScanAcrossSnapshot2(kv1, kv2); 277 } 278 279 protected void verifyScanAcrossSnapshot2(KeyValue kv1, KeyValue kv2) throws IOException { 280 List<KeyValueScanner> memstorescanners = this.memstore.getScanners(mvcc.getReadPoint()); 281 assertEquals(2, memstorescanners.size()); 282 final KeyValueScanner scanner0 = memstorescanners.get(0); 283 final KeyValueScanner scanner1 = memstorescanners.get(1); 284 scanner0.seek(KeyValueUtil.createFirstOnRow(HConstants.EMPTY_START_ROW)); 285 scanner1.seek(KeyValueUtil.createFirstOnRow(HConstants.EMPTY_START_ROW)); 286 Cell n0 = scanner0.next(); 287 Cell n1 = scanner1.next(); 288 assertTrue(kv1.equals(n0) || kv1.equals(n1)); 289 assertTrue(kv2.equals(n0) || kv2.equals(n1) || kv2.equals(scanner0.next()) 290 || kv2.equals(scanner1.next())); 291 assertNull(scanner0.next()); 292 assertNull(scanner1.next()); 293 } 294 295 protected void verifyOneScanAcrossSnapshot2(KeyValue kv1, KeyValue kv2) throws IOException { 296 List<KeyValueScanner> memstorescanners = this.memstore.getScanners(mvcc.getReadPoint()); 297 assertEquals(1, memstorescanners.size()); 298 final KeyValueScanner scanner0 = memstorescanners.get(0); 299 scanner0.seek(KeyValueUtil.createFirstOnRow(HConstants.EMPTY_START_ROW)); 300 Cell n0 = scanner0.next(); 301 Cell n1 = scanner0.next(); 302 assertTrue(kv1.equals(n0)); 303 assertTrue(kv2.equals(n1)); 304 assertNull(scanner0.next()); 305 } 306 307 protected void assertScannerResults(KeyValueScanner scanner, KeyValue[] expected) 308 throws IOException { 309 scanner.seek(KeyValueUtil.createFirstOnRow(new byte[] {})); 310 List<Cell> returned = Lists.newArrayList(); 311 312 while (true) { 313 Cell next = scanner.next(); 314 if (next == null) break; 315 returned.add(next); 316 } 317 318 assertTrue( 319 "Got:\n" + Joiner.on("\n").join(returned) + "\nExpected:\n" + Joiner.on("\n").join(expected), 320 Iterables.elementsEqual(Arrays.asList(expected), returned)); 321 assertNull(scanner.peek()); 322 } 323 324 @Test 325 public void testMemstoreConcurrentControl() throws IOException { 326 final byte[] row = Bytes.toBytes(1); 327 final byte[] f = Bytes.toBytes("family"); 328 final byte[] q1 = Bytes.toBytes("q1"); 329 final byte[] q2 = Bytes.toBytes("q2"); 330 final byte[] v = Bytes.toBytes("value"); 331 332 MultiVersionConcurrencyControl.WriteEntry w = mvcc.begin(); 333 334 KeyValue kv1 = new KeyValue(row, f, q1, v); 335 kv1.setSequenceId(w.getWriteNumber()); 336 memstore.add(kv1, null); 337 338 KeyValueScanner s = this.memstore.getScanners(mvcc.getReadPoint()).get(0); 339 assertScannerResults(s, new KeyValue[] {}); 340 341 mvcc.completeAndWait(w); 342 343 s = this.memstore.getScanners(mvcc.getReadPoint()).get(0); 344 assertScannerResults(s, new KeyValue[] { kv1 }); 345 346 w = mvcc.begin(); 347 KeyValue kv2 = new KeyValue(row, f, q2, v); 348 kv2.setSequenceId(w.getWriteNumber()); 349 memstore.add(kv2, null); 350 351 s = this.memstore.getScanners(mvcc.getReadPoint()).get(0); 352 assertScannerResults(s, new KeyValue[] { kv1 }); 353 354 mvcc.completeAndWait(w); 355 356 s = this.memstore.getScanners(mvcc.getReadPoint()).get(0); 357 assertScannerResults(s, new KeyValue[] { kv1, kv2 }); 358 } 359 360 private long getBytesReadFromMemstore() throws IOException { 361 final byte[] f = Bytes.toBytes("family"); 362 final byte[] q1 = Bytes.toBytes("q1"); 363 final byte[] v = Bytes.toBytes("value"); 364 int numKvs = 10; 365 366 MultiVersionConcurrencyControl.WriteEntry w = mvcc.begin(); 367 368 KeyValue kv; 369 KeyValue[] kvs = new KeyValue[numKvs]; 370 long totalCellSize = 0; 371 for (int i = 0; i < numKvs; i++) { 372 byte[] row = Bytes.toBytes(i); 373 kv = new KeyValue(row, f, q1, v); 374 kv.setSequenceId(w.getWriteNumber()); 375 memstore.add(kv, null); 376 kvs[i] = kv; 377 totalCellSize += Segment.getCellLength(kv); 378 } 379 mvcc.completeAndWait(w); 380 381 ThreadLocalServerSideScanMetrics.getBytesReadFromMemstoreAndReset(); 382 KeyValueScanner s = this.memstore.getScanners(mvcc.getReadPoint()).get(0); 383 assertScannerResults(s, kvs); 384 return totalCellSize; 385 } 386 387 @Test 388 public void testBytesReadFromMemstore() throws IOException { 389 ThreadLocalServerSideScanMetrics.setScanMetricsEnabled(true); 390 long totalCellSize = getBytesReadFromMemstore(); 391 Assert.assertEquals(totalCellSize, 392 ThreadLocalServerSideScanMetrics.getBytesReadFromMemstoreAndReset()); 393 } 394 395 @Test 396 public void testBytesReadFromMemstoreWithScanMetricsDisabled() throws IOException { 397 ThreadLocalServerSideScanMetrics.setScanMetricsEnabled(false); 398 getBytesReadFromMemstore(); 399 Assert.assertEquals(0, ThreadLocalServerSideScanMetrics.getBytesReadFromMemstoreAndReset()); 400 } 401 402 /** 403 * Regression test for HBASE-2616, HBASE-2670. When we insert a higher-memstoreTS version of a 404 * cell but with the same timestamp, we still need to provide consistent reads for the same 405 * scanner. 406 */ 407 @Test 408 public void testMemstoreEditsVisibilityWithSameKey() throws IOException { 409 final byte[] row = Bytes.toBytes(1); 410 final byte[] f = Bytes.toBytes("family"); 411 final byte[] q1 = Bytes.toBytes("q1"); 412 final byte[] q2 = Bytes.toBytes("q2"); 413 final byte[] v1 = Bytes.toBytes("value1"); 414 final byte[] v2 = Bytes.toBytes("value2"); 415 416 // INSERT 1: Write both columns val1 417 MultiVersionConcurrencyControl.WriteEntry w = mvcc.begin(); 418 419 KeyValue kv11 = new KeyValue(row, f, q1, v1); 420 kv11.setSequenceId(w.getWriteNumber()); 421 memstore.add(kv11, null); 422 423 KeyValue kv12 = new KeyValue(row, f, q2, v1); 424 kv12.setSequenceId(w.getWriteNumber()); 425 memstore.add(kv12, null); 426 mvcc.completeAndWait(w); 427 428 // BEFORE STARTING INSERT 2, SEE FIRST KVS 429 KeyValueScanner s = this.memstore.getScanners(mvcc.getReadPoint()).get(0); 430 assertScannerResults(s, new KeyValue[] { kv11, kv12 }); 431 432 // START INSERT 2: Write both columns val2 433 w = mvcc.begin(); 434 KeyValue kv21 = new KeyValue(row, f, q1, v2); 435 kv21.setSequenceId(w.getWriteNumber()); 436 memstore.add(kv21, null); 437 438 KeyValue kv22 = new KeyValue(row, f, q2, v2); 439 kv22.setSequenceId(w.getWriteNumber()); 440 memstore.add(kv22, null); 441 442 // BEFORE COMPLETING INSERT 2, SEE FIRST KVS 443 s = this.memstore.getScanners(mvcc.getReadPoint()).get(0); 444 assertScannerResults(s, new KeyValue[] { kv11, kv12 }); 445 446 // COMPLETE INSERT 2 447 mvcc.completeAndWait(w); 448 449 // NOW SHOULD SEE NEW KVS IN ADDITION TO OLD KVS. 450 // See HBASE-1485 for discussion about what we should do with 451 // the duplicate-TS inserts 452 s = this.memstore.getScanners(mvcc.getReadPoint()).get(0); 453 assertScannerResults(s, new KeyValue[] { kv21, kv11, kv22, kv12 }); 454 } 455 456 /** 457 * When we insert a higher-memstoreTS deletion of a cell but with the same timestamp, we still 458 * need to provide consistent reads for the same scanner. 459 */ 460 @Test 461 public void testMemstoreDeletesVisibilityWithSameKey() throws IOException { 462 final byte[] row = Bytes.toBytes(1); 463 final byte[] f = Bytes.toBytes("family"); 464 final byte[] q1 = Bytes.toBytes("q1"); 465 final byte[] q2 = Bytes.toBytes("q2"); 466 final byte[] v1 = Bytes.toBytes("value1"); 467 // INSERT 1: Write both columns val1 468 MultiVersionConcurrencyControl.WriteEntry w = mvcc.begin(); 469 470 KeyValue kv11 = new KeyValue(row, f, q1, v1); 471 kv11.setSequenceId(w.getWriteNumber()); 472 memstore.add(kv11, null); 473 474 KeyValue kv12 = new KeyValue(row, f, q2, v1); 475 kv12.setSequenceId(w.getWriteNumber()); 476 memstore.add(kv12, null); 477 mvcc.completeAndWait(w); 478 479 // BEFORE STARTING INSERT 2, SEE FIRST KVS 480 KeyValueScanner s = this.memstore.getScanners(mvcc.getReadPoint()).get(0); 481 assertScannerResults(s, new KeyValue[] { kv11, kv12 }); 482 483 // START DELETE: Insert delete for one of the columns 484 w = mvcc.begin(); 485 KeyValue kvDel = new KeyValue(row, f, q2, kv11.getTimestamp(), KeyValue.Type.DeleteColumn); 486 kvDel.setSequenceId(w.getWriteNumber()); 487 memstore.add(kvDel, null); 488 489 // BEFORE COMPLETING DELETE, SEE FIRST KVS 490 s = this.memstore.getScanners(mvcc.getReadPoint()).get(0); 491 assertScannerResults(s, new KeyValue[] { kv11, kv12 }); 492 493 // COMPLETE DELETE 494 mvcc.completeAndWait(w); 495 496 // NOW WE SHOULD SEE DELETE 497 s = this.memstore.getScanners(mvcc.getReadPoint()).get(0); 498 assertScannerResults(s, new KeyValue[] { kv11, kvDel, kv12 }); 499 } 500 501 private static class ReadOwnWritesTester extends Thread { 502 static final int NUM_TRIES = 1000; 503 504 final byte[] row; 505 506 final byte[] f = Bytes.toBytes("family"); 507 final byte[] q1 = Bytes.toBytes("q1"); 508 509 final MultiVersionConcurrencyControl mvcc; 510 final MemStore memstore; 511 512 AtomicReference<Throwable> caughtException; 513 514 public ReadOwnWritesTester(int id, MemStore memstore, MultiVersionConcurrencyControl mvcc, 515 AtomicReference<Throwable> caughtException) { 516 this.mvcc = mvcc; 517 this.memstore = memstore; 518 this.caughtException = caughtException; 519 row = Bytes.toBytes(id); 520 } 521 522 @Override 523 public void run() { 524 try { 525 internalRun(); 526 } catch (Throwable t) { 527 caughtException.compareAndSet(null, t); 528 } 529 } 530 531 private void internalRun() throws IOException { 532 for (long i = 0; i < NUM_TRIES && caughtException.get() == null; i++) { 533 MultiVersionConcurrencyControl.WriteEntry w = mvcc.begin(); 534 535 // Insert the sequence value (i) 536 byte[] v = Bytes.toBytes(i); 537 538 KeyValue kv = new KeyValue(row, f, q1, i, v); 539 kv.setSequenceId(w.getWriteNumber()); 540 memstore.add(kv, null); 541 mvcc.completeAndWait(w); 542 543 // Assert that we can read back 544 KeyValueScanner s = this.memstore.getScanners(mvcc.getReadPoint()).get(0); 545 s.seek(kv); 546 547 Cell ret = s.next(); 548 assertNotNull("Didnt find own write at all", ret); 549 assertEquals("Didnt read own writes", kv.getTimestamp(), ret.getTimestamp()); 550 } 551 } 552 } 553 554 @Test 555 public void testReadOwnWritesUnderConcurrency() throws Throwable { 556 int NUM_THREADS = 8; 557 558 ReadOwnWritesTester threads[] = new ReadOwnWritesTester[NUM_THREADS]; 559 AtomicReference<Throwable> caught = new AtomicReference<>(); 560 561 for (int i = 0; i < NUM_THREADS; i++) { 562 threads[i] = new ReadOwnWritesTester(i, memstore, mvcc, caught); 563 threads[i].start(); 564 } 565 566 for (int i = 0; i < NUM_THREADS; i++) { 567 threads[i].join(); 568 } 569 570 if (caught.get() != null) { 571 throw caught.get(); 572 } 573 } 574 575 /** 576 * Test memstore snapshots 577 */ 578 @Test 579 public void testSnapshotting() throws IOException { 580 final int snapshotCount = 5; 581 // Add some rows, run a snapshot. Do it a few times. 582 for (int i = 0; i < snapshotCount; i++) { 583 addRows(this.memstore); 584 runSnapshot(this.memstore); 585 assertEquals("History not being cleared", 0, this.memstore.getSnapshot().getCellsCount()); 586 } 587 } 588 589 @Test 590 public void testMultipleVersionsSimple() throws Exception { 591 DefaultMemStore m = new DefaultMemStore(new Configuration(), CellComparatorImpl.COMPARATOR); 592 byte[] row = Bytes.toBytes("testRow"); 593 byte[] family = Bytes.toBytes("testFamily"); 594 byte[] qf = Bytes.toBytes("testQualifier"); 595 long[] stamps = { 1, 2, 3 }; 596 byte[][] values = { Bytes.toBytes("value0"), Bytes.toBytes("value1"), Bytes.toBytes("value2") }; 597 KeyValue key0 = new KeyValue(row, family, qf, stamps[0], values[0]); 598 KeyValue key1 = new KeyValue(row, family, qf, stamps[1], values[1]); 599 KeyValue key2 = new KeyValue(row, family, qf, stamps[2], values[2]); 600 601 m.add(key0, null); 602 m.add(key1, null); 603 m.add(key2, null); 604 605 assertTrue("Expected memstore to hold 3 values, actually has " + m.getActive().getCellsCount(), 606 m.getActive().getCellsCount() == 3); 607 } 608 609 ////////////////////////////////////////////////////////////////////////////// 610 // Get tests 611 ////////////////////////////////////////////////////////////////////////////// 612 613 /** 614 * Test getNextRow from memstore 615 */ 616 @Test 617 public void testGetNextRow() throws Exception { 618 addRows(this.memstore); 619 // Add more versions to make it a little more interesting. 620 Thread.sleep(1); 621 addRows(this.memstore); 622 Cell closestToEmpty = ((DefaultMemStore) this.memstore).getNextRow(KeyValue.LOWESTKEY); 623 assertTrue(CellComparatorImpl.COMPARATOR.compareRows(closestToEmpty, 624 new KeyValue(Bytes.toBytes(0), EnvironmentEdgeManager.currentTime())) == 0); 625 for (int i = 0; i < ROW_COUNT; i++) { 626 Cell nr = ((DefaultMemStore) this.memstore) 627 .getNextRow(new KeyValue(Bytes.toBytes(i), EnvironmentEdgeManager.currentTime())); 628 if (i + 1 == ROW_COUNT) { 629 assertNull(nr); 630 } else { 631 assertTrue(CellComparatorImpl.COMPARATOR.compareRows(nr, 632 new KeyValue(Bytes.toBytes(i + 1), EnvironmentEdgeManager.currentTime())) == 0); 633 } 634 } 635 // starting from each row, validate results should contain the starting row 636 Configuration conf = HBaseConfiguration.create(); 637 for (int startRowId = 0; startRowId < ROW_COUNT; startRowId++) { 638 ScanInfo scanInfo = 639 new ScanInfo(conf, FAMILY, 0, 1, Integer.MAX_VALUE, KeepDeletedCells.FALSE, 640 HConstants.DEFAULT_BLOCKSIZE, 0, this.memstore.getComparator(), false); 641 try (InternalScanner scanner = 642 new StoreScanner(new Scan().withStartRow(Bytes.toBytes(startRowId)), scanInfo, null, 643 memstore.getScanners(0))) { 644 List<Cell> results = new ArrayList<>(); 645 for (int i = 0; scanner.next(results); i++) { 646 int rowId = startRowId + i; 647 Cell left = results.get(0); 648 byte[] row1 = Bytes.toBytes(rowId); 649 assertTrue("Row name", 650 CellComparatorImpl.COMPARATOR.compareRows(left, row1, 0, row1.length) == 0); 651 assertEquals("Count of columns", QUALIFIER_COUNT, results.size()); 652 List<Cell> row = new ArrayList<>(); 653 for (Cell kv : results) { 654 row.add(kv); 655 } 656 isExpectedRowWithoutTimestamps(rowId, row); 657 // Clear out set. Otherwise row results accumulate. 658 results.clear(); 659 } 660 } 661 } 662 } 663 664 @Test 665 public void testGet_memstoreAndSnapShot() throws IOException { 666 byte[] row = Bytes.toBytes("testrow"); 667 byte[] fam = Bytes.toBytes("testfamily"); 668 byte[] qf1 = Bytes.toBytes("testqualifier1"); 669 byte[] qf2 = Bytes.toBytes("testqualifier2"); 670 byte[] qf3 = Bytes.toBytes("testqualifier3"); 671 byte[] qf4 = Bytes.toBytes("testqualifier4"); 672 byte[] qf5 = Bytes.toBytes("testqualifier5"); 673 byte[] val = Bytes.toBytes("testval"); 674 675 // Setting up memstore 676 memstore.add(new KeyValue(row, fam, qf1, val), null); 677 memstore.add(new KeyValue(row, fam, qf2, val), null); 678 memstore.add(new KeyValue(row, fam, qf3, val), null); 679 // Creating a snapshot 680 memstore.snapshot(); 681 assertEquals(3, memstore.getSnapshot().getCellsCount()); 682 // Adding value to "new" memstore 683 assertEquals(0, memstore.getActive().getCellsCount()); 684 memstore.add(new KeyValue(row, fam, qf4, val), null); 685 memstore.add(new KeyValue(row, fam, qf5, val), null); 686 assertEquals(2, memstore.getActive().getCellsCount()); 687 } 688 689 ////////////////////////////////////////////////////////////////////////////// 690 // Delete tests 691 ////////////////////////////////////////////////////////////////////////////// 692 @Test 693 public void testGetWithDelete() throws IOException { 694 byte[] row = Bytes.toBytes("testrow"); 695 byte[] fam = Bytes.toBytes("testfamily"); 696 byte[] qf1 = Bytes.toBytes("testqualifier"); 697 byte[] val = Bytes.toBytes("testval"); 698 699 long ts1 = System.nanoTime(); 700 KeyValue put1 = new KeyValue(row, fam, qf1, ts1, val); 701 long ts2 = ts1 + 1; 702 KeyValue put2 = new KeyValue(row, fam, qf1, ts2, val); 703 long ts3 = ts2 + 1; 704 KeyValue put3 = new KeyValue(row, fam, qf1, ts3, val); 705 memstore.add(put1, null); 706 memstore.add(put2, null); 707 memstore.add(put3, null); 708 709 assertEquals(3, memstore.getActive().getCellsCount()); 710 711 KeyValue del2 = new KeyValue(row, fam, qf1, ts2, KeyValue.Type.Delete, val); 712 memstore.add(del2, null); 713 714 List<Cell> expected = new ArrayList<>(); 715 expected.add(put3); 716 expected.add(del2); 717 expected.add(put2); 718 expected.add(put1); 719 720 assertEquals(4, memstore.getActive().getCellsCount()); 721 int i = 0; 722 for (Cell cell : memstore.getActive().getCellSet()) { 723 assertEquals(expected.get(i++), cell); 724 } 725 } 726 727 @Test 728 public void testGetWithDeleteColumn() throws IOException { 729 byte[] row = Bytes.toBytes("testrow"); 730 byte[] fam = Bytes.toBytes("testfamily"); 731 byte[] qf1 = Bytes.toBytes("testqualifier"); 732 byte[] val = Bytes.toBytes("testval"); 733 734 long ts1 = System.nanoTime(); 735 KeyValue put1 = new KeyValue(row, fam, qf1, ts1, val); 736 long ts2 = ts1 + 1; 737 KeyValue put2 = new KeyValue(row, fam, qf1, ts2, val); 738 long ts3 = ts2 + 1; 739 KeyValue put3 = new KeyValue(row, fam, qf1, ts3, val); 740 memstore.add(put1, null); 741 memstore.add(put2, null); 742 memstore.add(put3, null); 743 744 assertEquals(3, memstore.getActive().getCellsCount()); 745 746 KeyValue del2 = new KeyValue(row, fam, qf1, ts2, KeyValue.Type.DeleteColumn, val); 747 memstore.add(del2, null); 748 749 List<Cell> expected = new ArrayList<>(); 750 expected.add(put3); 751 expected.add(del2); 752 expected.add(put2); 753 expected.add(put1); 754 755 assertEquals(4, memstore.getActive().getCellsCount()); 756 int i = 0; 757 for (Cell cell : memstore.getActive().getCellSet()) { 758 assertEquals(expected.get(i++), cell); 759 } 760 } 761 762 @Test 763 public void testGetWithDeleteFamily() throws IOException { 764 byte[] row = Bytes.toBytes("testrow"); 765 byte[] fam = Bytes.toBytes("testfamily"); 766 byte[] qf1 = Bytes.toBytes("testqualifier1"); 767 byte[] qf2 = Bytes.toBytes("testqualifier2"); 768 byte[] qf3 = Bytes.toBytes("testqualifier3"); 769 byte[] val = Bytes.toBytes("testval"); 770 long ts = System.nanoTime(); 771 772 KeyValue put1 = new KeyValue(row, fam, qf1, ts, val); 773 KeyValue put2 = new KeyValue(row, fam, qf2, ts, val); 774 KeyValue put3 = new KeyValue(row, fam, qf3, ts, val); 775 KeyValue put4 = new KeyValue(row, fam, qf3, ts + 1, val); 776 777 memstore.add(put1, null); 778 memstore.add(put2, null); 779 memstore.add(put3, null); 780 memstore.add(put4, null); 781 782 KeyValue del = new KeyValue(row, fam, null, ts, KeyValue.Type.DeleteFamily, val); 783 memstore.add(del, null); 784 785 List<Cell> expected = new ArrayList<>(); 786 expected.add(del); 787 expected.add(put1); 788 expected.add(put2); 789 expected.add(put4); 790 expected.add(put3); 791 792 assertEquals(5, memstore.getActive().getCellsCount()); 793 int i = 0; 794 for (Cell cell : memstore.getActive().getCellSet()) { 795 assertEquals(expected.get(i++), cell); 796 } 797 } 798 799 @Test 800 public void testKeepDeleteInmemstore() { 801 byte[] row = Bytes.toBytes("testrow"); 802 byte[] fam = Bytes.toBytes("testfamily"); 803 byte[] qf = Bytes.toBytes("testqualifier"); 804 byte[] val = Bytes.toBytes("testval"); 805 long ts = System.nanoTime(); 806 memstore.add(new KeyValue(row, fam, qf, ts, val), null); 807 KeyValue delete = new KeyValue(row, fam, qf, ts, KeyValue.Type.Delete, val); 808 memstore.add(delete, null); 809 assertEquals(2, memstore.getActive().getCellsCount()); 810 assertEquals(delete, memstore.getActive().first()); 811 } 812 813 @Test 814 public void testRetainsDeleteVersion() throws IOException { 815 // add a put to memstore 816 memstore.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care"), null); 817 818 // now process a specific delete: 819 KeyValue delete = 820 KeyValueTestUtil.create("row1", "fam", "a", 100, KeyValue.Type.Delete, "dont-care"); 821 memstore.add(delete, null); 822 823 assertEquals(2, memstore.getActive().getCellsCount()); 824 assertEquals(delete, memstore.getActive().first()); 825 } 826 827 @Test 828 public void testRetainsDeleteColumn() throws IOException { 829 // add a put to memstore 830 memstore.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care"), null); 831 832 // now process a specific delete: 833 KeyValue delete = 834 KeyValueTestUtil.create("row1", "fam", "a", 100, KeyValue.Type.DeleteColumn, "dont-care"); 835 memstore.add(delete, null); 836 837 assertEquals(2, memstore.getActive().getCellsCount()); 838 assertEquals(delete, memstore.getActive().first()); 839 } 840 841 @Test 842 public void testRetainsDeleteFamily() throws IOException { 843 // add a put to memstore 844 memstore.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care"), null); 845 846 // now process a specific delete: 847 KeyValue delete = 848 KeyValueTestUtil.create("row1", "fam", "a", 100, KeyValue.Type.DeleteFamily, "dont-care"); 849 memstore.add(delete, null); 850 851 assertEquals(2, memstore.getActive().getCellsCount()); 852 assertEquals(delete, memstore.getActive().first()); 853 } 854 855 ////////////////////////////////////////////////////////////////////////////// 856 // Helpers 857 ////////////////////////////////////////////////////////////////////////////// 858 private static byte[] makeQualifier(final int i1, final int i2) { 859 return Bytes.toBytes(Integer.toString(i1) + ";" + Integer.toString(i2)); 860 } 861 862 /** 863 * Add keyvalues with a fixed memstoreTs, and checks that memstore size is decreased as older 864 * keyvalues are deleted from the memstore. 865 */ 866 @Test 867 public void testUpsertMemstoreSize() throws Exception { 868 Configuration conf = HBaseConfiguration.create(); 869 memstore = new DefaultMemStore(conf, CellComparatorImpl.COMPARATOR); 870 MemStoreSize oldSize = memstore.size(); 871 872 List<ExtendedCell> l = new ArrayList<>(); 873 KeyValue kv1 = KeyValueTestUtil.create("r", "f", "q", 100, "v"); 874 KeyValue kv2 = KeyValueTestUtil.create("r", "f", "q", 101, "v"); 875 KeyValue kv3 = KeyValueTestUtil.create("r", "f", "q", 102, "v"); 876 877 kv1.setSequenceId(1); 878 kv2.setSequenceId(1); 879 kv3.setSequenceId(1); 880 l.add(kv1); 881 l.add(kv2); 882 l.add(kv3); 883 884 this.memstore.upsert(l, 2, null);// readpoint is 2 885 MemStoreSize newSize = this.memstore.size(); 886 assert (newSize.getDataSize() > oldSize.getDataSize()); 887 // The kv1 should be removed. 888 assert (memstore.getActive().getCellsCount() == 2); 889 890 KeyValue kv4 = KeyValueTestUtil.create("r", "f", "q", 104, "v"); 891 kv4.setSequenceId(1); 892 l.clear(); 893 l.add(kv4); 894 this.memstore.upsert(l, 3, null); 895 assertEquals(newSize, this.memstore.size()); 896 // The kv2 should be removed. 897 assert (memstore.getActive().getCellsCount() == 2); 898 // this.memstore = null; 899 } 900 901 //////////////////////////////////// 902 // Test for periodic memstore flushes 903 // based on time of oldest edit 904 //////////////////////////////////// 905 906 /** 907 * Tests that the timeOfOldestEdit is updated correctly for the various edit operations in 908 * memstore. 909 */ 910 @Test 911 public void testUpdateToTimeOfOldestEdit() throws Exception { 912 try { 913 EnvironmentEdgeForMemstoreTest edge = new EnvironmentEdgeForMemstoreTest(); 914 EnvironmentEdgeManager.injectEdge(edge); 915 DefaultMemStore memstore = new DefaultMemStore(); 916 long t = memstore.timeOfOldestEdit(); 917 assertEquals(Long.MAX_VALUE, t); 918 919 // test the case that the timeOfOldestEdit is updated after a KV add 920 memstore.add(KeyValueTestUtil.create("r", "f", "q", 100, "v"), null); 921 t = memstore.timeOfOldestEdit(); 922 assertTrue(t == 1234); 923 // snapshot() will reset timeOfOldestEdit. The method will also assert the 924 // value is reset to Long.MAX_VALUE 925 t = runSnapshot(memstore); 926 927 // test the case that the timeOfOldestEdit is updated after a KV delete 928 memstore.add(KeyValueTestUtil.create("r", "f", "q", 100, KeyValue.Type.Delete, "v"), null); 929 t = memstore.timeOfOldestEdit(); 930 assertTrue(t == 1234); 931 t = runSnapshot(memstore); 932 933 // test the case that the timeOfOldestEdit is updated after a KV upsert 934 List<ExtendedCell> l = new ArrayList<>(); 935 KeyValue kv1 = KeyValueTestUtil.create("r", "f", "q", 100, "v"); 936 kv1.setSequenceId(100); 937 l.add(kv1); 938 memstore.upsert(l, 1000, null); 939 t = memstore.timeOfOldestEdit(); 940 assertTrue(t == 1234); 941 } finally { 942 EnvironmentEdgeManager.reset(); 943 } 944 } 945 946 /** 947 * Tests the HRegion.shouldFlush method - adds an edit in the memstore and checks that shouldFlush 948 * returns true, and another where it disables the periodic flush functionality and tests whether 949 * shouldFlush returns false. 950 */ 951 @Test 952 public void testShouldFlush() throws Exception { 953 Configuration conf = new Configuration(); 954 conf.setInt(HRegion.MEMSTORE_PERIODIC_FLUSH_INTERVAL, 1000); 955 checkShouldFlush(conf, true); 956 // test disable flush 957 conf.setInt(HRegion.MEMSTORE_PERIODIC_FLUSH_INTERVAL, 0); 958 checkShouldFlush(conf, false); 959 } 960 961 protected void checkShouldFlush(Configuration conf, boolean expected) throws Exception { 962 try { 963 EnvironmentEdgeForMemstoreTest edge = new EnvironmentEdgeForMemstoreTest(); 964 EnvironmentEdgeManager.injectEdge(edge); 965 HBaseTestingUtil hbaseUtility = new HBaseTestingUtil(conf); 966 String cf = "foo"; 967 HRegion region = 968 hbaseUtility.createTestRegion("foobar", ColumnFamilyDescriptorBuilder.of(cf)); 969 970 edge.setCurrentTimeMillis(1234); 971 Put p = new Put(Bytes.toBytes("r")); 972 p.add(KeyValueTestUtil.create("r", cf, "q", 100, "v")); 973 region.put(p); 974 edge.setCurrentTimeMillis(1234 + 100); 975 StringBuilder sb = new StringBuilder(); 976 assertTrue(!region.shouldFlush(sb)); 977 edge.setCurrentTimeMillis(1234 + 10000); 978 assertTrue(region.shouldFlush(sb) == expected); 979 } finally { 980 EnvironmentEdgeManager.reset(); 981 } 982 } 983 984 @Test 985 public void testShouldFlushMeta() throws Exception { 986 // write an edit in the META and ensure the shouldFlush (that the periodic memstore 987 // flusher invokes) returns true after SYSTEM_CACHE_FLUSH_INTERVAL (even though 988 // the MEMSTORE_PERIODIC_FLUSH_INTERVAL is set to a higher value) 989 Configuration conf = new Configuration(); 990 conf.setInt(HRegion.MEMSTORE_PERIODIC_FLUSH_INTERVAL, HRegion.SYSTEM_CACHE_FLUSH_INTERVAL * 10); 991 HBaseTestingUtil hbaseUtility = new HBaseTestingUtil(conf); 992 Path testDir = hbaseUtility.getDataTestDir(); 993 EnvironmentEdgeForMemstoreTest edge = new EnvironmentEdgeForMemstoreTest(); 994 EnvironmentEdgeManager.injectEdge(edge); 995 edge.setCurrentTimeMillis(1234); 996 WALFactory wFactory = new WALFactory(conf, "1234"); 997 TableDescriptors tds = new FSTableDescriptors(conf); 998 FSTableDescriptors.tryUpdateMetaTableDescriptor(conf); 999 HRegion meta = HRegion.createHRegion(RegionInfoBuilder.FIRST_META_REGIONINFO, testDir, conf, 1000 tds.get(TableName.META_TABLE_NAME), wFactory.getWAL(RegionInfoBuilder.FIRST_META_REGIONINFO)); 1001 // parameterized tests add [#] suffix get rid of [ and ]. 1002 TableDescriptor desc = TableDescriptorBuilder 1003 .newBuilder(TableName.valueOf(name.getMethodName().replaceAll("[\\[\\]]", "_"))) 1004 .setColumnFamily(ColumnFamilyDescriptorBuilder.of("foo")).build(); 1005 RegionInfo hri = RegionInfoBuilder.newBuilder(desc.getTableName()) 1006 .setStartKey(Bytes.toBytes("row_0200")).setEndKey(Bytes.toBytes("row_0300")).build(); 1007 HRegion r = HRegion.createHRegion(hri, testDir, conf, desc, wFactory.getWAL(hri)); 1008 addRegionToMETA(meta, r); 1009 edge.setCurrentTimeMillis(1234 + 100); 1010 StringBuilder sb = new StringBuilder(); 1011 assertTrue(meta.shouldFlush(sb) == false); 1012 edge.setCurrentTimeMillis(edge.currentTime() + HRegion.SYSTEM_CACHE_FLUSH_INTERVAL + 1); 1013 assertTrue(meta.shouldFlush(sb) == true); 1014 } 1015 1016 /** 1017 * Inserts a new region's meta information into the passed <code>meta</code> region. 1018 * @param meta hbase:meta HRegion to be updated 1019 * @param r HRegion to add to <code>meta</code> 1020 */ 1021 private static void addRegionToMETA(final HRegion meta, final HRegion r) throws IOException { 1022 // The row key is the region name 1023 byte[] row = r.getRegionInfo().getRegionName(); 1024 final long now = EnvironmentEdgeManager.currentTime(); 1025 final List<Cell> cells = new ArrayList<>(2); 1026 cells.add(new KeyValue(row, HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, now, 1027 RegionInfo.toByteArray(r.getRegionInfo()))); 1028 // Set into the root table the version of the meta table. 1029 cells.add(new KeyValue(row, HConstants.CATALOG_FAMILY, HConstants.META_VERSION_QUALIFIER, now, 1030 Bytes.toBytes(HConstants.META_VERSION))); 1031 NavigableMap<byte[], List<Cell>> familyMap = new TreeMap<>(Bytes.BYTES_COMPARATOR); 1032 familyMap.put(HConstants.CATALOG_FAMILY, cells); 1033 meta.put(new Put(row, HConstants.LATEST_TIMESTAMP, familyMap)); 1034 } 1035 1036 private class EnvironmentEdgeForMemstoreTest implements EnvironmentEdge { 1037 long t = 1234; 1038 1039 @Override 1040 public long currentTime() { 1041 return t; 1042 } 1043 1044 public void setCurrentTimeMillis(long t) { 1045 this.t = t; 1046 } 1047 } 1048 1049 /** 1050 * Adds {@link #ROW_COUNT} rows and {@link #QUALIFIER_COUNT} 1051 * @param hmc Instance to add rows to. 1052 * @return How many rows we added. 1053 */ 1054 protected int addRows(final AbstractMemStore hmc) { 1055 return addRows(hmc, HConstants.LATEST_TIMESTAMP); 1056 } 1057 1058 /** 1059 * Adds {@link #ROW_COUNT} rows and {@link #QUALIFIER_COUNT} 1060 * @param hmc Instance to add rows to. 1061 * @return How many rows we added. 1062 */ 1063 protected int addRows(final MemStore hmc, final long ts) { 1064 for (int i = 0; i < ROW_COUNT; i++) { 1065 long timestamp = 1066 ts == HConstants.LATEST_TIMESTAMP ? EnvironmentEdgeManager.currentTime() : ts; 1067 for (int ii = 0; ii < QUALIFIER_COUNT; ii++) { 1068 byte[] row = Bytes.toBytes(i); 1069 byte[] qf = makeQualifier(i, ii); 1070 hmc.add(new KeyValue(row, FAMILY, qf, timestamp, qf), null); 1071 } 1072 } 1073 return ROW_COUNT; 1074 } 1075 1076 private long runSnapshot(final AbstractMemStore hmc) throws UnexpectedStateException { 1077 // Save off old state. 1078 int oldHistorySize = hmc.getSnapshot().getCellsCount(); 1079 MemStoreSnapshot snapshot = hmc.snapshot(); 1080 // Make some assertions about what just happened. 1081 assertTrue("History size has not increased", 1082 oldHistorySize < hmc.getSnapshot().getCellsCount()); 1083 long t = memstore.timeOfOldestEdit(); 1084 assertTrue("Time of oldest edit is not Long.MAX_VALUE", t == Long.MAX_VALUE); 1085 hmc.clearSnapshot(snapshot.getId()); 1086 return t; 1087 } 1088 1089 private void isExpectedRowWithoutTimestamps(final int rowIndex, List<Cell> kvs) { 1090 int i = 0; 1091 for (Cell kv : kvs) { 1092 byte[] expectedColname = makeQualifier(rowIndex, i++); 1093 assertTrue("Column name", CellUtil.matchingQualifier(kv, expectedColname)); 1094 // Value is column name as bytes. Usually result is 1095 // 100 bytes in size at least. This is the default size 1096 // for BytesWriteable. For comparison, convert bytes to 1097 // String and trim to remove trailing null bytes. 1098 assertTrue("Content", CellUtil.matchingValue(kv, expectedColname)); 1099 } 1100 } 1101 1102 private static void addRows(int count, final MemStore mem) { 1103 long nanos = System.nanoTime(); 1104 1105 for (int i = 0; i < count; i++) { 1106 if (i % 1000 == 0) { 1107 1108 System.out.println(i + " Took for 1k usec: " + (System.nanoTime() - nanos) / 1000); 1109 nanos = System.nanoTime(); 1110 } 1111 long timestamp = System.currentTimeMillis(); 1112 1113 for (int ii = 0; ii < QUALIFIER_COUNT; ii++) { 1114 byte[] row = Bytes.toBytes(i); 1115 byte[] qf = makeQualifier(i, ii); 1116 mem.add(new KeyValue(row, FAMILY, qf, timestamp, qf), null); 1117 } 1118 } 1119 } 1120 1121 static void doScan(MemStore ms, int iteration) throws IOException { 1122 long nanos = System.nanoTime(); 1123 KeyValueScanner s = ms.getScanners(0).get(0); 1124 s.seek(KeyValueUtil.createFirstOnRow(new byte[] {})); 1125 1126 System.out.println(iteration + " create/seek took: " + (System.nanoTime() - nanos) / 1000); 1127 int cnt = 0; 1128 while (s.next() != null) 1129 ++cnt; 1130 1131 System.out 1132 .println(iteration + " took usec: " + (System.nanoTime() - nanos) / 1000 + " for: " + cnt); 1133 1134 } 1135 1136 public static void main(String[] args) throws IOException { 1137 MemStore ms = new DefaultMemStore(); 1138 1139 long n1 = System.nanoTime(); 1140 addRows(25000, ms); 1141 System.out.println("Took for insert: " + (System.nanoTime() - n1) / 1000); 1142 1143 System.out.println("foo"); 1144 1145 for (int i = 0; i < 50; i++) 1146 doScan(ms, i); 1147 } 1148}