001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNotNull;
022import static org.junit.Assert.assertNull;
023import static org.junit.Assert.assertTrue;
024
025import java.io.IOException;
026import java.util.ArrayList;
027import java.util.Arrays;
028import java.util.List;
029import java.util.NavigableMap;
030import java.util.Objects;
031import java.util.TreeMap;
032import java.util.concurrent.atomic.AtomicLong;
033import java.util.concurrent.atomic.AtomicReference;
034import org.apache.hadoop.conf.Configuration;
035import org.apache.hadoop.fs.Path;
036import org.apache.hadoop.hbase.Cell;
037import org.apache.hadoop.hbase.CellComparatorImpl;
038import org.apache.hadoop.hbase.CellUtil;
039import org.apache.hadoop.hbase.ExtendedCell;
040import org.apache.hadoop.hbase.HBaseClassTestRule;
041import org.apache.hadoop.hbase.HBaseConfiguration;
042import org.apache.hadoop.hbase.HBaseTestingUtil;
043import org.apache.hadoop.hbase.HConstants;
044import org.apache.hadoop.hbase.KeepDeletedCells;
045import org.apache.hadoop.hbase.KeyValue;
046import org.apache.hadoop.hbase.KeyValueTestUtil;
047import org.apache.hadoop.hbase.KeyValueUtil;
048import org.apache.hadoop.hbase.TableDescriptors;
049import org.apache.hadoop.hbase.TableName;
050import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
051import org.apache.hadoop.hbase.client.Put;
052import org.apache.hadoop.hbase.client.RegionInfo;
053import org.apache.hadoop.hbase.client.RegionInfoBuilder;
054import org.apache.hadoop.hbase.client.Scan;
055import org.apache.hadoop.hbase.client.TableDescriptor;
056import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
057import org.apache.hadoop.hbase.exceptions.UnexpectedStateException;
058import org.apache.hadoop.hbase.monitoring.ThreadLocalServerSideScanMetrics;
059import org.apache.hadoop.hbase.testclassification.MediumTests;
060import org.apache.hadoop.hbase.testclassification.RegionServerTests;
061import org.apache.hadoop.hbase.util.Bytes;
062import org.apache.hadoop.hbase.util.EnvironmentEdge;
063import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
064import org.apache.hadoop.hbase.util.FSTableDescriptors;
065import org.apache.hadoop.hbase.wal.WALFactory;
066import org.junit.AfterClass;
067import org.junit.Assert;
068import org.junit.Before;
069import org.junit.ClassRule;
070import org.junit.Rule;
071import org.junit.Test;
072import org.junit.experimental.categories.Category;
073import org.junit.rules.TestName;
074import org.slf4j.Logger;
075import org.slf4j.LoggerFactory;
076
077import org.apache.hbase.thirdparty.com.google.common.base.Joiner;
078import org.apache.hbase.thirdparty.com.google.common.collect.Iterables;
079import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
080
081/** memstore test case */
082@Category({ RegionServerTests.class, MediumTests.class })
083public class TestDefaultMemStore {
084
085  @ClassRule
086  public static final HBaseClassTestRule CLASS_RULE =
087    HBaseClassTestRule.forClass(TestDefaultMemStore.class);
088
089  private static final Logger LOG = LoggerFactory.getLogger(TestDefaultMemStore.class);
090  @Rule
091  public TestName name = new TestName();
092  protected AbstractMemStore memstore;
093  protected static final int ROW_COUNT = 10;
094  protected static final int QUALIFIER_COUNT = ROW_COUNT;
095  protected static final byte[] FAMILY = Bytes.toBytes("column");
096  protected MultiVersionConcurrencyControl mvcc;
097  protected AtomicLong startSeqNum = new AtomicLong(0);
098  protected ChunkCreator chunkCreator;
099
100  private String getName() {
101    return this.name.getMethodName();
102  }
103
104  @Before
105  public void setUp() throws Exception {
106    internalSetUp();
107    // no pool
108    this.chunkCreator = ChunkCreator.initialize(MemStoreLAB.CHUNK_SIZE_DEFAULT, false, 0, 0, 0,
109      null, MemStoreLAB.INDEX_CHUNK_SIZE_PERCENTAGE_DEFAULT);
110    this.memstore = new DefaultMemStore();
111  }
112
113  @AfterClass
114  public static void tearDownClass() throws Exception {
115    ChunkCreator.getInstance().clearChunkIds();
116  }
117
118  protected void internalSetUp() throws Exception {
119    this.mvcc = new MultiVersionConcurrencyControl();
120  }
121
122  @Test
123  public void testPutSameKey() {
124    byte[] bytes = Bytes.toBytes(getName());
125    KeyValue kv = new KeyValue(bytes, bytes, bytes, bytes);
126    this.memstore.add(kv, null);
127    byte[] other = Bytes.toBytes("somethingelse");
128    KeyValue samekey = new KeyValue(bytes, bytes, bytes, other);
129    this.memstore.add(samekey, null);
130    Cell found = this.memstore.getActive().first();
131    assertEquals(1, this.memstore.getActive().getCellsCount());
132    assertTrue(Bytes.toString(found.getValueArray()), CellUtil.matchingValue(samekey, found));
133  }
134
135  @Test
136  public void testPutSameCell() {
137    byte[] bytes = Bytes.toBytes(getName());
138    KeyValue kv = new KeyValue(bytes, bytes, bytes, bytes);
139    MemStoreSizing sizeChangeForFirstCell = new NonThreadSafeMemStoreSizing();
140    this.memstore.add(kv, sizeChangeForFirstCell);
141    MemStoreSizing sizeChangeForSecondCell = new NonThreadSafeMemStoreSizing();
142    this.memstore.add(kv, sizeChangeForSecondCell);
143    // make sure memstore size increase won't double-count MSLAB chunk size
144    assertEquals(Segment.getCellLength(kv), sizeChangeForFirstCell.getMemStoreSize().getDataSize());
145    Segment segment = this.memstore.getActive();
146    MemStoreLAB msLab = segment.getMemStoreLAB();
147    if (msLab != null) {
148      // make sure memstore size increased even when writing the same cell, if using MSLAB
149      assertEquals(Segment.getCellLength(kv),
150        sizeChangeForSecondCell.getMemStoreSize().getDataSize());
151      // make sure chunk size increased even when writing the same cell, if using MSLAB
152      if (msLab instanceof MemStoreLABImpl) {
153        // since we add the chunkID at the 0th offset of the chunk and the
154        // chunkid is an int we need to account for those 4 bytes
155        assertEquals(2 * Segment.getCellLength(kv) + Bytes.SIZEOF_INT,
156          ((MemStoreLABImpl) msLab).getCurrentChunk().getNextFreeOffset());
157      }
158    } else {
159      // make sure no memstore size change w/o MSLAB
160      assertEquals(0, sizeChangeForSecondCell.getMemStoreSize().getDataSize());
161      assertEquals(0, sizeChangeForSecondCell.getMemStoreSize().getHeapSize());
162    }
163  }
164
165  /**
166   * Test memstore snapshot happening while scanning.
167   */
168  @Test
169  public void testScanAcrossSnapshot() throws IOException {
170    int rowCount = addRows(this.memstore);
171    List<KeyValueScanner> memstorescanners = this.memstore.getScanners(0);
172    Scan scan = new Scan();
173    List<Cell> result = new ArrayList<>();
174    Configuration conf = HBaseConfiguration.create();
175    ScanInfo scanInfo =
176      new ScanInfo(conf, null, 0, 1, HConstants.LATEST_TIMESTAMP, KeepDeletedCells.FALSE,
177        HConstants.DEFAULT_BLOCKSIZE, 0, this.memstore.getComparator(), false);
178    int count = 0;
179    try (StoreScanner s = new StoreScanner(scan, scanInfo, null, memstorescanners)) {
180      while (s.next(result)) {
181        LOG.info(Objects.toString(result));
182        count++;
183        // Row count is same as column count.
184        assertEquals(rowCount, result.size());
185        result.clear();
186      }
187    }
188    assertEquals(rowCount, count);
189    for (KeyValueScanner scanner : memstorescanners) {
190      scanner.close();
191    }
192
193    memstorescanners = this.memstore.getScanners(mvcc.getReadPoint());
194    // Now assert can count same number even if a snapshot mid-scan.
195    count = 0;
196    try (StoreScanner s = new StoreScanner(scan, scanInfo, null, memstorescanners)) {
197      while (s.next(result)) {
198        LOG.info(Objects.toString(result));
199        // Assert the stuff is coming out in right order.
200        assertTrue(CellUtil.matchingRows(result.get(0), Bytes.toBytes(count)));
201        count++;
202        // Row count is same as column count.
203        assertEquals(rowCount, result.size());
204        if (count == 2) {
205          this.memstore.snapshot();
206          LOG.info("Snapshotted");
207        }
208        result.clear();
209      }
210    }
211    assertEquals(rowCount, count);
212    for (KeyValueScanner scanner : memstorescanners) {
213      scanner.close();
214    }
215    memstorescanners = this.memstore.getScanners(mvcc.getReadPoint());
216    // Assert that new values are seen in kvset as we scan.
217    long ts = EnvironmentEdgeManager.currentTime();
218    count = 0;
219    int snapshotIndex = 5;
220    try (StoreScanner s = new StoreScanner(scan, scanInfo, null, memstorescanners)) {
221      while (s.next(result)) {
222        LOG.info(Objects.toString(result));
223        // Assert the stuff is coming out in right order.
224        assertTrue(CellUtil.matchingRows(result.get(0), Bytes.toBytes(count)));
225        // Row count is same as column count.
226        assertEquals("count=" + count + ", result=" + result, rowCount, result.size());
227        count++;
228        if (count == snapshotIndex) {
229          MemStoreSnapshot snapshot = this.memstore.snapshot();
230          this.memstore.clearSnapshot(snapshot.getId());
231          // Added more rows into kvset. But the scanner wont see these rows.
232          addRows(this.memstore, ts);
233          LOG.info("Snapshotted, cleared it and then added values (which wont be seen)");
234        }
235        result.clear();
236      }
237    }
238    assertEquals(rowCount, count);
239  }
240
241  /**
242   * A simple test which verifies the 3 possible states when scanning across snapshot.
243   */
244  @Test
245  public void testScanAcrossSnapshot2() throws IOException, CloneNotSupportedException {
246    // we are going to the scanning across snapshot with two kvs
247    // kv1 should always be returned before kv2
248    final byte[] one = Bytes.toBytes(1);
249    final byte[] two = Bytes.toBytes(2);
250    final byte[] f = Bytes.toBytes("f");
251    final byte[] q = Bytes.toBytes("q");
252    final byte[] v = Bytes.toBytes(3);
253
254    final KeyValue kv1 = new KeyValue(one, f, q, v);
255    final KeyValue kv2 = new KeyValue(two, f, q, v);
256
257    // use case 1: both kvs in kvset
258    this.memstore.add(kv1.clone(), null);
259    this.memstore.add(kv2.clone(), null);
260    // snapshot is empty,active segment is not empty,
261    // empty segment is skipped.
262    verifyOneScanAcrossSnapshot2(kv1, kv2);
263
264    // use case 2: both kvs in snapshot
265    // active segment is empty,snapshot is not empty,
266    // empty segment is skipped.
267    this.memstore.snapshot();
268    //
269    verifyOneScanAcrossSnapshot2(kv1, kv2);
270
271    // use case 3: first in snapshot second in kvset
272    this.memstore = new DefaultMemStore();
273    this.memstore.add(kv1.clone(), null);
274    this.memstore.snapshot();
275    this.memstore.add(kv2.clone(), null);
276    verifyScanAcrossSnapshot2(kv1, kv2);
277  }
278
279  protected void verifyScanAcrossSnapshot2(KeyValue kv1, KeyValue kv2) throws IOException {
280    List<KeyValueScanner> memstorescanners = this.memstore.getScanners(mvcc.getReadPoint());
281    assertEquals(2, memstorescanners.size());
282    final KeyValueScanner scanner0 = memstorescanners.get(0);
283    final KeyValueScanner scanner1 = memstorescanners.get(1);
284    scanner0.seek(KeyValueUtil.createFirstOnRow(HConstants.EMPTY_START_ROW));
285    scanner1.seek(KeyValueUtil.createFirstOnRow(HConstants.EMPTY_START_ROW));
286    Cell n0 = scanner0.next();
287    Cell n1 = scanner1.next();
288    assertTrue(kv1.equals(n0) || kv1.equals(n1));
289    assertTrue(kv2.equals(n0) || kv2.equals(n1) || kv2.equals(scanner0.next())
290      || kv2.equals(scanner1.next()));
291    assertNull(scanner0.next());
292    assertNull(scanner1.next());
293  }
294
295  protected void verifyOneScanAcrossSnapshot2(KeyValue kv1, KeyValue kv2) throws IOException {
296    List<KeyValueScanner> memstorescanners = this.memstore.getScanners(mvcc.getReadPoint());
297    assertEquals(1, memstorescanners.size());
298    final KeyValueScanner scanner0 = memstorescanners.get(0);
299    scanner0.seek(KeyValueUtil.createFirstOnRow(HConstants.EMPTY_START_ROW));
300    Cell n0 = scanner0.next();
301    Cell n1 = scanner0.next();
302    assertTrue(kv1.equals(n0));
303    assertTrue(kv2.equals(n1));
304    assertNull(scanner0.next());
305  }
306
307  protected void assertScannerResults(KeyValueScanner scanner, KeyValue[] expected)
308    throws IOException {
309    scanner.seek(KeyValueUtil.createFirstOnRow(new byte[] {}));
310    List<Cell> returned = Lists.newArrayList();
311
312    while (true) {
313      Cell next = scanner.next();
314      if (next == null) break;
315      returned.add(next);
316    }
317
318    assertTrue(
319      "Got:\n" + Joiner.on("\n").join(returned) + "\nExpected:\n" + Joiner.on("\n").join(expected),
320      Iterables.elementsEqual(Arrays.asList(expected), returned));
321    assertNull(scanner.peek());
322  }
323
324  @Test
325  public void testMemstoreConcurrentControl() throws IOException {
326    final byte[] row = Bytes.toBytes(1);
327    final byte[] f = Bytes.toBytes("family");
328    final byte[] q1 = Bytes.toBytes("q1");
329    final byte[] q2 = Bytes.toBytes("q2");
330    final byte[] v = Bytes.toBytes("value");
331
332    MultiVersionConcurrencyControl.WriteEntry w = mvcc.begin();
333
334    KeyValue kv1 = new KeyValue(row, f, q1, v);
335    kv1.setSequenceId(w.getWriteNumber());
336    memstore.add(kv1, null);
337
338    KeyValueScanner s = this.memstore.getScanners(mvcc.getReadPoint()).get(0);
339    assertScannerResults(s, new KeyValue[] {});
340
341    mvcc.completeAndWait(w);
342
343    s = this.memstore.getScanners(mvcc.getReadPoint()).get(0);
344    assertScannerResults(s, new KeyValue[] { kv1 });
345
346    w = mvcc.begin();
347    KeyValue kv2 = new KeyValue(row, f, q2, v);
348    kv2.setSequenceId(w.getWriteNumber());
349    memstore.add(kv2, null);
350
351    s = this.memstore.getScanners(mvcc.getReadPoint()).get(0);
352    assertScannerResults(s, new KeyValue[] { kv1 });
353
354    mvcc.completeAndWait(w);
355
356    s = this.memstore.getScanners(mvcc.getReadPoint()).get(0);
357    assertScannerResults(s, new KeyValue[] { kv1, kv2 });
358  }
359
360  private long getBytesReadFromMemstore() throws IOException {
361    final byte[] f = Bytes.toBytes("family");
362    final byte[] q1 = Bytes.toBytes("q1");
363    final byte[] v = Bytes.toBytes("value");
364    int numKvs = 10;
365
366    MultiVersionConcurrencyControl.WriteEntry w = mvcc.begin();
367
368    KeyValue kv;
369    KeyValue[] kvs = new KeyValue[numKvs];
370    long totalCellSize = 0;
371    for (int i = 0; i < numKvs; i++) {
372      byte[] row = Bytes.toBytes(i);
373      kv = new KeyValue(row, f, q1, v);
374      kv.setSequenceId(w.getWriteNumber());
375      memstore.add(kv, null);
376      kvs[i] = kv;
377      totalCellSize += Segment.getCellLength(kv);
378    }
379    mvcc.completeAndWait(w);
380
381    ThreadLocalServerSideScanMetrics.getBytesReadFromMemstoreAndReset();
382    KeyValueScanner s = this.memstore.getScanners(mvcc.getReadPoint()).get(0);
383    assertScannerResults(s, kvs);
384    return totalCellSize;
385  }
386
387  @Test
388  public void testBytesReadFromMemstore() throws IOException {
389    ThreadLocalServerSideScanMetrics.setScanMetricsEnabled(true);
390    long totalCellSize = getBytesReadFromMemstore();
391    Assert.assertEquals(totalCellSize,
392      ThreadLocalServerSideScanMetrics.getBytesReadFromMemstoreAndReset());
393  }
394
395  @Test
396  public void testBytesReadFromMemstoreWithScanMetricsDisabled() throws IOException {
397    ThreadLocalServerSideScanMetrics.setScanMetricsEnabled(false);
398    getBytesReadFromMemstore();
399    Assert.assertEquals(0, ThreadLocalServerSideScanMetrics.getBytesReadFromMemstoreAndReset());
400  }
401
402  /**
403   * Regression test for HBASE-2616, HBASE-2670. When we insert a higher-memstoreTS version of a
404   * cell but with the same timestamp, we still need to provide consistent reads for the same
405   * scanner.
406   */
407  @Test
408  public void testMemstoreEditsVisibilityWithSameKey() throws IOException {
409    final byte[] row = Bytes.toBytes(1);
410    final byte[] f = Bytes.toBytes("family");
411    final byte[] q1 = Bytes.toBytes("q1");
412    final byte[] q2 = Bytes.toBytes("q2");
413    final byte[] v1 = Bytes.toBytes("value1");
414    final byte[] v2 = Bytes.toBytes("value2");
415
416    // INSERT 1: Write both columns val1
417    MultiVersionConcurrencyControl.WriteEntry w = mvcc.begin();
418
419    KeyValue kv11 = new KeyValue(row, f, q1, v1);
420    kv11.setSequenceId(w.getWriteNumber());
421    memstore.add(kv11, null);
422
423    KeyValue kv12 = new KeyValue(row, f, q2, v1);
424    kv12.setSequenceId(w.getWriteNumber());
425    memstore.add(kv12, null);
426    mvcc.completeAndWait(w);
427
428    // BEFORE STARTING INSERT 2, SEE FIRST KVS
429    KeyValueScanner s = this.memstore.getScanners(mvcc.getReadPoint()).get(0);
430    assertScannerResults(s, new KeyValue[] { kv11, kv12 });
431
432    // START INSERT 2: Write both columns val2
433    w = mvcc.begin();
434    KeyValue kv21 = new KeyValue(row, f, q1, v2);
435    kv21.setSequenceId(w.getWriteNumber());
436    memstore.add(kv21, null);
437
438    KeyValue kv22 = new KeyValue(row, f, q2, v2);
439    kv22.setSequenceId(w.getWriteNumber());
440    memstore.add(kv22, null);
441
442    // BEFORE COMPLETING INSERT 2, SEE FIRST KVS
443    s = this.memstore.getScanners(mvcc.getReadPoint()).get(0);
444    assertScannerResults(s, new KeyValue[] { kv11, kv12 });
445
446    // COMPLETE INSERT 2
447    mvcc.completeAndWait(w);
448
449    // NOW SHOULD SEE NEW KVS IN ADDITION TO OLD KVS.
450    // See HBASE-1485 for discussion about what we should do with
451    // the duplicate-TS inserts
452    s = this.memstore.getScanners(mvcc.getReadPoint()).get(0);
453    assertScannerResults(s, new KeyValue[] { kv21, kv11, kv22, kv12 });
454  }
455
456  /**
457   * When we insert a higher-memstoreTS deletion of a cell but with the same timestamp, we still
458   * need to provide consistent reads for the same scanner.
459   */
460  @Test
461  public void testMemstoreDeletesVisibilityWithSameKey() throws IOException {
462    final byte[] row = Bytes.toBytes(1);
463    final byte[] f = Bytes.toBytes("family");
464    final byte[] q1 = Bytes.toBytes("q1");
465    final byte[] q2 = Bytes.toBytes("q2");
466    final byte[] v1 = Bytes.toBytes("value1");
467    // INSERT 1: Write both columns val1
468    MultiVersionConcurrencyControl.WriteEntry w = mvcc.begin();
469
470    KeyValue kv11 = new KeyValue(row, f, q1, v1);
471    kv11.setSequenceId(w.getWriteNumber());
472    memstore.add(kv11, null);
473
474    KeyValue kv12 = new KeyValue(row, f, q2, v1);
475    kv12.setSequenceId(w.getWriteNumber());
476    memstore.add(kv12, null);
477    mvcc.completeAndWait(w);
478
479    // BEFORE STARTING INSERT 2, SEE FIRST KVS
480    KeyValueScanner s = this.memstore.getScanners(mvcc.getReadPoint()).get(0);
481    assertScannerResults(s, new KeyValue[] { kv11, kv12 });
482
483    // START DELETE: Insert delete for one of the columns
484    w = mvcc.begin();
485    KeyValue kvDel = new KeyValue(row, f, q2, kv11.getTimestamp(), KeyValue.Type.DeleteColumn);
486    kvDel.setSequenceId(w.getWriteNumber());
487    memstore.add(kvDel, null);
488
489    // BEFORE COMPLETING DELETE, SEE FIRST KVS
490    s = this.memstore.getScanners(mvcc.getReadPoint()).get(0);
491    assertScannerResults(s, new KeyValue[] { kv11, kv12 });
492
493    // COMPLETE DELETE
494    mvcc.completeAndWait(w);
495
496    // NOW WE SHOULD SEE DELETE
497    s = this.memstore.getScanners(mvcc.getReadPoint()).get(0);
498    assertScannerResults(s, new KeyValue[] { kv11, kvDel, kv12 });
499  }
500
501  private static class ReadOwnWritesTester extends Thread {
502    static final int NUM_TRIES = 1000;
503
504    final byte[] row;
505
506    final byte[] f = Bytes.toBytes("family");
507    final byte[] q1 = Bytes.toBytes("q1");
508
509    final MultiVersionConcurrencyControl mvcc;
510    final MemStore memstore;
511
512    AtomicReference<Throwable> caughtException;
513
514    public ReadOwnWritesTester(int id, MemStore memstore, MultiVersionConcurrencyControl mvcc,
515      AtomicReference<Throwable> caughtException) {
516      this.mvcc = mvcc;
517      this.memstore = memstore;
518      this.caughtException = caughtException;
519      row = Bytes.toBytes(id);
520    }
521
522    @Override
523    public void run() {
524      try {
525        internalRun();
526      } catch (Throwable t) {
527        caughtException.compareAndSet(null, t);
528      }
529    }
530
531    private void internalRun() throws IOException {
532      for (long i = 0; i < NUM_TRIES && caughtException.get() == null; i++) {
533        MultiVersionConcurrencyControl.WriteEntry w = mvcc.begin();
534
535        // Insert the sequence value (i)
536        byte[] v = Bytes.toBytes(i);
537
538        KeyValue kv = new KeyValue(row, f, q1, i, v);
539        kv.setSequenceId(w.getWriteNumber());
540        memstore.add(kv, null);
541        mvcc.completeAndWait(w);
542
543        // Assert that we can read back
544        KeyValueScanner s = this.memstore.getScanners(mvcc.getReadPoint()).get(0);
545        s.seek(kv);
546
547        Cell ret = s.next();
548        assertNotNull("Didnt find own write at all", ret);
549        assertEquals("Didnt read own writes", kv.getTimestamp(), ret.getTimestamp());
550      }
551    }
552  }
553
554  @Test
555  public void testReadOwnWritesUnderConcurrency() throws Throwable {
556    int NUM_THREADS = 8;
557
558    ReadOwnWritesTester threads[] = new ReadOwnWritesTester[NUM_THREADS];
559    AtomicReference<Throwable> caught = new AtomicReference<>();
560
561    for (int i = 0; i < NUM_THREADS; i++) {
562      threads[i] = new ReadOwnWritesTester(i, memstore, mvcc, caught);
563      threads[i].start();
564    }
565
566    for (int i = 0; i < NUM_THREADS; i++) {
567      threads[i].join();
568    }
569
570    if (caught.get() != null) {
571      throw caught.get();
572    }
573  }
574
575  /**
576   * Test memstore snapshots
577   */
578  @Test
579  public void testSnapshotting() throws IOException {
580    final int snapshotCount = 5;
581    // Add some rows, run a snapshot. Do it a few times.
582    for (int i = 0; i < snapshotCount; i++) {
583      addRows(this.memstore);
584      runSnapshot(this.memstore);
585      assertEquals("History not being cleared", 0, this.memstore.getSnapshot().getCellsCount());
586    }
587  }
588
589  @Test
590  public void testMultipleVersionsSimple() throws Exception {
591    DefaultMemStore m = new DefaultMemStore(new Configuration(), CellComparatorImpl.COMPARATOR);
592    byte[] row = Bytes.toBytes("testRow");
593    byte[] family = Bytes.toBytes("testFamily");
594    byte[] qf = Bytes.toBytes("testQualifier");
595    long[] stamps = { 1, 2, 3 };
596    byte[][] values = { Bytes.toBytes("value0"), Bytes.toBytes("value1"), Bytes.toBytes("value2") };
597    KeyValue key0 = new KeyValue(row, family, qf, stamps[0], values[0]);
598    KeyValue key1 = new KeyValue(row, family, qf, stamps[1], values[1]);
599    KeyValue key2 = new KeyValue(row, family, qf, stamps[2], values[2]);
600
601    m.add(key0, null);
602    m.add(key1, null);
603    m.add(key2, null);
604
605    assertTrue("Expected memstore to hold 3 values, actually has " + m.getActive().getCellsCount(),
606      m.getActive().getCellsCount() == 3);
607  }
608
609  //////////////////////////////////////////////////////////////////////////////
610  // Get tests
611  //////////////////////////////////////////////////////////////////////////////
612
613  /**
614   * Test getNextRow from memstore
615   */
616  @Test
617  public void testGetNextRow() throws Exception {
618    addRows(this.memstore);
619    // Add more versions to make it a little more interesting.
620    Thread.sleep(1);
621    addRows(this.memstore);
622    Cell closestToEmpty = ((DefaultMemStore) this.memstore).getNextRow(KeyValue.LOWESTKEY);
623    assertTrue(CellComparatorImpl.COMPARATOR.compareRows(closestToEmpty,
624      new KeyValue(Bytes.toBytes(0), EnvironmentEdgeManager.currentTime())) == 0);
625    for (int i = 0; i < ROW_COUNT; i++) {
626      Cell nr = ((DefaultMemStore) this.memstore)
627        .getNextRow(new KeyValue(Bytes.toBytes(i), EnvironmentEdgeManager.currentTime()));
628      if (i + 1 == ROW_COUNT) {
629        assertNull(nr);
630      } else {
631        assertTrue(CellComparatorImpl.COMPARATOR.compareRows(nr,
632          new KeyValue(Bytes.toBytes(i + 1), EnvironmentEdgeManager.currentTime())) == 0);
633      }
634    }
635    // starting from each row, validate results should contain the starting row
636    Configuration conf = HBaseConfiguration.create();
637    for (int startRowId = 0; startRowId < ROW_COUNT; startRowId++) {
638      ScanInfo scanInfo =
639        new ScanInfo(conf, FAMILY, 0, 1, Integer.MAX_VALUE, KeepDeletedCells.FALSE,
640          HConstants.DEFAULT_BLOCKSIZE, 0, this.memstore.getComparator(), false);
641      try (InternalScanner scanner =
642        new StoreScanner(new Scan().withStartRow(Bytes.toBytes(startRowId)), scanInfo, null,
643          memstore.getScanners(0))) {
644        List<Cell> results = new ArrayList<>();
645        for (int i = 0; scanner.next(results); i++) {
646          int rowId = startRowId + i;
647          Cell left = results.get(0);
648          byte[] row1 = Bytes.toBytes(rowId);
649          assertTrue("Row name",
650            CellComparatorImpl.COMPARATOR.compareRows(left, row1, 0, row1.length) == 0);
651          assertEquals("Count of columns", QUALIFIER_COUNT, results.size());
652          List<Cell> row = new ArrayList<>();
653          for (Cell kv : results) {
654            row.add(kv);
655          }
656          isExpectedRowWithoutTimestamps(rowId, row);
657          // Clear out set. Otherwise row results accumulate.
658          results.clear();
659        }
660      }
661    }
662  }
663
664  @Test
665  public void testGet_memstoreAndSnapShot() throws IOException {
666    byte[] row = Bytes.toBytes("testrow");
667    byte[] fam = Bytes.toBytes("testfamily");
668    byte[] qf1 = Bytes.toBytes("testqualifier1");
669    byte[] qf2 = Bytes.toBytes("testqualifier2");
670    byte[] qf3 = Bytes.toBytes("testqualifier3");
671    byte[] qf4 = Bytes.toBytes("testqualifier4");
672    byte[] qf5 = Bytes.toBytes("testqualifier5");
673    byte[] val = Bytes.toBytes("testval");
674
675    // Setting up memstore
676    memstore.add(new KeyValue(row, fam, qf1, val), null);
677    memstore.add(new KeyValue(row, fam, qf2, val), null);
678    memstore.add(new KeyValue(row, fam, qf3, val), null);
679    // Creating a snapshot
680    memstore.snapshot();
681    assertEquals(3, memstore.getSnapshot().getCellsCount());
682    // Adding value to "new" memstore
683    assertEquals(0, memstore.getActive().getCellsCount());
684    memstore.add(new KeyValue(row, fam, qf4, val), null);
685    memstore.add(new KeyValue(row, fam, qf5, val), null);
686    assertEquals(2, memstore.getActive().getCellsCount());
687  }
688
689  //////////////////////////////////////////////////////////////////////////////
690  // Delete tests
691  //////////////////////////////////////////////////////////////////////////////
692  @Test
693  public void testGetWithDelete() throws IOException {
694    byte[] row = Bytes.toBytes("testrow");
695    byte[] fam = Bytes.toBytes("testfamily");
696    byte[] qf1 = Bytes.toBytes("testqualifier");
697    byte[] val = Bytes.toBytes("testval");
698
699    long ts1 = System.nanoTime();
700    KeyValue put1 = new KeyValue(row, fam, qf1, ts1, val);
701    long ts2 = ts1 + 1;
702    KeyValue put2 = new KeyValue(row, fam, qf1, ts2, val);
703    long ts3 = ts2 + 1;
704    KeyValue put3 = new KeyValue(row, fam, qf1, ts3, val);
705    memstore.add(put1, null);
706    memstore.add(put2, null);
707    memstore.add(put3, null);
708
709    assertEquals(3, memstore.getActive().getCellsCount());
710
711    KeyValue del2 = new KeyValue(row, fam, qf1, ts2, KeyValue.Type.Delete, val);
712    memstore.add(del2, null);
713
714    List<Cell> expected = new ArrayList<>();
715    expected.add(put3);
716    expected.add(del2);
717    expected.add(put2);
718    expected.add(put1);
719
720    assertEquals(4, memstore.getActive().getCellsCount());
721    int i = 0;
722    for (Cell cell : memstore.getActive().getCellSet()) {
723      assertEquals(expected.get(i++), cell);
724    }
725  }
726
727  @Test
728  public void testGetWithDeleteColumn() throws IOException {
729    byte[] row = Bytes.toBytes("testrow");
730    byte[] fam = Bytes.toBytes("testfamily");
731    byte[] qf1 = Bytes.toBytes("testqualifier");
732    byte[] val = Bytes.toBytes("testval");
733
734    long ts1 = System.nanoTime();
735    KeyValue put1 = new KeyValue(row, fam, qf1, ts1, val);
736    long ts2 = ts1 + 1;
737    KeyValue put2 = new KeyValue(row, fam, qf1, ts2, val);
738    long ts3 = ts2 + 1;
739    KeyValue put3 = new KeyValue(row, fam, qf1, ts3, val);
740    memstore.add(put1, null);
741    memstore.add(put2, null);
742    memstore.add(put3, null);
743
744    assertEquals(3, memstore.getActive().getCellsCount());
745
746    KeyValue del2 = new KeyValue(row, fam, qf1, ts2, KeyValue.Type.DeleteColumn, val);
747    memstore.add(del2, null);
748
749    List<Cell> expected = new ArrayList<>();
750    expected.add(put3);
751    expected.add(del2);
752    expected.add(put2);
753    expected.add(put1);
754
755    assertEquals(4, memstore.getActive().getCellsCount());
756    int i = 0;
757    for (Cell cell : memstore.getActive().getCellSet()) {
758      assertEquals(expected.get(i++), cell);
759    }
760  }
761
762  @Test
763  public void testGetWithDeleteFamily() throws IOException {
764    byte[] row = Bytes.toBytes("testrow");
765    byte[] fam = Bytes.toBytes("testfamily");
766    byte[] qf1 = Bytes.toBytes("testqualifier1");
767    byte[] qf2 = Bytes.toBytes("testqualifier2");
768    byte[] qf3 = Bytes.toBytes("testqualifier3");
769    byte[] val = Bytes.toBytes("testval");
770    long ts = System.nanoTime();
771
772    KeyValue put1 = new KeyValue(row, fam, qf1, ts, val);
773    KeyValue put2 = new KeyValue(row, fam, qf2, ts, val);
774    KeyValue put3 = new KeyValue(row, fam, qf3, ts, val);
775    KeyValue put4 = new KeyValue(row, fam, qf3, ts + 1, val);
776
777    memstore.add(put1, null);
778    memstore.add(put2, null);
779    memstore.add(put3, null);
780    memstore.add(put4, null);
781
782    KeyValue del = new KeyValue(row, fam, null, ts, KeyValue.Type.DeleteFamily, val);
783    memstore.add(del, null);
784
785    List<Cell> expected = new ArrayList<>();
786    expected.add(del);
787    expected.add(put1);
788    expected.add(put2);
789    expected.add(put4);
790    expected.add(put3);
791
792    assertEquals(5, memstore.getActive().getCellsCount());
793    int i = 0;
794    for (Cell cell : memstore.getActive().getCellSet()) {
795      assertEquals(expected.get(i++), cell);
796    }
797  }
798
799  @Test
800  public void testKeepDeleteInmemstore() {
801    byte[] row = Bytes.toBytes("testrow");
802    byte[] fam = Bytes.toBytes("testfamily");
803    byte[] qf = Bytes.toBytes("testqualifier");
804    byte[] val = Bytes.toBytes("testval");
805    long ts = System.nanoTime();
806    memstore.add(new KeyValue(row, fam, qf, ts, val), null);
807    KeyValue delete = new KeyValue(row, fam, qf, ts, KeyValue.Type.Delete, val);
808    memstore.add(delete, null);
809    assertEquals(2, memstore.getActive().getCellsCount());
810    assertEquals(delete, memstore.getActive().first());
811  }
812
813  @Test
814  public void testRetainsDeleteVersion() throws IOException {
815    // add a put to memstore
816    memstore.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care"), null);
817
818    // now process a specific delete:
819    KeyValue delete =
820      KeyValueTestUtil.create("row1", "fam", "a", 100, KeyValue.Type.Delete, "dont-care");
821    memstore.add(delete, null);
822
823    assertEquals(2, memstore.getActive().getCellsCount());
824    assertEquals(delete, memstore.getActive().first());
825  }
826
827  @Test
828  public void testRetainsDeleteColumn() throws IOException {
829    // add a put to memstore
830    memstore.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care"), null);
831
832    // now process a specific delete:
833    KeyValue delete =
834      KeyValueTestUtil.create("row1", "fam", "a", 100, KeyValue.Type.DeleteColumn, "dont-care");
835    memstore.add(delete, null);
836
837    assertEquals(2, memstore.getActive().getCellsCount());
838    assertEquals(delete, memstore.getActive().first());
839  }
840
841  @Test
842  public void testRetainsDeleteFamily() throws IOException {
843    // add a put to memstore
844    memstore.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care"), null);
845
846    // now process a specific delete:
847    KeyValue delete =
848      KeyValueTestUtil.create("row1", "fam", "a", 100, KeyValue.Type.DeleteFamily, "dont-care");
849    memstore.add(delete, null);
850
851    assertEquals(2, memstore.getActive().getCellsCount());
852    assertEquals(delete, memstore.getActive().first());
853  }
854
855  //////////////////////////////////////////////////////////////////////////////
856  // Helpers
857  //////////////////////////////////////////////////////////////////////////////
858  private static byte[] makeQualifier(final int i1, final int i2) {
859    return Bytes.toBytes(Integer.toString(i1) + ";" + Integer.toString(i2));
860  }
861
862  /**
863   * Add keyvalues with a fixed memstoreTs, and checks that memstore size is decreased as older
864   * keyvalues are deleted from the memstore.
865   */
866  @Test
867  public void testUpsertMemstoreSize() throws Exception {
868    Configuration conf = HBaseConfiguration.create();
869    memstore = new DefaultMemStore(conf, CellComparatorImpl.COMPARATOR);
870    MemStoreSize oldSize = memstore.size();
871
872    List<ExtendedCell> l = new ArrayList<>();
873    KeyValue kv1 = KeyValueTestUtil.create("r", "f", "q", 100, "v");
874    KeyValue kv2 = KeyValueTestUtil.create("r", "f", "q", 101, "v");
875    KeyValue kv3 = KeyValueTestUtil.create("r", "f", "q", 102, "v");
876
877    kv1.setSequenceId(1);
878    kv2.setSequenceId(1);
879    kv3.setSequenceId(1);
880    l.add(kv1);
881    l.add(kv2);
882    l.add(kv3);
883
884    this.memstore.upsert(l, 2, null);// readpoint is 2
885    MemStoreSize newSize = this.memstore.size();
886    assert (newSize.getDataSize() > oldSize.getDataSize());
887    // The kv1 should be removed.
888    assert (memstore.getActive().getCellsCount() == 2);
889
890    KeyValue kv4 = KeyValueTestUtil.create("r", "f", "q", 104, "v");
891    kv4.setSequenceId(1);
892    l.clear();
893    l.add(kv4);
894    this.memstore.upsert(l, 3, null);
895    assertEquals(newSize, this.memstore.size());
896    // The kv2 should be removed.
897    assert (memstore.getActive().getCellsCount() == 2);
898    // this.memstore = null;
899  }
900
901  ////////////////////////////////////
902  // Test for periodic memstore flushes
903  // based on time of oldest edit
904  ////////////////////////////////////
905
906  /**
907   * Tests that the timeOfOldestEdit is updated correctly for the various edit operations in
908   * memstore.
909   */
910  @Test
911  public void testUpdateToTimeOfOldestEdit() throws Exception {
912    try {
913      EnvironmentEdgeForMemstoreTest edge = new EnvironmentEdgeForMemstoreTest();
914      EnvironmentEdgeManager.injectEdge(edge);
915      DefaultMemStore memstore = new DefaultMemStore();
916      long t = memstore.timeOfOldestEdit();
917      assertEquals(Long.MAX_VALUE, t);
918
919      // test the case that the timeOfOldestEdit is updated after a KV add
920      memstore.add(KeyValueTestUtil.create("r", "f", "q", 100, "v"), null);
921      t = memstore.timeOfOldestEdit();
922      assertTrue(t == 1234);
923      // snapshot() will reset timeOfOldestEdit. The method will also assert the
924      // value is reset to Long.MAX_VALUE
925      t = runSnapshot(memstore);
926
927      // test the case that the timeOfOldestEdit is updated after a KV delete
928      memstore.add(KeyValueTestUtil.create("r", "f", "q", 100, KeyValue.Type.Delete, "v"), null);
929      t = memstore.timeOfOldestEdit();
930      assertTrue(t == 1234);
931      t = runSnapshot(memstore);
932
933      // test the case that the timeOfOldestEdit is updated after a KV upsert
934      List<ExtendedCell> l = new ArrayList<>();
935      KeyValue kv1 = KeyValueTestUtil.create("r", "f", "q", 100, "v");
936      kv1.setSequenceId(100);
937      l.add(kv1);
938      memstore.upsert(l, 1000, null);
939      t = memstore.timeOfOldestEdit();
940      assertTrue(t == 1234);
941    } finally {
942      EnvironmentEdgeManager.reset();
943    }
944  }
945
946  /**
947   * Tests the HRegion.shouldFlush method - adds an edit in the memstore and checks that shouldFlush
948   * returns true, and another where it disables the periodic flush functionality and tests whether
949   * shouldFlush returns false.
950   */
951  @Test
952  public void testShouldFlush() throws Exception {
953    Configuration conf = new Configuration();
954    conf.setInt(HRegion.MEMSTORE_PERIODIC_FLUSH_INTERVAL, 1000);
955    checkShouldFlush(conf, true);
956    // test disable flush
957    conf.setInt(HRegion.MEMSTORE_PERIODIC_FLUSH_INTERVAL, 0);
958    checkShouldFlush(conf, false);
959  }
960
961  protected void checkShouldFlush(Configuration conf, boolean expected) throws Exception {
962    try {
963      EnvironmentEdgeForMemstoreTest edge = new EnvironmentEdgeForMemstoreTest();
964      EnvironmentEdgeManager.injectEdge(edge);
965      HBaseTestingUtil hbaseUtility = new HBaseTestingUtil(conf);
966      String cf = "foo";
967      HRegion region =
968        hbaseUtility.createTestRegion("foobar", ColumnFamilyDescriptorBuilder.of(cf));
969
970      edge.setCurrentTimeMillis(1234);
971      Put p = new Put(Bytes.toBytes("r"));
972      p.add(KeyValueTestUtil.create("r", cf, "q", 100, "v"));
973      region.put(p);
974      edge.setCurrentTimeMillis(1234 + 100);
975      StringBuilder sb = new StringBuilder();
976      assertTrue(!region.shouldFlush(sb));
977      edge.setCurrentTimeMillis(1234 + 10000);
978      assertTrue(region.shouldFlush(sb) == expected);
979    } finally {
980      EnvironmentEdgeManager.reset();
981    }
982  }
983
984  @Test
985  public void testShouldFlushMeta() throws Exception {
986    // write an edit in the META and ensure the shouldFlush (that the periodic memstore
987    // flusher invokes) returns true after SYSTEM_CACHE_FLUSH_INTERVAL (even though
988    // the MEMSTORE_PERIODIC_FLUSH_INTERVAL is set to a higher value)
989    Configuration conf = new Configuration();
990    conf.setInt(HRegion.MEMSTORE_PERIODIC_FLUSH_INTERVAL, HRegion.SYSTEM_CACHE_FLUSH_INTERVAL * 10);
991    HBaseTestingUtil hbaseUtility = new HBaseTestingUtil(conf);
992    Path testDir = hbaseUtility.getDataTestDir();
993    EnvironmentEdgeForMemstoreTest edge = new EnvironmentEdgeForMemstoreTest();
994    EnvironmentEdgeManager.injectEdge(edge);
995    edge.setCurrentTimeMillis(1234);
996    WALFactory wFactory = new WALFactory(conf, "1234");
997    TableDescriptors tds = new FSTableDescriptors(conf);
998    FSTableDescriptors.tryUpdateMetaTableDescriptor(conf);
999    HRegion meta = HRegion.createHRegion(RegionInfoBuilder.FIRST_META_REGIONINFO, testDir, conf,
1000      tds.get(TableName.META_TABLE_NAME), wFactory.getWAL(RegionInfoBuilder.FIRST_META_REGIONINFO));
1001    // parameterized tests add [#] suffix get rid of [ and ].
1002    TableDescriptor desc = TableDescriptorBuilder
1003      .newBuilder(TableName.valueOf(name.getMethodName().replaceAll("[\\[\\]]", "_")))
1004      .setColumnFamily(ColumnFamilyDescriptorBuilder.of("foo")).build();
1005    RegionInfo hri = RegionInfoBuilder.newBuilder(desc.getTableName())
1006      .setStartKey(Bytes.toBytes("row_0200")).setEndKey(Bytes.toBytes("row_0300")).build();
1007    HRegion r = HRegion.createHRegion(hri, testDir, conf, desc, wFactory.getWAL(hri));
1008    addRegionToMETA(meta, r);
1009    edge.setCurrentTimeMillis(1234 + 100);
1010    StringBuilder sb = new StringBuilder();
1011    assertTrue(meta.shouldFlush(sb) == false);
1012    edge.setCurrentTimeMillis(edge.currentTime() + HRegion.SYSTEM_CACHE_FLUSH_INTERVAL + 1);
1013    assertTrue(meta.shouldFlush(sb) == true);
1014  }
1015
1016  /**
1017   * Inserts a new region's meta information into the passed <code>meta</code> region.
1018   * @param meta hbase:meta HRegion to be updated
1019   * @param r    HRegion to add to <code>meta</code>
1020   */
1021  private static void addRegionToMETA(final HRegion meta, final HRegion r) throws IOException {
1022    // The row key is the region name
1023    byte[] row = r.getRegionInfo().getRegionName();
1024    final long now = EnvironmentEdgeManager.currentTime();
1025    final List<Cell> cells = new ArrayList<>(2);
1026    cells.add(new KeyValue(row, HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, now,
1027      RegionInfo.toByteArray(r.getRegionInfo())));
1028    // Set into the root table the version of the meta table.
1029    cells.add(new KeyValue(row, HConstants.CATALOG_FAMILY, HConstants.META_VERSION_QUALIFIER, now,
1030      Bytes.toBytes(HConstants.META_VERSION)));
1031    NavigableMap<byte[], List<Cell>> familyMap = new TreeMap<>(Bytes.BYTES_COMPARATOR);
1032    familyMap.put(HConstants.CATALOG_FAMILY, cells);
1033    meta.put(new Put(row, HConstants.LATEST_TIMESTAMP, familyMap));
1034  }
1035
1036  private class EnvironmentEdgeForMemstoreTest implements EnvironmentEdge {
1037    long t = 1234;
1038
1039    @Override
1040    public long currentTime() {
1041      return t;
1042    }
1043
1044    public void setCurrentTimeMillis(long t) {
1045      this.t = t;
1046    }
1047  }
1048
1049  /**
1050   * Adds {@link #ROW_COUNT} rows and {@link #QUALIFIER_COUNT}
1051   * @param hmc Instance to add rows to.
1052   * @return How many rows we added.
1053   */
1054  protected int addRows(final AbstractMemStore hmc) {
1055    return addRows(hmc, HConstants.LATEST_TIMESTAMP);
1056  }
1057
1058  /**
1059   * Adds {@link #ROW_COUNT} rows and {@link #QUALIFIER_COUNT}
1060   * @param hmc Instance to add rows to.
1061   * @return How many rows we added.
1062   */
1063  protected int addRows(final MemStore hmc, final long ts) {
1064    for (int i = 0; i < ROW_COUNT; i++) {
1065      long timestamp =
1066        ts == HConstants.LATEST_TIMESTAMP ? EnvironmentEdgeManager.currentTime() : ts;
1067      for (int ii = 0; ii < QUALIFIER_COUNT; ii++) {
1068        byte[] row = Bytes.toBytes(i);
1069        byte[] qf = makeQualifier(i, ii);
1070        hmc.add(new KeyValue(row, FAMILY, qf, timestamp, qf), null);
1071      }
1072    }
1073    return ROW_COUNT;
1074  }
1075
1076  private long runSnapshot(final AbstractMemStore hmc) throws UnexpectedStateException {
1077    // Save off old state.
1078    int oldHistorySize = hmc.getSnapshot().getCellsCount();
1079    MemStoreSnapshot snapshot = hmc.snapshot();
1080    // Make some assertions about what just happened.
1081    assertTrue("History size has not increased",
1082      oldHistorySize < hmc.getSnapshot().getCellsCount());
1083    long t = memstore.timeOfOldestEdit();
1084    assertTrue("Time of oldest edit is not Long.MAX_VALUE", t == Long.MAX_VALUE);
1085    hmc.clearSnapshot(snapshot.getId());
1086    return t;
1087  }
1088
1089  private void isExpectedRowWithoutTimestamps(final int rowIndex, List<Cell> kvs) {
1090    int i = 0;
1091    for (Cell kv : kvs) {
1092      byte[] expectedColname = makeQualifier(rowIndex, i++);
1093      assertTrue("Column name", CellUtil.matchingQualifier(kv, expectedColname));
1094      // Value is column name as bytes. Usually result is
1095      // 100 bytes in size at least. This is the default size
1096      // for BytesWriteable. For comparison, convert bytes to
1097      // String and trim to remove trailing null bytes.
1098      assertTrue("Content", CellUtil.matchingValue(kv, expectedColname));
1099    }
1100  }
1101
1102  private static void addRows(int count, final MemStore mem) {
1103    long nanos = System.nanoTime();
1104
1105    for (int i = 0; i < count; i++) {
1106      if (i % 1000 == 0) {
1107
1108        System.out.println(i + " Took for 1k usec: " + (System.nanoTime() - nanos) / 1000);
1109        nanos = System.nanoTime();
1110      }
1111      long timestamp = System.currentTimeMillis();
1112
1113      for (int ii = 0; ii < QUALIFIER_COUNT; ii++) {
1114        byte[] row = Bytes.toBytes(i);
1115        byte[] qf = makeQualifier(i, ii);
1116        mem.add(new KeyValue(row, FAMILY, qf, timestamp, qf), null);
1117      }
1118    }
1119  }
1120
1121  static void doScan(MemStore ms, int iteration) throws IOException {
1122    long nanos = System.nanoTime();
1123    KeyValueScanner s = ms.getScanners(0).get(0);
1124    s.seek(KeyValueUtil.createFirstOnRow(new byte[] {}));
1125
1126    System.out.println(iteration + " create/seek took: " + (System.nanoTime() - nanos) / 1000);
1127    int cnt = 0;
1128    while (s.next() != null)
1129      ++cnt;
1130
1131    System.out
1132      .println(iteration + " took usec: " + (System.nanoTime() - nanos) / 1000 + " for: " + cnt);
1133
1134  }
1135
1136  public static void main(String[] args) throws IOException {
1137    MemStore ms = new DefaultMemStore();
1138
1139    long n1 = System.nanoTime();
1140    addRows(25000, ms);
1141    System.out.println("Took for insert: " + (System.nanoTime() - n1) / 1000);
1142
1143    System.out.println("foo");
1144
1145    for (int i = 0; i < 50; i++)
1146      doScan(ms, i);
1147  }
1148}