001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.client;
019
020import static org.junit.Assert.assertEquals;
021
022import java.io.IOException;
023import java.util.Arrays;
024import java.util.Collections;
025import java.util.List;
026import org.apache.hadoop.hbase.Cell;
027import org.apache.hadoop.hbase.CellUtil;
028import org.apache.hadoop.hbase.HBaseClassTestRule;
029import org.apache.hadoop.hbase.HBaseTestingUtility;
030import org.apache.hadoop.hbase.TableName;
031import org.apache.hadoop.hbase.testclassification.ClientTests;
032import org.apache.hadoop.hbase.testclassification.LargeTests;
033import org.apache.hadoop.hbase.util.Bytes;
034import org.junit.After;
035import org.junit.AfterClass;
036import org.junit.Before;
037import org.junit.BeforeClass;
038import org.junit.ClassRule;
039import org.junit.Rule;
040import org.junit.Test;
041import org.junit.experimental.categories.Category;
042import org.junit.rules.TestName;
043import org.slf4j.Logger;
044import org.slf4j.LoggerFactory;
045
046/**
047 * Run tests related to {@link org.apache.hadoop.hbase.filter.TimestampsFilter} using HBase client
048 * APIs. Sets up the HBase mini cluster once at start. Each creates a table named for the method and
049 * does its stuff against that.
050 */
051@Category({ LargeTests.class, ClientTests.class })
052public class TestMultipleTimestamps {
053
054  @ClassRule
055  public static final HBaseClassTestRule CLASS_RULE =
056    HBaseClassTestRule.forClass(TestMultipleTimestamps.class);
057
058  private static final Logger LOG = LoggerFactory.getLogger(TestMultipleTimestamps.class);
059  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
060
061  @Rule
062  public TestName name = new TestName();
063
064  /**
065   * @throws java.lang.Exception
066   */
067  @BeforeClass
068  public static void setUpBeforeClass() throws Exception {
069    TEST_UTIL.startMiniCluster();
070  }
071
072  /**
073   * @throws java.lang.Exception
074   */
075  @AfterClass
076  public static void tearDownAfterClass() throws Exception {
077    TEST_UTIL.shutdownMiniCluster();
078  }
079
080  /**
081   * @throws java.lang.Exception
082   */
083  @Before
084  public void setUp() throws Exception {
085    // Nothing to do.
086  }
087
088  /**
089   * @throws java.lang.Exception
090   */
091  @After
092  public void tearDown() throws Exception {
093    // Nothing to do.
094  }
095
096  @Test
097  public void testReseeksWithOneColumnMiltipleTimestamp() throws IOException {
098    final TableName tableName = TableName.valueOf(name.getMethodName());
099    byte[] FAMILY = Bytes.toBytes("event_log");
100    byte[][] FAMILIES = new byte[][] { FAMILY };
101
102    // create table; set versions to max...
103    Table ht = TEST_UTIL.createTable(tableName, FAMILIES, Integer.MAX_VALUE);
104
105    Integer[] putRows = new Integer[] { 1, 3, 5, 7 };
106    Integer[] putColumns = new Integer[] { 1, 3, 5 };
107    Long[] putTimestamps = new Long[] { 1L, 2L, 3L, 4L, 5L };
108
109    Integer[] scanRows = new Integer[] { 3, 5 };
110    Integer[] scanColumns = new Integer[] { 3 };
111    Long[] scanTimestamps = new Long[] { 3L, 4L };
112    int scanMaxVersions = 2;
113
114    put(ht, FAMILY, putRows, putColumns, putTimestamps);
115
116    TEST_UTIL.flush(tableName);
117
118    ResultScanner scanner =
119      scan(ht, FAMILY, scanRows, scanColumns, scanTimestamps, scanMaxVersions);
120
121    Cell[] kvs;
122
123    kvs = scanner.next().rawCells();
124    assertEquals(2, kvs.length);
125    checkOneCell(kvs[0], FAMILY, 3, 3, 4);
126    checkOneCell(kvs[1], FAMILY, 3, 3, 3);
127    kvs = scanner.next().rawCells();
128    assertEquals(2, kvs.length);
129    checkOneCell(kvs[0], FAMILY, 5, 3, 4);
130    checkOneCell(kvs[1], FAMILY, 5, 3, 3);
131
132    ht.close();
133  }
134
135  @Test
136  public void testReseeksWithMultipleColumnOneTimestamp() throws IOException {
137    LOG.info(name.getMethodName());
138    final TableName tableName = TableName.valueOf(name.getMethodName());
139    byte[] FAMILY = Bytes.toBytes("event_log");
140    byte[][] FAMILIES = new byte[][] { FAMILY };
141
142    // create table; set versions to max...
143    Table ht = TEST_UTIL.createTable(tableName, FAMILIES, Integer.MAX_VALUE);
144
145    Integer[] putRows = new Integer[] { 1, 3, 5, 7 };
146    Integer[] putColumns = new Integer[] { 1, 3, 5 };
147    Long[] putTimestamps = new Long[] { 1L, 2L, 3L, 4L, 5L };
148
149    Integer[] scanRows = new Integer[] { 3, 5 };
150    Integer[] scanColumns = new Integer[] { 3, 4 };
151    Long[] scanTimestamps = new Long[] { 3L };
152    int scanMaxVersions = 2;
153
154    put(ht, FAMILY, putRows, putColumns, putTimestamps);
155
156    TEST_UTIL.flush(tableName);
157
158    ResultScanner scanner =
159      scan(ht, FAMILY, scanRows, scanColumns, scanTimestamps, scanMaxVersions);
160
161    Cell[] kvs;
162
163    kvs = scanner.next().rawCells();
164    assertEquals(1, kvs.length);
165    checkOneCell(kvs[0], FAMILY, 3, 3, 3);
166    kvs = scanner.next().rawCells();
167    assertEquals(1, kvs.length);
168    checkOneCell(kvs[0], FAMILY, 5, 3, 3);
169
170    ht.close();
171  }
172
173  @Test
174  public void testReseeksWithMultipleColumnMultipleTimestamp() throws IOException {
175    LOG.info(name.getMethodName());
176
177    final TableName tableName = TableName.valueOf(name.getMethodName());
178    byte[] FAMILY = Bytes.toBytes("event_log");
179    byte[][] FAMILIES = new byte[][] { FAMILY };
180
181    // create table; set versions to max...
182    Table ht = TEST_UTIL.createTable(tableName, FAMILIES, Integer.MAX_VALUE);
183
184    Integer[] putRows = new Integer[] { 1, 3, 5, 7 };
185    Integer[] putColumns = new Integer[] { 1, 3, 5 };
186    Long[] putTimestamps = new Long[] { 1L, 2L, 3L, 4L, 5L };
187
188    Integer[] scanRows = new Integer[] { 5, 7 };
189    Integer[] scanColumns = new Integer[] { 3, 4, 5 };
190    Long[] scanTimestamps = new Long[] { 2L, 3L };
191    int scanMaxVersions = 2;
192
193    put(ht, FAMILY, putRows, putColumns, putTimestamps);
194
195    TEST_UTIL.flush(tableName);
196    Scan scan = new Scan();
197    scan.setMaxVersions(10);
198    ResultScanner scanner = ht.getScanner(scan);
199    while (true) {
200      Result r = scanner.next();
201      if (r == null) break;
202      LOG.info("r=" + r);
203    }
204    scanner = scan(ht, FAMILY, scanRows, scanColumns, scanTimestamps, scanMaxVersions);
205
206    Cell[] kvs;
207
208    // This looks like wrong answer. Should be 2. Even then we are returning wrong result,
209    // timestamps that are 3 whereas should be 2 since min is inclusive.
210    kvs = scanner.next().rawCells();
211    assertEquals(4, kvs.length);
212    checkOneCell(kvs[0], FAMILY, 5, 3, 3);
213    checkOneCell(kvs[1], FAMILY, 5, 3, 2);
214    checkOneCell(kvs[2], FAMILY, 5, 5, 3);
215    checkOneCell(kvs[3], FAMILY, 5, 5, 2);
216    kvs = scanner.next().rawCells();
217    assertEquals(4, kvs.length);
218    checkOneCell(kvs[0], FAMILY, 7, 3, 3);
219    checkOneCell(kvs[1], FAMILY, 7, 3, 2);
220    checkOneCell(kvs[2], FAMILY, 7, 5, 3);
221    checkOneCell(kvs[3], FAMILY, 7, 5, 2);
222
223    ht.close();
224  }
225
226  @Test
227  public void testReseeksWithMultipleFiles() throws IOException {
228    LOG.info(name.getMethodName());
229    final TableName tableName = TableName.valueOf(name.getMethodName());
230    byte[] FAMILY = Bytes.toBytes("event_log");
231    byte[][] FAMILIES = new byte[][] { FAMILY };
232
233    // create table; set versions to max...
234    Table ht = TEST_UTIL.createTable(tableName, FAMILIES, Integer.MAX_VALUE);
235
236    Integer[] putRows1 = new Integer[] { 1, 2, 3 };
237    Integer[] putColumns1 = new Integer[] { 2, 5, 6 };
238    Long[] putTimestamps1 = new Long[] { 1L, 2L, 5L };
239
240    Integer[] putRows2 = new Integer[] { 6, 7 };
241    Integer[] putColumns2 = new Integer[] { 3, 6 };
242    Long[] putTimestamps2 = new Long[] { 4L, 5L };
243
244    Integer[] putRows3 = new Integer[] { 2, 3, 5 };
245    Integer[] putColumns3 = new Integer[] { 1, 2, 3 };
246    Long[] putTimestamps3 = new Long[] { 4L, 8L };
247
248    Integer[] scanRows = new Integer[] { 3, 5, 7 };
249    Integer[] scanColumns = new Integer[] { 3, 4, 5 };
250    Long[] scanTimestamps = new Long[] { 2L, 4L };
251    int scanMaxVersions = 5;
252
253    put(ht, FAMILY, putRows1, putColumns1, putTimestamps1);
254    TEST_UTIL.flush(tableName);
255    put(ht, FAMILY, putRows2, putColumns2, putTimestamps2);
256    TEST_UTIL.flush(tableName);
257    put(ht, FAMILY, putRows3, putColumns3, putTimestamps3);
258
259    ResultScanner scanner =
260      scan(ht, FAMILY, scanRows, scanColumns, scanTimestamps, scanMaxVersions);
261
262    Cell[] kvs;
263
264    kvs = scanner.next().rawCells();
265    assertEquals(2, kvs.length);
266    checkOneCell(kvs[0], FAMILY, 3, 3, 4);
267    checkOneCell(kvs[1], FAMILY, 3, 5, 2);
268
269    kvs = scanner.next().rawCells();
270    assertEquals(1, kvs.length);
271    checkOneCell(kvs[0], FAMILY, 5, 3, 4);
272
273    kvs = scanner.next().rawCells();
274    assertEquals(1, kvs.length);
275    checkOneCell(kvs[0], FAMILY, 6, 3, 4);
276
277    kvs = scanner.next().rawCells();
278    assertEquals(1, kvs.length);
279    checkOneCell(kvs[0], FAMILY, 7, 3, 4);
280
281    ht.close();
282  }
283
284  @Test
285  public void testWithVersionDeletes() throws Exception {
286
287    // first test from memstore (without flushing).
288    testWithVersionDeletes(false);
289
290    // run same test against HFiles (by forcing a flush).
291    testWithVersionDeletes(true);
292  }
293
294  public void testWithVersionDeletes(boolean flushTables) throws IOException {
295    LOG.info(name.getMethodName() + "_" + (flushTables ? "flush" : "noflush"));
296    final TableName tableName =
297      TableName.valueOf(name.getMethodName() + "_" + (flushTables ? "flush" : "noflush"));
298    byte[] FAMILY = Bytes.toBytes("event_log");
299    byte[][] FAMILIES = new byte[][] { FAMILY };
300
301    // create table; set versions to max...
302    Table ht = TEST_UTIL.createTable(tableName, FAMILIES, Integer.MAX_VALUE);
303
304    // For row:0, col:0: insert versions 1 through 5.
305    putNVersions(ht, FAMILY, 0, 0, 1, 5);
306
307    if (flushTables) {
308      TEST_UTIL.flush(tableName);
309    }
310
311    // delete version 4.
312    deleteOneVersion(ht, FAMILY, 0, 0, 4);
313
314    // request a bunch of versions including the deleted version. We should
315    // only get back entries for the versions that exist.
316    Cell kvs[] = getNVersions(ht, FAMILY, 0, 0, Arrays.asList(2L, 3L, 4L, 5L));
317    assertEquals(3, kvs.length);
318    checkOneCell(kvs[0], FAMILY, 0, 0, 5);
319    checkOneCell(kvs[1], FAMILY, 0, 0, 3);
320    checkOneCell(kvs[2], FAMILY, 0, 0, 2);
321
322    ht.close();
323  }
324
325  @Test
326  public void testWithMultipleVersionDeletes() throws IOException {
327    LOG.info(name.getMethodName());
328
329    final TableName tableName = TableName.valueOf(name.getMethodName());
330    byte[] FAMILY = Bytes.toBytes("event_log");
331    byte[][] FAMILIES = new byte[][] { FAMILY };
332
333    // create table; set versions to max...
334    Table ht = TEST_UTIL.createTable(tableName, FAMILIES, Integer.MAX_VALUE);
335
336    // For row:0, col:0: insert versions 1 through 5.
337    putNVersions(ht, FAMILY, 0, 0, 1, 5);
338
339    TEST_UTIL.flush(tableName);
340
341    // delete all versions before 4.
342    deleteAllVersionsBefore(ht, FAMILY, 0, 0, 4);
343
344    // request a bunch of versions including the deleted version. We should
345    // only get back entries for the versions that exist.
346    Cell kvs[] = getNVersions(ht, FAMILY, 0, 0, Arrays.asList(2L, 3L));
347    assertEquals(0, kvs.length);
348
349    ht.close();
350  }
351
352  @Test
353  public void testWithColumnDeletes() throws IOException {
354    final TableName tableName = TableName.valueOf(name.getMethodName());
355    byte[] FAMILY = Bytes.toBytes("event_log");
356    byte[][] FAMILIES = new byte[][] { FAMILY };
357
358    // create table; set versions to max...
359    Table ht = TEST_UTIL.createTable(tableName, FAMILIES, Integer.MAX_VALUE);
360
361    // For row:0, col:0: insert versions 1 through 5.
362    putNVersions(ht, FAMILY, 0, 0, 1, 5);
363
364    TEST_UTIL.flush(tableName);
365
366    // delete all versions before 4.
367    deleteColumn(ht, FAMILY, 0, 0);
368
369    // request a bunch of versions including the deleted version. We should
370    // only get back entries for the versions that exist.
371    Cell kvs[] = getNVersions(ht, FAMILY, 0, 0, Arrays.asList(2L, 3L));
372    assertEquals(0, kvs.length);
373
374    ht.close();
375  }
376
377  @Test
378  public void testWithFamilyDeletes() throws IOException {
379    final TableName tableName = TableName.valueOf(name.getMethodName());
380    byte[] FAMILY = Bytes.toBytes("event_log");
381    byte[][] FAMILIES = new byte[][] { FAMILY };
382
383    // create table; set versions to max...
384    Table ht = TEST_UTIL.createTable(tableName, FAMILIES, Integer.MAX_VALUE);
385
386    // For row:0, col:0: insert versions 1 through 5.
387    putNVersions(ht, FAMILY, 0, 0, 1, 5);
388
389    TEST_UTIL.flush(tableName);
390
391    // delete all versions before 4.
392    deleteFamily(ht, FAMILY, 0);
393
394    // request a bunch of versions including the deleted version. We should
395    // only get back entries for the versions that exist.
396    Cell kvs[] = getNVersions(ht, FAMILY, 0, 0, Arrays.asList(2L, 3L));
397    assertEquals(0, kvs.length);
398
399    ht.close();
400  }
401
402  /**
403   * Assert that the passed in KeyValue has expected contents for the specified row, column &
404   * timestamp.
405   */
406  private void checkOneCell(Cell kv, byte[] cf, int rowIdx, int colIdx, long ts) {
407
408    String ctx = "rowIdx=" + rowIdx + "; colIdx=" + colIdx + "; ts=" + ts;
409
410    assertEquals("Row mismatch which checking: " + ctx, "row:" + rowIdx,
411      Bytes.toString(CellUtil.cloneRow(kv)));
412
413    assertEquals("ColumnFamily mismatch while checking: " + ctx, Bytes.toString(cf),
414      Bytes.toString(CellUtil.cloneFamily(kv)));
415
416    assertEquals("Column qualifier mismatch while checking: " + ctx, "column:" + colIdx,
417      Bytes.toString(CellUtil.cloneQualifier(kv)));
418
419    assertEquals("Timestamp mismatch while checking: " + ctx, ts, kv.getTimestamp());
420
421    assertEquals("Value mismatch while checking: " + ctx, "value-version-" + ts,
422      Bytes.toString(CellUtil.cloneValue(kv)));
423  }
424
425  /**
426   * Uses the TimestampFilter on a Get to request a specified list of versions for the row/column
427   * specified by rowIdx & colIdx.
428   */
429  private Cell[] getNVersions(Table ht, byte[] cf, int rowIdx, int colIdx, List<Long> versions)
430    throws IOException {
431    byte row[] = Bytes.toBytes("row:" + rowIdx);
432    byte column[] = Bytes.toBytes("column:" + colIdx);
433    Get get = new Get(row);
434    get.addColumn(cf, column);
435    get.setMaxVersions();
436    get.setTimeRange(Collections.min(versions), Collections.max(versions) + 1);
437    Result result = ht.get(get);
438
439    return result.rawCells();
440  }
441
442  private ResultScanner scan(Table ht, byte[] cf, Integer[] rowIndexes, Integer[] columnIndexes,
443    Long[] versions, int maxVersions) throws IOException {
444    byte startRow[] = Bytes.toBytes("row:" + Collections.min(Arrays.asList(rowIndexes)));
445    byte endRow[] = Bytes.toBytes("row:" + Collections.max(Arrays.asList(rowIndexes)) + 1);
446    Scan scan = new Scan(startRow, endRow);
447    for (Integer colIdx : columnIndexes) {
448      byte column[] = Bytes.toBytes("column:" + colIdx);
449      scan.addColumn(cf, column);
450    }
451    scan.setMaxVersions(maxVersions);
452    scan.setTimeRange(Collections.min(Arrays.asList(versions)),
453      Collections.max(Arrays.asList(versions)) + 1);
454    ResultScanner scanner = ht.getScanner(scan);
455    return scanner;
456  }
457
458  private void put(Table ht, byte[] cf, Integer[] rowIndexes, Integer[] columnIndexes,
459    Long[] versions) throws IOException {
460    for (int rowIdx : rowIndexes) {
461      byte row[] = Bytes.toBytes("row:" + rowIdx);
462      Put put = new Put(row);
463      put.setDurability(Durability.SKIP_WAL);
464      for (int colIdx : columnIndexes) {
465        byte column[] = Bytes.toBytes("column:" + colIdx);
466        for (long version : versions) {
467          put.addColumn(cf, column, version, Bytes.toBytes("value-version-" + version));
468        }
469      }
470      ht.put(put);
471    }
472  }
473
474  /**
475   * Insert in specific row/column versions with timestamps versionStart..versionEnd.
476   */
477  private void putNVersions(Table ht, byte[] cf, int rowIdx, int colIdx, long versionStart,
478    long versionEnd) throws IOException {
479    byte row[] = Bytes.toBytes("row:" + rowIdx);
480    byte column[] = Bytes.toBytes("column:" + colIdx);
481    Put put = new Put(row);
482    put.setDurability(Durability.SKIP_WAL);
483
484    for (long idx = versionStart; idx <= versionEnd; idx++) {
485      put.addColumn(cf, column, idx, Bytes.toBytes("value-version-" + idx));
486    }
487
488    ht.put(put);
489  }
490
491  /**
492   * For row/column specified by rowIdx/colIdx, delete the cell corresponding to the specified
493   * version.
494   */
495  private void deleteOneVersion(Table ht, byte[] cf, int rowIdx, int colIdx, long version)
496    throws IOException {
497    byte row[] = Bytes.toBytes("row:" + rowIdx);
498    byte column[] = Bytes.toBytes("column:" + colIdx);
499    Delete del = new Delete(row);
500    del.addColumn(cf, column, version);
501    ht.delete(del);
502  }
503
504  /**
505   * For row/column specified by rowIdx/colIdx, delete all cells preceeding the specified version.
506   */
507  private void deleteAllVersionsBefore(Table ht, byte[] cf, int rowIdx, int colIdx, long version)
508    throws IOException {
509    byte row[] = Bytes.toBytes("row:" + rowIdx);
510    byte column[] = Bytes.toBytes("column:" + colIdx);
511    Delete del = new Delete(row);
512    del.addColumns(cf, column, version);
513    ht.delete(del);
514  }
515
516  private void deleteColumn(Table ht, byte[] cf, int rowIdx, int colIdx) throws IOException {
517    byte row[] = Bytes.toBytes("row:" + rowIdx);
518    byte column[] = Bytes.toBytes("column:" + colIdx);
519    Delete del = new Delete(row);
520    del.addColumns(cf, column);
521    ht.delete(del);
522  }
523
524  private void deleteFamily(Table ht, byte[] cf, int rowIdx) throws IOException {
525    byte row[] = Bytes.toBytes("row:" + rowIdx);
526    Delete del = new Delete(row);
527    del.addFamily(cf);
528    ht.delete(del);
529  }
530
531}