001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021
022import java.io.IOException;
023import org.apache.hadoop.hbase.client.Delete;
024import org.apache.hadoop.hbase.client.Durability;
025import org.apache.hadoop.hbase.client.Get;
026import org.apache.hadoop.hbase.client.Put;
027import org.apache.hadoop.hbase.client.Result;
028import org.apache.hadoop.hbase.client.ResultScanner;
029import org.apache.hadoop.hbase.client.Scan;
030import org.apache.hadoop.hbase.client.Table;
031import org.apache.hadoop.hbase.util.Bytes;
032
033/**
034 * Tests user specifiable time stamps putting, getting and scanning. Also tests same in presence of
035 * deletes. Test cores are written so can be run against an HRegion and against an HTable: i.e. both
036 * local and remote.
037 */
038public class TimestampTestBase {
039  private static final long T0 = 10L;
040  private static final long T1 = 100L;
041  private static final long T2 = 200L;
042
043  public static final byte[] FAMILY_NAME = Bytes.toBytes("colfamily11");
044  private static final byte[] QUALIFIER_NAME = Bytes.toBytes("contents");
045
046  private static final byte[] ROW = Bytes.toBytes("row");
047
048  interface FlushCache {
049    void flushcache() throws IOException;
050  }
051
052  /*
053   * Run test that delete works according to description in <a
054   * href="https://issues.apache.org/jira/browse/HADOOP-1784">hadoop-1784</a>.
055   */
056  public static void doTestDelete(final Table table, FlushCache flusher) throws IOException {
057    // Add values at various timestamps (Values are timestampes as bytes).
058    put(table, T0);
059    put(table, T1);
060    put(table, T2);
061    put(table);
062    // Verify that returned versions match passed timestamps.
063    assertVersions(table, new long[] { HConstants.LATEST_TIMESTAMP, T2, T1 });
064
065    // If I delete w/o specifying a timestamp, this means I'm deleting the latest.
066    delete(table);
067    // Verify that I get back T2 through T1 -- that the latest version has been deleted.
068    assertVersions(table, new long[] { T2, T1, T0 });
069
070    // Flush everything out to disk and then retry
071    flusher.flushcache();
072    assertVersions(table, new long[] { T2, T1, T0 });
073
074    // Now add, back a latest so I can test remove other than the latest.
075    put(table);
076    assertVersions(table, new long[] { HConstants.LATEST_TIMESTAMP, T2, T1 });
077    delete(table, T2);
078    assertVersions(table, new long[] { HConstants.LATEST_TIMESTAMP, T1, T0 });
079    // Flush everything out to disk and then retry
080    flusher.flushcache();
081    assertVersions(table, new long[] { HConstants.LATEST_TIMESTAMP, T1, T0 });
082
083    // Now try deleting all from T2 back inclusive (We first need to add T2
084    // back into the mix and to make things a little interesting, delete and then readd T1.
085    put(table, T2);
086    delete(table, T1);
087    put(table, T1);
088
089    Delete delete = new Delete(ROW);
090    delete.addColumns(FAMILY_NAME, QUALIFIER_NAME, T2);
091    table.delete(delete);
092
093    // Should only be current value in set. Assert this is so
094    assertOnlyLatest(table, HConstants.LATEST_TIMESTAMP);
095
096    // Flush everything out to disk and then redo above tests
097    flusher.flushcache();
098    assertOnlyLatest(table, HConstants.LATEST_TIMESTAMP);
099  }
100
101  private static void assertOnlyLatest(final Table incommon, final long currentTime)
102    throws IOException {
103    Get get = null;
104    get = new Get(ROW);
105    get.addColumn(FAMILY_NAME, QUALIFIER_NAME);
106    get.readVersions(3);
107    Result result = incommon.get(get);
108    assertEquals(1, result.size());
109    long time = Bytes.toLong(CellUtil.cloneValue(result.rawCells()[0]));
110    assertEquals(time, currentTime);
111  }
112
113  /*
114   * Assert that returned versions match passed in timestamps and that results are returned in the
115   * right order. Assert that values when converted to longs match the corresponding passed
116   * timestamp.
117   */
118  public static void assertVersions(final Table incommon, final long[] tss) throws IOException {
119    // Assert that 'latest' is what we expect.
120    Get get = null;
121    get = new Get(ROW);
122    get.addColumn(FAMILY_NAME, QUALIFIER_NAME);
123    Result r = incommon.get(get);
124    byte[] bytes = r.getValue(FAMILY_NAME, QUALIFIER_NAME);
125    long t = Bytes.toLong(bytes);
126    assertEquals(tss[0], t);
127
128    // Now assert that if we ask for multiple versions, that they come out in
129    // order.
130    get = new Get(ROW);
131    get.addColumn(FAMILY_NAME, QUALIFIER_NAME);
132    get.readVersions(tss.length);
133    Result result = incommon.get(get);
134    Cell[] kvs = result.rawCells();
135    assertEquals(kvs.length, tss.length);
136    for (int i = 0; i < kvs.length; i++) {
137      t = Bytes.toLong(CellUtil.cloneValue(kvs[i]));
138      assertEquals(tss[i], t);
139    }
140
141    // Determine highest stamp to set as next max stamp
142    long maxStamp = kvs[0].getTimestamp();
143
144    // Specify a timestamp get multiple versions.
145    get = new Get(ROW);
146    get.addColumn(FAMILY_NAME, QUALIFIER_NAME);
147    get.setTimeRange(0, maxStamp);
148    get.readVersions(kvs.length - 1);
149    result = incommon.get(get);
150    kvs = result.rawCells();
151    assertEquals(kvs.length, tss.length - 1);
152    for (int i = 1; i < kvs.length; i++) {
153      t = Bytes.toLong(CellUtil.cloneValue(kvs[i - 1]));
154      assertEquals(tss[i], t);
155    }
156
157    // Test scanner returns expected version
158    assertScanContentTimestamp(incommon, tss[0]);
159  }
160
161  /*
162   * Run test scanning different timestamps.
163   */
164  public static void doTestTimestampScanning(final Table incommon, final FlushCache flusher)
165    throws IOException {
166    // Add a couple of values for three different timestamps.
167    put(incommon, T0);
168    put(incommon, T1);
169    put(incommon, HConstants.LATEST_TIMESTAMP);
170    // Get count of latest items.
171    int count = assertScanContentTimestamp(incommon, HConstants.LATEST_TIMESTAMP);
172    // Assert I get same count when I scan at each timestamp.
173    assertEquals(count, assertScanContentTimestamp(incommon, T0));
174    assertEquals(count, assertScanContentTimestamp(incommon, T1));
175    // Flush everything out to disk and then retry
176    flusher.flushcache();
177    assertEquals(count, assertScanContentTimestamp(incommon, T0));
178    assertEquals(count, assertScanContentTimestamp(incommon, T1));
179  }
180
181  /*
182   * Assert that the scan returns only values < timestamp.
183   * @return Count of items scanned.
184   */
185  public static int assertScanContentTimestamp(final Table in, final long ts) throws IOException {
186    Scan scan = new Scan().withStartRow(HConstants.EMPTY_START_ROW);
187    scan.addFamily(FAMILY_NAME);
188    scan.setTimeRange(0, ts);
189    ResultScanner scanner = in.getScanner(scan);
190    int count = 0;
191    try {
192      // TODO FIX
193      // HStoreKey key = new HStoreKey();
194      // TreeMap<byte [], Cell>value =
195      // new TreeMap<byte [], Cell>(Bytes.BYTES_COMPARATOR);
196      // while (scanner.next(key, value)) {
197      // assertTrue(key.getTimestamp() <= ts);
198      // // Content matches the key or HConstants.LATEST_TIMESTAMP.
199      // // (Key does not match content if we 'put' with LATEST_TIMESTAMP).
200      // long l = Bytes.toLong(value.get(COLUMN).getValue());
201      // assertTrue(key.getTimestamp() == l ||
202      // HConstants.LATEST_TIMESTAMP == l);
203      // count++;
204      // value.clear();
205      // }
206    } finally {
207      scanner.close();
208    }
209    return count;
210  }
211
212  public static void put(final Table loader, final long ts) throws IOException {
213    put(loader, Bytes.toBytes(ts), ts);
214  }
215
216  public static void put(final Table loader) throws IOException {
217    long ts = HConstants.LATEST_TIMESTAMP;
218    put(loader, Bytes.toBytes(ts), ts);
219  }
220
221  /*
222   * Put values.
223   */
224  public static void put(final Table loader, final byte[] bytes, final long ts) throws IOException {
225    Put put = new Put(ROW, ts);
226    put.setDurability(Durability.SKIP_WAL);
227    put.addColumn(FAMILY_NAME, QUALIFIER_NAME, bytes);
228    loader.put(put);
229  }
230
231  public static void delete(final Table loader) throws IOException {
232    delete(loader, null);
233  }
234
235  public static void delete(final Table loader, final byte[] column) throws IOException {
236    delete(loader, column, HConstants.LATEST_TIMESTAMP);
237  }
238
239  public static void delete(final Table loader, final long ts) throws IOException {
240    delete(loader, null, ts);
241  }
242
243  public static void delete(final Table loader, final byte[] column, final long ts)
244    throws IOException {
245    Delete delete = ts == HConstants.LATEST_TIMESTAMP ? new Delete(ROW) : new Delete(ROW, ts);
246    delete.addColumn(FAMILY_NAME, QUALIFIER_NAME, ts);
247    loader.delete(delete);
248  }
249
250  public static Result get(final Table loader) throws IOException {
251    return loader.get(new Get(ROW));
252  }
253}