001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019package org.apache.hadoop.hbase;
020
021import java.io.IOException;
022
023import org.apache.hadoop.hbase.client.Delete;
024import org.apache.hadoop.hbase.client.Get;
025import org.apache.hadoop.hbase.client.Put;
026import org.apache.hadoop.hbase.client.Result;
027import org.apache.hadoop.hbase.client.ResultScanner;
028import org.apache.hadoop.hbase.client.Scan;
029import org.apache.hadoop.hbase.client.Table;
030import org.apache.hadoop.hbase.client.Durability;
031import org.apache.hadoop.hbase.util.Bytes;
032import org.junit.Assert;
033
034/**
035 * Tests user specifiable time stamps putting, getting and scanning.  Also
036 * tests same in presence of deletes.  Test cores are written so can be
037 * run against an HRegion and against an HTable: i.e. both local and remote.
038 */
039public class TimestampTestBase {
040  private static final long T0 = 10L;
041  private static final long T1 = 100L;
042  private static final long T2 = 200L;
043
044  public static final byte [] FAMILY_NAME = Bytes.toBytes("colfamily11");
045  private static final byte [] QUALIFIER_NAME = Bytes.toBytes("contents");
046
047  private static final byte [] ROW = Bytes.toBytes("row");
048
049  interface FlushCache {
050    void flushcache() throws IOException;
051  }
052
053  /*
054   * Run test that delete works according to description in <a
055   * href="https://issues.apache.org/jira/browse/HADOOP-1784">hadoop-1784</a>.
056   * @param incommon
057   * @param flusher
058   * @throws IOException
059   */
060  public static void doTestDelete(final Table table, FlushCache flusher)
061  throws IOException {
062    // Add values at various timestamps (Values are timestampes as bytes).
063    put(table, T0);
064    put(table, T1);
065    put(table, T2);
066    put(table);
067    // Verify that returned versions match passed timestamps.
068    assertVersions(table, new long [] {HConstants.LATEST_TIMESTAMP, T2, T1});
069
070    // If I delete w/o specifying a timestamp, this means I'm deleting the latest.
071    delete(table);
072    // Verify that I get back T2 through T1 -- that the latest version has been deleted.
073    assertVersions(table, new long [] {T2, T1, T0});
074
075    // Flush everything out to disk and then retry
076    flusher.flushcache();
077    assertVersions(table, new long [] {T2, T1, T0});
078
079    // Now add, back a latest so I can test remove other than the latest.
080    put(table);
081    assertVersions(table, new long [] {HConstants.LATEST_TIMESTAMP, T2, T1});
082    delete(table, T2);
083    assertVersions(table, new long [] {HConstants.LATEST_TIMESTAMP, T1, T0});
084    // Flush everything out to disk and then retry
085    flusher.flushcache();
086    assertVersions(table, new long [] {HConstants.LATEST_TIMESTAMP, T1, T0});
087
088    // Now try deleting all from T2 back inclusive (We first need to add T2
089    // back into the mix and to make things a little interesting, delete and then readd T1.
090    put(table, T2);
091    delete(table, T1);
092    put(table, T1);
093
094    Delete delete = new Delete(ROW);
095    delete.addColumns(FAMILY_NAME, QUALIFIER_NAME, T2);
096    table.delete(delete);
097
098    // Should only be current value in set.  Assert this is so
099    assertOnlyLatest(table, HConstants.LATEST_TIMESTAMP);
100
101    // Flush everything out to disk and then redo above tests
102    flusher.flushcache();
103    assertOnlyLatest(table, HConstants.LATEST_TIMESTAMP);
104  }
105
106  private static void assertOnlyLatest(final Table incommon, final long currentTime)
107  throws IOException {
108    Get get = null;
109    get = new Get(ROW);
110    get.addColumn(FAMILY_NAME, QUALIFIER_NAME);
111    get.setMaxVersions(3);
112    Result result = incommon.get(get);
113    Assert.assertEquals(1, result.size());
114    long time = Bytes.toLong(CellUtil.cloneValue(result.rawCells()[0]));
115    Assert.assertEquals(time, currentTime);
116  }
117
118  /*
119   * Assert that returned versions match passed in timestamps and that results
120   * are returned in the right order.  Assert that values when converted to
121   * longs match the corresponding passed timestamp.
122   * @param r
123   * @param tss
124   * @throws IOException
125   */
126  public static void assertVersions(final Table incommon, final long [] tss)
127  throws IOException {
128    // Assert that 'latest' is what we expect.
129    Get get = null;
130    get = new Get(ROW);
131    get.addColumn(FAMILY_NAME, QUALIFIER_NAME);
132    Result r = incommon.get(get);
133    byte [] bytes = r.getValue(FAMILY_NAME, QUALIFIER_NAME);
134    long t = Bytes.toLong(bytes);
135    Assert.assertEquals(tss[0], t);
136
137    // Now assert that if we ask for multiple versions, that they come out in
138    // order.
139    get = new Get(ROW);
140    get.addColumn(FAMILY_NAME, QUALIFIER_NAME);
141    get.setMaxVersions(tss.length);
142    Result result = incommon.get(get);
143    Cell [] kvs = result.rawCells();
144    Assert.assertEquals(kvs.length, tss.length);
145    for(int i=0;i<kvs.length;i++) {
146      t = Bytes.toLong(CellUtil.cloneValue(kvs[i]));
147      Assert.assertEquals(tss[i], t);
148    }
149
150    // Determine highest stamp to set as next max stamp
151    long maxStamp = kvs[0].getTimestamp();
152
153    // Specify a timestamp get multiple versions.
154    get = new Get(ROW);
155    get.addColumn(FAMILY_NAME, QUALIFIER_NAME);
156    get.setTimeRange(0, maxStamp);
157    get.setMaxVersions(kvs.length - 1);
158    result = incommon.get(get);
159    kvs = result.rawCells();
160    Assert.assertEquals(kvs.length, tss.length - 1);
161    for(int i=1;i<kvs.length;i++) {
162      t = Bytes.toLong(CellUtil.cloneValue(kvs[i-1]));
163      Assert.assertEquals(tss[i], t);
164    }
165
166    // Test scanner returns expected version
167    assertScanContentTimestamp(incommon, tss[0]);
168  }
169
170  /*
171   * Run test scanning different timestamps.
172   * @param incommon
173   * @param flusher
174   * @throws IOException
175   */
176  public static void doTestTimestampScanning(final Table incommon,
177    final FlushCache flusher)
178  throws IOException {
179    // Add a couple of values for three different timestamps.
180    put(incommon, T0);
181    put(incommon, T1);
182    put(incommon, HConstants.LATEST_TIMESTAMP);
183    // Get count of latest items.
184    int count = assertScanContentTimestamp(incommon,
185      HConstants.LATEST_TIMESTAMP);
186    // Assert I get same count when I scan at each timestamp.
187    Assert.assertEquals(count, assertScanContentTimestamp(incommon, T0));
188    Assert.assertEquals(count, assertScanContentTimestamp(incommon, T1));
189    // Flush everything out to disk and then retry
190    flusher.flushcache();
191    Assert.assertEquals(count, assertScanContentTimestamp(incommon, T0));
192    Assert.assertEquals(count, assertScanContentTimestamp(incommon, T1));
193  }
194
195  /*
196   * Assert that the scan returns only values < timestamp.
197   * @param r
198   * @param ts
199   * @return Count of items scanned.
200   * @throws IOException
201   */
202  public static int assertScanContentTimestamp(final Table in, final long ts)
203  throws IOException {
204    Scan scan = new Scan(HConstants.EMPTY_START_ROW);
205    scan.addFamily(FAMILY_NAME);
206    scan.setTimeRange(0, ts);
207    ResultScanner scanner = in.getScanner(scan);
208    int count = 0;
209    try {
210      // TODO FIX
211//      HStoreKey key = new HStoreKey();
212//      TreeMap<byte [], Cell>value =
213//        new TreeMap<byte [], Cell>(Bytes.BYTES_COMPARATOR);
214//      while (scanner.next(key, value)) {
215//        assertTrue(key.getTimestamp() <= ts);
216//        // Content matches the key or HConstants.LATEST_TIMESTAMP.
217//        // (Key does not match content if we 'put' with LATEST_TIMESTAMP).
218//        long l = Bytes.toLong(value.get(COLUMN).getValue());
219//        assertTrue(key.getTimestamp() == l ||
220//          HConstants.LATEST_TIMESTAMP == l);
221//        count++;
222//        value.clear();
223//      }
224    } finally {
225      scanner.close();
226    }
227    return count;
228  }
229
230  public static void put(final Table loader, final long ts)
231  throws IOException {
232    put(loader, Bytes.toBytes(ts), ts);
233  }
234
235  public static void put(final Table loader)
236  throws IOException {
237    long ts = HConstants.LATEST_TIMESTAMP;
238    put(loader, Bytes.toBytes(ts), ts);
239  }
240
241  /*
242   * Put values.
243   * @param loader
244   * @param bytes
245   * @param ts
246   * @throws IOException
247   */
248  public static void put(final Table loader, final byte [] bytes,
249    final long ts)
250  throws IOException {
251    Put put = new Put(ROW, ts);
252    put.setDurability(Durability.SKIP_WAL);
253    put.addColumn(FAMILY_NAME, QUALIFIER_NAME, bytes);
254    loader.put(put);
255  }
256
257  public static void delete(final Table loader) throws IOException {
258    delete(loader, null);
259  }
260
261  public static void delete(final Table loader, final byte [] column)
262  throws IOException {
263    delete(loader, column, HConstants.LATEST_TIMESTAMP);
264  }
265
266  public static void delete(final Table loader, final long ts)
267  throws IOException {
268    delete(loader, null, ts);
269  }
270
271  public static void delete(final Table loader, final byte [] column,
272      final long ts)
273  throws IOException {
274    Delete delete = ts == HConstants.LATEST_TIMESTAMP?
275      new Delete(ROW): new Delete(ROW, ts);
276    delete.addColumn(FAMILY_NAME, QUALIFIER_NAME, ts);
277    loader.delete(delete);
278  }
279
280  public static Result get(final Table loader) throws IOException {
281    return loader.get(new Get(ROW));
282  }
283}