001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.apache.hadoop.hbase.HBaseTestingUtility.COLUMNS;
021import static org.junit.Assert.assertEquals;
022import static org.junit.Assert.assertTrue;
023
024import java.util.ArrayList;
025import java.util.List;
026import org.apache.hadoop.hbase.Cell;
027import org.apache.hadoop.hbase.CellUtil;
028import org.apache.hadoop.hbase.HBaseClassTestRule;
029import org.apache.hadoop.hbase.HBaseTestingUtility;
030import org.apache.hadoop.hbase.KeepDeletedCells;
031import org.apache.hadoop.hbase.TableName;
032import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
033import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
034import org.apache.hadoop.hbase.client.Delete;
035import org.apache.hadoop.hbase.client.Get;
036import org.apache.hadoop.hbase.client.Put;
037import org.apache.hadoop.hbase.client.Result;
038import org.apache.hadoop.hbase.client.TableDescriptor;
039import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
040import org.apache.hadoop.hbase.filter.TimestampsFilter;
041import org.apache.hadoop.hbase.testclassification.RegionServerTests;
042import org.apache.hadoop.hbase.testclassification.SmallTests;
043import org.apache.hadoop.hbase.util.Bytes;
044import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
045import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
046import org.junit.Assert;
047import org.junit.ClassRule;
048import org.junit.Rule;
049import org.junit.Test;
050import org.junit.experimental.categories.Category;
051import org.junit.rules.TestName;
052
053/**
054 * Test Minimum Versions feature (HBASE-4071).
055 */
056@Category({RegionServerTests.class, SmallTests.class})
057public class TestMinVersions {
058
059  @ClassRule
060  public static final HBaseClassTestRule CLASS_RULE =
061      HBaseClassTestRule.forClass(TestMinVersions.class);
062
063  HBaseTestingUtility hbu = new HBaseTestingUtility();
064  private final byte[] T0 = Bytes.toBytes("0");
065  private final byte[] T1 = Bytes.toBytes("1");
066  private final byte[] T2 = Bytes.toBytes("2");
067  private final byte[] T3 = Bytes.toBytes("3");
068  private final byte[] T4 = Bytes.toBytes("4");
069  private final byte[] T5 = Bytes.toBytes("5");
070
071  private final byte[] c0 = COLUMNS[0];
072
073  @Rule public TestName name = new TestName();
074
075  /**
076   * Verify behavior of getClosestBefore(...)
077   */
078  @Test
079  public void testGetClosestBefore() throws Exception {
080
081    ColumnFamilyDescriptor cfd =
082      ColumnFamilyDescriptorBuilder.newBuilder(c0)
083      .setMinVersions(1).setMaxVersions(1000).setTimeToLive(1).
084        setKeepDeletedCells(KeepDeletedCells.FALSE).build();
085
086    TableDescriptor htd = TableDescriptorBuilder.
087      newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build();
088    HRegion region = hbu.createLocalHRegion(htd, null, null);
089    try {
090
091      // 2s in the past
092      long ts = EnvironmentEdgeManager.currentTime() - 2000;
093
094      Put p = new Put(T1, ts);
095      p.addColumn(c0, c0, T1);
096      region.put(p);
097
098      p = new Put(T1, ts+1);
099      p.addColumn(c0, c0, T4);
100      region.put(p);
101
102      p = new Put(T3, ts);
103      p.addColumn(c0, c0, T3);
104      region.put(p);
105
106      // now make sure that getClosestBefore(...) get can
107      // rows that would be expired without minVersion.
108      // also make sure it gets the latest version
109      Result r = hbu.getClosestRowBefore(region, T1, c0);
110      checkResult(r, c0, T4);
111
112      r = hbu.getClosestRowBefore(region, T2, c0);
113      checkResult(r, c0, T4);
114
115      // now flush/compact
116      region.flush(true);
117      region.compact(true);
118
119      r = hbu.getClosestRowBefore(region, T1, c0);
120      checkResult(r, c0, T4);
121
122      r = hbu.getClosestRowBefore(region, T2, c0);
123      checkResult(r, c0, T4);
124    } finally {
125      HBaseTestingUtility.closeRegionAndWAL(region);
126    }
127  }
128
129  /**
130   * Test mixed memstore and storefile scanning
131   * with minimum versions.
132   */
133  @Test
134  public void testStoreMemStore() throws Exception {
135    // keep 3 versions minimum
136
137    ColumnFamilyDescriptor cfd =
138      ColumnFamilyDescriptorBuilder.newBuilder(c0)
139        .setMinVersions(3).setMaxVersions(1000).setTimeToLive(1).
140        setKeepDeletedCells(KeepDeletedCells.FALSE).build();
141
142    TableDescriptor htd = TableDescriptorBuilder.
143      newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build();
144
145    HRegion region = hbu.createLocalHRegion(htd, null, null);
146    // 2s in the past
147    long ts = EnvironmentEdgeManager.currentTime() - 2000;
148
149    try {
150      Put p = new Put(T1, ts-1);
151      p.addColumn(c0, c0, T2);
152      region.put(p);
153
154      p = new Put(T1, ts-3);
155      p.addColumn(c0, c0, T0);
156      region.put(p);
157
158      // now flush/compact
159      region.flush(true);
160      region.compact(true);
161
162      p = new Put(T1, ts);
163      p.addColumn(c0, c0, T3);
164      region.put(p);
165
166      p = new Put(T1, ts-2);
167      p.addColumn(c0, c0, T1);
168      region.put(p);
169
170      p = new Put(T1, ts-3);
171      p.addColumn(c0, c0, T0);
172      region.put(p);
173
174      // newest version in the memstore
175      // the 2nd oldest in the store file
176      // and the 3rd, 4th oldest also in the memstore
177
178      Get g = new Get(T1);
179      g.setMaxVersions();
180      Result r = region.get(g); // this'll use ScanWildcardColumnTracker
181      checkResult(r, c0, T3,T2,T1);
182
183      g = new Get(T1);
184      g.setMaxVersions();
185      g.addColumn(c0, c0);
186      r = region.get(g);  // this'll use ExplicitColumnTracker
187      checkResult(r, c0, T3,T2,T1);
188    } finally {
189      HBaseTestingUtility.closeRegionAndWAL(region);
190    }
191  }
192
193  /**
194   * Make sure the Deletes behave as expected with minimum versions
195   */
196  @Test
197  public void testDelete() throws Exception {
198    ColumnFamilyDescriptor cfd =
199      ColumnFamilyDescriptorBuilder.newBuilder(c0)
200        .setMinVersions(3).setMaxVersions(1000).setTimeToLive(1).
201        setKeepDeletedCells(KeepDeletedCells.FALSE).build();
202
203    TableDescriptor htd = TableDescriptorBuilder.
204      newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build();
205
206    HRegion region = hbu.createLocalHRegion(htd, null, null);
207
208    // 2s in the past
209    long ts = EnvironmentEdgeManager.currentTime() - 2000;
210
211    try {
212      Put p = new Put(T1, ts-2);
213      p.addColumn(c0, c0, T1);
214      region.put(p);
215
216      p = new Put(T1, ts-1);
217      p.addColumn(c0, c0, T2);
218      region.put(p);
219
220      p = new Put(T1, ts);
221      p.addColumn(c0, c0, T3);
222      region.put(p);
223
224      Delete d = new Delete(T1, ts-1);
225      region.delete(d);
226
227      Get g = new Get(T1);
228      g.setMaxVersions();
229      Result r = region.get(g);  // this'll use ScanWildcardColumnTracker
230      checkResult(r, c0, T3);
231
232      g = new Get(T1);
233      g.setMaxVersions();
234      g.addColumn(c0, c0);
235      r = region.get(g);  // this'll use ExplicitColumnTracker
236      checkResult(r, c0, T3);
237
238      // now flush/compact
239      region.flush(true);
240      region.compact(true);
241
242      // try again
243      g = new Get(T1);
244      g.setMaxVersions();
245      r = region.get(g);  // this'll use ScanWildcardColumnTracker
246      checkResult(r, c0, T3);
247
248      g = new Get(T1);
249      g.setMaxVersions();
250      g.addColumn(c0, c0);
251      r = region.get(g);  // this'll use ExplicitColumnTracker
252      checkResult(r, c0, T3);
253    } finally {
254      HBaseTestingUtility.closeRegionAndWAL(region);
255    }
256  }
257
258  /**
259   * Make sure the memstor behaves correctly with minimum versions
260   */
261  @Test
262  public void testMemStore() throws Exception {
263    ColumnFamilyDescriptor cfd =
264      ColumnFamilyDescriptorBuilder.newBuilder(c0)
265        .setMinVersions(2).setMaxVersions(1000).setTimeToLive(1).
266        setKeepDeletedCells(KeepDeletedCells.FALSE).build();
267
268    TableDescriptor htd = TableDescriptorBuilder.
269      newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build();
270    HRegion region = hbu.createLocalHRegion(htd, null, null);
271
272    // 2s in the past
273    long ts = EnvironmentEdgeManager.currentTime() - 2000;
274
275    try {
276      // 2nd version
277      Put p = new Put(T1, ts-2);
278      p.addColumn(c0, c0, T2);
279      region.put(p);
280
281      // 3rd version
282      p = new Put(T1, ts-1);
283      p.addColumn(c0, c0, T3);
284      region.put(p);
285
286      // 4th version
287      p = new Put(T1, ts);
288      p.addColumn(c0, c0, T4);
289      region.put(p);
290
291      // now flush/compact
292      region.flush(true);
293      region.compact(true);
294
295      // now put the first version (backdated)
296      p = new Put(T1, ts-3);
297      p.addColumn(c0, c0, T1);
298      region.put(p);
299
300      // now the latest change is in the memstore,
301      // but it is not the latest version
302
303      Result r = region.get(new Get(T1));
304      checkResult(r, c0, T4);
305
306      Get g = new Get(T1);
307      g.setMaxVersions();
308      r = region.get(g); // this'll use ScanWildcardColumnTracker
309      checkResult(r, c0, T4,T3);
310
311      g = new Get(T1);
312      g.setMaxVersions();
313      g.addColumn(c0, c0);
314      r = region.get(g);  // this'll use ExplicitColumnTracker
315      checkResult(r, c0, T4,T3);
316
317      p = new Put(T1, ts+1);
318      p.addColumn(c0, c0, T5);
319      region.put(p);
320
321      // now the latest version is in the memstore
322
323      g = new Get(T1);
324      g.setMaxVersions();
325      r = region.get(g);  // this'll use ScanWildcardColumnTracker
326      checkResult(r, c0, T5,T4);
327
328      g = new Get(T1);
329      g.setMaxVersions();
330      g.addColumn(c0, c0);
331      r = region.get(g);  // this'll use ExplicitColumnTracker
332      checkResult(r, c0, T5,T4);
333    } finally {
334      HBaseTestingUtility.closeRegionAndWAL(region);
335    }
336  }
337
338  /**
339   * Verify basic minimum versions functionality
340   */
341  @Test
342  public void testBaseCase() throws Exception {
343    // 2 version minimum, 1000 versions maximum, ttl = 1s
344    ColumnFamilyDescriptor cfd =
345      ColumnFamilyDescriptorBuilder.newBuilder(c0)
346        .setMinVersions(2).setMaxVersions(1000).setTimeToLive(1).
347        setKeepDeletedCells(KeepDeletedCells.FALSE).build();
348
349    TableDescriptor htd = TableDescriptorBuilder.
350      newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build();
351    HRegion region = hbu.createLocalHRegion(htd, null, null);
352    try {
353
354      // 2s in the past
355      long ts = EnvironmentEdgeManager.currentTime() - 2000;
356
357       // 1st version
358      Put p = new Put(T1, ts-3);
359      p.addColumn(c0, c0, T1);
360      region.put(p);
361
362      // 2nd version
363      p = new Put(T1, ts-2);
364      p.addColumn(c0, c0, T2);
365      region.put(p);
366
367      // 3rd version
368      p = new Put(T1, ts-1);
369      p.addColumn(c0, c0, T3);
370      region.put(p);
371
372      // 4th version
373      p = new Put(T1, ts);
374      p.addColumn(c0, c0, T4);
375      region.put(p);
376
377      Result r = region.get(new Get(T1));
378      checkResult(r, c0, T4);
379
380      Get g = new Get(T1);
381      g.setTimeRange(0L, ts+1);
382      r = region.get(g);
383      checkResult(r, c0, T4);
384
385  // oldest version still exists
386      g.setTimeRange(0L, ts-2);
387      r = region.get(g);
388      checkResult(r, c0, T1);
389
390      // gets see only available versions
391      // even before compactions
392      g = new Get(T1);
393      g.setMaxVersions();
394      r = region.get(g); // this'll use ScanWildcardColumnTracker
395      checkResult(r, c0, T4,T3);
396
397      g = new Get(T1);
398      g.setMaxVersions();
399      g.addColumn(c0, c0);
400      r = region.get(g);  // this'll use ExplicitColumnTracker
401      checkResult(r, c0, T4,T3);
402
403      // now flush
404      region.flush(true);
405
406      // with HBASE-4241 a flush will eliminate the expired rows
407      g = new Get(T1);
408      g.setTimeRange(0L, ts-2);
409      r = region.get(g);
410      assertTrue(r.isEmpty());
411
412      // major compaction
413      region.compact(true);
414
415      // after compaction the 4th version is still available
416      g = new Get(T1);
417      g.setTimeRange(0L, ts+1);
418      r = region.get(g);
419      checkResult(r, c0, T4);
420
421      // so is the 3rd
422      g.setTimeRange(0L, ts);
423      r = region.get(g);
424      checkResult(r, c0, T3);
425
426      // but the 2nd and earlier versions are gone
427      g.setTimeRange(0L, ts-1);
428      r = region.get(g);
429      assertTrue(r.isEmpty());
430    } finally {
431      HBaseTestingUtility.closeRegionAndWAL(region);
432    }
433  }
434
435  /**
436   * Verify that basic filters still behave correctly with
437   * minimum versions enabled.
438   */
439  @Test
440  public void testFilters() throws Exception {
441    final byte [] c1 = COLUMNS[1];
442    ColumnFamilyDescriptor cfd =
443      ColumnFamilyDescriptorBuilder.newBuilder(c0)
444        .setMinVersions(2).setMaxVersions(1000).setTimeToLive(1).
445        setKeepDeletedCells(KeepDeletedCells.FALSE).build();
446
447    ColumnFamilyDescriptor cfd2 =
448      ColumnFamilyDescriptorBuilder.newBuilder(c1)
449        .setMinVersions(2).setMaxVersions(1000).setTimeToLive(1).
450        setKeepDeletedCells(KeepDeletedCells.FALSE).build();
451    List<ColumnFamilyDescriptor> cfdList = new ArrayList();
452    cfdList.add(cfd);
453    cfdList.add(cfd2);
454
455    TableDescriptor htd = TableDescriptorBuilder.
456      newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamilies(cfdList).build();
457    HRegion region = hbu.createLocalHRegion(htd, null, null);
458
459    // 2s in the past
460    long ts = EnvironmentEdgeManager.currentTime() - 2000;
461    try {
462
463      Put p = new Put(T1, ts-3);
464      p.addColumn(c0, c0, T0);
465      p.addColumn(c1, c1, T0);
466      region.put(p);
467
468      p = new Put(T1, ts-2);
469      p.addColumn(c0, c0, T1);
470      p.addColumn(c1, c1, T1);
471      region.put(p);
472
473      p = new Put(T1, ts-1);
474      p.addColumn(c0, c0, T2);
475      p.addColumn(c1, c1, T2);
476      region.put(p);
477
478      p = new Put(T1, ts);
479      p.addColumn(c0, c0, T3);
480      p.addColumn(c1, c1, T3);
481      region.put(p);
482
483      List<Long> tss = new ArrayList<>();
484      tss.add(ts-1);
485      tss.add(ts-2);
486
487      // Sholud only get T2, versions is 2, so T1 is gone from user view.
488      Get g = new Get(T1);
489      g.addColumn(c1,c1);
490      g.setFilter(new TimestampsFilter(tss));
491      g.setMaxVersions();
492      Result r = region.get(g);
493      checkResult(r, c1, T2);
494
495      // Sholud only get T2, versions is 2, so T1 is gone from user view.
496      g = new Get(T1);
497      g.addColumn(c0,c0);
498      g.setFilter(new TimestampsFilter(tss));
499      g.setMaxVersions();
500      r = region.get(g);
501      checkResult(r, c0, T2);
502
503      // now flush/compact
504      region.flush(true);
505      region.compact(true);
506
507      // After flush/compact, the result should be consistent with previous result
508      g = new Get(T1);
509      g.addColumn(c1,c1);
510      g.setFilter(new TimestampsFilter(tss));
511      g.setMaxVersions();
512      r = region.get(g);
513      checkResult(r, c1, T2);
514
515      // After flush/compact, the result should be consistent with previous result
516      g = new Get(T1);
517      g.addColumn(c0,c0);
518      g.setFilter(new TimestampsFilter(tss));
519      g.setMaxVersions();
520      r = region.get(g);
521      checkResult(r, c0, T2);
522    } finally {
523      HBaseTestingUtility.closeRegionAndWAL(region);
524    }
525  }
526
527  @Test
528  public void testMinVersionsWithKeepDeletedCellsTTL() throws Exception {
529    int ttl = 4;
530    ColumnFamilyDescriptor cfd =
531      ColumnFamilyDescriptorBuilder.newBuilder(c0)
532        .setMinVersions(2).setMaxVersions(Integer.MAX_VALUE).setTimeToLive(ttl).
533        setKeepDeletedCells(KeepDeletedCells.TTL).build();
534
535    TableDescriptor htd = TableDescriptorBuilder.
536      newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build();
537
538    HRegion region = hbu.createLocalHRegion(htd, null, null);
539
540    long startTS = EnvironmentEdgeManager.currentTime();
541    ManualEnvironmentEdge injectEdge = new ManualEnvironmentEdge();
542    injectEdge.setValue(startTS);
543    EnvironmentEdgeManager.injectEdge(injectEdge);
544
545    long ts = startTS - 2000;
546    // 1st version
547    Put p = new Put(T1, ts-3);
548    p.addColumn(c0, c0, T1);
549    region.put(p);
550
551    // 2nd version
552    p = new Put(T1, ts-2);
553    p.addColumn(c0, c0, T2);
554    region.put(p);
555
556    // 3rd version
557    p = new Put(T1, ts-1);
558    p.addColumn(c0, c0, T3);
559    region.put(p);
560
561    Get g;
562    Result r;
563
564    //check we can still see all versions before compaction
565    g = new Get(T1);
566    g.readAllVersions();
567    g.setTimeRange(0, ts);
568    r = region.get(g);
569    checkResult(r, c0, T3, T2, T1);
570
571    region.flush(true);
572    region.compact(true);
573    Assert.assertEquals(startTS, EnvironmentEdgeManager.currentTime());
574    long expiredTime = EnvironmentEdgeManager.currentTime() - ts - 3;
575    Assert.assertTrue("TTL for T1 has expired", expiredTime < (ttl * 1000));
576    //check that nothing was purged yet
577    g = new Get(T1);
578    g.readAllVersions();
579    g.setTimeRange(0, ts);
580    r = region.get(g);
581    checkResult(r, c0, T3, T2, T1);
582
583    g = new Get(T1);
584    g.readAllVersions();
585    g.setTimeRange(0, ts -1);
586    r = region.get(g);
587    checkResult(r, c0, T2, T1);
588
589    injectEdge.incValue(ttl * 1000);
590
591    region.flush(true);
592    region.compact(true);
593
594    //check that after compaction (which is after TTL) that only T1 was purged
595    g = new Get(T1);
596    g.readAllVersions();
597    g.setTimeRange(0, ts);
598    r = region.get(g);
599    checkResult(r, c0, T3, T2);
600
601    g = new Get(T1);
602    g.readAllVersions();
603    g.setTimestamp(ts -2);
604    r = region.get(g);
605    checkResult(r, c0, T2);
606  }
607
608  private void checkResult(Result r, byte[] col, byte[] ... vals) {
609    assertEquals(vals.length, r.size());
610    List<Cell> kvs = r.getColumnCells(col, col);
611    assertEquals(kvs.size(), vals.length);
612    for (int i=0;i<vals.length;i++) {
613      String expected = Bytes.toString(vals[i]);
614      String actual = Bytes.toString(CellUtil.cloneValue(kvs.get(i)));
615      assertTrue(expected + " was expected but doesn't match " + actual,
616          CellUtil.matchingValue(kvs.get(i), vals[i]));
617    }
618  }
619
620}
621