001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.apache.hadoop.hbase.HBaseTestingUtility.COLUMNS;
021import static org.junit.Assert.assertEquals;
022import static org.junit.Assert.assertTrue;
023
024import java.util.ArrayList;
025import java.util.List;
026import org.apache.hadoop.hbase.Cell;
027import org.apache.hadoop.hbase.CellUtil;
028import org.apache.hadoop.hbase.HBaseClassTestRule;
029import org.apache.hadoop.hbase.HBaseTestingUtility;
030import org.apache.hadoop.hbase.HTableDescriptor;
031import org.apache.hadoop.hbase.KeepDeletedCells;
032import org.apache.hadoop.hbase.client.Delete;
033import org.apache.hadoop.hbase.client.Get;
034import org.apache.hadoop.hbase.client.Put;
035import org.apache.hadoop.hbase.client.Result;
036import org.apache.hadoop.hbase.filter.TimestampsFilter;
037import org.apache.hadoop.hbase.testclassification.RegionServerTests;
038import org.apache.hadoop.hbase.testclassification.SmallTests;
039import org.apache.hadoop.hbase.util.Bytes;
040import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
041import org.junit.ClassRule;
042import org.junit.Rule;
043import org.junit.Test;
044import org.junit.experimental.categories.Category;
045import org.junit.rules.TestName;
046
047/**
048 * Test Minimum Versions feature (HBASE-4071).
049 */
050@Category({RegionServerTests.class, SmallTests.class})
051public class TestMinVersions {
052
053  @ClassRule
054  public static final HBaseClassTestRule CLASS_RULE =
055      HBaseClassTestRule.forClass(TestMinVersions.class);
056
057  HBaseTestingUtility hbu = HBaseTestingUtility.createLocalHTU();
058  private final byte[] T0 = Bytes.toBytes("0");
059  private final byte[] T1 = Bytes.toBytes("1");
060  private final byte[] T2 = Bytes.toBytes("2");
061  private final byte[] T3 = Bytes.toBytes("3");
062  private final byte[] T4 = Bytes.toBytes("4");
063  private final byte[] T5 = Bytes.toBytes("5");
064
065  private final byte[] c0 = COLUMNS[0];
066
067  @Rule public TestName name = new TestName();
068
069  /**
070   * Verify behavior of getClosestBefore(...)
071   */
072  @Test
073  public void testGetClosestBefore() throws Exception {
074    HTableDescriptor htd =
075        hbu.createTableDescriptor(name.getMethodName(), 1, 1000, 1, KeepDeletedCells.FALSE);
076    HRegion region = hbu.createLocalHRegion(htd, null, null);
077    try {
078
079      // 2s in the past
080      long ts = EnvironmentEdgeManager.currentTime() - 2000;
081
082      Put p = new Put(T1, ts);
083      p.addColumn(c0, c0, T1);
084      region.put(p);
085
086      p = new Put(T1, ts+1);
087      p.addColumn(c0, c0, T4);
088      region.put(p);
089
090      p = new Put(T3, ts);
091      p.addColumn(c0, c0, T3);
092      region.put(p);
093
094      // now make sure that getClosestBefore(...) get can
095      // rows that would be expired without minVersion.
096      // also make sure it gets the latest version
097      Result r = hbu.getClosestRowBefore(region, T1, c0);
098      checkResult(r, c0, T4);
099
100      r = hbu.getClosestRowBefore(region, T2, c0);
101      checkResult(r, c0, T4);
102
103      // now flush/compact
104      region.flush(true);
105      region.compact(true);
106
107      r = hbu.getClosestRowBefore(region, T1, c0);
108      checkResult(r, c0, T4);
109
110      r = hbu.getClosestRowBefore(region, T2, c0);
111      checkResult(r, c0, T4);
112    } finally {
113      HBaseTestingUtility.closeRegionAndWAL(region);
114    }
115  }
116
117  /**
118   * Test mixed memstore and storefile scanning
119   * with minimum versions.
120   */
121  @Test
122  public void testStoreMemStore() throws Exception {
123    // keep 3 versions minimum
124    HTableDescriptor htd =
125        hbu.createTableDescriptor(name.getMethodName(), 3, 1000, 1, KeepDeletedCells.FALSE);
126    HRegion region = hbu.createLocalHRegion(htd, null, null);
127    // 2s in the past
128    long ts = EnvironmentEdgeManager.currentTime() - 2000;
129
130    try {
131      Put p = new Put(T1, ts-1);
132      p.addColumn(c0, c0, T2);
133      region.put(p);
134
135      p = new Put(T1, ts-3);
136      p.addColumn(c0, c0, T0);
137      region.put(p);
138
139      // now flush/compact
140      region.flush(true);
141      region.compact(true);
142
143      p = new Put(T1, ts);
144      p.addColumn(c0, c0, T3);
145      region.put(p);
146
147      p = new Put(T1, ts-2);
148      p.addColumn(c0, c0, T1);
149      region.put(p);
150
151      p = new Put(T1, ts-3);
152      p.addColumn(c0, c0, T0);
153      region.put(p);
154
155      // newest version in the memstore
156      // the 2nd oldest in the store file
157      // and the 3rd, 4th oldest also in the memstore
158
159      Get g = new Get(T1);
160      g.setMaxVersions();
161      Result r = region.get(g); // this'll use ScanWildcardColumnTracker
162      checkResult(r, c0, T3,T2,T1);
163
164      g = new Get(T1);
165      g.setMaxVersions();
166      g.addColumn(c0, c0);
167      r = region.get(g);  // this'll use ExplicitColumnTracker
168      checkResult(r, c0, T3,T2,T1);
169    } finally {
170      HBaseTestingUtility.closeRegionAndWAL(region);
171    }
172  }
173
174  /**
175   * Make sure the Deletes behave as expected with minimum versions
176   */
177  @Test
178  public void testDelete() throws Exception {
179    HTableDescriptor htd =
180        hbu.createTableDescriptor(name.getMethodName(), 3, 1000, 1, KeepDeletedCells.FALSE);
181    HRegion region = hbu.createLocalHRegion(htd, null, null);
182
183    // 2s in the past
184    long ts = EnvironmentEdgeManager.currentTime() - 2000;
185
186    try {
187      Put p = new Put(T1, ts-2);
188      p.addColumn(c0, c0, T1);
189      region.put(p);
190
191      p = new Put(T1, ts-1);
192      p.addColumn(c0, c0, T2);
193      region.put(p);
194
195      p = new Put(T1, ts);
196      p.addColumn(c0, c0, T3);
197      region.put(p);
198
199      Delete d = new Delete(T1, ts-1);
200      region.delete(d);
201
202      Get g = new Get(T1);
203      g.setMaxVersions();
204      Result r = region.get(g);  // this'll use ScanWildcardColumnTracker
205      checkResult(r, c0, T3);
206
207      g = new Get(T1);
208      g.setMaxVersions();
209      g.addColumn(c0, c0);
210      r = region.get(g);  // this'll use ExplicitColumnTracker
211      checkResult(r, c0, T3);
212
213      // now flush/compact
214      region.flush(true);
215      region.compact(true);
216
217      // try again
218      g = new Get(T1);
219      g.setMaxVersions();
220      r = region.get(g);  // this'll use ScanWildcardColumnTracker
221      checkResult(r, c0, T3);
222
223      g = new Get(T1);
224      g.setMaxVersions();
225      g.addColumn(c0, c0);
226      r = region.get(g);  // this'll use ExplicitColumnTracker
227      checkResult(r, c0, T3);
228    } finally {
229      HBaseTestingUtility.closeRegionAndWAL(region);
230    }
231  }
232
233  /**
234   * Make sure the memstor behaves correctly with minimum versions
235   */
236  @Test
237  public void testMemStore() throws Exception {
238    HTableDescriptor htd =
239        hbu.createTableDescriptor(name.getMethodName(), 2, 1000, 1, KeepDeletedCells.FALSE);
240    HRegion region = hbu.createLocalHRegion(htd, null, null);
241
242    // 2s in the past
243    long ts = EnvironmentEdgeManager.currentTime() - 2000;
244
245    try {
246      // 2nd version
247      Put p = new Put(T1, ts-2);
248      p.addColumn(c0, c0, T2);
249      region.put(p);
250
251      // 3rd version
252      p = new Put(T1, ts-1);
253      p.addColumn(c0, c0, T3);
254      region.put(p);
255
256      // 4th version
257      p = new Put(T1, ts);
258      p.addColumn(c0, c0, T4);
259      region.put(p);
260
261      // now flush/compact
262      region.flush(true);
263      region.compact(true);
264
265      // now put the first version (backdated)
266      p = new Put(T1, ts-3);
267      p.addColumn(c0, c0, T1);
268      region.put(p);
269
270      // now the latest change is in the memstore,
271      // but it is not the latest version
272
273      Result r = region.get(new Get(T1));
274      checkResult(r, c0, T4);
275
276      Get g = new Get(T1);
277      g.setMaxVersions();
278      r = region.get(g); // this'll use ScanWildcardColumnTracker
279      checkResult(r, c0, T4,T3);
280
281      g = new Get(T1);
282      g.setMaxVersions();
283      g.addColumn(c0, c0);
284      r = region.get(g);  // this'll use ExplicitColumnTracker
285      checkResult(r, c0, T4,T3);
286
287      p = new Put(T1, ts+1);
288      p.addColumn(c0, c0, T5);
289      region.put(p);
290
291      // now the latest version is in the memstore
292
293      g = new Get(T1);
294      g.setMaxVersions();
295      r = region.get(g);  // this'll use ScanWildcardColumnTracker
296      checkResult(r, c0, T5,T4);
297
298      g = new Get(T1);
299      g.setMaxVersions();
300      g.addColumn(c0, c0);
301      r = region.get(g);  // this'll use ExplicitColumnTracker
302      checkResult(r, c0, T5,T4);
303    } finally {
304      HBaseTestingUtility.closeRegionAndWAL(region);
305    }
306  }
307
308  /**
309   * Verify basic minimum versions functionality
310   */
311  @Test
312  public void testBaseCase() throws Exception {
313    // 1 version minimum, 1000 versions maximum, ttl = 1s
314    HTableDescriptor htd =
315        hbu.createTableDescriptor(name.getMethodName(), 2, 1000, 1, KeepDeletedCells.FALSE);
316    HRegion region = hbu.createLocalHRegion(htd, null, null);
317    try {
318
319      // 2s in the past
320      long ts = EnvironmentEdgeManager.currentTime() - 2000;
321
322       // 1st version
323      Put p = new Put(T1, ts-3);
324      p.addColumn(c0, c0, T1);
325      region.put(p);
326
327      // 2nd version
328      p = new Put(T1, ts-2);
329      p.addColumn(c0, c0, T2);
330      region.put(p);
331
332      // 3rd version
333      p = new Put(T1, ts-1);
334      p.addColumn(c0, c0, T3);
335      region.put(p);
336
337      // 4th version
338      p = new Put(T1, ts);
339      p.addColumn(c0, c0, T4);
340      region.put(p);
341
342      Result r = region.get(new Get(T1));
343      checkResult(r, c0, T4);
344
345      Get g = new Get(T1);
346      g.setTimeRange(0L, ts+1);
347      r = region.get(g);
348      checkResult(r, c0, T4);
349
350  // oldest version still exists
351      g.setTimeRange(0L, ts-2);
352      r = region.get(g);
353      checkResult(r, c0, T1);
354
355      // gets see only available versions
356      // even before compactions
357      g = new Get(T1);
358      g.setMaxVersions();
359      r = region.get(g); // this'll use ScanWildcardColumnTracker
360      checkResult(r, c0, T4,T3);
361
362      g = new Get(T1);
363      g.setMaxVersions();
364      g.addColumn(c0, c0);
365      r = region.get(g);  // this'll use ExplicitColumnTracker
366      checkResult(r, c0, T4,T3);
367
368      // now flush
369      region.flush(true);
370
371      // with HBASE-4241 a flush will eliminate the expired rows
372      g = new Get(T1);
373      g.setTimeRange(0L, ts-2);
374      r = region.get(g);
375      assertTrue(r.isEmpty());
376
377      // major compaction
378      region.compact(true);
379
380      // after compaction the 4th version is still available
381      g = new Get(T1);
382      g.setTimeRange(0L, ts+1);
383      r = region.get(g);
384      checkResult(r, c0, T4);
385
386      // so is the 3rd
387      g.setTimeRange(0L, ts);
388      r = region.get(g);
389      checkResult(r, c0, T3);
390
391      // but the 2nd and earlier versions are gone
392      g.setTimeRange(0L, ts-1);
393      r = region.get(g);
394      assertTrue(r.isEmpty());
395    } finally {
396      HBaseTestingUtility.closeRegionAndWAL(region);
397    }
398  }
399
400  /**
401   * Verify that basic filters still behave correctly with
402   * minimum versions enabled.
403   */
404  @Test
405  public void testFilters() throws Exception {
406    HTableDescriptor htd =
407        hbu.createTableDescriptor(name.getMethodName(), 2, 1000, 1, KeepDeletedCells.FALSE);
408    HRegion region = hbu.createLocalHRegion(htd, null, null);
409    final byte [] c1 = COLUMNS[1];
410
411    // 2s in the past
412    long ts = EnvironmentEdgeManager.currentTime() - 2000;
413    try {
414
415      Put p = new Put(T1, ts-3);
416      p.addColumn(c0, c0, T0);
417      p.addColumn(c1, c1, T0);
418      region.put(p);
419
420      p = new Put(T1, ts-2);
421      p.addColumn(c0, c0, T1);
422      p.addColumn(c1, c1, T1);
423      region.put(p);
424
425      p = new Put(T1, ts-1);
426      p.addColumn(c0, c0, T2);
427      p.addColumn(c1, c1, T2);
428      region.put(p);
429
430      p = new Put(T1, ts);
431      p.addColumn(c0, c0, T3);
432      p.addColumn(c1, c1, T3);
433      region.put(p);
434
435      List<Long> tss = new ArrayList<>();
436      tss.add(ts-1);
437      tss.add(ts-2);
438
439      // Sholud only get T2, versions is 2, so T1 is gone from user view.
440      Get g = new Get(T1);
441      g.addColumn(c1,c1);
442      g.setFilter(new TimestampsFilter(tss));
443      g.setMaxVersions();
444      Result r = region.get(g);
445      checkResult(r, c1, T2);
446
447      // Sholud only get T2, versions is 2, so T1 is gone from user view.
448      g = new Get(T1);
449      g.addColumn(c0,c0);
450      g.setFilter(new TimestampsFilter(tss));
451      g.setMaxVersions();
452      r = region.get(g);
453      checkResult(r, c0, T2);
454
455      // now flush/compact
456      region.flush(true);
457      region.compact(true);
458
459      // After flush/compact, the result should be consistent with previous result
460      g = new Get(T1);
461      g.addColumn(c1,c1);
462      g.setFilter(new TimestampsFilter(tss));
463      g.setMaxVersions();
464      r = region.get(g);
465      checkResult(r, c1, T2);
466
467      // After flush/compact, the result should be consistent with previous result
468      g = new Get(T1);
469      g.addColumn(c0,c0);
470      g.setFilter(new TimestampsFilter(tss));
471      g.setMaxVersions();
472      r = region.get(g);
473      checkResult(r, c0, T2);
474    } finally {
475      HBaseTestingUtility.closeRegionAndWAL(region);
476    }
477  }
478
479  private void checkResult(Result r, byte[] col, byte[] ... vals) {
480    assertEquals(r.size(), vals.length);
481    List<Cell> kvs = r.getColumnCells(col, col);
482    assertEquals(kvs.size(), vals.length);
483    for (int i=0;i<vals.length;i++) {
484      assertTrue(CellUtil.matchingValue(kvs.get(i), vals[i]));
485    }
486  }
487
488}
489