001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.regionserver; 019 020import static org.apache.hadoop.hbase.HBaseTestingUtility.COLUMNS; 021import static org.junit.Assert.assertEquals; 022import static org.junit.Assert.assertTrue; 023 024import java.util.ArrayList; 025import java.util.List; 026import org.apache.hadoop.hbase.Cell; 027import org.apache.hadoop.hbase.CellUtil; 028import org.apache.hadoop.hbase.HBaseClassTestRule; 029import org.apache.hadoop.hbase.HBaseTestingUtility; 030import org.apache.hadoop.hbase.KeepDeletedCells; 031import org.apache.hadoop.hbase.TableName; 032import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; 033import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; 034import org.apache.hadoop.hbase.client.Delete; 035import org.apache.hadoop.hbase.client.Get; 036import org.apache.hadoop.hbase.client.Put; 037import org.apache.hadoop.hbase.client.Result; 038import org.apache.hadoop.hbase.client.TableDescriptor; 039import org.apache.hadoop.hbase.client.TableDescriptorBuilder; 040import org.apache.hadoop.hbase.filter.TimestampsFilter; 041import org.apache.hadoop.hbase.testclassification.RegionServerTests; 042import org.apache.hadoop.hbase.testclassification.SmallTests; 043import org.apache.hadoop.hbase.util.Bytes; 044import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; 045import org.apache.hadoop.hbase.util.ManualEnvironmentEdge; 046import org.junit.Assert; 047import org.junit.ClassRule; 048import org.junit.Rule; 049import org.junit.Test; 050import org.junit.experimental.categories.Category; 051import org.junit.rules.TestName; 052 053/** 054 * Test Minimum Versions feature (HBASE-4071). 055 */ 056@Category({RegionServerTests.class, SmallTests.class}) 057public class TestMinVersions { 058 059 @ClassRule 060 public static final HBaseClassTestRule CLASS_RULE = 061 HBaseClassTestRule.forClass(TestMinVersions.class); 062 063 HBaseTestingUtility hbu = new HBaseTestingUtility(); 064 private final byte[] T0 = Bytes.toBytes("0"); 065 private final byte[] T1 = Bytes.toBytes("1"); 066 private final byte[] T2 = Bytes.toBytes("2"); 067 private final byte[] T3 = Bytes.toBytes("3"); 068 private final byte[] T4 = Bytes.toBytes("4"); 069 private final byte[] T5 = Bytes.toBytes("5"); 070 071 private final byte[] c0 = COLUMNS[0]; 072 073 @Rule public TestName name = new TestName(); 074 075 /** 076 * Verify behavior of getClosestBefore(...) 077 */ 078 @Test 079 public void testGetClosestBefore() throws Exception { 080 081 ColumnFamilyDescriptor cfd = 082 ColumnFamilyDescriptorBuilder.newBuilder(c0) 083 .setMinVersions(1).setMaxVersions(1000).setTimeToLive(1). 084 setKeepDeletedCells(KeepDeletedCells.FALSE).build(); 085 086 TableDescriptor htd = TableDescriptorBuilder. 087 newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build(); 088 HRegion region = hbu.createLocalHRegion(htd, null, null); 089 try { 090 091 // 2s in the past 092 long ts = EnvironmentEdgeManager.currentTime() - 2000; 093 094 Put p = new Put(T1, ts); 095 p.addColumn(c0, c0, T1); 096 region.put(p); 097 098 p = new Put(T1, ts+1); 099 p.addColumn(c0, c0, T4); 100 region.put(p); 101 102 p = new Put(T3, ts); 103 p.addColumn(c0, c0, T3); 104 region.put(p); 105 106 // now make sure that getClosestBefore(...) get can 107 // rows that would be expired without minVersion. 108 // also make sure it gets the latest version 109 Result r = hbu.getClosestRowBefore(region, T1, c0); 110 checkResult(r, c0, T4); 111 112 r = hbu.getClosestRowBefore(region, T2, c0); 113 checkResult(r, c0, T4); 114 115 // now flush/compact 116 region.flush(true); 117 region.compact(true); 118 119 r = hbu.getClosestRowBefore(region, T1, c0); 120 checkResult(r, c0, T4); 121 122 r = hbu.getClosestRowBefore(region, T2, c0); 123 checkResult(r, c0, T4); 124 } finally { 125 HBaseTestingUtility.closeRegionAndWAL(region); 126 } 127 } 128 129 /** 130 * Test mixed memstore and storefile scanning 131 * with minimum versions. 132 */ 133 @Test 134 public void testStoreMemStore() throws Exception { 135 // keep 3 versions minimum 136 137 ColumnFamilyDescriptor cfd = 138 ColumnFamilyDescriptorBuilder.newBuilder(c0) 139 .setMinVersions(3).setMaxVersions(1000).setTimeToLive(1). 140 setKeepDeletedCells(KeepDeletedCells.FALSE).build(); 141 142 TableDescriptor htd = TableDescriptorBuilder. 143 newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build(); 144 145 HRegion region = hbu.createLocalHRegion(htd, null, null); 146 // 2s in the past 147 long ts = EnvironmentEdgeManager.currentTime() - 2000; 148 149 try { 150 Put p = new Put(T1, ts-1); 151 p.addColumn(c0, c0, T2); 152 region.put(p); 153 154 p = new Put(T1, ts-3); 155 p.addColumn(c0, c0, T0); 156 region.put(p); 157 158 // now flush/compact 159 region.flush(true); 160 region.compact(true); 161 162 p = new Put(T1, ts); 163 p.addColumn(c0, c0, T3); 164 region.put(p); 165 166 p = new Put(T1, ts-2); 167 p.addColumn(c0, c0, T1); 168 region.put(p); 169 170 p = new Put(T1, ts-3); 171 p.addColumn(c0, c0, T0); 172 region.put(p); 173 174 // newest version in the memstore 175 // the 2nd oldest in the store file 176 // and the 3rd, 4th oldest also in the memstore 177 178 Get g = new Get(T1); 179 g.setMaxVersions(); 180 Result r = region.get(g); // this'll use ScanWildcardColumnTracker 181 checkResult(r, c0, T3,T2,T1); 182 183 g = new Get(T1); 184 g.setMaxVersions(); 185 g.addColumn(c0, c0); 186 r = region.get(g); // this'll use ExplicitColumnTracker 187 checkResult(r, c0, T3,T2,T1); 188 } finally { 189 HBaseTestingUtility.closeRegionAndWAL(region); 190 } 191 } 192 193 /** 194 * Make sure the Deletes behave as expected with minimum versions 195 */ 196 @Test 197 public void testDelete() throws Exception { 198 ColumnFamilyDescriptor cfd = 199 ColumnFamilyDescriptorBuilder.newBuilder(c0) 200 .setMinVersions(3).setMaxVersions(1000).setTimeToLive(1). 201 setKeepDeletedCells(KeepDeletedCells.FALSE).build(); 202 203 TableDescriptor htd = TableDescriptorBuilder. 204 newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build(); 205 206 HRegion region = hbu.createLocalHRegion(htd, null, null); 207 208 // 2s in the past 209 long ts = EnvironmentEdgeManager.currentTime() - 2000; 210 211 try { 212 Put p = new Put(T1, ts-2); 213 p.addColumn(c0, c0, T1); 214 region.put(p); 215 216 p = new Put(T1, ts-1); 217 p.addColumn(c0, c0, T2); 218 region.put(p); 219 220 p = new Put(T1, ts); 221 p.addColumn(c0, c0, T3); 222 region.put(p); 223 224 Delete d = new Delete(T1, ts-1); 225 region.delete(d); 226 227 Get g = new Get(T1); 228 g.setMaxVersions(); 229 Result r = region.get(g); // this'll use ScanWildcardColumnTracker 230 checkResult(r, c0, T3); 231 232 g = new Get(T1); 233 g.setMaxVersions(); 234 g.addColumn(c0, c0); 235 r = region.get(g); // this'll use ExplicitColumnTracker 236 checkResult(r, c0, T3); 237 238 // now flush/compact 239 region.flush(true); 240 region.compact(true); 241 242 // try again 243 g = new Get(T1); 244 g.setMaxVersions(); 245 r = region.get(g); // this'll use ScanWildcardColumnTracker 246 checkResult(r, c0, T3); 247 248 g = new Get(T1); 249 g.setMaxVersions(); 250 g.addColumn(c0, c0); 251 r = region.get(g); // this'll use ExplicitColumnTracker 252 checkResult(r, c0, T3); 253 } finally { 254 HBaseTestingUtility.closeRegionAndWAL(region); 255 } 256 } 257 258 /** 259 * Make sure the memstor behaves correctly with minimum versions 260 */ 261 @Test 262 public void testMemStore() throws Exception { 263 ColumnFamilyDescriptor cfd = 264 ColumnFamilyDescriptorBuilder.newBuilder(c0) 265 .setMinVersions(2).setMaxVersions(1000).setTimeToLive(1). 266 setKeepDeletedCells(KeepDeletedCells.FALSE).build(); 267 268 TableDescriptor htd = TableDescriptorBuilder. 269 newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build(); 270 HRegion region = hbu.createLocalHRegion(htd, null, null); 271 272 // 2s in the past 273 long ts = EnvironmentEdgeManager.currentTime() - 2000; 274 275 try { 276 // 2nd version 277 Put p = new Put(T1, ts-2); 278 p.addColumn(c0, c0, T2); 279 region.put(p); 280 281 // 3rd version 282 p = new Put(T1, ts-1); 283 p.addColumn(c0, c0, T3); 284 region.put(p); 285 286 // 4th version 287 p = new Put(T1, ts); 288 p.addColumn(c0, c0, T4); 289 region.put(p); 290 291 // now flush/compact 292 region.flush(true); 293 region.compact(true); 294 295 // now put the first version (backdated) 296 p = new Put(T1, ts-3); 297 p.addColumn(c0, c0, T1); 298 region.put(p); 299 300 // now the latest change is in the memstore, 301 // but it is not the latest version 302 303 Result r = region.get(new Get(T1)); 304 checkResult(r, c0, T4); 305 306 Get g = new Get(T1); 307 g.setMaxVersions(); 308 r = region.get(g); // this'll use ScanWildcardColumnTracker 309 checkResult(r, c0, T4,T3); 310 311 g = new Get(T1); 312 g.setMaxVersions(); 313 g.addColumn(c0, c0); 314 r = region.get(g); // this'll use ExplicitColumnTracker 315 checkResult(r, c0, T4,T3); 316 317 p = new Put(T1, ts+1); 318 p.addColumn(c0, c0, T5); 319 region.put(p); 320 321 // now the latest version is in the memstore 322 323 g = new Get(T1); 324 g.setMaxVersions(); 325 r = region.get(g); // this'll use ScanWildcardColumnTracker 326 checkResult(r, c0, T5,T4); 327 328 g = new Get(T1); 329 g.setMaxVersions(); 330 g.addColumn(c0, c0); 331 r = region.get(g); // this'll use ExplicitColumnTracker 332 checkResult(r, c0, T5,T4); 333 } finally { 334 HBaseTestingUtility.closeRegionAndWAL(region); 335 } 336 } 337 338 /** 339 * Verify basic minimum versions functionality 340 */ 341 @Test 342 public void testBaseCase() throws Exception { 343 // 2 version minimum, 1000 versions maximum, ttl = 1s 344 ColumnFamilyDescriptor cfd = 345 ColumnFamilyDescriptorBuilder.newBuilder(c0) 346 .setMinVersions(2).setMaxVersions(1000).setTimeToLive(1). 347 setKeepDeletedCells(KeepDeletedCells.FALSE).build(); 348 349 TableDescriptor htd = TableDescriptorBuilder. 350 newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build(); 351 HRegion region = hbu.createLocalHRegion(htd, null, null); 352 try { 353 354 // 2s in the past 355 long ts = EnvironmentEdgeManager.currentTime() - 2000; 356 357 // 1st version 358 Put p = new Put(T1, ts-3); 359 p.addColumn(c0, c0, T1); 360 region.put(p); 361 362 // 2nd version 363 p = new Put(T1, ts-2); 364 p.addColumn(c0, c0, T2); 365 region.put(p); 366 367 // 3rd version 368 p = new Put(T1, ts-1); 369 p.addColumn(c0, c0, T3); 370 region.put(p); 371 372 // 4th version 373 p = new Put(T1, ts); 374 p.addColumn(c0, c0, T4); 375 region.put(p); 376 377 Result r = region.get(new Get(T1)); 378 checkResult(r, c0, T4); 379 380 Get g = new Get(T1); 381 g.setTimeRange(0L, ts+1); 382 r = region.get(g); 383 checkResult(r, c0, T4); 384 385 // oldest version still exists 386 g.setTimeRange(0L, ts-2); 387 r = region.get(g); 388 checkResult(r, c0, T1); 389 390 // gets see only available versions 391 // even before compactions 392 g = new Get(T1); 393 g.setMaxVersions(); 394 r = region.get(g); // this'll use ScanWildcardColumnTracker 395 checkResult(r, c0, T4,T3); 396 397 g = new Get(T1); 398 g.setMaxVersions(); 399 g.addColumn(c0, c0); 400 r = region.get(g); // this'll use ExplicitColumnTracker 401 checkResult(r, c0, T4,T3); 402 403 // now flush 404 region.flush(true); 405 406 // with HBASE-4241 a flush will eliminate the expired rows 407 g = new Get(T1); 408 g.setTimeRange(0L, ts-2); 409 r = region.get(g); 410 assertTrue(r.isEmpty()); 411 412 // major compaction 413 region.compact(true); 414 415 // after compaction the 4th version is still available 416 g = new Get(T1); 417 g.setTimeRange(0L, ts+1); 418 r = region.get(g); 419 checkResult(r, c0, T4); 420 421 // so is the 3rd 422 g.setTimeRange(0L, ts); 423 r = region.get(g); 424 checkResult(r, c0, T3); 425 426 // but the 2nd and earlier versions are gone 427 g.setTimeRange(0L, ts-1); 428 r = region.get(g); 429 assertTrue(r.isEmpty()); 430 } finally { 431 HBaseTestingUtility.closeRegionAndWAL(region); 432 } 433 } 434 435 /** 436 * Verify that basic filters still behave correctly with 437 * minimum versions enabled. 438 */ 439 @Test 440 public void testFilters() throws Exception { 441 final byte [] c1 = COLUMNS[1]; 442 ColumnFamilyDescriptor cfd = 443 ColumnFamilyDescriptorBuilder.newBuilder(c0) 444 .setMinVersions(2).setMaxVersions(1000).setTimeToLive(1). 445 setKeepDeletedCells(KeepDeletedCells.FALSE).build(); 446 447 ColumnFamilyDescriptor cfd2 = 448 ColumnFamilyDescriptorBuilder.newBuilder(c1) 449 .setMinVersions(2).setMaxVersions(1000).setTimeToLive(1). 450 setKeepDeletedCells(KeepDeletedCells.FALSE).build(); 451 List<ColumnFamilyDescriptor> cfdList = new ArrayList(); 452 cfdList.add(cfd); 453 cfdList.add(cfd2); 454 455 TableDescriptor htd = TableDescriptorBuilder. 456 newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamilies(cfdList).build(); 457 HRegion region = hbu.createLocalHRegion(htd, null, null); 458 459 // 2s in the past 460 long ts = EnvironmentEdgeManager.currentTime() - 2000; 461 try { 462 463 Put p = new Put(T1, ts-3); 464 p.addColumn(c0, c0, T0); 465 p.addColumn(c1, c1, T0); 466 region.put(p); 467 468 p = new Put(T1, ts-2); 469 p.addColumn(c0, c0, T1); 470 p.addColumn(c1, c1, T1); 471 region.put(p); 472 473 p = new Put(T1, ts-1); 474 p.addColumn(c0, c0, T2); 475 p.addColumn(c1, c1, T2); 476 region.put(p); 477 478 p = new Put(T1, ts); 479 p.addColumn(c0, c0, T3); 480 p.addColumn(c1, c1, T3); 481 region.put(p); 482 483 List<Long> tss = new ArrayList<>(); 484 tss.add(ts-1); 485 tss.add(ts-2); 486 487 // Sholud only get T2, versions is 2, so T1 is gone from user view. 488 Get g = new Get(T1); 489 g.addColumn(c1,c1); 490 g.setFilter(new TimestampsFilter(tss)); 491 g.setMaxVersions(); 492 Result r = region.get(g); 493 checkResult(r, c1, T2); 494 495 // Sholud only get T2, versions is 2, so T1 is gone from user view. 496 g = new Get(T1); 497 g.addColumn(c0,c0); 498 g.setFilter(new TimestampsFilter(tss)); 499 g.setMaxVersions(); 500 r = region.get(g); 501 checkResult(r, c0, T2); 502 503 // now flush/compact 504 region.flush(true); 505 region.compact(true); 506 507 // After flush/compact, the result should be consistent with previous result 508 g = new Get(T1); 509 g.addColumn(c1,c1); 510 g.setFilter(new TimestampsFilter(tss)); 511 g.setMaxVersions(); 512 r = region.get(g); 513 checkResult(r, c1, T2); 514 515 // After flush/compact, the result should be consistent with previous result 516 g = new Get(T1); 517 g.addColumn(c0,c0); 518 g.setFilter(new TimestampsFilter(tss)); 519 g.setMaxVersions(); 520 r = region.get(g); 521 checkResult(r, c0, T2); 522 } finally { 523 HBaseTestingUtility.closeRegionAndWAL(region); 524 } 525 } 526 527 @Test 528 public void testMinVersionsWithKeepDeletedCellsTTL() throws Exception { 529 int ttl = 4; 530 ColumnFamilyDescriptor cfd = 531 ColumnFamilyDescriptorBuilder.newBuilder(c0) 532 .setMinVersions(2).setMaxVersions(Integer.MAX_VALUE).setTimeToLive(ttl). 533 setKeepDeletedCells(KeepDeletedCells.TTL).build(); 534 535 TableDescriptor htd = TableDescriptorBuilder. 536 newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build(); 537 538 HRegion region = hbu.createLocalHRegion(htd, null, null); 539 540 long startTS = EnvironmentEdgeManager.currentTime(); 541 ManualEnvironmentEdge injectEdge = new ManualEnvironmentEdge(); 542 injectEdge.setValue(startTS); 543 EnvironmentEdgeManager.injectEdge(injectEdge); 544 545 long ts = startTS - 2000; 546 // 1st version 547 Put p = new Put(T1, ts-3); 548 p.addColumn(c0, c0, T1); 549 region.put(p); 550 551 // 2nd version 552 p = new Put(T1, ts-2); 553 p.addColumn(c0, c0, T2); 554 region.put(p); 555 556 // 3rd version 557 p = new Put(T1, ts-1); 558 p.addColumn(c0, c0, T3); 559 region.put(p); 560 561 Get g; 562 Result r; 563 564 //check we can still see all versions before compaction 565 g = new Get(T1); 566 g.readAllVersions(); 567 g.setTimeRange(0, ts); 568 r = region.get(g); 569 checkResult(r, c0, T3, T2, T1); 570 571 region.flush(true); 572 region.compact(true); 573 Assert.assertEquals(startTS, EnvironmentEdgeManager.currentTime()); 574 long expiredTime = EnvironmentEdgeManager.currentTime() - ts - 3; 575 Assert.assertTrue("TTL for T1 has expired", expiredTime < (ttl * 1000)); 576 //check that nothing was purged yet 577 g = new Get(T1); 578 g.readAllVersions(); 579 g.setTimeRange(0, ts); 580 r = region.get(g); 581 checkResult(r, c0, T3, T2, T1); 582 583 g = new Get(T1); 584 g.readAllVersions(); 585 g.setTimeRange(0, ts -1); 586 r = region.get(g); 587 checkResult(r, c0, T2, T1); 588 589 injectEdge.incValue(ttl * 1000); 590 591 region.flush(true); 592 region.compact(true); 593 594 //check that after compaction (which is after TTL) that only T1 was purged 595 g = new Get(T1); 596 g.readAllVersions(); 597 g.setTimeRange(0, ts); 598 r = region.get(g); 599 checkResult(r, c0, T3, T2); 600 601 g = new Get(T1); 602 g.readAllVersions(); 603 g.setTimestamp(ts -2); 604 r = region.get(g); 605 checkResult(r, c0, T2); 606 } 607 608 private void checkResult(Result r, byte[] col, byte[] ... vals) { 609 assertEquals(vals.length, r.size()); 610 List<Cell> kvs = r.getColumnCells(col, col); 611 assertEquals(kvs.size(), vals.length); 612 for (int i=0;i<vals.length;i++) { 613 String expected = Bytes.toString(vals[i]); 614 String actual = Bytes.toString(CellUtil.cloneValue(kvs.get(i))); 615 assertTrue(expected + " was expected but doesn't match " + actual, 616 CellUtil.matchingValue(kvs.get(i), vals[i])); 617 } 618 } 619 620} 621