001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.regionserver; 019 020import static org.apache.hadoop.hbase.HBaseTestingUtility.COLUMNS; 021import static org.junit.Assert.assertEquals; 022import static org.junit.Assert.assertTrue; 023 024import java.util.ArrayList; 025import java.util.List; 026import org.apache.hadoop.hbase.Cell; 027import org.apache.hadoop.hbase.CellUtil; 028import org.apache.hadoop.hbase.HBaseClassTestRule; 029import org.apache.hadoop.hbase.HBaseTestingUtility; 030import org.apache.hadoop.hbase.HTableDescriptor; 031import org.apache.hadoop.hbase.KeepDeletedCells; 032import org.apache.hadoop.hbase.client.Delete; 033import org.apache.hadoop.hbase.client.Get; 034import org.apache.hadoop.hbase.client.Put; 035import org.apache.hadoop.hbase.client.Result; 036import org.apache.hadoop.hbase.filter.TimestampsFilter; 037import org.apache.hadoop.hbase.testclassification.RegionServerTests; 038import org.apache.hadoop.hbase.testclassification.SmallTests; 039import org.apache.hadoop.hbase.util.Bytes; 040import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; 041import org.junit.ClassRule; 042import org.junit.Rule; 043import org.junit.Test; 044import org.junit.experimental.categories.Category; 045import org.junit.rules.TestName; 046 047/** 048 * Test Minimum Versions feature (HBASE-4071). 049 */ 050@Category({RegionServerTests.class, SmallTests.class}) 051public class TestMinVersions { 052 053 @ClassRule 054 public static final HBaseClassTestRule CLASS_RULE = 055 HBaseClassTestRule.forClass(TestMinVersions.class); 056 057 HBaseTestingUtility hbu = HBaseTestingUtility.createLocalHTU(); 058 private final byte[] T0 = Bytes.toBytes("0"); 059 private final byte[] T1 = Bytes.toBytes("1"); 060 private final byte[] T2 = Bytes.toBytes("2"); 061 private final byte[] T3 = Bytes.toBytes("3"); 062 private final byte[] T4 = Bytes.toBytes("4"); 063 private final byte[] T5 = Bytes.toBytes("5"); 064 065 private final byte[] c0 = COLUMNS[0]; 066 067 @Rule public TestName name = new TestName(); 068 069 /** 070 * Verify behavior of getClosestBefore(...) 071 */ 072 @Test 073 public void testGetClosestBefore() throws Exception { 074 HTableDescriptor htd = 075 hbu.createTableDescriptor(name.getMethodName(), 1, 1000, 1, KeepDeletedCells.FALSE); 076 HRegion region = hbu.createLocalHRegion(htd, null, null); 077 try { 078 079 // 2s in the past 080 long ts = EnvironmentEdgeManager.currentTime() - 2000; 081 082 Put p = new Put(T1, ts); 083 p.addColumn(c0, c0, T1); 084 region.put(p); 085 086 p = new Put(T1, ts+1); 087 p.addColumn(c0, c0, T4); 088 region.put(p); 089 090 p = new Put(T3, ts); 091 p.addColumn(c0, c0, T3); 092 region.put(p); 093 094 // now make sure that getClosestBefore(...) get can 095 // rows that would be expired without minVersion. 096 // also make sure it gets the latest version 097 Result r = hbu.getClosestRowBefore(region, T1, c0); 098 checkResult(r, c0, T4); 099 100 r = hbu.getClosestRowBefore(region, T2, c0); 101 checkResult(r, c0, T4); 102 103 // now flush/compact 104 region.flush(true); 105 region.compact(true); 106 107 r = hbu.getClosestRowBefore(region, T1, c0); 108 checkResult(r, c0, T4); 109 110 r = hbu.getClosestRowBefore(region, T2, c0); 111 checkResult(r, c0, T4); 112 } finally { 113 HBaseTestingUtility.closeRegionAndWAL(region); 114 } 115 } 116 117 /** 118 * Test mixed memstore and storefile scanning 119 * with minimum versions. 120 */ 121 @Test 122 public void testStoreMemStore() throws Exception { 123 // keep 3 versions minimum 124 HTableDescriptor htd = 125 hbu.createTableDescriptor(name.getMethodName(), 3, 1000, 1, KeepDeletedCells.FALSE); 126 HRegion region = hbu.createLocalHRegion(htd, null, null); 127 // 2s in the past 128 long ts = EnvironmentEdgeManager.currentTime() - 2000; 129 130 try { 131 Put p = new Put(T1, ts-1); 132 p.addColumn(c0, c0, T2); 133 region.put(p); 134 135 p = new Put(T1, ts-3); 136 p.addColumn(c0, c0, T0); 137 region.put(p); 138 139 // now flush/compact 140 region.flush(true); 141 region.compact(true); 142 143 p = new Put(T1, ts); 144 p.addColumn(c0, c0, T3); 145 region.put(p); 146 147 p = new Put(T1, ts-2); 148 p.addColumn(c0, c0, T1); 149 region.put(p); 150 151 p = new Put(T1, ts-3); 152 p.addColumn(c0, c0, T0); 153 region.put(p); 154 155 // newest version in the memstore 156 // the 2nd oldest in the store file 157 // and the 3rd, 4th oldest also in the memstore 158 159 Get g = new Get(T1); 160 g.setMaxVersions(); 161 Result r = region.get(g); // this'll use ScanWildcardColumnTracker 162 checkResult(r, c0, T3,T2,T1); 163 164 g = new Get(T1); 165 g.setMaxVersions(); 166 g.addColumn(c0, c0); 167 r = region.get(g); // this'll use ExplicitColumnTracker 168 checkResult(r, c0, T3,T2,T1); 169 } finally { 170 HBaseTestingUtility.closeRegionAndWAL(region); 171 } 172 } 173 174 /** 175 * Make sure the Deletes behave as expected with minimum versions 176 */ 177 @Test 178 public void testDelete() throws Exception { 179 HTableDescriptor htd = 180 hbu.createTableDescriptor(name.getMethodName(), 3, 1000, 1, KeepDeletedCells.FALSE); 181 HRegion region = hbu.createLocalHRegion(htd, null, null); 182 183 // 2s in the past 184 long ts = EnvironmentEdgeManager.currentTime() - 2000; 185 186 try { 187 Put p = new Put(T1, ts-2); 188 p.addColumn(c0, c0, T1); 189 region.put(p); 190 191 p = new Put(T1, ts-1); 192 p.addColumn(c0, c0, T2); 193 region.put(p); 194 195 p = new Put(T1, ts); 196 p.addColumn(c0, c0, T3); 197 region.put(p); 198 199 Delete d = new Delete(T1, ts-1); 200 region.delete(d); 201 202 Get g = new Get(T1); 203 g.setMaxVersions(); 204 Result r = region.get(g); // this'll use ScanWildcardColumnTracker 205 checkResult(r, c0, T3); 206 207 g = new Get(T1); 208 g.setMaxVersions(); 209 g.addColumn(c0, c0); 210 r = region.get(g); // this'll use ExplicitColumnTracker 211 checkResult(r, c0, T3); 212 213 // now flush/compact 214 region.flush(true); 215 region.compact(true); 216 217 // try again 218 g = new Get(T1); 219 g.setMaxVersions(); 220 r = region.get(g); // this'll use ScanWildcardColumnTracker 221 checkResult(r, c0, T3); 222 223 g = new Get(T1); 224 g.setMaxVersions(); 225 g.addColumn(c0, c0); 226 r = region.get(g); // this'll use ExplicitColumnTracker 227 checkResult(r, c0, T3); 228 } finally { 229 HBaseTestingUtility.closeRegionAndWAL(region); 230 } 231 } 232 233 /** 234 * Make sure the memstor behaves correctly with minimum versions 235 */ 236 @Test 237 public void testMemStore() throws Exception { 238 HTableDescriptor htd = 239 hbu.createTableDescriptor(name.getMethodName(), 2, 1000, 1, KeepDeletedCells.FALSE); 240 HRegion region = hbu.createLocalHRegion(htd, null, null); 241 242 // 2s in the past 243 long ts = EnvironmentEdgeManager.currentTime() - 2000; 244 245 try { 246 // 2nd version 247 Put p = new Put(T1, ts-2); 248 p.addColumn(c0, c0, T2); 249 region.put(p); 250 251 // 3rd version 252 p = new Put(T1, ts-1); 253 p.addColumn(c0, c0, T3); 254 region.put(p); 255 256 // 4th version 257 p = new Put(T1, ts); 258 p.addColumn(c0, c0, T4); 259 region.put(p); 260 261 // now flush/compact 262 region.flush(true); 263 region.compact(true); 264 265 // now put the first version (backdated) 266 p = new Put(T1, ts-3); 267 p.addColumn(c0, c0, T1); 268 region.put(p); 269 270 // now the latest change is in the memstore, 271 // but it is not the latest version 272 273 Result r = region.get(new Get(T1)); 274 checkResult(r, c0, T4); 275 276 Get g = new Get(T1); 277 g.setMaxVersions(); 278 r = region.get(g); // this'll use ScanWildcardColumnTracker 279 checkResult(r, c0, T4,T3); 280 281 g = new Get(T1); 282 g.setMaxVersions(); 283 g.addColumn(c0, c0); 284 r = region.get(g); // this'll use ExplicitColumnTracker 285 checkResult(r, c0, T4,T3); 286 287 p = new Put(T1, ts+1); 288 p.addColumn(c0, c0, T5); 289 region.put(p); 290 291 // now the latest version is in the memstore 292 293 g = new Get(T1); 294 g.setMaxVersions(); 295 r = region.get(g); // this'll use ScanWildcardColumnTracker 296 checkResult(r, c0, T5,T4); 297 298 g = new Get(T1); 299 g.setMaxVersions(); 300 g.addColumn(c0, c0); 301 r = region.get(g); // this'll use ExplicitColumnTracker 302 checkResult(r, c0, T5,T4); 303 } finally { 304 HBaseTestingUtility.closeRegionAndWAL(region); 305 } 306 } 307 308 /** 309 * Verify basic minimum versions functionality 310 */ 311 @Test 312 public void testBaseCase() throws Exception { 313 // 1 version minimum, 1000 versions maximum, ttl = 1s 314 HTableDescriptor htd = 315 hbu.createTableDescriptor(name.getMethodName(), 2, 1000, 1, KeepDeletedCells.FALSE); 316 HRegion region = hbu.createLocalHRegion(htd, null, null); 317 try { 318 319 // 2s in the past 320 long ts = EnvironmentEdgeManager.currentTime() - 2000; 321 322 // 1st version 323 Put p = new Put(T1, ts-3); 324 p.addColumn(c0, c0, T1); 325 region.put(p); 326 327 // 2nd version 328 p = new Put(T1, ts-2); 329 p.addColumn(c0, c0, T2); 330 region.put(p); 331 332 // 3rd version 333 p = new Put(T1, ts-1); 334 p.addColumn(c0, c0, T3); 335 region.put(p); 336 337 // 4th version 338 p = new Put(T1, ts); 339 p.addColumn(c0, c0, T4); 340 region.put(p); 341 342 Result r = region.get(new Get(T1)); 343 checkResult(r, c0, T4); 344 345 Get g = new Get(T1); 346 g.setTimeRange(0L, ts+1); 347 r = region.get(g); 348 checkResult(r, c0, T4); 349 350 // oldest version still exists 351 g.setTimeRange(0L, ts-2); 352 r = region.get(g); 353 checkResult(r, c0, T1); 354 355 // gets see only available versions 356 // even before compactions 357 g = new Get(T1); 358 g.setMaxVersions(); 359 r = region.get(g); // this'll use ScanWildcardColumnTracker 360 checkResult(r, c0, T4,T3); 361 362 g = new Get(T1); 363 g.setMaxVersions(); 364 g.addColumn(c0, c0); 365 r = region.get(g); // this'll use ExplicitColumnTracker 366 checkResult(r, c0, T4,T3); 367 368 // now flush 369 region.flush(true); 370 371 // with HBASE-4241 a flush will eliminate the expired rows 372 g = new Get(T1); 373 g.setTimeRange(0L, ts-2); 374 r = region.get(g); 375 assertTrue(r.isEmpty()); 376 377 // major compaction 378 region.compact(true); 379 380 // after compaction the 4th version is still available 381 g = new Get(T1); 382 g.setTimeRange(0L, ts+1); 383 r = region.get(g); 384 checkResult(r, c0, T4); 385 386 // so is the 3rd 387 g.setTimeRange(0L, ts); 388 r = region.get(g); 389 checkResult(r, c0, T3); 390 391 // but the 2nd and earlier versions are gone 392 g.setTimeRange(0L, ts-1); 393 r = region.get(g); 394 assertTrue(r.isEmpty()); 395 } finally { 396 HBaseTestingUtility.closeRegionAndWAL(region); 397 } 398 } 399 400 /** 401 * Verify that basic filters still behave correctly with 402 * minimum versions enabled. 403 */ 404 @Test 405 public void testFilters() throws Exception { 406 HTableDescriptor htd = 407 hbu.createTableDescriptor(name.getMethodName(), 2, 1000, 1, KeepDeletedCells.FALSE); 408 HRegion region = hbu.createLocalHRegion(htd, null, null); 409 final byte [] c1 = COLUMNS[1]; 410 411 // 2s in the past 412 long ts = EnvironmentEdgeManager.currentTime() - 2000; 413 try { 414 415 Put p = new Put(T1, ts-3); 416 p.addColumn(c0, c0, T0); 417 p.addColumn(c1, c1, T0); 418 region.put(p); 419 420 p = new Put(T1, ts-2); 421 p.addColumn(c0, c0, T1); 422 p.addColumn(c1, c1, T1); 423 region.put(p); 424 425 p = new Put(T1, ts-1); 426 p.addColumn(c0, c0, T2); 427 p.addColumn(c1, c1, T2); 428 region.put(p); 429 430 p = new Put(T1, ts); 431 p.addColumn(c0, c0, T3); 432 p.addColumn(c1, c1, T3); 433 region.put(p); 434 435 List<Long> tss = new ArrayList<>(); 436 tss.add(ts-1); 437 tss.add(ts-2); 438 439 // Sholud only get T2, versions is 2, so T1 is gone from user view. 440 Get g = new Get(T1); 441 g.addColumn(c1,c1); 442 g.setFilter(new TimestampsFilter(tss)); 443 g.setMaxVersions(); 444 Result r = region.get(g); 445 checkResult(r, c1, T2); 446 447 // Sholud only get T2, versions is 2, so T1 is gone from user view. 448 g = new Get(T1); 449 g.addColumn(c0,c0); 450 g.setFilter(new TimestampsFilter(tss)); 451 g.setMaxVersions(); 452 r = region.get(g); 453 checkResult(r, c0, T2); 454 455 // now flush/compact 456 region.flush(true); 457 region.compact(true); 458 459 // After flush/compact, the result should be consistent with previous result 460 g = new Get(T1); 461 g.addColumn(c1,c1); 462 g.setFilter(new TimestampsFilter(tss)); 463 g.setMaxVersions(); 464 r = region.get(g); 465 checkResult(r, c1, T2); 466 467 // After flush/compact, the result should be consistent with previous result 468 g = new Get(T1); 469 g.addColumn(c0,c0); 470 g.setFilter(new TimestampsFilter(tss)); 471 g.setMaxVersions(); 472 r = region.get(g); 473 checkResult(r, c0, T2); 474 } finally { 475 HBaseTestingUtility.closeRegionAndWAL(region); 476 } 477 } 478 479 private void checkResult(Result r, byte[] col, byte[] ... vals) { 480 assertEquals(r.size(), vals.length); 481 List<Cell> kvs = r.getColumnCells(col, col); 482 assertEquals(kvs.size(), vals.length); 483 for (int i=0;i<vals.length;i++) { 484 assertTrue(CellUtil.matchingValue(kvs.get(i), vals[i])); 485 } 486 } 487 488} 489