001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.regionserver; 019 020import static org.apache.hadoop.hbase.regionserver.Store.PRIORITY_USER; 021 022import java.io.IOException; 023import java.security.Key; 024import java.security.SecureRandom; 025import java.util.ArrayList; 026import java.util.Collection; 027import java.util.Collections; 028import java.util.Date; 029import java.util.Iterator; 030import java.util.List; 031import java.util.NavigableSet; 032import java.util.Optional; 033import java.util.concurrent.ConcurrentSkipListSet; 034import javax.crypto.spec.SecretKeySpec; 035 036import org.apache.hadoop.conf.Configuration; 037import org.apache.hadoop.fs.FileSystem; 038import org.apache.hadoop.fs.Path; 039import org.apache.hadoop.hbase.ArrayBackedTag; 040import org.apache.hadoop.hbase.Cell; 041import org.apache.hadoop.hbase.CellComparatorImpl; 042import org.apache.hadoop.hbase.CellUtil; 043import org.apache.hadoop.hbase.HBaseClassTestRule; 044import org.apache.hadoop.hbase.HBaseConfiguration; 045import org.apache.hadoop.hbase.HBaseTestingUtility; 046import org.apache.hadoop.hbase.HConstants; 047import org.apache.hadoop.hbase.KeyValue; 048import org.apache.hadoop.hbase.TableName; 049import org.apache.hadoop.hbase.Tag; 050import org.apache.hadoop.hbase.TagType; 051import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; 052import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; 053import org.apache.hadoop.hbase.client.Get; 054import org.apache.hadoop.hbase.client.RegionInfo; 055import org.apache.hadoop.hbase.client.RegionInfoBuilder; 056import org.apache.hadoop.hbase.client.Scan; 057import org.apache.hadoop.hbase.client.TableDescriptor; 058import org.apache.hadoop.hbase.client.TableDescriptorBuilder; 059import org.apache.hadoop.hbase.io.crypto.KeyProviderForTesting; 060import org.apache.hadoop.hbase.io.crypto.aes.AES; 061import org.apache.hadoop.hbase.io.hfile.HFile; 062import org.apache.hadoop.hbase.mob.MobConstants; 063import org.apache.hadoop.hbase.mob.MobFileCache; 064import org.apache.hadoop.hbase.mob.MobUtils; 065import org.apache.hadoop.hbase.monitoring.MonitoredTask; 066import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext; 067import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker; 068import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController; 069import org.apache.hadoop.hbase.security.EncryptionUtil; 070import org.apache.hadoop.hbase.security.User; 071import org.apache.hadoop.hbase.testclassification.MediumTests; 072import org.apache.hadoop.hbase.util.Bytes; 073import org.apache.hadoop.hbase.util.FSUtils; 074import org.apache.hadoop.hbase.wal.WALFactory; 075import org.junit.Assert; 076import org.junit.Before; 077import org.junit.ClassRule; 078import org.junit.Rule; 079import org.junit.Test; 080import org.junit.experimental.categories.Category; 081import org.junit.rules.TestName; 082import org.mockito.Mockito; 083import org.slf4j.Logger; 084import org.slf4j.LoggerFactory; 085 086@Category(MediumTests.class) 087public class TestHMobStore { 088 089 @ClassRule 090 public static final HBaseClassTestRule CLASS_RULE = 091 HBaseClassTestRule.forClass(TestHMobStore.class); 092 093 public static final Logger LOG = LoggerFactory.getLogger(TestHMobStore.class); 094 @Rule public TestName name = new TestName(); 095 096 private HMobStore store; 097 private HRegion region; 098 private FileSystem fs; 099 private byte [] table = Bytes.toBytes("table"); 100 private byte [] family = Bytes.toBytes("family"); 101 private byte [] row = Bytes.toBytes("row"); 102 private byte [] row2 = Bytes.toBytes("row2"); 103 private byte [] qf1 = Bytes.toBytes("qf1"); 104 private byte [] qf2 = Bytes.toBytes("qf2"); 105 private byte [] qf3 = Bytes.toBytes("qf3"); 106 private byte [] qf4 = Bytes.toBytes("qf4"); 107 private byte [] qf5 = Bytes.toBytes("qf5"); 108 private byte [] qf6 = Bytes.toBytes("qf6"); 109 private byte[] value = Bytes.toBytes("value"); 110 private byte[] value2 = Bytes.toBytes("value2"); 111 private Path mobFilePath; 112 private Date currentDate = new Date(); 113 private Cell seekKey1; 114 private Cell seekKey2; 115 private Cell seekKey3; 116 private NavigableSet<byte[]> qualifiers = new ConcurrentSkipListSet<>(Bytes.BYTES_COMPARATOR); 117 private List<Cell> expected = new ArrayList<>(); 118 private long id = System.currentTimeMillis(); 119 private Get get = new Get(row); 120 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); 121 private final String DIR = TEST_UTIL.getDataTestDir("TestHMobStore").toString(); 122 123 /** 124 * Setup 125 * @throws Exception 126 */ 127 @Before 128 public void setUp() throws Exception { 129 qualifiers.add(qf1); 130 qualifiers.add(qf3); 131 qualifiers.add(qf5); 132 133 Iterator<byte[]> iter = qualifiers.iterator(); 134 while(iter.hasNext()){ 135 byte [] next = iter.next(); 136 expected.add(new KeyValue(row, family, next, 1, value)); 137 get.addColumn(family, next); 138 get.readAllVersions(); 139 } 140 } 141 142 private void init(String methodName, Configuration conf, boolean testStore) throws IOException { 143 ColumnFamilyDescriptor cfd = 144 ColumnFamilyDescriptorBuilder.newBuilder(family).setMobEnabled(true).setMobThreshold(3L) 145 .setMaxVersions(4).build(); 146 init(methodName, conf, cfd, testStore); 147 } 148 149 private void init(String methodName, Configuration conf, ColumnFamilyDescriptor cfd, 150 boolean testStore) throws IOException { 151 TableDescriptor td = 152 TableDescriptorBuilder.newBuilder(TableName.valueOf(table)).setColumnFamily(cfd).build(); 153 154 //Setting up tje Region and Store 155 Path basedir = new Path(DIR + methodName); 156 Path tableDir = FSUtils.getTableDir(basedir, td.getTableName()); 157 String logName = "logs"; 158 Path logdir = new Path(basedir, logName); 159 FileSystem fs = FileSystem.get(conf); 160 fs.delete(logdir, true); 161 162 RegionInfo info = RegionInfoBuilder.newBuilder(td.getTableName()).build(); 163 ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null); 164 final Configuration walConf = new Configuration(conf); 165 FSUtils.setRootDir(walConf, basedir); 166 final WALFactory wals = new WALFactory(walConf, methodName); 167 region = new HRegion(tableDir, wals.getWAL(info), fs, conf, info, td, null); 168 region.setMobFileCache(new MobFileCache(conf)); 169 store = new HMobStore(region, cfd, conf, false); 170 if (testStore) { 171 init(conf, cfd); 172 } 173 } 174 175 private void init(Configuration conf, ColumnFamilyDescriptor cfd) 176 throws IOException { 177 Path basedir = FSUtils.getRootDir(conf); 178 fs = FileSystem.get(conf); 179 Path homePath = new Path(basedir, Bytes.toString(family) + Path.SEPARATOR 180 + Bytes.toString(family)); 181 fs.mkdirs(homePath); 182 183 KeyValue key1 = new KeyValue(row, family, qf1, 1, value); 184 KeyValue key2 = new KeyValue(row, family, qf2, 1, value); 185 KeyValue key3 = new KeyValue(row2, family, qf3, 1, value2); 186 KeyValue[] keys = new KeyValue[] { key1, key2, key3 }; 187 int maxKeyCount = keys.length; 188 StoreFileWriter mobWriter = store.createWriterInTmp(currentDate, maxKeyCount, 189 cfd.getCompactionCompressionType(), region.getRegionInfo().getStartKey(), false); 190 mobFilePath = mobWriter.getPath(); 191 192 mobWriter.append(key1); 193 mobWriter.append(key2); 194 mobWriter.append(key3); 195 mobWriter.close(); 196 197 String targetPathName = MobUtils.formatDate(currentDate); 198 byte[] referenceValue = Bytes.toBytes(targetPathName + Path.SEPARATOR + mobFilePath.getName()); 199 Tag tableNameTag = new ArrayBackedTag(TagType.MOB_TABLE_NAME_TAG_TYPE, 200 store.getTableName().getName()); 201 KeyValue kv1 = new KeyValue(row, family, qf1, Long.MAX_VALUE, referenceValue); 202 KeyValue kv2 = new KeyValue(row, family, qf2, Long.MAX_VALUE, referenceValue); 203 KeyValue kv3 = new KeyValue(row2, family, qf3, Long.MAX_VALUE, referenceValue); 204 seekKey1 = MobUtils.createMobRefCell(kv1, referenceValue, tableNameTag); 205 seekKey2 = MobUtils.createMobRefCell(kv2, referenceValue, tableNameTag); 206 seekKey3 = MobUtils.createMobRefCell(kv3, referenceValue, tableNameTag); 207 } 208 209 /** 210 * Getting data from memstore 211 * @throws IOException 212 */ 213 @Test 214 public void testGetFromMemStore() throws IOException { 215 final Configuration conf = HBaseConfiguration.create(); 216 init(name.getMethodName(), conf, false); 217 218 //Put data in memstore 219 this.store.add(new KeyValue(row, family, qf1, 1, value), null); 220 this.store.add(new KeyValue(row, family, qf2, 1, value), null); 221 this.store.add(new KeyValue(row, family, qf3, 1, value), null); 222 this.store.add(new KeyValue(row, family, qf4, 1, value), null); 223 this.store.add(new KeyValue(row, family, qf5, 1, value), null); 224 this.store.add(new KeyValue(row, family, qf6, 1, value), null); 225 226 Scan scan = new Scan(get); 227 InternalScanner scanner = (InternalScanner) store.getScanner(scan, 228 scan.getFamilyMap().get(store.getColumnFamilyDescriptor().getName()), 229 0); 230 231 List<Cell> results = new ArrayList<>(); 232 scanner.next(results); 233 Collections.sort(results, CellComparatorImpl.COMPARATOR); 234 scanner.close(); 235 236 //Compare 237 Assert.assertEquals(expected.size(), results.size()); 238 for(int i=0; i<results.size(); i++) { 239 // Verify the values 240 Assert.assertEquals(expected.get(i), results.get(i)); 241 } 242 } 243 244 /** 245 * Getting MOB data from files 246 * @throws IOException 247 */ 248 @Test 249 public void testGetFromFiles() throws IOException { 250 final Configuration conf = TEST_UTIL.getConfiguration(); 251 init(name.getMethodName(), conf, false); 252 253 //Put data in memstore 254 this.store.add(new KeyValue(row, family, qf1, 1, value), null); 255 this.store.add(new KeyValue(row, family, qf2, 1, value), null); 256 //flush 257 flush(1); 258 259 //Add more data 260 this.store.add(new KeyValue(row, family, qf3, 1, value), null); 261 this.store.add(new KeyValue(row, family, qf4, 1, value), null); 262 //flush 263 flush(2); 264 265 //Add more data 266 this.store.add(new KeyValue(row, family, qf5, 1, value), null); 267 this.store.add(new KeyValue(row, family, qf6, 1, value), null); 268 //flush 269 flush(3); 270 271 Scan scan = new Scan(get); 272 InternalScanner scanner = (InternalScanner) store.getScanner(scan, 273 scan.getFamilyMap().get(store.getColumnFamilyDescriptor().getName()), 274 0); 275 276 List<Cell> results = new ArrayList<>(); 277 scanner.next(results); 278 Collections.sort(results, CellComparatorImpl.COMPARATOR); 279 scanner.close(); 280 281 //Compare 282 Assert.assertEquals(expected.size(), results.size()); 283 for(int i=0; i<results.size(); i++) { 284 Assert.assertEquals(expected.get(i), results.get(i)); 285 } 286 } 287 288 /** 289 * Getting the reference data from files 290 * @throws IOException 291 */ 292 @Test 293 public void testGetReferencesFromFiles() throws IOException { 294 final Configuration conf = HBaseConfiguration.create(); 295 init(name.getMethodName(), conf, false); 296 297 //Put data in memstore 298 this.store.add(new KeyValue(row, family, qf1, 1, value), null); 299 this.store.add(new KeyValue(row, family, qf2, 1, value), null); 300 //flush 301 flush(1); 302 303 //Add more data 304 this.store.add(new KeyValue(row, family, qf3, 1, value), null); 305 this.store.add(new KeyValue(row, family, qf4, 1, value), null); 306 //flush 307 flush(2); 308 309 //Add more data 310 this.store.add(new KeyValue(row, family, qf5, 1, value), null); 311 this.store.add(new KeyValue(row, family, qf6, 1, value), null); 312 //flush 313 flush(3); 314 315 Scan scan = new Scan(get); 316 scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE)); 317 InternalScanner scanner = (InternalScanner) store.getScanner(scan, 318 scan.getFamilyMap().get(store.getColumnFamilyDescriptor().getName()), 319 0); 320 321 List<Cell> results = new ArrayList<>(); 322 scanner.next(results); 323 Collections.sort(results, CellComparatorImpl.COMPARATOR); 324 scanner.close(); 325 326 //Compare 327 Assert.assertEquals(expected.size(), results.size()); 328 for(int i=0; i<results.size(); i++) { 329 Cell cell = results.get(i); 330 Assert.assertTrue(MobUtils.isMobReferenceCell(cell)); 331 } 332 } 333 334 /** 335 * Getting data from memstore and files 336 * @throws IOException 337 */ 338 @Test 339 public void testGetFromMemStoreAndFiles() throws IOException { 340 341 final Configuration conf = HBaseConfiguration.create(); 342 343 init(name.getMethodName(), conf, false); 344 345 //Put data in memstore 346 this.store.add(new KeyValue(row, family, qf1, 1, value), null); 347 this.store.add(new KeyValue(row, family, qf2, 1, value), null); 348 //flush 349 flush(1); 350 351 //Add more data 352 this.store.add(new KeyValue(row, family, qf3, 1, value), null); 353 this.store.add(new KeyValue(row, family, qf4, 1, value), null); 354 //flush 355 flush(2); 356 357 //Add more data 358 this.store.add(new KeyValue(row, family, qf5, 1, value), null); 359 this.store.add(new KeyValue(row, family, qf6, 1, value), null); 360 361 Scan scan = new Scan(get); 362 InternalScanner scanner = (InternalScanner) store.getScanner(scan, 363 scan.getFamilyMap().get(store.getColumnFamilyDescriptor().getName()), 364 0); 365 366 List<Cell> results = new ArrayList<>(); 367 scanner.next(results); 368 Collections.sort(results, CellComparatorImpl.COMPARATOR); 369 scanner.close(); 370 371 //Compare 372 Assert.assertEquals(expected.size(), results.size()); 373 for(int i=0; i<results.size(); i++) { 374 Assert.assertEquals(expected.get(i), results.get(i)); 375 } 376 } 377 378 /** 379 * Getting data from memstore and files 380 * @throws IOException 381 */ 382 @Test 383 public void testMobCellSizeThreshold() throws IOException { 384 final Configuration conf = HBaseConfiguration.create(); 385 ColumnFamilyDescriptor cfd = 386 ColumnFamilyDescriptorBuilder.newBuilder(family).setMobEnabled(true).setMobThreshold(100) 387 .setMaxVersions(4).build(); 388 init(name.getMethodName(), conf, cfd, false); 389 390 //Put data in memstore 391 this.store.add(new KeyValue(row, family, qf1, 1, value), null); 392 this.store.add(new KeyValue(row, family, qf2, 1, value), null); 393 //flush 394 flush(1); 395 396 //Add more data 397 this.store.add(new KeyValue(row, family, qf3, 1, value), null); 398 this.store.add(new KeyValue(row, family, qf4, 1, value), null); 399 //flush 400 flush(2); 401 402 //Add more data 403 this.store.add(new KeyValue(row, family, qf5, 1, value), null); 404 this.store.add(new KeyValue(row, family, qf6, 1, value), null); 405 //flush 406 flush(3); 407 408 Scan scan = new Scan(get); 409 scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE)); 410 InternalScanner scanner = (InternalScanner) store.getScanner(scan, 411 scan.getFamilyMap().get(store.getColumnFamilyDescriptor().getName()), 412 0); 413 414 List<Cell> results = new ArrayList<>(); 415 scanner.next(results); 416 Collections.sort(results, CellComparatorImpl.COMPARATOR); 417 scanner.close(); 418 419 //Compare 420 Assert.assertEquals(expected.size(), results.size()); 421 for(int i=0; i<results.size(); i++) { 422 Cell cell = results.get(i); 423 //this is not mob reference cell. 424 Assert.assertFalse(MobUtils.isMobReferenceCell(cell)); 425 Assert.assertEquals(expected.get(i), results.get(i)); 426 Assert.assertEquals(100, store.getColumnFamilyDescriptor().getMobThreshold()); 427 } 428 } 429 430 @Test 431 public void testCommitFile() throws Exception { 432 final Configuration conf = HBaseConfiguration.create(); 433 init(name.getMethodName(), conf, true); 434 String targetPathName = MobUtils.formatDate(new Date()); 435 Path targetPath = new Path(store.getPath(), (targetPathName 436 + Path.SEPARATOR + mobFilePath.getName())); 437 fs.delete(targetPath, true); 438 Assert.assertFalse(fs.exists(targetPath)); 439 //commit file 440 store.commitFile(mobFilePath, targetPath); 441 Assert.assertTrue(fs.exists(targetPath)); 442 } 443 444 @Test 445 public void testResolve() throws Exception { 446 final Configuration conf = HBaseConfiguration.create(); 447 init(name.getMethodName(), conf, true); 448 String targetPathName = MobUtils.formatDate(currentDate); 449 Path targetPath = new Path(store.getPath(), targetPathName); 450 store.commitFile(mobFilePath, targetPath); 451 //resolve 452 Cell resultCell1 = store.resolve(seekKey1, false); 453 Cell resultCell2 = store.resolve(seekKey2, false); 454 Cell resultCell3 = store.resolve(seekKey3, false); 455 //compare 456 Assert.assertEquals(Bytes.toString(value), 457 Bytes.toString(CellUtil.cloneValue(resultCell1))); 458 Assert.assertEquals(Bytes.toString(value), 459 Bytes.toString(CellUtil.cloneValue(resultCell2))); 460 Assert.assertEquals(Bytes.toString(value2), 461 Bytes.toString(CellUtil.cloneValue(resultCell3))); 462 } 463 464 /** 465 * Flush the memstore 466 * @param storeFilesSize 467 * @throws IOException 468 */ 469 private void flush(int storeFilesSize) throws IOException{ 470 this.store.snapshot(); 471 flushStore(store, id++); 472 Assert.assertEquals(storeFilesSize, this.store.getStorefiles().size()); 473 Assert.assertEquals(0, ((AbstractMemStore)this.store.memstore).getActive().getCellsCount()); 474 } 475 476 /** 477 * Flush the memstore 478 * @param store 479 * @param id 480 * @throws IOException 481 */ 482 private static void flushStore(HMobStore store, long id) throws IOException { 483 StoreFlushContext storeFlushCtx = store.createFlushContext(id, FlushLifeCycleTracker.DUMMY); 484 storeFlushCtx.prepare(); 485 storeFlushCtx.flushCache(Mockito.mock(MonitoredTask.class)); 486 storeFlushCtx.commit(Mockito.mock(MonitoredTask.class)); 487 } 488 489 @Test 490 public void testMOBStoreEncryption() throws Exception { 491 final Configuration conf = TEST_UTIL.getConfiguration(); 492 493 conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyProviderForTesting.class.getName()); 494 conf.set(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, "hbase"); 495 SecureRandom rng = new SecureRandom(); 496 byte[] keyBytes = new byte[AES.KEY_LENGTH]; 497 rng.nextBytes(keyBytes); 498 String algorithm = conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES); 499 Key cfKey = new SecretKeySpec(keyBytes, algorithm); 500 501 ColumnFamilyDescriptor cfd = 502 ColumnFamilyDescriptorBuilder.newBuilder(family).setMobEnabled(true).setMobThreshold(100) 503 .setMaxVersions(4).setEncryptionType(algorithm).setEncryptionKey(EncryptionUtil 504 .wrapKey(conf, conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, 505 User.getCurrent().getShortName()), cfKey)).build(); 506 init(name.getMethodName(), conf, cfd, false); 507 508 this.store.add(new KeyValue(row, family, qf1, 1, value), null); 509 this.store.add(new KeyValue(row, family, qf2, 1, value), null); 510 this.store.add(new KeyValue(row, family, qf3, 1, value), null); 511 flush(1); 512 513 this.store.add(new KeyValue(row, family, qf4, 1, value), null); 514 this.store.add(new KeyValue(row, family, qf5, 1, value), null); 515 this.store.add(new KeyValue(row, family, qf6, 1, value), null); 516 flush(2); 517 518 Collection<HStoreFile> storefiles = this.store.getStorefiles(); 519 checkMobHFileEncrytption(storefiles); 520 521 // Scan the values 522 Scan scan = new Scan(get); 523 InternalScanner scanner = (InternalScanner) store.getScanner(scan, 524 scan.getFamilyMap().get(store.getColumnFamilyDescriptor().getName()), 525 0); 526 527 List<Cell> results = new ArrayList<>(); 528 scanner.next(results); 529 Collections.sort(results, CellComparatorImpl.COMPARATOR); 530 scanner.close(); 531 Assert.assertEquals(expected.size(), results.size()); 532 for(int i=0; i<results.size(); i++) { 533 Assert.assertEquals(expected.get(i), results.get(i)); 534 } 535 536 // Trigger major compaction 537 this.store.triggerMajorCompaction(); 538 Optional<CompactionContext> requestCompaction = 539 this.store.requestCompaction(PRIORITY_USER, CompactionLifeCycleTracker.DUMMY, null); 540 this.store.compact(requestCompaction.get(), NoLimitThroughputController.INSTANCE, null); 541 Assert.assertEquals(1, this.store.getStorefiles().size()); 542 543 //Check encryption after compaction 544 checkMobHFileEncrytption(this.store.getStorefiles()); 545 } 546 547 private void checkMobHFileEncrytption(Collection<HStoreFile> storefiles) { 548 HStoreFile storeFile = storefiles.iterator().next(); 549 HFile.Reader reader = storeFile.getReader().getHFileReader(); 550 byte[] encryptionKey = reader.getTrailer().getEncryptionKey(); 551 Assert.assertTrue(null != encryptionKey); 552 Assert.assertTrue(reader.getFileContext().getEncryptionContext().getCipher().getName() 553 .equals(HConstants.CIPHER_AES)); 554 } 555 556}