001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.regionserver; 019 020import static org.apache.hadoop.hbase.regionserver.Store.PRIORITY_USER; 021 022import java.io.IOException; 023import java.security.Key; 024import java.security.SecureRandom; 025import java.util.ArrayList; 026import java.util.Collection; 027import java.util.Collections; 028import java.util.Date; 029import java.util.Iterator; 030import java.util.List; 031import java.util.NavigableSet; 032import java.util.Optional; 033import java.util.concurrent.ConcurrentSkipListSet; 034import javax.crypto.spec.SecretKeySpec; 035import org.apache.hadoop.conf.Configuration; 036import org.apache.hadoop.fs.FileSystem; 037import org.apache.hadoop.fs.Path; 038import org.apache.hadoop.hbase.ArrayBackedTag; 039import org.apache.hadoop.hbase.Cell; 040import org.apache.hadoop.hbase.CellComparatorImpl; 041import org.apache.hadoop.hbase.CellUtil; 042import org.apache.hadoop.hbase.HBaseClassTestRule; 043import org.apache.hadoop.hbase.HBaseConfiguration; 044import org.apache.hadoop.hbase.HBaseTestingUtility; 045import org.apache.hadoop.hbase.HConstants; 046import org.apache.hadoop.hbase.KeyValue; 047import org.apache.hadoop.hbase.TableName; 048import org.apache.hadoop.hbase.Tag; 049import org.apache.hadoop.hbase.TagType; 050import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; 051import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; 052import org.apache.hadoop.hbase.client.Get; 053import org.apache.hadoop.hbase.client.RegionInfo; 054import org.apache.hadoop.hbase.client.RegionInfoBuilder; 055import org.apache.hadoop.hbase.client.Scan; 056import org.apache.hadoop.hbase.client.TableDescriptor; 057import org.apache.hadoop.hbase.client.TableDescriptorBuilder; 058import org.apache.hadoop.hbase.io.crypto.KeyProviderForTesting; 059import org.apache.hadoop.hbase.io.crypto.aes.AES; 060import org.apache.hadoop.hbase.io.hfile.HFile; 061import org.apache.hadoop.hbase.mob.MobConstants; 062import org.apache.hadoop.hbase.mob.MobFileCache; 063import org.apache.hadoop.hbase.mob.MobUtils; 064import org.apache.hadoop.hbase.monitoring.MonitoredTask; 065import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext; 066import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker; 067import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController; 068import org.apache.hadoop.hbase.security.EncryptionUtil; 069import org.apache.hadoop.hbase.security.User; 070import org.apache.hadoop.hbase.testclassification.MediumTests; 071import org.apache.hadoop.hbase.util.Bytes; 072import org.apache.hadoop.hbase.util.CommonFSUtils; 073import org.apache.hadoop.hbase.wal.WALFactory; 074import org.junit.Assert; 075import org.junit.Before; 076import org.junit.ClassRule; 077import org.junit.Rule; 078import org.junit.Test; 079import org.junit.experimental.categories.Category; 080import org.junit.rules.TestName; 081import org.mockito.Mockito; 082import org.slf4j.Logger; 083import org.slf4j.LoggerFactory; 084 085@Category(MediumTests.class) 086public class TestHMobStore { 087 088 @ClassRule 089 public static final HBaseClassTestRule CLASS_RULE = 090 HBaseClassTestRule.forClass(TestHMobStore.class); 091 092 public static final Logger LOG = LoggerFactory.getLogger(TestHMobStore.class); 093 @Rule public TestName name = new TestName(); 094 095 private HMobStore store; 096 private HRegion region; 097 private FileSystem fs; 098 private byte [] table = Bytes.toBytes("table"); 099 private byte [] family = Bytes.toBytes("family"); 100 private byte [] row = Bytes.toBytes("row"); 101 private byte [] row2 = Bytes.toBytes("row2"); 102 private byte [] qf1 = Bytes.toBytes("qf1"); 103 private byte [] qf2 = Bytes.toBytes("qf2"); 104 private byte [] qf3 = Bytes.toBytes("qf3"); 105 private byte [] qf4 = Bytes.toBytes("qf4"); 106 private byte [] qf5 = Bytes.toBytes("qf5"); 107 private byte [] qf6 = Bytes.toBytes("qf6"); 108 private byte[] value = Bytes.toBytes("value"); 109 private byte[] value2 = Bytes.toBytes("value2"); 110 private Path mobFilePath; 111 private Date currentDate = new Date(); 112 private Cell seekKey1; 113 private Cell seekKey2; 114 private Cell seekKey3; 115 private NavigableSet<byte[]> qualifiers = new ConcurrentSkipListSet<>(Bytes.BYTES_COMPARATOR); 116 private List<Cell> expected = new ArrayList<>(); 117 private long id = System.currentTimeMillis(); 118 private Get get = new Get(row); 119 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); 120 private final String DIR = TEST_UTIL.getDataTestDir("TestHMobStore").toString(); 121 122 /** 123 * Setup 124 * @throws Exception 125 */ 126 @Before 127 public void setUp() throws Exception { 128 qualifiers.add(qf1); 129 qualifiers.add(qf3); 130 qualifiers.add(qf5); 131 132 Iterator<byte[]> iter = qualifiers.iterator(); 133 while(iter.hasNext()){ 134 byte [] next = iter.next(); 135 expected.add(new KeyValue(row, family, next, 1, value)); 136 get.addColumn(family, next); 137 get.readAllVersions(); 138 } 139 } 140 141 private void init(String methodName, Configuration conf, boolean testStore) throws IOException { 142 ColumnFamilyDescriptor cfd = 143 ColumnFamilyDescriptorBuilder.newBuilder(family).setMobEnabled(true).setMobThreshold(3L) 144 .setMaxVersions(4).build(); 145 init(methodName, conf, cfd, testStore); 146 } 147 148 private void init(String methodName, Configuration conf, ColumnFamilyDescriptor cfd, 149 boolean testStore) throws IOException { 150 TableDescriptor td = 151 TableDescriptorBuilder.newBuilder(TableName.valueOf(table)).setColumnFamily(cfd).build(); 152 153 //Setting up tje Region and Store 154 Path basedir = new Path(DIR + methodName); 155 Path tableDir = CommonFSUtils.getTableDir(basedir, td.getTableName()); 156 String logName = "logs"; 157 Path logdir = new Path(basedir, logName); 158 FileSystem fs = FileSystem.get(conf); 159 fs.delete(logdir, true); 160 161 RegionInfo info = RegionInfoBuilder.newBuilder(td.getTableName()).build(); 162 ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null); 163 final Configuration walConf = new Configuration(conf); 164 CommonFSUtils.setRootDir(walConf, basedir); 165 final WALFactory wals = new WALFactory(walConf, methodName); 166 region = new HRegion(tableDir, wals.getWAL(info), fs, conf, info, td, null); 167 region.setMobFileCache(new MobFileCache(conf)); 168 store = new HMobStore(region, cfd, conf, false); 169 if (testStore) { 170 init(conf, cfd); 171 } 172 } 173 174 private void init(Configuration conf, ColumnFamilyDescriptor cfd) 175 throws IOException { 176 Path basedir = CommonFSUtils.getRootDir(conf); 177 fs = FileSystem.get(conf); 178 Path homePath = new Path(basedir, Bytes.toString(family) + Path.SEPARATOR 179 + Bytes.toString(family)); 180 fs.mkdirs(homePath); 181 182 KeyValue key1 = new KeyValue(row, family, qf1, 1, value); 183 KeyValue key2 = new KeyValue(row, family, qf2, 1, value); 184 KeyValue key3 = new KeyValue(row2, family, qf3, 1, value2); 185 KeyValue[] keys = new KeyValue[] { key1, key2, key3 }; 186 int maxKeyCount = keys.length; 187 StoreFileWriter mobWriter = store.createWriterInTmp(currentDate, maxKeyCount, 188 cfd.getCompactionCompressionType(), region.getRegionInfo().getStartKey(), false); 189 mobFilePath = mobWriter.getPath(); 190 191 mobWriter.append(key1); 192 mobWriter.append(key2); 193 mobWriter.append(key3); 194 mobWriter.close(); 195 196 String targetPathName = MobUtils.formatDate(currentDate); 197 byte[] referenceValue = Bytes.toBytes(targetPathName + Path.SEPARATOR + mobFilePath.getName()); 198 Tag tableNameTag = new ArrayBackedTag(TagType.MOB_TABLE_NAME_TAG_TYPE, 199 store.getTableName().getName()); 200 KeyValue kv1 = new KeyValue(row, family, qf1, Long.MAX_VALUE, referenceValue); 201 KeyValue kv2 = new KeyValue(row, family, qf2, Long.MAX_VALUE, referenceValue); 202 KeyValue kv3 = new KeyValue(row2, family, qf3, Long.MAX_VALUE, referenceValue); 203 seekKey1 = MobUtils.createMobRefCell(kv1, referenceValue, tableNameTag); 204 seekKey2 = MobUtils.createMobRefCell(kv2, referenceValue, tableNameTag); 205 seekKey3 = MobUtils.createMobRefCell(kv3, referenceValue, tableNameTag); 206 } 207 208 /** 209 * Getting data from memstore 210 * @throws IOException 211 */ 212 @Test 213 public void testGetFromMemStore() throws IOException { 214 final Configuration conf = HBaseConfiguration.create(); 215 init(name.getMethodName(), conf, false); 216 217 //Put data in memstore 218 this.store.add(new KeyValue(row, family, qf1, 1, value), null); 219 this.store.add(new KeyValue(row, family, qf2, 1, value), null); 220 this.store.add(new KeyValue(row, family, qf3, 1, value), null); 221 this.store.add(new KeyValue(row, family, qf4, 1, value), null); 222 this.store.add(new KeyValue(row, family, qf5, 1, value), null); 223 this.store.add(new KeyValue(row, family, qf6, 1, value), null); 224 225 Scan scan = new Scan(get); 226 InternalScanner scanner = (InternalScanner) store.getScanner(scan, 227 scan.getFamilyMap().get(store.getColumnFamilyDescriptor().getName()), 228 0); 229 230 List<Cell> results = new ArrayList<>(); 231 scanner.next(results); 232 Collections.sort(results, CellComparatorImpl.COMPARATOR); 233 scanner.close(); 234 235 //Compare 236 Assert.assertEquals(expected.size(), results.size()); 237 for(int i=0; i<results.size(); i++) { 238 // Verify the values 239 Assert.assertEquals(expected.get(i), results.get(i)); 240 } 241 } 242 243 /** 244 * Getting MOB data from files 245 * @throws IOException 246 */ 247 @Test 248 public void testGetFromFiles() throws IOException { 249 final Configuration conf = TEST_UTIL.getConfiguration(); 250 init(name.getMethodName(), conf, false); 251 252 //Put data in memstore 253 this.store.add(new KeyValue(row, family, qf1, 1, value), null); 254 this.store.add(new KeyValue(row, family, qf2, 1, value), null); 255 //flush 256 flush(1); 257 258 //Add more data 259 this.store.add(new KeyValue(row, family, qf3, 1, value), null); 260 this.store.add(new KeyValue(row, family, qf4, 1, value), null); 261 //flush 262 flush(2); 263 264 //Add more data 265 this.store.add(new KeyValue(row, family, qf5, 1, value), null); 266 this.store.add(new KeyValue(row, family, qf6, 1, value), null); 267 //flush 268 flush(3); 269 270 Scan scan = new Scan(get); 271 InternalScanner scanner = (InternalScanner) store.getScanner(scan, 272 scan.getFamilyMap().get(store.getColumnFamilyDescriptor().getName()), 273 0); 274 275 List<Cell> results = new ArrayList<>(); 276 scanner.next(results); 277 Collections.sort(results, CellComparatorImpl.COMPARATOR); 278 scanner.close(); 279 280 //Compare 281 Assert.assertEquals(expected.size(), results.size()); 282 for(int i=0; i<results.size(); i++) { 283 Assert.assertEquals(expected.get(i), results.get(i)); 284 } 285 } 286 287 /** 288 * Getting the reference data from files 289 * @throws IOException 290 */ 291 @Test 292 public void testGetReferencesFromFiles() throws IOException { 293 final Configuration conf = HBaseConfiguration.create(); 294 init(name.getMethodName(), conf, false); 295 296 //Put data in memstore 297 this.store.add(new KeyValue(row, family, qf1, 1, value), null); 298 this.store.add(new KeyValue(row, family, qf2, 1, value), null); 299 //flush 300 flush(1); 301 302 //Add more data 303 this.store.add(new KeyValue(row, family, qf3, 1, value), null); 304 this.store.add(new KeyValue(row, family, qf4, 1, value), null); 305 //flush 306 flush(2); 307 308 //Add more data 309 this.store.add(new KeyValue(row, family, qf5, 1, value), null); 310 this.store.add(new KeyValue(row, family, qf6, 1, value), null); 311 //flush 312 flush(3); 313 314 Scan scan = new Scan(get); 315 scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE)); 316 InternalScanner scanner = (InternalScanner) store.getScanner(scan, 317 scan.getFamilyMap().get(store.getColumnFamilyDescriptor().getName()), 318 0); 319 320 List<Cell> results = new ArrayList<>(); 321 scanner.next(results); 322 Collections.sort(results, CellComparatorImpl.COMPARATOR); 323 scanner.close(); 324 325 //Compare 326 Assert.assertEquals(expected.size(), results.size()); 327 for(int i=0; i<results.size(); i++) { 328 Cell cell = results.get(i); 329 Assert.assertTrue(MobUtils.isMobReferenceCell(cell)); 330 } 331 } 332 333 /** 334 * Getting data from memstore and files 335 * @throws IOException 336 */ 337 @Test 338 public void testGetFromMemStoreAndFiles() throws IOException { 339 340 final Configuration conf = HBaseConfiguration.create(); 341 342 init(name.getMethodName(), conf, false); 343 344 //Put data in memstore 345 this.store.add(new KeyValue(row, family, qf1, 1, value), null); 346 this.store.add(new KeyValue(row, family, qf2, 1, value), null); 347 //flush 348 flush(1); 349 350 //Add more data 351 this.store.add(new KeyValue(row, family, qf3, 1, value), null); 352 this.store.add(new KeyValue(row, family, qf4, 1, value), null); 353 //flush 354 flush(2); 355 356 //Add more data 357 this.store.add(new KeyValue(row, family, qf5, 1, value), null); 358 this.store.add(new KeyValue(row, family, qf6, 1, value), null); 359 360 Scan scan = new Scan(get); 361 InternalScanner scanner = (InternalScanner) store.getScanner(scan, 362 scan.getFamilyMap().get(store.getColumnFamilyDescriptor().getName()), 363 0); 364 365 List<Cell> results = new ArrayList<>(); 366 scanner.next(results); 367 Collections.sort(results, CellComparatorImpl.COMPARATOR); 368 scanner.close(); 369 370 //Compare 371 Assert.assertEquals(expected.size(), results.size()); 372 for(int i=0; i<results.size(); i++) { 373 Assert.assertEquals(expected.get(i), results.get(i)); 374 } 375 } 376 377 /** 378 * Getting data from memstore and files 379 * @throws IOException 380 */ 381 @Test 382 public void testMobCellSizeThreshold() throws IOException { 383 final Configuration conf = HBaseConfiguration.create(); 384 ColumnFamilyDescriptor cfd = 385 ColumnFamilyDescriptorBuilder.newBuilder(family).setMobEnabled(true).setMobThreshold(100) 386 .setMaxVersions(4).build(); 387 init(name.getMethodName(), conf, cfd, false); 388 389 //Put data in memstore 390 this.store.add(new KeyValue(row, family, qf1, 1, value), null); 391 this.store.add(new KeyValue(row, family, qf2, 1, value), null); 392 //flush 393 flush(1); 394 395 //Add more data 396 this.store.add(new KeyValue(row, family, qf3, 1, value), null); 397 this.store.add(new KeyValue(row, family, qf4, 1, value), null); 398 //flush 399 flush(2); 400 401 //Add more data 402 this.store.add(new KeyValue(row, family, qf5, 1, value), null); 403 this.store.add(new KeyValue(row, family, qf6, 1, value), null); 404 //flush 405 flush(3); 406 407 Scan scan = new Scan(get); 408 scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE)); 409 InternalScanner scanner = (InternalScanner) store.getScanner(scan, 410 scan.getFamilyMap().get(store.getColumnFamilyDescriptor().getName()), 411 0); 412 413 List<Cell> results = new ArrayList<>(); 414 scanner.next(results); 415 Collections.sort(results, CellComparatorImpl.COMPARATOR); 416 scanner.close(); 417 418 //Compare 419 Assert.assertEquals(expected.size(), results.size()); 420 for(int i=0; i<results.size(); i++) { 421 Cell cell = results.get(i); 422 //this is not mob reference cell. 423 Assert.assertFalse(MobUtils.isMobReferenceCell(cell)); 424 Assert.assertEquals(expected.get(i), results.get(i)); 425 Assert.assertEquals(100, store.getColumnFamilyDescriptor().getMobThreshold()); 426 } 427 } 428 429 @Test 430 public void testCommitFile() throws Exception { 431 final Configuration conf = HBaseConfiguration.create(); 432 init(name.getMethodName(), conf, true); 433 String targetPathName = MobUtils.formatDate(new Date()); 434 Path targetPath = new Path(store.getPath(), (targetPathName 435 + Path.SEPARATOR + mobFilePath.getName())); 436 fs.delete(targetPath, true); 437 Assert.assertFalse(fs.exists(targetPath)); 438 //commit file 439 store.commitFile(mobFilePath, targetPath); 440 Assert.assertTrue(fs.exists(targetPath)); 441 } 442 443 @Test 444 public void testResolve() throws Exception { 445 final Configuration conf = HBaseConfiguration.create(); 446 init(name.getMethodName(), conf, true); 447 String targetPathName = MobUtils.formatDate(currentDate); 448 Path targetPath = new Path(store.getPath(), targetPathName); 449 store.commitFile(mobFilePath, targetPath); 450 // resolve 451 Cell resultCell1 = store.resolve(seekKey1, false).getCell(); 452 Cell resultCell2 = store.resolve(seekKey2, false).getCell(); 453 Cell resultCell3 = store.resolve(seekKey3, false).getCell(); 454 // compare 455 Assert.assertEquals(Bytes.toString(value), Bytes.toString(CellUtil.cloneValue(resultCell1))); 456 Assert.assertEquals(Bytes.toString(value), Bytes.toString(CellUtil.cloneValue(resultCell2))); 457 Assert.assertEquals(Bytes.toString(value2), Bytes.toString(CellUtil.cloneValue(resultCell3))); 458 } 459 460 /** 461 * Flush the memstore 462 * @param storeFilesSize 463 * @throws IOException 464 */ 465 private void flush(int storeFilesSize) throws IOException{ 466 this.store.snapshot(); 467 flushStore(store, id++); 468 Assert.assertEquals(storeFilesSize, this.store.getStorefiles().size()); 469 Assert.assertEquals(0, ((AbstractMemStore)this.store.memstore).getActive().getCellsCount()); 470 } 471 472 /** 473 * Flush the memstore 474 * @param store 475 * @param id 476 * @throws IOException 477 */ 478 private static void flushStore(HMobStore store, long id) throws IOException { 479 StoreFlushContext storeFlushCtx = store.createFlushContext(id, FlushLifeCycleTracker.DUMMY); 480 storeFlushCtx.prepare(); 481 storeFlushCtx.flushCache(Mockito.mock(MonitoredTask.class)); 482 storeFlushCtx.commit(Mockito.mock(MonitoredTask.class)); 483 } 484 485 @Test 486 public void testMOBStoreEncryption() throws Exception { 487 final Configuration conf = TEST_UTIL.getConfiguration(); 488 489 conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyProviderForTesting.class.getName()); 490 conf.set(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, "hbase"); 491 SecureRandom rng = new SecureRandom(); 492 byte[] keyBytes = new byte[AES.KEY_LENGTH]; 493 rng.nextBytes(keyBytes); 494 String algorithm = conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES); 495 Key cfKey = new SecretKeySpec(keyBytes, algorithm); 496 497 ColumnFamilyDescriptor cfd = 498 ColumnFamilyDescriptorBuilder.newBuilder(family).setMobEnabled(true).setMobThreshold(100) 499 .setMaxVersions(4).setEncryptionType(algorithm).setEncryptionKey(EncryptionUtil 500 .wrapKey(conf, conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, 501 User.getCurrent().getShortName()), cfKey)).build(); 502 init(name.getMethodName(), conf, cfd, false); 503 504 this.store.add(new KeyValue(row, family, qf1, 1, value), null); 505 this.store.add(new KeyValue(row, family, qf2, 1, value), null); 506 this.store.add(new KeyValue(row, family, qf3, 1, value), null); 507 flush(1); 508 509 this.store.add(new KeyValue(row, family, qf4, 1, value), null); 510 this.store.add(new KeyValue(row, family, qf5, 1, value), null); 511 this.store.add(new KeyValue(row, family, qf6, 1, value), null); 512 flush(2); 513 514 Collection<HStoreFile> storefiles = this.store.getStorefiles(); 515 checkMobHFileEncrytption(storefiles); 516 517 // Scan the values 518 Scan scan = new Scan(get); 519 InternalScanner scanner = (InternalScanner) store.getScanner(scan, 520 scan.getFamilyMap().get(store.getColumnFamilyDescriptor().getName()), 521 0); 522 523 List<Cell> results = new ArrayList<>(); 524 scanner.next(results); 525 Collections.sort(results, CellComparatorImpl.COMPARATOR); 526 scanner.close(); 527 Assert.assertEquals(expected.size(), results.size()); 528 for(int i=0; i<results.size(); i++) { 529 Assert.assertEquals(expected.get(i), results.get(i)); 530 } 531 532 // Trigger major compaction 533 this.store.triggerMajorCompaction(); 534 Optional<CompactionContext> requestCompaction = 535 this.store.requestCompaction(PRIORITY_USER, CompactionLifeCycleTracker.DUMMY, null); 536 this.store.compact(requestCompaction.get(), NoLimitThroughputController.INSTANCE, null); 537 Assert.assertEquals(1, this.store.getStorefiles().size()); 538 539 //Check encryption after compaction 540 checkMobHFileEncrytption(this.store.getStorefiles()); 541 } 542 543 private void checkMobHFileEncrytption(Collection<HStoreFile> storefiles) { 544 HStoreFile storeFile = storefiles.iterator().next(); 545 HFile.Reader reader = storeFile.getReader().getHFileReader(); 546 byte[] encryptionKey = reader.getTrailer().getEncryptionKey(); 547 Assert.assertTrue(null != encryptionKey); 548 Assert.assertTrue(reader.getFileContext().getEncryptionContext().getCipher().getName() 549 .equals(HConstants.CIPHER_AES)); 550 } 551 552}