001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.apache.hadoop.hbase.regionserver.Store.PRIORITY_USER;
021
022import java.io.IOException;
023import java.security.Key;
024import java.util.ArrayList;
025import java.util.Collection;
026import java.util.Collections;
027import java.util.Date;
028import java.util.Iterator;
029import java.util.List;
030import java.util.NavigableSet;
031import java.util.Optional;
032import java.util.concurrent.ConcurrentSkipListSet;
033import javax.crypto.spec.SecretKeySpec;
034import org.apache.hadoop.conf.Configuration;
035import org.apache.hadoop.fs.FileSystem;
036import org.apache.hadoop.fs.Path;
037import org.apache.hadoop.hbase.ArrayBackedTag;
038import org.apache.hadoop.hbase.Cell;
039import org.apache.hadoop.hbase.CellComparatorImpl;
040import org.apache.hadoop.hbase.CellUtil;
041import org.apache.hadoop.hbase.HBaseClassTestRule;
042import org.apache.hadoop.hbase.HBaseConfiguration;
043import org.apache.hadoop.hbase.HBaseTestingUtility;
044import org.apache.hadoop.hbase.HConstants;
045import org.apache.hadoop.hbase.KeyValue;
046import org.apache.hadoop.hbase.TableName;
047import org.apache.hadoop.hbase.Tag;
048import org.apache.hadoop.hbase.TagType;
049import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
050import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
051import org.apache.hadoop.hbase.client.Get;
052import org.apache.hadoop.hbase.client.RegionInfo;
053import org.apache.hadoop.hbase.client.RegionInfoBuilder;
054import org.apache.hadoop.hbase.client.Scan;
055import org.apache.hadoop.hbase.client.TableDescriptor;
056import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
057import org.apache.hadoop.hbase.io.crypto.KeyProviderForTesting;
058import org.apache.hadoop.hbase.io.crypto.aes.AES;
059import org.apache.hadoop.hbase.io.hfile.HFile;
060import org.apache.hadoop.hbase.mob.MobConstants;
061import org.apache.hadoop.hbase.mob.MobFileCache;
062import org.apache.hadoop.hbase.mob.MobUtils;
063import org.apache.hadoop.hbase.monitoring.MonitoredTask;
064import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
065import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker;
066import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController;
067import org.apache.hadoop.hbase.security.EncryptionUtil;
068import org.apache.hadoop.hbase.security.User;
069import org.apache.hadoop.hbase.testclassification.MediumTests;
070import org.apache.hadoop.hbase.util.Bytes;
071import org.apache.hadoop.hbase.util.CommonFSUtils;
072import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
073import org.apache.hadoop.hbase.wal.WALFactory;
074import org.junit.Assert;
075import org.junit.Before;
076import org.junit.ClassRule;
077import org.junit.Rule;
078import org.junit.Test;
079import org.junit.experimental.categories.Category;
080import org.junit.rules.TestName;
081import org.mockito.Mockito;
082import org.slf4j.Logger;
083import org.slf4j.LoggerFactory;
084
085@Category(MediumTests.class)
086public class TestHMobStore {
087
088  @ClassRule
089  public static final HBaseClassTestRule CLASS_RULE =
090    HBaseClassTestRule.forClass(TestHMobStore.class);
091
092  public static final Logger LOG = LoggerFactory.getLogger(TestHMobStore.class);
093  @Rule
094  public TestName name = new TestName();
095
096  private HMobStore store;
097  private HRegion region;
098  private FileSystem fs;
099  private byte[] table = Bytes.toBytes("table");
100  private byte[] family = Bytes.toBytes("family");
101  private byte[] row = Bytes.toBytes("row");
102  private byte[] row2 = Bytes.toBytes("row2");
103  private byte[] qf1 = Bytes.toBytes("qf1");
104  private byte[] qf2 = Bytes.toBytes("qf2");
105  private byte[] qf3 = Bytes.toBytes("qf3");
106  private byte[] qf4 = Bytes.toBytes("qf4");
107  private byte[] qf5 = Bytes.toBytes("qf5");
108  private byte[] qf6 = Bytes.toBytes("qf6");
109  private byte[] value = Bytes.toBytes("value");
110  private byte[] value2 = Bytes.toBytes("value2");
111  private Path mobFilePath;
112  private Date currentDate = new Date();
113  private Cell seekKey1;
114  private Cell seekKey2;
115  private Cell seekKey3;
116  private NavigableSet<byte[]> qualifiers = new ConcurrentSkipListSet<>(Bytes.BYTES_COMPARATOR);
117  private List<Cell> expected = new ArrayList<>();
118  private long id = EnvironmentEdgeManager.currentTime();
119  private Get get = new Get(row);
120  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
121  private final String DIR = TEST_UTIL.getDataTestDir("TestHMobStore").toString();
122
123  /**
124   * Setup n
125   */
126  @Before
127  public void setUp() throws Exception {
128    qualifiers.add(qf1);
129    qualifiers.add(qf3);
130    qualifiers.add(qf5);
131
132    Iterator<byte[]> iter = qualifiers.iterator();
133    while (iter.hasNext()) {
134      byte[] next = iter.next();
135      expected.add(new KeyValue(row, family, next, 1, value));
136      get.addColumn(family, next);
137      get.readAllVersions();
138    }
139  }
140
141  private void init(String methodName, Configuration conf, boolean testStore) throws IOException {
142    ColumnFamilyDescriptor cfd = ColumnFamilyDescriptorBuilder.newBuilder(family)
143      .setMobEnabled(true).setMobThreshold(3L).setMaxVersions(4).build();
144    init(methodName, conf, cfd, testStore);
145  }
146
147  private void init(String methodName, Configuration conf, ColumnFamilyDescriptor cfd,
148    boolean testStore) throws IOException {
149    TableDescriptor td =
150      TableDescriptorBuilder.newBuilder(TableName.valueOf(table)).setColumnFamily(cfd).build();
151
152    // Setting up tje Region and Store
153    Path basedir = new Path(DIR + methodName);
154    Path tableDir = CommonFSUtils.getTableDir(basedir, td.getTableName());
155    String logName = "logs";
156    Path logdir = new Path(basedir, logName);
157    FileSystem fs = FileSystem.get(conf);
158    fs.delete(logdir, true);
159
160    RegionInfo info = RegionInfoBuilder.newBuilder(td.getTableName()).build();
161    ChunkCreator.initialize(MemStoreLAB.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null,
162      MemStoreLAB.INDEX_CHUNK_SIZE_PERCENTAGE_DEFAULT);
163    final Configuration walConf = new Configuration(conf);
164    CommonFSUtils.setRootDir(walConf, basedir);
165    final WALFactory wals = new WALFactory(walConf, methodName);
166    region = new HRegion(tableDir, wals.getWAL(info), fs, conf, info, td, null);
167    region.setMobFileCache(new MobFileCache(conf));
168    store = new HMobStore(region, cfd, conf, false);
169    if (testStore) {
170      init(conf, cfd);
171    }
172  }
173
174  private void init(Configuration conf, ColumnFamilyDescriptor cfd) throws IOException {
175    Path basedir = CommonFSUtils.getRootDir(conf);
176    fs = FileSystem.get(conf);
177    Path homePath =
178      new Path(basedir, Bytes.toString(family) + Path.SEPARATOR + Bytes.toString(family));
179    fs.mkdirs(homePath);
180
181    KeyValue key1 = new KeyValue(row, family, qf1, 1, value);
182    KeyValue key2 = new KeyValue(row, family, qf2, 1, value);
183    KeyValue key3 = new KeyValue(row2, family, qf3, 1, value2);
184    KeyValue[] keys = new KeyValue[] { key1, key2, key3 };
185    int maxKeyCount = keys.length;
186    StoreFileWriter mobWriter = store.createWriterInTmp(currentDate, maxKeyCount,
187      cfd.getCompactionCompressionType(), region.getRegionInfo().getStartKey(), false);
188    mobFilePath = mobWriter.getPath();
189
190    mobWriter.append(key1);
191    mobWriter.append(key2);
192    mobWriter.append(key3);
193    mobWriter.close();
194
195    String targetPathName = MobUtils.formatDate(currentDate);
196    byte[] referenceValue = Bytes.toBytes(targetPathName + Path.SEPARATOR + mobFilePath.getName());
197    Tag tableNameTag =
198      new ArrayBackedTag(TagType.MOB_TABLE_NAME_TAG_TYPE, store.getTableName().getName());
199    KeyValue kv1 = new KeyValue(row, family, qf1, Long.MAX_VALUE, referenceValue);
200    KeyValue kv2 = new KeyValue(row, family, qf2, Long.MAX_VALUE, referenceValue);
201    KeyValue kv3 = new KeyValue(row2, family, qf3, Long.MAX_VALUE, referenceValue);
202    seekKey1 = MobUtils.createMobRefCell(kv1, referenceValue, tableNameTag);
203    seekKey2 = MobUtils.createMobRefCell(kv2, referenceValue, tableNameTag);
204    seekKey3 = MobUtils.createMobRefCell(kv3, referenceValue, tableNameTag);
205  }
206
207  /**
208   * Getting data from memstore n
209   */
210  @Test
211  public void testGetFromMemStore() throws IOException {
212    final Configuration conf = HBaseConfiguration.create();
213    init(name.getMethodName(), conf, false);
214
215    // Put data in memstore
216    this.store.add(new KeyValue(row, family, qf1, 1, value), null);
217    this.store.add(new KeyValue(row, family, qf2, 1, value), null);
218    this.store.add(new KeyValue(row, family, qf3, 1, value), null);
219    this.store.add(new KeyValue(row, family, qf4, 1, value), null);
220    this.store.add(new KeyValue(row, family, qf5, 1, value), null);
221    this.store.add(new KeyValue(row, family, qf6, 1, value), null);
222
223    Scan scan = new Scan(get);
224    InternalScanner scanner = (InternalScanner) store.getScanner(scan,
225      scan.getFamilyMap().get(store.getColumnFamilyDescriptor().getName()), 0);
226
227    List<Cell> results = new ArrayList<>();
228    scanner.next(results);
229    Collections.sort(results, CellComparatorImpl.COMPARATOR);
230    scanner.close();
231
232    // Compare
233    Assert.assertEquals(expected.size(), results.size());
234    for (int i = 0; i < results.size(); i++) {
235      // Verify the values
236      Assert.assertEquals(expected.get(i), results.get(i));
237    }
238  }
239
240  /**
241   * Getting MOB data from files n
242   */
243  @Test
244  public void testGetFromFiles() throws IOException {
245    final Configuration conf = TEST_UTIL.getConfiguration();
246    init(name.getMethodName(), conf, false);
247
248    // Put data in memstore
249    this.store.add(new KeyValue(row, family, qf1, 1, value), null);
250    this.store.add(new KeyValue(row, family, qf2, 1, value), null);
251    // flush
252    flush(1);
253
254    // Add more data
255    this.store.add(new KeyValue(row, family, qf3, 1, value), null);
256    this.store.add(new KeyValue(row, family, qf4, 1, value), null);
257    // flush
258    flush(2);
259
260    // Add more data
261    this.store.add(new KeyValue(row, family, qf5, 1, value), null);
262    this.store.add(new KeyValue(row, family, qf6, 1, value), null);
263    // flush
264    flush(3);
265
266    Scan scan = new Scan(get);
267    InternalScanner scanner = (InternalScanner) store.getScanner(scan,
268      scan.getFamilyMap().get(store.getColumnFamilyDescriptor().getName()), 0);
269
270    List<Cell> results = new ArrayList<>();
271    scanner.next(results);
272    Collections.sort(results, CellComparatorImpl.COMPARATOR);
273    scanner.close();
274
275    // Compare
276    Assert.assertEquals(expected.size(), results.size());
277    for (int i = 0; i < results.size(); i++) {
278      Assert.assertEquals(expected.get(i), results.get(i));
279    }
280  }
281
282  /**
283   * Getting the reference data from files n
284   */
285  @Test
286  public void testGetReferencesFromFiles() throws IOException {
287    final Configuration conf = HBaseConfiguration.create();
288    init(name.getMethodName(), conf, false);
289
290    // Put data in memstore
291    this.store.add(new KeyValue(row, family, qf1, 1, value), null);
292    this.store.add(new KeyValue(row, family, qf2, 1, value), null);
293    // flush
294    flush(1);
295
296    // Add more data
297    this.store.add(new KeyValue(row, family, qf3, 1, value), null);
298    this.store.add(new KeyValue(row, family, qf4, 1, value), null);
299    // flush
300    flush(2);
301
302    // Add more data
303    this.store.add(new KeyValue(row, family, qf5, 1, value), null);
304    this.store.add(new KeyValue(row, family, qf6, 1, value), null);
305    // flush
306    flush(3);
307
308    Scan scan = new Scan(get);
309    scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
310    InternalScanner scanner = (InternalScanner) store.getScanner(scan,
311      scan.getFamilyMap().get(store.getColumnFamilyDescriptor().getName()), 0);
312
313    List<Cell> results = new ArrayList<>();
314    scanner.next(results);
315    Collections.sort(results, CellComparatorImpl.COMPARATOR);
316    scanner.close();
317
318    // Compare
319    Assert.assertEquals(expected.size(), results.size());
320    for (int i = 0; i < results.size(); i++) {
321      Cell cell = results.get(i);
322      Assert.assertTrue(MobUtils.isMobReferenceCell(cell));
323    }
324  }
325
326  /**
327   * Getting data from memstore and files n
328   */
329  @Test
330  public void testGetFromMemStoreAndFiles() throws IOException {
331
332    final Configuration conf = HBaseConfiguration.create();
333
334    init(name.getMethodName(), conf, false);
335
336    // Put data in memstore
337    this.store.add(new KeyValue(row, family, qf1, 1, value), null);
338    this.store.add(new KeyValue(row, family, qf2, 1, value), null);
339    // flush
340    flush(1);
341
342    // Add more data
343    this.store.add(new KeyValue(row, family, qf3, 1, value), null);
344    this.store.add(new KeyValue(row, family, qf4, 1, value), null);
345    // flush
346    flush(2);
347
348    // Add more data
349    this.store.add(new KeyValue(row, family, qf5, 1, value), null);
350    this.store.add(new KeyValue(row, family, qf6, 1, value), null);
351
352    Scan scan = new Scan(get);
353    InternalScanner scanner = (InternalScanner) store.getScanner(scan,
354      scan.getFamilyMap().get(store.getColumnFamilyDescriptor().getName()), 0);
355
356    List<Cell> results = new ArrayList<>();
357    scanner.next(results);
358    Collections.sort(results, CellComparatorImpl.COMPARATOR);
359    scanner.close();
360
361    // Compare
362    Assert.assertEquals(expected.size(), results.size());
363    for (int i = 0; i < results.size(); i++) {
364      Assert.assertEquals(expected.get(i), results.get(i));
365    }
366  }
367
368  /**
369   * Getting data from memstore and files n
370   */
371  @Test
372  public void testMobCellSizeThreshold() throws IOException {
373    final Configuration conf = HBaseConfiguration.create();
374    ColumnFamilyDescriptor cfd = ColumnFamilyDescriptorBuilder.newBuilder(family)
375      .setMobEnabled(true).setMobThreshold(100).setMaxVersions(4).build();
376    init(name.getMethodName(), conf, cfd, false);
377
378    // Put data in memstore
379    this.store.add(new KeyValue(row, family, qf1, 1, value), null);
380    this.store.add(new KeyValue(row, family, qf2, 1, value), null);
381    // flush
382    flush(1);
383
384    // Add more data
385    this.store.add(new KeyValue(row, family, qf3, 1, value), null);
386    this.store.add(new KeyValue(row, family, qf4, 1, value), null);
387    // flush
388    flush(2);
389
390    // Add more data
391    this.store.add(new KeyValue(row, family, qf5, 1, value), null);
392    this.store.add(new KeyValue(row, family, qf6, 1, value), null);
393    // flush
394    flush(3);
395
396    Scan scan = new Scan(get);
397    scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
398    InternalScanner scanner = (InternalScanner) store.getScanner(scan,
399      scan.getFamilyMap().get(store.getColumnFamilyDescriptor().getName()), 0);
400
401    List<Cell> results = new ArrayList<>();
402    scanner.next(results);
403    Collections.sort(results, CellComparatorImpl.COMPARATOR);
404    scanner.close();
405
406    // Compare
407    Assert.assertEquals(expected.size(), results.size());
408    for (int i = 0; i < results.size(); i++) {
409      Cell cell = results.get(i);
410      // this is not mob reference cell.
411      Assert.assertFalse(MobUtils.isMobReferenceCell(cell));
412      Assert.assertEquals(expected.get(i), results.get(i));
413      Assert.assertEquals(100, store.getColumnFamilyDescriptor().getMobThreshold());
414    }
415  }
416
417  @Test
418  public void testCommitFile() throws Exception {
419    final Configuration conf = HBaseConfiguration.create();
420    init(name.getMethodName(), conf, true);
421    String targetPathName = MobUtils.formatDate(new Date());
422    Path targetPath =
423      new Path(store.getPath(), (targetPathName + Path.SEPARATOR + mobFilePath.getName()));
424    fs.delete(targetPath, true);
425    Assert.assertFalse(fs.exists(targetPath));
426    // commit file
427    store.commitFile(mobFilePath, targetPath);
428    Assert.assertTrue(fs.exists(targetPath));
429  }
430
431  @Test
432  public void testResolve() throws Exception {
433    final Configuration conf = HBaseConfiguration.create();
434    init(name.getMethodName(), conf, true);
435    String targetPathName = MobUtils.formatDate(currentDate);
436    Path targetPath = new Path(store.getPath(), targetPathName);
437    store.commitFile(mobFilePath, targetPath);
438    // resolve
439    Cell resultCell1 = store.resolve(seekKey1, false).getCell();
440    Cell resultCell2 = store.resolve(seekKey2, false).getCell();
441    Cell resultCell3 = store.resolve(seekKey3, false).getCell();
442    // compare
443    Assert.assertEquals(Bytes.toString(value), Bytes.toString(CellUtil.cloneValue(resultCell1)));
444    Assert.assertEquals(Bytes.toString(value), Bytes.toString(CellUtil.cloneValue(resultCell2)));
445    Assert.assertEquals(Bytes.toString(value2), Bytes.toString(CellUtil.cloneValue(resultCell3)));
446  }
447
448  /**
449   * Flush the memstore
450   */
451  private void flush(int storeFilesSize) throws IOException {
452    flushStore(store, id++);
453    Assert.assertEquals(storeFilesSize, this.store.getStorefiles().size());
454    Assert.assertEquals(0, ((AbstractMemStore) this.store.memstore).getActive().getCellsCount());
455  }
456
457  /**
458   * Flush the memstore
459   */
460  private static void flushStore(HMobStore store, long id) throws IOException {
461    StoreFlushContext storeFlushCtx = store.createFlushContext(id, FlushLifeCycleTracker.DUMMY);
462    storeFlushCtx.prepare();
463    storeFlushCtx.flushCache(Mockito.mock(MonitoredTask.class));
464    storeFlushCtx.commit(Mockito.mock(MonitoredTask.class));
465  }
466
467  @Test
468  public void testMOBStoreEncryption() throws Exception {
469    final Configuration conf = TEST_UTIL.getConfiguration();
470
471    conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyProviderForTesting.class.getName());
472    conf.set(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, "hbase");
473    byte[] keyBytes = new byte[AES.KEY_LENGTH];
474    Bytes.secureRandom(keyBytes);
475    String algorithm = conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES);
476    Key cfKey = new SecretKeySpec(keyBytes, algorithm);
477
478    ColumnFamilyDescriptor cfd = ColumnFamilyDescriptorBuilder.newBuilder(family)
479      .setMobEnabled(true).setMobThreshold(100).setMaxVersions(4).setEncryptionType(algorithm)
480      .setEncryptionKey(EncryptionUtil.wrapKey(conf,
481        conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, User.getCurrent().getShortName()),
482        cfKey))
483      .build();
484    init(name.getMethodName(), conf, cfd, false);
485
486    this.store.add(new KeyValue(row, family, qf1, 1, value), null);
487    this.store.add(new KeyValue(row, family, qf2, 1, value), null);
488    this.store.add(new KeyValue(row, family, qf3, 1, value), null);
489    flush(1);
490
491    this.store.add(new KeyValue(row, family, qf4, 1, value), null);
492    this.store.add(new KeyValue(row, family, qf5, 1, value), null);
493    this.store.add(new KeyValue(row, family, qf6, 1, value), null);
494    flush(2);
495
496    Collection<HStoreFile> storefiles = this.store.getStorefiles();
497    checkMobHFileEncrytption(storefiles);
498
499    // Scan the values
500    Scan scan = new Scan(get);
501    InternalScanner scanner = (InternalScanner) store.getScanner(scan,
502      scan.getFamilyMap().get(store.getColumnFamilyDescriptor().getName()), 0);
503
504    List<Cell> results = new ArrayList<>();
505    scanner.next(results);
506    Collections.sort(results, CellComparatorImpl.COMPARATOR);
507    scanner.close();
508    Assert.assertEquals(expected.size(), results.size());
509    for (int i = 0; i < results.size(); i++) {
510      Assert.assertEquals(expected.get(i), results.get(i));
511    }
512
513    // Trigger major compaction
514    this.store.triggerMajorCompaction();
515    Optional<CompactionContext> requestCompaction =
516      this.store.requestCompaction(PRIORITY_USER, CompactionLifeCycleTracker.DUMMY, null);
517    this.store.compact(requestCompaction.get(), NoLimitThroughputController.INSTANCE, null);
518    Assert.assertEquals(1, this.store.getStorefiles().size());
519
520    // Check encryption after compaction
521    checkMobHFileEncrytption(this.store.getStorefiles());
522  }
523
524  private void checkMobHFileEncrytption(Collection<HStoreFile> storefiles) {
525    HStoreFile storeFile = storefiles.iterator().next();
526    HFile.Reader reader = storeFile.getReader().getHFileReader();
527    byte[] encryptionKey = reader.getTrailer().getEncryptionKey();
528    Assert.assertTrue(null != encryptionKey);
529    Assert.assertTrue(reader.getFileContext().getEncryptionContext().getCipher().getName()
530      .equals(HConstants.CIPHER_AES));
531  }
532
533}