001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.apache.hadoop.hbase.regionserver.Store.PRIORITY_USER;
021
022import java.io.IOException;
023import java.security.Key;
024import java.security.SecureRandom;
025import java.util.ArrayList;
026import java.util.Collection;
027import java.util.Collections;
028import java.util.Date;
029import java.util.Iterator;
030import java.util.List;
031import java.util.NavigableSet;
032import java.util.Optional;
033import java.util.concurrent.ConcurrentSkipListSet;
034import javax.crypto.spec.SecretKeySpec;
035import org.apache.hadoop.conf.Configuration;
036import org.apache.hadoop.fs.FileSystem;
037import org.apache.hadoop.fs.Path;
038import org.apache.hadoop.hbase.ArrayBackedTag;
039import org.apache.hadoop.hbase.Cell;
040import org.apache.hadoop.hbase.CellComparatorImpl;
041import org.apache.hadoop.hbase.CellUtil;
042import org.apache.hadoop.hbase.HBaseClassTestRule;
043import org.apache.hadoop.hbase.HBaseConfiguration;
044import org.apache.hadoop.hbase.HBaseTestingUtility;
045import org.apache.hadoop.hbase.HConstants;
046import org.apache.hadoop.hbase.KeyValue;
047import org.apache.hadoop.hbase.TableName;
048import org.apache.hadoop.hbase.Tag;
049import org.apache.hadoop.hbase.TagType;
050import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
051import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
052import org.apache.hadoop.hbase.client.Get;
053import org.apache.hadoop.hbase.client.RegionInfo;
054import org.apache.hadoop.hbase.client.RegionInfoBuilder;
055import org.apache.hadoop.hbase.client.Scan;
056import org.apache.hadoop.hbase.client.TableDescriptor;
057import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
058import org.apache.hadoop.hbase.io.crypto.KeyProviderForTesting;
059import org.apache.hadoop.hbase.io.crypto.aes.AES;
060import org.apache.hadoop.hbase.io.hfile.HFile;
061import org.apache.hadoop.hbase.mob.MobConstants;
062import org.apache.hadoop.hbase.mob.MobFileCache;
063import org.apache.hadoop.hbase.mob.MobUtils;
064import org.apache.hadoop.hbase.monitoring.MonitoredTask;
065import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
066import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker;
067import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController;
068import org.apache.hadoop.hbase.security.EncryptionUtil;
069import org.apache.hadoop.hbase.security.User;
070import org.apache.hadoop.hbase.testclassification.MediumTests;
071import org.apache.hadoop.hbase.util.Bytes;
072import org.apache.hadoop.hbase.util.CommonFSUtils;
073import org.apache.hadoop.hbase.wal.WALFactory;
074import org.junit.Assert;
075import org.junit.Before;
076import org.junit.ClassRule;
077import org.junit.Rule;
078import org.junit.Test;
079import org.junit.experimental.categories.Category;
080import org.junit.rules.TestName;
081import org.mockito.Mockito;
082import org.slf4j.Logger;
083import org.slf4j.LoggerFactory;
084
085@Category(MediumTests.class)
086public class TestHMobStore {
087
088  @ClassRule
089  public static final HBaseClassTestRule CLASS_RULE =
090      HBaseClassTestRule.forClass(TestHMobStore.class);
091
092  public static final Logger LOG = LoggerFactory.getLogger(TestHMobStore.class);
093  @Rule public TestName name = new TestName();
094
095  private HMobStore store;
096  private HRegion region;
097  private FileSystem fs;
098  private byte [] table = Bytes.toBytes("table");
099  private byte [] family = Bytes.toBytes("family");
100  private byte [] row = Bytes.toBytes("row");
101  private byte [] row2 = Bytes.toBytes("row2");
102  private byte [] qf1 = Bytes.toBytes("qf1");
103  private byte [] qf2 = Bytes.toBytes("qf2");
104  private byte [] qf3 = Bytes.toBytes("qf3");
105  private byte [] qf4 = Bytes.toBytes("qf4");
106  private byte [] qf5 = Bytes.toBytes("qf5");
107  private byte [] qf6 = Bytes.toBytes("qf6");
108  private byte[] value = Bytes.toBytes("value");
109  private byte[] value2 = Bytes.toBytes("value2");
110  private Path mobFilePath;
111  private Date currentDate = new Date();
112  private Cell seekKey1;
113  private Cell seekKey2;
114  private Cell seekKey3;
115  private NavigableSet<byte[]> qualifiers = new ConcurrentSkipListSet<>(Bytes.BYTES_COMPARATOR);
116  private List<Cell> expected = new ArrayList<>();
117  private long id = System.currentTimeMillis();
118  private Get get = new Get(row);
119  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
120  private final String DIR = TEST_UTIL.getDataTestDir("TestHMobStore").toString();
121
122  /**
123   * Setup
124   * @throws Exception
125   */
126  @Before
127  public void setUp() throws Exception {
128    qualifiers.add(qf1);
129    qualifiers.add(qf3);
130    qualifiers.add(qf5);
131
132    Iterator<byte[]> iter = qualifiers.iterator();
133    while(iter.hasNext()){
134      byte [] next = iter.next();
135      expected.add(new KeyValue(row, family, next, 1, value));
136      get.addColumn(family, next);
137      get.readAllVersions();
138    }
139  }
140
141  private void init(String methodName, Configuration conf, boolean testStore) throws IOException {
142    ColumnFamilyDescriptor cfd =
143        ColumnFamilyDescriptorBuilder.newBuilder(family).setMobEnabled(true).setMobThreshold(3L)
144            .setMaxVersions(4).build();
145    init(methodName, conf, cfd, testStore);
146  }
147
148  private void init(String methodName, Configuration conf, ColumnFamilyDescriptor cfd,
149      boolean testStore) throws IOException {
150    TableDescriptor td =
151        TableDescriptorBuilder.newBuilder(TableName.valueOf(table)).setColumnFamily(cfd).build();
152
153    //Setting up tje Region and Store
154    Path basedir = new Path(DIR + methodName);
155    Path tableDir = CommonFSUtils.getTableDir(basedir, td.getTableName());
156    String logName = "logs";
157    Path logdir = new Path(basedir, logName);
158    FileSystem fs = FileSystem.get(conf);
159    fs.delete(logdir, true);
160
161    RegionInfo info = RegionInfoBuilder.newBuilder(td.getTableName()).build();
162    ChunkCreator.initialize(MemStoreLAB.CHUNK_SIZE_DEFAULT, false, 0, 0,
163      0, null, MemStoreLAB.INDEX_CHUNK_SIZE_PERCENTAGE_DEFAULT);
164    final Configuration walConf = new Configuration(conf);
165    CommonFSUtils.setRootDir(walConf, basedir);
166    final WALFactory wals = new WALFactory(walConf, methodName);
167    region = new HRegion(tableDir, wals.getWAL(info), fs, conf, info, td, null);
168    region.setMobFileCache(new MobFileCache(conf));
169    store = new HMobStore(region, cfd, conf, false);
170    if (testStore) {
171      init(conf, cfd);
172    }
173  }
174
175  private void init(Configuration conf, ColumnFamilyDescriptor cfd)
176      throws IOException {
177    Path basedir = CommonFSUtils.getRootDir(conf);
178    fs = FileSystem.get(conf);
179    Path homePath = new Path(basedir, Bytes.toString(family) + Path.SEPARATOR
180        + Bytes.toString(family));
181    fs.mkdirs(homePath);
182
183    KeyValue key1 = new KeyValue(row, family, qf1, 1, value);
184    KeyValue key2 = new KeyValue(row, family, qf2, 1, value);
185    KeyValue key3 = new KeyValue(row2, family, qf3, 1, value2);
186    KeyValue[] keys = new KeyValue[] { key1, key2, key3 };
187    int maxKeyCount = keys.length;
188    StoreFileWriter mobWriter = store.createWriterInTmp(currentDate, maxKeyCount,
189        cfd.getCompactionCompressionType(), region.getRegionInfo().getStartKey(), false);
190    mobFilePath = mobWriter.getPath();
191
192    mobWriter.append(key1);
193    mobWriter.append(key2);
194    mobWriter.append(key3);
195    mobWriter.close();
196
197    String targetPathName = MobUtils.formatDate(currentDate);
198    byte[] referenceValue = Bytes.toBytes(targetPathName + Path.SEPARATOR + mobFilePath.getName());
199    Tag tableNameTag = new ArrayBackedTag(TagType.MOB_TABLE_NAME_TAG_TYPE,
200        store.getTableName().getName());
201    KeyValue kv1 = new KeyValue(row, family, qf1, Long.MAX_VALUE, referenceValue);
202    KeyValue kv2 = new KeyValue(row, family, qf2, Long.MAX_VALUE, referenceValue);
203    KeyValue kv3 = new KeyValue(row2, family, qf3, Long.MAX_VALUE, referenceValue);
204    seekKey1 = MobUtils.createMobRefCell(kv1, referenceValue, tableNameTag);
205    seekKey2 = MobUtils.createMobRefCell(kv2, referenceValue, tableNameTag);
206    seekKey3 = MobUtils.createMobRefCell(kv3, referenceValue, tableNameTag);
207  }
208
209  /**
210   * Getting data from memstore
211   * @throws IOException
212   */
213  @Test
214  public void testGetFromMemStore() throws IOException {
215    final Configuration conf = HBaseConfiguration.create();
216    init(name.getMethodName(), conf, false);
217
218    //Put data in memstore
219    this.store.add(new KeyValue(row, family, qf1, 1, value), null);
220    this.store.add(new KeyValue(row, family, qf2, 1, value), null);
221    this.store.add(new KeyValue(row, family, qf3, 1, value), null);
222    this.store.add(new KeyValue(row, family, qf4, 1, value), null);
223    this.store.add(new KeyValue(row, family, qf5, 1, value), null);
224    this.store.add(new KeyValue(row, family, qf6, 1, value), null);
225
226    Scan scan = new Scan(get);
227    InternalScanner scanner = (InternalScanner) store.getScanner(scan,
228        scan.getFamilyMap().get(store.getColumnFamilyDescriptor().getName()),
229        0);
230
231    List<Cell> results = new ArrayList<>();
232    scanner.next(results);
233    Collections.sort(results, CellComparatorImpl.COMPARATOR);
234    scanner.close();
235
236    //Compare
237    Assert.assertEquals(expected.size(), results.size());
238    for(int i=0; i<results.size(); i++) {
239      // Verify the values
240      Assert.assertEquals(expected.get(i), results.get(i));
241    }
242  }
243
244  /**
245   * Getting MOB data from files
246   * @throws IOException
247   */
248  @Test
249  public void testGetFromFiles() throws IOException {
250    final Configuration conf = TEST_UTIL.getConfiguration();
251    init(name.getMethodName(), conf, false);
252
253    //Put data in memstore
254    this.store.add(new KeyValue(row, family, qf1, 1, value), null);
255    this.store.add(new KeyValue(row, family, qf2, 1, value), null);
256    //flush
257    flush(1);
258
259    //Add more data
260    this.store.add(new KeyValue(row, family, qf3, 1, value), null);
261    this.store.add(new KeyValue(row, family, qf4, 1, value), null);
262    //flush
263    flush(2);
264
265    //Add more data
266    this.store.add(new KeyValue(row, family, qf5, 1, value), null);
267    this.store.add(new KeyValue(row, family, qf6, 1, value), null);
268    //flush
269    flush(3);
270
271    Scan scan = new Scan(get);
272    InternalScanner scanner = (InternalScanner) store.getScanner(scan,
273        scan.getFamilyMap().get(store.getColumnFamilyDescriptor().getName()),
274        0);
275
276    List<Cell> results = new ArrayList<>();
277    scanner.next(results);
278    Collections.sort(results, CellComparatorImpl.COMPARATOR);
279    scanner.close();
280
281    //Compare
282    Assert.assertEquals(expected.size(), results.size());
283    for(int i=0; i<results.size(); i++) {
284      Assert.assertEquals(expected.get(i), results.get(i));
285    }
286  }
287
288  /**
289   * Getting the reference data from files
290   * @throws IOException
291   */
292  @Test
293  public void testGetReferencesFromFiles() throws IOException {
294    final Configuration conf = HBaseConfiguration.create();
295    init(name.getMethodName(), conf, false);
296
297    //Put data in memstore
298    this.store.add(new KeyValue(row, family, qf1, 1, value), null);
299    this.store.add(new KeyValue(row, family, qf2, 1, value), null);
300    //flush
301    flush(1);
302
303    //Add more data
304    this.store.add(new KeyValue(row, family, qf3, 1, value), null);
305    this.store.add(new KeyValue(row, family, qf4, 1, value), null);
306    //flush
307    flush(2);
308
309    //Add more data
310    this.store.add(new KeyValue(row, family, qf5, 1, value), null);
311    this.store.add(new KeyValue(row, family, qf6, 1, value), null);
312    //flush
313    flush(3);
314
315    Scan scan = new Scan(get);
316    scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
317    InternalScanner scanner = (InternalScanner) store.getScanner(scan,
318      scan.getFamilyMap().get(store.getColumnFamilyDescriptor().getName()),
319      0);
320
321    List<Cell> results = new ArrayList<>();
322    scanner.next(results);
323    Collections.sort(results, CellComparatorImpl.COMPARATOR);
324    scanner.close();
325
326    //Compare
327    Assert.assertEquals(expected.size(), results.size());
328    for(int i=0; i<results.size(); i++) {
329      Cell cell = results.get(i);
330      Assert.assertTrue(MobUtils.isMobReferenceCell(cell));
331    }
332  }
333
334  /**
335   * Getting data from memstore and files
336   * @throws IOException
337   */
338  @Test
339  public void testGetFromMemStoreAndFiles() throws IOException {
340
341    final Configuration conf = HBaseConfiguration.create();
342
343    init(name.getMethodName(), conf, false);
344
345    //Put data in memstore
346    this.store.add(new KeyValue(row, family, qf1, 1, value), null);
347    this.store.add(new KeyValue(row, family, qf2, 1, value), null);
348    //flush
349    flush(1);
350
351    //Add more data
352    this.store.add(new KeyValue(row, family, qf3, 1, value), null);
353    this.store.add(new KeyValue(row, family, qf4, 1, value), null);
354    //flush
355    flush(2);
356
357    //Add more data
358    this.store.add(new KeyValue(row, family, qf5, 1, value), null);
359    this.store.add(new KeyValue(row, family, qf6, 1, value), null);
360
361    Scan scan = new Scan(get);
362    InternalScanner scanner = (InternalScanner) store.getScanner(scan,
363        scan.getFamilyMap().get(store.getColumnFamilyDescriptor().getName()),
364        0);
365
366    List<Cell> results = new ArrayList<>();
367    scanner.next(results);
368    Collections.sort(results, CellComparatorImpl.COMPARATOR);
369    scanner.close();
370
371    //Compare
372    Assert.assertEquals(expected.size(), results.size());
373    for(int i=0; i<results.size(); i++) {
374      Assert.assertEquals(expected.get(i), results.get(i));
375    }
376  }
377
378  /**
379   * Getting data from memstore and files
380   * @throws IOException
381   */
382  @Test
383  public void testMobCellSizeThreshold() throws IOException {
384    final Configuration conf = HBaseConfiguration.create();
385    ColumnFamilyDescriptor cfd =
386        ColumnFamilyDescriptorBuilder.newBuilder(family).setMobEnabled(true).setMobThreshold(100)
387            .setMaxVersions(4).build();
388    init(name.getMethodName(), conf, cfd, false);
389
390    //Put data in memstore
391    this.store.add(new KeyValue(row, family, qf1, 1, value), null);
392    this.store.add(new KeyValue(row, family, qf2, 1, value), null);
393    //flush
394    flush(1);
395
396    //Add more data
397    this.store.add(new KeyValue(row, family, qf3, 1, value), null);
398    this.store.add(new KeyValue(row, family, qf4, 1, value), null);
399    //flush
400    flush(2);
401
402    //Add more data
403    this.store.add(new KeyValue(row, family, qf5, 1, value), null);
404    this.store.add(new KeyValue(row, family, qf6, 1, value), null);
405    //flush
406    flush(3);
407
408    Scan scan = new Scan(get);
409    scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
410    InternalScanner scanner = (InternalScanner) store.getScanner(scan,
411      scan.getFamilyMap().get(store.getColumnFamilyDescriptor().getName()),
412      0);
413
414    List<Cell> results = new ArrayList<>();
415    scanner.next(results);
416    Collections.sort(results, CellComparatorImpl.COMPARATOR);
417    scanner.close();
418
419    //Compare
420    Assert.assertEquals(expected.size(), results.size());
421    for(int i=0; i<results.size(); i++) {
422      Cell cell = results.get(i);
423      //this is not mob reference cell.
424      Assert.assertFalse(MobUtils.isMobReferenceCell(cell));
425      Assert.assertEquals(expected.get(i), results.get(i));
426      Assert.assertEquals(100, store.getColumnFamilyDescriptor().getMobThreshold());
427    }
428  }
429
430  @Test
431  public void testCommitFile() throws Exception {
432    final Configuration conf = HBaseConfiguration.create();
433    init(name.getMethodName(), conf, true);
434    String targetPathName = MobUtils.formatDate(new Date());
435    Path targetPath = new Path(store.getPath(), (targetPathName
436        + Path.SEPARATOR + mobFilePath.getName()));
437    fs.delete(targetPath, true);
438    Assert.assertFalse(fs.exists(targetPath));
439    //commit file
440    store.commitFile(mobFilePath, targetPath);
441    Assert.assertTrue(fs.exists(targetPath));
442  }
443
444  @Test
445  public void testResolve() throws Exception {
446    final Configuration conf = HBaseConfiguration.create();
447    init(name.getMethodName(), conf, true);
448    String targetPathName = MobUtils.formatDate(currentDate);
449    Path targetPath = new Path(store.getPath(), targetPathName);
450    store.commitFile(mobFilePath, targetPath);
451    // resolve
452    Cell resultCell1 = store.resolve(seekKey1, false).getCell();
453    Cell resultCell2 = store.resolve(seekKey2, false).getCell();
454    Cell resultCell3 = store.resolve(seekKey3, false).getCell();
455    // compare
456    Assert.assertEquals(Bytes.toString(value), Bytes.toString(CellUtil.cloneValue(resultCell1)));
457    Assert.assertEquals(Bytes.toString(value), Bytes.toString(CellUtil.cloneValue(resultCell2)));
458    Assert.assertEquals(Bytes.toString(value2), Bytes.toString(CellUtil.cloneValue(resultCell3)));
459  }
460
461  /**
462   * Flush the memstore
463   * @param storeFilesSize
464   * @throws IOException
465   */
466  private void flush(int storeFilesSize) throws IOException{
467    this.store.snapshot();
468    flushStore(store, id++);
469    Assert.assertEquals(storeFilesSize, this.store.getStorefiles().size());
470    Assert.assertEquals(0, ((AbstractMemStore)this.store.memstore).getActive().getCellsCount());
471  }
472
473  /**
474   * Flush the memstore
475   * @param store
476   * @param id
477   * @throws IOException
478   */
479  private static void flushStore(HMobStore store, long id) throws IOException {
480    StoreFlushContext storeFlushCtx = store.createFlushContext(id, FlushLifeCycleTracker.DUMMY);
481    storeFlushCtx.prepare();
482    storeFlushCtx.flushCache(Mockito.mock(MonitoredTask.class));
483    storeFlushCtx.commit(Mockito.mock(MonitoredTask.class));
484  }
485
486  @Test
487  public void testMOBStoreEncryption() throws Exception {
488    final Configuration conf = TEST_UTIL.getConfiguration();
489
490    conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyProviderForTesting.class.getName());
491    conf.set(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, "hbase");
492    SecureRandom rng = new SecureRandom();
493    byte[] keyBytes = new byte[AES.KEY_LENGTH];
494    rng.nextBytes(keyBytes);
495    String algorithm = conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES);
496    Key cfKey = new SecretKeySpec(keyBytes, algorithm);
497
498    ColumnFamilyDescriptor cfd =
499        ColumnFamilyDescriptorBuilder.newBuilder(family).setMobEnabled(true).setMobThreshold(100)
500            .setMaxVersions(4).setEncryptionType(algorithm).setEncryptionKey(EncryptionUtil
501            .wrapKey(conf, conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY,
502                User.getCurrent().getShortName()), cfKey)).build();
503    init(name.getMethodName(), conf, cfd, false);
504
505    this.store.add(new KeyValue(row, family, qf1, 1, value), null);
506    this.store.add(new KeyValue(row, family, qf2, 1, value), null);
507    this.store.add(new KeyValue(row, family, qf3, 1, value), null);
508    flush(1);
509
510    this.store.add(new KeyValue(row, family, qf4, 1, value), null);
511    this.store.add(new KeyValue(row, family, qf5, 1, value), null);
512    this.store.add(new KeyValue(row, family, qf6, 1, value), null);
513    flush(2);
514
515    Collection<HStoreFile> storefiles = this.store.getStorefiles();
516    checkMobHFileEncrytption(storefiles);
517
518    // Scan the values
519    Scan scan = new Scan(get);
520    InternalScanner scanner = (InternalScanner) store.getScanner(scan,
521        scan.getFamilyMap().get(store.getColumnFamilyDescriptor().getName()),
522        0);
523
524    List<Cell> results = new ArrayList<>();
525    scanner.next(results);
526    Collections.sort(results, CellComparatorImpl.COMPARATOR);
527    scanner.close();
528    Assert.assertEquals(expected.size(), results.size());
529    for(int i=0; i<results.size(); i++) {
530      Assert.assertEquals(expected.get(i), results.get(i));
531    }
532
533    // Trigger major compaction
534    this.store.triggerMajorCompaction();
535    Optional<CompactionContext> requestCompaction =
536        this.store.requestCompaction(PRIORITY_USER, CompactionLifeCycleTracker.DUMMY, null);
537    this.store.compact(requestCompaction.get(), NoLimitThroughputController.INSTANCE, null);
538    Assert.assertEquals(1, this.store.getStorefiles().size());
539
540    //Check encryption after compaction
541    checkMobHFileEncrytption(this.store.getStorefiles());
542  }
543
544  private void checkMobHFileEncrytption(Collection<HStoreFile> storefiles) {
545    HStoreFile storeFile = storefiles.iterator().next();
546    HFile.Reader reader = storeFile.getReader().getHFileReader();
547    byte[] encryptionKey = reader.getTrailer().getEncryptionKey();
548    Assert.assertTrue(null != encryptionKey);
549    Assert.assertTrue(reader.getFileContext().getEncryptionContext().getCipher().getName()
550        .equals(HConstants.CIPHER_AES));
551  }
552
553}