Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.apache.hadoop.hbase.regionserver.Store.PRIORITY_USER;
021
022import java.io.IOException;
023import java.security.Key;
024import java.security.SecureRandom;
025import java.util.ArrayList;
026import java.util.Collection;
027import java.util.Collections;
028import java.util.Date;
029import java.util.Iterator;
030import java.util.List;
031import java.util.NavigableSet;
032import java.util.Optional;
033import java.util.concurrent.ConcurrentSkipListSet;
034import javax.crypto.spec.SecretKeySpec;
035import org.apache.hadoop.conf.Configuration;
036import org.apache.hadoop.fs.FileSystem;
037import org.apache.hadoop.fs.Path;
038import org.apache.hadoop.hbase.ArrayBackedTag;
039import org.apache.hadoop.hbase.Cell;
040import org.apache.hadoop.hbase.CellComparatorImpl;
041import org.apache.hadoop.hbase.CellUtil;
042import org.apache.hadoop.hbase.HBaseClassTestRule;
043import org.apache.hadoop.hbase.HBaseConfiguration;
044import org.apache.hadoop.hbase.HBaseTestingUtility;
045import org.apache.hadoop.hbase.HColumnDescriptor;
046import org.apache.hadoop.hbase.HConstants;
047import org.apache.hadoop.hbase.HRegionInfo;
048import org.apache.hadoop.hbase.HTableDescriptor;
049import org.apache.hadoop.hbase.KeyValue;
050import org.apache.hadoop.hbase.TableName;
051import org.apache.hadoop.hbase.Tag;
052import org.apache.hadoop.hbase.TagType;
053import org.apache.hadoop.hbase.client.Get;
054import org.apache.hadoop.hbase.client.Scan;
055import org.apache.hadoop.hbase.io.crypto.KeyProviderForTesting;
056import org.apache.hadoop.hbase.io.crypto.aes.AES;
057import org.apache.hadoop.hbase.io.hfile.HFile;
058import org.apache.hadoop.hbase.mob.MobConstants;
059import org.apache.hadoop.hbase.mob.MobUtils;
060import org.apache.hadoop.hbase.monitoring.MonitoredTask;
061import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
062import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker;
063import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController;
064import org.apache.hadoop.hbase.security.EncryptionUtil;
065import org.apache.hadoop.hbase.security.User;
066import org.apache.hadoop.hbase.testclassification.MediumTests;
067import org.apache.hadoop.hbase.util.Bytes;
068import org.apache.hadoop.hbase.util.FSUtils;
069import org.apache.hadoop.hbase.wal.WALFactory;
070import org.junit.Assert;
071import org.junit.Before;
072import org.junit.ClassRule;
073import org.junit.Rule;
074import org.junit.Test;
075import org.junit.experimental.categories.Category;
076import org.junit.rules.TestName;
077import org.mockito.Mockito;
078import org.slf4j.Logger;
079import org.slf4j.LoggerFactory;
080
081@Category(MediumTests.class)
082public class TestHMobStore {
083
084  @ClassRule
085  public static final HBaseClassTestRule CLASS_RULE =
086      HBaseClassTestRule.forClass(TestHMobStore.class);
087
088  public static final Logger LOG = LoggerFactory.getLogger(TestHMobStore.class);
089  @Rule public TestName name = new TestName();
090
091  private HMobStore store;
092  private HRegion region;
093  private HColumnDescriptor hcd;
094  private FileSystem fs;
095  private byte [] table = Bytes.toBytes("table");
096  private byte [] family = Bytes.toBytes("family");
097  private byte [] row = Bytes.toBytes("row");
098  private byte [] row2 = Bytes.toBytes("row2");
099  private byte [] qf1 = Bytes.toBytes("qf1");
100  private byte [] qf2 = Bytes.toBytes("qf2");
101  private byte [] qf3 = Bytes.toBytes("qf3");
102  private byte [] qf4 = Bytes.toBytes("qf4");
103  private byte [] qf5 = Bytes.toBytes("qf5");
104  private byte [] qf6 = Bytes.toBytes("qf6");
105  private byte[] value = Bytes.toBytes("value");
106  private byte[] value2 = Bytes.toBytes("value2");
107  private Path mobFilePath;
108  private Date currentDate = new Date();
109  private Cell seekKey1;
110  private Cell seekKey2;
111  private Cell seekKey3;
112  private NavigableSet<byte[]> qualifiers = new ConcurrentSkipListSet<>(Bytes.BYTES_COMPARATOR);
113  private List<Cell> expected = new ArrayList<>();
114  private long id = System.currentTimeMillis();
115  private Get get = new Get(row);
116  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
117  private final String DIR = TEST_UTIL.getDataTestDir("TestHMobStore").toString();
118
119  /**
120   * Setup
121   * @throws Exception
122   */
123  @Before
124  public void setUp() throws Exception {
125    qualifiers.add(qf1);
126    qualifiers.add(qf3);
127    qualifiers.add(qf5);
128
129    Iterator<byte[]> iter = qualifiers.iterator();
130    while(iter.hasNext()){
131      byte [] next = iter.next();
132      expected.add(new KeyValue(row, family, next, 1, value));
133      get.addColumn(family, next);
134      get.setMaxVersions(); // all versions.
135    }
136  }
137
138  private void init(String methodName, Configuration conf, boolean testStore)
139  throws IOException {
140    hcd = new HColumnDescriptor(family);
141    hcd.setMobEnabled(true);
142    hcd.setMobThreshold(3L);
143    hcd.setMaxVersions(4);
144    init(methodName, conf, hcd, testStore);
145  }
146
147  private void init(String methodName, Configuration conf,
148      HColumnDescriptor hcd, boolean testStore) throws IOException {
149    HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table));
150    init(methodName, conf, htd, hcd, testStore);
151  }
152
153  private void init(String methodName, Configuration conf, HTableDescriptor htd,
154      HColumnDescriptor hcd, boolean testStore) throws IOException {
155    //Setting up tje Region and Store
156    Path basedir = new Path(DIR+methodName);
157    Path tableDir = FSUtils.getTableDir(basedir, htd.getTableName());
158    String logName = "logs";
159    Path logdir = new Path(basedir, logName);
160    FileSystem fs = FileSystem.get(conf);
161    fs.delete(logdir, true);
162
163    htd.addFamily(hcd);
164    HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
165    ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null);
166    final Configuration walConf = new Configuration(conf);
167    FSUtils.setRootDir(walConf, basedir);
168    final WALFactory wals = new WALFactory(walConf, methodName);
169    region = new HRegion(tableDir, wals.getWAL(info), fs, conf, info, htd, null);
170    store = new HMobStore(region, hcd, conf);
171    if(testStore) {
172      init(conf, hcd);
173    }
174  }
175
176  private void init(Configuration conf, HColumnDescriptor hcd)
177      throws IOException {
178    Path basedir = FSUtils.getRootDir(conf);
179    fs = FileSystem.get(conf);
180    Path homePath = new Path(basedir, Bytes.toString(family) + Path.SEPARATOR
181        + Bytes.toString(family));
182    fs.mkdirs(homePath);
183
184    KeyValue key1 = new KeyValue(row, family, qf1, 1, value);
185    KeyValue key2 = new KeyValue(row, family, qf2, 1, value);
186    KeyValue key3 = new KeyValue(row2, family, qf3, 1, value2);
187    KeyValue[] keys = new KeyValue[] { key1, key2, key3 };
188    int maxKeyCount = keys.length;
189    StoreFileWriter mobWriter = store.createWriterInTmp(currentDate, maxKeyCount,
190        hcd.getCompactionCompressionType(), region.getRegionInfo().getStartKey(), false);
191    mobFilePath = mobWriter.getPath();
192
193    mobWriter.append(key1);
194    mobWriter.append(key2);
195    mobWriter.append(key3);
196    mobWriter.close();
197
198    String targetPathName = MobUtils.formatDate(currentDate);
199    byte[] referenceValue = Bytes.toBytes(targetPathName + Path.SEPARATOR + mobFilePath.getName());
200    Tag tableNameTag = new ArrayBackedTag(TagType.MOB_TABLE_NAME_TAG_TYPE,
201        store.getTableName().getName());
202    KeyValue kv1 = new KeyValue(row, family, qf1, Long.MAX_VALUE, referenceValue);
203    KeyValue kv2 = new KeyValue(row, family, qf2, Long.MAX_VALUE, referenceValue);
204    KeyValue kv3 = new KeyValue(row2, family, qf3, Long.MAX_VALUE, referenceValue);
205    seekKey1 = MobUtils.createMobRefCell(kv1, referenceValue, tableNameTag);
206    seekKey2 = MobUtils.createMobRefCell(kv2, referenceValue, tableNameTag);
207    seekKey3 = MobUtils.createMobRefCell(kv3, referenceValue, tableNameTag);
208  }
209
210  /**
211   * Getting data from memstore
212   * @throws IOException
213   */
214  @Test
215  public void testGetFromMemStore() throws IOException {
216    final Configuration conf = HBaseConfiguration.create();
217    init(name.getMethodName(), conf, false);
218
219    //Put data in memstore
220    this.store.add(new KeyValue(row, family, qf1, 1, value), null);
221    this.store.add(new KeyValue(row, family, qf2, 1, value), null);
222    this.store.add(new KeyValue(row, family, qf3, 1, value), null);
223    this.store.add(new KeyValue(row, family, qf4, 1, value), null);
224    this.store.add(new KeyValue(row, family, qf5, 1, value), null);
225    this.store.add(new KeyValue(row, family, qf6, 1, value), null);
226
227    Scan scan = new Scan(get);
228    InternalScanner scanner = (InternalScanner) store.getScanner(scan,
229        scan.getFamilyMap().get(store.getColumnFamilyDescriptor().getName()),
230        0);
231
232    List<Cell> results = new ArrayList<>();
233    scanner.next(results);
234    Collections.sort(results, CellComparatorImpl.COMPARATOR);
235    scanner.close();
236
237    //Compare
238    Assert.assertEquals(expected.size(), results.size());
239    for(int i=0; i<results.size(); i++) {
240      // Verify the values
241      Assert.assertEquals(expected.get(i), results.get(i));
242    }
243  }
244
245  /**
246   * Getting MOB data from files
247   * @throws IOException
248   */
249  @Test
250  public void testGetFromFiles() throws IOException {
251    final Configuration conf = TEST_UTIL.getConfiguration();
252    init(name.getMethodName(), conf, false);
253
254    //Put data in memstore
255    this.store.add(new KeyValue(row, family, qf1, 1, value), null);
256    this.store.add(new KeyValue(row, family, qf2, 1, value), null);
257    //flush
258    flush(1);
259
260    //Add more data
261    this.store.add(new KeyValue(row, family, qf3, 1, value), null);
262    this.store.add(new KeyValue(row, family, qf4, 1, value), null);
263    //flush
264    flush(2);
265
266    //Add more data
267    this.store.add(new KeyValue(row, family, qf5, 1, value), null);
268    this.store.add(new KeyValue(row, family, qf6, 1, value), null);
269    //flush
270    flush(3);
271
272    Scan scan = new Scan(get);
273    InternalScanner scanner = (InternalScanner) store.getScanner(scan,
274        scan.getFamilyMap().get(store.getColumnFamilyDescriptor().getName()),
275        0);
276
277    List<Cell> results = new ArrayList<>();
278    scanner.next(results);
279    Collections.sort(results, CellComparatorImpl.COMPARATOR);
280    scanner.close();
281
282    //Compare
283    Assert.assertEquals(expected.size(), results.size());
284    for(int i=0; i<results.size(); i++) {
285      Assert.assertEquals(expected.get(i), results.get(i));
286    }
287  }
288
289  /**
290   * Getting the reference data from files
291   * @throws IOException
292   */
293  @Test
294  public void testGetReferencesFromFiles() throws IOException {
295    final Configuration conf = HBaseConfiguration.create();
296    init(name.getMethodName(), conf, false);
297
298    //Put data in memstore
299    this.store.add(new KeyValue(row, family, qf1, 1, value), null);
300    this.store.add(new KeyValue(row, family, qf2, 1, value), null);
301    //flush
302    flush(1);
303
304    //Add more data
305    this.store.add(new KeyValue(row, family, qf3, 1, value), null);
306    this.store.add(new KeyValue(row, family, qf4, 1, value), null);
307    //flush
308    flush(2);
309
310    //Add more data
311    this.store.add(new KeyValue(row, family, qf5, 1, value), null);
312    this.store.add(new KeyValue(row, family, qf6, 1, value), null);
313    //flush
314    flush(3);
315
316    Scan scan = new Scan(get);
317    scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
318    InternalScanner scanner = (InternalScanner) store.getScanner(scan,
319      scan.getFamilyMap().get(store.getColumnFamilyDescriptor().getName()),
320      0);
321
322    List<Cell> results = new ArrayList<>();
323    scanner.next(results);
324    Collections.sort(results, CellComparatorImpl.COMPARATOR);
325    scanner.close();
326
327    //Compare
328    Assert.assertEquals(expected.size(), results.size());
329    for(int i=0; i<results.size(); i++) {
330      Cell cell = results.get(i);
331      Assert.assertTrue(MobUtils.isMobReferenceCell(cell));
332    }
333  }
334
335  /**
336   * Getting data from memstore and files
337   * @throws IOException
338   */
339  @Test
340  public void testGetFromMemStoreAndFiles() throws IOException {
341
342    final Configuration conf = HBaseConfiguration.create();
343
344    init(name.getMethodName(), conf, false);
345
346    //Put data in memstore
347    this.store.add(new KeyValue(row, family, qf1, 1, value), null);
348    this.store.add(new KeyValue(row, family, qf2, 1, value), null);
349    //flush
350    flush(1);
351
352    //Add more data
353    this.store.add(new KeyValue(row, family, qf3, 1, value), null);
354    this.store.add(new KeyValue(row, family, qf4, 1, value), null);
355    //flush
356    flush(2);
357
358    //Add more data
359    this.store.add(new KeyValue(row, family, qf5, 1, value), null);
360    this.store.add(new KeyValue(row, family, qf6, 1, value), null);
361
362    Scan scan = new Scan(get);
363    InternalScanner scanner = (InternalScanner) store.getScanner(scan,
364        scan.getFamilyMap().get(store.getColumnFamilyDescriptor().getName()),
365        0);
366
367    List<Cell> results = new ArrayList<>();
368    scanner.next(results);
369    Collections.sort(results, CellComparatorImpl.COMPARATOR);
370    scanner.close();
371
372    //Compare
373    Assert.assertEquals(expected.size(), results.size());
374    for(int i=0; i<results.size(); i++) {
375      Assert.assertEquals(expected.get(i), results.get(i));
376    }
377  }
378
379  /**
380   * Getting data from memstore and files
381   * @throws IOException
382   */
383  @Test
384  public void testMobCellSizeThreshold() throws IOException {
385
386    final Configuration conf = HBaseConfiguration.create();
387
388    HColumnDescriptor hcd;
389    hcd = new HColumnDescriptor(family);
390    hcd.setMobEnabled(true);
391    hcd.setMobThreshold(100);
392    hcd.setMaxVersions(4);
393    init(name.getMethodName(), conf, hcd, false);
394
395    //Put data in memstore
396    this.store.add(new KeyValue(row, family, qf1, 1, value), null);
397    this.store.add(new KeyValue(row, family, qf2, 1, value), null);
398    //flush
399    flush(1);
400
401    //Add more data
402    this.store.add(new KeyValue(row, family, qf3, 1, value), null);
403    this.store.add(new KeyValue(row, family, qf4, 1, value), null);
404    //flush
405    flush(2);
406
407    //Add more data
408    this.store.add(new KeyValue(row, family, qf5, 1, value), null);
409    this.store.add(new KeyValue(row, family, qf6, 1, value), null);
410    //flush
411    flush(3);
412
413    Scan scan = new Scan(get);
414    scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
415    InternalScanner scanner = (InternalScanner) store.getScanner(scan,
416      scan.getFamilyMap().get(store.getColumnFamilyDescriptor().getName()),
417      0);
418
419    List<Cell> results = new ArrayList<>();
420    scanner.next(results);
421    Collections.sort(results, CellComparatorImpl.COMPARATOR);
422    scanner.close();
423
424    //Compare
425    Assert.assertEquals(expected.size(), results.size());
426    for(int i=0; i<results.size(); i++) {
427      Cell cell = results.get(i);
428      //this is not mob reference cell.
429      Assert.assertFalse(MobUtils.isMobReferenceCell(cell));
430      Assert.assertEquals(expected.get(i), results.get(i));
431      Assert.assertEquals(100, store.getColumnFamilyDescriptor().getMobThreshold());
432    }
433  }
434
435  @Test
436  public void testCommitFile() throws Exception {
437    final Configuration conf = HBaseConfiguration.create();
438    init(name.getMethodName(), conf, true);
439    String targetPathName = MobUtils.formatDate(new Date());
440    Path targetPath = new Path(store.getPath(), (targetPathName
441        + Path.SEPARATOR + mobFilePath.getName()));
442    fs.delete(targetPath, true);
443    Assert.assertFalse(fs.exists(targetPath));
444    //commit file
445    store.commitFile(mobFilePath, targetPath);
446    Assert.assertTrue(fs.exists(targetPath));
447  }
448
449  @Test
450  public void testResolve() throws Exception {
451    final Configuration conf = HBaseConfiguration.create();
452    init(name.getMethodName(), conf, true);
453    String targetPathName = MobUtils.formatDate(currentDate);
454    Path targetPath = new Path(store.getPath(), targetPathName);
455    store.commitFile(mobFilePath, targetPath);
456    //resolve
457    Cell resultCell1 = store.resolve(seekKey1, false);
458    Cell resultCell2 = store.resolve(seekKey2, false);
459    Cell resultCell3 = store.resolve(seekKey3, false);
460    //compare
461    Assert.assertEquals(Bytes.toString(value),
462        Bytes.toString(CellUtil.cloneValue(resultCell1)));
463    Assert.assertEquals(Bytes.toString(value),
464        Bytes.toString(CellUtil.cloneValue(resultCell2)));
465    Assert.assertEquals(Bytes.toString(value2),
466        Bytes.toString(CellUtil.cloneValue(resultCell3)));
467  }
468
469  /**
470   * Flush the memstore
471   * @param storeFilesSize
472   * @throws IOException
473   */
474  private void flush(int storeFilesSize) throws IOException{
475    this.store.snapshot();
476    flushStore(store, id++);
477    Assert.assertEquals(storeFilesSize, this.store.getStorefiles().size());
478    Assert.assertEquals(0, ((AbstractMemStore)this.store.memstore).getActive().getCellsCount());
479  }
480
481  /**
482   * Flush the memstore
483   * @param store
484   * @param id
485   * @throws IOException
486   */
487  private static void flushStore(HMobStore store, long id) throws IOException {
488    StoreFlushContext storeFlushCtx = store.createFlushContext(id, FlushLifeCycleTracker.DUMMY);
489    storeFlushCtx.prepare();
490    storeFlushCtx.flushCache(Mockito.mock(MonitoredTask.class));
491    storeFlushCtx.commit(Mockito.mock(MonitoredTask.class));
492  }
493
494  @Test
495  public void testMOBStoreEncryption() throws Exception {
496    final Configuration conf = TEST_UTIL.getConfiguration();
497
498    conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyProviderForTesting.class.getName());
499    conf.set(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, "hbase");
500    SecureRandom rng = new SecureRandom();
501    byte[] keyBytes = new byte[AES.KEY_LENGTH];
502    rng.nextBytes(keyBytes);
503    String algorithm = conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES);
504    Key cfKey = new SecretKeySpec(keyBytes, algorithm);
505
506    HColumnDescriptor hcd = new HColumnDescriptor(family);
507    hcd.setMobEnabled(true);
508    hcd.setMobThreshold(100);
509    hcd.setMaxVersions(4);
510    hcd.setEncryptionType(algorithm);
511    hcd.setEncryptionKey(EncryptionUtil.wrapKey(conf,
512      conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, User.getCurrent().getShortName()),cfKey));
513
514    init(name.getMethodName(), conf, hcd, false);
515
516    this.store.add(new KeyValue(row, family, qf1, 1, value), null);
517    this.store.add(new KeyValue(row, family, qf2, 1, value), null);
518    this.store.add(new KeyValue(row, family, qf3, 1, value), null);
519    flush(1);
520
521    this.store.add(new KeyValue(row, family, qf4, 1, value), null);
522    this.store.add(new KeyValue(row, family, qf5, 1, value), null);
523    this.store.add(new KeyValue(row, family, qf6, 1, value), null);
524    flush(2);
525
526    Collection<HStoreFile> storefiles = this.store.getStorefiles();
527    checkMobHFileEncrytption(storefiles);
528
529    // Scan the values
530    Scan scan = new Scan(get);
531    InternalScanner scanner = (InternalScanner) store.getScanner(scan,
532        scan.getFamilyMap().get(store.getColumnFamilyDescriptor().getName()),
533        0);
534
535    List<Cell> results = new ArrayList<>();
536    scanner.next(results);
537    Collections.sort(results, CellComparatorImpl.COMPARATOR);
538    scanner.close();
539    Assert.assertEquals(expected.size(), results.size());
540    for(int i=0; i<results.size(); i++) {
541      Assert.assertEquals(expected.get(i), results.get(i));
542    }
543
544    // Trigger major compaction
545    this.store.triggerMajorCompaction();
546    Optional<CompactionContext> requestCompaction =
547        this.store.requestCompaction(PRIORITY_USER, CompactionLifeCycleTracker.DUMMY, null);
548    this.store.compact(requestCompaction.get(), NoLimitThroughputController.INSTANCE, null);
549    Assert.assertEquals(1, this.store.getStorefiles().size());
550
551    //Check encryption after compaction
552    checkMobHFileEncrytption(this.store.getStorefiles());
553  }
554
555  private void checkMobHFileEncrytption(Collection<HStoreFile> storefiles) {
556    HStoreFile storeFile = storefiles.iterator().next();
557    HFile.Reader reader = storeFile.getReader().getHFileReader();
558    byte[] encryptionKey = reader.getTrailer().getEncryptionKey();
559    Assert.assertTrue(null != encryptionKey);
560    Assert.assertTrue(reader.getFileContext().getEncryptionContext().getCipher().getName()
561        .equals(HConstants.CIPHER_AES));
562  }
563
564}