001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.mob;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertNotNull;
022
023import java.io.IOException;
024import java.util.Date;
025import org.apache.hadoop.conf.Configuration;
026import org.apache.hadoop.fs.FileSystem;
027import org.apache.hadoop.fs.Path;
028import org.apache.hadoop.hbase.HBaseTestingUtil;
029import org.apache.hadoop.hbase.HConstants;
030import org.apache.hadoop.hbase.KeyValue;
031import org.apache.hadoop.hbase.TableName;
032import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
033import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
034import org.apache.hadoop.hbase.client.RegionInfo;
035import org.apache.hadoop.hbase.client.RegionInfoBuilder;
036import org.apache.hadoop.hbase.client.TableDescriptor;
037import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
038import org.apache.hadoop.hbase.io.hfile.CacheConfig;
039import org.apache.hadoop.hbase.regionserver.HMobStore;
040import org.apache.hadoop.hbase.regionserver.HRegion;
041import org.apache.hadoop.hbase.regionserver.StoreContext;
042import org.apache.hadoop.hbase.regionserver.StoreFileWriter;
043import org.apache.hadoop.hbase.testclassification.SmallTests;
044import org.apache.hadoop.hbase.util.Bytes;
045import org.apache.hadoop.hbase.util.Pair;
046import org.junit.jupiter.api.AfterEach;
047import org.junit.jupiter.api.BeforeEach;
048import org.junit.jupiter.api.Tag;
049import org.junit.jupiter.api.Test;
050
051@Tag(SmallTests.TAG)
052public class TestMobFileCache {
053
054  private HBaseTestingUtil UTIL;
055  private HRegion region;
056  private Configuration conf;
057  private MobFileCache mobFileCache;
058  private Date currentDate = new Date();
059  private static final String TEST_CACHE_SIZE = "2";
060  private static final int EXPECTED_CACHE_SIZE_ZERO = 0;
061  private static final int EXPECTED_CACHE_SIZE_ONE = 1;
062  private static final int EXPECTED_CACHE_SIZE_TWO = 2;
063  private static final int EXPECTED_CACHE_SIZE_THREE = 3;
064  private static final long EXPECTED_REFERENCE_ONE = 1;
065  private static final long EXPECTED_REFERENCE_TWO = 2;
066
067  private static final String TABLE = "tableName";
068  private static final String FAMILY1 = "family1";
069  private static final String FAMILY2 = "family2";
070  private static final String FAMILY3 = "family3";
071
072  private static final byte[] ROW = Bytes.toBytes("row");
073  private static final byte[] ROW2 = Bytes.toBytes("row2");
074  private static final byte[] VALUE = Bytes.toBytes("value");
075  private static final byte[] VALUE2 = Bytes.toBytes("value2");
076  private static final byte[] QF1 = Bytes.toBytes("qf1");
077  private static final byte[] QF2 = Bytes.toBytes("qf2");
078  private static final byte[] QF3 = Bytes.toBytes("qf3");
079
080  @BeforeEach
081  public void setUp() throws Exception {
082    UTIL = new HBaseTestingUtil();
083    conf = UTIL.getConfiguration();
084    conf.set(MobConstants.MOB_FILE_CACHE_SIZE_KEY, TEST_CACHE_SIZE);
085    TableDescriptorBuilder tableDescriptorBuilder =
086      TableDescriptorBuilder.newBuilder(UTIL.createTableDescriptor(
087        TableName.valueOf("testMobFileCache"), ColumnFamilyDescriptorBuilder.DEFAULT_MIN_VERSIONS,
088        3, HConstants.FOREVER, ColumnFamilyDescriptorBuilder.DEFAULT_KEEP_DELETED));
089    ColumnFamilyDescriptor columnFamilyDescriptor = ColumnFamilyDescriptorBuilder
090      .newBuilder(Bytes.toBytes(FAMILY1)).setMobEnabled(true).setMobThreshold(0).build();
091    tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
092    columnFamilyDescriptor = ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(FAMILY2))
093      .setMobEnabled(true).setMobThreshold(0).build();
094    tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
095    columnFamilyDescriptor = ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(FAMILY3))
096      .setMobEnabled(true).setMobThreshold(0).build();
097    tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
098    TableDescriptor tableDescriptor = tableDescriptorBuilder.build();
099    RegionInfo regionInfo = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
100    mobFileCache = new MobFileCache(conf);
101    region = HBaseTestingUtil.createRegionAndWAL(regionInfo, UTIL.getDataTestDir(), conf,
102      tableDescriptor, mobFileCache);
103  }
104
105  @AfterEach
106  public void tearDown() throws Exception {
107    region.close();
108    region.getFilesystem().delete(UTIL.getDataTestDir(), true);
109  }
110
111  /**
112   * Create the mob store file.
113   */
114  private Pair<Path, StoreContext> createAndGetMobStoreFileContextPair(String family)
115    throws IOException {
116    ColumnFamilyDescriptor columnFamilyDescriptor = ColumnFamilyDescriptorBuilder
117      .newBuilder(Bytes.toBytes(family)).setMaxVersions(4).setMobEnabled(true).build();
118    // Setting up a Store
119    TableName tn = TableName.valueOf(TABLE);
120    TableDescriptorBuilder tableDescriptorBuilder = TableDescriptorBuilder.newBuilder(tn);
121    tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
122    HMobStore mobStore = (HMobStore) region.getStore(columnFamilyDescriptor.getName());
123    KeyValue key1 = new KeyValue(ROW, columnFamilyDescriptor.getName(), QF1, 1, VALUE);
124    KeyValue key2 = new KeyValue(ROW, columnFamilyDescriptor.getName(), QF2, 1, VALUE);
125    KeyValue key3 = new KeyValue(ROW2, columnFamilyDescriptor.getName(), QF3, 1, VALUE2);
126    KeyValue[] keys = new KeyValue[] { key1, key2, key3 };
127    int maxKeyCount = keys.length;
128    RegionInfo regionInfo = RegionInfoBuilder.newBuilder(tn).build();
129    StoreFileWriter mobWriter = mobStore.createWriterInTmp(currentDate, maxKeyCount,
130      columnFamilyDescriptor.getCompactionCompressionType(), regionInfo.getStartKey(), false);
131    Path mobFilePath = mobWriter.getPath();
132    String fileName = mobFilePath.getName();
133    mobWriter.append(key1);
134    mobWriter.append(key2);
135    mobWriter.append(key3);
136    mobWriter.close();
137    String targetPathName = MobUtils.formatDate(currentDate);
138    Path targetPath = new Path(mobStore.getPath(), targetPathName);
139    mobStore.commitFile(mobFilePath, targetPath);
140    return new Pair(new Path(targetPath, fileName), mobStore.getStoreContext());
141  }
142
143  @Test
144  public void testMobFileCache() throws Exception {
145    FileSystem fs = FileSystem.get(conf);
146    Pair<Path, StoreContext> fileAndContextPair1 = createAndGetMobStoreFileContextPair(FAMILY1);
147    Pair<Path, StoreContext> fileAndContextPair2 = createAndGetMobStoreFileContextPair(FAMILY2);
148    Pair<Path, StoreContext> fileAndContextPair3 = createAndGetMobStoreFileContextPair(FAMILY3);
149
150    Path file1Path = fileAndContextPair1.getFirst();
151    Path file2Path = fileAndContextPair2.getFirst();
152    Path file3Path = fileAndContextPair3.getFirst();
153
154    CacheConfig cacheConf = new CacheConfig(conf);
155    // Before open one file by the MobFileCache
156    assertEquals(EXPECTED_CACHE_SIZE_ZERO, mobFileCache.getCacheSize());
157    // Open one file by the MobFileCache
158    CachedMobFile cachedMobFile1 = (CachedMobFile) mobFileCache.openFile(fs, file1Path, cacheConf,
159      fileAndContextPair1.getSecond());
160    assertEquals(EXPECTED_CACHE_SIZE_ONE, mobFileCache.getCacheSize());
161    assertNotNull(cachedMobFile1);
162    assertEquals(EXPECTED_REFERENCE_TWO, cachedMobFile1.getReferenceCount());
163
164    // The evict is also managed by a schedule thread pool.
165    // And its check period is set as 3600 seconds by default.
166    // This evict should get the lock at the most time
167    mobFileCache.evict(); // Cache not full, evict it
168    assertEquals(EXPECTED_CACHE_SIZE_ONE, mobFileCache.getCacheSize());
169    assertEquals(EXPECTED_REFERENCE_TWO, cachedMobFile1.getReferenceCount());
170
171    mobFileCache.evictFile(file1Path.getName()); // Evict one file
172    assertEquals(EXPECTED_CACHE_SIZE_ZERO, mobFileCache.getCacheSize());
173    assertEquals(EXPECTED_REFERENCE_ONE, cachedMobFile1.getReferenceCount());
174
175    cachedMobFile1.close(); // Close the cached mob file
176
177    // Reopen three cached file
178    cachedMobFile1 = (CachedMobFile) mobFileCache.openFile(fs, file1Path, cacheConf,
179      fileAndContextPair1.getSecond());
180    assertEquals(EXPECTED_CACHE_SIZE_ONE, mobFileCache.getCacheSize());
181    CachedMobFile cachedMobFile2 = (CachedMobFile) mobFileCache.openFile(fs, file2Path, cacheConf,
182      fileAndContextPair2.getSecond());
183    assertEquals(EXPECTED_CACHE_SIZE_TWO, mobFileCache.getCacheSize());
184    CachedMobFile cachedMobFile3 = (CachedMobFile) mobFileCache.openFile(fs, file3Path, cacheConf,
185      fileAndContextPair3.getSecond());
186    // Before the evict
187    // Evict the cache, should close the first file 1
188    assertEquals(EXPECTED_CACHE_SIZE_THREE, mobFileCache.getCacheSize());
189    assertEquals(EXPECTED_REFERENCE_TWO, cachedMobFile1.getReferenceCount());
190    assertEquals(EXPECTED_REFERENCE_TWO, cachedMobFile2.getReferenceCount());
191    assertEquals(EXPECTED_REFERENCE_TWO, cachedMobFile3.getReferenceCount());
192    mobFileCache.evict();
193    assertEquals(EXPECTED_CACHE_SIZE_ONE, mobFileCache.getCacheSize());
194    assertEquals(EXPECTED_REFERENCE_ONE, cachedMobFile1.getReferenceCount());
195    assertEquals(EXPECTED_REFERENCE_ONE, cachedMobFile2.getReferenceCount());
196    assertEquals(EXPECTED_REFERENCE_TWO, cachedMobFile3.getReferenceCount());
197  }
198}