001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.mob;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNotNull;
022
023import java.io.IOException;
024import java.util.Date;
025import org.apache.hadoop.conf.Configuration;
026import org.apache.hadoop.fs.FileSystem;
027import org.apache.hadoop.fs.Path;
028import org.apache.hadoop.hbase.HBaseClassTestRule;
029import org.apache.hadoop.hbase.HBaseConfiguration;
030import org.apache.hadoop.hbase.HBaseTestingUtility;
031import org.apache.hadoop.hbase.HColumnDescriptor;
032import org.apache.hadoop.hbase.HRegionInfo;
033import org.apache.hadoop.hbase.HTableDescriptor;
034import org.apache.hadoop.hbase.KeyValue;
035import org.apache.hadoop.hbase.TableName;
036import org.apache.hadoop.hbase.regionserver.HMobStore;
037import org.apache.hadoop.hbase.regionserver.HRegion;
038import org.apache.hadoop.hbase.regionserver.StoreFileWriter;
039import org.apache.hadoop.hbase.testclassification.SmallTests;
040import org.apache.hadoop.hbase.util.Bytes;
041import org.junit.After;
042import org.junit.Before;
043import org.junit.ClassRule;
044import org.junit.Test;
045import org.junit.experimental.categories.Category;
046import org.slf4j.Logger;
047import org.slf4j.LoggerFactory;
048
049@Category(SmallTests.class)
050public class TestMobFileCache {
051
052  @ClassRule
053  public static final HBaseClassTestRule CLASS_RULE =
054      HBaseClassTestRule.forClass(TestMobFileCache.class);
055
056  static final Logger LOG = LoggerFactory.getLogger(TestMobFileCache.class);
057  private HBaseTestingUtility UTIL;
058  private HRegion region;
059  private Configuration conf;
060  private MobCacheConfig mobCacheConf;
061  private MobFileCache mobFileCache;
062  private Date currentDate = new Date();
063  private static final String TEST_CACHE_SIZE = "2";
064  private static final int EXPECTED_CACHE_SIZE_ZERO = 0;
065  private static final int EXPECTED_CACHE_SIZE_ONE = 1;
066  private static final int EXPECTED_CACHE_SIZE_TWO = 2;
067  private static final int EXPECTED_CACHE_SIZE_THREE = 3;
068  private static final long EXPECTED_REFERENCE_ONE = 1;
069  private static final long EXPECTED_REFERENCE_TWO = 2;
070
071  private static final String TABLE = "tableName";
072  private static final String FAMILY1 = "family1";
073  private static final String FAMILY2 = "family2";
074  private static final String FAMILY3 = "family3";
075
076  private static final byte[] ROW = Bytes.toBytes("row");
077  private static final byte[] ROW2 = Bytes.toBytes("row2");
078  private static final byte[] VALUE = Bytes.toBytes("value");
079  private static final byte[] VALUE2 = Bytes.toBytes("value2");
080  private static final byte[] QF1 = Bytes.toBytes("qf1");
081  private static final byte[] QF2 = Bytes.toBytes("qf2");
082  private static final byte[] QF3 = Bytes.toBytes("qf3");
083
084  @Before
085  public void setUp() throws Exception {
086    UTIL = HBaseTestingUtility.createLocalHTU();
087    conf = UTIL.getConfiguration();
088    HTableDescriptor htd = UTIL.createTableDescriptor("testMobFileCache");
089    HColumnDescriptor hcd1 = new HColumnDescriptor(FAMILY1);
090    hcd1.setMobEnabled(true);
091    hcd1.setMobThreshold(0);
092    HColumnDescriptor hcd2 = new HColumnDescriptor(FAMILY2);
093    hcd2.setMobEnabled(true);
094    hcd2.setMobThreshold(0);
095    HColumnDescriptor hcd3 = new HColumnDescriptor(FAMILY3);
096    hcd3.setMobEnabled(true);
097    hcd3.setMobThreshold(0);
098    htd.addFamily(hcd1);
099    htd.addFamily(hcd2);
100    htd.addFamily(hcd3);
101    region = UTIL.createLocalHRegion(htd, null, null);
102  }
103
104  @After
105  public void tearDown() throws Exception {
106    region.close();
107    region.getFilesystem().delete(UTIL.getDataTestDir(), true);
108  }
109
110  /**
111   * Create the mob store file.
112   */
113  private Path createMobStoreFile(String family) throws IOException {
114    return createMobStoreFile(HBaseConfiguration.create(), family);
115  }
116
117  /**
118   * Create the mob store file
119   */
120  private Path createMobStoreFile(Configuration conf, String family) throws IOException {
121    HColumnDescriptor hcd = new HColumnDescriptor(family);
122    hcd.setMaxVersions(4);
123    hcd.setMobEnabled(true);
124    mobCacheConf = new MobCacheConfig(conf, hcd);
125    return createMobStoreFile(hcd);
126  }
127
128  /**
129   * Create the mob store file
130   */
131  private Path createMobStoreFile(HColumnDescriptor hcd)
132      throws IOException {
133    // Setting up a Store
134    TableName tn = TableName.valueOf(TABLE);
135    HTableDescriptor htd = new HTableDescriptor(tn);
136    htd.addFamily(hcd);
137    HMobStore mobStore = (HMobStore) region.getStore(hcd.getName());
138    KeyValue key1 = new KeyValue(ROW, hcd.getName(), QF1, 1, VALUE);
139    KeyValue key2 = new KeyValue(ROW, hcd.getName(), QF2, 1, VALUE);
140    KeyValue key3 = new KeyValue(ROW2, hcd.getName(), QF3, 1, VALUE2);
141    KeyValue[] keys = new KeyValue[] { key1, key2, key3 };
142    int maxKeyCount = keys.length;
143    HRegionInfo regionInfo = new HRegionInfo(tn);
144    StoreFileWriter mobWriter = mobStore.createWriterInTmp(currentDate,
145        maxKeyCount, hcd.getCompactionCompression(), regionInfo.getStartKey(), false);
146    Path mobFilePath = mobWriter.getPath();
147    String fileName = mobFilePath.getName();
148    mobWriter.append(key1);
149    mobWriter.append(key2);
150    mobWriter.append(key3);
151    mobWriter.close();
152    String targetPathName = MobUtils.formatDate(currentDate);
153    Path targetPath = new Path(mobStore.getPath(), targetPathName);
154    mobStore.commitFile(mobFilePath, targetPath);
155    return new Path(targetPath, fileName);
156  }
157
158  @Test
159  public void testMobFileCache() throws Exception {
160    FileSystem fs = FileSystem.get(conf);
161    conf.set(MobConstants.MOB_FILE_CACHE_SIZE_KEY, TEST_CACHE_SIZE);
162    mobFileCache = new MobFileCache(conf);
163    Path file1Path = createMobStoreFile(FAMILY1);
164    Path file2Path = createMobStoreFile(FAMILY2);
165    Path file3Path = createMobStoreFile(FAMILY3);
166
167    // Before open one file by the MobFileCache
168    assertEquals(EXPECTED_CACHE_SIZE_ZERO, mobFileCache.getCacheSize());
169    // Open one file by the MobFileCache
170    CachedMobFile cachedMobFile1 = (CachedMobFile) mobFileCache.openFile(
171        fs, file1Path, mobCacheConf);
172    assertEquals(EXPECTED_CACHE_SIZE_ONE, mobFileCache.getCacheSize());
173    assertNotNull(cachedMobFile1);
174    assertEquals(EXPECTED_REFERENCE_TWO, cachedMobFile1.getReferenceCount());
175
176    // The evict is also managed by a schedule thread pool.
177    // And its check period is set as 3600 seconds by default.
178    // This evict should get the lock at the most time
179    mobFileCache.evict();  // Cache not full, evict it
180    assertEquals(EXPECTED_CACHE_SIZE_ONE, mobFileCache.getCacheSize());
181    assertEquals(EXPECTED_REFERENCE_TWO, cachedMobFile1.getReferenceCount());
182
183    mobFileCache.evictFile(file1Path.getName());  // Evict one file
184    assertEquals(EXPECTED_CACHE_SIZE_ZERO, mobFileCache.getCacheSize());
185    assertEquals(EXPECTED_REFERENCE_ONE, cachedMobFile1.getReferenceCount());
186
187    cachedMobFile1.close();  // Close the cached mob file
188
189    // Reopen three cached file
190    cachedMobFile1 = (CachedMobFile) mobFileCache.openFile(
191        fs, file1Path, mobCacheConf);
192    assertEquals(EXPECTED_CACHE_SIZE_ONE, mobFileCache.getCacheSize());
193    CachedMobFile cachedMobFile2 = (CachedMobFile) mobFileCache.openFile(
194        fs, file2Path, mobCacheConf);
195    assertEquals(EXPECTED_CACHE_SIZE_TWO, mobFileCache.getCacheSize());
196    CachedMobFile cachedMobFile3 = (CachedMobFile) mobFileCache.openFile(
197        fs, file3Path, mobCacheConf);
198    // Before the evict
199    // Evict the cache, should close the first file 1
200    assertEquals(EXPECTED_CACHE_SIZE_THREE, mobFileCache.getCacheSize());
201    assertEquals(EXPECTED_REFERENCE_TWO, cachedMobFile1.getReferenceCount());
202    assertEquals(EXPECTED_REFERENCE_TWO, cachedMobFile2.getReferenceCount());
203    assertEquals(EXPECTED_REFERENCE_TWO, cachedMobFile3.getReferenceCount());
204    mobFileCache.evict();
205    assertEquals(EXPECTED_CACHE_SIZE_ONE, mobFileCache.getCacheSize());
206    assertEquals(EXPECTED_REFERENCE_ONE, cachedMobFile1.getReferenceCount());
207    assertEquals(EXPECTED_REFERENCE_ONE, cachedMobFile2.getReferenceCount());
208    assertEquals(EXPECTED_REFERENCE_TWO, cachedMobFile3.getReferenceCount());
209  }
210}