001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.mob; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertNotNull; 022 023import java.io.IOException; 024import java.util.Date; 025import org.apache.hadoop.conf.Configuration; 026import org.apache.hadoop.fs.FileSystem; 027import org.apache.hadoop.fs.Path; 028import org.apache.hadoop.hbase.HBaseClassTestRule; 029import org.apache.hadoop.hbase.HBaseConfiguration; 030import org.apache.hadoop.hbase.HBaseTestingUtility; 031import org.apache.hadoop.hbase.HColumnDescriptor; 032import org.apache.hadoop.hbase.HRegionInfo; 033import org.apache.hadoop.hbase.HTableDescriptor; 034import org.apache.hadoop.hbase.KeyValue; 035import org.apache.hadoop.hbase.TableName; 036import org.apache.hadoop.hbase.regionserver.HMobStore; 037import org.apache.hadoop.hbase.regionserver.HRegion; 038import org.apache.hadoop.hbase.regionserver.StoreFileWriter; 039import org.apache.hadoop.hbase.testclassification.SmallTests; 040import org.apache.hadoop.hbase.util.Bytes; 041import org.junit.After; 042import org.junit.Before; 043import org.junit.ClassRule; 044import org.junit.Test; 045import org.junit.experimental.categories.Category; 046import org.slf4j.Logger; 047import org.slf4j.LoggerFactory; 048 049@Category(SmallTests.class) 050public class TestMobFileCache { 051 052 @ClassRule 053 public static final HBaseClassTestRule CLASS_RULE = 054 HBaseClassTestRule.forClass(TestMobFileCache.class); 055 056 static final Logger LOG = LoggerFactory.getLogger(TestMobFileCache.class); 057 private HBaseTestingUtility UTIL; 058 private HRegion region; 059 private Configuration conf; 060 private MobCacheConfig mobCacheConf; 061 private MobFileCache mobFileCache; 062 private Date currentDate = new Date(); 063 private static final String TEST_CACHE_SIZE = "2"; 064 private static final int EXPECTED_CACHE_SIZE_ZERO = 0; 065 private static final int EXPECTED_CACHE_SIZE_ONE = 1; 066 private static final int EXPECTED_CACHE_SIZE_TWO = 2; 067 private static final int EXPECTED_CACHE_SIZE_THREE = 3; 068 private static final long EXPECTED_REFERENCE_ONE = 1; 069 private static final long EXPECTED_REFERENCE_TWO = 2; 070 071 private static final String TABLE = "tableName"; 072 private static final String FAMILY1 = "family1"; 073 private static final String FAMILY2 = "family2"; 074 private static final String FAMILY3 = "family3"; 075 076 private static final byte[] ROW = Bytes.toBytes("row"); 077 private static final byte[] ROW2 = Bytes.toBytes("row2"); 078 private static final byte[] VALUE = Bytes.toBytes("value"); 079 private static final byte[] VALUE2 = Bytes.toBytes("value2"); 080 private static final byte[] QF1 = Bytes.toBytes("qf1"); 081 private static final byte[] QF2 = Bytes.toBytes("qf2"); 082 private static final byte[] QF3 = Bytes.toBytes("qf3"); 083 084 @Before 085 public void setUp() throws Exception { 086 UTIL = HBaseTestingUtility.createLocalHTU(); 087 conf = UTIL.getConfiguration(); 088 HTableDescriptor htd = UTIL.createTableDescriptor("testMobFileCache"); 089 HColumnDescriptor hcd1 = new HColumnDescriptor(FAMILY1); 090 hcd1.setMobEnabled(true); 091 hcd1.setMobThreshold(0); 092 HColumnDescriptor hcd2 = new HColumnDescriptor(FAMILY2); 093 hcd2.setMobEnabled(true); 094 hcd2.setMobThreshold(0); 095 HColumnDescriptor hcd3 = new HColumnDescriptor(FAMILY3); 096 hcd3.setMobEnabled(true); 097 hcd3.setMobThreshold(0); 098 htd.addFamily(hcd1); 099 htd.addFamily(hcd2); 100 htd.addFamily(hcd3); 101 region = UTIL.createLocalHRegion(htd, null, null); 102 } 103 104 @After 105 public void tearDown() throws Exception { 106 region.close(); 107 region.getFilesystem().delete(UTIL.getDataTestDir(), true); 108 } 109 110 /** 111 * Create the mob store file. 112 */ 113 private Path createMobStoreFile(String family) throws IOException { 114 return createMobStoreFile(HBaseConfiguration.create(), family); 115 } 116 117 /** 118 * Create the mob store file 119 */ 120 private Path createMobStoreFile(Configuration conf, String family) throws IOException { 121 HColumnDescriptor hcd = new HColumnDescriptor(family); 122 hcd.setMaxVersions(4); 123 hcd.setMobEnabled(true); 124 mobCacheConf = new MobCacheConfig(conf, hcd); 125 return createMobStoreFile(hcd); 126 } 127 128 /** 129 * Create the mob store file 130 */ 131 private Path createMobStoreFile(HColumnDescriptor hcd) 132 throws IOException { 133 // Setting up a Store 134 TableName tn = TableName.valueOf(TABLE); 135 HTableDescriptor htd = new HTableDescriptor(tn); 136 htd.addFamily(hcd); 137 HMobStore mobStore = (HMobStore) region.getStore(hcd.getName()); 138 KeyValue key1 = new KeyValue(ROW, hcd.getName(), QF1, 1, VALUE); 139 KeyValue key2 = new KeyValue(ROW, hcd.getName(), QF2, 1, VALUE); 140 KeyValue key3 = new KeyValue(ROW2, hcd.getName(), QF3, 1, VALUE2); 141 KeyValue[] keys = new KeyValue[] { key1, key2, key3 }; 142 int maxKeyCount = keys.length; 143 HRegionInfo regionInfo = new HRegionInfo(tn); 144 StoreFileWriter mobWriter = mobStore.createWriterInTmp(currentDate, 145 maxKeyCount, hcd.getCompactionCompression(), regionInfo.getStartKey(), false); 146 Path mobFilePath = mobWriter.getPath(); 147 String fileName = mobFilePath.getName(); 148 mobWriter.append(key1); 149 mobWriter.append(key2); 150 mobWriter.append(key3); 151 mobWriter.close(); 152 String targetPathName = MobUtils.formatDate(currentDate); 153 Path targetPath = new Path(mobStore.getPath(), targetPathName); 154 mobStore.commitFile(mobFilePath, targetPath); 155 return new Path(targetPath, fileName); 156 } 157 158 @Test 159 public void testMobFileCache() throws Exception { 160 FileSystem fs = FileSystem.get(conf); 161 conf.set(MobConstants.MOB_FILE_CACHE_SIZE_KEY, TEST_CACHE_SIZE); 162 mobFileCache = new MobFileCache(conf); 163 Path file1Path = createMobStoreFile(FAMILY1); 164 Path file2Path = createMobStoreFile(FAMILY2); 165 Path file3Path = createMobStoreFile(FAMILY3); 166 167 // Before open one file by the MobFileCache 168 assertEquals(EXPECTED_CACHE_SIZE_ZERO, mobFileCache.getCacheSize()); 169 // Open one file by the MobFileCache 170 CachedMobFile cachedMobFile1 = (CachedMobFile) mobFileCache.openFile( 171 fs, file1Path, mobCacheConf); 172 assertEquals(EXPECTED_CACHE_SIZE_ONE, mobFileCache.getCacheSize()); 173 assertNotNull(cachedMobFile1); 174 assertEquals(EXPECTED_REFERENCE_TWO, cachedMobFile1.getReferenceCount()); 175 176 // The evict is also managed by a schedule thread pool. 177 // And its check period is set as 3600 seconds by default. 178 // This evict should get the lock at the most time 179 mobFileCache.evict(); // Cache not full, evict it 180 assertEquals(EXPECTED_CACHE_SIZE_ONE, mobFileCache.getCacheSize()); 181 assertEquals(EXPECTED_REFERENCE_TWO, cachedMobFile1.getReferenceCount()); 182 183 mobFileCache.evictFile(file1Path.getName()); // Evict one file 184 assertEquals(EXPECTED_CACHE_SIZE_ZERO, mobFileCache.getCacheSize()); 185 assertEquals(EXPECTED_REFERENCE_ONE, cachedMobFile1.getReferenceCount()); 186 187 cachedMobFile1.close(); // Close the cached mob file 188 189 // Reopen three cached file 190 cachedMobFile1 = (CachedMobFile) mobFileCache.openFile( 191 fs, file1Path, mobCacheConf); 192 assertEquals(EXPECTED_CACHE_SIZE_ONE, mobFileCache.getCacheSize()); 193 CachedMobFile cachedMobFile2 = (CachedMobFile) mobFileCache.openFile( 194 fs, file2Path, mobCacheConf); 195 assertEquals(EXPECTED_CACHE_SIZE_TWO, mobFileCache.getCacheSize()); 196 CachedMobFile cachedMobFile3 = (CachedMobFile) mobFileCache.openFile( 197 fs, file3Path, mobCacheConf); 198 // Before the evict 199 // Evict the cache, should close the first file 1 200 assertEquals(EXPECTED_CACHE_SIZE_THREE, mobFileCache.getCacheSize()); 201 assertEquals(EXPECTED_REFERENCE_TWO, cachedMobFile1.getReferenceCount()); 202 assertEquals(EXPECTED_REFERENCE_TWO, cachedMobFile2.getReferenceCount()); 203 assertEquals(EXPECTED_REFERENCE_TWO, cachedMobFile3.getReferenceCount()); 204 mobFileCache.evict(); 205 assertEquals(EXPECTED_CACHE_SIZE_ONE, mobFileCache.getCacheSize()); 206 assertEquals(EXPECTED_REFERENCE_ONE, cachedMobFile1.getReferenceCount()); 207 assertEquals(EXPECTED_REFERENCE_ONE, cachedMobFile2.getReferenceCount()); 208 assertEquals(EXPECTED_REFERENCE_TWO, cachedMobFile3.getReferenceCount()); 209 } 210}