001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.mob; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertNotNull; 022 023import java.io.IOException; 024import java.util.Date; 025import org.apache.hadoop.conf.Configuration; 026import org.apache.hadoop.fs.FileSystem; 027import org.apache.hadoop.fs.Path; 028import org.apache.hadoop.hbase.HBaseClassTestRule; 029import org.apache.hadoop.hbase.HBaseConfiguration; 030import org.apache.hadoop.hbase.HBaseTestingUtil; 031import org.apache.hadoop.hbase.HConstants; 032import org.apache.hadoop.hbase.KeyValue; 033import org.apache.hadoop.hbase.TableName; 034import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; 035import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; 036import org.apache.hadoop.hbase.client.RegionInfo; 037import org.apache.hadoop.hbase.client.RegionInfoBuilder; 038import org.apache.hadoop.hbase.client.TableDescriptor; 039import org.apache.hadoop.hbase.client.TableDescriptorBuilder; 040import org.apache.hadoop.hbase.io.hfile.CacheConfig; 041import org.apache.hadoop.hbase.regionserver.HMobStore; 042import org.apache.hadoop.hbase.regionserver.HRegion; 043import org.apache.hadoop.hbase.regionserver.StoreFileWriter; 044import org.apache.hadoop.hbase.testclassification.SmallTests; 045import org.apache.hadoop.hbase.util.Bytes; 046import org.junit.After; 047import org.junit.Before; 048import org.junit.ClassRule; 049import org.junit.Test; 050import org.junit.experimental.categories.Category; 051 052@Category(SmallTests.class) 053public class TestMobFileCache { 054 055 @ClassRule 056 public static final HBaseClassTestRule CLASS_RULE = 057 HBaseClassTestRule.forClass(TestMobFileCache.class); 058 059 private HBaseTestingUtil UTIL; 060 private HRegion region; 061 private Configuration conf; 062 private MobFileCache mobFileCache; 063 private Date currentDate = new Date(); 064 private static final String TEST_CACHE_SIZE = "2"; 065 private static final int EXPECTED_CACHE_SIZE_ZERO = 0; 066 private static final int EXPECTED_CACHE_SIZE_ONE = 1; 067 private static final int EXPECTED_CACHE_SIZE_TWO = 2; 068 private static final int EXPECTED_CACHE_SIZE_THREE = 3; 069 private static final long EXPECTED_REFERENCE_ONE = 1; 070 private static final long EXPECTED_REFERENCE_TWO = 2; 071 072 private static final String TABLE = "tableName"; 073 private static final String FAMILY1 = "family1"; 074 private static final String FAMILY2 = "family2"; 075 private static final String FAMILY3 = "family3"; 076 077 private static final byte[] ROW = Bytes.toBytes("row"); 078 private static final byte[] ROW2 = Bytes.toBytes("row2"); 079 private static final byte[] VALUE = Bytes.toBytes("value"); 080 private static final byte[] VALUE2 = Bytes.toBytes("value2"); 081 private static final byte[] QF1 = Bytes.toBytes("qf1"); 082 private static final byte[] QF2 = Bytes.toBytes("qf2"); 083 private static final byte[] QF3 = Bytes.toBytes("qf3"); 084 085 @Before 086 public void setUp() throws Exception { 087 UTIL = new HBaseTestingUtil(); 088 conf = UTIL.getConfiguration(); 089 conf.set(MobConstants.MOB_FILE_CACHE_SIZE_KEY, TEST_CACHE_SIZE); 090 TableDescriptorBuilder tableDescriptorBuilder = 091 TableDescriptorBuilder.newBuilder(UTIL.createTableDescriptor( 092 TableName.valueOf("testMobFileCache"), ColumnFamilyDescriptorBuilder.DEFAULT_MIN_VERSIONS, 093 3, HConstants.FOREVER, ColumnFamilyDescriptorBuilder.DEFAULT_KEEP_DELETED)); 094 ColumnFamilyDescriptor columnFamilyDescriptor = ColumnFamilyDescriptorBuilder 095 .newBuilder(Bytes.toBytes(FAMILY1)).setMobEnabled(true).setMobThreshold(0).build(); 096 tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor); 097 columnFamilyDescriptor = ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(FAMILY2)) 098 .setMobEnabled(true).setMobThreshold(0).build(); 099 tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor); 100 columnFamilyDescriptor = ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(FAMILY3)) 101 .setMobEnabled(true).setMobThreshold(0).build(); 102 tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor); 103 TableDescriptor tableDescriptor = tableDescriptorBuilder.build(); 104 RegionInfo regionInfo = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build(); 105 mobFileCache = new MobFileCache(conf); 106 region = HBaseTestingUtil.createRegionAndWAL(regionInfo, UTIL.getDataTestDir(), conf, 107 tableDescriptor, mobFileCache); 108 } 109 110 @After 111 public void tearDown() throws Exception { 112 region.close(); 113 region.getFilesystem().delete(UTIL.getDataTestDir(), true); 114 } 115 116 /** 117 * Create the mob store file. 118 */ 119 private Path createMobStoreFile(String family) throws IOException { 120 return createMobStoreFile(HBaseConfiguration.create(), family); 121 } 122 123 /** 124 * Create the mob store file 125 */ 126 private Path createMobStoreFile(Configuration conf, String family) throws IOException { 127 ColumnFamilyDescriptor columnFamilyDescriptor = ColumnFamilyDescriptorBuilder 128 .newBuilder(Bytes.toBytes(family)).setMaxVersions(4).setMobEnabled(true).build(); 129 return createMobStoreFile(columnFamilyDescriptor); 130 } 131 132 /** 133 * Create the mob store file 134 */ 135 private Path createMobStoreFile(ColumnFamilyDescriptor columnFamilyDescriptor) 136 throws IOException { 137 // Setting up a Store 138 TableName tn = TableName.valueOf(TABLE); 139 TableDescriptorBuilder tableDescriptorBuilder = TableDescriptorBuilder.newBuilder(tn); 140 tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor); 141 HMobStore mobStore = (HMobStore) region.getStore(columnFamilyDescriptor.getName()); 142 KeyValue key1 = new KeyValue(ROW, columnFamilyDescriptor.getName(), QF1, 1, VALUE); 143 KeyValue key2 = new KeyValue(ROW, columnFamilyDescriptor.getName(), QF2, 1, VALUE); 144 KeyValue key3 = new KeyValue(ROW2, columnFamilyDescriptor.getName(), QF3, 1, VALUE2); 145 KeyValue[] keys = new KeyValue[] { key1, key2, key3 }; 146 int maxKeyCount = keys.length; 147 RegionInfo regionInfo = RegionInfoBuilder.newBuilder(tn).build(); 148 StoreFileWriter mobWriter = mobStore.createWriterInTmp(currentDate, maxKeyCount, 149 columnFamilyDescriptor.getCompactionCompressionType(), regionInfo.getStartKey(), false); 150 Path mobFilePath = mobWriter.getPath(); 151 String fileName = mobFilePath.getName(); 152 mobWriter.append(key1); 153 mobWriter.append(key2); 154 mobWriter.append(key3); 155 mobWriter.close(); 156 String targetPathName = MobUtils.formatDate(currentDate); 157 Path targetPath = new Path(mobStore.getPath(), targetPathName); 158 mobStore.commitFile(mobFilePath, targetPath); 159 return new Path(targetPath, fileName); 160 } 161 162 @Test 163 public void testMobFileCache() throws Exception { 164 FileSystem fs = FileSystem.get(conf); 165 Path file1Path = createMobStoreFile(FAMILY1); 166 Path file2Path = createMobStoreFile(FAMILY2); 167 Path file3Path = createMobStoreFile(FAMILY3); 168 169 CacheConfig cacheConf = new CacheConfig(conf); 170 // Before open one file by the MobFileCache 171 assertEquals(EXPECTED_CACHE_SIZE_ZERO, mobFileCache.getCacheSize()); 172 // Open one file by the MobFileCache 173 CachedMobFile cachedMobFile1 = (CachedMobFile) mobFileCache.openFile(fs, file1Path, cacheConf); 174 assertEquals(EXPECTED_CACHE_SIZE_ONE, mobFileCache.getCacheSize()); 175 assertNotNull(cachedMobFile1); 176 assertEquals(EXPECTED_REFERENCE_TWO, cachedMobFile1.getReferenceCount()); 177 178 // The evict is also managed by a schedule thread pool. 179 // And its check period is set as 3600 seconds by default. 180 // This evict should get the lock at the most time 181 mobFileCache.evict(); // Cache not full, evict it 182 assertEquals(EXPECTED_CACHE_SIZE_ONE, mobFileCache.getCacheSize()); 183 assertEquals(EXPECTED_REFERENCE_TWO, cachedMobFile1.getReferenceCount()); 184 185 mobFileCache.evictFile(file1Path.getName()); // Evict one file 186 assertEquals(EXPECTED_CACHE_SIZE_ZERO, mobFileCache.getCacheSize()); 187 assertEquals(EXPECTED_REFERENCE_ONE, cachedMobFile1.getReferenceCount()); 188 189 cachedMobFile1.close(); // Close the cached mob file 190 191 // Reopen three cached file 192 cachedMobFile1 = (CachedMobFile) mobFileCache.openFile(fs, file1Path, cacheConf); 193 assertEquals(EXPECTED_CACHE_SIZE_ONE, mobFileCache.getCacheSize()); 194 CachedMobFile cachedMobFile2 = (CachedMobFile) mobFileCache.openFile(fs, file2Path, cacheConf); 195 assertEquals(EXPECTED_CACHE_SIZE_TWO, mobFileCache.getCacheSize()); 196 CachedMobFile cachedMobFile3 = (CachedMobFile) mobFileCache.openFile(fs, file3Path, cacheConf); 197 // Before the evict 198 // Evict the cache, should close the first file 1 199 assertEquals(EXPECTED_CACHE_SIZE_THREE, mobFileCache.getCacheSize()); 200 assertEquals(EXPECTED_REFERENCE_TWO, cachedMobFile1.getReferenceCount()); 201 assertEquals(EXPECTED_REFERENCE_TWO, cachedMobFile2.getReferenceCount()); 202 assertEquals(EXPECTED_REFERENCE_TWO, cachedMobFile3.getReferenceCount()); 203 mobFileCache.evict(); 204 assertEquals(EXPECTED_CACHE_SIZE_ONE, mobFileCache.getCacheSize()); 205 assertEquals(EXPECTED_REFERENCE_ONE, cachedMobFile1.getReferenceCount()); 206 assertEquals(EXPECTED_REFERENCE_ONE, cachedMobFile2.getReferenceCount()); 207 assertEquals(EXPECTED_REFERENCE_TWO, cachedMobFile3.getReferenceCount()); 208 } 209}