001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.regionserver; 019 020import java.io.IOException; 021import java.util.ArrayList; 022import java.util.Arrays; 023import java.util.List; 024import org.apache.hadoop.conf.Configuration; 025import org.apache.hadoop.fs.FileSystem; 026import org.apache.hadoop.fs.Path; 027import org.apache.hadoop.hbase.HBaseTestingUtil; 028import org.apache.hadoop.hbase.HConstants; 029import org.apache.hadoop.hbase.TableName; 030import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; 031import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; 032import org.apache.hadoop.hbase.client.RegionInfo; 033import org.apache.hadoop.hbase.client.RegionInfoBuilder; 034import org.apache.hadoop.hbase.client.TableDescriptor; 035import org.apache.hadoop.hbase.client.TableDescriptorBuilder; 036import org.apache.hadoop.hbase.regionserver.compactions.CompactionConfiguration; 037import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequestImpl; 038import org.apache.hadoop.hbase.regionserver.compactions.RatioBasedCompactionPolicy; 039import org.apache.hadoop.hbase.regionserver.wal.FSHLog; 040import org.apache.hadoop.hbase.util.Bytes; 041import org.apache.hadoop.hbase.util.CommonFSUtils; 042import org.junit.After; 043import org.junit.Assert; 044import org.junit.Before; 045import org.slf4j.Logger; 046import org.slf4j.LoggerFactory; 047 048import org.apache.hbase.thirdparty.com.google.common.collect.Lists; 049 050public class TestCompactionPolicy { 051 private final static Logger LOG = LoggerFactory.getLogger(TestCompactionPolicy.class); 052 protected final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil(); 053 054 protected Configuration conf; 055 protected HStore store; 056 private static final String DIR = 057 TEST_UTIL.getDataTestDir(TestCompactionPolicy.class.getSimpleName()).toString(); 058 protected static Path TEST_FILE; 059 protected static final int minFiles = 3; 060 protected static final int maxFiles = 5; 061 062 protected static final long minSize = 10; 063 protected static final long maxSize = 2100; 064 065 private FSHLog hlog; 066 private HRegion region; 067 068 @Before 069 public void setUp() throws Exception { 070 config(); 071 initialize(); 072 } 073 074 /** 075 * setup config values necessary for store 076 */ 077 protected void config() { 078 this.conf = TEST_UTIL.getConfiguration(); 079 this.conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 0); 080 this.conf.setInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MIN_KEY, minFiles); 081 this.conf.setInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MAX_KEY, maxFiles); 082 this.conf.setLong(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MIN_SIZE_KEY, minSize); 083 this.conf.setLong(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MAX_SIZE_KEY, maxSize); 084 this.conf.setFloat(CompactionConfiguration.HBASE_HSTORE_COMPACTION_RATIO_KEY, 1.0F); 085 } 086 087 /** 088 * Setting up a Store 089 * @throws IOException with error 090 */ 091 protected void initialize() throws IOException { 092 Path basedir = new Path(DIR); 093 String logName = "logs"; 094 Path logdir = new Path(DIR, logName); 095 ColumnFamilyDescriptor familyDescriptor = 096 ColumnFamilyDescriptorBuilder.of(Bytes.toBytes("family")); 097 FileSystem fs = FileSystem.get(conf); 098 099 fs.delete(logdir, true); 100 101 TableDescriptor tableDescriptor = 102 TableDescriptorBuilder.newBuilder(TableName.valueOf(Bytes.toBytes("table"))) 103 .setColumnFamily(familyDescriptor).build(); 104 RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build(); 105 106 hlog = new FSHLog(fs, basedir, logName, conf); 107 hlog.init(); 108 ChunkCreator.initialize(MemStoreLAB.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null, 109 MemStoreLAB.INDEX_CHUNK_SIZE_PERCENTAGE_DEFAULT); 110 region = HRegion.createHRegion(info, basedir, conf, tableDescriptor, hlog); 111 region.close(); 112 Path tableDir = CommonFSUtils.getTableDir(basedir, tableDescriptor.getTableName()); 113 region = new HRegion(tableDir, hlog, fs, conf, info, tableDescriptor, null); 114 115 store = new HStore(region, familyDescriptor, conf, false); 116 117 TEST_FILE = region.getRegionFileSystem().createTempName(); 118 fs.createNewFile(TEST_FILE); 119 } 120 121 @After 122 public void tearDown() throws IOException { 123 IOException ex = null; 124 try { 125 region.close(); 126 } catch (IOException e) { 127 LOG.warn("Caught Exception", e); 128 ex = e; 129 } 130 try { 131 hlog.close(); 132 } catch (IOException e) { 133 LOG.warn("Caught Exception", e); 134 ex = e; 135 } 136 if (ex != null) { 137 throw ex; 138 } 139 } 140 141 ArrayList<Long> toArrayList(long... numbers) { 142 ArrayList<Long> result = new ArrayList<>(); 143 for (long i : numbers) { 144 result.add(i); 145 } 146 return result; 147 } 148 149 List<HStoreFile> sfCreate(long... sizes) throws IOException { 150 ArrayList<Long> ageInDisk = new ArrayList<>(); 151 for (int i = 0; i < sizes.length; i++) { 152 ageInDisk.add(0L); 153 } 154 return sfCreate(toArrayList(sizes), ageInDisk); 155 } 156 157 List<HStoreFile> sfCreate(ArrayList<Long> sizes, ArrayList<Long> ageInDisk) throws IOException { 158 return sfCreate(false, sizes, ageInDisk); 159 } 160 161 List<HStoreFile> sfCreate(boolean isReference, long... sizes) throws IOException { 162 ArrayList<Long> ageInDisk = new ArrayList<>(sizes.length); 163 for (int i = 0; i < sizes.length; i++) { 164 ageInDisk.add(0L); 165 } 166 return sfCreate(isReference, toArrayList(sizes), ageInDisk); 167 } 168 169 List<HStoreFile> sfCreate(boolean isReference, ArrayList<Long> sizes, ArrayList<Long> ageInDisk) 170 throws IOException { 171 List<HStoreFile> ret = Lists.newArrayList(); 172 for (int i = 0; i < sizes.size(); i++) { 173 ret.add( 174 new MockHStoreFile(TEST_UTIL, TEST_FILE, sizes.get(i), ageInDisk.get(i), isReference, i)); 175 } 176 return ret; 177 } 178 179 long[] getSizes(List<HStoreFile> sfList) { 180 long[] aNums = new long[sfList.size()]; 181 for (int i = 0; i < sfList.size(); ++i) { 182 aNums[i] = sfList.get(i).getReader().length(); 183 } 184 return aNums; 185 } 186 187 void compactEquals(List<HStoreFile> candidates, long... expected) throws IOException { 188 compactEquals(candidates, false, false, expected); 189 } 190 191 void compactEquals(List<HStoreFile> candidates, boolean forcemajor, long... expected) 192 throws IOException { 193 compactEquals(candidates, forcemajor, false, expected); 194 } 195 196 void compactEquals(List<HStoreFile> candidates, boolean forcemajor, boolean isOffPeak, 197 long... expected) throws IOException { 198 store.forceMajor = forcemajor; 199 // Test Default compactions 200 CompactionRequestImpl result = 201 ((RatioBasedCompactionPolicy) store.storeEngine.getCompactionPolicy()) 202 .selectCompaction(candidates, new ArrayList<>(), false, isOffPeak, forcemajor); 203 List<HStoreFile> actual = new ArrayList<>(result.getFiles()); 204 if (isOffPeak && !forcemajor) { 205 Assert.assertTrue(result.isOffPeak()); 206 } 207 Assert.assertEquals(Arrays.toString(expected), Arrays.toString(getSizes(actual))); 208 store.forceMajor = false; 209 } 210}