001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019package org.apache.hadoop.hbase.regionserver.compactions;
020
021import static org.mockito.Mockito.mock;
022import static org.mockito.Mockito.when;
023
024import java.io.IOException;
025import java.lang.reflect.InvocationTargetException;
026import java.util.ArrayList;
027import java.util.Collection;
028import java.util.List;
029
030import org.apache.hadoop.conf.Configuration;
031import org.apache.hadoop.hbase.HBaseConfiguration;
032import org.apache.hadoop.hbase.regionserver.HStore;
033import org.apache.hadoop.hbase.regionserver.HStoreFile;
034import org.apache.hadoop.hbase.regionserver.StoreConfigInformation;
035import org.apache.hadoop.hbase.testclassification.MediumTests;
036import org.apache.hadoop.hbase.testclassification.RegionServerTests;
037import org.apache.hadoop.hbase.util.ReflectionUtils;
038import org.junit.Test;
039import org.junit.experimental.categories.Category;
040import org.junit.runner.RunWith;
041import org.junit.runners.Parameterized;
042
043@Category({RegionServerTests.class, MediumTests.class})
044@RunWith(Parameterized.class)
045public class PerfTestCompactionPolicies extends MockStoreFileGenerator {
046
047  private final RatioBasedCompactionPolicy cp;
048  private final StoreFileListGenerator generator;
049  private final HStore store;
050  private Class<? extends StoreFileListGenerator> fileGenClass;
051  private final int max;
052  private final int min;
053  private final float ratio;
054  private long written = 0;
055
056  @Parameterized.Parameters
057  public static Collection<Object[]> data() {
058
059
060
061    Class<?>[] policyClasses = new Class[]{
062        EverythingPolicy.class,
063        RatioBasedCompactionPolicy.class,
064        ExploringCompactionPolicy.class,
065    };
066
067    Class<?>[] fileListGenClasses = new Class[]{
068        ExplicitFileListGenerator.class,
069        ConstantSizeFileListGenerator.class,
070        SemiConstantSizeFileListGenerator.class,
071        GaussianFileListGenerator.class,
072        SinusoidalFileListGenerator.class,
073        SpikyFileListGenerator.class
074     };
075
076    int[] maxFileValues = new int[] {10};
077    int[] minFilesValues = new int[] {3};
078    float[] ratioValues = new float[] {1.2f};
079
080    List<Object[]> params = new ArrayList<>(
081        maxFileValues.length
082        * minFilesValues.length
083        * fileListGenClasses.length
084        * policyClasses.length);
085
086
087    for (Class<?> policyClass : policyClasses) {
088      for (Class<?> genClass : fileListGenClasses) {
089        for (int maxFile : maxFileValues) {
090          for (int minFile : minFilesValues) {
091            for (float ratio : ratioValues) {
092              params.add(new Object[] { policyClass, genClass, maxFile, minFile, ratio });
093            }
094          }
095        }
096      }
097    }
098
099    return params;
100  }
101
102  /**
103   * Test the perf of a CompactionPolicy with settings.
104   * @param cpClass The compaction policy to test
105   * @param inMmax The maximum number of file to compact
106   * @param inMin The min number of files to compact
107   * @param inRatio The ratio that files must be under to be compacted.
108   */
109  public PerfTestCompactionPolicies(
110      final Class<? extends CompactionPolicy> cpClass,
111      final Class<? extends StoreFileListGenerator> fileGenClass,
112      final int inMmax,
113      final int inMin,
114      final float inRatio) throws IllegalAccessException, InstantiationException,
115      NoSuchMethodException, InvocationTargetException {
116    super(PerfTestCompactionPolicies.class);
117    this.fileGenClass = fileGenClass;
118    this.max = inMmax;
119    this.min = inMin;
120    this.ratio = inRatio;
121
122    // Hide lots of logging so the system out is usable as a tab delimited file.
123    org.apache.log4j.Logger.getLogger(CompactionConfiguration.class).
124        setLevel(org.apache.log4j.Level.ERROR);
125    org.apache.log4j.Logger.getLogger(RatioBasedCompactionPolicy.class).
126        setLevel(org.apache.log4j.Level.ERROR);
127
128    org.apache.log4j.Logger.getLogger(cpClass).setLevel(org.apache.log4j.Level.ERROR);
129
130
131    Configuration configuration = HBaseConfiguration.create();
132
133    // Make sure that this doesn't include every file.
134    configuration.setInt("hbase.hstore.compaction.max", max);
135    configuration.setInt("hbase.hstore.compaction.min", min);
136    configuration.setFloat("hbase.hstore.compaction.ratio", ratio);
137
138    store = createMockStore();
139    this.cp = ReflectionUtils.instantiateWithCustomCtor(cpClass.getName(),
140        new Class[] {Configuration.class, StoreConfigInformation.class },
141        new Object[] {configuration, store });
142
143    this.generator = fileGenClass.getDeclaredConstructor().newInstance();
144    // Used for making paths
145  }
146
147  @Test
148  public final void testSelection() throws Exception {
149    long fileDiff = 0;
150    for (List<HStoreFile> storeFileList : generator) {
151      List<HStoreFile> currentFiles = new ArrayList<>(18);
152      for (HStoreFile file : storeFileList) {
153        currentFiles.add(file);
154        currentFiles = runIteration(currentFiles);
155      }
156      fileDiff += (storeFileList.size() - currentFiles.size());
157    }
158
159    // print out tab delimited so that it can be used in excel/gdocs.
160    System.out.println(
161            cp.getClass().getSimpleName()
162            + "\t" + fileGenClass.getSimpleName()
163            + "\t" + max
164            + "\t" + min
165            + "\t" + ratio
166            + "\t" + written
167            + "\t" + fileDiff
168    );
169  }
170
171
172  private List<HStoreFile> runIteration(List<HStoreFile> startingStoreFiles) throws IOException {
173    List<HStoreFile> storeFiles = new ArrayList<>(startingStoreFiles);
174    CompactionRequestImpl req = cp.selectCompaction(
175        storeFiles, new ArrayList<>(), false, false, false);
176    long newFileSize = 0;
177
178    Collection<HStoreFile> filesToCompact = req.getFiles();
179
180    if (!filesToCompact.isEmpty()) {
181
182      storeFiles = new ArrayList<>(storeFiles);
183      storeFiles.removeAll(filesToCompact);
184
185      for (HStoreFile storeFile : filesToCompact) {
186        newFileSize += storeFile.getReader().length();
187      }
188
189      storeFiles.add(createMockStoreFileBytes(newFileSize));
190    }
191
192    written += newFileSize;
193    return storeFiles;
194  }
195
196  private HStore createMockStore() {
197    HStore s = mock(HStore.class);
198    when(s.getStoreFileTtl()).thenReturn(Long.MAX_VALUE);
199    when(s.getBlockingFileCount()).thenReturn(7L);
200    return s;
201  }
202
203}