001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver.compactions;
019
020import static org.mockito.Mockito.mock;
021import static org.mockito.Mockito.when;
022
023import java.io.IOException;
024import java.lang.reflect.InvocationTargetException;
025import java.util.ArrayList;
026import java.util.Collection;
027import java.util.List;
028import org.apache.hadoop.conf.Configuration;
029import org.apache.hadoop.hbase.HBaseClassTestRule;
030import org.apache.hadoop.hbase.HBaseConfiguration;
031import org.apache.hadoop.hbase.client.RegionInfoBuilder;
032import org.apache.hadoop.hbase.logging.Log4jUtils;
033import org.apache.hadoop.hbase.regionserver.HStore;
034import org.apache.hadoop.hbase.regionserver.HStoreFile;
035import org.apache.hadoop.hbase.regionserver.StoreConfigInformation;
036import org.apache.hadoop.hbase.testclassification.MediumTests;
037import org.apache.hadoop.hbase.testclassification.RegionServerTests;
038import org.apache.hadoop.hbase.util.ReflectionUtils;
039import org.junit.ClassRule;
040import org.junit.Test;
041import org.junit.experimental.categories.Category;
042import org.junit.runner.RunWith;
043import org.junit.runners.Parameterized;
044
045/**
046 * This is not a unit test. It is not run as part of the general unit test suite. It is for
047 * comparing compaction policies. You must run it explicitly; e.g. mvn test
048 * -Dtest=PerfTestCompactionPolicies
049 */
050@Category({ RegionServerTests.class, MediumTests.class })
051@RunWith(Parameterized.class)
052public class PerfTestCompactionPolicies extends MockStoreFileGenerator {
053
054  @ClassRule
055  public static final HBaseClassTestRule CLASS_RULE =
056    HBaseClassTestRule.forClass(PerfTestCompactionPolicies.class);
057
058  private final RatioBasedCompactionPolicy cp;
059  private final StoreFileListGenerator generator;
060  private final HStore store;
061  private Class<? extends StoreFileListGenerator> fileGenClass;
062  private final int max;
063  private final int min;
064  private final float ratio;
065  private long written = 0;
066
067  @Parameterized.Parameters
068  public static Collection<Object[]> data() {
069
070    Class<?>[] policyClasses = new Class[] { EverythingPolicy.class,
071      RatioBasedCompactionPolicy.class, ExploringCompactionPolicy.class, };
072
073    Class<?>[] fileListGenClasses =
074      new Class[] { ExplicitFileListGenerator.class, ConstantSizeFileListGenerator.class,
075        SemiConstantSizeFileListGenerator.class, GaussianFileListGenerator.class,
076        SinusoidalFileListGenerator.class, SpikyFileListGenerator.class };
077
078    int[] maxFileValues = new int[] { 10 };
079    int[] minFilesValues = new int[] { 3 };
080    float[] ratioValues = new float[] { 1.2f };
081
082    List<Object[]> params = new ArrayList<>(maxFileValues.length * minFilesValues.length
083      * fileListGenClasses.length * policyClasses.length);
084
085    for (Class<?> policyClass : policyClasses) {
086      for (Class<?> genClass : fileListGenClasses) {
087        for (int maxFile : maxFileValues) {
088          for (int minFile : minFilesValues) {
089            for (float ratio : ratioValues) {
090              params.add(new Object[] { policyClass, genClass, maxFile, minFile, ratio });
091            }
092          }
093        }
094      }
095    }
096
097    return params;
098  }
099
100  /**
101   * Test the perf of a CompactionPolicy with settings.
102   * @param cpClass The compaction policy to test
103   * @param inMmax  The maximum number of file to compact
104   * @param inMin   The min number of files to compact
105   * @param inRatio The ratio that files must be under to be compacted.
106   */
107  public PerfTestCompactionPolicies(final Class<? extends CompactionPolicy> cpClass,
108    final Class<? extends StoreFileListGenerator> fileGenClass, final int inMmax, final int inMin,
109    final float inRatio) throws IllegalAccessException, InstantiationException,
110    NoSuchMethodException, InvocationTargetException {
111    this.fileGenClass = fileGenClass;
112    this.max = inMmax;
113    this.min = inMin;
114    this.ratio = inRatio;
115
116    // Hide lots of logging so the system out is usable as a tab delimited file.
117    Log4jUtils.setLogLevel(CompactionConfiguration.class.getName(), "ERROR");
118    Log4jUtils.setLogLevel(RatioBasedCompactionPolicy.class.getName(), "ERROR");
119    Log4jUtils.setLogLevel(cpClass.getName(), "ERROR");
120
121    Configuration configuration = HBaseConfiguration.create();
122
123    // Make sure that this doesn't include every file.
124    configuration.setInt("hbase.hstore.compaction.max", max);
125    configuration.setInt("hbase.hstore.compaction.min", min);
126    configuration.setFloat("hbase.hstore.compaction.ratio", ratio);
127
128    store = createMockStore();
129    this.cp = ReflectionUtils.instantiateWithCustomCtor(cpClass.getName(),
130      new Class[] { Configuration.class, StoreConfigInformation.class },
131      new Object[] { configuration, store });
132
133    this.generator = fileGenClass.getDeclaredConstructor().newInstance();
134    // Used for making paths
135  }
136
137  @Test
138  public final void testSelection() throws Exception {
139    long fileDiff = 0;
140    for (List<HStoreFile> storeFileList : generator) {
141      List<HStoreFile> currentFiles = new ArrayList<>(18);
142      for (HStoreFile file : storeFileList) {
143        currentFiles.add(file);
144        currentFiles = runIteration(currentFiles);
145      }
146      fileDiff += (storeFileList.size() - currentFiles.size());
147    }
148
149    // print out tab delimited so that it can be used in excel/gdocs.
150    System.out.println(cp.getClass().getSimpleName() + "\t" + fileGenClass.getSimpleName() + "\t"
151      + max + "\t" + min + "\t" + ratio + "\t" + written + "\t" + fileDiff);
152  }
153
154  private List<HStoreFile> runIteration(List<HStoreFile> startingStoreFiles) throws IOException {
155    List<HStoreFile> storeFiles = new ArrayList<>(startingStoreFiles);
156    CompactionRequestImpl req =
157      cp.selectCompaction(storeFiles, new ArrayList<>(), false, false, false);
158    long newFileSize = 0;
159
160    Collection<HStoreFile> filesToCompact = req.getFiles();
161
162    if (!filesToCompact.isEmpty()) {
163
164      storeFiles = new ArrayList<>(storeFiles);
165      storeFiles.removeAll(filesToCompact);
166
167      for (HStoreFile storeFile : filesToCompact) {
168        newFileSize += storeFile.getReader().length();
169      }
170
171      storeFiles.add(createMockStoreFileBytes(newFileSize));
172    }
173
174    written += newFileSize;
175    return storeFiles;
176  }
177
178  private HStore createMockStore() {
179    HStore s = mock(HStore.class);
180    when(s.getStoreFileTtl()).thenReturn(Long.MAX_VALUE);
181    when(s.getBlockingFileCount()).thenReturn(7L);
182    when(s.getRegionInfo()).thenReturn(RegionInfoBuilder.FIRST_META_REGIONINFO);
183    return s;
184  }
185
186}