001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver.compactions;
019
020import static org.mockito.Mockito.mock;
021import static org.mockito.Mockito.when;
022
023import java.io.IOException;
024import java.lang.reflect.InvocationTargetException;
025import java.util.ArrayList;
026import java.util.Collection;
027import java.util.List;
028import java.util.stream.Stream;
029import org.apache.hadoop.conf.Configuration;
030import org.apache.hadoop.hbase.HBaseConfiguration;
031import org.apache.hadoop.hbase.HBaseParameterizedTestTemplate;
032import org.apache.hadoop.hbase.client.RegionInfoBuilder;
033import org.apache.hadoop.hbase.logging.Log4jUtils;
034import org.apache.hadoop.hbase.regionserver.HStore;
035import org.apache.hadoop.hbase.regionserver.HStoreFile;
036import org.apache.hadoop.hbase.regionserver.StoreConfigInformation;
037import org.apache.hadoop.hbase.testclassification.MediumTests;
038import org.apache.hadoop.hbase.testclassification.RegionServerTests;
039import org.apache.hadoop.hbase.util.ReflectionUtils;
040import org.junit.jupiter.api.Tag;
041import org.junit.jupiter.api.TestTemplate;
042import org.junit.jupiter.params.provider.Arguments;
043
044/**
045 * This test compares compaction policy behavior with different generated store file lists and
046 * settings. It can be run explicitly; e.g. mvn test -Dtest=PerfTestCompactionPolicies
047 */
048@Tag(RegionServerTests.TAG)
049@Tag(MediumTests.TAG)
050@HBaseParameterizedTestTemplate(
051    name = "{index}: policy={0}, fileGen={1}, max={2}, min={3}, " + "ratio={4}")
052public class PerfTestCompactionPolicies extends MockStoreFileGenerator {
053
054  private final RatioBasedCompactionPolicy cp;
055  private final StoreFileListGenerator generator;
056  private final HStore store;
057  private Class<? extends StoreFileListGenerator> fileGenClass;
058  private final int max;
059  private final int min;
060  private final float ratio;
061  private long written = 0;
062
063  public static Stream<Arguments> parameters() {
064
065    Class<?>[] policyClasses = new Class[] { EverythingPolicy.class,
066      RatioBasedCompactionPolicy.class, ExploringCompactionPolicy.class, };
067
068    Class<?>[] fileListGenClasses =
069      new Class[] { ExplicitFileListGenerator.class, ConstantSizeFileListGenerator.class,
070        SemiConstantSizeFileListGenerator.class, GaussianFileListGenerator.class,
071        SinusoidalFileListGenerator.class, SpikyFileListGenerator.class };
072
073    int[] maxFileValues = new int[] { 10 };
074    int[] minFilesValues = new int[] { 3 };
075    float[] ratioValues = new float[] { 1.2f };
076
077    List<Arguments> params = new ArrayList<>(maxFileValues.length * minFilesValues.length
078      * fileListGenClasses.length * policyClasses.length);
079
080    for (Class<?> policyClass : policyClasses) {
081      for (Class<?> genClass : fileListGenClasses) {
082        for (int maxFile : maxFileValues) {
083          for (int minFile : minFilesValues) {
084            for (float ratio : ratioValues) {
085              params.add(Arguments.of(policyClass, genClass, maxFile, minFile, ratio));
086            }
087          }
088        }
089      }
090    }
091
092    return params.stream();
093  }
094
095  /**
096   * Test the perf of a CompactionPolicy with settings.
097   * @param cpClass      The compaction policy to test
098   * @param fileGenClass The store file list generator to test with
099   * @param inMmax       The maximum number of files to compact
100   * @param inMin        The min number of files to compact
101   * @param inRatio      The ratio that files must be under to be compacted.
102   */
103  public PerfTestCompactionPolicies(final Class<? extends CompactionPolicy> cpClass,
104    final Class<? extends StoreFileListGenerator> fileGenClass, final int inMmax, final int inMin,
105    final float inRatio) throws IllegalAccessException, InstantiationException,
106    NoSuchMethodException, InvocationTargetException {
107    this.fileGenClass = fileGenClass;
108    this.max = inMmax;
109    this.min = inMin;
110    this.ratio = inRatio;
111
112    // Hide lots of logging so the system out is usable as a tab delimited file.
113    Log4jUtils.setLogLevel(CompactionConfiguration.class.getName(), "ERROR");
114    Log4jUtils.setLogLevel(RatioBasedCompactionPolicy.class.getName(), "ERROR");
115    Log4jUtils.setLogLevel(cpClass.getName(), "ERROR");
116
117    Configuration configuration = HBaseConfiguration.create();
118
119    // Make sure that this doesn't include every file.
120    configuration.setInt("hbase.hstore.compaction.max", max);
121    configuration.setInt("hbase.hstore.compaction.min", min);
122    configuration.setFloat("hbase.hstore.compaction.ratio", ratio);
123
124    store = createMockStore();
125    this.cp = ReflectionUtils.instantiateWithCustomCtor(cpClass.getName(),
126      new Class[] { Configuration.class, StoreConfigInformation.class },
127      new Object[] { configuration, store });
128
129    this.generator = fileGenClass.getDeclaredConstructor().newInstance();
130    // Used for making paths
131  }
132
133  @TestTemplate
134  public final void testSelection() throws Exception {
135    long fileDiff = 0;
136    for (List<HStoreFile> storeFileList : generator) {
137      List<HStoreFile> currentFiles = new ArrayList<>(18);
138      for (HStoreFile file : storeFileList) {
139        currentFiles.add(file);
140        currentFiles = runIteration(currentFiles);
141      }
142      fileDiff += (storeFileList.size() - currentFiles.size());
143    }
144
145    // print out tab delimited so that it can be used in excel/gdocs.
146    System.out.println(cp.getClass().getSimpleName() + "\t" + fileGenClass.getSimpleName() + "\t"
147      + max + "\t" + min + "\t" + ratio + "\t" + written + "\t" + fileDiff);
148  }
149
150  private List<HStoreFile> runIteration(List<HStoreFile> startingStoreFiles) throws IOException {
151    List<HStoreFile> storeFiles = new ArrayList<>(startingStoreFiles);
152    CompactionRequestImpl req =
153      cp.selectCompaction(storeFiles, new ArrayList<>(), false, false, false);
154    long newFileSize = 0;
155
156    Collection<HStoreFile> filesToCompact = req.getFiles();
157
158    if (!filesToCompact.isEmpty()) {
159
160      storeFiles = new ArrayList<>(storeFiles);
161      storeFiles.removeAll(filesToCompact);
162
163      for (HStoreFile storeFile : filesToCompact) {
164        newFileSize += storeFile.getReader().length();
165      }
166
167      storeFiles.add(createMockStoreFileBytes(newFileSize));
168    }
169
170    written += newFileSize;
171    return storeFiles;
172  }
173
174  private HStore createMockStore() {
175    HStore s = mock(HStore.class);
176    when(s.getStoreFileTtl()).thenReturn(Long.MAX_VALUE);
177    when(s.getBlockingFileCount()).thenReturn(7L);
178    when(s.getRegionInfo()).thenReturn(RegionInfoBuilder.FIRST_META_REGIONINFO);
179    return s;
180  }
181
182}