001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import java.io.IOException;
021import java.util.ArrayList;
022import java.util.List;
023import org.apache.hadoop.hbase.HBaseClassTestRule;
024import org.apache.hadoop.hbase.HConstants;
025import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequestImpl;
026import org.apache.hadoop.hbase.regionserver.compactions.RatioBasedCompactionPolicy;
027import org.apache.hadoop.hbase.testclassification.SmallTests;
028import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
029import org.apache.hadoop.hbase.util.TimeOffsetEnvironmentEdge;
030import org.junit.Assert;
031import org.junit.ClassRule;
032import org.junit.Test;
033import org.junit.experimental.categories.Category;
034
035@Category(SmallTests.class)
036public class TestDefaultCompactSelection extends TestCompactionPolicy {
037
038  @ClassRule
039  public static final HBaseClassTestRule CLASS_RULE =
040      HBaseClassTestRule.forClass(TestDefaultCompactSelection.class);
041
042  @Override
043  protected void config() {
044    super.config();
045    // DON'T change this config since all test cases assume HStore.BLOCKING_STOREFILES_KEY is 10.
046    this.conf.setLong(HStore.BLOCKING_STOREFILES_KEY, 10);
047  }
048
049  @Test
050  public void testCompactionRatio() throws IOException {
051    TimeOffsetEnvironmentEdge edge = new TimeOffsetEnvironmentEdge();
052    EnvironmentEdgeManager.injectEdge(edge);
053    /**
054     * NOTE: these tests are specific to describe the implementation of the
055     * current compaction algorithm.  Developed to ensure that refactoring
056     * doesn't implicitly alter this.
057     */
058    long tooBig = maxSize + 1;
059
060    // default case. preserve user ratio on size
061    compactEquals(sfCreate(100,50,23,12,12), 23, 12, 12);
062    // less than compact threshold = don't compact
063    compactEquals(sfCreate(100,50,25,12,12) /* empty */);
064    // greater than compact size = skip those
065    compactEquals(sfCreate(tooBig, tooBig, 700, 700, 700), 700, 700, 700);
066    // big size + threshold
067    compactEquals(sfCreate(tooBig, tooBig, 700,700) /* empty */);
068    // small files = don't care about ratio
069    compactEquals(sfCreate(7,1,1), 7,1,1);
070
071    // don't exceed max file compact threshold
072    // note:  file selection starts with largest to smallest.
073    compactEquals(sfCreate(7, 6, 5, 4, 3, 2, 1), 5, 4, 3, 2, 1);
074
075    compactEquals(sfCreate(50, 10, 10 ,10, 10), 10, 10, 10, 10);
076
077    compactEquals(sfCreate(10, 10, 10, 10, 50), 10, 10, 10, 10);
078
079    compactEquals(sfCreate(251, 253, 251, maxSize -1), 251, 253, 251);
080
081    compactEquals(sfCreate(maxSize -1,maxSize -1,maxSize -1) /* empty */);
082
083    // Always try and compact something to get below blocking storefile count
084    this.conf.setLong("hbase.hstore.compaction.min.size", 1);
085    store.storeEngine.getCompactionPolicy().setConf(conf);
086    compactEquals(sfCreate(512,256,128,64,32,16,8,4,2,1), 4,2,1);
087    this.conf.setLong("hbase.hstore.compaction.min.size", minSize);
088    store.storeEngine.getCompactionPolicy().setConf(conf);
089
090    /* MAJOR COMPACTION */
091    // if a major compaction has been forced, then compact everything
092    compactEquals(sfCreate(50,25,12,12), true, 50, 25, 12, 12);
093    // also choose files < threshold on major compaction
094    compactEquals(sfCreate(12,12), true, 12, 12);
095    // even if one of those files is too big
096    compactEquals(sfCreate(tooBig, 12,12), true, tooBig, 12, 12);
097    // don't exceed max file compact threshold, even with major compaction
098    store.forceMajor = true;
099    compactEquals(sfCreate(7, 6, 5, 4, 3, 2, 1), 5, 4, 3, 2, 1);
100    store.forceMajor = false;
101    // if we exceed maxCompactSize, downgrade to minor
102    // if not, it creates a 'snowball effect' when files >> maxCompactSize:
103    // the last file in compaction is the aggregate of all previous compactions
104    compactEquals(sfCreate(100,50,23,12,12), true, 23, 12, 12);
105    conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 1);
106    conf.setFloat("hbase.hregion.majorcompaction.jitter", 0);
107    store.storeEngine.getCompactionPolicy().setConf(conf);
108    try {
109      // The modTime of the mocked store file is currentTimeMillis, so we need to increase the
110      // timestamp a bit to make sure that now - lowestModTime is greater than major compaction
111      // period(1ms).
112      // trigger an aged major compaction
113      List<HStoreFile> candidates = sfCreate(50, 25, 12, 12);
114      edge.increment(2);
115      compactEquals(candidates, 50, 25, 12, 12);
116      // major sure exceeding maxCompactSize also downgrades aged minors
117      candidates = sfCreate(100, 50, 23, 12, 12);
118      edge.increment(2);
119      compactEquals(candidates, 23, 12, 12);
120    } finally {
121      conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 1000*60*60*24);
122      conf.setFloat("hbase.hregion.majorcompaction.jitter", 0.20F);
123    }
124
125    /* REFERENCES == file is from a region that was split */
126    // treat storefiles that have references like a major compaction
127    compactEquals(sfCreate(true, 100,50,25,12,12), 100, 50, 25, 12, 12);
128    // reference files shouldn't obey max threshold
129    compactEquals(sfCreate(true, tooBig, 12,12), tooBig, 12, 12);
130    // reference files should obey max file compact to avoid OOM
131    compactEquals(sfCreate(true, 7, 6, 5, 4, 3, 2, 1), 7, 6, 5, 4, 3);
132
133    // empty case
134    compactEquals(new ArrayList<>() /* empty */);
135    // empty case (because all files are too big)
136    compactEquals(sfCreate(tooBig, tooBig) /* empty */);
137  }
138
139  @Test
140  public void testOffPeakCompactionRatio() throws IOException {
141    /*
142     * NOTE: these tests are specific to describe the implementation of the
143     * current compaction algorithm.  Developed to ensure that refactoring
144     * doesn't implicitly alter this.
145     */
146    // set an off-peak compaction threshold
147    this.conf.setFloat("hbase.hstore.compaction.ratio.offpeak", 5.0F);
148    store.storeEngine.getCompactionPolicy().setConf(this.conf);
149    // Test with and without the flag.
150    compactEquals(sfCreate(999, 50, 12, 12, 1), false, true, 50, 12, 12, 1);
151    compactEquals(sfCreate(999, 50, 12, 12, 1), 12, 12, 1);
152  }
153
154  @Test
155  public void testStuckStoreCompaction() throws IOException {
156    // Select the smallest compaction if the store is stuck.
157    compactEquals(sfCreate(99,99,99,99,99,99, 30,30,30,30), 30, 30, 30);
158    // If not stuck, standard policy applies.
159    compactEquals(sfCreate(99,99,99,99,99, 30,30,30,30), 99, 30, 30, 30, 30);
160
161    // Add sufficiently small files to compaction, though
162    compactEquals(sfCreate(99,99,99,99,99,99, 30,30,30,15), 30, 30, 30, 15);
163    // Prefer earlier compaction to latter if the benefit is not significant
164    compactEquals(sfCreate(99,99,99,99, 30,26,26,29,25,25), 30, 26, 26);
165    // Prefer later compaction if the benefit is significant.
166    compactEquals(sfCreate(99,99,99,99, 27,27,27,20,20,20), 20, 20, 20);
167  }
168
169  @Test
170  public void testCompactionEmptyHFile() throws IOException {
171    // Set TTL
172    ScanInfo oldScanInfo = store.getScanInfo();
173    ScanInfo newScanInfo = oldScanInfo.customize(oldScanInfo.getMaxVersions(), 600,
174        oldScanInfo.getKeepDeletedCells());
175    store.setScanInfo(newScanInfo);
176    // Do not compact empty store file
177    List<HStoreFile> candidates = sfCreate(0);
178    for (HStoreFile file : candidates) {
179      if (file instanceof MockHStoreFile) {
180        MockHStoreFile mockFile = (MockHStoreFile) file;
181        mockFile.setTimeRangeTracker(TimeRangeTracker.create(TimeRangeTracker.Type.SYNC, -1, -1));
182        mockFile.setEntries(0);
183      }
184    }
185    // Test Default compactions
186    CompactionRequestImpl result = ((RatioBasedCompactionPolicy) store.storeEngine
187        .getCompactionPolicy()).selectCompaction(candidates,
188        new ArrayList<>(), false, false, false);
189    Assert.assertTrue(result.getFiles().isEmpty());
190    store.setScanInfo(oldScanInfo);
191  }
192}