001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.junit.jupiter.api.Assertions.assertTrue;
021
022import java.io.IOException;
023import java.util.ArrayList;
024import java.util.List;
025import org.apache.hadoop.hbase.HConstants;
026import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequestImpl;
027import org.apache.hadoop.hbase.regionserver.compactions.RatioBasedCompactionPolicy;
028import org.apache.hadoop.hbase.testclassification.SmallTests;
029import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
030import org.apache.hadoop.hbase.util.TimeOffsetEnvironmentEdge;
031import org.junit.jupiter.api.Tag;
032import org.junit.jupiter.api.Test;
033
034@Tag(SmallTests.TAG)
035public class TestDefaultCompactSelection extends TestCompactionPolicy {
036
037  @Override
038  protected void config() {
039    super.config();
040    // DON'T change this config since all test cases assume HStore.BLOCKING_STOREFILES_KEY is 10.
041    this.conf.setLong(HStore.BLOCKING_STOREFILES_KEY, 10);
042  }
043
044  @Test
045  public void testCompactionRatio() throws IOException {
046    TimeOffsetEnvironmentEdge edge = new TimeOffsetEnvironmentEdge();
047    EnvironmentEdgeManager.injectEdge(edge);
048    /**
049     * NOTE: these tests are specific to describe the implementation of the current compaction
050     * algorithm. Developed to ensure that refactoring doesn't implicitly alter this.
051     */
052    long tooBig = maxSize + 1;
053
054    // default case. preserve user ratio on size
055    compactEquals(sfCreate(100, 50, 23, 12, 12), 23, 12, 12);
056    // less than compact threshold = don't compact
057    compactEquals(sfCreate(100, 50, 25, 12, 12) /* empty */);
058    // greater than compact size = skip those
059    compactEquals(sfCreate(tooBig, tooBig, 700, 700, 700), 700, 700, 700);
060    // big size + threshold
061    compactEquals(sfCreate(tooBig, tooBig, 700, 700) /* empty */);
062    // small files = don't care about ratio
063    compactEquals(sfCreate(7, 1, 1), 7, 1, 1);
064
065    // don't exceed max file compact threshold
066    // note: file selection starts with largest to smallest.
067    compactEquals(sfCreate(7, 6, 5, 4, 3, 2, 1), 5, 4, 3, 2, 1);
068
069    compactEquals(sfCreate(50, 10, 10, 10, 10), 10, 10, 10, 10);
070
071    compactEquals(sfCreate(10, 10, 10, 10, 50), 10, 10, 10, 10);
072
073    compactEquals(sfCreate(251, 253, 251, maxSize - 1), 251, 253, 251);
074
075    compactEquals(sfCreate(maxSize - 1, maxSize - 1, maxSize - 1) /* empty */);
076
077    // Always try and compact something to get below blocking storefile count
078    this.conf.setLong("hbase.hstore.compaction.min.size", 1);
079    store.storeEngine.getCompactionPolicy().setConf(conf);
080    compactEquals(sfCreate(512, 256, 128, 64, 32, 16, 8, 4, 2, 1), 4, 2, 1);
081    this.conf.setLong("hbase.hstore.compaction.min.size", minSize);
082    store.storeEngine.getCompactionPolicy().setConf(conf);
083
084    /* MAJOR COMPACTION */
085    // if a major compaction has been forced, then compact everything
086    compactEquals(sfCreate(50, 25, 12, 12), true, 50, 25, 12, 12);
087    // also choose files < threshold on major compaction
088    compactEquals(sfCreate(12, 12), true, 12, 12);
089    // even if one of those files is too big
090    compactEquals(sfCreate(tooBig, 12, 12), true, tooBig, 12, 12);
091    // don't exceed max file compact threshold, even with major compaction
092    store.forceMajor = true;
093    compactEquals(sfCreate(7, 6, 5, 4, 3, 2, 1), 5, 4, 3, 2, 1);
094    store.forceMajor = false;
095    // if we exceed maxCompactSize, downgrade to minor
096    // if not, it creates a 'snowball effect' when files >> maxCompactSize:
097    // the last file in compaction is the aggregate of all previous compactions
098    compactEquals(sfCreate(100, 50, 23, 12, 12), true, 23, 12, 12);
099    conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 1);
100    conf.setFloat("hbase.hregion.majorcompaction.jitter", 0);
101    store.storeEngine.getCompactionPolicy().setConf(conf);
102    try {
103      // The modTime of the mocked store file is the current time, so we need to increase the
104      // timestamp a bit to make sure that now - lowestModTime is greater than major compaction
105      // period(1ms).
106      // trigger an aged major compaction
107      List<HStoreFile> candidates = sfCreate(50, 25, 12, 12);
108      edge.increment(2);
109      compactEquals(candidates, 50, 25, 12, 12);
110      // major sure exceeding maxCompactSize also downgrades aged minors
111      candidates = sfCreate(100, 50, 23, 12, 12);
112      edge.increment(2);
113      compactEquals(candidates, 23, 12, 12);
114    } finally {
115      conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 1000 * 60 * 60 * 24);
116      conf.setFloat("hbase.hregion.majorcompaction.jitter", 0.20F);
117    }
118
119    /* REFERENCES == file is from a region that was split */
120    // treat storefiles that have references like a major compaction
121    compactEquals(sfCreate(true, 100, 50, 25, 12, 12), 100, 50, 25, 12, 12);
122    // reference files shouldn't obey max threshold
123    compactEquals(sfCreate(true, tooBig, 12, 12), tooBig, 12, 12);
124    // reference files should obey max file compact to avoid OOM
125    compactEquals(sfCreate(true, 7, 6, 5, 4, 3, 2, 1), 7, 6, 5, 4, 3);
126
127    // empty case
128    compactEquals(new ArrayList<>() /* empty */);
129    // empty case (because all files are too big)
130    compactEquals(sfCreate(tooBig, tooBig) /* empty */);
131  }
132
133  @Test
134  public void testOffPeakCompactionRatio() throws IOException {
135    /*
136     * NOTE: these tests are specific to describe the implementation of the current compaction
137     * algorithm. Developed to ensure that refactoring doesn't implicitly alter this.
138     */
139    // set an off-peak compaction threshold
140    this.conf.setFloat("hbase.hstore.compaction.ratio.offpeak", 5.0F);
141    store.storeEngine.getCompactionPolicy().setConf(this.conf);
142    // Test with and without the flag.
143    compactEquals(sfCreate(999, 50, 12, 12, 1), false, true, 50, 12, 12, 1);
144    compactEquals(sfCreate(999, 50, 12, 12, 1), 12, 12, 1);
145  }
146
147  @Test
148  public void testStuckStoreCompaction() throws IOException {
149    // Select the smallest compaction if the store is stuck.
150    compactEquals(sfCreate(99, 99, 99, 99, 99, 99, 30, 30, 30, 30), 30, 30, 30);
151    // If not stuck, standard policy applies.
152    compactEquals(sfCreate(99, 99, 99, 99, 99, 30, 30, 30, 30), 99, 30, 30, 30, 30);
153
154    // Add sufficiently small files to compaction, though
155    compactEquals(sfCreate(99, 99, 99, 99, 99, 99, 30, 30, 30, 15), 30, 30, 30, 15);
156    // Prefer earlier compaction to latter if the benefit is not significant
157    compactEquals(sfCreate(99, 99, 99, 99, 30, 26, 26, 29, 25, 25), 30, 26, 26);
158    // Prefer later compaction if the benefit is significant.
159    compactEquals(sfCreate(99, 99, 99, 99, 27, 27, 27, 20, 20, 20), 20, 20, 20);
160  }
161
162  @Test
163  public void testCompactionEmptyHFile() throws IOException {
164    // Set TTL
165    ScanInfo oldScanInfo = store.getScanInfo();
166    ScanInfo newScanInfo =
167      oldScanInfo.customize(oldScanInfo.getMaxVersions(), 600, oldScanInfo.getKeepDeletedCells());
168    store.setScanInfo(newScanInfo);
169    // Do not compact empty store file
170    List<HStoreFile> candidates = sfCreate(0);
171    for (HStoreFile file : candidates) {
172      if (file instanceof MockHStoreFile) {
173        MockHStoreFile mockFile = (MockHStoreFile) file;
174        mockFile.setTimeRangeTracker(TimeRangeTracker.create(TimeRangeTracker.Type.SYNC, -1, -1));
175        mockFile.setEntries(0);
176      }
177    }
178    // Test Default compactions
179    CompactionRequestImpl result =
180      ((RatioBasedCompactionPolicy) store.storeEngine.getCompactionPolicy())
181        .selectCompaction(candidates, new ArrayList<>(), false, false, false);
182    assertTrue(result.getFiles().isEmpty());
183    store.setScanInfo(oldScanInfo);
184  }
185}