001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.apache.hadoop.hbase.regionserver.CustomTieringMultiFileWriter.CUSTOM_TIERING_TIME_RANGE;
021import static org.junit.jupiter.api.Assertions.assertEquals;
022import static org.junit.jupiter.api.Assertions.assertFalse;
023import static org.junit.jupiter.api.Assertions.assertTrue;
024import static org.mockito.Mockito.mock;
025import static org.mockito.Mockito.when;
026
027import java.io.IOException;
028import java.util.ArrayList;
029import java.util.UUID;
030import org.apache.hadoop.fs.FileSystem;
031import org.apache.hadoop.fs.Path;
032import org.apache.hadoop.hbase.HBaseTestingUtil;
033import org.apache.hadoop.hbase.HConstants;
034import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
035import org.apache.hadoop.hbase.client.RegionInfo;
036import org.apache.hadoop.hbase.regionserver.compactions.CustomDateTieredCompactionPolicy;
037import org.apache.hadoop.hbase.regionserver.compactions.DateTieredCompactionRequest;
038import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerForTest;
039import org.apache.hadoop.hbase.testclassification.RegionServerTests;
040import org.apache.hadoop.hbase.testclassification.SmallTests;
041import org.apache.hadoop.hbase.util.Bytes;
042import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
043import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
044import org.junit.jupiter.api.Tag;
045import org.junit.jupiter.api.Test;
046
047@Tag(RegionServerTests.TAG)
048@Tag(SmallTests.TAG)
049public class TestCustomCellTieredCompactionPolicy {
050
051  private final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
052
053  public static final byte[] FAMILY = Bytes.toBytes("cf");
054
055  private HStoreFile createFile(Path file, long minValue, long maxValue, long size, int seqId)
056    throws IOException {
057    return createFile(mockRegionInfo(), file, minValue, maxValue, size, seqId, 0);
058  }
059
060  private HStoreFile createFile(RegionInfo regionInfo, Path file, long minValue, long maxValue,
061    long size, int seqId, long ageInDisk) throws IOException {
062    FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
063    HRegionFileSystem regionFileSystem =
064      new HRegionFileSystem(TEST_UTIL.getConfiguration(), fs, file, regionInfo);
065    StoreContext ctx = new StoreContext.Builder()
066      .withColumnFamilyDescriptor(ColumnFamilyDescriptorBuilder.newBuilder(FAMILY).build())
067      .withRegionFileSystem(regionFileSystem).build();
068    StoreFileTrackerForTest sftForTest =
069      new StoreFileTrackerForTest(TEST_UTIL.getConfiguration(), true, ctx);
070    MockHStoreFile msf =
071      new MockHStoreFile(TEST_UTIL, file, size, ageInDisk, false, (long) seqId, sftForTest);
072    TimeRangeTracker timeRangeTracker = TimeRangeTracker.create(TimeRangeTracker.Type.NON_SYNC);
073    timeRangeTracker.setMin(minValue);
074    timeRangeTracker.setMax(maxValue);
075    msf.setMetadataValue(CUSTOM_TIERING_TIME_RANGE, TimeRangeTracker.toByteArray(timeRangeTracker));
076    return msf;
077  }
078
079  private CustomDateTieredCompactionPolicy mockAndCreatePolicy() throws Exception {
080    RegionInfo mockedRegionInfo = mockRegionInfo();
081    return mockAndCreatePolicy(mockedRegionInfo);
082  }
083
084  private CustomDateTieredCompactionPolicy mockAndCreatePolicy(RegionInfo regionInfo)
085    throws Exception {
086    StoreConfigInformation mockedStoreConfig = mock(StoreConfigInformation.class);
087    when(mockedStoreConfig.getRegionInfo()).thenReturn(regionInfo);
088    CustomDateTieredCompactionPolicy policy =
089      new CustomDateTieredCompactionPolicy(TEST_UTIL.getConfiguration(), mockedStoreConfig);
090    return policy;
091  }
092
093  private RegionInfo mockRegionInfo() {
094    RegionInfo mockedRegionInfo = mock(RegionInfo.class);
095    when(mockedRegionInfo.getEncodedName()).thenReturn("1234567890987654321");
096    return mockedRegionInfo;
097  }
098
099  private Path preparePath() throws Exception {
100    FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
101    Path file =
102      new Path(TEST_UTIL.getDataTestDir(), UUID.randomUUID().toString().replaceAll("-", ""));
103    fs.create(file);
104    return file;
105  }
106
107  @Test
108  public void testGetCompactBoundariesForMajorNoOld() throws Exception {
109    CustomDateTieredCompactionPolicy policy = mockAndCreatePolicy();
110    Path file = preparePath();
111    ArrayList<HStoreFile> files = new ArrayList<>();
112    files.add(createFile(file, EnvironmentEdgeManager.currentTime(),
113      EnvironmentEdgeManager.currentTime(), 1024, 0));
114    files.add(createFile(file, EnvironmentEdgeManager.currentTime(),
115      EnvironmentEdgeManager.currentTime(), 1024, 1));
116    // getCompactBoundariesForMajor always offers [MIN_VALUE, cutOffTimestamp] now, regardless of
117    // the files being compacted, so the boundary count is always 2.
118    assertEquals(2,
119      ((DateTieredCompactionRequest) policy.selectMajorCompaction(files)).getBoundaries().size());
120  }
121
122  @Test
123  public void testGetCompactBoundariesForMajorAllOld() throws Exception {
124    CustomDateTieredCompactionPolicy policy = mockAndCreatePolicy();
125    Path file = preparePath();
126    ArrayList<HStoreFile> files = new ArrayList<>();
127    // The default cut off age is 10 years, so any of the min/max value there should get in the old
128    // tier
129    files.add(createFile(file, 0, 1, 1024, 0));
130    files.add(createFile(file, 2, 3, 1024, 1));
131    assertEquals(2,
132      ((DateTieredCompactionRequest) policy.selectMajorCompaction(files)).getBoundaries().size());
133  }
134
135  @Test
136  public void testGetCompactBoundariesForMajorOneOnEachSide() throws Exception {
137    CustomDateTieredCompactionPolicy policy = mockAndCreatePolicy();
138    Path file = preparePath();
139    ArrayList<HStoreFile> files = new ArrayList<>();
140    files.add(createFile(file, 0, 1, 1024, 0));
141    files.add(createFile(file, EnvironmentEdgeManager.currentTime(),
142      EnvironmentEdgeManager.currentTime(), 1024, 1));
143    assertEquals(2,
144      ((DateTieredCompactionRequest) policy.selectMajorCompaction(files)).getBoundaries().size());
145  }
146
147  @Test
148  public void testGetCompactBoundariesForMajorOneCrossing() throws Exception {
149    CustomDateTieredCompactionPolicy policy = mockAndCreatePolicy();
150    Path file = preparePath();
151    ArrayList<HStoreFile> files = new ArrayList<>();
152    files.add(createFile(file, 0, EnvironmentEdgeManager.currentTime(), 1024, 0));
153    assertEquals(2,
154      ((DateTieredCompactionRequest) policy.selectMajorCompaction(files)).getBoundaries().size());
155  }
156
157  @FunctionalInterface
158  interface PolicyValidator<T, U> {
159    void accept(T t, U u) throws Exception;
160  }
161
162  private void testShouldPerformMajorCompaction(long min, long max, int numFiles,
163    PolicyValidator<CustomDateTieredCompactionPolicy, ArrayList<HStoreFile>> validation)
164    throws Exception {
165    CustomDateTieredCompactionPolicy policy = mockAndCreatePolicy();
166    RegionInfo mockedRegionInfo = mockRegionInfo();
167    Path file = preparePath();
168    ArrayList<HStoreFile> files = new ArrayList<>();
169    ManualEnvironmentEdge timeMachine = new ManualEnvironmentEdge();
170    EnvironmentEdgeManager.injectEdge(timeMachine);
171    for (int i = 0; i < numFiles; i++) {
172      MockHStoreFile mockedSFile = (MockHStoreFile) createFile(mockedRegionInfo, file, min, max,
173        1024, 0, HConstants.DEFAULT_MAJOR_COMPACTION_PERIOD);
174      mockedSFile.setIsMajor(true);
175      files.add(mockedSFile);
176    }
177    EnvironmentEdgeManager.reset();
178    validation.accept(policy, files);
179  }
180
181  @Test
182  public void testShouldPerformMajorCompactionOneFileCrossing() throws Exception {
183    long max = EnvironmentEdgeManager.currentTime();
184    testShouldPerformMajorCompaction(0, max, 1,
185      (p, f) -> assertTrue(p.shouldPerformMajorCompaction(f)));
186  }
187
188  @Test
189  public void testShouldPerformMajorCompactionOneFileMinMaxLow() throws Exception {
190    testShouldPerformMajorCompaction(0, 1, 1,
191      (p, f) -> assertFalse(p.shouldPerformMajorCompaction(f)));
192  }
193
194  @Test
195  public void testShouldPerformMajorCompactionOneFileMinMaxHigh() throws Exception {
196    long currentTime = EnvironmentEdgeManager.currentTime();
197    testShouldPerformMajorCompaction(currentTime, currentTime, 1,
198      (p, f) -> assertFalse(p.shouldPerformMajorCompaction(f)));
199  }
200
201  @Test
202  public void testShouldPerformMajorCompactionTwoFilesMinMaxHigh() throws Exception {
203    long currentTime = EnvironmentEdgeManager.currentTime();
204    testShouldPerformMajorCompaction(currentTime, currentTime, 2,
205      (p, f) -> assertTrue(p.shouldPerformMajorCompaction(f)));
206  }
207
208  @Test
209  public void testSelectMinorCompactionTwoFilesNoOld() throws Exception {
210    CustomDateTieredCompactionPolicy policy = mockAndCreatePolicy();
211    Path file = preparePath();
212    ArrayList<HStoreFile> files = new ArrayList<>();
213    files.add(createFile(file, EnvironmentEdgeManager.currentTime(),
214      EnvironmentEdgeManager.currentTime(), 1024, 0));
215    files.add(createFile(file, EnvironmentEdgeManager.currentTime(),
216      EnvironmentEdgeManager.currentTime(), 1024, 1));
217    // Shouldn't do minor compaction, as minimum number of files
218    // for minor compactions is 3
219    assertEquals(0, policy.selectMinorCompaction(files, true, true).getFiles().size());
220  }
221
222  @Test
223  public void testSelectMinorCompactionThreeFilesNoOld() throws Exception {
224    CustomDateTieredCompactionPolicy policy = mockAndCreatePolicy();
225    Path file = preparePath();
226    ArrayList<HStoreFile> files = new ArrayList<>();
227    files.add(createFile(file, EnvironmentEdgeManager.currentTime(),
228      EnvironmentEdgeManager.currentTime(), 1024, 0));
229    files.add(createFile(file, EnvironmentEdgeManager.currentTime(),
230      EnvironmentEdgeManager.currentTime(), 1024, 1));
231    files.add(createFile(file, EnvironmentEdgeManager.currentTime(),
232      EnvironmentEdgeManager.currentTime(), 1024, 2));
233    assertEquals(3, policy.selectMinorCompaction(files, true, true).getFiles().size());
234  }
235
236  @Test
237  public void testSelectMinorCompactionThreeFilesAllOld() throws Exception {
238    CustomDateTieredCompactionPolicy policy = mockAndCreatePolicy();
239    Path file = preparePath();
240    ArrayList<HStoreFile> files = new ArrayList<>();
241    files.add(createFile(file, 0, 1, 1024, 0));
242    files.add(createFile(file, 1, 2, 1024, 1));
243    files.add(createFile(file, 3, 4, 1024, 2));
244    assertEquals(3, policy.selectMinorCompaction(files, true, true).getFiles().size());
245  }
246
247  @Test
248  public void testSelectMinorCompactionThreeFilesOneOldTwoNew() throws Exception {
249    CustomDateTieredCompactionPolicy policy = mockAndCreatePolicy();
250    Path file = preparePath();
251    ArrayList<HStoreFile> files = new ArrayList<>();
252    files.add(createFile(file, 0, 1, 1024, 0));
253    files.add(createFile(file, EnvironmentEdgeManager.currentTime(),
254      EnvironmentEdgeManager.currentTime(), 1024, 1));
255    files.add(createFile(file, EnvironmentEdgeManager.currentTime(),
256      EnvironmentEdgeManager.currentTime(), 1024, 2));
257    assertEquals(3, policy.selectMinorCompaction(files, true, true).getFiles().size());
258  }
259
260  @Test
261  public void testSelectMinorCompactionThreeFilesTwoOldOneNew() throws Exception {
262    CustomDateTieredCompactionPolicy policy = mockAndCreatePolicy();
263    Path file = preparePath();
264    ArrayList<HStoreFile> files = new ArrayList<>();
265    files.add(createFile(file, 0, 1, 1024, 0));
266    files.add(createFile(file, EnvironmentEdgeManager.currentTime(),
267      EnvironmentEdgeManager.currentTime(), 1024, 1));
268    files.add(createFile(file, EnvironmentEdgeManager.currentTime(),
269      EnvironmentEdgeManager.currentTime(), 1024, 2));
270    assertEquals(3, policy.selectMinorCompaction(files, true, true).getFiles().size());
271  }
272}