Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.apache.hadoop.hbase.regionserver.CustomTieringMultiFileWriter.CUSTOM_TIERING_TIME_RANGE;
021import static org.junit.jupiter.api.Assertions.assertEquals;
022import static org.junit.jupiter.api.Assertions.assertFalse;
023import static org.junit.jupiter.api.Assertions.assertTrue;
024import static org.mockito.Mockito.mock;
025import static org.mockito.Mockito.when;
026
027import java.io.IOException;
028import java.util.ArrayList;
029import java.util.UUID;
030import org.apache.hadoop.fs.FileSystem;
031import org.apache.hadoop.fs.Path;
032import org.apache.hadoop.hbase.HBaseTestingUtil;
033import org.apache.hadoop.hbase.HConstants;
034import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
035import org.apache.hadoop.hbase.client.RegionInfo;
036import org.apache.hadoop.hbase.regionserver.compactions.CustomDateTieredCompactionPolicy;
037import org.apache.hadoop.hbase.regionserver.compactions.DateTieredCompactionRequest;
038import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerForTest;
039import org.apache.hadoop.hbase.testclassification.RegionServerTests;
040import org.apache.hadoop.hbase.testclassification.SmallTests;
041import org.apache.hadoop.hbase.util.Bytes;
042import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
043import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
044import org.junit.jupiter.api.Tag;
045import org.junit.jupiter.api.Test;
046
047@Tag(RegionServerTests.TAG)
048@Tag(SmallTests.TAG)
049public class TestCustomCellTieredCompactionPolicy {
050
051  private final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
052
053  public static final byte[] FAMILY = Bytes.toBytes("cf");
054
055  private HStoreFile createFile(Path file, long minValue, long maxValue, long size, int seqId)
056    throws IOException {
057    return createFile(mockRegionInfo(), file, minValue, maxValue, size, seqId, 0);
058  }
059
060  private HStoreFile createFile(RegionInfo regionInfo, Path file, long minValue, long maxValue,
061    long size, int seqId, long ageInDisk) throws IOException {
062    FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
063    HRegionFileSystem regionFileSystem =
064      new HRegionFileSystem(TEST_UTIL.getConfiguration(), fs, file, regionInfo);
065    StoreContext ctx = new StoreContext.Builder()
066      .withColumnFamilyDescriptor(ColumnFamilyDescriptorBuilder.newBuilder(FAMILY).build())
067      .withRegionFileSystem(regionFileSystem).build();
068    StoreFileTrackerForTest sftForTest =
069      new StoreFileTrackerForTest(TEST_UTIL.getConfiguration(), true, ctx);
070    MockHStoreFile msf =
071      new MockHStoreFile(TEST_UTIL, file, size, ageInDisk, false, (long) seqId, sftForTest);
072    TimeRangeTracker timeRangeTracker = TimeRangeTracker.create(TimeRangeTracker.Type.NON_SYNC);
073    timeRangeTracker.setMin(minValue);
074    timeRangeTracker.setMax(maxValue);
075    msf.setMetadataValue(CUSTOM_TIERING_TIME_RANGE, TimeRangeTracker.toByteArray(timeRangeTracker));
076    return msf;
077  }
078
079  private CustomDateTieredCompactionPolicy mockAndCreatePolicy() throws Exception {
080    RegionInfo mockedRegionInfo = mockRegionInfo();
081    return mockAndCreatePolicy(mockedRegionInfo);
082  }
083
084  private CustomDateTieredCompactionPolicy mockAndCreatePolicy(RegionInfo regionInfo)
085    throws Exception {
086    StoreConfigInformation mockedStoreConfig = mock(StoreConfigInformation.class);
087    when(mockedStoreConfig.getRegionInfo()).thenReturn(regionInfo);
088    CustomDateTieredCompactionPolicy policy =
089      new CustomDateTieredCompactionPolicy(TEST_UTIL.getConfiguration(), mockedStoreConfig);
090    return policy;
091  }
092
093  private RegionInfo mockRegionInfo() {
094    RegionInfo mockedRegionInfo = mock(RegionInfo.class);
095    when(mockedRegionInfo.getEncodedName()).thenReturn("1234567890987654321");
096    return mockedRegionInfo;
097  }
098
099  private Path preparePath() throws Exception {
100    FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
101    Path file =
102      new Path(TEST_UTIL.getDataTestDir(), UUID.randomUUID().toString().replaceAll("-", ""));
103    fs.create(file);
104    return file;
105  }
106
107  @Test
108  public void testGetCompactBoundariesForMajorNoOld() throws Exception {
109    CustomDateTieredCompactionPolicy policy = mockAndCreatePolicy();
110    Path file = preparePath();
111    ArrayList<HStoreFile> files = new ArrayList<>();
112    files.add(createFile(file, EnvironmentEdgeManager.currentTime(),
113      EnvironmentEdgeManager.currentTime(), 1024, 0));
114    files.add(createFile(file, EnvironmentEdgeManager.currentTime(),
115      EnvironmentEdgeManager.currentTime(), 1024, 1));
116    assertEquals(1,
117      ((DateTieredCompactionRequest) policy.selectMajorCompaction(files)).getBoundaries().size());
118  }
119
120  @Test
121  public void testGetCompactBoundariesForMajorAllOld() throws Exception {
122    CustomDateTieredCompactionPolicy policy = mockAndCreatePolicy();
123    Path file = preparePath();
124    ArrayList<HStoreFile> files = new ArrayList<>();
125    // The default cut off age is 10 years, so any of the min/max value there should get in the old
126    // tier
127    files.add(createFile(file, 0, 1, 1024, 0));
128    files.add(createFile(file, 2, 3, 1024, 1));
129    assertEquals(2,
130      ((DateTieredCompactionRequest) policy.selectMajorCompaction(files)).getBoundaries().size());
131  }
132
133  @Test
134  public void testGetCompactBoundariesForMajorOneOnEachSide() throws Exception {
135    CustomDateTieredCompactionPolicy policy = mockAndCreatePolicy();
136    Path file = preparePath();
137    ArrayList<HStoreFile> files = new ArrayList<>();
138    files.add(createFile(file, 0, 1, 1024, 0));
139    files.add(createFile(file, EnvironmentEdgeManager.currentTime(),
140      EnvironmentEdgeManager.currentTime(), 1024, 1));
141    assertEquals(3,
142      ((DateTieredCompactionRequest) policy.selectMajorCompaction(files)).getBoundaries().size());
143  }
144
145  @Test
146  public void testGetCompactBoundariesForMajorOneCrossing() throws Exception {
147    CustomDateTieredCompactionPolicy policy = mockAndCreatePolicy();
148    Path file = preparePath();
149    ArrayList<HStoreFile> files = new ArrayList<>();
150    files.add(createFile(file, 0, EnvironmentEdgeManager.currentTime(), 1024, 0));
151    assertEquals(3,
152      ((DateTieredCompactionRequest) policy.selectMajorCompaction(files)).getBoundaries().size());
153  }
154
155  @FunctionalInterface
156  interface PolicyValidator<T, U> {
157    void accept(T t, U u) throws Exception;
158  }
159
160  private void testShouldPerformMajorCompaction(long min, long max, int numFiles,
161    PolicyValidator<CustomDateTieredCompactionPolicy, ArrayList<HStoreFile>> validation)
162    throws Exception {
163    CustomDateTieredCompactionPolicy policy = mockAndCreatePolicy();
164    RegionInfo mockedRegionInfo = mockRegionInfo();
165    Path file = preparePath();
166    ArrayList<HStoreFile> files = new ArrayList<>();
167    ManualEnvironmentEdge timeMachine = new ManualEnvironmentEdge();
168    EnvironmentEdgeManager.injectEdge(timeMachine);
169    for (int i = 0; i < numFiles; i++) {
170      MockHStoreFile mockedSFile = (MockHStoreFile) createFile(mockedRegionInfo, file, min, max,
171        1024, 0, HConstants.DEFAULT_MAJOR_COMPACTION_PERIOD);
172      mockedSFile.setIsMajor(true);
173      files.add(mockedSFile);
174    }
175    EnvironmentEdgeManager.reset();
176    validation.accept(policy, files);
177  }
178
179  @Test
180  public void testShouldPerformMajorCompactionOneFileCrossing() throws Exception {
181    long max = EnvironmentEdgeManager.currentTime();
182    testShouldPerformMajorCompaction(0, max, 1,
183      (p, f) -> assertTrue(p.shouldPerformMajorCompaction(f)));
184  }
185
186  @Test
187  public void testShouldPerformMajorCompactionOneFileMinMaxLow() throws Exception {
188    testShouldPerformMajorCompaction(0, 1, 1,
189      (p, f) -> assertFalse(p.shouldPerformMajorCompaction(f)));
190  }
191
192  @Test
193  public void testShouldPerformMajorCompactionOneFileMinMaxHigh() throws Exception {
194    long currentTime = EnvironmentEdgeManager.currentTime();
195    testShouldPerformMajorCompaction(currentTime, currentTime, 1,
196      (p, f) -> assertFalse(p.shouldPerformMajorCompaction(f)));
197  }
198
199  @Test
200  public void testShouldPerformMajorCompactionTwoFilesMinMaxHigh() throws Exception {
201    long currentTime = EnvironmentEdgeManager.currentTime();
202    testShouldPerformMajorCompaction(currentTime, currentTime, 2,
203      (p, f) -> assertTrue(p.shouldPerformMajorCompaction(f)));
204  }
205
206  @Test
207  public void testSelectMinorCompactionTwoFilesNoOld() throws Exception {
208    CustomDateTieredCompactionPolicy policy = mockAndCreatePolicy();
209    Path file = preparePath();
210    ArrayList<HStoreFile> files = new ArrayList<>();
211    files.add(createFile(file, EnvironmentEdgeManager.currentTime(),
212      EnvironmentEdgeManager.currentTime(), 1024, 0));
213    files.add(createFile(file, EnvironmentEdgeManager.currentTime(),
214      EnvironmentEdgeManager.currentTime(), 1024, 1));
215    // Shouldn't do minor compaction, as minimum number of files
216    // for minor compactions is 3
217    assertEquals(0, policy.selectMinorCompaction(files, true, true).getFiles().size());
218  }
219
220  @Test
221  public void testSelectMinorCompactionThreeFilesNoOld() throws Exception {
222    CustomDateTieredCompactionPolicy policy = mockAndCreatePolicy();
223    Path file = preparePath();
224    ArrayList<HStoreFile> files = new ArrayList<>();
225    files.add(createFile(file, EnvironmentEdgeManager.currentTime(),
226      EnvironmentEdgeManager.currentTime(), 1024, 0));
227    files.add(createFile(file, EnvironmentEdgeManager.currentTime(),
228      EnvironmentEdgeManager.currentTime(), 1024, 1));
229    files.add(createFile(file, EnvironmentEdgeManager.currentTime(),
230      EnvironmentEdgeManager.currentTime(), 1024, 2));
231    assertEquals(3, policy.selectMinorCompaction(files, true, true).getFiles().size());
232  }
233
234  @Test
235  public void testSelectMinorCompactionThreeFilesAllOld() throws Exception {
236    CustomDateTieredCompactionPolicy policy = mockAndCreatePolicy();
237    Path file = preparePath();
238    ArrayList<HStoreFile> files = new ArrayList<>();
239    files.add(createFile(file, 0, 1, 1024, 0));
240    files.add(createFile(file, 1, 2, 1024, 1));
241    files.add(createFile(file, 3, 4, 1024, 2));
242    assertEquals(3, policy.selectMinorCompaction(files, true, true).getFiles().size());
243  }
244
245  @Test
246  public void testSelectMinorCompactionThreeFilesOneOldTwoNew() throws Exception {
247    CustomDateTieredCompactionPolicy policy = mockAndCreatePolicy();
248    Path file = preparePath();
249    ArrayList<HStoreFile> files = new ArrayList<>();
250    files.add(createFile(file, 0, 1, 1024, 0));
251    files.add(createFile(file, EnvironmentEdgeManager.currentTime(),
252      EnvironmentEdgeManager.currentTime(), 1024, 1));
253    files.add(createFile(file, EnvironmentEdgeManager.currentTime(),
254      EnvironmentEdgeManager.currentTime(), 1024, 2));
255    assertEquals(3, policy.selectMinorCompaction(files, true, true).getFiles().size());
256  }
257
258  @Test
259  public void testSelectMinorCompactionThreeFilesTwoOldOneNew() throws Exception {
260    CustomDateTieredCompactionPolicy policy = mockAndCreatePolicy();
261    Path file = preparePath();
262    ArrayList<HStoreFile> files = new ArrayList<>();
263    files.add(createFile(file, 0, 1, 1024, 0));
264    files.add(createFile(file, EnvironmentEdgeManager.currentTime(),
265      EnvironmentEdgeManager.currentTime(), 1024, 1));
266    files.add(createFile(file, EnvironmentEdgeManager.currentTime(),
267      EnvironmentEdgeManager.currentTime(), 1024, 2));
268    assertEquals(3, policy.selectMinorCompaction(files, true, true).getFiles().size());
269  }
270}