001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.apache.hadoop.hbase.regionserver.CustomTieringMultiFileWriter.CUSTOM_TIERING_TIME_RANGE;
021import static org.junit.Assert.assertEquals;
022import static org.junit.Assert.assertFalse;
023import static org.junit.Assert.assertTrue;
024import static org.mockito.Mockito.mock;
025import static org.mockito.Mockito.when;
026
027import java.io.IOException;
028import java.util.ArrayList;
029import java.util.UUID;
030import org.apache.hadoop.fs.FileSystem;
031import org.apache.hadoop.fs.Path;
032import org.apache.hadoop.hbase.HBaseClassTestRule;
033import org.apache.hadoop.hbase.HBaseTestingUtil;
034import org.apache.hadoop.hbase.HConstants;
035import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
036import org.apache.hadoop.hbase.client.RegionInfo;
037import org.apache.hadoop.hbase.regionserver.compactions.CustomDateTieredCompactionPolicy;
038import org.apache.hadoop.hbase.regionserver.compactions.DateTieredCompactionRequest;
039import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerForTest;
040import org.apache.hadoop.hbase.testclassification.RegionServerTests;
041import org.apache.hadoop.hbase.testclassification.SmallTests;
042import org.apache.hadoop.hbase.util.Bytes;
043import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
044import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
045import org.junit.ClassRule;
046import org.junit.Test;
047import org.junit.experimental.categories.Category;
048
049@Category({ RegionServerTests.class, SmallTests.class })
050public class TestCustomCellTieredCompactionPolicy {
051
052  @ClassRule
053  public static final HBaseClassTestRule CLASS_RULE =
054    HBaseClassTestRule.forClass(TestCustomCellTieredCompactionPolicy.class);
055
056  private final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
057
058  public static final byte[] FAMILY = Bytes.toBytes("cf");
059
060  private HStoreFile createFile(Path file, long minValue, long maxValue, long size, int seqId)
061    throws IOException {
062    return createFile(mockRegionInfo(), file, minValue, maxValue, size, seqId, 0);
063  }
064
065  private HStoreFile createFile(RegionInfo regionInfo, Path file, long minValue, long maxValue,
066    long size, int seqId, long ageInDisk) throws IOException {
067    FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
068    HRegionFileSystem regionFileSystem =
069      new HRegionFileSystem(TEST_UTIL.getConfiguration(), fs, file, regionInfo);
070    StoreContext ctx = new StoreContext.Builder()
071      .withColumnFamilyDescriptor(ColumnFamilyDescriptorBuilder.newBuilder(FAMILY).build())
072      .withRegionFileSystem(regionFileSystem).build();
073    StoreFileTrackerForTest sftForTest =
074      new StoreFileTrackerForTest(TEST_UTIL.getConfiguration(), true, ctx);
075    MockHStoreFile msf =
076      new MockHStoreFile(TEST_UTIL, file, size, ageInDisk, false, (long) seqId, sftForTest);
077    TimeRangeTracker timeRangeTracker = TimeRangeTracker.create(TimeRangeTracker.Type.NON_SYNC);
078    timeRangeTracker.setMin(minValue);
079    timeRangeTracker.setMax(maxValue);
080    msf.setMetadataValue(CUSTOM_TIERING_TIME_RANGE, TimeRangeTracker.toByteArray(timeRangeTracker));
081    return msf;
082  }
083
084  private CustomDateTieredCompactionPolicy mockAndCreatePolicy() throws Exception {
085    RegionInfo mockedRegionInfo = mockRegionInfo();
086    return mockAndCreatePolicy(mockedRegionInfo);
087  }
088
089  private CustomDateTieredCompactionPolicy mockAndCreatePolicy(RegionInfo regionInfo)
090    throws Exception {
091    StoreConfigInformation mockedStoreConfig = mock(StoreConfigInformation.class);
092    when(mockedStoreConfig.getRegionInfo()).thenReturn(regionInfo);
093    CustomDateTieredCompactionPolicy policy =
094      new CustomDateTieredCompactionPolicy(TEST_UTIL.getConfiguration(), mockedStoreConfig);
095    return policy;
096  }
097
098  private RegionInfo mockRegionInfo() {
099    RegionInfo mockedRegionInfo = mock(RegionInfo.class);
100    when(mockedRegionInfo.getEncodedName()).thenReturn("1234567890987654321");
101    return mockedRegionInfo;
102  }
103
104  private Path preparePath() throws Exception {
105    FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
106    Path file =
107      new Path(TEST_UTIL.getDataTestDir(), UUID.randomUUID().toString().replaceAll("-", ""));
108    fs.create(file);
109    return file;
110  }
111
112  @Test
113  public void testGetCompactBoundariesForMajorNoOld() throws Exception {
114    CustomDateTieredCompactionPolicy policy = mockAndCreatePolicy();
115    Path file = preparePath();
116    ArrayList<HStoreFile> files = new ArrayList<>();
117    files.add(createFile(file, EnvironmentEdgeManager.currentTime(),
118      EnvironmentEdgeManager.currentTime(), 1024, 0));
119    files.add(createFile(file, EnvironmentEdgeManager.currentTime(),
120      EnvironmentEdgeManager.currentTime(), 1024, 1));
121    assertEquals(1,
122      ((DateTieredCompactionRequest) policy.selectMajorCompaction(files)).getBoundaries().size());
123  }
124
125  @Test
126  public void testGetCompactBoundariesForMajorAllOld() throws Exception {
127    CustomDateTieredCompactionPolicy policy = mockAndCreatePolicy();
128    Path file = preparePath();
129    ArrayList<HStoreFile> files = new ArrayList<>();
130    // The default cut off age is 10 years, so any of the min/max value there should get in the old
131    // tier
132    files.add(createFile(file, 0, 1, 1024, 0));
133    files.add(createFile(file, 2, 3, 1024, 1));
134    assertEquals(2,
135      ((DateTieredCompactionRequest) policy.selectMajorCompaction(files)).getBoundaries().size());
136  }
137
138  @Test
139  public void testGetCompactBoundariesForMajorOneOnEachSide() throws Exception {
140    CustomDateTieredCompactionPolicy policy = mockAndCreatePolicy();
141    Path file = preparePath();
142    ArrayList<HStoreFile> files = new ArrayList<>();
143    files.add(createFile(file, 0, 1, 1024, 0));
144    files.add(createFile(file, EnvironmentEdgeManager.currentTime(),
145      EnvironmentEdgeManager.currentTime(), 1024, 1));
146    assertEquals(3,
147      ((DateTieredCompactionRequest) policy.selectMajorCompaction(files)).getBoundaries().size());
148  }
149
150  @Test
151  public void testGetCompactBoundariesForMajorOneCrossing() throws Exception {
152    CustomDateTieredCompactionPolicy policy = mockAndCreatePolicy();
153    Path file = preparePath();
154    ArrayList<HStoreFile> files = new ArrayList<>();
155    files.add(createFile(file, 0, EnvironmentEdgeManager.currentTime(), 1024, 0));
156    assertEquals(3,
157      ((DateTieredCompactionRequest) policy.selectMajorCompaction(files)).getBoundaries().size());
158  }
159
160  @FunctionalInterface
161  interface PolicyValidator<T, U> {
162    void accept(T t, U u) throws Exception;
163  }
164
165  private void testShouldPerformMajorCompaction(long min, long max, int numFiles,
166    PolicyValidator<CustomDateTieredCompactionPolicy, ArrayList<HStoreFile>> validation)
167    throws Exception {
168    CustomDateTieredCompactionPolicy policy = mockAndCreatePolicy();
169    RegionInfo mockedRegionInfo = mockRegionInfo();
170    Path file = preparePath();
171    ArrayList<HStoreFile> files = new ArrayList<>();
172    ManualEnvironmentEdge timeMachine = new ManualEnvironmentEdge();
173    EnvironmentEdgeManager.injectEdge(timeMachine);
174    for (int i = 0; i < numFiles; i++) {
175      MockHStoreFile mockedSFile = (MockHStoreFile) createFile(mockedRegionInfo, file, min, max,
176        1024, 0, HConstants.DEFAULT_MAJOR_COMPACTION_PERIOD);
177      mockedSFile.setIsMajor(true);
178      files.add(mockedSFile);
179    }
180    EnvironmentEdgeManager.reset();
181    validation.accept(policy, files);
182  }
183
184  @Test
185  public void testShouldPerformMajorCompactionOneFileCrossing() throws Exception {
186    long max = EnvironmentEdgeManager.currentTime();
187    testShouldPerformMajorCompaction(0, max, 1,
188      (p, f) -> assertTrue(p.shouldPerformMajorCompaction(f)));
189  }
190
191  @Test
192  public void testShouldPerformMajorCompactionOneFileMinMaxLow() throws Exception {
193    testShouldPerformMajorCompaction(0, 1, 1,
194      (p, f) -> assertFalse(p.shouldPerformMajorCompaction(f)));
195  }
196
197  @Test
198  public void testShouldPerformMajorCompactionOneFileMinMaxHigh() throws Exception {
199    long currentTime = EnvironmentEdgeManager.currentTime();
200    testShouldPerformMajorCompaction(currentTime, currentTime, 1,
201      (p, f) -> assertFalse(p.shouldPerformMajorCompaction(f)));
202  }
203
204  @Test
205  public void testShouldPerformMajorCompactionTwoFilesMinMaxHigh() throws Exception {
206    long currentTime = EnvironmentEdgeManager.currentTime();
207    testShouldPerformMajorCompaction(currentTime, currentTime, 2,
208      (p, f) -> assertTrue(p.shouldPerformMajorCompaction(f)));
209  }
210
211  @Test
212  public void testSelectMinorCompactionTwoFilesNoOld() throws Exception {
213    CustomDateTieredCompactionPolicy policy = mockAndCreatePolicy();
214    Path file = preparePath();
215    ArrayList<HStoreFile> files = new ArrayList<>();
216    files.add(createFile(file, EnvironmentEdgeManager.currentTime(),
217      EnvironmentEdgeManager.currentTime(), 1024, 0));
218    files.add(createFile(file, EnvironmentEdgeManager.currentTime(),
219      EnvironmentEdgeManager.currentTime(), 1024, 1));
220    // Shouldn't do minor compaction, as minimum number of files
221    // for minor compactions is 3
222    assertEquals(0, policy.selectMinorCompaction(files, true, true).getFiles().size());
223  }
224
225  @Test
226  public void testSelectMinorCompactionThreeFilesNoOld() throws Exception {
227    CustomDateTieredCompactionPolicy policy = mockAndCreatePolicy();
228    Path file = preparePath();
229    ArrayList<HStoreFile> files = new ArrayList<>();
230    files.add(createFile(file, EnvironmentEdgeManager.currentTime(),
231      EnvironmentEdgeManager.currentTime(), 1024, 0));
232    files.add(createFile(file, EnvironmentEdgeManager.currentTime(),
233      EnvironmentEdgeManager.currentTime(), 1024, 1));
234    files.add(createFile(file, EnvironmentEdgeManager.currentTime(),
235      EnvironmentEdgeManager.currentTime(), 1024, 2));
236    assertEquals(3, policy.selectMinorCompaction(files, true, true).getFiles().size());
237  }
238
239  @Test
240  public void testSelectMinorCompactionThreeFilesAllOld() throws Exception {
241    CustomDateTieredCompactionPolicy policy = mockAndCreatePolicy();
242    Path file = preparePath();
243    ArrayList<HStoreFile> files = new ArrayList<>();
244    files.add(createFile(file, 0, 1, 1024, 0));
245    files.add(createFile(file, 1, 2, 1024, 1));
246    files.add(createFile(file, 3, 4, 1024, 2));
247    assertEquals(3, policy.selectMinorCompaction(files, true, true).getFiles().size());
248  }
249
250  @Test
251  public void testSelectMinorCompactionThreeFilesOneOldTwoNew() throws Exception {
252    CustomDateTieredCompactionPolicy policy = mockAndCreatePolicy();
253    Path file = preparePath();
254    ArrayList<HStoreFile> files = new ArrayList<>();
255    files.add(createFile(file, 0, 1, 1024, 0));
256    files.add(createFile(file, EnvironmentEdgeManager.currentTime(),
257      EnvironmentEdgeManager.currentTime(), 1024, 1));
258    files.add(createFile(file, EnvironmentEdgeManager.currentTime(),
259      EnvironmentEdgeManager.currentTime(), 1024, 2));
260    assertEquals(3, policy.selectMinorCompaction(files, true, true).getFiles().size());
261  }
262
263  @Test
264  public void testSelectMinorCompactionThreeFilesTwoOldOneNew() throws Exception {
265    CustomDateTieredCompactionPolicy policy = mockAndCreatePolicy();
266    Path file = preparePath();
267    ArrayList<HStoreFile> files = new ArrayList<>();
268    files.add(createFile(file, 0, 1, 1024, 0));
269    files.add(createFile(file, EnvironmentEdgeManager.currentTime(),
270      EnvironmentEdgeManager.currentTime(), 1024, 1));
271    files.add(createFile(file, EnvironmentEdgeManager.currentTime(),
272      EnvironmentEdgeManager.currentTime(), 1024, 2));
273    assertEquals(3, policy.selectMinorCompaction(files, true, true).getFiles().size());
274  }
275}