001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.util.compaction;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertFalse;
022import static org.junit.jupiter.api.Assertions.assertTrue;
023import static org.mockito.ArgumentMatchers.any;
024import static org.mockito.ArgumentMatchers.isA;
025import static org.mockito.Mockito.doReturn;
026import static org.mockito.Mockito.mock;
027import static org.mockito.Mockito.spy;
028import static org.mockito.Mockito.when;
029
030import java.io.IOException;
031import java.util.List;
032import java.util.Optional;
033import java.util.Set;
034import java.util.stream.Collectors;
035import org.apache.commons.lang3.RandomStringUtils;
036import org.apache.hadoop.fs.FileStatus;
037import org.apache.hadoop.fs.FileSystem;
038import org.apache.hadoop.fs.Path;
039import org.apache.hadoop.hbase.HBaseTestingUtil;
040import org.apache.hadoop.hbase.TableName;
041import org.apache.hadoop.hbase.client.Connection;
042import org.apache.hadoop.hbase.client.RegionInfo;
043import org.apache.hadoop.hbase.client.RegionInfoBuilder;
044import org.apache.hadoop.hbase.client.TableDescriptor;
045import org.apache.hadoop.hbase.regionserver.HRegion;
046import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
047import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
048import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerForTest;
049import org.apache.hadoop.hbase.testclassification.SmallTests;
050import org.apache.hadoop.hbase.util.Bytes;
051import org.junit.jupiter.api.BeforeEach;
052import org.junit.jupiter.api.Tag;
053import org.junit.jupiter.api.Test;
054
055import org.apache.hbase.thirdparty.com.google.common.collect.Iterables;
056import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
057import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
058
059@Tag(SmallTests.TAG)
060public class TestMajorCompactionRequest {
061
062  protected static final HBaseTestingUtil UTILITY = new HBaseTestingUtil();
063  protected static final String FAMILY = "a";
064  protected Path rootRegionDir;
065  protected Path regionStoreDir;
066
067  @BeforeEach
068  public void setUp() throws Exception {
069    rootRegionDir = UTILITY.getDataTestDirOnTestFS("TestMajorCompactionRequest");
070    regionStoreDir = new Path(rootRegionDir, FAMILY);
071  }
072
073  @Test
074  public void testStoresNeedingCompaction() throws Exception {
075    // store files older than timestamp
076    List<StoreFileInfo> storeFiles = mockStoreFiles(regionStoreDir, 5, 10);
077    MajorCompactionRequest request = makeMockRequest(storeFiles, false);
078    Optional<MajorCompactionRequest> result =
079      request.createRequest(mock(Connection.class), Sets.newHashSet(FAMILY), 100);
080    assertTrue(result.isPresent());
081
082    // store files newer than timestamp
083    storeFiles = mockStoreFiles(regionStoreDir, 5, 101);
084    request = makeMockRequest(storeFiles, false);
085    result = request.createRequest(mock(Connection.class), Sets.newHashSet(FAMILY), 100);
086    assertFalse(result.isPresent());
087  }
088
089  @Test
090  public void testIfWeHaveNewReferenceFilesButOldStoreFiles() throws Exception {
091    // this tests that reference files that are new, but have older timestamps for the files
092    // they reference still will get compacted.
093    TableName tableName = TableName.valueOf("TestMajorCompactor");
094    TableDescriptor htd = UTILITY.createTableDescriptor(tableName, Bytes.toBytes(FAMILY));
095    RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build();
096    HRegion region =
097      HBaseTestingUtil.createRegionAndWAL(hri, rootRegionDir, UTILITY.getConfiguration(), htd);
098
099    Connection connection = mock(Connection.class);
100    // the reference file timestamp is newer
101    List<StoreFileInfo> storeFiles = mockStoreFiles(regionStoreDir, 4, 101);
102    List<Path> paths = storeFiles.stream().map(StoreFileInfo::getPath).collect(Collectors.toList());
103    // the files that are referenced are older, thus we still compact.
104    HRegionFileSystem fileSystem = mockFileSystem(region.getRegionInfo(), true, storeFiles, 50);
105    MajorCompactionRequest majorCompactionRequest =
106      spy(new MajorCompactionRequest(connection, region.getRegionInfo(), Sets.newHashSet(FAMILY)));
107    doReturn(paths).when(majorCompactionRequest).getReferenceFilePaths(any(FileSystem.class),
108      any(Path.class));
109    StoreFileTrackerForTest sft = mockSFT(true, storeFiles);
110    doReturn(fileSystem).when(majorCompactionRequest).getFileSystem();
111    doReturn(sft).when(majorCompactionRequest).getStoreFileTracker(any(), any());
112    doReturn(UTILITY.getConfiguration()).when(connection).getConfiguration();
113    Set<String> result =
114      majorCompactionRequest.getStoresRequiringCompaction(Sets.newHashSet("a"), 100);
115    assertEquals(FAMILY, Iterables.getOnlyElement(result));
116  }
117
118  protected StoreFileTrackerForTest mockSFT(boolean references, List<StoreFileInfo> storeFiles)
119    throws IOException {
120    StoreFileTrackerForTest sft = mock(StoreFileTrackerForTest.class);
121    doReturn(references).when(sft).hasReferences();
122    doReturn(storeFiles).when(sft).load();
123    return sft;
124  }
125
126  protected HRegionFileSystem mockFileSystem(RegionInfo info, boolean hasReferenceFiles,
127    List<StoreFileInfo> storeFiles) throws IOException {
128    long timestamp = storeFiles.stream().findFirst().get().getModificationTime();
129    return mockFileSystem(info, hasReferenceFiles, storeFiles, timestamp);
130  }
131
132  private HRegionFileSystem mockFileSystem(RegionInfo info, boolean hasReferenceFiles,
133    List<StoreFileInfo> storeFiles, long referenceFileTimestamp) throws IOException {
134    FileSystem fileSystem = mock(FileSystem.class);
135    if (hasReferenceFiles) {
136      FileStatus fileStatus = mock(FileStatus.class);
137      doReturn(referenceFileTimestamp).when(fileStatus).getModificationTime();
138      doReturn(fileStatus).when(fileSystem).getFileLinkStatus(isA(Path.class));
139    }
140    HRegionFileSystem mockSystem = mock(HRegionFileSystem.class);
141    doReturn(info).when(mockSystem).getRegionInfo();
142    doReturn(regionStoreDir).when(mockSystem).getStoreDir(FAMILY);
143    doReturn(fileSystem).when(mockSystem).getFileSystem();
144    return mockSystem;
145  }
146
147  protected List<StoreFileInfo> mockStoreFiles(Path regionStoreDir, int howMany, long timestamp)
148    throws IOException {
149    List<StoreFileInfo> infos = Lists.newArrayList();
150    int i = 0;
151    while (i < howMany) {
152      StoreFileInfo storeFileInfo = mock(StoreFileInfo.class);
153      doReturn(timestamp).doReturn(timestamp).when(storeFileInfo).getModificationTime();
154      doReturn(new Path(regionStoreDir, RandomStringUtils.randomAlphabetic(10))).when(storeFileInfo)
155        .getPath();
156      infos.add(storeFileInfo);
157      i++;
158    }
159    return infos;
160  }
161
162  private MajorCompactionRequest makeMockRequest(List<StoreFileInfo> storeFiles, boolean references)
163    throws IOException {
164    Connection connection = mock(Connection.class);
165    RegionInfo regionInfo = mock(RegionInfo.class);
166    when(regionInfo.getEncodedName()).thenReturn("HBase");
167    when(regionInfo.getTable()).thenReturn(TableName.valueOf("foo"));
168    MajorCompactionRequest request =
169      new MajorCompactionRequest(connection, regionInfo, Sets.newHashSet("a"));
170    MajorCompactionRequest spy = spy(request);
171    HRegionFileSystem fileSystem = mockFileSystem(regionInfo, references, storeFiles);
172    StoreFileTrackerForTest sft = mockSFT(references, storeFiles);
173    doReturn(sft).when(spy).getStoreFileTracker(any(), any());
174    doReturn(fileSystem).when(spy).getFileSystem();
175    return spy;
176  }
177}