001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.util.compaction;
019
020import java.io.IOException;
021import java.util.List;
022import java.util.Optional;
023import java.util.Set;
024import java.util.stream.Collectors;
025import org.apache.commons.lang3.RandomStringUtils;
026import org.apache.hadoop.conf.Configuration;
027import org.apache.hadoop.fs.FileStatus;
028import org.apache.hadoop.fs.FileSystem;
029import org.apache.hadoop.fs.Path;
030import org.apache.hadoop.hbase.HBaseClassTestRule;
031import org.apache.hadoop.hbase.HBaseTestingUtility;
032import org.apache.hadoop.hbase.TableName;
033import org.apache.hadoop.hbase.client.Connection;
034import org.apache.hadoop.hbase.client.RegionInfo;
035import org.apache.hadoop.hbase.client.RegionInfoBuilder;
036import org.apache.hadoop.hbase.client.TableDescriptor;
037import org.apache.hadoop.hbase.regionserver.HRegion;
038import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
039import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
040import org.apache.hadoop.hbase.testclassification.SmallTests;
041import org.apache.hadoop.hbase.util.Bytes;
042import org.apache.hbase.thirdparty.com.google.common.collect.Iterables;
043import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
044import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
045import static org.junit.Assert.assertEquals;
046import static org.junit.Assert.assertFalse;
047import static org.junit.Assert.assertTrue;
048import org.junit.Before;
049import org.junit.ClassRule;
050import org.junit.Test;
051import org.junit.experimental.categories.Category;
052import static org.mockito.ArgumentMatchers.any;
053import static org.mockito.ArgumentMatchers.eq;
054import static org.mockito.ArgumentMatchers.isA;
055import static org.mockito.Matchers.anyString;
056import static org.mockito.Mockito.doReturn;
057import static org.mockito.Mockito.mock;
058import static org.mockito.Mockito.spy;
059import static org.mockito.Mockito.when;
060
061@Category({SmallTests.class})
062public class TestMajorCompactionRequest {
063  @ClassRule
064  public static final HBaseClassTestRule CLASS_RULE =
065      HBaseClassTestRule.forClass(TestMajorCompactionRequest.class);
066
067  private static final HBaseTestingUtility UTILITY = new HBaseTestingUtility();
068  private static final String FAMILY = "a";
069  private Path rootRegionDir;
070  private Path regionStoreDir;
071
072  @Before public void setUp() throws Exception {
073    rootRegionDir = UTILITY.getDataTestDirOnTestFS("TestMajorCompactionRequest");
074    regionStoreDir = new Path(rootRegionDir, FAMILY);
075  }
076
077  @Test public void testStoresNeedingCompaction() throws Exception {
078    // store files older than timestamp
079    List<StoreFileInfo> storeFiles = mockStoreFiles(regionStoreDir, 5, 10);
080    MajorCompactionRequest request = makeMockRequest(100, storeFiles, false);
081    Optional<MajorCompactionRequest> result =
082        request.createRequest(mock(Configuration.class), Sets.newHashSet(FAMILY));
083    assertTrue(result.isPresent());
084
085    // store files newer than timestamp
086    storeFiles = mockStoreFiles(regionStoreDir, 5, 101);
087    request = makeMockRequest(100, storeFiles, false);
088    result = request.createRequest(mock(Configuration.class), Sets.newHashSet(FAMILY));
089    assertFalse(result.isPresent());
090  }
091
092  @Test public void testIfWeHaveNewReferenceFilesButOldStoreFiles() throws Exception {
093    // this tests that reference files that are new, but have older timestamps for the files
094    // they reference still will get compacted.
095    TableName table = TableName.valueOf("TestMajorCompactor");
096    TableDescriptor htd = UTILITY.createTableDescriptor(table, Bytes.toBytes(FAMILY));
097    RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build();
098    HRegion region =
099        HBaseTestingUtility.createRegionAndWAL(hri, rootRegionDir, UTILITY.getConfiguration(), htd);
100
101    Configuration configuration = mock(Configuration.class);
102    // the reference file timestamp is newer
103    List<StoreFileInfo> storeFiles = mockStoreFiles(regionStoreDir, 4, 101);
104    List<Path> paths = storeFiles.stream().map(StoreFileInfo::getPath).collect(Collectors.toList());
105    // the files that are referenced are older, thus we still compact.
106    HRegionFileSystem fileSystem =
107        mockFileSystem(region.getRegionInfo(), true, storeFiles, 50);
108    MajorCompactionRequest majorCompactionRequest = spy(new MajorCompactionRequest(configuration,
109        region.getRegionInfo(), Sets.newHashSet(FAMILY), 100));
110    doReturn(mock(Connection.class)).when(majorCompactionRequest).getConnection(eq(configuration));
111    doReturn(paths).when(majorCompactionRequest).getReferenceFilePaths(any(FileSystem.class),
112        any(Path.class));
113    doReturn(fileSystem).when(majorCompactionRequest).getFileSystem(any(Connection.class));
114    Set<String> result = majorCompactionRequest.getStoresRequiringCompaction(Sets.newHashSet("a"));
115    assertEquals(FAMILY, Iterables.getOnlyElement(result));
116  }
117
118  private HRegionFileSystem mockFileSystem(RegionInfo info, boolean hasReferenceFiles,
119      List<StoreFileInfo> storeFiles) throws IOException {
120    long timestamp = storeFiles.stream().findFirst().get().getModificationTime();
121    return mockFileSystem(info, hasReferenceFiles, storeFiles, timestamp);
122  }
123
124  private HRegionFileSystem mockFileSystem(RegionInfo info, boolean hasReferenceFiles,
125      List<StoreFileInfo> storeFiles, long referenceFileTimestamp) throws IOException {
126    FileSystem fileSystem = mock(FileSystem.class);
127    if (hasReferenceFiles) {
128      FileStatus fileStatus = mock(FileStatus.class);
129      doReturn(referenceFileTimestamp).when(fileStatus).getModificationTime();
130      doReturn(fileStatus).when(fileSystem).getFileLinkStatus(isA(Path.class));
131    }
132    HRegionFileSystem mockSystem = mock(HRegionFileSystem.class);
133    doReturn(info).when(mockSystem).getRegionInfo();
134    doReturn(regionStoreDir).when(mockSystem).getStoreDir(FAMILY);
135    doReturn(hasReferenceFiles).when(mockSystem).hasReferences(anyString());
136    doReturn(storeFiles).when(mockSystem).getStoreFiles(anyString());
137    doReturn(fileSystem).when(mockSystem).getFileSystem();
138    return mockSystem;
139  }
140
141  private List<StoreFileInfo> mockStoreFiles(Path regionStoreDir, int howMany, long timestamp)
142      throws IOException {
143    List<StoreFileInfo> infos = Lists.newArrayList();
144    int i = 0;
145    while (i < howMany) {
146      StoreFileInfo storeFileInfo = mock(StoreFileInfo.class);
147      doReturn(timestamp).doReturn(timestamp).when(storeFileInfo).getModificationTime();
148      doReturn(new Path(regionStoreDir, RandomStringUtils.randomAlphabetic(10))).when(storeFileInfo)
149          .getPath();
150      infos.add(storeFileInfo);
151      i++;
152    }
153    return infos;
154  }
155
156  private MajorCompactionRequest makeMockRequest(long timestamp, List<StoreFileInfo> storeFiles,
157      boolean references) throws IOException {
158    Configuration configuration = mock(Configuration.class);
159    RegionInfo regionInfo = mock(RegionInfo.class);
160    when(regionInfo.getEncodedName()).thenReturn("HBase");
161    when(regionInfo.getTable()).thenReturn(TableName.valueOf("foo"));
162    MajorCompactionRequest request =
163        new MajorCompactionRequest(configuration, regionInfo, Sets.newHashSet("a"), timestamp);
164    MajorCompactionRequest spy = spy(request);
165    HRegionFileSystem fileSystem = mockFileSystem(regionInfo, references, storeFiles);
166    doReturn(fileSystem).when(spy).getFileSystem(isA(Connection.class));
167    doReturn(mock(Connection.class)).when(spy).getConnection(eq(configuration));
168    return spy;
169  }
170}