001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.util.compaction; 019 020import java.io.IOException; 021import java.util.List; 022import java.util.Optional; 023import java.util.Set; 024import java.util.stream.Collectors; 025import org.apache.commons.lang3.RandomStringUtils; 026import org.apache.hadoop.conf.Configuration; 027import org.apache.hadoop.fs.FileStatus; 028import org.apache.hadoop.fs.FileSystem; 029import org.apache.hadoop.fs.Path; 030import org.apache.hadoop.hbase.HBaseClassTestRule; 031import org.apache.hadoop.hbase.HBaseTestingUtility; 032import org.apache.hadoop.hbase.TableName; 033import org.apache.hadoop.hbase.client.Connection; 034import org.apache.hadoop.hbase.client.RegionInfo; 035import org.apache.hadoop.hbase.client.RegionInfoBuilder; 036import org.apache.hadoop.hbase.client.TableDescriptor; 037import org.apache.hadoop.hbase.regionserver.HRegion; 038import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; 039import org.apache.hadoop.hbase.regionserver.StoreFileInfo; 040import org.apache.hadoop.hbase.testclassification.SmallTests; 041import org.apache.hadoop.hbase.util.Bytes; 042import org.apache.hbase.thirdparty.com.google.common.collect.Iterables; 043import org.apache.hbase.thirdparty.com.google.common.collect.Lists; 044import org.apache.hbase.thirdparty.com.google.common.collect.Sets; 045import static org.junit.Assert.assertEquals; 046import static org.junit.Assert.assertFalse; 047import static org.junit.Assert.assertTrue; 048import org.junit.Before; 049import org.junit.ClassRule; 050import org.junit.Test; 051import org.junit.experimental.categories.Category; 052import static org.mockito.ArgumentMatchers.any; 053import static org.mockito.ArgumentMatchers.eq; 054import static org.mockito.ArgumentMatchers.isA; 055import static org.mockito.Matchers.anyString; 056import static org.mockito.Mockito.doReturn; 057import static org.mockito.Mockito.mock; 058import static org.mockito.Mockito.spy; 059import static org.mockito.Mockito.when; 060 061@Category({SmallTests.class}) 062public class TestMajorCompactionRequest { 063 @ClassRule 064 public static final HBaseClassTestRule CLASS_RULE = 065 HBaseClassTestRule.forClass(TestMajorCompactionRequest.class); 066 067 private static final HBaseTestingUtility UTILITY = new HBaseTestingUtility(); 068 private static final String FAMILY = "a"; 069 private Path rootRegionDir; 070 private Path regionStoreDir; 071 072 @Before public void setUp() throws Exception { 073 rootRegionDir = UTILITY.getDataTestDirOnTestFS("TestMajorCompactionRequest"); 074 regionStoreDir = new Path(rootRegionDir, FAMILY); 075 } 076 077 @Test public void testStoresNeedingCompaction() throws Exception { 078 // store files older than timestamp 079 List<StoreFileInfo> storeFiles = mockStoreFiles(regionStoreDir, 5, 10); 080 MajorCompactionRequest request = makeMockRequest(100, storeFiles, false); 081 Optional<MajorCompactionRequest> result = 082 request.createRequest(mock(Configuration.class), Sets.newHashSet(FAMILY)); 083 assertTrue(result.isPresent()); 084 085 // store files newer than timestamp 086 storeFiles = mockStoreFiles(regionStoreDir, 5, 101); 087 request = makeMockRequest(100, storeFiles, false); 088 result = request.createRequest(mock(Configuration.class), Sets.newHashSet(FAMILY)); 089 assertFalse(result.isPresent()); 090 } 091 092 @Test public void testIfWeHaveNewReferenceFilesButOldStoreFiles() throws Exception { 093 // this tests that reference files that are new, but have older timestamps for the files 094 // they reference still will get compacted. 095 TableName table = TableName.valueOf("TestMajorCompactor"); 096 TableDescriptor htd = UTILITY.createTableDescriptor(table, Bytes.toBytes(FAMILY)); 097 RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build(); 098 HRegion region = 099 HBaseTestingUtility.createRegionAndWAL(hri, rootRegionDir, UTILITY.getConfiguration(), htd); 100 101 Configuration configuration = mock(Configuration.class); 102 // the reference file timestamp is newer 103 List<StoreFileInfo> storeFiles = mockStoreFiles(regionStoreDir, 4, 101); 104 List<Path> paths = storeFiles.stream().map(StoreFileInfo::getPath).collect(Collectors.toList()); 105 // the files that are referenced are older, thus we still compact. 106 HRegionFileSystem fileSystem = 107 mockFileSystem(region.getRegionInfo(), true, storeFiles, 50); 108 MajorCompactionRequest majorCompactionRequest = spy(new MajorCompactionRequest(configuration, 109 region.getRegionInfo(), Sets.newHashSet(FAMILY), 100)); 110 doReturn(mock(Connection.class)).when(majorCompactionRequest).getConnection(eq(configuration)); 111 doReturn(paths).when(majorCompactionRequest).getReferenceFilePaths(any(FileSystem.class), 112 any(Path.class)); 113 doReturn(fileSystem).when(majorCompactionRequest).getFileSystem(any(Connection.class)); 114 Set<String> result = majorCompactionRequest.getStoresRequiringCompaction(Sets.newHashSet("a")); 115 assertEquals(FAMILY, Iterables.getOnlyElement(result)); 116 } 117 118 private HRegionFileSystem mockFileSystem(RegionInfo info, boolean hasReferenceFiles, 119 List<StoreFileInfo> storeFiles) throws IOException { 120 long timestamp = storeFiles.stream().findFirst().get().getModificationTime(); 121 return mockFileSystem(info, hasReferenceFiles, storeFiles, timestamp); 122 } 123 124 private HRegionFileSystem mockFileSystem(RegionInfo info, boolean hasReferenceFiles, 125 List<StoreFileInfo> storeFiles, long referenceFileTimestamp) throws IOException { 126 FileSystem fileSystem = mock(FileSystem.class); 127 if (hasReferenceFiles) { 128 FileStatus fileStatus = mock(FileStatus.class); 129 doReturn(referenceFileTimestamp).when(fileStatus).getModificationTime(); 130 doReturn(fileStatus).when(fileSystem).getFileLinkStatus(isA(Path.class)); 131 } 132 HRegionFileSystem mockSystem = mock(HRegionFileSystem.class); 133 doReturn(info).when(mockSystem).getRegionInfo(); 134 doReturn(regionStoreDir).when(mockSystem).getStoreDir(FAMILY); 135 doReturn(hasReferenceFiles).when(mockSystem).hasReferences(anyString()); 136 doReturn(storeFiles).when(mockSystem).getStoreFiles(anyString()); 137 doReturn(fileSystem).when(mockSystem).getFileSystem(); 138 return mockSystem; 139 } 140 141 private List<StoreFileInfo> mockStoreFiles(Path regionStoreDir, int howMany, long timestamp) 142 throws IOException { 143 List<StoreFileInfo> infos = Lists.newArrayList(); 144 int i = 0; 145 while (i < howMany) { 146 StoreFileInfo storeFileInfo = mock(StoreFileInfo.class); 147 doReturn(timestamp).doReturn(timestamp).when(storeFileInfo).getModificationTime(); 148 doReturn(new Path(regionStoreDir, RandomStringUtils.randomAlphabetic(10))).when(storeFileInfo) 149 .getPath(); 150 infos.add(storeFileInfo); 151 i++; 152 } 153 return infos; 154 } 155 156 private MajorCompactionRequest makeMockRequest(long timestamp, List<StoreFileInfo> storeFiles, 157 boolean references) throws IOException { 158 Configuration configuration = mock(Configuration.class); 159 RegionInfo regionInfo = mock(RegionInfo.class); 160 when(regionInfo.getEncodedName()).thenReturn("HBase"); 161 when(regionInfo.getTable()).thenReturn(TableName.valueOf("foo")); 162 MajorCompactionRequest request = 163 new MajorCompactionRequest(configuration, regionInfo, Sets.newHashSet("a"), timestamp); 164 MajorCompactionRequest spy = spy(request); 165 HRegionFileSystem fileSystem = mockFileSystem(regionInfo, references, storeFiles); 166 doReturn(fileSystem).when(spy).getFileSystem(isA(Connection.class)); 167 doReturn(mock(Connection.class)).when(spy).getConnection(eq(configuration)); 168 return spy; 169 } 170}