001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.mob; 019 020import static org.awaitility.Awaitility.await; 021import static org.junit.jupiter.api.Assertions.assertAll; 022import static org.junit.jupiter.api.Assertions.assertArrayEquals; 023import static org.junit.jupiter.api.Assertions.assertEquals; 024import static org.junit.jupiter.api.Assertions.assertFalse; 025import static org.junit.jupiter.api.Assertions.assertNotNull; 026import static org.junit.jupiter.api.Assertions.assertTrue; 027 028import java.io.IOException; 029import java.time.Duration; 030import java.util.ArrayList; 031import java.util.HashSet; 032import java.util.List; 033import java.util.Set; 034import java.util.concurrent.TimeUnit; 035import org.apache.hadoop.conf.Configuration; 036import org.apache.hadoop.fs.FileStatus; 037import org.apache.hadoop.fs.FileSystem; 038import org.apache.hadoop.fs.Path; 039import org.apache.hadoop.hbase.HBaseTestingUtil; 040import org.apache.hadoop.hbase.TableName; 041import org.apache.hadoop.hbase.client.Admin; 042import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; 043import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; 044import org.apache.hadoop.hbase.client.Put; 045import org.apache.hadoop.hbase.client.RegionInfo; 046import org.apache.hadoop.hbase.client.Result; 047import org.apache.hadoop.hbase.client.ResultScanner; 048import org.apache.hadoop.hbase.client.Table; 049import org.apache.hadoop.hbase.client.TableDescriptor; 050import org.apache.hadoop.hbase.client.TableDescriptorBuilder; 051import org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner; 052import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; 053import org.apache.hadoop.hbase.regionserver.HRegion; 054import org.apache.hadoop.hbase.regionserver.HStore; 055import org.apache.hadoop.hbase.regionserver.HStoreFile; 056import org.apache.hadoop.hbase.testclassification.MediumTests; 057import org.apache.hadoop.hbase.util.Bytes; 058import org.apache.hadoop.hbase.util.CommonFSUtils; 059import org.junit.jupiter.api.AfterAll; 060import org.junit.jupiter.api.BeforeAll; 061import org.junit.jupiter.api.Tag; 062import org.junit.jupiter.api.Test; 063import org.junit.jupiter.api.TestInfo; 064import org.slf4j.Logger; 065import org.slf4j.LoggerFactory; 066 067/** 068 * Verifies that the master side obsolete MOB file cleanup does not archive MOB files which are 069 * named after a split or merge parent region but are still referenced by the daughter regions. 070 * <p> 071 * A non user compaction does not rewrite MOB cells, it only carries the MOB references into the new 072 * store file. So after the parent region is garbage collected, the MOB files whose name suffix is 073 * the parent region are still live. 074 */ 075@Tag(MediumTests.TAG) 076public class TestMobFileCleanupUtilAfterSplitOrMerge { 077 078 private static final Logger LOG = 079 LoggerFactory.getLogger(TestMobFileCleanupUtilAfterSplitOrMerge.class); 080 081 private static final HBaseTestingUtil HTU = new HBaseTestingUtil(); 082 083 private static final byte[] FAMILY = Bytes.toBytes("f"); 084 private static final byte[] QUALIFIER = Bytes.toBytes("q"); 085 private static final long MOB_THRESHOLD = 10; 086 private static final int ROWS = 20; 087 private static final byte[] SPLIT_KEY = Bytes.toBytes(ROWS / 2); 088 089 private static Configuration conf; 090 private static Admin admin; 091 092 @BeforeAll 093 public static void setUpBeforeClass() throws Exception { 094 conf = HTU.getConfiguration(); 095 // Disable the automatic MOB compaction and MOB cleaner chores, the test runs them explicitly 096 conf.setLong(MobConstants.MOB_COMPACTION_CHORE_PERIOD, 0); 097 conf.setLong(MobConstants.MOB_CLEANER_PERIOD, 0); 098 // Every MOB file which exists before the cleanup is old enough to be archived 099 conf.setLong(MobConstants.MIN_AGE_TO_ARCHIVE_KEY, 0); 100 // Archived files are deleted as soon as the HFile cleaner runs 101 conf.setLong(TimeToLiveHFileCleaner.TTL_CONF_KEY, 0); 102 // Always open the MOB file from the file system when reading a MOB cell 103 conf.setInt(MobConstants.MOB_FILE_CACHE_SIZE_KEY, 0); 104 HTU.startMiniCluster(); 105 admin = HTU.getAdmin(); 106 } 107 108 @AfterAll 109 public static void tearDownAfterClass() throws Exception { 110 HTU.shutdownMiniCluster(); 111 } 112 113 @Test 114 public void testReferencedMobFilesOfSplitParentAreKept(TestInfo testInfo) throws Exception { 115 TableName tableName = TableName.valueOf(testInfo.getTestMethod().get().getName()); 116 try (Table table = createMobTable(tableName, null)) { 117 loadAndFlush(table); 118 List<HRegion> parents = HTU.getMiniHBaseCluster().getRegions(tableName); 119 assertEquals(1, parents.size()); 120 RegionInfo parent = parents.get(0).getRegionInfo(); 121 Set<String> parentMobFiles = listMobFiles(tableName); 122 assertMobFilesNamedAfter(parentMobFiles, parent); 123 124 admin.splitRegionAsync(parent.getRegionName(), SPLIT_KEY).get(2, TimeUnit.MINUTES); 125 await().atMost(Duration.ofMinutes(1)).untilAsserted( 126 () -> assertEquals(2, HTU.getMiniHBaseCluster().getRegions(tableName).size())); 127 128 verifyCleanupKeepsReferencedMobFiles(table, parentMobFiles, parent); 129 } 130 } 131 132 @Test 133 public void testReferencedMobFilesOfMergeParentsAreKept(TestInfo testInfo) throws Exception { 134 TableName tableName = TableName.valueOf(testInfo.getTestMethod().get().getName()); 135 try (Table table = createMobTable(tableName, new byte[][] { SPLIT_KEY })) { 136 loadAndFlush(table); 137 List<HRegion> parents = HTU.getMiniHBaseCluster().getRegions(tableName); 138 assertEquals(2, parents.size()); 139 RegionInfo parentA = parents.get(0).getRegionInfo(); 140 RegionInfo parentB = parents.get(1).getRegionInfo(); 141 Set<String> parentMobFiles = listMobFiles(tableName); 142 assertMobFilesNamedAfter(parentMobFiles, parentA, parentB); 143 144 admin 145 .mergeRegionsAsync( 146 new byte[][] { parentA.getEncodedNameAsBytes(), parentB.getEncodedNameAsBytes() }, false) 147 .get(2, TimeUnit.MINUTES); 148 await().atMost(Duration.ofMinutes(1)).untilAsserted( 149 () -> assertEquals(1, HTU.getMiniHBaseCluster().getRegions(tableName).size())); 150 151 verifyCleanupKeepsReferencedMobFiles(table, parentMobFiles, parentA, parentB); 152 } 153 } 154 155 private void verifyCleanupKeepsReferencedMobFiles(Table table, Set<String> parentMobFiles, 156 RegionInfo... parents) throws Exception { 157 TableName tableName = table.getName(); 158 List<HRegion> children = HTU.getMiniHBaseCluster().getRegions(tableName); 159 // Wait for the system compactions triggered on region open to replace the reference files 160 for (HRegion child : children) { 161 waitUntilNoReferences(child); 162 } 163 // The children did not rewrite the MOB cells, they still refer to the parent MOB files 164 assertEquals(parentMobFiles, getReferencedMobFiles(children)); 165 assertEquals(parentMobFiles, listMobFiles(tableName)); 166 167 // Garbage collect the parent regions 168 FileSystem fs = HTU.getTestFileSystem(); 169 Path tableDir = CommonFSUtils.getTableDir(CommonFSUtils.getRootDir(conf), tableName); 170 await().atMost(Duration.ofMinutes(1)).untilAsserted(() -> { 171 admin.runCatalogJanitor(); 172 ProcedureTestingUtility 173 .waitNoProcedureRunning(HTU.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor()); 174 for (RegionInfo parent : parents) { 175 Path parentDir = new Path(tableDir, parent.getEncodedName()); 176 assertFalse(fs.exists(parentDir), "Parent region dir not cleaned: " + parentDir); 177 } 178 }); 179 180 MobFileCleanupUtil.cleanupObsoleteMobFiles(conf, tableName, admin); 181 Set<String> mobFilesAfterCleanup = listMobFiles(tableName); 182 // Remove whatever has been archived, then read all the rows back 183 HTU.getMiniHBaseCluster().getMaster().getHFileCleaner().triggerCleanerNow().get(); 184 185 assertAll( 186 () -> assertEquals(parentMobFiles, mobFilesAfterCleanup, 187 "MOB files still referenced by the children were archived"), 188 () -> assertEquals(ROWS, verifyRows(table))); 189 } 190 191 private Table createMobTable(TableName tableName, byte[][] splitKeys) throws IOException { 192 ColumnFamilyDescriptor cfd = ColumnFamilyDescriptorBuilder.newBuilder(FAMILY) 193 .setMobEnabled(true).setMobThreshold(MOB_THRESHOLD).setMaxVersions(1).build(); 194 TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName).setColumnFamily(cfd).build(); 195 return HTU.createTable(htd, splitKeys); 196 } 197 198 private static byte[] value(int row) { 199 return Bytes.toBytes(String.format("mob-value-%050d", row)); 200 } 201 202 private void loadAndFlush(Table table) throws IOException { 203 List<Put> puts = new ArrayList<>(ROWS); 204 for (int i = 0; i < ROWS; i++) { 205 puts.add(new Put(Bytes.toBytes(i)).addColumn(FAMILY, QUALIFIER, value(i))); 206 } 207 table.put(puts); 208 admin.flush(table.getName()); 209 } 210 211 private int verifyRows(Table table) throws IOException { 212 int count = 0; 213 try (ResultScanner scanner = table.getScanner(FAMILY)) { 214 for (Result result; (result = scanner.next()) != null;) { 215 int row = Bytes.toInt(result.getRow()); 216 assertArrayEquals(value(row), result.getValue(FAMILY, QUALIFIER)); 217 count++; 218 } 219 } 220 return count; 221 } 222 223 private Set<String> listMobFiles(TableName tableName) throws IOException { 224 Path mobFamilyDir = MobUtils.getMobFamilyPath(conf, tableName, Bytes.toString(FAMILY)); 225 Set<String> names = new HashSet<>(); 226 for (FileStatus status : HTU.getTestFileSystem().listStatus(mobFamilyDir)) { 227 names.add(status.getPath().getName()); 228 } 229 LOG.info("MOB files of {}: {}", tableName, names); 230 return names; 231 } 232 233 private static void assertMobFilesNamedAfter(Set<String> mobFiles, RegionInfo... regions) { 234 Set<String> regionNames = new HashSet<>(); 235 for (RegionInfo region : regions) { 236 regionNames.add(region.getEncodedName()); 237 } 238 assertFalse(mobFiles.isEmpty()); 239 for (String mobFile : mobFiles) { 240 assertTrue(regionNames.contains(MobFileName.create(mobFile).getRegionName()), mobFile); 241 } 242 } 243 244 private static Set<String> getReferencedMobFiles(List<HRegion> regions) { 245 Set<String> referenced = new HashSet<>(); 246 for (HRegion region : regions) { 247 for (HStoreFile sf : region.getStore(FAMILY).getStorefiles()) { 248 byte[] refs = sf.getMetadataValue(HStoreFile.MOB_FILE_REFS); 249 assertNotNull(refs, "No MOB_FILE_REFS in " + sf); 250 referenced.addAll(MobUtils.deserializeMobFileRefs(refs).build().values()); 251 } 252 } 253 LOG.info("MOB files referenced by {}: {}", regions, referenced); 254 return referenced; 255 } 256 257 private void waitUntilNoReferences(HRegion region) { 258 HStore store = region.getStore(FAMILY); 259 await().atMost(Duration.ofMinutes(1)).until(() -> { 260 if (store.getStoreEngine().getCompactor().isCompacting()) { 261 return false; 262 } 263 store.closeAndArchiveCompactedFiles(); 264 if (!store.hasReferences()) { 265 return true; 266 } 267 // The compaction requested on region open may have been skipped, request it again. This is 268 // still a system compaction, which does not rewrite the MOB cells. 269 region.compact(true); 270 return false; 271 }); 272 } 273}