001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.mob;
019
020import static org.awaitility.Awaitility.await;
021import static org.junit.jupiter.api.Assertions.assertAll;
022import static org.junit.jupiter.api.Assertions.assertArrayEquals;
023import static org.junit.jupiter.api.Assertions.assertEquals;
024import static org.junit.jupiter.api.Assertions.assertFalse;
025import static org.junit.jupiter.api.Assertions.assertNotNull;
026import static org.junit.jupiter.api.Assertions.assertTrue;
027
028import java.io.IOException;
029import java.time.Duration;
030import java.util.ArrayList;
031import java.util.HashSet;
032import java.util.List;
033import java.util.Set;
034import java.util.concurrent.TimeUnit;
035import org.apache.hadoop.conf.Configuration;
036import org.apache.hadoop.fs.FileStatus;
037import org.apache.hadoop.fs.FileSystem;
038import org.apache.hadoop.fs.Path;
039import org.apache.hadoop.hbase.HBaseTestingUtil;
040import org.apache.hadoop.hbase.TableName;
041import org.apache.hadoop.hbase.client.Admin;
042import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
043import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
044import org.apache.hadoop.hbase.client.Put;
045import org.apache.hadoop.hbase.client.RegionInfo;
046import org.apache.hadoop.hbase.client.Result;
047import org.apache.hadoop.hbase.client.ResultScanner;
048import org.apache.hadoop.hbase.client.Table;
049import org.apache.hadoop.hbase.client.TableDescriptor;
050import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
051import org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner;
052import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
053import org.apache.hadoop.hbase.regionserver.HRegion;
054import org.apache.hadoop.hbase.regionserver.HStore;
055import org.apache.hadoop.hbase.regionserver.HStoreFile;
056import org.apache.hadoop.hbase.testclassification.MediumTests;
057import org.apache.hadoop.hbase.util.Bytes;
058import org.apache.hadoop.hbase.util.CommonFSUtils;
059import org.junit.jupiter.api.AfterAll;
060import org.junit.jupiter.api.BeforeAll;
061import org.junit.jupiter.api.Tag;
062import org.junit.jupiter.api.Test;
063import org.junit.jupiter.api.TestInfo;
064import org.slf4j.Logger;
065import org.slf4j.LoggerFactory;
066
067/**
068 * Verifies that the master side obsolete MOB file cleanup does not archive MOB files which are
069 * named after a split or merge parent region but are still referenced by the daughter regions.
070 * <p>
071 * A non user compaction does not rewrite MOB cells, it only carries the MOB references into the new
072 * store file. So after the parent region is garbage collected, the MOB files whose name suffix is
073 * the parent region are still live.
074 */
075@Tag(MediumTests.TAG)
076public class TestMobFileCleanupUtilAfterSplitOrMerge {
077
078  private static final Logger LOG =
079    LoggerFactory.getLogger(TestMobFileCleanupUtilAfterSplitOrMerge.class);
080
081  private static final HBaseTestingUtil HTU = new HBaseTestingUtil();
082
083  private static final byte[] FAMILY = Bytes.toBytes("f");
084  private static final byte[] QUALIFIER = Bytes.toBytes("q");
085  private static final long MOB_THRESHOLD = 10;
086  private static final int ROWS = 20;
087  private static final byte[] SPLIT_KEY = Bytes.toBytes(ROWS / 2);
088
089  private static Configuration conf;
090  private static Admin admin;
091
092  @BeforeAll
093  public static void setUpBeforeClass() throws Exception {
094    conf = HTU.getConfiguration();
095    // Disable the automatic MOB compaction and MOB cleaner chores, the test runs them explicitly
096    conf.setLong(MobConstants.MOB_COMPACTION_CHORE_PERIOD, 0);
097    conf.setLong(MobConstants.MOB_CLEANER_PERIOD, 0);
098    // Every MOB file which exists before the cleanup is old enough to be archived
099    conf.setLong(MobConstants.MIN_AGE_TO_ARCHIVE_KEY, 0);
100    // Archived files are deleted as soon as the HFile cleaner runs
101    conf.setLong(TimeToLiveHFileCleaner.TTL_CONF_KEY, 0);
102    // Always open the MOB file from the file system when reading a MOB cell
103    conf.setInt(MobConstants.MOB_FILE_CACHE_SIZE_KEY, 0);
104    HTU.startMiniCluster();
105    admin = HTU.getAdmin();
106  }
107
108  @AfterAll
109  public static void tearDownAfterClass() throws Exception {
110    HTU.shutdownMiniCluster();
111  }
112
113  @Test
114  public void testReferencedMobFilesOfSplitParentAreKept(TestInfo testInfo) throws Exception {
115    TableName tableName = TableName.valueOf(testInfo.getTestMethod().get().getName());
116    try (Table table = createMobTable(tableName, null)) {
117      loadAndFlush(table);
118      List<HRegion> parents = HTU.getMiniHBaseCluster().getRegions(tableName);
119      assertEquals(1, parents.size());
120      RegionInfo parent = parents.get(0).getRegionInfo();
121      Set<String> parentMobFiles = listMobFiles(tableName);
122      assertMobFilesNamedAfter(parentMobFiles, parent);
123
124      admin.splitRegionAsync(parent.getRegionName(), SPLIT_KEY).get(2, TimeUnit.MINUTES);
125      await().atMost(Duration.ofMinutes(1)).untilAsserted(
126        () -> assertEquals(2, HTU.getMiniHBaseCluster().getRegions(tableName).size()));
127
128      verifyCleanupKeepsReferencedMobFiles(table, parentMobFiles, parent);
129    }
130  }
131
132  @Test
133  public void testReferencedMobFilesOfMergeParentsAreKept(TestInfo testInfo) throws Exception {
134    TableName tableName = TableName.valueOf(testInfo.getTestMethod().get().getName());
135    try (Table table = createMobTable(tableName, new byte[][] { SPLIT_KEY })) {
136      loadAndFlush(table);
137      List<HRegion> parents = HTU.getMiniHBaseCluster().getRegions(tableName);
138      assertEquals(2, parents.size());
139      RegionInfo parentA = parents.get(0).getRegionInfo();
140      RegionInfo parentB = parents.get(1).getRegionInfo();
141      Set<String> parentMobFiles = listMobFiles(tableName);
142      assertMobFilesNamedAfter(parentMobFiles, parentA, parentB);
143
144      admin
145        .mergeRegionsAsync(
146          new byte[][] { parentA.getEncodedNameAsBytes(), parentB.getEncodedNameAsBytes() }, false)
147        .get(2, TimeUnit.MINUTES);
148      await().atMost(Duration.ofMinutes(1)).untilAsserted(
149        () -> assertEquals(1, HTU.getMiniHBaseCluster().getRegions(tableName).size()));
150
151      verifyCleanupKeepsReferencedMobFiles(table, parentMobFiles, parentA, parentB);
152    }
153  }
154
155  private void verifyCleanupKeepsReferencedMobFiles(Table table, Set<String> parentMobFiles,
156    RegionInfo... parents) throws Exception {
157    TableName tableName = table.getName();
158    List<HRegion> children = HTU.getMiniHBaseCluster().getRegions(tableName);
159    // Wait for the system compactions triggered on region open to replace the reference files
160    for (HRegion child : children) {
161      waitUntilNoReferences(child);
162    }
163    // The children did not rewrite the MOB cells, they still refer to the parent MOB files
164    assertEquals(parentMobFiles, getReferencedMobFiles(children));
165    assertEquals(parentMobFiles, listMobFiles(tableName));
166
167    // Garbage collect the parent regions
168    FileSystem fs = HTU.getTestFileSystem();
169    Path tableDir = CommonFSUtils.getTableDir(CommonFSUtils.getRootDir(conf), tableName);
170    await().atMost(Duration.ofMinutes(1)).untilAsserted(() -> {
171      admin.runCatalogJanitor();
172      ProcedureTestingUtility
173        .waitNoProcedureRunning(HTU.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor());
174      for (RegionInfo parent : parents) {
175        Path parentDir = new Path(tableDir, parent.getEncodedName());
176        assertFalse(fs.exists(parentDir), "Parent region dir not cleaned: " + parentDir);
177      }
178    });
179
180    MobFileCleanupUtil.cleanupObsoleteMobFiles(conf, tableName, admin);
181    Set<String> mobFilesAfterCleanup = listMobFiles(tableName);
182    // Remove whatever has been archived, then read all the rows back
183    HTU.getMiniHBaseCluster().getMaster().getHFileCleaner().triggerCleanerNow().get();
184
185    assertAll(
186      () -> assertEquals(parentMobFiles, mobFilesAfterCleanup,
187        "MOB files still referenced by the children were archived"),
188      () -> assertEquals(ROWS, verifyRows(table)));
189  }
190
191  private Table createMobTable(TableName tableName, byte[][] splitKeys) throws IOException {
192    ColumnFamilyDescriptor cfd = ColumnFamilyDescriptorBuilder.newBuilder(FAMILY)
193      .setMobEnabled(true).setMobThreshold(MOB_THRESHOLD).setMaxVersions(1).build();
194    TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName).setColumnFamily(cfd).build();
195    return HTU.createTable(htd, splitKeys);
196  }
197
198  private static byte[] value(int row) {
199    return Bytes.toBytes(String.format("mob-value-%050d", row));
200  }
201
202  private void loadAndFlush(Table table) throws IOException {
203    List<Put> puts = new ArrayList<>(ROWS);
204    for (int i = 0; i < ROWS; i++) {
205      puts.add(new Put(Bytes.toBytes(i)).addColumn(FAMILY, QUALIFIER, value(i)));
206    }
207    table.put(puts);
208    admin.flush(table.getName());
209  }
210
211  private int verifyRows(Table table) throws IOException {
212    int count = 0;
213    try (ResultScanner scanner = table.getScanner(FAMILY)) {
214      for (Result result; (result = scanner.next()) != null;) {
215        int row = Bytes.toInt(result.getRow());
216        assertArrayEquals(value(row), result.getValue(FAMILY, QUALIFIER));
217        count++;
218      }
219    }
220    return count;
221  }
222
223  private Set<String> listMobFiles(TableName tableName) throws IOException {
224    Path mobFamilyDir = MobUtils.getMobFamilyPath(conf, tableName, Bytes.toString(FAMILY));
225    Set<String> names = new HashSet<>();
226    for (FileStatus status : HTU.getTestFileSystem().listStatus(mobFamilyDir)) {
227      names.add(status.getPath().getName());
228    }
229    LOG.info("MOB files of {}: {}", tableName, names);
230    return names;
231  }
232
233  private static void assertMobFilesNamedAfter(Set<String> mobFiles, RegionInfo... regions) {
234    Set<String> regionNames = new HashSet<>();
235    for (RegionInfo region : regions) {
236      regionNames.add(region.getEncodedName());
237    }
238    assertFalse(mobFiles.isEmpty());
239    for (String mobFile : mobFiles) {
240      assertTrue(regionNames.contains(MobFileName.create(mobFile).getRegionName()), mobFile);
241    }
242  }
243
244  private static Set<String> getReferencedMobFiles(List<HRegion> regions) {
245    Set<String> referenced = new HashSet<>();
246    for (HRegion region : regions) {
247      for (HStoreFile sf : region.getStore(FAMILY).getStorefiles()) {
248        byte[] refs = sf.getMetadataValue(HStoreFile.MOB_FILE_REFS);
249        assertNotNull(refs, "No MOB_FILE_REFS in " + sf);
250        referenced.addAll(MobUtils.deserializeMobFileRefs(refs).build().values());
251      }
252    }
253    LOG.info("MOB files referenced by {}: {}", regions, referenced);
254    return referenced;
255  }
256
257  private void waitUntilNoReferences(HRegion region) {
258    HStore store = region.getStore(FAMILY);
259    await().atMost(Duration.ofMinutes(1)).until(() -> {
260      if (store.getStoreEngine().getCompactor().isCompacting()) {
261        return false;
262      }
263      store.closeAndArchiveCompactedFiles();
264      if (!store.hasReferences()) {
265        return true;
266      }
267      // The compaction requested on region open may have been skipped, request it again. This is
268      // still a system compaction, which does not rewrite the MOB cells.
269      region.compact(true);
270      return false;
271    });
272  }
273}