001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.mob;
019
020import static org.junit.jupiter.api.Assertions.assertTrue;
021import static org.mockito.Mockito.mock;
022import static org.mockito.Mockito.when;
023
024import java.util.Date;
025import org.apache.hadoop.conf.Configuration;
026import org.apache.hadoop.fs.FileSystem;
027import org.apache.hadoop.fs.Path;
028import org.apache.hadoop.hbase.ArrayBackedTag;
029import org.apache.hadoop.hbase.ExtendedCell;
030import org.apache.hadoop.hbase.HBaseTestingUtil;
031import org.apache.hadoop.hbase.KeyValue;
032import org.apache.hadoop.hbase.TableName;
033import org.apache.hadoop.hbase.TagType;
034import org.apache.hadoop.hbase.client.Admin;
035import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
036import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
037import org.apache.hadoop.hbase.client.RegionInfo;
038import org.apache.hadoop.hbase.client.RegionInfoBuilder;
039import org.apache.hadoop.hbase.client.TableDescriptor;
040import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
041import org.apache.hadoop.hbase.io.compress.Compression;
042import org.apache.hadoop.hbase.io.crypto.Encryption;
043import org.apache.hadoop.hbase.io.hfile.CacheConfig;
044import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
045import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
046import org.apache.hadoop.hbase.regionserver.StoreFileWriter;
047import org.apache.hadoop.hbase.testclassification.SmallTests;
048import org.apache.hadoop.hbase.util.Bytes;
049import org.apache.hadoop.hbase.util.CommonFSUtils;
050import org.junit.jupiter.api.Tag;
051import org.junit.jupiter.api.Test;
052
053@Tag(SmallTests.TAG)
054public class TestMobFileCleanupUtilWithOldStoreFile {
055
056  private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
057  private static final byte[] FAMILY = Bytes.toBytes("f");
058
059  @Test
060  public void testCleanupWithOldStoreFile() throws Exception {
061    Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
062    Path rootDir = TEST_UTIL.getDataTestDir("old-store-file");
063    FileSystem fs = rootDir.getFileSystem(conf);
064    fs.delete(rootDir, true);
065    CommonFSUtils.setRootDir(conf, rootDir);
066    conf.setLong(MobConstants.MIN_AGE_TO_ARCHIVE_KEY, 0);
067
068    TableName tableName = TableName.valueOf("oldStoreFile");
069    ColumnFamilyDescriptor familyDescriptor =
070      ColumnFamilyDescriptorBuilder.newBuilder(FAMILY).setMobEnabled(true).build();
071    TableDescriptor tableDescriptor =
072      TableDescriptorBuilder.newBuilder(tableName).setColumnFamily(familyDescriptor).build();
073    RegionInfo regionInfo = RegionInfoBuilder.newBuilder(tableName).build();
074    Path tableDir = CommonFSUtils.getTableDir(rootDir, tableName);
075    HRegionFileSystem regionFs =
076      HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, regionInfo);
077
078    String nonexistentRegionName = "deadbeef";
079    Path mobFamilyDir =
080      MobUtils.getMobFamilyPath(conf, tableName, familyDescriptor.getNameAsString());
081    StoreFileWriter mobWriter = MobUtils.createWriter(conf, fs, familyDescriptor,
082      MobUtils.formatDate(new Date()), mobFamilyDir, 1, Compression.Algorithm.NONE, "start",
083      CacheConfig.DISABLED, Encryption.Context.NONE, false, nonexistentRegionName);
084    mobWriter.append(
085      new KeyValue(Bytes.toBytes("row"), FAMILY, Bytes.toBytes("q"), Bytes.toBytes("mob-value")));
086    mobWriter.close();
087
088    // Old MOB file names do not contain the region suffix used by the current cleanup logic.
089    Path mobFileWithRegionSuffix = mobWriter.getPath();
090    Path oldMobFile = new Path(mobFileWithRegionSuffix.getParent(),
091      mobFileWithRegionSuffix.getName().split("_")[0]);
092    assertTrue(fs.rename(mobFileWithRegionSuffix, oldMobFile));
093    fs.setTimes(oldMobFile, 1, -1);
094
095    // This represents a store file written before MOB_FILE_REFS metadata was added.
096    StoreFileWriter oldStoreFileWriter = new StoreFileWriter.Builder(conf, CacheConfig.DISABLED, fs)
097      .withOutputDir(regionFs.getStoreDir(familyDescriptor.getNameAsString()))
098      .withFileContext(new HFileContextBuilder().withIncludesTags(true).build()).build();
099    ExtendedCell originalCell =
100      new KeyValue(Bytes.toBytes("row"), FAMILY, Bytes.toBytes("q"), Bytes.toBytes("mob-value"));
101    oldStoreFileWriter
102      .append(MobUtils.createMobRefCell(originalCell, Bytes.toBytes(oldMobFile.getName()),
103        new ArrayBackedTag(TagType.MOB_TABLE_NAME_TAG_TYPE, tableName.getName())));
104    oldStoreFileWriter.appendMetadata(1, false);
105    oldStoreFileWriter.close();
106
107    Admin admin = mock(Admin.class);
108    when(admin.getDescriptor(tableName)).thenReturn(tableDescriptor);
109
110    MobFileCleanupUtil.cleanupObsoleteMobFiles(conf, tableName, admin);
111
112    assertTrue(fs.exists(oldMobFile), "MOB file referenced by an old store file was archived");
113  }
114}