001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.cleaner;
019
020import static org.apache.hadoop.hbase.master.HMaster.HBASE_MASTER_CLEANER_INTERVAL;
021import static org.apache.hadoop.hbase.master.cleaner.HFileCleaner.HFILE_CLEANER_CUSTOM_PATHS_PLUGINS;
022
023import java.io.FileNotFoundException;
024import java.io.IOException;
025import org.apache.hadoop.conf.Configuration;
026import org.apache.hadoop.fs.Path;
027import org.apache.hadoop.hbase.HBaseTestingUtil;
028import org.apache.hadoop.hbase.TableName;
029import org.apache.hadoop.hbase.client.Admin;
030import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
031import org.apache.hadoop.hbase.client.Table;
032import org.apache.hadoop.hbase.client.TableDescriptor;
033import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
034import org.apache.hadoop.hbase.testclassification.LargeTests;
035import org.apache.hadoop.hbase.util.Bytes;
036import org.apache.hadoop.hbase.util.HFileArchiveUtil;
037import org.apache.hadoop.hdfs.DistributedFileSystem;
038import org.junit.jupiter.api.AfterAll;
039import org.junit.jupiter.api.BeforeAll;
040import org.junit.jupiter.api.Tag;
041import org.junit.jupiter.api.Test;
042
043@Tag(LargeTests.TAG)
044public class TestCleanerClearHFiles {
045
046  private static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
047  private static Configuration conf = TEST_UTIL.getConfiguration();
048  private static Admin admin = null;
049
050  private static final byte[] COLUMN_FAMILY = Bytes.toBytes("CF");
051
052  private static final String TABLE1 = "table1";
053  private static final String TABLE2 = "table2";
054  private static final String DEFAULT_ARCHIVE_SUBDIRS_PREFIX = "data/default/";
055
056  @BeforeAll
057  public static void setupBeforeClass() throws Exception {
058    conf.setStrings(HFileCleaner.HFILE_CLEANER_CUSTOM_PATHS,
059      DEFAULT_ARCHIVE_SUBDIRS_PREFIX + TABLE1);
060    conf.setStrings(HFILE_CLEANER_CUSTOM_PATHS_PLUGINS, HFileLinkCleaner.class.getName());
061
062    conf.setInt(TimeToLiveHFileCleaner.TTL_CONF_KEY, 10);
063    conf.setInt(HBASE_MASTER_CLEANER_INTERVAL, 20000);
064
065    TEST_UTIL.startMiniCluster();
066    admin = TEST_UTIL.getAdmin();
067  }
068
069  @AfterAll
070  public static void tearDownAfterClass() throws Exception {
071    TEST_UTIL.shutdownMiniCluster();
072  }
073
074  @Test
075  public void testClearArchive() throws Exception {
076    DistributedFileSystem fs = TEST_UTIL.getDFSCluster().getFileSystem();
077    Table table1 = createTable(TEST_UTIL, TableName.valueOf(TABLE1));
078    Table table2 = createTable(TEST_UTIL, TableName.valueOf(TABLE2));
079
080    admin.disableTable(table1.getName());
081    admin.deleteTable(table1.getName());
082    admin.disableTable(table2.getName());
083    admin.deleteTable(table2.getName());
084
085    Path archiveDir = HFileArchiveUtil.getArchivePath(conf);
086    Path archiveTable1Path = new Path(archiveDir, DEFAULT_ARCHIVE_SUBDIRS_PREFIX + TABLE1);
087    Path archiveTable2Path = new Path(archiveDir, DEFAULT_ARCHIVE_SUBDIRS_PREFIX + TABLE2);
088
089    TEST_UTIL.waitFor(10000, () -> !notExistOrEmptyDir(archiveTable1Path, fs)
090      && !notExistOrEmptyDir(archiveTable2Path, fs));
091
092    TEST_UTIL.waitFor(30000,
093      () -> notExistOrEmptyDir(archiveTable1Path, fs) && notExistOrEmptyDir(archiveTable2Path, fs));
094  }
095
096  private boolean notExistOrEmptyDir(Path dir, DistributedFileSystem fs) {
097    try {
098      return fs.listStatus(dir).length == 0;
099    } catch (Exception e) {
100      return e instanceof FileNotFoundException;
101    }
102  }
103
104  private Table createTable(HBaseTestingUtil util, TableName tableName) throws IOException {
105    TableDescriptor td = TableDescriptorBuilder.newBuilder(tableName)
106      .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(COLUMN_FAMILY).build()).build();
107    return util.createTable(td, null);
108  }
109}