001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.backup;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertTrue;
022
023import java.io.IOException;
024import java.util.List;
025import java.util.Map;
026import java.util.Set;
027import org.apache.hadoop.conf.Configuration;
028import org.apache.hadoop.fs.FileStatus;
029import org.apache.hadoop.fs.FileSystem;
030import org.apache.hadoop.fs.Path;
031import org.apache.hadoop.hbase.HBaseTestingUtil;
032import org.apache.hadoop.hbase.TableName;
033import org.apache.hadoop.hbase.backup.impl.BackupSystemTable;
034import org.apache.hadoop.hbase.testclassification.MasterTests;
035import org.apache.hadoop.hbase.testclassification.MediumTests;
036import org.junit.jupiter.api.AfterAll;
037import org.junit.jupiter.api.AfterEach;
038import org.junit.jupiter.api.BeforeAll;
039import org.junit.jupiter.api.BeforeEach;
040import org.junit.jupiter.api.Tag;
041import org.junit.jupiter.api.Test;
042import org.slf4j.Logger;
043import org.slf4j.LoggerFactory;
044
045import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
046
047@Tag(MasterTests.TAG)
048@Tag(MediumTests.TAG)
049public class TestBackupHFileCleaner {
050
051  private static final Logger LOG = LoggerFactory.getLogger(TestBackupHFileCleaner.class);
052  private final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
053  private final static Configuration conf = TEST_UTIL.getConfiguration();
054  private final static TableName tableNameWithBackup = TableName.valueOf("backup.hfile.cleaner");
055  private final static TableName tableNameWithoutBackup =
056    TableName.valueOf("backup.hfile.cleaner2");
057
058  private static FileSystem fs = null;
059
060  private Path root;
061
062  @BeforeAll
063  public static void setUpBeforeClass() throws Exception {
064    conf.setBoolean(BackupRestoreConstants.BACKUP_ENABLE_KEY, true);
065    TEST_UTIL.startMiniCluster(1);
066    fs = FileSystem.get(conf);
067  }
068
069  @AfterAll
070  public static void tearDownAfterClass() throws Exception {
071    TEST_UTIL.shutdownMiniCluster();
072  }
073
074  @BeforeEach
075  public void setup() throws IOException {
076    root = TEST_UTIL.getDataTestDirOnTestFS();
077  }
078
079  @AfterEach
080  public void cleanup() {
081    try {
082      fs.delete(root, true);
083    } catch (IOException e) {
084      LOG.warn("Failed to delete files recursively from path " + root);
085    }
086  }
087
088  @Test
089  public void testGetDeletableFiles() throws IOException {
090    FileStatus file1 = createFile("file1");
091    FileStatus file1Archived = createFile("archived/file1");
092    FileStatus file2 = createFile("file2");
093    FileStatus file3 = createFile("file3");
094
095    BackupHFileCleaner cleaner = new BackupHFileCleaner() {
096      @Override
097      protected Set<TableName> fetchFullyBackedUpTables(BackupSystemTable tbl) {
098        return Set.of(tableNameWithBackup);
099      }
100    };
101    cleaner.setConf(conf);
102
103    Iterable<FileStatus> deletable;
104
105    // The first call will not allow any deletions because of the timestamp mechanism.
106    deletable = callCleaner(cleaner, List.of(file1, file1Archived, file2, file3));
107    assertEquals(Set.of(), Sets.newHashSet(deletable));
108
109    // No bulk loads registered, so all files can be deleted.
110    deletable = callCleaner(cleaner, List.of(file1, file1Archived, file2, file3));
111    assertEquals(Set.of(file1, file1Archived, file2, file3), Sets.newHashSet(deletable));
112
113    // Register some bulk loads.
114    try (BackupSystemTable backupSystem = new BackupSystemTable(TEST_UTIL.getConnection())) {
115      byte[] unused = new byte[] { 0 };
116      backupSystem.registerBulkLoad(tableNameWithBackup, unused,
117        Map.of(unused, List.of(file1.getPath())));
118      backupSystem.registerBulkLoad(tableNameWithoutBackup, unused,
119        Map.of(unused, List.of(file2.getPath())));
120    }
121
122    // File 1 can no longer be deleted, because it is registered as a bulk load.
123    deletable = callCleaner(cleaner, List.of(file1, file1Archived, file2, file3));
124    assertEquals(Set.of(file2, file3), Sets.newHashSet(deletable));
125  }
126
127  private Iterable<FileStatus> callCleaner(BackupHFileCleaner cleaner, Iterable<FileStatus> files) {
128    cleaner.preClean();
129    Iterable<FileStatus> deletable = cleaner.getDeletableFiles(files);
130    cleaner.postClean();
131    return deletable;
132  }
133
134  private FileStatus createFile(String fileName) throws IOException {
135    Path file = new Path(root, fileName);
136    fs.createNewFile(file);
137    assertTrue(fs.exists(file), "Test file not created!");
138    return fs.getFileStatus(file);
139  }
140}