Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.backup;
019
020import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.CONF_CONTINUOUS_BACKUP_WAL_DIR;
021import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.CONTINUOUS_BACKUP_REPLICATION_PEER;
022import static org.apache.hadoop.hbase.backup.replication.ContinuousBackupReplicationEndpoint.ONE_DAY_IN_MILLISECONDS;
023import static org.apache.hadoop.hbase.backup.util.BackupFileSystemManager.BULKLOAD_FILES_DIR;
024import static org.apache.hadoop.hbase.backup.util.BackupFileSystemManager.WALS_DIR;
025import static org.apache.hadoop.hbase.backup.util.BackupUtils.DATE_FORMAT;
026import static org.junit.jupiter.api.Assertions.assertEquals;
027import static org.junit.jupiter.api.Assertions.assertFalse;
028import static org.junit.jupiter.api.Assertions.assertTrue;
029
030import java.io.IOException;
031import java.text.SimpleDateFormat;
032import java.util.ArrayList;
033import java.util.Date;
034import java.util.List;
035import java.util.Map;
036import java.util.Set;
037import org.apache.hadoop.conf.Configuration;
038import org.apache.hadoop.fs.FileStatus;
039import org.apache.hadoop.fs.FileSystem;
040import org.apache.hadoop.fs.Path;
041import org.apache.hadoop.hbase.TableName;
042import org.apache.hadoop.hbase.backup.impl.BackupAdminImpl;
043import org.apache.hadoop.hbase.backup.impl.BackupSystemTable;
044import org.apache.hadoop.hbase.backup.util.BackupFileSystemManager;
045import org.apache.hadoop.hbase.testclassification.LargeTests;
046import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
047import org.apache.hadoop.util.ToolRunner;
048import org.junit.jupiter.api.AfterEach;
049import org.junit.jupiter.api.BeforeEach;
050import org.junit.jupiter.api.Tag;
051import org.junit.jupiter.api.Test;
052
053import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
054
055@Tag(LargeTests.TAG)
056public class TestBackupDeleteWithCleanup extends TestBackupBase {
057
058  String backupWalDirName = "TestBackupDeleteWithCleanup";
059
060  private FileSystem fs;
061  private Path backupWalDir;
062  private BackupSystemTable backupSystemTable;
063
064  @BeforeEach
065  public void setUpTest() throws Exception {
066    Path root = TEST_UTIL.getDataTestDirOnTestFS();
067    backupWalDir = new Path(root, backupWalDirName);
068    conf1.set(CONF_CONTINUOUS_BACKUP_WAL_DIR, backupWalDir.toString());
069    fs = FileSystem.get(conf1);
070    fs.mkdirs(backupWalDir);
071    backupSystemTable = new BackupSystemTable(TEST_UTIL.getConnection());
072  }
073
074  @AfterEach
075  public void tearDownTest() throws Exception {
076    if (backupSystemTable != null) {
077      backupSystemTable.close();
078    }
079    if (fs != null && backupWalDir != null) {
080      fs.delete(backupWalDir, true);
081    }
082
083    EnvironmentEdgeManager.reset();
084  }
085
086  @Test
087  public void testBackupDeleteWithCleanupLogic() throws Exception {
088    // Step 1: Setup Backup Folders
089    long currentTime = EnvironmentEdgeManager.getDelegate().currentTime();
090    setupBackupFolders(currentTime);
091
092    // Log the directory structure before cleanup
093    logDirectoryStructure(fs, backupWalDir, "Directory structure BEFORE cleanup:");
094
095    // Step 2: Simulate Backup Creation
096    backupSystemTable.addContinuousBackupTableSet(Set.of(table1),
097      currentTime - (2 * ONE_DAY_IN_MILLISECONDS));
098
099    EnvironmentEdgeManager
100      .injectEdge(() -> System.currentTimeMillis() - (2 * ONE_DAY_IN_MILLISECONDS));
101
102    String backupId = fullTableBackup(Lists.newArrayList(table1));
103    assertTrue(checkSucceeded(backupId));
104    String anotherBackupId = fullTableBackup(Lists.newArrayList(table1));
105    assertTrue(checkSucceeded(anotherBackupId));
106
107    // Step 3: Run Delete Command
108    deleteBackup(backupId);
109
110    // Log the directory structure after cleanup
111    logDirectoryStructure(fs, backupWalDir, "Directory structure AFTER cleanup:");
112
113    // Step 4: Verify Cleanup
114    verifyBackupCleanup(fs, backupWalDir, currentTime);
115
116    // Step 5: Verify System Table Update
117    verifySystemTableUpdate(backupSystemTable, currentTime);
118
119    // Cleanup
120    deleteBackup(anotherBackupId);
121  }
122
123  @Test
124  public void testSingleBackupForceDelete() throws Exception {
125    // Step 1: Setup Backup Folders
126    long currentTime = EnvironmentEdgeManager.getDelegate().currentTime();
127    setupBackupFolders(currentTime);
128
129    // Log the directory structure before cleanup
130    logDirectoryStructure(fs, backupWalDir, "Directory structure BEFORE cleanup:");
131
132    // Step 2: Simulate Backup Creation
133    backupSystemTable.addContinuousBackupTableSet(Set.of(table1),
134      currentTime - (2 * ONE_DAY_IN_MILLISECONDS));
135
136    EnvironmentEdgeManager
137      .injectEdge(() -> System.currentTimeMillis() - (2 * ONE_DAY_IN_MILLISECONDS));
138
139    String backupId = fullTableBackupWithContinuous(Lists.newArrayList(table1));
140    assertTrue(checkSucceeded(backupId));
141
142    assertTrue(continuousBackupReplicationPeerExistsAndEnabled(),
143      "Backup replication peer should be enabled after the backup");
144
145    // Step 3: Run Delete Command
146    deleteBackup(backupId);
147
148    // Log the directory structure after cleanup
149    logDirectoryStructure(fs, backupWalDir, "Directory structure AFTER cleanup:");
150
151    // Step 4: Verify CONTINUOUS_BACKUP_REPLICATION_PEER is disabled
152    assertFalse(continuousBackupReplicationPeerExistsAndEnabled(),
153      "Backup replication peer should be disabled or removed");
154
155    // Step 5: Verify that system table is updated to remove all the tables
156    Set<TableName> remainingTables = backupSystemTable.getContinuousBackupTableSet().keySet();
157    assertTrue(remainingTables.isEmpty(),
158      "System table should have no tables after all full backups are clear");
159
160    // Step 6: Verify that the backup WAL directory is empty
161    assertTrue(areWalAndBulkloadDirsEmpty(conf1, backupWalDir.toString()),
162      "WAL backup directory should be empty after force delete");
163
164    // Step 7: Take new full backup with continuous backup enabled
165    String backupIdContinuous = fullTableBackupWithContinuous(Lists.newArrayList(table1));
166
167    // Step 8: Verify CONTINUOUS_BACKUP_REPLICATION_PEER is enabled again
168    assertTrue(continuousBackupReplicationPeerExistsAndEnabled(),
169      "Backup replication peer should be re-enabled after new backup");
170
171    // And system table has new entry
172    Set<TableName> newTables = backupSystemTable.getContinuousBackupTableSet().keySet();
173    assertTrue(newTables.contains(table1),
174      "System table should contain the table after new backup");
175
176    // Cleanup
177    deleteBackup(backupIdContinuous);
178  }
179
180  private void setupBackupFolders(long currentTime) throws IOException {
181    setupBackupFolders(fs, backupWalDir, currentTime);
182  }
183
184  public static void setupBackupFolders(FileSystem fs, Path backupWalDir, long currentTime)
185    throws IOException {
186    Path walsDir = new Path(backupWalDir, WALS_DIR);
187    Path bulkLoadDir = new Path(backupWalDir, BULKLOAD_FILES_DIR);
188
189    fs.mkdirs(walsDir);
190    fs.mkdirs(bulkLoadDir);
191
192    SimpleDateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT);
193
194    for (int i = 0; i < 5; i++) {
195      String dateStr = dateFormat.format(new Date(currentTime - (i * ONE_DAY_IN_MILLISECONDS)));
196      fs.mkdirs(new Path(walsDir, dateStr));
197      fs.mkdirs(new Path(bulkLoadDir, dateStr));
198    }
199  }
200
201  private static void verifyBackupCleanup(FileSystem fs, Path backupWalDir, long currentTime)
202    throws IOException {
203    Path walsDir = new Path(backupWalDir, WALS_DIR);
204    Path bulkLoadDir = new Path(backupWalDir, BULKLOAD_FILES_DIR);
205    SimpleDateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT);
206
207    // Expect folders older than 3 days to be deleted
208    for (int i = 3; i < 5; i++) {
209      String oldDateStr = dateFormat.format(new Date(currentTime - (i * ONE_DAY_IN_MILLISECONDS)));
210      Path walPath = new Path(walsDir, oldDateStr);
211      Path bulkLoadPath = new Path(bulkLoadDir, oldDateStr);
212      assertFalse(fs.exists(walPath),
213        "Old WAL directory (" + walPath + ") should be deleted, but it exists!");
214      assertFalse(fs.exists(bulkLoadPath),
215        "Old BulkLoad directory (" + bulkLoadPath + ") should be deleted, but it exists!");
216    }
217
218    // Expect folders within the last 3 days to exist
219    for (int i = 0; i < 3; i++) {
220      String recentDateStr =
221        dateFormat.format(new Date(currentTime - (i * ONE_DAY_IN_MILLISECONDS)));
222      Path walPath = new Path(walsDir, recentDateStr);
223      Path bulkLoadPath = new Path(bulkLoadDir, recentDateStr);
224
225      assertTrue(fs.exists(walPath),
226        "Recent WAL directory (" + walPath + ") should exist, but it is missing!");
227      assertTrue(fs.exists(bulkLoadPath),
228        "Recent BulkLoad directory (" + bulkLoadPath + ") should exist, but it is missing!");
229    }
230  }
231
232  private void verifySystemTableUpdate(BackupSystemTable backupSystemTable, long currentTime)
233    throws IOException {
234    Map<TableName, Long> updatedTables = backupSystemTable.getContinuousBackupTableSet();
235
236    for (Map.Entry<TableName, Long> entry : updatedTables.entrySet()) {
237      long updatedStartTime = entry.getValue();
238
239      // Ensure that the updated start time is not earlier than the expected cutoff time
240      assertTrue(updatedStartTime >= (currentTime - (3 * ONE_DAY_IN_MILLISECONDS)),
241        "System table update failed!");
242    }
243  }
244
245  public static void logDirectoryStructure(FileSystem fs, Path dir, String message)
246    throws IOException {
247    System.out.println(message);
248    listDirectory(fs, dir, "  ");
249  }
250
251  public static void listDirectory(FileSystem fs, Path dir, String indent) throws IOException {
252    if (!fs.exists(dir)) {
253      System.out.println(indent + "[Missing] " + dir);
254      return;
255    }
256    FileStatus[] files = fs.listStatus(dir);
257    System.out.println(indent + dir);
258    for (FileStatus file : files) {
259      if (file.isDirectory()) {
260        listDirectory(fs, file.getPath(), indent + "  ");
261      } else {
262        System.out.println(indent + "  " + file.getPath());
263      }
264    }
265  }
266
267  private boolean continuousBackupReplicationPeerExistsAndEnabled() throws IOException {
268    return TEST_UTIL.getAdmin().listReplicationPeers().stream().anyMatch(
269      peer -> peer.getPeerId().equals(CONTINUOUS_BACKUP_REPLICATION_PEER) && peer.isEnabled());
270  }
271
272  private static boolean areWalAndBulkloadDirsEmpty(Configuration conf, String backupWalDir)
273    throws IOException {
274    BackupFileSystemManager manager =
275      new BackupFileSystemManager(CONTINUOUS_BACKUP_REPLICATION_PEER, conf, backupWalDir);
276
277    FileSystem fs = manager.getBackupFs();
278    Path walDir = manager.getWalsDir();
279    Path bulkloadDir = manager.getBulkLoadFilesDir();
280
281    return isDirectoryEmpty(fs, walDir) && isDirectoryEmpty(fs, bulkloadDir);
282  }
283
284  private static boolean isDirectoryEmpty(FileSystem fs, Path dirPath) throws IOException {
285    if (!fs.exists(dirPath)) {
286      // Directory doesn't exist — treat as empty
287      return true;
288    }
289    FileStatus[] entries = fs.listStatus(dirPath);
290    return entries == null || entries.length == 0;
291  }
292
293  private static void deleteBackup(String backupId) throws Exception {
294    int ret =
295      ToolRunner.run(conf1, new BackupDriver(), new String[] { "delete", "-l", backupId, "-fd" });
296    assertEquals(0, ret);
297  }
298
299  private String fullTableBackupWithContinuous(List<TableName> tables) throws IOException {
300    try (BackupAdmin admin = new BackupAdminImpl(TEST_UTIL.getConnection())) {
301      BackupRequest request =
302        createBackupRequest(BackupType.FULL, new ArrayList<>(tables), BACKUP_ROOT_DIR, false, true);
303      return admin.backupTables(request);
304    }
305  }
306
307}