001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.backup.impl;
019
020import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.CONF_CONTINUOUS_BACKUP_WAL_DIR;
021import static org.apache.hadoop.hbase.backup.TestBackupDeleteWithCleanup.logDirectoryStructure;
022import static org.apache.hadoop.hbase.backup.TestBackupDeleteWithCleanup.setupBackupFolders;
023import static org.apache.hadoop.hbase.backup.replication.ContinuousBackupReplicationEndpoint.ONE_DAY_IN_MILLISECONDS;
024import static org.apache.hadoop.hbase.backup.util.BackupFileSystemManager.BULKLOAD_FILES_DIR;
025import static org.apache.hadoop.hbase.backup.util.BackupFileSystemManager.WALS_DIR;
026import static org.apache.hadoop.hbase.backup.util.BackupUtils.DATE_FORMAT;
027import static org.junit.jupiter.api.Assertions.assertEquals;
028import static org.junit.jupiter.api.Assertions.assertFalse;
029import static org.junit.jupiter.api.Assertions.assertTrue;
030import static org.mockito.ArgumentMatchers.any;
031import static org.mockito.Mockito.mock;
032import static org.mockito.Mockito.verify;
033import static org.mockito.Mockito.when;
034
035import java.io.IOException;
036import java.text.SimpleDateFormat;
037import java.util.ArrayList;
038import java.util.Date;
039import java.util.List;
040import java.util.Map;
041import java.util.Set;
042import java.util.TimeZone;
043import org.apache.hadoop.fs.FileSystem;
044import org.apache.hadoop.fs.Path;
045import org.apache.hadoop.hbase.TableName;
046import org.apache.hadoop.hbase.backup.BackupInfo;
047import org.apache.hadoop.hbase.backup.BackupType;
048import org.apache.hadoop.hbase.backup.TestBackupBase;
049import org.apache.hadoop.hbase.testclassification.SmallTests;
050import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
051import org.junit.jupiter.api.Tag;
052import org.junit.jupiter.api.Test;
053
054@Tag(SmallTests.TAG)
055public class TestBackupCommands extends TestBackupBase {
056  String backupWalDirName = "TestBackupWalDir";
057
058  /**
059   * Tests whether determineWALCleanupCutoffTime returns the correct FULL backup start timestamp.
060   */
061  @Test
062  public void testDetermineWALCleanupCutoffTimeOfCleanupCommand() throws IOException {
063    // GIVEN
064    BackupSystemTable sysTable = mock(BackupSystemTable.class);
065
066    BackupInfo full1 = new BackupInfo();
067    full1.setType(BackupType.FULL);
068    full1.setStartTs(1111L);
069    full1.setState(BackupInfo.BackupState.COMPLETE);
070
071    BackupInfo inc = new BackupInfo();
072    inc.setType(BackupType.INCREMENTAL);
073    inc.setStartTs(2222L);
074    inc.setState(BackupInfo.BackupState.COMPLETE);
075
076    BackupInfo full2 = new BackupInfo();
077    full2.setType(BackupType.FULL);
078    full2.setStartTs(3333L);
079    full2.setState(BackupInfo.BackupState.COMPLETE);
080
081    // Ordered as newest to oldest, will be reversed in the method
082    List<BackupInfo> backupInfos = List.of(full2, inc, full1);
083    when(sysTable.getBackupHistory(any())).thenReturn(new ArrayList<>(backupInfos));
084
085    // WHEN
086    BackupCommands.DeleteCommand command = new BackupCommands.DeleteCommand(conf1, null);
087    long cutoff = command.determineWALCleanupCutoffTime(sysTable);
088
089    // THEN
090    assertEquals(1111L, cutoff, "Expected oldest FULL backup timestamp");
091  }
092
093  @Test
094  public void testUpdateBackupTableStartTimesOfCleanupCommand() throws IOException {
095    // GIVEN
096    BackupSystemTable mockSysTable = mock(BackupSystemTable.class);
097
098    TableName tableA = TableName.valueOf("ns", "tableA");
099    TableName tableB = TableName.valueOf("ns", "tableB");
100    TableName tableC = TableName.valueOf("ns", "tableC");
101
102    long cutoffTimestamp = 1_000_000L;
103
104    // Simulate current table start times
105    Map<TableName, Long> tableSet = Map.of(tableA, 900_000L, // Before cutoff → should be updated
106      tableB, 1_100_000L, // After cutoff → should NOT be updated
107      tableC, 800_000L // Before cutoff → should be updated
108    );
109
110    when(mockSysTable.getContinuousBackupTableSet()).thenReturn(tableSet);
111
112    // WHEN
113    BackupCommands.DeleteCommand command = new BackupCommands.DeleteCommand(conf1, null);
114    command.updateBackupTableStartTimes(mockSysTable, cutoffTimestamp);
115
116    // THEN
117    Set<TableName> expectedUpdated = Set.of(tableA, tableC);
118    verify(mockSysTable).updateContinuousBackupTableSet(expectedUpdated, cutoffTimestamp);
119  }
120
121  @Test
122  public void testDeleteOldWALFilesOfCleanupCommand() throws IOException {
123    // GIVEN
124    Path root = TEST_UTIL.getDataTestDirOnTestFS();
125    Path backupWalDir = new Path(root, backupWalDirName);
126    conf1.set(CONF_CONTINUOUS_BACKUP_WAL_DIR, backupWalDir.toString());
127
128    FileSystem fs = FileSystem.get(conf1);
129    fs.mkdirs(backupWalDir);
130
131    long currentTime = EnvironmentEdgeManager.getDelegate().currentTime();
132    setupBackupFolders(fs, backupWalDir, currentTime); // Create 5 days of WAL/bulkload-files folder
133
134    logDirectoryStructure(fs, backupWalDir, "Before cleanup:");
135
136    // Delete files older than 2 days from current time
137    long cutoffTime = currentTime - (2 * ONE_DAY_IN_MILLISECONDS);
138
139    // WHEN
140    BackupCommands.DeleteCommand command = new BackupCommands.DeleteCommand(conf1, null);
141    command.deleteOldWALFiles(conf1, backupWalDir.toString(), cutoffTime);
142
143    logDirectoryStructure(fs, backupWalDir, "After cleanup:");
144
145    // THEN
146    verifyCleanupOutcome(fs, backupWalDir, currentTime, cutoffTime);
147  }
148
149  private static void verifyCleanupOutcome(FileSystem fs, Path backupWalDir, long currentTime,
150    long cutoffTime) throws IOException {
151    Path walsDir = new Path(backupWalDir, WALS_DIR);
152    Path bulkLoadDir = new Path(backupWalDir, BULKLOAD_FILES_DIR);
153    SimpleDateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT);
154    dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
155
156    for (int i = 0; i < 5; i++) {
157      long dayTime = currentTime - (i * ONE_DAY_IN_MILLISECONDS);
158      String dayDir = dateFormat.format(new Date(dayTime));
159      Path walPath = new Path(walsDir, dayDir);
160      Path bulkPath = new Path(bulkLoadDir, dayDir);
161
162      if (dayTime + ONE_DAY_IN_MILLISECONDS - 1 < cutoffTime) {
163        assertFalse(fs.exists(walPath), "Old WAL dir should be deleted: " + walPath);
164        assertFalse(fs.exists(bulkPath), "Old BulkLoad dir should be deleted: " + bulkPath);
165      } else {
166        assertTrue(fs.exists(walPath), "Recent WAL dir should exist: " + walPath);
167        assertTrue(fs.exists(bulkPath), "Recent BulkLoad dir should exist: " + bulkPath);
168      }
169    }
170  }
171}