001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.backup;
019
020import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.CONF_CONTINUOUS_BACKUP_WAL_DIR;
021import static org.apache.hadoop.hbase.backup.replication.ContinuousBackupReplicationEndpoint.ONE_DAY_IN_MILLISECONDS;
022import static org.junit.jupiter.api.Assertions.assertEquals;
023import static org.junit.jupiter.api.Assertions.assertNotEquals;
024
025import java.io.IOException;
026import org.apache.hadoop.fs.FileSystem;
027import org.apache.hadoop.fs.Path;
028import org.apache.hadoop.hbase.TableName;
029import org.apache.hadoop.hbase.testclassification.LargeTests;
030import org.apache.hadoop.hbase.util.Bytes;
031import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
032import org.apache.hadoop.util.ToolRunner;
033import org.junit.jupiter.api.AfterAll;
034import org.junit.jupiter.api.BeforeAll;
035import org.junit.jupiter.api.Tag;
036import org.junit.jupiter.api.Test;
037
038/**
039 * Integration-style tests for Point-in-Time Restore (PITR).
040 * <p>
041 * These tests exercise the full backup / continuous backup / restore flow: - create backups at
042 * multiple historical points in time (via {@code BackupDriver}) - exercise WAL-based
043 * replication/continuous backup - validate Point-in-Time Restore behavior (successful restores,
044 * failure cases)
045 * </p>
046 * <p>
047 * NOTE: Some tests also create HFiles and perform HBase bulk-loads (HFile -> table) so the restore
048 * flow is validated when bulk-loaded storefiles are present in WALs. This ensures the
049 * BulkLoadCollector/BulkFilesCollector logic (discovering bulk-loaded store files referenced from
050 * WAL bulk-load descriptors) is exercised by the test suite.
051 * </p>
052 */
053@Tag(LargeTests.TAG)
054public class TestPointInTimeRestore extends TestBackupBase {
055
056  private static final String backupWalDirName = "TestPointInTimeRestoreWalDir";
057  static Path backupWalDir;
058  static FileSystem fs;
059
060  @BeforeAll
061  public static void setupBeforeClass() throws Exception {
062    Path root = TEST_UTIL.getDataTestDirOnTestFS();
063    backupWalDir = new Path(root, backupWalDirName);
064    fs = FileSystem.get(conf1);
065    fs.mkdirs(backupWalDir);
066    conf1.set(CONF_CONTINUOUS_BACKUP_WAL_DIR, backupWalDir.toString());
067
068    setUpBackups();
069  }
070
071  /**
072   * Sets up multiple backups at different timestamps by: 1. Adjusting the system time to simulate
073   * past backup points. 2. Loading data into tables to create meaningful snapshots. 3. Running full
074   * backups with or without continuous backup enabled. 4. Ensuring replication is complete before
075   * proceeding.
076   */
077  private static void setUpBackups() throws Exception {
078    // Simulate a backup taken 20 days ago
079    EnvironmentEdgeManager
080      .injectEdge(() -> System.currentTimeMillis() - 20 * ONE_DAY_IN_MILLISECONDS);
081    // Insert initial data into table1
082    PITRTestUtil.loadRandomData(TEST_UTIL, table1, famName, 1000);
083
084    // Perform a full backup for table1 with continuous backup enabled
085    String[] args =
086      PITRTestUtil.buildBackupArgs("full", new TableName[] { table1 }, true, BACKUP_ROOT_DIR);
087    int ret = ToolRunner.run(conf1, new BackupDriver(), args);
088    assertEquals(0, ret, "Backup should succeed");
089
090    // Move time forward to simulate 15 days ago
091    EnvironmentEdgeManager
092      .injectEdge(() -> System.currentTimeMillis() - 15 * ONE_DAY_IN_MILLISECONDS);
093    PITRTestUtil.loadRandomData(TEST_UTIL, table1, famName, 1000); // Add more data to table1
094
095    Path dir = TEST_UTIL.getDataTestDirOnTestFS("testBulkLoadByFamily");
096    PITRTestUtil.generateHFiles(dir, TEST_UTIL.getConfiguration(), Bytes.toString(famName));
097    PITRTestUtil.bulkLoadHFiles(table1, dir, TEST_UTIL.getConnection(),
098      TEST_UTIL.getConfiguration());
099
100    PITRTestUtil.loadRandomData(TEST_UTIL, table2, famName, 500); // Insert data into table2
101
102    PITRTestUtil.waitForReplication(); // Ensure replication is complete
103
104    // Perform a full backup for table2 with continuous backup enabled
105    args = PITRTestUtil.buildBackupArgs("full", new TableName[] { table2 }, true, BACKUP_ROOT_DIR);
106    ret = ToolRunner.run(conf1, new BackupDriver(), args);
107    assertEquals(0, ret, "Backup should succeed");
108
109    // Move time forward to simulate 10 days ago
110    EnvironmentEdgeManager
111      .injectEdge(() -> System.currentTimeMillis() - 10 * ONE_DAY_IN_MILLISECONDS);
112    PITRTestUtil.loadRandomData(TEST_UTIL, table2, famName, 500); // Add more data to table2
113    PITRTestUtil.loadRandomData(TEST_UTIL, table3, famName, 500); // Insert data into table3
114
115    // Perform a full backup for table3 and table4 (without continuous backup)
116    args = PITRTestUtil.buildBackupArgs("full", new TableName[] { table3, table4 }, false,
117      BACKUP_ROOT_DIR);
118    ret = ToolRunner.run(conf1, new BackupDriver(), args);
119    assertEquals(0, ret, "Backup should succeed");
120
121    PITRTestUtil.waitForReplication(); // Ensure replication is complete before concluding setup
122
123    // Reset time mocking to avoid affecting other tests
124    EnvironmentEdgeManager.reset();
125  }
126
127  @AfterAll
128  public static void setupAfterClass() throws IOException {
129    Path root = TEST_UTIL.getDataTestDirOnTestFS();
130    Path backupWalDir = new Path(root, backupWalDirName);
131    FileSystem fs = FileSystem.get(conf1);
132
133    if (fs.exists(backupWalDir)) {
134      fs.delete(backupWalDir, true);
135    }
136
137    conf1.unset(CONF_CONTINUOUS_BACKUP_WAL_DIR);
138  }
139
140  /**
141   * Verifies that PITR (Point-in-Time Restore) fails when the requested restore time is either in
142   * the future or outside the allowed retention window.
143   */
144  @Test
145  public void testPITR_FailsOutsideWindow() throws Exception {
146    // Case 1: Requested restore time is in the future (should fail)
147    String[] args = PITRTestUtil.buildPITRArgs(new TableName[] { table1 },
148      new TableName[] { TableName.valueOf("restoredTable1") },
149      EnvironmentEdgeManager.currentTime() + ONE_DAY_IN_MILLISECONDS, null);
150
151    int ret = ToolRunner.run(conf1, new PointInTimeRestoreDriver(), args);
152    assertNotEquals(0, ret,
153      "Restore should fail since the requested restore time is in the future");
154
155    // Case 2: Requested restore time is too old (beyond the retention window, should fail)
156    args = PITRTestUtil.buildPITRArgs(new TableName[] { table1 },
157      new TableName[] { TableName.valueOf("restoredTable1") },
158      EnvironmentEdgeManager.currentTime() - 40 * ONE_DAY_IN_MILLISECONDS, null);
159
160    ret = ToolRunner.run(conf1, new PointInTimeRestoreDriver(), args);
161    assertNotEquals(0, ret,
162      "Restore should fail since the requested restore time is outside the retention window");
163  }
164
165  /**
166   * Ensures that PITR fails when attempting to restore tables where continuous backup was not
167   * enabled.
168   */
169  @Test
170  public void testPointInTimeRestore_ContinuousBackupNotEnabledTables() throws Exception {
171    String[] args = PITRTestUtil.buildPITRArgs(new TableName[] { table3 },
172      new TableName[] { TableName.valueOf("restoredTable1") },
173      EnvironmentEdgeManager.currentTime() - 10 * ONE_DAY_IN_MILLISECONDS, null);
174
175    int ret = ToolRunner.run(conf1, new PointInTimeRestoreDriver(), args);
176    assertNotEquals(0, ret,
177      "Restore should fail since continuous backup is not enabled for the table");
178  }
179
180  /**
181   * Ensures that PITR fails when trying to restore from a point before continuous backup started.
182   */
183  @Test
184  public void testPointInTimeRestore_TablesWithNoProperBackupOrWals() throws Exception {
185    String[] args = PITRTestUtil.buildPITRArgs(new TableName[] { table2 },
186      new TableName[] { TableName.valueOf("restoredTable1") },
187      EnvironmentEdgeManager.currentTime() - 16 * ONE_DAY_IN_MILLISECONDS, null);
188
189    int ret = ToolRunner.run(conf1, new PointInTimeRestoreDriver(), args);
190    assertNotEquals(0, ret,
191      "Restore should fail since the requested restore point is before the start of continuous backup");
192  }
193
194  /**
195   * Verifies that PITR successfully restores data for a single table.
196   */
197  @Test
198  public void testPointInTimeRestore_SuccessfulRestoreForOneTable() throws Exception {
199    TableName restoredTable = TableName.valueOf("restoredTable");
200
201    // Perform restore operation
202    String[] args =
203      PITRTestUtil.buildPITRArgs(new TableName[] { table1 }, new TableName[] { restoredTable },
204        EnvironmentEdgeManager.currentTime() - 5 * ONE_DAY_IN_MILLISECONDS, null);
205
206    int ret = ToolRunner.run(conf1, new PointInTimeRestoreDriver(), args);
207    assertEquals(0, ret, "Restore should succeed");
208
209    // Validate that the restored table contains the same number of rows as the original table
210    assertEquals(PITRTestUtil.getRowCount(TEST_UTIL, table1),
211      PITRTestUtil.getRowCount(TEST_UTIL, restoredTable),
212      "Restored table should have the same row count as the original");
213  }
214
215  /**
216   * Verifies that PITR successfully restores multiple tables at once.
217   */
218  @Test
219  public void testPointInTimeRestore_SuccessfulRestoreForMultipleTables() throws Exception {
220    TableName restoredTable1 = TableName.valueOf("restoredTable1");
221    TableName restoredTable2 = TableName.valueOf("restoredTable2");
222
223    // Perform restore operation for multiple tables
224    String[] args = PITRTestUtil.buildPITRArgs(new TableName[] { table1, table2 },
225      new TableName[] { restoredTable1, restoredTable2 },
226      EnvironmentEdgeManager.currentTime() - 5 * ONE_DAY_IN_MILLISECONDS, null);
227
228    int ret = ToolRunner.run(conf1, new PointInTimeRestoreDriver(), args);
229    assertEquals(0, ret, "Restore should succeed");
230
231    // Validate that the restored tables contain the same number of rows as the originals
232    assertEquals(PITRTestUtil.getRowCount(TEST_UTIL, table1),
233      PITRTestUtil.getRowCount(TEST_UTIL, restoredTable1),
234      "Restored table1 should have the same row count as the original");
235    assertEquals(PITRTestUtil.getRowCount(TEST_UTIL, table2),
236      PITRTestUtil.getRowCount(TEST_UTIL, restoredTable2),
237      "Restored table2 should have the same row count as the original");
238  }
239}