001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.backup;
019
020import static org.junit.jupiter.api.Assertions.assertAll;
021import static org.junit.jupiter.api.Assertions.assertFalse;
022import static org.junit.jupiter.api.Assertions.assertNotNull;
023import static org.junit.jupiter.api.Assertions.assertTrue;
024
025import java.util.ArrayList;
026import java.util.List;
027import java.util.Map;
028import org.apache.hadoop.conf.Configuration;
029import org.apache.hadoop.fs.FileSystem;
030import org.apache.hadoop.fs.Path;
031import org.apache.hadoop.hbase.HBaseTestingUtil;
032import org.apache.hadoop.hbase.ServerName;
033import org.apache.hadoop.hbase.TableName;
034import org.apache.hadoop.hbase.backup.impl.BackupAdminImpl;
035import org.apache.hadoop.hbase.backup.impl.IncrementalBackupManager;
036import org.apache.hadoop.hbase.backup.util.BackupUtils;
037import org.apache.hadoop.hbase.client.Connection;
038import org.apache.hadoop.hbase.client.ConnectionFactory;
039import org.apache.hadoop.hbase.regionserver.HRegionServer;
040import org.apache.hadoop.hbase.testclassification.LargeTests;
041import org.apache.hadoop.hbase.util.CommonFSUtils;
042import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
043import org.apache.hadoop.hbase.util.JVMClusterUtil;
044import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
045import org.junit.jupiter.api.BeforeAll;
046import org.junit.jupiter.api.Tag;
047import org.junit.jupiter.api.Test;
048
049@Tag(LargeTests.TAG)
050public class TestIncrementalBackupManager extends TestBackupBase {
051
052  @BeforeAll
053  public static void setUp() throws Exception {
054    TEST_UTIL = new HBaseTestingUtil();
055    conf1 = TEST_UTIL.getConfiguration();
056    conf1.setBoolean(AbstractFSWALProvider.SEPARATE_OLDLOGDIR, true);
057    autoRestoreOnFailure = true;
058    useSecondCluster = false;
059    setUpHelper();
060  }
061
062  @Test
063  public void testCollectWALFilesFromRegionServerDirectories() throws Exception {
064    testCollectWALFiles(true);
065  }
066
067  @Test
068  public void testCollectWALFilesFromFlatOldWALDirectory() throws Exception {
069    testCollectWALFiles(false);
070  }
071
072  private void testCollectWALFiles(boolean separateOldLogDir) throws Exception {
073    Configuration testConf = new Configuration(conf1);
074    testConf.setBoolean(AbstractFSWALProvider.SEPARATE_OLDLOGDIR, separateOldLogDir);
075    List<TableName> tables = List.of(table1);
076    try (Connection conn = ConnectionFactory.createConnection(testConf);
077      BackupAdminImpl backupAdmin = new BackupAdminImpl(conn)) {
078      String fullBackupId =
079        backupAdmin.backupTables(createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR));
080      assertTrue(checkSucceeded(fullBackupId));
081
082      try (IncrementalBackupManager manager = new IncrementalBackupManager(conn, testConf)) {
083        BackupInfo backupInfo = manager.createBackupInfo("backup_test", BackupType.INCREMENTAL,
084          tables, BACKUP_ROOT_DIR, -1, -1, false, false);
085        Map<String, Long> previousTimestamps =
086          BackupUtils.getRSLogTimestampMins(manager.readLogTimestampMap());
087        ServerName serverName = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0).getServerName();
088        Long previousTimestamp =
089          previousTimestamps.get(BackupUtils.toHostKey(serverName.getAddress().toString()));
090        assertNotNull(previousTimestamp);
091
092        TEST_UTIL.waitFor(30_000,
093          () -> EnvironmentEdgeManager.currentTime() > previousTimestamp + 1);
094        Path walRootDir = CommonFSUtils.getWALRootDir(conf1);
095        Path archiveDir = new Path(walRootDir,
096          AbstractFSWALProvider.getWALArchiveDirectoryName(testConf, serverName.toString()));
097        String walName = (separateOldLogDir ? "wal" : serverName.toString())
098          + BackupUtils.LOGNAME_SEPARATOR + (previousTimestamp + 1);
099        Path archivedWAL = new Path(archiveDir, walName);
100        FileSystem fs = walRootDir.getFileSystem(conf1);
101        fs.mkdirs(archiveDir);
102        fs.create(archivedWAL).close();
103
104        manager.getIncrBackupLogFileMap();
105
106        assertTrue(backupInfo.getIncrBackupFileList().contains(archivedWAL.toString()));
107      }
108    }
109  }
110
111  /**
112   * WALs can be archived out of order, so a region server that took part in the log roll can have
113   * an archived WAL newer than its roll result while an older WAL is still in the WALs directory.
114   * The newer archived WAL must be deferred to a later backup instead of being backed up twice, and
115   * the older WAL must still end up in a backup.
116   */
117  @Test
118  public void testOutOfOrderArchivedWALDoesNotSkipOlderWAL() throws Exception {
119    List<TableName> tables = List.of(table1);
120    HRegionServer rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0);
121    ServerName serverName = rs.getServerName();
122    Path walRootDir = CommonFSUtils.getWALRootDir(conf1);
123    FileSystem fs = walRootDir.getFileSystem(conf1);
124
125    try (Connection conn = ConnectionFactory.createConnection(conf1);
126      BackupAdminImpl backupAdmin = new BackupAdminImpl(conn)) {
127      String fullBackupId =
128        backupAdmin.backupTables(createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR));
129      assertTrue(checkSucceeded(fullBackupId));
130
131      long olderWALTs = EnvironmentEdgeManager.currentTime() + 1;
132      long newerWALTs = olderWALTs + 1;
133      Path walDir =
134        new Path(walRootDir, AbstractFSWALProvider.getWALDirectoryName(serverName.toString()));
135      Path olderWAL =
136        new Path(walDir, serverName.toString() + BackupUtils.LOGNAME_SEPARATOR + olderWALTs);
137      Path archiveDir = new Path(walRootDir,
138        AbstractFSWALProvider.getWALArchiveDirectoryName(conf1, serverName.toString()));
139      Path newerArchivedWAL =
140        new Path(archiveDir, "wal" + BackupUtils.LOGNAME_SEPARATOR + newerWALTs);
141      fs.create(olderWAL).close();
142      fs.mkdirs(archiveDir);
143      fs.create(newerArchivedWAL).close();
144
145      try {
146        List<String> firstBackupFiles;
147        try (IncrementalBackupManager manager = new IncrementalBackupManager(conn, conf1)) {
148          BackupInfo backupInfo = manager.createBackupInfo("backup_incr_1", BackupType.INCREMENTAL,
149            tables, BACKUP_ROOT_DIR, -1, -1, false, false);
150          Map<String, Long> boundaries = manager.getIncrBackupLogFileMap();
151          manager.writeRegionServerLogTimestamp(backupInfo.getTables(), boundaries);
152          firstBackupFiles = backupInfo.getIncrBackupFileList();
153        }
154        assertFalse(firstBackupFiles.contains(newerArchivedWAL.toString()),
155          "Archived WAL newer than the roll result should be deferred to a later backup: "
156            + firstBackupFiles);
157
158        TEST_UTIL.waitFor(30_000, () -> EnvironmentEdgeManager.currentTime() > newerWALTs);
159        rs.getWalRoller().requestRollAll();
160        rs.getWalRoller().waitUntilWalRollFinished();
161
162        List<String> secondBackupFiles;
163        try (IncrementalBackupManager manager = new IncrementalBackupManager(conn, conf1)) {
164          BackupInfo backupInfo = manager.createBackupInfo("backup_incr_2", BackupType.INCREMENTAL,
165            tables, BACKUP_ROOT_DIR, -1, -1, false, false);
166          manager.getIncrBackupLogFileMap();
167          secondBackupFiles = backupInfo.getIncrBackupFileList();
168        }
169
170        assertTrue(
171          firstBackupFiles.contains(olderWAL.toString())
172            || secondBackupFiles.contains(olderWAL.toString()),
173          "WAL " + olderWAL + " was not included in any backup. First backup: " + firstBackupFiles
174            + ", second backup: " + secondBackupFiles);
175      } finally {
176        fs.delete(olderWAL, false);
177        fs.delete(newerArchivedWAL, false);
178      }
179    }
180  }
181
182  /**
183   * A dead region server's WALs stay in its -splitting directory until WAL splitting finishes. An
184   * incremental backup that runs during the split must not lose them once they are archived.
185   */
186  @Test
187  public void testWALOfDeadServerStillSplittingIsBackedUpAfterArchiving() throws Exception {
188    List<TableName> tables = List.of(table1);
189    ServerName deadServer = ServerName.valueOf("deadhost", 16020, 1001L);
190    Path walRootDir = CommonFSUtils.getWALRootDir(conf1);
191    FileSystem fs = walRootDir.getFileSystem(conf1);
192    Path splittingDir = splittingDir(walRootDir, deadServer);
193    Path archiveDir = new Path(walRootDir,
194      AbstractFSWALProvider.getWALArchiveDirectoryName(conf1, deadServer.toString()));
195
196    try (Connection conn = ConnectionFactory.createConnection(conf1);
197      BackupAdminImpl backupAdmin = new BackupAdminImpl(conn)) {
198      String fullBackupId =
199        backupAdmin.backupTables(createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR));
200      assertTrue(checkSucceeded(fullBackupId));
201
202      long walTs = EnvironmentEdgeManager.currentTime() + 1;
203      Path splittingWAL = new Path(splittingDir, walName(deadServer, walTs));
204      Path archivedWAL = archivedWAL(archiveDir, walTs);
205      fs.create(splittingWAL).close();
206
207      try {
208        // Make the live servers' roll results, and so the next boundaries, newer than the WAL.
209        TEST_UTIL.waitFor(30_000, () -> EnvironmentEdgeManager.currentTime() > walTs);
210        rollAllLiveRegionServers();
211
212        List<String> firstBackupFiles = runIncrementalBackup(conn, tables, "backup_split_1");
213
214        fs.mkdirs(archiveDir);
215        assertTrue(fs.rename(splittingWAL, archivedWAL));
216        fs.delete(splittingDir, true);
217
218        List<String> secondBackupFiles = runIncrementalBackup(conn, tables, "backup_split_2");
219
220        assertTrue(
221          firstBackupFiles.contains(splittingWAL.toString())
222            || secondBackupFiles.contains(archivedWAL.toString()),
223          "WAL of the dead server was not included in any backup. First backup: " + firstBackupFiles
224            + ", second backup: " + secondBackupFiles);
225      } finally {
226        fs.delete(splittingDir, true);
227        fs.delete(archivedWAL, false);
228      }
229    }
230  }
231
232  /**
233   * WAL splitting archives a dead region server's WALs in parallel, so a newer WAL can be archived
234   * while older ones are still in the -splitting directory. Backing up the newer WAL must not move
235   * the boundary past the older ones.
236   */
237  @Test
238  public void testOlderWALsOfDeadServerAreNotSkippedWhenNewerWALIsArchivedFirst() throws Exception {
239    List<TableName> tables = List.of(table1);
240    ServerName deadServer = ServerName.valueOf("deadhost", 16020, 2002L);
241    Path walRootDir = CommonFSUtils.getWALRootDir(conf1);
242    FileSystem fs = walRootDir.getFileSystem(conf1);
243    Path splittingDir = splittingDir(walRootDir, deadServer);
244    Path archiveDir = new Path(walRootDir,
245      AbstractFSWALProvider.getWALArchiveDirectoryName(conf1, deadServer.toString()));
246
247    try (Connection conn = ConnectionFactory.createConnection(conf1);
248      BackupAdminImpl backupAdmin = new BackupAdminImpl(conn)) {
249      String fullBackupId =
250        backupAdmin.backupTables(createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR));
251      assertTrue(checkSucceeded(fullBackupId));
252
253      long oldestTs = EnvironmentEdgeManager.currentTime() + 1;
254      long middleTs = oldestTs + 1;
255      long newestTs = oldestTs + 2;
256      Path oldestSplittingWAL = new Path(splittingDir, walName(deadServer, oldestTs));
257      Path middleSplittingWAL = new Path(splittingDir, walName(deadServer, middleTs));
258      Path oldestArchivedWAL = archivedWAL(archiveDir, oldestTs);
259      Path middleArchivedWAL = archivedWAL(archiveDir, middleTs);
260      Path newestArchivedWAL = archivedWAL(archiveDir, newestTs);
261      fs.create(oldestSplittingWAL).close();
262      fs.create(middleSplittingWAL).close();
263      fs.mkdirs(archiveDir);
264      fs.create(newestArchivedWAL).close();
265
266      try {
267        TEST_UTIL.waitFor(30_000, () -> EnvironmentEdgeManager.currentTime() > newestTs);
268
269        List<String> firstBackupFiles = runIncrementalBackup(conn, tables, "backup_order_1");
270
271        assertTrue(fs.rename(oldestSplittingWAL, oldestArchivedWAL));
272        assertTrue(fs.rename(middleSplittingWAL, middleArchivedWAL));
273        fs.delete(splittingDir, true);
274
275        List<String> secondBackupFiles = runIncrementalBackup(conn, tables, "backup_order_2");
276
277        assertAll(
278          () -> assertTrue(
279            firstBackupFiles.contains(oldestSplittingWAL.toString())
280              || secondBackupFiles.contains(oldestArchivedWAL.toString()),
281            "Oldest WAL of the dead server was not included in any backup. First backup: "
282              + firstBackupFiles + ", second backup: " + secondBackupFiles),
283          () -> assertTrue(
284            firstBackupFiles.contains(middleSplittingWAL.toString())
285              || secondBackupFiles.contains(middleArchivedWAL.toString()),
286            "Middle WAL of the dead server was not included in any backup. First backup: "
287              + firstBackupFiles + ", second backup: " + secondBackupFiles));
288      } finally {
289        fs.delete(splittingDir, true);
290        fs.delete(oldestArchivedWAL, false);
291        fs.delete(middleArchivedWAL, false);
292        fs.delete(newestArchivedWAL, false);
293      }
294    }
295  }
296
297  /**
298   * A dead region server can keep an old WAL, already covered by its boundary, in its -splitting
299   * directory across several incremental backups. Its boundary must survive those backups, or a
300   * later backup would include that WAL again and could bring back deleted data.
301   */
302  @Test
303  public void testDeadServerKeepsBoundaryWhileOldWALIsStillSplitting() throws Exception {
304    List<TableName> tables = List.of(table1);
305    ServerName deadServer = ServerName.valueOf("deadhost", 16020, 3003L);
306    Path walRootDir = CommonFSUtils.getWALRootDir(conf1);
307    FileSystem fs = walRootDir.getFileSystem(conf1);
308    Path splittingDir = splittingDir(walRootDir, deadServer);
309    Path archiveDir = new Path(walRootDir,
310      AbstractFSWALProvider.getWALArchiveDirectoryName(conf1, deadServer.toString()));
311
312    try (Connection conn = ConnectionFactory.createConnection(conf1);
313      BackupAdminImpl backupAdmin = new BackupAdminImpl(conn)) {
314      String fullBackupId =
315        backupAdmin.backupTables(createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR));
316      assertTrue(checkSucceeded(fullBackupId));
317
318      long stuckTs = EnvironmentEdgeManager.currentTime() + 1;
319      long archivedTs = stuckTs + 1;
320      Path archivedWAL = archivedWAL(archiveDir, archivedTs);
321      Path stuckSplittingWAL = new Path(splittingDir, walName(deadServer, stuckTs));
322      Path stuckArchivedWAL = archivedWAL(archiveDir, stuckTs);
323      fs.mkdirs(archiveDir);
324      fs.create(archivedWAL).close();
325
326      try {
327        TEST_UTIL.waitFor(30_000, () -> EnvironmentEdgeManager.currentTime() > archivedTs);
328        List<String> firstBackupFiles = runIncrementalBackup(conn, tables, "backup_stuck_1");
329        assertTrue(firstBackupFiles.contains(archivedWAL.toString()),
330          "Archived WAL of the dead server should be in the first backup: " + firstBackupFiles);
331
332        fs.create(stuckSplittingWAL).close();
333        List<String> laterBackupFiles = new ArrayList<>();
334        laterBackupFiles.addAll(runIncrementalBackup(conn, tables, "backup_stuck_2"));
335        laterBackupFiles.addAll(runIncrementalBackup(conn, tables, "backup_stuck_3"));
336
337        assertTrue(fs.rename(stuckSplittingWAL, stuckArchivedWAL));
338        fs.delete(splittingDir, true);
339        laterBackupFiles.addAll(runIncrementalBackup(conn, tables, "backup_stuck_4"));
340
341        assertFalse(
342          laterBackupFiles.contains(stuckSplittingWAL.toString())
343            || laterBackupFiles.contains(stuckArchivedWAL.toString()),
344          "WAL already covered by the dead server's boundary was backed up again: "
345            + laterBackupFiles);
346      } finally {
347        fs.delete(splittingDir, true);
348        fs.delete(archivedWAL, false);
349        fs.delete(stuckArchivedWAL, false);
350      }
351    }
352  }
353
354  private static Path splittingDir(Path walRootDir, ServerName serverName) {
355    return new Path(walRootDir, AbstractFSWALProvider.getWALDirectoryName(serverName.toString())
356      + AbstractFSWALProvider.SPLITTING_EXT);
357  }
358
359  private static Path archivedWAL(Path archiveDir, long ts) {
360    return new Path(archiveDir, "wal" + BackupUtils.LOGNAME_SEPARATOR + ts);
361  }
362
363  private static String walName(ServerName serverName, long ts) {
364    return serverName.toString() + BackupUtils.LOGNAME_SEPARATOR + ts;
365  }
366
367  private static void rollAllLiveRegionServers() throws Exception {
368    for (JVMClusterUtil.RegionServerThread rst : TEST_UTIL.getMiniHBaseCluster()
369      .getLiveRegionServerThreads()) {
370      rst.getRegionServer().getWalRoller().requestRollAll();
371      rst.getRegionServer().getWalRoller().waitUntilWalRollFinished();
372    }
373  }
374
375  private static List<String> runIncrementalBackup(Connection conn, List<TableName> tables,
376    String backupId) throws Exception {
377    try (IncrementalBackupManager manager = new IncrementalBackupManager(conn, conf1)) {
378      BackupInfo backupInfo = manager.createBackupInfo(backupId, BackupType.INCREMENTAL, tables,
379        BACKUP_ROOT_DIR, -1, -1, false, false);
380      Map<String, Long> boundaries = manager.getIncrBackupLogFileMap();
381      manager.writeRegionServerLogTimestamp(backupInfo.getTables(), boundaries);
382      return backupInfo.getIncrBackupFileList();
383    }
384  }
385}