001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.backup; 019 020import static org.junit.jupiter.api.Assertions.assertAll; 021import static org.junit.jupiter.api.Assertions.assertFalse; 022import static org.junit.jupiter.api.Assertions.assertNotNull; 023import static org.junit.jupiter.api.Assertions.assertTrue; 024 025import java.util.ArrayList; 026import java.util.List; 027import java.util.Map; 028import org.apache.hadoop.conf.Configuration; 029import org.apache.hadoop.fs.FileSystem; 030import org.apache.hadoop.fs.Path; 031import org.apache.hadoop.hbase.HBaseTestingUtil; 032import org.apache.hadoop.hbase.ServerName; 033import org.apache.hadoop.hbase.TableName; 034import org.apache.hadoop.hbase.backup.impl.BackupAdminImpl; 035import org.apache.hadoop.hbase.backup.impl.IncrementalBackupManager; 036import org.apache.hadoop.hbase.backup.util.BackupUtils; 037import org.apache.hadoop.hbase.client.Connection; 038import org.apache.hadoop.hbase.client.ConnectionFactory; 039import org.apache.hadoop.hbase.regionserver.HRegionServer; 040import org.apache.hadoop.hbase.testclassification.LargeTests; 041import org.apache.hadoop.hbase.util.CommonFSUtils; 042import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; 043import org.apache.hadoop.hbase.util.JVMClusterUtil; 044import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; 045import org.junit.jupiter.api.BeforeAll; 046import org.junit.jupiter.api.Tag; 047import org.junit.jupiter.api.Test; 048 049@Tag(LargeTests.TAG) 050public class TestIncrementalBackupManager extends TestBackupBase { 051 052 @BeforeAll 053 public static void setUp() throws Exception { 054 TEST_UTIL = new HBaseTestingUtil(); 055 conf1 = TEST_UTIL.getConfiguration(); 056 conf1.setBoolean(AbstractFSWALProvider.SEPARATE_OLDLOGDIR, true); 057 autoRestoreOnFailure = true; 058 useSecondCluster = false; 059 setUpHelper(); 060 } 061 062 @Test 063 public void testCollectWALFilesFromRegionServerDirectories() throws Exception { 064 testCollectWALFiles(true); 065 } 066 067 @Test 068 public void testCollectWALFilesFromFlatOldWALDirectory() throws Exception { 069 testCollectWALFiles(false); 070 } 071 072 private void testCollectWALFiles(boolean separateOldLogDir) throws Exception { 073 Configuration testConf = new Configuration(conf1); 074 testConf.setBoolean(AbstractFSWALProvider.SEPARATE_OLDLOGDIR, separateOldLogDir); 075 List<TableName> tables = List.of(table1); 076 try (Connection conn = ConnectionFactory.createConnection(testConf); 077 BackupAdminImpl backupAdmin = new BackupAdminImpl(conn)) { 078 String fullBackupId = 079 backupAdmin.backupTables(createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR)); 080 assertTrue(checkSucceeded(fullBackupId)); 081 082 try (IncrementalBackupManager manager = new IncrementalBackupManager(conn, testConf)) { 083 BackupInfo backupInfo = manager.createBackupInfo("backup_test", BackupType.INCREMENTAL, 084 tables, BACKUP_ROOT_DIR, -1, -1, false, false); 085 Map<String, Long> previousTimestamps = 086 BackupUtils.getRSLogTimestampMins(manager.readLogTimestampMap()); 087 ServerName serverName = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0).getServerName(); 088 Long previousTimestamp = 089 previousTimestamps.get(BackupUtils.toHostKey(serverName.getAddress().toString())); 090 assertNotNull(previousTimestamp); 091 092 TEST_UTIL.waitFor(30_000, 093 () -> EnvironmentEdgeManager.currentTime() > previousTimestamp + 1); 094 Path walRootDir = CommonFSUtils.getWALRootDir(conf1); 095 Path archiveDir = new Path(walRootDir, 096 AbstractFSWALProvider.getWALArchiveDirectoryName(testConf, serverName.toString())); 097 String walName = (separateOldLogDir ? "wal" : serverName.toString()) 098 + BackupUtils.LOGNAME_SEPARATOR + (previousTimestamp + 1); 099 Path archivedWAL = new Path(archiveDir, walName); 100 FileSystem fs = walRootDir.getFileSystem(conf1); 101 fs.mkdirs(archiveDir); 102 fs.create(archivedWAL).close(); 103 104 manager.getIncrBackupLogFileMap(); 105 106 assertTrue(backupInfo.getIncrBackupFileList().contains(archivedWAL.toString())); 107 } 108 } 109 } 110 111 /** 112 * WALs can be archived out of order, so a region server that took part in the log roll can have 113 * an archived WAL newer than its roll result while an older WAL is still in the WALs directory. 114 * The newer archived WAL must be deferred to a later backup instead of being backed up twice, and 115 * the older WAL must still end up in a backup. 116 */ 117 @Test 118 public void testOutOfOrderArchivedWALDoesNotSkipOlderWAL() throws Exception { 119 List<TableName> tables = List.of(table1); 120 HRegionServer rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0); 121 ServerName serverName = rs.getServerName(); 122 Path walRootDir = CommonFSUtils.getWALRootDir(conf1); 123 FileSystem fs = walRootDir.getFileSystem(conf1); 124 125 try (Connection conn = ConnectionFactory.createConnection(conf1); 126 BackupAdminImpl backupAdmin = new BackupAdminImpl(conn)) { 127 String fullBackupId = 128 backupAdmin.backupTables(createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR)); 129 assertTrue(checkSucceeded(fullBackupId)); 130 131 long olderWALTs = EnvironmentEdgeManager.currentTime() + 1; 132 long newerWALTs = olderWALTs + 1; 133 Path walDir = 134 new Path(walRootDir, AbstractFSWALProvider.getWALDirectoryName(serverName.toString())); 135 Path olderWAL = 136 new Path(walDir, serverName.toString() + BackupUtils.LOGNAME_SEPARATOR + olderWALTs); 137 Path archiveDir = new Path(walRootDir, 138 AbstractFSWALProvider.getWALArchiveDirectoryName(conf1, serverName.toString())); 139 Path newerArchivedWAL = 140 new Path(archiveDir, "wal" + BackupUtils.LOGNAME_SEPARATOR + newerWALTs); 141 fs.create(olderWAL).close(); 142 fs.mkdirs(archiveDir); 143 fs.create(newerArchivedWAL).close(); 144 145 try { 146 List<String> firstBackupFiles; 147 try (IncrementalBackupManager manager = new IncrementalBackupManager(conn, conf1)) { 148 BackupInfo backupInfo = manager.createBackupInfo("backup_incr_1", BackupType.INCREMENTAL, 149 tables, BACKUP_ROOT_DIR, -1, -1, false, false); 150 Map<String, Long> boundaries = manager.getIncrBackupLogFileMap(); 151 manager.writeRegionServerLogTimestamp(backupInfo.getTables(), boundaries); 152 firstBackupFiles = backupInfo.getIncrBackupFileList(); 153 } 154 assertFalse(firstBackupFiles.contains(newerArchivedWAL.toString()), 155 "Archived WAL newer than the roll result should be deferred to a later backup: " 156 + firstBackupFiles); 157 158 TEST_UTIL.waitFor(30_000, () -> EnvironmentEdgeManager.currentTime() > newerWALTs); 159 rs.getWalRoller().requestRollAll(); 160 rs.getWalRoller().waitUntilWalRollFinished(); 161 162 List<String> secondBackupFiles; 163 try (IncrementalBackupManager manager = new IncrementalBackupManager(conn, conf1)) { 164 BackupInfo backupInfo = manager.createBackupInfo("backup_incr_2", BackupType.INCREMENTAL, 165 tables, BACKUP_ROOT_DIR, -1, -1, false, false); 166 manager.getIncrBackupLogFileMap(); 167 secondBackupFiles = backupInfo.getIncrBackupFileList(); 168 } 169 170 assertTrue( 171 firstBackupFiles.contains(olderWAL.toString()) 172 || secondBackupFiles.contains(olderWAL.toString()), 173 "WAL " + olderWAL + " was not included in any backup. First backup: " + firstBackupFiles 174 + ", second backup: " + secondBackupFiles); 175 } finally { 176 fs.delete(olderWAL, false); 177 fs.delete(newerArchivedWAL, false); 178 } 179 } 180 } 181 182 /** 183 * A dead region server's WALs stay in its -splitting directory until WAL splitting finishes. An 184 * incremental backup that runs during the split must not lose them once they are archived. 185 */ 186 @Test 187 public void testWALOfDeadServerStillSplittingIsBackedUpAfterArchiving() throws Exception { 188 List<TableName> tables = List.of(table1); 189 ServerName deadServer = ServerName.valueOf("deadhost", 16020, 1001L); 190 Path walRootDir = CommonFSUtils.getWALRootDir(conf1); 191 FileSystem fs = walRootDir.getFileSystem(conf1); 192 Path splittingDir = splittingDir(walRootDir, deadServer); 193 Path archiveDir = new Path(walRootDir, 194 AbstractFSWALProvider.getWALArchiveDirectoryName(conf1, deadServer.toString())); 195 196 try (Connection conn = ConnectionFactory.createConnection(conf1); 197 BackupAdminImpl backupAdmin = new BackupAdminImpl(conn)) { 198 String fullBackupId = 199 backupAdmin.backupTables(createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR)); 200 assertTrue(checkSucceeded(fullBackupId)); 201 202 long walTs = EnvironmentEdgeManager.currentTime() + 1; 203 Path splittingWAL = new Path(splittingDir, walName(deadServer, walTs)); 204 Path archivedWAL = archivedWAL(archiveDir, walTs); 205 fs.create(splittingWAL).close(); 206 207 try { 208 // Make the live servers' roll results, and so the next boundaries, newer than the WAL. 209 TEST_UTIL.waitFor(30_000, () -> EnvironmentEdgeManager.currentTime() > walTs); 210 rollAllLiveRegionServers(); 211 212 List<String> firstBackupFiles = runIncrementalBackup(conn, tables, "backup_split_1"); 213 214 fs.mkdirs(archiveDir); 215 assertTrue(fs.rename(splittingWAL, archivedWAL)); 216 fs.delete(splittingDir, true); 217 218 List<String> secondBackupFiles = runIncrementalBackup(conn, tables, "backup_split_2"); 219 220 assertTrue( 221 firstBackupFiles.contains(splittingWAL.toString()) 222 || secondBackupFiles.contains(archivedWAL.toString()), 223 "WAL of the dead server was not included in any backup. First backup: " + firstBackupFiles 224 + ", second backup: " + secondBackupFiles); 225 } finally { 226 fs.delete(splittingDir, true); 227 fs.delete(archivedWAL, false); 228 } 229 } 230 } 231 232 /** 233 * WAL splitting archives a dead region server's WALs in parallel, so a newer WAL can be archived 234 * while older ones are still in the -splitting directory. Backing up the newer WAL must not move 235 * the boundary past the older ones. 236 */ 237 @Test 238 public void testOlderWALsOfDeadServerAreNotSkippedWhenNewerWALIsArchivedFirst() throws Exception { 239 List<TableName> tables = List.of(table1); 240 ServerName deadServer = ServerName.valueOf("deadhost", 16020, 2002L); 241 Path walRootDir = CommonFSUtils.getWALRootDir(conf1); 242 FileSystem fs = walRootDir.getFileSystem(conf1); 243 Path splittingDir = splittingDir(walRootDir, deadServer); 244 Path archiveDir = new Path(walRootDir, 245 AbstractFSWALProvider.getWALArchiveDirectoryName(conf1, deadServer.toString())); 246 247 try (Connection conn = ConnectionFactory.createConnection(conf1); 248 BackupAdminImpl backupAdmin = new BackupAdminImpl(conn)) { 249 String fullBackupId = 250 backupAdmin.backupTables(createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR)); 251 assertTrue(checkSucceeded(fullBackupId)); 252 253 long oldestTs = EnvironmentEdgeManager.currentTime() + 1; 254 long middleTs = oldestTs + 1; 255 long newestTs = oldestTs + 2; 256 Path oldestSplittingWAL = new Path(splittingDir, walName(deadServer, oldestTs)); 257 Path middleSplittingWAL = new Path(splittingDir, walName(deadServer, middleTs)); 258 Path oldestArchivedWAL = archivedWAL(archiveDir, oldestTs); 259 Path middleArchivedWAL = archivedWAL(archiveDir, middleTs); 260 Path newestArchivedWAL = archivedWAL(archiveDir, newestTs); 261 fs.create(oldestSplittingWAL).close(); 262 fs.create(middleSplittingWAL).close(); 263 fs.mkdirs(archiveDir); 264 fs.create(newestArchivedWAL).close(); 265 266 try { 267 TEST_UTIL.waitFor(30_000, () -> EnvironmentEdgeManager.currentTime() > newestTs); 268 269 List<String> firstBackupFiles = runIncrementalBackup(conn, tables, "backup_order_1"); 270 271 assertTrue(fs.rename(oldestSplittingWAL, oldestArchivedWAL)); 272 assertTrue(fs.rename(middleSplittingWAL, middleArchivedWAL)); 273 fs.delete(splittingDir, true); 274 275 List<String> secondBackupFiles = runIncrementalBackup(conn, tables, "backup_order_2"); 276 277 assertAll( 278 () -> assertTrue( 279 firstBackupFiles.contains(oldestSplittingWAL.toString()) 280 || secondBackupFiles.contains(oldestArchivedWAL.toString()), 281 "Oldest WAL of the dead server was not included in any backup. First backup: " 282 + firstBackupFiles + ", second backup: " + secondBackupFiles), 283 () -> assertTrue( 284 firstBackupFiles.contains(middleSplittingWAL.toString()) 285 || secondBackupFiles.contains(middleArchivedWAL.toString()), 286 "Middle WAL of the dead server was not included in any backup. First backup: " 287 + firstBackupFiles + ", second backup: " + secondBackupFiles)); 288 } finally { 289 fs.delete(splittingDir, true); 290 fs.delete(oldestArchivedWAL, false); 291 fs.delete(middleArchivedWAL, false); 292 fs.delete(newestArchivedWAL, false); 293 } 294 } 295 } 296 297 /** 298 * A dead region server can keep an old WAL, already covered by its boundary, in its -splitting 299 * directory across several incremental backups. Its boundary must survive those backups, or a 300 * later backup would include that WAL again and could bring back deleted data. 301 */ 302 @Test 303 public void testDeadServerKeepsBoundaryWhileOldWALIsStillSplitting() throws Exception { 304 List<TableName> tables = List.of(table1); 305 ServerName deadServer = ServerName.valueOf("deadhost", 16020, 3003L); 306 Path walRootDir = CommonFSUtils.getWALRootDir(conf1); 307 FileSystem fs = walRootDir.getFileSystem(conf1); 308 Path splittingDir = splittingDir(walRootDir, deadServer); 309 Path archiveDir = new Path(walRootDir, 310 AbstractFSWALProvider.getWALArchiveDirectoryName(conf1, deadServer.toString())); 311 312 try (Connection conn = ConnectionFactory.createConnection(conf1); 313 BackupAdminImpl backupAdmin = new BackupAdminImpl(conn)) { 314 String fullBackupId = 315 backupAdmin.backupTables(createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR)); 316 assertTrue(checkSucceeded(fullBackupId)); 317 318 long stuckTs = EnvironmentEdgeManager.currentTime() + 1; 319 long archivedTs = stuckTs + 1; 320 Path archivedWAL = archivedWAL(archiveDir, archivedTs); 321 Path stuckSplittingWAL = new Path(splittingDir, walName(deadServer, stuckTs)); 322 Path stuckArchivedWAL = archivedWAL(archiveDir, stuckTs); 323 fs.mkdirs(archiveDir); 324 fs.create(archivedWAL).close(); 325 326 try { 327 TEST_UTIL.waitFor(30_000, () -> EnvironmentEdgeManager.currentTime() > archivedTs); 328 List<String> firstBackupFiles = runIncrementalBackup(conn, tables, "backup_stuck_1"); 329 assertTrue(firstBackupFiles.contains(archivedWAL.toString()), 330 "Archived WAL of the dead server should be in the first backup: " + firstBackupFiles); 331 332 fs.create(stuckSplittingWAL).close(); 333 List<String> laterBackupFiles = new ArrayList<>(); 334 laterBackupFiles.addAll(runIncrementalBackup(conn, tables, "backup_stuck_2")); 335 laterBackupFiles.addAll(runIncrementalBackup(conn, tables, "backup_stuck_3")); 336 337 assertTrue(fs.rename(stuckSplittingWAL, stuckArchivedWAL)); 338 fs.delete(splittingDir, true); 339 laterBackupFiles.addAll(runIncrementalBackup(conn, tables, "backup_stuck_4")); 340 341 assertFalse( 342 laterBackupFiles.contains(stuckSplittingWAL.toString()) 343 || laterBackupFiles.contains(stuckArchivedWAL.toString()), 344 "WAL already covered by the dead server's boundary was backed up again: " 345 + laterBackupFiles); 346 } finally { 347 fs.delete(splittingDir, true); 348 fs.delete(archivedWAL, false); 349 fs.delete(stuckArchivedWAL, false); 350 } 351 } 352 } 353 354 private static Path splittingDir(Path walRootDir, ServerName serverName) { 355 return new Path(walRootDir, AbstractFSWALProvider.getWALDirectoryName(serverName.toString()) 356 + AbstractFSWALProvider.SPLITTING_EXT); 357 } 358 359 private static Path archivedWAL(Path archiveDir, long ts) { 360 return new Path(archiveDir, "wal" + BackupUtils.LOGNAME_SEPARATOR + ts); 361 } 362 363 private static String walName(ServerName serverName, long ts) { 364 return serverName.toString() + BackupUtils.LOGNAME_SEPARATOR + ts; 365 } 366 367 private static void rollAllLiveRegionServers() throws Exception { 368 for (JVMClusterUtil.RegionServerThread rst : TEST_UTIL.getMiniHBaseCluster() 369 .getLiveRegionServerThreads()) { 370 rst.getRegionServer().getWalRoller().requestRollAll(); 371 rst.getRegionServer().getWalRoller().waitUntilWalRollFinished(); 372 } 373 } 374 375 private static List<String> runIncrementalBackup(Connection conn, List<TableName> tables, 376 String backupId) throws Exception { 377 try (IncrementalBackupManager manager = new IncrementalBackupManager(conn, conf1)) { 378 BackupInfo backupInfo = manager.createBackupInfo(backupId, BackupType.INCREMENTAL, tables, 379 BACKUP_ROOT_DIR, -1, -1, false, false); 380 Map<String, Long> boundaries = manager.getIncrBackupLogFileMap(); 381 manager.writeRegionServerLogTimestamp(backupInfo.getTables(), boundaries); 382 return backupInfo.getIncrBackupFileList(); 383 } 384 } 385}