001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.backup;
019
020import static org.junit.jupiter.api.Assertions.assertNotNull;
021import static org.junit.jupiter.api.Assertions.assertTrue;
022
023import java.util.List;
024import java.util.Map;
025import org.apache.hadoop.conf.Configuration;
026import org.apache.hadoop.fs.FileSystem;
027import org.apache.hadoop.fs.Path;
028import org.apache.hadoop.hbase.HBaseTestingUtil;
029import org.apache.hadoop.hbase.ServerName;
030import org.apache.hadoop.hbase.TableName;
031import org.apache.hadoop.hbase.backup.impl.BackupAdminImpl;
032import org.apache.hadoop.hbase.backup.impl.IncrementalBackupManager;
033import org.apache.hadoop.hbase.backup.util.BackupUtils;
034import org.apache.hadoop.hbase.client.Connection;
035import org.apache.hadoop.hbase.client.ConnectionFactory;
036import org.apache.hadoop.hbase.testclassification.LargeTests;
037import org.apache.hadoop.hbase.util.CommonFSUtils;
038import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
039import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
040import org.junit.jupiter.api.BeforeAll;
041import org.junit.jupiter.api.Tag;
042import org.junit.jupiter.api.Test;
043
044@Tag(LargeTests.TAG)
045public class TestIncrementalBackupManager extends TestBackupBase {
046
047  @BeforeAll
048  public static void setUp() throws Exception {
049    TEST_UTIL = new HBaseTestingUtil();
050    conf1 = TEST_UTIL.getConfiguration();
051    conf1.setBoolean(AbstractFSWALProvider.SEPARATE_OLDLOGDIR, true);
052    autoRestoreOnFailure = true;
053    useSecondCluster = false;
054    setUpHelper();
055  }
056
057  @Test
058  public void testCollectWALFilesFromRegionServerDirectories() throws Exception {
059    testCollectWALFiles(true);
060  }
061
062  @Test
063  public void testCollectWALFilesFromFlatOldWALDirectory() throws Exception {
064    testCollectWALFiles(false);
065  }
066
067  private void testCollectWALFiles(boolean separateOldLogDir) throws Exception {
068    Configuration testConf = new Configuration(conf1);
069    testConf.setBoolean(AbstractFSWALProvider.SEPARATE_OLDLOGDIR, separateOldLogDir);
070    List<TableName> tables = List.of(table1);
071    try (Connection conn = ConnectionFactory.createConnection(testConf);
072      BackupAdminImpl backupAdmin = new BackupAdminImpl(conn)) {
073      String fullBackupId =
074        backupAdmin.backupTables(createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR));
075      assertTrue(checkSucceeded(fullBackupId));
076
077      try (IncrementalBackupManager manager = new IncrementalBackupManager(conn, testConf)) {
078        BackupInfo backupInfo = manager.createBackupInfo("backup_test", BackupType.INCREMENTAL,
079          tables, BACKUP_ROOT_DIR, -1, -1, false);
080        Map<String, Long> previousTimestamps =
081          BackupUtils.getRSLogTimestampMins(manager.readLogTimestampMap());
082        ServerName serverName = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0).getServerName();
083        Long previousTimestamp = previousTimestamps.get(serverName.getAddress().toString());
084        assertNotNull(previousTimestamp);
085
086        TEST_UTIL.waitFor(30_000,
087          () -> EnvironmentEdgeManager.currentTime() > previousTimestamp + 1);
088        Path walRootDir = CommonFSUtils.getWALRootDir(conf1);
089        Path archiveDir = new Path(walRootDir,
090          AbstractFSWALProvider.getWALArchiveDirectoryName(testConf, serverName.toString()));
091        String walName = (separateOldLogDir ? "wal" : serverName.toString())
092          + BackupUtils.LOGNAME_SEPARATOR + (previousTimestamp + 1);
093        Path archivedWAL = new Path(archiveDir, walName);
094        FileSystem fs = walRootDir.getFileSystem(conf1);
095        fs.mkdirs(archiveDir);
096        fs.create(archivedWAL).close();
097
098        manager.getIncrBackupLogFileMap();
099
100        assertTrue(backupInfo.getIncrBackupFileList().contains(archivedWAL.toString()));
101      }
102    }
103  }
104}