001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.backup;
019
020import static org.apache.hadoop.hbase.HConstants.REPLICATION_BULKLOAD_ENABLE_KEY;
021import static org.apache.hadoop.hbase.HConstants.REPLICATION_CLUSTER_ID;
022import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.CONTINUOUS_BACKUP_REPLICATION_PEER;
023import static org.apache.hadoop.hbase.mapreduce.WALPlayer.IGNORE_EMPTY_FILES;
024
025import java.io.IOException;
026import java.util.ArrayList;
027import java.util.Arrays;
028import java.util.HashMap;
029import java.util.Iterator;
030import java.util.List;
031import java.util.Map;
032import java.util.Map.Entry;
033import java.util.Objects;
034import org.apache.hadoop.conf.Configuration;
035import org.apache.hadoop.fs.FileStatus;
036import org.apache.hadoop.fs.FileSystem;
037import org.apache.hadoop.fs.LocatedFileStatus;
038import org.apache.hadoop.fs.Path;
039import org.apache.hadoop.fs.RemoteIterator;
040import org.apache.hadoop.hbase.HBaseConfiguration;
041import org.apache.hadoop.hbase.HBaseTestingUtil;
042import org.apache.hadoop.hbase.HConstants;
043import org.apache.hadoop.hbase.NamespaceDescriptor;
044import org.apache.hadoop.hbase.TableName;
045import org.apache.hadoop.hbase.backup.BackupInfo.BackupPhase;
046import org.apache.hadoop.hbase.backup.BackupInfo.BackupState;
047import org.apache.hadoop.hbase.backup.impl.BackupAdminImpl;
048import org.apache.hadoop.hbase.backup.impl.BackupManager;
049import org.apache.hadoop.hbase.backup.impl.BackupManifest;
050import org.apache.hadoop.hbase.backup.impl.BackupSystemTable;
051import org.apache.hadoop.hbase.backup.impl.FullTableBackupClient;
052import org.apache.hadoop.hbase.backup.impl.IncrementalBackupManager;
053import org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient;
054import org.apache.hadoop.hbase.backup.util.BackupUtils;
055import org.apache.hadoop.hbase.client.Admin;
056import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
057import org.apache.hadoop.hbase.client.Connection;
058import org.apache.hadoop.hbase.client.ConnectionFactory;
059import org.apache.hadoop.hbase.client.Durability;
060import org.apache.hadoop.hbase.client.Put;
061import org.apache.hadoop.hbase.client.Table;
062import org.apache.hadoop.hbase.client.TableDescriptor;
063import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
064import org.apache.hadoop.hbase.master.cleaner.LogCleaner;
065import org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner;
066import org.apache.hadoop.hbase.regionserver.LogRoller;
067import org.apache.hadoop.hbase.security.HadoopSecurityEnabledUserProviderForTesting;
068import org.apache.hadoop.hbase.security.UserProvider;
069import org.apache.hadoop.hbase.security.access.SecureTestUtil;
070import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
071import org.apache.hadoop.hbase.util.Bytes;
072import org.apache.hadoop.hbase.util.CommonFSUtils;
073import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
074import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
075import org.apache.hadoop.hbase.wal.WALFactory;
076import org.junit.jupiter.api.AfterAll;
077import org.junit.jupiter.api.BeforeAll;
078import org.junit.jupiter.api.BeforeEach;
079import org.slf4j.Logger;
080import org.slf4j.LoggerFactory;
081
082/**
083 * This class is only a base for other integration-level backup tests. Do not add tests here.
084 * TestBackupSmallTests is where tests that don't require bring machines up/down should go All other
085 * tests should have their own classes and extend this one
086 */
087public class TestBackupBase {
088  private static final Logger LOG = LoggerFactory.getLogger(TestBackupBase.class);
089
090  protected static HBaseTestingUtil TEST_UTIL;
091  protected static HBaseTestingUtil TEST_UTIL2;
092  protected static Configuration conf1;
093  protected static Configuration conf2;
094
095  protected static TableName table1 = TableName.valueOf("table1");
096  protected static TableDescriptor table1Desc;
097  protected static TableName table2 = TableName.valueOf("table2");
098  protected static TableName table3 = TableName.valueOf("table3");
099  protected static TableName table4 = TableName.valueOf("table4");
100
101  protected static TableName table1_restore = TableName.valueOf("default:table1");
102  protected static TableName table2_restore = TableName.valueOf("ns2:table2");
103  protected static TableName table3_restore = TableName.valueOf("ns3:table3_restore");
104
105  protected static final int NB_ROWS_IN_BATCH = 99;
106  protected static final byte[] qualName = Bytes.toBytes("q1");
107  protected static final byte[] famName = Bytes.toBytes("f");
108
109  protected static String BACKUP_ROOT_DIR;
110  protected static String BACKUP_REMOTE_ROOT_DIR;
111  protected static String provider = "defaultProvider";
112  protected static boolean secure = false;
113
114  protected static boolean autoRestoreOnFailure;
115  protected static boolean useSecondCluster;
116
117  static class IncrementalTableBackupClientForTest extends IncrementalTableBackupClient {
118    public IncrementalTableBackupClientForTest() {
119    }
120
121    public IncrementalTableBackupClientForTest(Connection conn, String backupId,
122      BackupRequest request) throws IOException {
123      super(conn, backupId, request);
124    }
125
126    @BeforeEach
127    public void ensurePreviousBackupTestsAreCleanedUp() throws Exception {
128      // Every operation here may not be necessary for any given test,
129      // some often being no-ops. the goal is to help ensure atomicity
130      // of that tests that implement TestBackupBase
131      try (BackupAdmin backupAdmin = getBackupAdmin()) {
132        backupManager.finishBackupSession();
133        backupAdmin.listBackupSets().forEach(backupSet -> {
134          try {
135            backupAdmin.deleteBackupSet(backupSet.getName());
136          } catch (IOException ignored) {
137          }
138        });
139      } catch (Exception ignored) {
140      }
141      Arrays.stream(TEST_UTIL.getAdmin().listTableNames())
142        .filter(tableName -> !tableName.isSystemTable()).forEach(tableName -> {
143          try {
144            TEST_UTIL.truncateTable(tableName);
145          } catch (IOException ignored) {
146          }
147        });
148      TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads().forEach(rst -> {
149        try {
150          LogRoller walRoller = rst.getRegionServer().getWalRoller();
151          walRoller.requestRollAll();
152          walRoller.waitUntilWalRollFinished();
153        } catch (Exception ignored) {
154        }
155      });
156    }
157
158    @Override
159    public void execute() throws IOException {
160      // case INCREMENTAL_COPY:
161      try {
162        // case PREPARE_INCREMENTAL:
163        failStageIf(Stage.stage_0);
164        beginBackup(backupManager, backupInfo);
165
166        failStageIf(Stage.stage_1);
167        backupInfo.setPhase(BackupPhase.PREPARE_INCREMENTAL);
168        LOG.debug("For incremental backup, current table set is "
169          + backupManager.getIncrementalBackupTableSet());
170        newTimestamps = ((IncrementalBackupManager) backupManager).getIncrBackupLogFileMap();
171        // copy out the table and region info files for each table
172        BackupUtils.copyTableRegionInfo(conn, backupInfo, conf);
173        // convert WAL to HFiles and copy them to .tmp under BACKUP_ROOT
174        convertWALsToHFiles(new HashMap<>(), new HashMap<>());
175        incrementalCopyHFiles(new String[] { getBulkOutputDir().toString() },
176          backupInfo.getBackupRootDir());
177        failStageIf(Stage.stage_2);
178
179        // case INCR_BACKUP_COMPLETE:
180        // set overall backup status: complete. Here we make sure to complete the backup.
181        // After this checkpoint, even if entering cancel process, will let the backup finished
182        // Set the previousTimestampMap which is before this current log roll to the manifest.
183        Map<TableName, Map<String, Long>> previousTimestampMap =
184          backupManager.readLogTimestampMap();
185        backupInfo.setIncrTimestampMap(previousTimestampMap);
186
187        // The table list in backupInfo is good for both full backup and incremental backup.
188        // For incremental backup, it contains the incremental backup table set.
189        backupManager.writeRegionServerLogTimestamp(backupInfo.getTables(), newTimestamps);
190        failStageIf(Stage.stage_3);
191
192        Map<TableName, Map<String, Long>> newTableSetTimestampMap =
193          backupManager.readLogTimestampMap();
194
195        handleBulkLoad(backupInfo.getTableNames(), new HashMap<>(), new HashMap<>());
196        failStageIf(Stage.stage_4);
197
198        // backup complete
199        backupInfo.setTableSetTimestampMap(newTableSetTimestampMap);
200        completeBackup(conn, backupInfo, BackupType.INCREMENTAL, conf);
201
202      } catch (Exception e) {
203        failBackup(conn, backupInfo, backupManager, e, "Unexpected Exception : ",
204          BackupType.INCREMENTAL, conf);
205        throw new IOException(e);
206      }
207    }
208  }
209
210  static class FullTableBackupClientForTest extends FullTableBackupClient {
211    public FullTableBackupClientForTest() {
212    }
213
214    public FullTableBackupClientForTest(Connection conn, String backupId, BackupRequest request)
215      throws IOException {
216      super(conn, backupId, request);
217    }
218
219    @Override
220    public void execute() throws IOException {
221      // Get the stage ID to fail on
222      try (Admin admin = conn.getAdmin()) {
223        // Begin BACKUP
224        beginBackup(backupManager, backupInfo);
225        failStageIf(Stage.stage_0);
226        // do snapshot for full table backup
227        failStageIf(Stage.stage_1);
228        // We roll log here before we do the snapshot. It is possible there is duplicate data
229        // in the log that is already in the snapshot. But if we do it after the snapshot, we
230        // could have data loss.
231        // A better approach is to do the roll log on each RS in the same global procedure as
232        // the snapshot.
233        LOG.info("Execute roll log procedure for full backup ...");
234
235        BackupUtils.logRoll(conn, backupInfo.getBackupRootDir(), conf);
236        failStageIf(Stage.stage_2);
237        newTimestamps = backupManager.readRegionServerLastLogRollResult();
238
239        // SNAPSHOT_TABLES:
240        backupInfo.setPhase(BackupPhase.SNAPSHOT);
241        for (TableName tableName : tableList) {
242          String snapshotName = "snapshot_" + Long.toString(EnvironmentEdgeManager.currentTime())
243            + "_" + tableName.getNamespaceAsString() + "_" + tableName.getQualifierAsString();
244
245          snapshotTable(admin, tableName, snapshotName);
246          backupInfo.setSnapshotName(tableName, snapshotName);
247        }
248        failStageIf(Stage.stage_3);
249        // SNAPSHOT_COPY:
250        // do snapshot copy
251        LOG.debug("snapshot copy for " + backupId);
252        snapshotCopy(backupInfo);
253        // Updates incremental backup table set
254        backupManager.addIncrementalBackupTableSet(backupInfo.getTables());
255
256        // BACKUP_COMPLETE:
257        // set overall backup status: complete. Here we make sure to complete the backup.
258        // After this checkpoint, even if entering cancel process, will let the backup finished
259        backupInfo.setState(BackupState.COMPLETE);
260        // The table list in backupInfo is good for both full backup and incremental backup.
261        // For incremental backup, it contains the incremental backup table set.
262        backupManager.writeRegionServerLogTimestamp(backupInfo.getTables(), newTimestamps);
263
264        Map<TableName, Map<String, Long>> newTableSetTimestampMap =
265          backupManager.readLogTimestampMap();
266
267        failStageIf(Stage.stage_4);
268        // backup complete
269        backupInfo.setTableSetTimestampMap(newTableSetTimestampMap);
270        completeBackup(conn, backupInfo, BackupType.FULL, conf);
271
272      } catch (Exception e) {
273
274        if (autoRestoreOnFailure) {
275          failBackup(conn, backupInfo, backupManager, e, "Unexpected BackupException : ",
276            BackupType.FULL, conf);
277        }
278        throw new IOException(e);
279      }
280    }
281  }
282
283  public static void setUpHelper() throws Exception {
284    BACKUP_ROOT_DIR = Path.SEPARATOR + "backupUT";
285    BACKUP_REMOTE_ROOT_DIR = Path.SEPARATOR + "backupUT";
286    conf1.setBoolean(REPLICATION_BULKLOAD_ENABLE_KEY, true);
287    conf1.set(REPLICATION_CLUSTER_ID, "clusterId1");
288    conf1.setBoolean(IGNORE_EMPTY_FILES, true);
289
290    if (secure) {
291      // set the always on security provider
292      UserProvider.setUserProviderForTesting(TEST_UTIL.getConfiguration(),
293        HadoopSecurityEnabledUserProviderForTesting.class);
294      // setup configuration
295      SecureTestUtil.enableSecurity(TEST_UTIL.getConfiguration());
296    }
297    conf1.setBoolean(BackupRestoreConstants.BACKUP_ENABLE_KEY, true);
298    BackupManager.decorateMasterConfiguration(conf1);
299    BackupManager.decorateRegionServerConfiguration(conf1);
300    conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
301    // Set TTL for old WALs to 1 sec to enforce fast cleaning of an archived
302    // WAL files
303    conf1.setLong(TimeToLiveLogCleaner.TTL_CONF_KEY, 1000);
304    conf1.setLong(LogCleaner.OLD_WALS_CLEANER_THREAD_TIMEOUT_MSEC, 1000);
305
306    // Set MultiWAL (with 2 default WAL files per RS)
307    conf1.set(WALFactory.WAL_PROVIDER, provider);
308    TEST_UTIL.startMiniCluster();
309    conf1 = TEST_UTIL.getConfiguration();
310    TEST_UTIL.startMiniMapReduceCluster();
311
312    if (useSecondCluster) {
313      conf2 = HBaseConfiguration.create(conf1);
314      conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2");
315      TEST_UTIL2 = new HBaseTestingUtil(conf2);
316      TEST_UTIL2.setZkCluster(TEST_UTIL.getZkCluster());
317      TEST_UTIL2.startMiniDFSCluster(3);
318      String root2 = TEST_UTIL2.getConfiguration().get("fs.defaultFS");
319      Path p = new Path(new Path(root2), "/tmp/wal");
320      CommonFSUtils.setWALRootDir(TEST_UTIL2.getConfiguration(), p);
321      TEST_UTIL2.startMiniCluster();
322    }
323
324    BACKUP_ROOT_DIR =
325      new Path(new Path(TEST_UTIL.getConfiguration().get("fs.defaultFS")), BACKUP_ROOT_DIR)
326        .toString();
327    LOG.info("ROOTDIR " + BACKUP_ROOT_DIR);
328    if (useSecondCluster) {
329      BACKUP_REMOTE_ROOT_DIR = new Path(
330        new Path(TEST_UTIL2.getConfiguration().get("fs.defaultFS")) + BACKUP_REMOTE_ROOT_DIR)
331        .toString();
332      LOG.info("REMOTE ROOTDIR " + BACKUP_REMOTE_ROOT_DIR);
333    }
334    createTables();
335    populateFromMasterConfig(TEST_UTIL.getHBaseCluster().getMaster().getConfiguration(), conf1);
336  }
337
338  /**
339   * Setup Cluster with appropriate configurations before running tests.
340   * @throws Exception if starting the mini cluster or setting up the tables fails
341   */
342  @BeforeAll
343  public static void setUp() throws Exception {
344    TEST_UTIL = new HBaseTestingUtil();
345    conf1 = TEST_UTIL.getConfiguration();
346    autoRestoreOnFailure = true;
347    useSecondCluster = false;
348    setUpHelper();
349  }
350
351  private static void populateFromMasterConfig(Configuration masterConf, Configuration conf) {
352    Iterator<Entry<String, String>> it = masterConf.iterator();
353    while (it.hasNext()) {
354      Entry<String, String> e = it.next();
355      conf.set(e.getKey(), e.getValue());
356    }
357  }
358
359  @AfterAll
360  public static void tearDown() throws Exception {
361    try {
362      SnapshotTestingUtils.deleteAllSnapshots(TEST_UTIL.getAdmin());
363    } catch (Exception e) {
364    }
365    SnapshotTestingUtils.deleteArchiveDirectory(TEST_UTIL);
366    if (useSecondCluster) {
367      TEST_UTIL2.shutdownMiniCluster();
368    }
369    TEST_UTIL.shutdownMiniCluster();
370    TEST_UTIL.shutdownMiniMapReduceCluster();
371    autoRestoreOnFailure = true;
372    useSecondCluster = false;
373  }
374
375  Table insertIntoTable(Connection conn, TableName table, byte[] family, int id, int numRows)
376    throws IOException {
377    Table t = conn.getTable(table);
378    Put p1;
379    for (int i = 0; i < numRows; i++) {
380      p1 = new Put(Bytes.toBytes("row-" + table + "-" + id + "-" + i));
381      p1.addColumn(family, qualName, Bytes.toBytes("val" + i));
382      t.put(p1);
383    }
384    return t;
385  }
386
387  protected BackupRequest createBackupRequest(BackupType type, List<TableName> tables,
388    String path) {
389    return createBackupRequest(type, tables, path, false);
390  }
391
392  protected BackupRequest createBackupRequest(BackupType type, List<TableName> tables, String path,
393    boolean noChecksumVerify) {
394    BackupRequest.Builder builder = new BackupRequest.Builder();
395    BackupRequest request = builder.withBackupType(type).withTableList(tables)
396      .withTargetRootDir(path).withNoChecksumVerify(noChecksumVerify).build();
397    return request;
398  }
399
400  protected BackupRequest createBackupRequest(BackupType type, List<TableName> tables,
401    String rootDir, boolean noChecksumVerify, boolean isContinuousBackupEnabled) {
402    BackupRequest.Builder builder = new BackupRequest.Builder();
403    return builder.withBackupType(type).withTableList(tables).withTargetRootDir(rootDir)
404      .withNoChecksumVerify(noChecksumVerify).withContinuousBackupEnabled(isContinuousBackupEnabled)
405      .build();
406  }
407
408  protected String backupTables(BackupType type, List<TableName> tables, String path)
409    throws IOException {
410    return backupTables(type, tables, path, false);
411  }
412
413  protected String backupTables(BackupType type, List<TableName> tables, String path,
414    boolean isContinuousBackup) throws IOException {
415    Connection conn = null;
416    BackupAdmin badmin = null;
417    String backupId;
418    try {
419      conn = ConnectionFactory.createConnection(conf1);
420      badmin = new BackupAdminImpl(conn);
421
422      BackupRequest request =
423        createBackupRequest(type, new ArrayList<>(tables), path, false, isContinuousBackup);
424      backupId = badmin.backupTables(request);
425    } finally {
426      if (badmin != null) {
427        badmin.close();
428      }
429      if (conn != null) {
430        conn.close();
431      }
432    }
433    return backupId;
434  }
435
436  protected String fullTableBackup(List<TableName> tables) throws IOException {
437    return backupTables(BackupType.FULL, tables, BACKUP_ROOT_DIR);
438  }
439
440  protected String incrementalTableBackup(List<TableName> tables) throws IOException {
441    return backupTables(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR);
442  }
443
444  protected static void loadTable(Table table) throws Exception {
445    Put p; // 100 + 1 row to t1_syncup
446    for (int i = 0; i < NB_ROWS_IN_BATCH; i++) {
447      p = new Put(Bytes.toBytes("row" + i));
448      p.setDurability(Durability.SKIP_WAL);
449      p.addColumn(famName, qualName, Bytes.toBytes("val" + i));
450      table.put(p);
451    }
452  }
453
454  protected static void createTables() throws Exception {
455    long tid = EnvironmentEdgeManager.currentTime();
456    table1 = TableName.valueOf("test-" + tid);
457    Admin ha = TEST_UTIL.getAdmin();
458
459    // Create namespaces
460    ha.createNamespace(NamespaceDescriptor.create("ns1").build());
461    ha.createNamespace(NamespaceDescriptor.create("ns2").build());
462    ha.createNamespace(NamespaceDescriptor.create("ns3").build());
463    ha.createNamespace(NamespaceDescriptor.create("ns4").build());
464
465    TableDescriptor desc = TableDescriptorBuilder.newBuilder(table1)
466      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(famName)).build();
467    ha.createTable(desc);
468    table1Desc = desc;
469    Connection conn = ConnectionFactory.createConnection(conf1);
470    Table table = conn.getTable(table1);
471    loadTable(table);
472    table.close();
473    table2 = TableName.valueOf("ns2:test-" + tid + 1);
474    desc = TableDescriptorBuilder.newBuilder(table2)
475      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(famName)).build();
476    ha.createTable(desc);
477    table = conn.getTable(table2);
478    loadTable(table);
479    table.close();
480    table3 = TableName.valueOf("ns3:test-" + tid + 2);
481    table = TEST_UTIL.createTable(table3, famName);
482    table.close();
483    table4 = TableName.valueOf("ns4:test-" + tid + 3);
484    table = TEST_UTIL.createTable(table4, famName);
485    table.close();
486    ha.close();
487    conn.close();
488  }
489
490  protected boolean checkSucceeded(String backupId) throws IOException {
491    BackupInfo status = getBackupInfo(backupId);
492
493    if (status == null) {
494      return false;
495    }
496
497    return status.getState() == BackupState.COMPLETE;
498  }
499
500  protected boolean checkFailed(String backupId) throws IOException {
501    BackupInfo status = getBackupInfo(backupId);
502
503    if (status == null) {
504      return false;
505    }
506
507    return status.getState() == BackupState.FAILED;
508  }
509
510  private BackupInfo getBackupInfo(String backupId) throws IOException {
511    try (BackupSystemTable table = new BackupSystemTable(TEST_UTIL.getConnection())) {
512      BackupInfo status = table.readBackupInfo(backupId);
513      return status;
514    }
515  }
516
517  protected static BackupAdmin getBackupAdmin() throws IOException {
518    return new BackupAdminImpl(TEST_UTIL.getConnection());
519  }
520
521  /**
522   * Helper method
523   */
524  protected List<TableName> toList(String... args) {
525    List<TableName> ret = new ArrayList<>();
526    for (int i = 0; i < args.length; i++) {
527      ret.add(TableName.valueOf(args[i]));
528    }
529    return ret;
530  }
531
532  protected List<FileStatus> getListOfWALFiles(Configuration c) throws IOException {
533    Path logRoot = new Path(CommonFSUtils.getWALRootDir(c), HConstants.HREGION_LOGDIR_NAME);
534    FileSystem fs = logRoot.getFileSystem(c);
535    RemoteIterator<LocatedFileStatus> it = fs.listFiles(logRoot, true);
536    List<FileStatus> logFiles = new ArrayList<FileStatus>();
537    while (it.hasNext()) {
538      LocatedFileStatus lfs = it.next();
539      if (lfs.isFile() && !AbstractFSWALProvider.isMetaFile(lfs.getPath())) {
540        logFiles.add(lfs);
541        LOG.info(Objects.toString(lfs));
542      }
543    }
544    return logFiles;
545  }
546
547  protected void dumpBackupDir() throws IOException {
548    // Dump Backup Dir
549    FileSystem fs = FileSystem.get(conf1);
550    RemoteIterator<LocatedFileStatus> it = fs.listFiles(new Path(BACKUP_ROOT_DIR), true);
551    while (it.hasNext()) {
552      LOG.debug(Objects.toString(it.next().getPath()));
553    }
554  }
555
556  BackupManifest getLatestBackupManifest(List<BackupInfo> backups) throws IOException {
557    BackupInfo newestBackup = backups.get(0);
558    return HBackupFileSystem.getManifest(conf1, new Path(BACKUP_ROOT_DIR),
559      newestBackup.getBackupId());
560  }
561
562  void deleteContinuousBackupReplicationPeerIfExists(Admin admin) throws IOException {
563    if (
564      admin.listReplicationPeers().stream()
565        .anyMatch(peer -> peer.getPeerId().equals(CONTINUOUS_BACKUP_REPLICATION_PEER))
566    ) {
567      admin.disableReplicationPeer(CONTINUOUS_BACKUP_REPLICATION_PEER);
568      admin.removeReplicationPeer(CONTINUOUS_BACKUP_REPLICATION_PEER);
569    }
570  }
571}