001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.backup;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertFalse;
022import static org.junit.jupiter.api.Assertions.assertTrue;
023
024import java.io.File;
025import java.util.List;
026import org.apache.hadoop.hbase.HBaseTestingUtil;
027import org.apache.hadoop.hbase.TableName;
028import org.apache.hadoop.hbase.backup.impl.BackupAdminImpl;
029import org.apache.hadoop.hbase.backup.util.BackupUtils;
030import org.apache.hadoop.hbase.client.Admin;
031import org.apache.hadoop.hbase.client.Connection;
032import org.apache.hadoop.hbase.client.ConnectionFactory;
033import org.apache.hadoop.hbase.client.Table;
034import org.apache.hadoop.hbase.testclassification.LargeTests;
035import org.junit.jupiter.api.Tag;
036import org.junit.jupiter.api.Test;
037import org.slf4j.Logger;
038import org.slf4j.LoggerFactory;
039
040import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
041
042@Tag(LargeTests.TAG)
043public class TestBackupMerge extends TestBackupBase {
044
045  private static final Logger LOG = LoggerFactory.getLogger(TestBackupMerge.class);
046
047  @Test
048  public void TestIncBackupMergeRestore() throws Exception {
049    int ADD_ROWS = 99;
050    // #1 - create full backup for all tables
051    LOG.info("create full backup image for all tables");
052
053    List<TableName> tables = Lists.newArrayList(table1, table2);
054    // Set custom Merge Job implementation
055
056    Connection conn = ConnectionFactory.createConnection(conf1);
057
058    Admin admin = conn.getAdmin();
059    BackupAdminImpl client = new BackupAdminImpl(conn);
060
061    BackupRequest request = createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR);
062    String backupIdFull = client.backupTables(request);
063
064    assertTrue(checkSucceeded(backupIdFull));
065
066    // #2 - insert some data to table1
067    Table t1 = insertIntoTable(conn, table1, famName, 1, ADD_ROWS);
068    LOG.debug("writing {} rows to {}", ADD_ROWS, table1);
069
070    assertEquals(NB_ROWS_IN_BATCH + ADD_ROWS, HBaseTestingUtil.countRows(t1));
071    t1.close();
072    LOG.debug("written {} rows to {}", ADD_ROWS, table1);
073
074    Table t2 = insertIntoTable(conn, table2, famName, 1, ADD_ROWS);
075
076    assertEquals(NB_ROWS_IN_BATCH + ADD_ROWS, HBaseTestingUtil.countRows(t2));
077    t2.close();
078    LOG.debug("written {} rows to {}", ADD_ROWS, table2);
079
080    // #3 - incremental backup for multiple tables
081    tables = Lists.newArrayList(table1, table2);
082    request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR);
083    String backupIdIncMultiple = client.backupTables(request);
084
085    assertTrue(checkSucceeded(backupIdIncMultiple));
086
087    t1 = insertIntoTable(conn, table1, famName, 2, ADD_ROWS);
088    t1.close();
089
090    t2 = insertIntoTable(conn, table2, famName, 2, ADD_ROWS);
091    t2.close();
092
093    // #3 - incremental backup for multiple tables
094    request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR);
095    String backupIdIncMultiple2 = client.backupTables(request);
096    assertTrue(checkSucceeded(backupIdIncMultiple2));
097
098    try (BackupAdmin bAdmin = new BackupAdminImpl(conn)) {
099      String[] backups = new String[] { backupIdIncMultiple, backupIdIncMultiple2 };
100      bAdmin.mergeBackups(backups);
101    }
102
103    // #6 - restore incremental backup for multiple tables, with overwrite
104    TableName[] tablesRestoreIncMultiple = new TableName[] { table1, table2 };
105    TableName[] tablesMapIncMultiple = new TableName[] { table1_restore, table2_restore };
106    client.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, backupIdIncMultiple2, false,
107      tablesRestoreIncMultiple, tablesMapIncMultiple, true));
108
109    Table hTable = conn.getTable(table1_restore);
110    LOG.debug("After incremental restore: {}", hTable.getDescriptor());
111    int countRows = HBaseTestingUtil.countRows(hTable, famName);
112    LOG.debug("f1 has {} rows", countRows);
113    assertEquals(NB_ROWS_IN_BATCH + 2 * ADD_ROWS, countRows);
114
115    hTable.close();
116
117    hTable = conn.getTable(table2_restore);
118    assertEquals(NB_ROWS_IN_BATCH + 2 * ADD_ROWS, HBaseTestingUtil.countRows(hTable));
119    hTable.close();
120
121    admin.close();
122    conn.close();
123  }
124
125  @Test
126  public void testIncBackupMergeRestoreSeparateFs() throws Exception {
127    String originalBackupRoot = BACKUP_ROOT_DIR;
128    // prepare BACKUP_ROOT_DIR on a different filesystem from HBase.
129    String backupTargetDir = TEST_UTIL.getDataTestDir("backupTarget").toString();
130    BACKUP_ROOT_DIR = new File(backupTargetDir).toURI().toString();
131
132    try (Connection conn = ConnectionFactory.createConnection(conf1)) {
133      BackupAdminImpl client = new BackupAdminImpl(conn);
134      List<TableName> tables = Lists.newArrayList(table1, table2);
135
136      BackupRequest request = createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR, true);
137      String backupIdFull = client.backupTables(request);
138      assertTrue(checkSucceeded(backupIdFull));
139
140      request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR, true);
141      String backupIdIncMultiple = client.backupTables(request);
142      assertTrue(checkSucceeded(backupIdIncMultiple));
143
144      request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR, true);
145      String backupIdIncMultiple2 = client.backupTables(request);
146      assertTrue(checkSucceeded(backupIdIncMultiple2));
147
148      try (BackupAdmin bAdmin = new BackupAdminImpl(conn)) {
149        String[] backups = new String[] { backupIdIncMultiple, backupIdIncMultiple2 };
150        // this throws java.lang.IllegalArgumentException: Wrong FS prior to HBASE-28539
151        bAdmin.mergeBackups(backups);
152      }
153
154      assertTrue(
155        new File(HBackupFileSystem.getBackupPath(BACKUP_ROOT_DIR, backupIdFull).toUri()).exists());
156      assertFalse(
157        new File(HBackupFileSystem.getBackupPath(BACKUP_ROOT_DIR, backupIdIncMultiple).toUri())
158          .exists());
159      assertTrue(
160        new File(HBackupFileSystem.getBackupPath(BACKUP_ROOT_DIR, backupIdIncMultiple2).toUri())
161          .exists());
162    } finally {
163      BACKUP_ROOT_DIR = originalBackupRoot;
164    }
165  }
166}