001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.backup;
019
020import static org.junit.Assert.assertFalse;
021import static org.junit.Assert.assertTrue;
022
023import java.io.File;
024import java.util.List;
025import org.apache.hadoop.hbase.HBaseClassTestRule;
026import org.apache.hadoop.hbase.TableName;
027import org.apache.hadoop.hbase.backup.impl.BackupAdminImpl;
028import org.apache.hadoop.hbase.backup.util.BackupUtils;
029import org.apache.hadoop.hbase.client.Admin;
030import org.apache.hadoop.hbase.client.Connection;
031import org.apache.hadoop.hbase.client.ConnectionFactory;
032import org.apache.hadoop.hbase.client.Table;
033import org.apache.hadoop.hbase.testclassification.LargeTests;
034import org.junit.Assert;
035import org.junit.ClassRule;
036import org.junit.Test;
037import org.junit.experimental.categories.Category;
038import org.slf4j.Logger;
039import org.slf4j.LoggerFactory;
040
041import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
042
043@Category(LargeTests.class)
044public class TestBackupMerge extends TestBackupBase {
045
046  @ClassRule
047  public static final HBaseClassTestRule CLASS_RULE =
048    HBaseClassTestRule.forClass(TestBackupMerge.class);
049
050  private static final Logger LOG = LoggerFactory.getLogger(TestBackupMerge.class);
051
052  @Test
053  public void TestIncBackupMergeRestore() throws Exception {
054    int ADD_ROWS = 99;
055    // #1 - create full backup for all tables
056    LOG.info("create full backup image for all tables");
057
058    List<TableName> tables = Lists.newArrayList(table1, table2);
059    // Set custom Merge Job implementation
060
061    Connection conn = ConnectionFactory.createConnection(conf1);
062
063    Admin admin = conn.getAdmin();
064    BackupAdminImpl client = new BackupAdminImpl(conn);
065
066    BackupRequest request = createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR);
067    String backupIdFull = client.backupTables(request);
068
069    assertTrue(checkSucceeded(backupIdFull));
070
071    // #2 - insert some data to table1
072    Table t1 = insertIntoTable(conn, table1, famName, 1, ADD_ROWS);
073    LOG.debug("writing " + ADD_ROWS + " rows to " + table1);
074
075    Assert.assertEquals(TEST_UTIL.countRows(t1), NB_ROWS_IN_BATCH + ADD_ROWS);
076    t1.close();
077    LOG.debug("written " + ADD_ROWS + " rows to " + table1);
078
079    Table t2 = insertIntoTable(conn, table2, famName, 1, ADD_ROWS);
080
081    Assert.assertEquals(TEST_UTIL.countRows(t2), NB_ROWS_IN_BATCH + ADD_ROWS);
082    t2.close();
083    LOG.debug("written " + ADD_ROWS + " rows to " + table2);
084
085    // #3 - incremental backup for multiple tables
086    tables = Lists.newArrayList(table1, table2);
087    request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR);
088    String backupIdIncMultiple = client.backupTables(request);
089
090    assertTrue(checkSucceeded(backupIdIncMultiple));
091
092    t1 = insertIntoTable(conn, table1, famName, 2, ADD_ROWS);
093    t1.close();
094
095    t2 = insertIntoTable(conn, table2, famName, 2, ADD_ROWS);
096    t2.close();
097
098    // #3 - incremental backup for multiple tables
099    request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR);
100    String backupIdIncMultiple2 = client.backupTables(request);
101    assertTrue(checkSucceeded(backupIdIncMultiple2));
102
103    try (BackupAdmin bAdmin = new BackupAdminImpl(conn)) {
104      String[] backups = new String[] { backupIdIncMultiple, backupIdIncMultiple2 };
105      bAdmin.mergeBackups(backups);
106    }
107
108    // #6 - restore incremental backup for multiple tables, with overwrite
109    TableName[] tablesRestoreIncMultiple = new TableName[] { table1, table2 };
110    TableName[] tablesMapIncMultiple = new TableName[] { table1_restore, table2_restore };
111    client.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, backupIdIncMultiple2, false,
112      tablesRestoreIncMultiple, tablesMapIncMultiple, true));
113
114    Table hTable = conn.getTable(table1_restore);
115    LOG.debug("After incremental restore: " + hTable.getDescriptor());
116    int countRows = TEST_UTIL.countRows(hTable, famName);
117    LOG.debug("f1 has " + countRows + " rows");
118    Assert.assertEquals(NB_ROWS_IN_BATCH + 2 * ADD_ROWS, countRows);
119
120    hTable.close();
121
122    hTable = conn.getTable(table2_restore);
123    Assert.assertEquals(TEST_UTIL.countRows(hTable), NB_ROWS_IN_BATCH + 2 * ADD_ROWS);
124    hTable.close();
125
126    admin.close();
127    conn.close();
128  }
129
130  @Test
131  public void testIncBackupMergeRestoreSeparateFs() throws Exception {
132    String originalBackupRoot = BACKUP_ROOT_DIR;
133    // prepare BACKUP_ROOT_DIR on a different filesystem from HBase.
134    String backupTargetDir = TEST_UTIL.getDataTestDir("backupTarget").toString();
135    BACKUP_ROOT_DIR = new File(backupTargetDir).toURI().toString();
136
137    try (Connection conn = ConnectionFactory.createConnection(conf1)) {
138      BackupAdminImpl client = new BackupAdminImpl(conn);
139      List<TableName> tables = Lists.newArrayList(table1, table2);
140
141      BackupRequest request = createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR, true);
142      String backupIdFull = client.backupTables(request);
143      assertTrue(checkSucceeded(backupIdFull));
144
145      request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR, true);
146      String backupIdIncMultiple = client.backupTables(request);
147      assertTrue(checkSucceeded(backupIdIncMultiple));
148
149      request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR, true);
150      String backupIdIncMultiple2 = client.backupTables(request);
151      assertTrue(checkSucceeded(backupIdIncMultiple2));
152
153      try (BackupAdmin bAdmin = new BackupAdminImpl(conn)) {
154        String[] backups = new String[] { backupIdIncMultiple, backupIdIncMultiple2 };
155        // this throws java.lang.IllegalArgumentException: Wrong FS prior to HBASE-28539
156        bAdmin.mergeBackups(backups);
157      }
158
159      assertTrue(
160        new File(HBackupFileSystem.getBackupPath(BACKUP_ROOT_DIR, backupIdFull).toUri()).exists());
161      assertFalse(
162        new File(HBackupFileSystem.getBackupPath(BACKUP_ROOT_DIR, backupIdIncMultiple).toUri())
163          .exists());
164      assertTrue(
165        new File(HBackupFileSystem.getBackupPath(BACKUP_ROOT_DIR, backupIdIncMultiple2).toUri())
166          .exists());
167    } finally {
168      BACKUP_ROOT_DIR = originalBackupRoot;
169    }
170  }
171}