001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.backup;
019
020import static org.junit.jupiter.api.Assertions.assertNull;
021import static org.junit.jupiter.api.Assertions.assertTrue;
022
023import java.io.IOException;
024import java.util.ArrayList;
025import java.util.List;
026import java.util.Optional;
027import org.apache.hadoop.fs.FileSystem;
028import org.apache.hadoop.fs.Path;
029import org.apache.hadoop.hbase.HBaseTestingUtil;
030import org.apache.hadoop.hbase.HConstants;
031import org.apache.hadoop.hbase.TableName;
032import org.apache.hadoop.hbase.backup.impl.BackupSystemTable;
033import org.apache.hadoop.hbase.client.Admin;
034import org.apache.hadoop.hbase.client.Connection;
035import org.apache.hadoop.hbase.client.SnapshotDescription;
036import org.apache.hadoop.hbase.client.TableDescriptor;
037import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
038import org.apache.hadoop.hbase.coprocessor.MasterCoprocessor;
039import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
040import org.apache.hadoop.hbase.coprocessor.MasterObserver;
041import org.apache.hadoop.hbase.coprocessor.ObserverContext;
042import org.apache.hadoop.hbase.testclassification.LargeTests;
043import org.apache.hadoop.util.ToolRunner;
044import org.junit.jupiter.api.BeforeAll;
045import org.junit.jupiter.api.Tag;
046import org.junit.jupiter.api.Test;
047import org.slf4j.Logger;
048import org.slf4j.LoggerFactory;
049
050import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
051
052/**
053 * This class is only a base for other integration-level backup tests. Do not add tests here.
054 * TestBackupSmallTests is where tests that don't require bring machines up/down should go All other
055 * tests should have their own classes and extend this one
056 */
057@Tag(LargeTests.TAG)
058public class TestBackupDeleteWithFailures extends TestBackupBase {
059
060  private static final Logger LOG = LoggerFactory.getLogger(TestBackupDeleteWithFailures.class);
061
062  public enum Failure {
063    NO_FAILURES,
064    PRE_SNAPSHOT_FAILURE,
065    PRE_DELETE_SNAPSHOT_FAILURE,
066    POST_DELETE_SNAPSHOT_FAILURE
067  }
068
069  public static class MasterSnapshotObserver implements MasterCoprocessor, MasterObserver {
070    List<Failure> failures = new ArrayList<>();
071
072    public void setFailures(Failure... f) {
073      failures.clear();
074      for (int i = 0; i < f.length; i++) {
075        failures.add(f[i]);
076      }
077    }
078
079    @Override
080    public Optional<MasterObserver> getMasterObserver() {
081      return Optional.of(this);
082    }
083
084    @Override
085    public void preSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
086      final SnapshotDescription snapshot, final TableDescriptor hTableDescriptor)
087      throws IOException {
088      if (failures.contains(Failure.PRE_SNAPSHOT_FAILURE)) {
089        throw new IOException("preSnapshot");
090      }
091    }
092
093    @Override
094    public void preDeleteSnapshot(ObserverContext<MasterCoprocessorEnvironment> ctx,
095      SnapshotDescription snapshot) throws IOException {
096      if (failures.contains(Failure.PRE_DELETE_SNAPSHOT_FAILURE)) {
097        throw new IOException("preDeleteSnapshot");
098      }
099    }
100
101    @Override
102    public void postDeleteSnapshot(ObserverContext<MasterCoprocessorEnvironment> ctx,
103      SnapshotDescription snapshot) throws IOException {
104      if (failures.contains(Failure.POST_DELETE_SNAPSHOT_FAILURE)) {
105        throw new IOException("postDeleteSnapshot");
106      }
107    }
108  }
109
110  /**
111   * Setup Cluster with appropriate configurations before running tests.
112   * @throws Exception if starting the mini cluster or setting up the tables fails
113   */
114  @BeforeAll
115  public static void setUp() throws Exception {
116    TEST_UTIL = new HBaseTestingUtil();
117    conf1 = TEST_UTIL.getConfiguration();
118    conf1.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, MasterSnapshotObserver.class.getName());
119    conf1.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
120    setUpHelper();
121  }
122
123  private MasterSnapshotObserver getMasterSnapshotObserver() {
124    return TEST_UTIL.getHBaseCluster().getMaster().getMasterCoprocessorHost()
125      .findCoprocessor(MasterSnapshotObserver.class);
126  }
127
128  @Test
129  public void testBackupDeleteWithFailures() throws Exception {
130    testBackupDeleteWithFailuresAfter(1, Failure.PRE_DELETE_SNAPSHOT_FAILURE);
131    testBackupDeleteWithFailuresAfter(0, Failure.POST_DELETE_SNAPSHOT_FAILURE);
132    testBackupDeleteWithFailuresAfter(1, Failure.PRE_SNAPSHOT_FAILURE);
133  }
134
135  private void testBackupDeleteWithFailuresAfter(int expected, Failure... failures)
136    throws Exception {
137    LOG.info("test repair backup delete on a single table with data and failures " + failures[0]);
138    List<TableName> tableList = Lists.newArrayList(table1);
139    String backupId = fullTableBackup(tableList);
140    assertTrue(checkSucceeded(backupId));
141    LOG.info("backup complete");
142    String[] backupIds = new String[] { backupId };
143    BackupSystemTable table = new BackupSystemTable(TEST_UTIL.getConnection());
144    BackupInfo info = table.readBackupInfo(backupId);
145    Path path = new Path(info.getBackupRootDir(), backupId);
146    FileSystem fs = FileSystem.get(path.toUri(), conf1);
147    assertTrue(fs.exists(path));
148
149    Connection conn = TEST_UTIL.getConnection();
150    Admin admin = conn.getAdmin();
151    MasterSnapshotObserver observer = getMasterSnapshotObserver();
152
153    observer.setFailures(failures);
154    try {
155      getBackupAdmin().deleteBackups(backupIds);
156    } catch (IOException e) {
157      if (expected != 1) {
158        assertTrue(false);
159      }
160    }
161
162    // Verify that history length == expected after delete failure
163    assertTrue(table.getBackupHistory().size() == expected);
164
165    String[] ids = table.getListOfBackupIdsFromDeleteOperation();
166
167    // Verify that we still have delete record in backup system table
168    if (expected == 1) {
169      assertTrue(ids.length == 1);
170      assertTrue(ids[0].equals(backupId));
171    } else {
172      assertNull(ids);
173    }
174
175    // Now run repair command to repair "failed" delete operation
176    String[] args = new String[] { "repair" };
177
178    observer.setFailures(Failure.NO_FAILURES);
179
180    // Run repair
181    int ret = ToolRunner.run(conf1, new BackupDriver(), args);
182    assertTrue(ret == 0);
183    // Verify that history length == 0
184    assertTrue(table.getBackupHistory().size() == 0);
185    ids = table.getListOfBackupIdsFromDeleteOperation();
186
187    // Verify that we do not have delete record in backup system table
188    assertNull(ids);
189
190    table.close();
191    admin.close();
192  }
193}