001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.procedure;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertFalse;
022import static org.junit.jupiter.api.Assertions.assertNotEquals;
023import static org.junit.jupiter.api.Assertions.assertTrue;
024
025import java.io.IOException;
026import java.util.List;
027import org.apache.hadoop.hbase.TableName;
028import org.apache.hadoop.hbase.TableNotDisabledException;
029import org.apache.hadoop.hbase.TableNotFoundException;
030import org.apache.hadoop.hbase.client.Admin;
031import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
032import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
033import org.apache.hadoop.hbase.client.SnapshotDescription;
034import org.apache.hadoop.hbase.client.TableDescriptor;
035import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
036import org.apache.hadoop.hbase.procedure2.Procedure;
037import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
038import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
039import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
040import org.apache.hadoop.hbase.testclassification.LargeTests;
041import org.apache.hadoop.hbase.testclassification.MasterTests;
042import org.apache.hadoop.hbase.util.Bytes;
043import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
044import org.junit.jupiter.api.AfterAll;
045import org.junit.jupiter.api.AfterEach;
046import org.junit.jupiter.api.BeforeAll;
047import org.junit.jupiter.api.BeforeEach;
048import org.junit.jupiter.api.Tag;
049import org.junit.jupiter.api.Test;
050import org.junit.jupiter.api.TestInfo;
051import org.slf4j.Logger;
052import org.slf4j.LoggerFactory;
053
054import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
055import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos;
056
057@Tag(MasterTests.TAG)
058@Tag(LargeTests.TAG)
059public class TestRestoreSnapshotProcedure extends TestTableDDLProcedureBase {
060
061  private static final Logger LOG = LoggerFactory.getLogger(TestRestoreSnapshotProcedure.class);
062
063  @BeforeAll
064  public static void setupCluster() throws Exception {
065    TestTableDDLProcedureBase.setupCluster();
066  }
067
068  @AfterAll
069  public static void cleanupTest() throws Exception {
070    TestTableDDLProcedureBase.cleanupTest();
071  }
072
073  protected final TableName snapshotTableName = TableName.valueOf("testRestoreSnapshot");
074  protected final byte[] CF1 = Bytes.toBytes("cf1");
075  protected final byte[] CF2 = Bytes.toBytes("cf2");
076  protected final byte[] CF3 = Bytes.toBytes("cf3");
077  protected final byte[] CF4 = Bytes.toBytes("cf4");
078  protected final int rowCountCF1 = 10;
079  protected final int rowCountCF2 = 40;
080  protected final int rowCountCF3 = 40;
081  protected final int rowCountCF4 = 40;
082  protected final int rowCountCF1addition = 10;
083
084  private SnapshotProtos.SnapshotDescription snapshot = null;
085  private TableDescriptor snapshotHTD = null;
086  private String testMethodName;
087
088  @BeforeEach
089  public void setTestMethod(TestInfo testInfo) {
090    testMethodName = testInfo.getTestMethod().get().getName();
091  }
092
093  @BeforeEach
094  @Override
095  public void setup() throws Exception {
096    super.setup();
097    setupSnapshotAndUpdateTable();
098  }
099
100  @AfterEach
101  @Override
102  public void tearDown() throws Exception {
103    super.tearDown();
104    SnapshotTestingUtils.deleteAllSnapshots(UTIL.getAdmin());
105    SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
106  }
107
108  private int getNumReplicas() {
109    return 1;
110  }
111
112  private void setupSnapshotAndUpdateTable() throws Exception {
113    long tid = EnvironmentEdgeManager.currentTime();
114    final String snapshotName = "snapshot-" + tid;
115    Admin admin = UTIL.getAdmin();
116    // create Table
117    SnapshotTestingUtils.createTable(UTIL, snapshotTableName, getNumReplicas(), CF1, CF2);
118    // Load data
119    SnapshotTestingUtils.loadData(UTIL, snapshotTableName, rowCountCF1, CF1);
120    SnapshotTestingUtils.loadData(UTIL, snapshotTableName, rowCountCF2, CF2);
121    SnapshotTestingUtils.verifyRowCount(UTIL, snapshotTableName, rowCountCF1 + rowCountCF2);
122
123    snapshotHTD = admin.getDescriptor(snapshotTableName);
124
125    admin.disableTable(snapshotTableName);
126    // take a snapshot
127    admin.snapshot(snapshotName, snapshotTableName);
128
129    List<SnapshotDescription> snapshotList = admin.listSnapshots();
130    snapshot = ProtobufUtil.createHBaseProtosSnapshotDesc(snapshotList.get(0));
131
132    // modify the table
133    ColumnFamilyDescriptor columnFamilyDescriptor3 = ColumnFamilyDescriptorBuilder.of(CF3);
134    ColumnFamilyDescriptor columnFamilyDescriptor4 = ColumnFamilyDescriptorBuilder.of(CF4);
135    admin.addColumnFamily(snapshotTableName, columnFamilyDescriptor3);
136    admin.addColumnFamily(snapshotTableName, columnFamilyDescriptor4);
137    admin.deleteColumnFamily(snapshotTableName, CF2);
138    // enable table and insert data
139    admin.enableTable(snapshotTableName);
140    SnapshotTestingUtils.loadData(UTIL, snapshotTableName, rowCountCF3, CF3);
141    SnapshotTestingUtils.loadData(UTIL, snapshotTableName, rowCountCF4, CF4);
142    SnapshotTestingUtils.loadData(UTIL, snapshotTableName, rowCountCF1addition, CF1);
143    TableDescriptor currentHTD = admin.getDescriptor(snapshotTableName);
144    assertTrue(currentHTD.hasColumnFamily(CF1));
145    assertFalse(currentHTD.hasColumnFamily(CF2));
146    assertTrue(currentHTD.hasColumnFamily(CF3));
147    assertTrue(currentHTD.hasColumnFamily(CF4));
148    assertNotEquals(currentHTD.getColumnFamilyNames().size(),
149      snapshotHTD.getColumnFamilies().length);
150    SnapshotTestingUtils.verifyRowCount(UTIL, snapshotTableName,
151      rowCountCF1 + rowCountCF3 + rowCountCF4 + rowCountCF1addition);
152    admin.disableTable(snapshotTableName);
153  }
154
155  private static TableDescriptor createHTableDescriptor(final TableName tableName,
156    final byte[]... family) {
157    TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName);
158
159    for (int i = 0; i < family.length; ++i) {
160      builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(family[i]));
161    }
162    return builder.build();
163  }
164
165  @Test
166  public void testRestoreSnapshot() throws Exception {
167    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
168
169    long procId = ProcedureTestingUtility.submitAndWait(procExec,
170      new RestoreSnapshotProcedure(procExec.getEnvironment(), snapshotHTD, snapshot));
171    ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId));
172
173    validateSnapshotRestore();
174  }
175
176  @Test
177  public void testRestoreSnapshotToDifferentTable() throws Exception {
178    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
179    final TableName restoredTableName = TableName.valueOf(testMethodName);
180    final TableDescriptor tableDescriptor = createHTableDescriptor(restoredTableName, CF1, CF2);
181
182    long procId = ProcedureTestingUtility.submitAndWait(procExec,
183      new RestoreSnapshotProcedure(procExec.getEnvironment(), tableDescriptor, snapshot));
184    Procedure<?> result = procExec.getResult(procId);
185    assertTrue(result.isFailed());
186    LOG.debug("Restore snapshot failed with exception: " + result.getException());
187    assertTrue(ProcedureTestingUtility.getExceptionCause(result) instanceof TableNotFoundException);
188  }
189
190  @Test
191  public void testRestoreSnapshotToEnabledTable() throws Exception {
192    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
193
194    try {
195      UTIL.getAdmin().enableTable(snapshotTableName);
196
197      long procId = ProcedureTestingUtility.submitAndWait(procExec,
198        new RestoreSnapshotProcedure(procExec.getEnvironment(), snapshotHTD, snapshot));
199      Procedure<?> result = procExec.getResult(procId);
200      assertTrue(result.isFailed());
201      LOG.debug("Restore snapshot failed with exception: " + result.getException());
202      assertTrue(
203        ProcedureTestingUtility.getExceptionCause(result) instanceof TableNotDisabledException);
204    } finally {
205      UTIL.getAdmin().disableTable(snapshotTableName);
206    }
207  }
208
209  @Test
210  public void testRecoveryAndDoubleExecution() throws Exception {
211    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
212
213    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
214
215    // Start the Restore snapshot procedure && kill the executor
216    long procId = procExec.submitProcedure(
217      new RestoreSnapshotProcedure(procExec.getEnvironment(), snapshotHTD, snapshot));
218
219    // Restart the executor and execute the step twice
220    MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId);
221
222    resetProcExecutorTestingKillFlag();
223    validateSnapshotRestore();
224  }
225
226  @Test
227  public void testRecoverWithRestoreAclFlag() throws Exception {
228    // This test is to solve the problems mentioned in HBASE-26462,
229    // this needs to simulate the case of RestoreSnapshotProcedure failure and recovery,
230    // and verify whether 'restoreAcl' flag can obtain the correct value.
231
232    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
233    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
234
235    // Start the Restore snapshot procedure (with restoreAcl 'true') && kill the executor
236    long procId = procExec.submitProcedure(
237      new RestoreSnapshotProcedure(procExec.getEnvironment(), snapshotHTD, snapshot, true));
238    MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId);
239
240    RestoreSnapshotProcedure result = (RestoreSnapshotProcedure) procExec.getResult(procId);
241    // check whether the restoreAcl flag is true after deserialization from Pb.
242    assertEquals(true, result.getRestoreAcl());
243  }
244
245  private void validateSnapshotRestore() throws IOException {
246    try {
247      UTIL.getAdmin().enableTable(snapshotTableName);
248
249      TableDescriptor currentHTD = UTIL.getAdmin().getDescriptor(snapshotTableName);
250      assertTrue(currentHTD.hasColumnFamily(CF1));
251      assertTrue(currentHTD.hasColumnFamily(CF2));
252      assertFalse(currentHTD.hasColumnFamily(CF3));
253      assertFalse(currentHTD.hasColumnFamily(CF4));
254      assertEquals(currentHTD.getColumnFamilyNames().size(),
255        snapshotHTD.getColumnFamilies().length);
256      SnapshotTestingUtils.verifyRowCount(UTIL, snapshotTableName, rowCountCF1 + rowCountCF2);
257    } finally {
258      UTIL.getAdmin().disableTable(snapshotTableName);
259    }
260  }
261}