001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.procedure;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertTrue;
022
023import java.util.List;
024import java.util.stream.Stream;
025import org.apache.hadoop.hbase.TableExistsException;
026import org.apache.hadoop.hbase.TableName;
027import org.apache.hadoop.hbase.client.Admin;
028import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
029import org.apache.hadoop.hbase.client.SnapshotDescription;
030import org.apache.hadoop.hbase.client.TableDescriptor;
031import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
032import org.apache.hadoop.hbase.procedure2.Procedure;
033import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
034import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
035import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
036import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
037import org.apache.hadoop.hbase.testclassification.MasterTests;
038import org.apache.hadoop.hbase.testclassification.MediumTests;
039import org.apache.hadoop.hbase.util.Bytes;
040import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
041import org.junit.jupiter.api.AfterAll;
042import org.junit.jupiter.api.AfterEach;
043import org.junit.jupiter.api.BeforeAll;
044import org.junit.jupiter.api.Tag;
045import org.junit.jupiter.api.Test;
046import org.slf4j.Logger;
047import org.slf4j.LoggerFactory;
048
049import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
050import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos;
051
052@Tag(MasterTests.TAG)
053@Tag(MediumTests.TAG)
054public class TestCloneSnapshotProcedure extends TestTableDDLProcedureBase {
055
056  private static final Logger LOG = LoggerFactory.getLogger(TestCloneSnapshotProcedure.class);
057
058  protected final byte[] CF = Bytes.toBytes("cf1");
059
060  private static SnapshotProtos.SnapshotDescription snapshot = null;
061
062  @BeforeAll
063  public static void setupCluster() throws Exception {
064    TestTableDDLProcedureBase.setupCluster();
065  }
066
067  @AfterAll
068  public static void cleanupTest() throws Exception {
069    TestTableDDLProcedureBase.cleanupTest();
070  }
071
072  @AfterEach
073  @Override
074  public void tearDown() throws Exception {
075    super.tearDown();
076    SnapshotTestingUtils.deleteAllSnapshots(UTIL.getAdmin());
077    snapshot = null;
078  }
079
080  private SnapshotProtos.SnapshotDescription getSnapshot() throws Exception {
081    if (snapshot == null) {
082      final TableName snapshotTableName = TableName.valueOf("testCloneSnapshot");
083      long tid = EnvironmentEdgeManager.currentTime();
084      final String snapshotName = "snapshot-" + tid;
085
086      Admin admin = UTIL.getAdmin();
087      // create Table
088      SnapshotTestingUtils.createTable(UTIL, snapshotTableName, getNumReplicas(), CF);
089      // Load data
090      SnapshotTestingUtils.loadData(UTIL, snapshotTableName, 500, CF);
091      admin.disableTable(snapshotTableName);
092      // take a snapshot
093      admin.snapshot(snapshotName, snapshotTableName);
094      admin.enableTable(snapshotTableName);
095
096      List<SnapshotDescription> snapshotList = admin.listSnapshots();
097      snapshot = ProtobufUtil.createHBaseProtosSnapshotDesc(snapshotList.get(0));
098    }
099    return snapshot;
100  }
101
102  private int getNumReplicas() {
103    return 1;
104  }
105
106  private static TableDescriptor createTableDescriptor(TableName tableName, byte[]... family) {
107    TableDescriptorBuilder builder =
108      TableDescriptorBuilder.newBuilder(tableName).setValue(StoreFileTrackerFactory.TRACKER_IMPL,
109        UTIL.getConfiguration().get(StoreFileTrackerFactory.TRACKER_IMPL,
110          StoreFileTrackerFactory.Trackers.DEFAULT.name()));
111    Stream.of(family).map(ColumnFamilyDescriptorBuilder::of)
112      .forEachOrdered(builder::setColumnFamily);
113    return builder.build();
114  }
115
116  @Test
117  public void testCloneSnapshot() throws Exception {
118    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
119    final TableName clonedTableName = TableName.valueOf("testCloneSnapshot2");
120    final TableDescriptor htd = createTableDescriptor(clonedTableName, CF);
121
122    // take the snapshot
123    SnapshotProtos.SnapshotDescription snapshotDesc = getSnapshot();
124
125    long procId = ProcedureTestingUtility.submitAndWait(procExec,
126      new CloneSnapshotProcedure(procExec.getEnvironment(), htd, snapshotDesc));
127    ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId));
128    MasterProcedureTestingUtility.validateTableIsEnabled(UTIL.getHBaseCluster().getMaster(),
129      clonedTableName);
130  }
131
132  @Test
133  public void testCloneSnapshotToSameTable() throws Exception {
134    // take the snapshot
135    SnapshotProtos.SnapshotDescription snapshotDesc = getSnapshot();
136
137    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
138    final TableName clonedTableName = TableName.valueOf(snapshotDesc.getTable());
139    final TableDescriptor htd = createTableDescriptor(clonedTableName, CF);
140
141    long procId = ProcedureTestingUtility.submitAndWait(procExec,
142      new CloneSnapshotProcedure(procExec.getEnvironment(), htd, snapshotDesc));
143    Procedure<?> result = procExec.getResult(procId);
144    assertTrue(result.isFailed());
145    LOG.debug("Clone snapshot failed with exception: " + result.getException());
146    assertTrue(ProcedureTestingUtility.getExceptionCause(result) instanceof TableExistsException);
147  }
148
149  @Test
150  public void testRecoveryAndDoubleExecution() throws Exception {
151    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
152    final TableName clonedTableName = TableName.valueOf("testRecoveryAndDoubleExecution");
153    final TableDescriptor htd = createTableDescriptor(clonedTableName, CF);
154
155    // take the snapshot
156    SnapshotProtos.SnapshotDescription snapshotDesc = getSnapshot();
157
158    // Here if you enable this then we will enter an infinite loop, as we will fail either after
159    // TRSP.openRegion or after OpenRegionProcedure.execute, so we can never finish the TRSP...
160    ProcedureTestingUtility.setKillIfHasParent(procExec, false);
161    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
162
163    // Start the Clone snapshot procedure && kill the executor
164    long procId = procExec
165      .submitProcedure(new CloneSnapshotProcedure(procExec.getEnvironment(), htd, snapshotDesc));
166
167    // Restart the executor and execute the step twice
168    MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId);
169
170    MasterProcedureTestingUtility.validateTableIsEnabled(UTIL.getHBaseCluster().getMaster(),
171      clonedTableName);
172  }
173
174  @Test
175  public void testRecoverWithRestoreAclFlag() throws Exception {
176    // This test is to solve the problems mentioned in HBASE-26462,
177    // this needs to simulate the case of CloneSnapshotProcedure failure and recovery,
178    // and verify whether 'restoreAcl' flag can obtain the correct value.
179
180    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
181    final TableName clonedTableName = TableName.valueOf("testRecoverWithRestoreAclFlag");
182    final TableDescriptor htd = createTableDescriptor(clonedTableName, CF);
183
184    SnapshotProtos.SnapshotDescription snapshotDesc = getSnapshot();
185    ProcedureTestingUtility.setKillIfHasParent(procExec, false);
186    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
187
188    // Start the Clone snapshot procedure (with restoreAcl 'true') && kill the executor
189    long procId = procExec.submitProcedure(
190      new CloneSnapshotProcedure(procExec.getEnvironment(), htd, snapshotDesc, true));
191
192    MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId);
193
194    CloneSnapshotProcedure result = (CloneSnapshotProcedure) procExec.getResult(procId);
195    // check whether the 'restoreAcl' flag is true after deserialization from Pb.
196    assertEquals(true, result.getRestoreAcl());
197  }
198
199  @Test
200  public void testRollbackAndDoubleExecution() throws Exception {
201    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
202    final TableName clonedTableName = TableName.valueOf("testRollbackAndDoubleExecution");
203    final TableDescriptor htd = createTableDescriptor(clonedTableName, CF);
204
205    // take the snapshot
206    SnapshotProtos.SnapshotDescription snapshotDesc = getSnapshot();
207
208    ProcedureTestingUtility.waitNoProcedureRunning(procExec);
209    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
210
211    // Start the Clone snapshot procedure && kill the executor
212    long procId = procExec
213      .submitProcedure(new CloneSnapshotProcedure(procExec.getEnvironment(), htd, snapshotDesc));
214
215    int lastStep = 2; // failing before CLONE_SNAPSHOT_WRITE_FS_LAYOUT
216    MasterProcedureTestingUtility.testRollbackAndDoubleExecution(procExec, procId, lastStep);
217
218    MasterProcedureTestingUtility.validateTableDeletion(UTIL.getHBaseCluster().getMaster(),
219      clonedTableName);
220  }
221}