001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.procedure;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertTrue;
022
023import java.util.List;
024import java.util.stream.Stream;
025import org.apache.hadoop.hbase.HBaseClassTestRule;
026import org.apache.hadoop.hbase.TableExistsException;
027import org.apache.hadoop.hbase.TableName;
028import org.apache.hadoop.hbase.client.Admin;
029import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
030import org.apache.hadoop.hbase.client.SnapshotDescription;
031import org.apache.hadoop.hbase.client.TableDescriptor;
032import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
033import org.apache.hadoop.hbase.procedure2.Procedure;
034import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
035import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
036import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
037import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
038import org.apache.hadoop.hbase.testclassification.MasterTests;
039import org.apache.hadoop.hbase.testclassification.MediumTests;
040import org.apache.hadoop.hbase.util.Bytes;
041import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
042import org.junit.After;
043import org.junit.ClassRule;
044import org.junit.Test;
045import org.junit.experimental.categories.Category;
046import org.slf4j.Logger;
047import org.slf4j.LoggerFactory;
048
049import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
050import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos;
051
052@Category({ MasterTests.class, MediumTests.class })
053public class TestCloneSnapshotProcedure extends TestTableDDLProcedureBase {
054
055  @ClassRule
056  public static final HBaseClassTestRule CLASS_RULE =
057    HBaseClassTestRule.forClass(TestCloneSnapshotProcedure.class);
058
059  private static final Logger LOG = LoggerFactory.getLogger(TestCloneSnapshotProcedure.class);
060
061  protected final byte[] CF = Bytes.toBytes("cf1");
062
063  private static SnapshotProtos.SnapshotDescription snapshot = null;
064
065  @After
066  @Override
067  public void tearDown() throws Exception {
068    super.tearDown();
069    SnapshotTestingUtils.deleteAllSnapshots(UTIL.getAdmin());
070    snapshot = null;
071  }
072
073  private SnapshotProtos.SnapshotDescription getSnapshot() throws Exception {
074    if (snapshot == null) {
075      final TableName snapshotTableName = TableName.valueOf("testCloneSnapshot");
076      long tid = EnvironmentEdgeManager.currentTime();
077      final String snapshotName = "snapshot-" + tid;
078
079      Admin admin = UTIL.getAdmin();
080      // create Table
081      SnapshotTestingUtils.createTable(UTIL, snapshotTableName, getNumReplicas(), CF);
082      // Load data
083      SnapshotTestingUtils.loadData(UTIL, snapshotTableName, 500, CF);
084      admin.disableTable(snapshotTableName);
085      // take a snapshot
086      admin.snapshot(snapshotName, snapshotTableName);
087      admin.enableTable(snapshotTableName);
088
089      List<SnapshotDescription> snapshotList = admin.listSnapshots();
090      snapshot = ProtobufUtil.createHBaseProtosSnapshotDesc(snapshotList.get(0));
091    }
092    return snapshot;
093  }
094
095  private int getNumReplicas() {
096    return 1;
097  }
098
099  private static TableDescriptor createTableDescriptor(TableName tableName, byte[]... family) {
100    TableDescriptorBuilder builder =
101      TableDescriptorBuilder.newBuilder(tableName).setValue(StoreFileTrackerFactory.TRACKER_IMPL,
102        UTIL.getConfiguration().get(StoreFileTrackerFactory.TRACKER_IMPL,
103          StoreFileTrackerFactory.Trackers.DEFAULT.name()));
104    Stream.of(family).map(ColumnFamilyDescriptorBuilder::of)
105      .forEachOrdered(builder::setColumnFamily);
106    return builder.build();
107  }
108
109  @Test
110  public void testCloneSnapshot() throws Exception {
111    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
112    final TableName clonedTableName = TableName.valueOf("testCloneSnapshot2");
113    final TableDescriptor htd = createTableDescriptor(clonedTableName, CF);
114
115    // take the snapshot
116    SnapshotProtos.SnapshotDescription snapshotDesc = getSnapshot();
117
118    long procId = ProcedureTestingUtility.submitAndWait(procExec,
119      new CloneSnapshotProcedure(procExec.getEnvironment(), htd, snapshotDesc));
120    ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId));
121    MasterProcedureTestingUtility.validateTableIsEnabled(UTIL.getHBaseCluster().getMaster(),
122      clonedTableName);
123  }
124
125  @Test
126  public void testCloneSnapshotToSameTable() throws Exception {
127    // take the snapshot
128    SnapshotProtos.SnapshotDescription snapshotDesc = getSnapshot();
129
130    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
131    final TableName clonedTableName = TableName.valueOf(snapshotDesc.getTable());
132    final TableDescriptor htd = createTableDescriptor(clonedTableName, CF);
133
134    long procId = ProcedureTestingUtility.submitAndWait(procExec,
135      new CloneSnapshotProcedure(procExec.getEnvironment(), htd, snapshotDesc));
136    Procedure<?> result = procExec.getResult(procId);
137    assertTrue(result.isFailed());
138    LOG.debug("Clone snapshot failed with exception: " + result.getException());
139    assertTrue(ProcedureTestingUtility.getExceptionCause(result) instanceof TableExistsException);
140  }
141
142  @Test
143  public void testRecoveryAndDoubleExecution() throws Exception {
144    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
145    final TableName clonedTableName = TableName.valueOf("testRecoveryAndDoubleExecution");
146    final TableDescriptor htd = createTableDescriptor(clonedTableName, CF);
147
148    // take the snapshot
149    SnapshotProtos.SnapshotDescription snapshotDesc = getSnapshot();
150
151    // Here if you enable this then we will enter an infinite loop, as we will fail either after
152    // TRSP.openRegion or after OpenRegionProcedure.execute, so we can never finish the TRSP...
153    ProcedureTestingUtility.setKillIfHasParent(procExec, false);
154    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
155
156    // Start the Clone snapshot procedure && kill the executor
157    long procId = procExec
158      .submitProcedure(new CloneSnapshotProcedure(procExec.getEnvironment(), htd, snapshotDesc));
159
160    // Restart the executor and execute the step twice
161    MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId);
162
163    MasterProcedureTestingUtility.validateTableIsEnabled(UTIL.getHBaseCluster().getMaster(),
164      clonedTableName);
165  }
166
167  @Test
168  public void testRecoverWithRestoreAclFlag() throws Exception {
169    // This test is to solve the problems mentioned in HBASE-26462,
170    // this needs to simulate the case of CloneSnapshotProcedure failure and recovery,
171    // and verify whether 'restoreAcl' flag can obtain the correct value.
172
173    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
174    final TableName clonedTableName = TableName.valueOf("testRecoverWithRestoreAclFlag");
175    final TableDescriptor htd = createTableDescriptor(clonedTableName, CF);
176
177    SnapshotProtos.SnapshotDescription snapshotDesc = getSnapshot();
178    ProcedureTestingUtility.setKillIfHasParent(procExec, false);
179    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
180
181    // Start the Clone snapshot procedure (with restoreAcl 'true') && kill the executor
182    long procId = procExec.submitProcedure(
183      new CloneSnapshotProcedure(procExec.getEnvironment(), htd, snapshotDesc, true));
184
185    MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId);
186
187    CloneSnapshotProcedure result = (CloneSnapshotProcedure) procExec.getResult(procId);
188    // check whether the 'restoreAcl' flag is true after deserialization from Pb.
189    assertEquals(true, result.getRestoreAcl());
190  }
191
192  @Test
193  public void testRollbackAndDoubleExecution() throws Exception {
194    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
195    final TableName clonedTableName = TableName.valueOf("testRollbackAndDoubleExecution");
196    final TableDescriptor htd = createTableDescriptor(clonedTableName, CF);
197
198    // take the snapshot
199    SnapshotProtos.SnapshotDescription snapshotDesc = getSnapshot();
200
201    ProcedureTestingUtility.waitNoProcedureRunning(procExec);
202    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
203
204    // Start the Clone snapshot procedure && kill the executor
205    long procId = procExec
206      .submitProcedure(new CloneSnapshotProcedure(procExec.getEnvironment(), htd, snapshotDesc));
207
208    int lastStep = 2; // failing before CLONE_SNAPSHOT_WRITE_FS_LAYOUT
209    MasterProcedureTestingUtility.testRollbackAndDoubleExecution(procExec, procId, lastStep);
210
211    MasterProcedureTestingUtility.validateTableDeletion(UTIL.getHBaseCluster().getMaster(),
212      clonedTableName);
213  }
214}