001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.procedure;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertNotEquals;
023import static org.junit.Assert.assertTrue;
024import java.io.IOException;
025import java.util.List;
026import org.apache.hadoop.hbase.HBaseClassTestRule;
027import org.apache.hadoop.hbase.HColumnDescriptor;
028import org.apache.hadoop.hbase.HTableDescriptor;
029import org.apache.hadoop.hbase.TableName;
030import org.apache.hadoop.hbase.TableNotDisabledException;
031import org.apache.hadoop.hbase.TableNotFoundException;
032import org.apache.hadoop.hbase.client.Admin;
033import org.apache.hadoop.hbase.client.SnapshotDescription;
034import org.apache.hadoop.hbase.procedure2.Procedure;
035import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
036import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
037import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
038import org.apache.hadoop.hbase.testclassification.LargeTests;
039import org.apache.hadoop.hbase.testclassification.MasterTests;
040import org.apache.hadoop.hbase.util.Bytes;
041import org.junit.After;
042import org.junit.Before;
043import org.junit.ClassRule;
044import org.junit.Rule;
045import org.junit.Test;
046import org.junit.experimental.categories.Category;
047import org.junit.rules.TestName;
048import org.slf4j.Logger;
049import org.slf4j.LoggerFactory;
050import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
051import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos;
052
053@Category({MasterTests.class, LargeTests.class})
054public class TestRestoreSnapshotProcedure extends TestTableDDLProcedureBase {
055
056  @ClassRule
057  public static final HBaseClassTestRule CLASS_RULE =
058      HBaseClassTestRule.forClass(TestRestoreSnapshotProcedure.class);
059
060  private static final Logger LOG = LoggerFactory.getLogger(TestRestoreSnapshotProcedure.class);
061
062  protected final TableName snapshotTableName = TableName.valueOf("testRestoreSnapshot");
063  protected final byte[] CF1 = Bytes.toBytes("cf1");
064  protected final byte[] CF2 = Bytes.toBytes("cf2");
065  protected final byte[] CF3 = Bytes.toBytes("cf3");
066  protected final byte[] CF4 = Bytes.toBytes("cf4");
067  protected final int rowCountCF1 = 10;
068  protected final int rowCountCF2 = 40;
069  protected final int rowCountCF3 = 40;
070  protected final int rowCountCF4 = 40;
071  protected final int rowCountCF1addition = 10;
072
073  private SnapshotProtos.SnapshotDescription snapshot = null;
074  private HTableDescriptor snapshotHTD = null;
075
076  @Rule
077  public TestName name = new TestName();
078
079  @Before
080  @Override
081  public void setup() throws Exception {
082    super.setup();
083    setupSnapshotAndUpdateTable();
084  }
085
086  @After
087  @Override
088  public void tearDown() throws Exception {
089    super.tearDown();
090    SnapshotTestingUtils.deleteAllSnapshots(UTIL.getAdmin());
091    SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
092  }
093
094  private int getNumReplicas() {
095    return 1;
096  }
097
098  private void setupSnapshotAndUpdateTable() throws Exception {
099    long tid = System.currentTimeMillis();
100    final byte[] snapshotName = Bytes.toBytes("snapshot-" + tid);
101    Admin admin = UTIL.getAdmin();
102    // create Table
103    SnapshotTestingUtils.createTable(UTIL, snapshotTableName, getNumReplicas(), CF1, CF2);
104    // Load data
105    SnapshotTestingUtils.loadData(UTIL, snapshotTableName, rowCountCF1, CF1);
106    SnapshotTestingUtils.loadData(UTIL, snapshotTableName, rowCountCF2, CF2);
107    SnapshotTestingUtils.verifyRowCount(UTIL, snapshotTableName, rowCountCF1 + rowCountCF2);
108
109    snapshotHTD = admin.getTableDescriptor(snapshotTableName);
110
111    admin.disableTable(snapshotTableName);
112    // take a snapshot
113    admin.snapshot(snapshotName, snapshotTableName);
114
115    List<SnapshotDescription> snapshotList = admin.listSnapshots();
116    snapshot = ProtobufUtil.createHBaseProtosSnapshotDesc(snapshotList.get(0));
117
118    // modify the table
119    HColumnDescriptor columnFamilyDescriptor3 = new HColumnDescriptor(CF3);
120    HColumnDescriptor columnFamilyDescriptor4 = new HColumnDescriptor(CF4);
121    admin.addColumnFamily(snapshotTableName, columnFamilyDescriptor3);
122    admin.addColumnFamily(snapshotTableName, columnFamilyDescriptor4);
123    admin.deleteColumnFamily(snapshotTableName, CF2);
124    // enable table and insert data
125    admin.enableTable(snapshotTableName);
126    SnapshotTestingUtils.loadData(UTIL, snapshotTableName, rowCountCF3, CF3);
127    SnapshotTestingUtils.loadData(UTIL, snapshotTableName, rowCountCF4, CF4);
128    SnapshotTestingUtils.loadData(UTIL, snapshotTableName, rowCountCF1addition, CF1);
129    HTableDescriptor currentHTD = admin.getTableDescriptor(snapshotTableName);
130    assertTrue(currentHTD.hasFamily(CF1));
131    assertFalse(currentHTD.hasFamily(CF2));
132    assertTrue(currentHTD.hasFamily(CF3));
133    assertTrue(currentHTD.hasFamily(CF4));
134    assertNotEquals(currentHTD.getFamiliesKeys().size(), snapshotHTD.getFamiliesKeys().size());
135    SnapshotTestingUtils.verifyRowCount(
136      UTIL, snapshotTableName, rowCountCF1 + rowCountCF3 + rowCountCF4 + rowCountCF1addition);
137    admin.disableTable(snapshotTableName);
138  }
139
140  private static HTableDescriptor createHTableDescriptor(
141      final TableName tableName, final byte[] ... family) {
142    HTableDescriptor htd = new HTableDescriptor(tableName);
143    for (int i = 0; i < family.length; ++i) {
144      htd.addFamily(new HColumnDescriptor(family[i]));
145    }
146    return htd;
147  }
148
149  @Test
150  public void testRestoreSnapshot() throws Exception {
151    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
152
153    long procId = ProcedureTestingUtility.submitAndWait(
154      procExec,
155      new RestoreSnapshotProcedure(procExec.getEnvironment(), snapshotHTD, snapshot));
156    ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId));
157
158    validateSnapshotRestore();
159  }
160
161  @Test
162  public void testRestoreSnapshotToDifferentTable() throws Exception {
163    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
164    final TableName restoredTableName = TableName.valueOf(name.getMethodName());
165    final HTableDescriptor newHTD = createHTableDescriptor(restoredTableName, CF1, CF2);
166
167    long procId = ProcedureTestingUtility.submitAndWait(
168      procExec, new RestoreSnapshotProcedure(procExec.getEnvironment(), newHTD, snapshot));
169    Procedure<?> result = procExec.getResult(procId);
170    assertTrue(result.isFailed());
171    LOG.debug("Restore snapshot failed with exception: " + result.getException());
172    assertTrue(
173      ProcedureTestingUtility.getExceptionCause(result) instanceof TableNotFoundException);
174  }
175
176  @Test
177  public void testRestoreSnapshotToEnabledTable() throws Exception {
178    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
179
180    try {
181      UTIL.getAdmin().enableTable(snapshotTableName);
182
183      long procId = ProcedureTestingUtility.submitAndWait(
184        procExec,
185        new RestoreSnapshotProcedure(procExec.getEnvironment(), snapshotHTD, snapshot));
186      Procedure<?> result = procExec.getResult(procId);
187      assertTrue(result.isFailed());
188      LOG.debug("Restore snapshot failed with exception: " + result.getException());
189      assertTrue(
190        ProcedureTestingUtility.getExceptionCause(result) instanceof TableNotDisabledException);
191    } finally {
192      UTIL.getAdmin().disableTable(snapshotTableName);
193    }
194  }
195
196  @Test
197  public void testRecoveryAndDoubleExecution() throws Exception {
198    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
199
200    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
201
202    // Start the Restore snapshot procedure && kill the executor
203    long procId = procExec.submitProcedure(
204      new RestoreSnapshotProcedure(procExec.getEnvironment(), snapshotHTD, snapshot));
205
206    // Restart the executor and execute the step twice
207    MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId);
208
209    resetProcExecutorTestingKillFlag();
210    validateSnapshotRestore();
211  }
212
213  private void validateSnapshotRestore() throws IOException {
214    try {
215      UTIL.getAdmin().enableTable(snapshotTableName);
216
217      HTableDescriptor currentHTD = UTIL.getAdmin().getTableDescriptor(snapshotTableName);
218      assertTrue(currentHTD.hasFamily(CF1));
219      assertTrue(currentHTD.hasFamily(CF2));
220      assertFalse(currentHTD.hasFamily(CF3));
221      assertFalse(currentHTD.hasFamily(CF4));
222      assertEquals(currentHTD.getFamiliesKeys().size(), snapshotHTD.getFamiliesKeys().size());
223      SnapshotTestingUtils.verifyRowCount(UTIL, snapshotTableName, rowCountCF1 + rowCountCF2);
224    } finally {
225      UTIL.getAdmin().disableTable(snapshotTableName);
226    }
227  }
228}