001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.procedure;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertTrue;
022import static org.junit.Assert.fail;
023
024import java.io.IOException;
025import org.apache.hadoop.conf.Configuration;
026import org.apache.hadoop.fs.FileSystem;
027import org.apache.hadoop.fs.Path;
028import org.apache.hadoop.hbase.DoNotRetryIOException;
029import org.apache.hadoop.hbase.HBaseClassTestRule;
030import org.apache.hadoop.hbase.HBaseIOException;
031import org.apache.hadoop.hbase.TableExistsException;
032import org.apache.hadoop.hbase.TableName;
033import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
034import org.apache.hadoop.hbase.client.RegionInfo;
035import org.apache.hadoop.hbase.client.TableDescriptor;
036import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
037import org.apache.hadoop.hbase.master.MasterFileSystem;
038import org.apache.hadoop.hbase.procedure2.Procedure;
039import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
040import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
041import org.apache.hadoop.hbase.testclassification.MasterTests;
042import org.apache.hadoop.hbase.testclassification.MediumTests;
043import org.apache.hadoop.hbase.util.Bytes;
044import org.apache.hadoop.hbase.util.FSUtils;
045import org.apache.hadoop.hbase.util.ModifyRegionUtils;
046import org.junit.ClassRule;
047import org.junit.Rule;
048import org.junit.Test;
049import org.junit.experimental.categories.Category;
050import org.junit.rules.TestName;
051
052import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos;
053
054@Category({MasterTests.class, MediumTests.class})
055public class TestCreateTableProcedure extends TestTableDDLProcedureBase {
056
057  @ClassRule
058  public static final HBaseClassTestRule CLASS_RULE =
059    HBaseClassTestRule.forClass(TestCreateTableProcedure.class);
060
061  private static final String F1 = "f1";
062  private static final String F2 = "f2";
063
064  @Rule
065  public TestName name = new TestName();
066
067  @Test
068  public void testSimpleCreate() throws Exception {
069    final TableName tableName = TableName.valueOf(name.getMethodName());
070    final byte[][] splitKeys = null;
071    testSimpleCreate(tableName, splitKeys);
072  }
073
074  @Test
075  public void testSimpleCreateWithSplits() throws Exception {
076    final TableName tableName = TableName.valueOf(name.getMethodName());
077    final byte[][] splitKeys = new byte[][] {
078      Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c")
079    };
080    testSimpleCreate(tableName, splitKeys);
081  }
082
083  private void testSimpleCreate(final TableName tableName, byte[][] splitKeys) throws Exception {
084    RegionInfo[] regions = MasterProcedureTestingUtility.createTable(
085      getMasterProcedureExecutor(), tableName, splitKeys, F1, F2);
086    MasterProcedureTestingUtility.validateTableCreation(getMaster(), tableName, regions, F1, F2);
087  }
088
089  @Test
090  public void testCreateWithoutColumnFamily() throws Exception {
091    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
092    final TableName tableName = TableName.valueOf(name.getMethodName());
093    // create table with 0 families will fail
094    final TableDescriptorBuilder builder =
095      TableDescriptorBuilder.newBuilder(MasterProcedureTestingUtility.createHTD(tableName));
096
097    // disable sanity check
098    builder.setValue("hbase.table.sanity.checks", Boolean.FALSE.toString());
099    TableDescriptor htd = builder.build();
100    final RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, null);
101
102    long procId =
103        ProcedureTestingUtility.submitAndWait(procExec,
104            new CreateTableProcedure(procExec.getEnvironment(), htd, regions));
105    final Procedure<?> result = procExec.getResult(procId);
106    assertEquals(true, result.isFailed());
107    Throwable cause = ProcedureTestingUtility.getExceptionCause(result);
108    assertTrue("expected DoNotRetryIOException, got " + cause,
109        cause instanceof DoNotRetryIOException);
110  }
111
112  @Test(expected=TableExistsException.class)
113  public void testCreateExisting() throws Exception {
114    final TableName tableName = TableName.valueOf(name.getMethodName());
115    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
116    final TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, "f");
117    final RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, null);
118
119    // create the table
120    long procId1 = procExec.submitProcedure(
121      new CreateTableProcedure(procExec.getEnvironment(), htd, regions));
122
123    // create another with the same name
124    ProcedurePrepareLatch latch2 = new ProcedurePrepareLatch.CompatibilityLatch();
125    long procId2 = procExec.submitProcedure(
126      new CreateTableProcedure(procExec.getEnvironment(), htd, regions, latch2));
127
128    ProcedureTestingUtility.waitProcedure(procExec, procId1);
129    ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId1));
130
131    ProcedureTestingUtility.waitProcedure(procExec, procId2);
132    latch2.await();
133  }
134
135  @Test
136  public void testRecoveryAndDoubleExecution() throws Exception {
137    final TableName tableName = TableName.valueOf(name.getMethodName());
138
139    // create the table
140    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
141    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
142
143    // Start the Create procedure && kill the executor
144    byte[][] splitKeys = null;
145    TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, "f1", "f2");
146    RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, splitKeys);
147    long procId = procExec.submitProcedure(
148      new CreateTableProcedure(procExec.getEnvironment(), htd, regions));
149
150    // Restart the executor and execute the step twice
151    MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId);
152    MasterProcedureTestingUtility.validateTableCreation(getMaster(), tableName, regions, F1, F2);
153  }
154
155  @Test
156  public void testRollbackAndDoubleExecution() throws Exception {
157    final TableName tableName = TableName.valueOf(name.getMethodName());
158    testRollbackAndDoubleExecution(TableDescriptorBuilder
159      .newBuilder(MasterProcedureTestingUtility.createHTD(tableName, F1, F2)));
160  }
161
162  @Test
163  public void testRollbackAndDoubleExecutionOnMobTable() throws Exception {
164    final TableName tableName = TableName.valueOf(name.getMethodName());
165    TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, F1, F2);
166    TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(htd)
167            .modifyColumnFamily(ColumnFamilyDescriptorBuilder
168              .newBuilder(htd.getColumnFamily(Bytes.toBytes(F1)))
169              .setMobEnabled(true)
170              .build());
171    testRollbackAndDoubleExecution(builder);
172  }
173
174  private void testRollbackAndDoubleExecution(TableDescriptorBuilder builder) throws Exception {
175    // create the table
176    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
177    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
178
179    // Start the Create procedure && kill the executor
180    final byte[][] splitKeys = new byte[][] {
181      Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c")
182    };
183    builder.setRegionReplication(3);
184    TableDescriptor htd = builder.build();
185    RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, splitKeys);
186    long procId = procExec.submitProcedure(
187      new CreateTableProcedure(procExec.getEnvironment(), htd, regions));
188
189    int lastStep = 2; // failing before CREATE_TABLE_WRITE_FS_LAYOUT
190    MasterProcedureTestingUtility.testRollbackAndDoubleExecution(procExec, procId, lastStep);
191
192    TableName tableName = htd.getTableName();
193    MasterProcedureTestingUtility.validateTableDeletion(getMaster(), tableName);
194
195    // are we able to create the table after a rollback?
196    resetProcExecutorTestingKillFlag();
197    testSimpleCreate(tableName, splitKeys);
198  }
199
200  public static class CreateTableProcedureOnHDFSFailure extends CreateTableProcedure {
201    private boolean failOnce = false;
202
203    public CreateTableProcedureOnHDFSFailure() {
204      // Required by the Procedure framework to create the procedure on replay
205      super();
206    }
207
208    public CreateTableProcedureOnHDFSFailure(final MasterProcedureEnv env,
209      final TableDescriptor tableDescriptor, final RegionInfo[] newRegions)
210      throws HBaseIOException {
211      super(env, tableDescriptor, newRegions);
212    }
213
214    @Override
215    protected Flow executeFromState(MasterProcedureEnv env,
216      MasterProcedureProtos.CreateTableState state) throws InterruptedException {
217
218      if (!failOnce &&
219        state == MasterProcedureProtos.CreateTableState.CREATE_TABLE_WRITE_FS_LAYOUT) {
220        try {
221          // To emulate an HDFS failure, create only the first region directory
222          RegionInfo regionInfo = getFirstRegionInfo();
223          Configuration conf = env.getMasterConfiguration();
224          MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
225          Path tempdir = mfs.getTempDir();
226          Path tableDir = FSUtils.getTableDir(tempdir, regionInfo.getTable());
227          Path regionDir = FSUtils.getRegionDirFromTableDir(tableDir, regionInfo);
228          FileSystem fs = FileSystem.get(conf);
229          fs.mkdirs(regionDir);
230
231          failOnce = true;
232          return Flow.HAS_MORE_STATE;
233        } catch (IOException e) {
234          fail("failed to create a region directory: " + e);
235        }
236      }
237
238      return super.executeFromState(env, state);
239    }
240  }
241
242  @Test
243  public void testOnHDFSFailure() throws Exception {
244    final TableName tableName = TableName.valueOf(name.getMethodName());
245
246    // create the table
247    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
248    final byte[][] splitKeys = new byte[][] {
249      Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c")
250    };
251    TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, "f1", "f2");
252    RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, splitKeys);
253    long procId = ProcedureTestingUtility.submitAndWait(procExec,
254      new CreateTableProcedureOnHDFSFailure(procExec.getEnvironment(), htd, regions));
255    ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
256  }
257}