001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.procedure;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertTrue;
022import static org.junit.Assert.fail;
023
024import java.io.IOException;
025import org.apache.hadoop.conf.Configuration;
026import org.apache.hadoop.fs.FileSystem;
027import org.apache.hadoop.fs.Path;
028import org.apache.hadoop.hbase.DoNotRetryIOException;
029import org.apache.hadoop.hbase.HBaseClassTestRule;
030import org.apache.hadoop.hbase.HBaseIOException;
031import org.apache.hadoop.hbase.TableExistsException;
032import org.apache.hadoop.hbase.TableName;
033import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
034import org.apache.hadoop.hbase.client.RegionInfo;
035import org.apache.hadoop.hbase.client.TableDescriptor;
036import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
037import org.apache.hadoop.hbase.master.MasterFileSystem;
038import org.apache.hadoop.hbase.procedure2.Procedure;
039import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
040import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
041import org.apache.hadoop.hbase.testclassification.MasterTests;
042import org.apache.hadoop.hbase.testclassification.MediumTests;
043import org.apache.hadoop.hbase.util.Bytes;
044import org.apache.hadoop.hbase.util.FSUtils;
045import org.apache.hadoop.hbase.util.ModifyRegionUtils;
046import org.apache.hadoop.hbase.util.TableDescriptorChecker;
047import org.junit.ClassRule;
048import org.junit.Rule;
049import org.junit.Test;
050import org.junit.experimental.categories.Category;
051import org.junit.rules.TestName;
052
053import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos;
054
055@Category({MasterTests.class, MediumTests.class})
056public class TestCreateTableProcedure extends TestTableDDLProcedureBase {
057
058  @ClassRule
059  public static final HBaseClassTestRule CLASS_RULE =
060    HBaseClassTestRule.forClass(TestCreateTableProcedure.class);
061
062  private static final String F1 = "f1";
063  private static final String F2 = "f2";
064
065  @Rule
066  public TestName name = new TestName();
067
068  @Test
069  public void testSimpleCreate() throws Exception {
070    final TableName tableName = TableName.valueOf(name.getMethodName());
071    final byte[][] splitKeys = null;
072    testSimpleCreate(tableName, splitKeys);
073  }
074
075  @Test
076  public void testSimpleCreateWithSplits() throws Exception {
077    final TableName tableName = TableName.valueOf(name.getMethodName());
078    final byte[][] splitKeys = new byte[][] {
079      Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c")
080    };
081    testSimpleCreate(tableName, splitKeys);
082  }
083
084  private void testSimpleCreate(final TableName tableName, byte[][] splitKeys) throws Exception {
085    RegionInfo[] regions = MasterProcedureTestingUtility.createTable(
086      getMasterProcedureExecutor(), tableName, splitKeys, F1, F2);
087    MasterProcedureTestingUtility.validateTableCreation(getMaster(), tableName, regions, F1, F2);
088  }
089
090  @Test
091  public void testCreateWithoutColumnFamily() throws Exception {
092    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
093    final TableName tableName = TableName.valueOf(name.getMethodName());
094    // create table with 0 families will fail
095    final TableDescriptorBuilder builder =
096      TableDescriptorBuilder.newBuilder(MasterProcedureTestingUtility.createHTD(tableName));
097
098    // disable sanity check
099    builder.setValue(TableDescriptorChecker.TABLE_SANITY_CHECKS, Boolean.FALSE.toString());
100    TableDescriptor htd = builder.build();
101    final RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, null);
102
103    long procId =
104        ProcedureTestingUtility.submitAndWait(procExec,
105            new CreateTableProcedure(procExec.getEnvironment(), htd, regions));
106    final Procedure<?> result = procExec.getResult(procId);
107    assertEquals(true, result.isFailed());
108    Throwable cause = ProcedureTestingUtility.getExceptionCause(result);
109    assertTrue("expected DoNotRetryIOException, got " + cause,
110        cause instanceof DoNotRetryIOException);
111  }
112
113  @Test(expected=TableExistsException.class)
114  public void testCreateExisting() throws Exception {
115    final TableName tableName = TableName.valueOf(name.getMethodName());
116    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
117    final TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, "f");
118    final RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, null);
119
120    // create the table
121    long procId1 = procExec.submitProcedure(
122      new CreateTableProcedure(procExec.getEnvironment(), htd, regions));
123
124    // create another with the same name
125    ProcedurePrepareLatch latch2 = new ProcedurePrepareLatch.CompatibilityLatch();
126    long procId2 = procExec.submitProcedure(
127      new CreateTableProcedure(procExec.getEnvironment(), htd, regions, latch2));
128
129    ProcedureTestingUtility.waitProcedure(procExec, procId1);
130    ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId1));
131
132    ProcedureTestingUtility.waitProcedure(procExec, procId2);
133    latch2.await();
134  }
135
136  @Test
137  public void testRecoveryAndDoubleExecution() throws Exception {
138    final TableName tableName = TableName.valueOf(name.getMethodName());
139
140    // create the table
141    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
142    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
143
144    // Start the Create procedure && kill the executor
145    byte[][] splitKeys = null;
146    TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, "f1", "f2");
147    RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, splitKeys);
148    long procId = procExec.submitProcedure(
149      new CreateTableProcedure(procExec.getEnvironment(), htd, regions));
150
151    // Restart the executor and execute the step twice
152    MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId);
153    MasterProcedureTestingUtility.validateTableCreation(getMaster(), tableName, regions, F1, F2);
154  }
155
156  @Test
157  public void testRollbackAndDoubleExecution() throws Exception {
158    final TableName tableName = TableName.valueOf(name.getMethodName());
159    testRollbackAndDoubleExecution(TableDescriptorBuilder
160      .newBuilder(MasterProcedureTestingUtility.createHTD(tableName, F1, F2)));
161  }
162
163  @Test
164  public void testRollbackAndDoubleExecutionOnMobTable() throws Exception {
165    final TableName tableName = TableName.valueOf(name.getMethodName());
166    TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, F1, F2);
167    TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(htd)
168            .modifyColumnFamily(ColumnFamilyDescriptorBuilder
169              .newBuilder(htd.getColumnFamily(Bytes.toBytes(F1)))
170              .setMobEnabled(true)
171              .build());
172    testRollbackAndDoubleExecution(builder);
173  }
174
175  private void testRollbackAndDoubleExecution(TableDescriptorBuilder builder) throws Exception {
176    // create the table
177    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
178    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
179
180    // Start the Create procedure && kill the executor
181    final byte[][] splitKeys = new byte[][] {
182      Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c")
183    };
184    builder.setRegionReplication(3);
185    TableDescriptor htd = builder.build();
186    RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, splitKeys);
187    long procId = procExec.submitProcedure(
188      new CreateTableProcedure(procExec.getEnvironment(), htd, regions));
189
190    int lastStep = 2; // failing before CREATE_TABLE_WRITE_FS_LAYOUT
191    MasterProcedureTestingUtility.testRollbackAndDoubleExecution(procExec, procId, lastStep);
192
193    TableName tableName = htd.getTableName();
194    MasterProcedureTestingUtility.validateTableDeletion(getMaster(), tableName);
195
196    // are we able to create the table after a rollback?
197    resetProcExecutorTestingKillFlag();
198    testSimpleCreate(tableName, splitKeys);
199  }
200
201  public static class CreateTableProcedureOnHDFSFailure extends CreateTableProcedure {
202    private boolean failOnce = false;
203
204    public CreateTableProcedureOnHDFSFailure() {
205      // Required by the Procedure framework to create the procedure on replay
206      super();
207    }
208
209    public CreateTableProcedureOnHDFSFailure(final MasterProcedureEnv env,
210      final TableDescriptor tableDescriptor, final RegionInfo[] newRegions)
211      throws HBaseIOException {
212      super(env, tableDescriptor, newRegions);
213    }
214
215    @Override
216    protected Flow executeFromState(MasterProcedureEnv env,
217      MasterProcedureProtos.CreateTableState state) throws InterruptedException {
218
219      if (!failOnce &&
220        state == MasterProcedureProtos.CreateTableState.CREATE_TABLE_WRITE_FS_LAYOUT) {
221        try {
222          // To emulate an HDFS failure, create only the first region directory
223          RegionInfo regionInfo = getFirstRegionInfo();
224          Configuration conf = env.getMasterConfiguration();
225          MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
226          Path tempdir = mfs.getTempDir();
227          Path tableDir = FSUtils.getTableDir(tempdir, regionInfo.getTable());
228          Path regionDir = FSUtils.getRegionDirFromTableDir(tableDir, regionInfo);
229          FileSystem fs = FileSystem.get(conf);
230          fs.mkdirs(regionDir);
231
232          failOnce = true;
233          return Flow.HAS_MORE_STATE;
234        } catch (IOException e) {
235          fail("failed to create a region directory: " + e);
236        }
237      }
238
239      return super.executeFromState(env, state);
240    }
241  }
242
243  @Test
244  public void testOnHDFSFailure() throws Exception {
245    final TableName tableName = TableName.valueOf(name.getMethodName());
246
247    // create the table
248    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
249    final byte[][] splitKeys = new byte[][] {
250      Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c")
251    };
252    TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, "f1", "f2");
253    RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, splitKeys);
254    long procId = ProcedureTestingUtility.submitAndWait(procExec,
255      new CreateTableProcedureOnHDFSFailure(procExec.getEnvironment(), htd, regions));
256    ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
257  }
258}