001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.procedure;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertTrue;
022import static org.junit.Assert.fail;
023
024import java.io.IOException;
025import org.apache.hadoop.conf.Configuration;
026import org.apache.hadoop.fs.FileSystem;
027import org.apache.hadoop.fs.Path;
028import org.apache.hadoop.hbase.DoNotRetryIOException;
029import org.apache.hadoop.hbase.HBaseClassTestRule;
030import org.apache.hadoop.hbase.HBaseIOException;
031import org.apache.hadoop.hbase.TableExistsException;
032import org.apache.hadoop.hbase.TableName;
033import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
034import org.apache.hadoop.hbase.client.RegionInfo;
035import org.apache.hadoop.hbase.client.TableDescriptor;
036import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
037import org.apache.hadoop.hbase.master.MasterFileSystem;
038import org.apache.hadoop.hbase.procedure2.Procedure;
039import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
040import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
041import org.apache.hadoop.hbase.testclassification.MasterTests;
042import org.apache.hadoop.hbase.testclassification.MediumTests;
043import org.apache.hadoop.hbase.util.Bytes;
044import org.apache.hadoop.hbase.util.CommonFSUtils;
045import org.apache.hadoop.hbase.util.FSUtils;
046import org.apache.hadoop.hbase.util.ModifyRegionUtils;
047import org.apache.hadoop.hbase.util.TableDescriptorChecker;
048import org.junit.ClassRule;
049import org.junit.Rule;
050import org.junit.Test;
051import org.junit.experimental.categories.Category;
052import org.junit.rules.TestName;
053
054import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos;
055
056@Category({MasterTests.class, MediumTests.class})
057public class TestCreateTableProcedure extends TestTableDDLProcedureBase {
058
059  @ClassRule
060  public static final HBaseClassTestRule CLASS_RULE =
061    HBaseClassTestRule.forClass(TestCreateTableProcedure.class);
062
063  private static final String F1 = "f1";
064  private static final String F2 = "f2";
065
066  @Rule
067  public TestName name = new TestName();
068
069  @Test
070  public void testSimpleCreate() throws Exception {
071    final TableName tableName = TableName.valueOf(name.getMethodName());
072    final byte[][] splitKeys = null;
073    testSimpleCreate(tableName, splitKeys);
074  }
075
076  @Test
077  public void testSimpleCreateWithSplits() throws Exception {
078    final TableName tableName = TableName.valueOf(name.getMethodName());
079    final byte[][] splitKeys = new byte[][] {
080      Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c")
081    };
082    testSimpleCreate(tableName, splitKeys);
083  }
084
085  private void testSimpleCreate(final TableName tableName, byte[][] splitKeys) throws Exception {
086    RegionInfo[] regions = MasterProcedureTestingUtility.createTable(
087      getMasterProcedureExecutor(), tableName, splitKeys, F1, F2);
088    MasterProcedureTestingUtility.validateTableCreation(getMaster(), tableName, regions, F1, F2);
089  }
090
091  @Test
092  public void testCreateWithoutColumnFamily() throws Exception {
093    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
094    final TableName tableName = TableName.valueOf(name.getMethodName());
095    // create table with 0 families will fail
096    final TableDescriptorBuilder builder =
097      TableDescriptorBuilder.newBuilder(MasterProcedureTestingUtility.createHTD(tableName));
098
099    // disable sanity check
100    builder.setValue(TableDescriptorChecker.TABLE_SANITY_CHECKS, Boolean.FALSE.toString());
101    TableDescriptor htd = builder.build();
102    final RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, null);
103
104    long procId =
105        ProcedureTestingUtility.submitAndWait(procExec,
106            new CreateTableProcedure(procExec.getEnvironment(), htd, regions));
107    final Procedure<?> result = procExec.getResult(procId);
108    assertEquals(true, result.isFailed());
109    Throwable cause = ProcedureTestingUtility.getExceptionCause(result);
110    assertTrue("expected DoNotRetryIOException, got " + cause,
111        cause instanceof DoNotRetryIOException);
112  }
113
114  @Test(expected=TableExistsException.class)
115  public void testCreateExisting() throws Exception {
116    final TableName tableName = TableName.valueOf(name.getMethodName());
117    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
118    final TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, "f");
119    final RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, null);
120
121    // create the table
122    long procId1 = procExec.submitProcedure(
123      new CreateTableProcedure(procExec.getEnvironment(), htd, regions));
124
125    // create another with the same name
126    ProcedurePrepareLatch latch2 = new ProcedurePrepareLatch.CompatibilityLatch();
127    long procId2 = procExec.submitProcedure(
128      new CreateTableProcedure(procExec.getEnvironment(), htd, regions, latch2));
129
130    ProcedureTestingUtility.waitProcedure(procExec, procId1);
131    ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId1));
132
133    ProcedureTestingUtility.waitProcedure(procExec, procId2);
134    latch2.await();
135  }
136
137  @Test
138  public void testRecoveryAndDoubleExecution() throws Exception {
139    final TableName tableName = TableName.valueOf(name.getMethodName());
140
141    // create the table
142    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
143    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
144
145    // Start the Create procedure && kill the executor
146    byte[][] splitKeys = null;
147    TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, "f1", "f2");
148    RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, splitKeys);
149    long procId = procExec.submitProcedure(
150      new CreateTableProcedure(procExec.getEnvironment(), htd, regions));
151
152    // Restart the executor and execute the step twice
153    MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId);
154    MasterProcedureTestingUtility.validateTableCreation(getMaster(), tableName, regions, F1, F2);
155  }
156
157  @Test
158  public void testRollbackAndDoubleExecution() throws Exception {
159    final TableName tableName = TableName.valueOf(name.getMethodName());
160    testRollbackAndDoubleExecution(TableDescriptorBuilder
161      .newBuilder(MasterProcedureTestingUtility.createHTD(tableName, F1, F2)));
162  }
163
164  @Test
165  public void testRollbackAndDoubleExecutionOnMobTable() throws Exception {
166    final TableName tableName = TableName.valueOf(name.getMethodName());
167    TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, F1, F2);
168    TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(htd)
169            .modifyColumnFamily(ColumnFamilyDescriptorBuilder
170              .newBuilder(htd.getColumnFamily(Bytes.toBytes(F1)))
171              .setMobEnabled(true)
172              .build());
173    testRollbackAndDoubleExecution(builder);
174  }
175
176  private void testRollbackAndDoubleExecution(TableDescriptorBuilder builder) throws Exception {
177    // create the table
178    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
179    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
180
181    // Start the Create procedure && kill the executor
182    final byte[][] splitKeys = new byte[][] {
183      Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c")
184    };
185    builder.setRegionReplication(3);
186    TableDescriptor htd = builder.build();
187    RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, splitKeys);
188    long procId = procExec.submitProcedure(
189      new CreateTableProcedure(procExec.getEnvironment(), htd, regions));
190
191    int lastStep = 2; // failing before CREATE_TABLE_WRITE_FS_LAYOUT
192    MasterProcedureTestingUtility.testRollbackAndDoubleExecution(procExec, procId, lastStep);
193
194    TableName tableName = htd.getTableName();
195    MasterProcedureTestingUtility.validateTableDeletion(getMaster(), tableName);
196
197    // are we able to create the table after a rollback?
198    resetProcExecutorTestingKillFlag();
199    testSimpleCreate(tableName, splitKeys);
200  }
201
202  public static class CreateTableProcedureOnHDFSFailure extends CreateTableProcedure {
203    private boolean failOnce = false;
204
205    public CreateTableProcedureOnHDFSFailure() {
206      // Required by the Procedure framework to create the procedure on replay
207      super();
208    }
209
210    public CreateTableProcedureOnHDFSFailure(final MasterProcedureEnv env,
211      final TableDescriptor tableDescriptor, final RegionInfo[] newRegions)
212      throws HBaseIOException {
213      super(env, tableDescriptor, newRegions);
214    }
215
216    @Override
217    protected Flow executeFromState(MasterProcedureEnv env,
218      MasterProcedureProtos.CreateTableState state) throws InterruptedException {
219
220      if (!failOnce &&
221        state == MasterProcedureProtos.CreateTableState.CREATE_TABLE_WRITE_FS_LAYOUT) {
222        try {
223          // To emulate an HDFS failure, create only the first region directory
224          RegionInfo regionInfo = getFirstRegionInfo();
225          Configuration conf = env.getMasterConfiguration();
226          MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
227          Path tempdir = mfs.getTempDir();
228          Path tableDir = CommonFSUtils.getTableDir(tempdir, regionInfo.getTable());
229          Path regionDir = FSUtils.getRegionDirFromTableDir(tableDir, regionInfo);
230          FileSystem fs = FileSystem.get(conf);
231          fs.mkdirs(regionDir);
232
233          failOnce = true;
234          return Flow.HAS_MORE_STATE;
235        } catch (IOException e) {
236          fail("failed to create a region directory: " + e);
237        }
238      }
239
240      return super.executeFromState(env, state);
241    }
242  }
243
244  @Test
245  public void testOnHDFSFailure() throws Exception {
246    final TableName tableName = TableName.valueOf(name.getMethodName());
247
248    // create the table
249    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
250    final byte[][] splitKeys = new byte[][] {
251      Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c")
252    };
253    TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, "f1", "f2");
254    RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, splitKeys);
255    long procId = ProcedureTestingUtility.submitAndWait(procExec,
256      new CreateTableProcedureOnHDFSFailure(procExec.getEnvironment(), htd, regions));
257    ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
258  }
259}