001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.procedure;
019
020import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACKER_IMPL;
021import static org.junit.Assert.assertEquals;
022import static org.junit.Assert.assertTrue;
023import static org.junit.Assert.fail;
024
025import java.io.IOException;
026import org.apache.hadoop.conf.Configuration;
027import org.apache.hadoop.fs.FileSystem;
028import org.apache.hadoop.fs.Path;
029import org.apache.hadoop.hbase.DoNotRetryIOException;
030import org.apache.hadoop.hbase.HBaseClassTestRule;
031import org.apache.hadoop.hbase.HBaseIOException;
032import org.apache.hadoop.hbase.TableExistsException;
033import org.apache.hadoop.hbase.TableName;
034import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
035import org.apache.hadoop.hbase.client.RegionInfo;
036import org.apache.hadoop.hbase.client.TableDescriptor;
037import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
038import org.apache.hadoop.hbase.master.MasterFileSystem;
039import org.apache.hadoop.hbase.procedure2.Procedure;
040import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
041import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
042import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
043import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerForTest;
044import org.apache.hadoop.hbase.testclassification.MasterTests;
045import org.apache.hadoop.hbase.testclassification.MediumTests;
046import org.apache.hadoop.hbase.util.Bytes;
047import org.apache.hadoop.hbase.util.CommonFSUtils;
048import org.apache.hadoop.hbase.util.FSUtils;
049import org.apache.hadoop.hbase.util.ModifyRegionUtils;
050import org.apache.hadoop.hbase.util.TableDescriptorChecker;
051import org.junit.ClassRule;
052import org.junit.Rule;
053import org.junit.Test;
054import org.junit.experimental.categories.Category;
055import org.junit.rules.TestName;
056
057import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos;
058
059@Category({ MasterTests.class, MediumTests.class })
060public class TestCreateTableProcedure extends TestTableDDLProcedureBase {
061
062  @ClassRule
063  public static final HBaseClassTestRule CLASS_RULE =
064    HBaseClassTestRule.forClass(TestCreateTableProcedure.class);
065
066  private static final String F1 = "f1";
067  private static final String F2 = "f2";
068
069  @Rule
070  public TestName name = new TestName();
071
072  @Test
073  public void testSimpleCreate() throws Exception {
074    final TableName tableName = TableName.valueOf(name.getMethodName());
075    final byte[][] splitKeys = null;
076    testSimpleCreate(tableName, splitKeys);
077  }
078
079  @Test
080  public void testSimpleCreateWithSplits() throws Exception {
081    final TableName tableName = TableName.valueOf(name.getMethodName());
082    final byte[][] splitKeys =
083      new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c") };
084    testSimpleCreate(tableName, splitKeys);
085  }
086
087  private void testSimpleCreate(final TableName tableName, byte[][] splitKeys) throws Exception {
088    RegionInfo[] regions = MasterProcedureTestingUtility.createTable(getMasterProcedureExecutor(),
089      tableName, splitKeys, F1, F2);
090    MasterProcedureTestingUtility.validateTableCreation(getMaster(), tableName, regions, F1, F2);
091  }
092
093  @Test
094  public void testCreateWithTrackImpl() throws Exception {
095    final TableName tableName = TableName.valueOf(name.getMethodName());
096    ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
097    TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, F1);
098    String trackerName = StoreFileTrackerForTest.class.getName();
099    htd = TableDescriptorBuilder.newBuilder(htd).setValue(TRACKER_IMPL, trackerName).build();
100    RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, null);
101    long procId = ProcedureTestingUtility.submitAndWait(procExec,
102      new CreateTableProcedure(procExec.getEnvironment(), htd, regions));
103    ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId));
104    htd = getMaster().getTableDescriptors().get(tableName);
105    assertEquals(trackerName, htd.getValue(TRACKER_IMPL));
106  }
107
108  @Test
109  public void testCreateWithFileBasedStoreTrackerImpl() throws Exception {
110    ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
111    procExec.getEnvironment().getMasterConfiguration().set(StoreFileTrackerFactory.TRACKER_IMPL,
112      StoreFileTrackerFactory.Trackers.FILE.name());
113    final TableName tableName = TableName.valueOf(name.getMethodName());
114    TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, F1);
115    RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, null);
116    long procId = ProcedureTestingUtility.submitAndWait(procExec,
117      new CreateTableProcedure(procExec.getEnvironment(), htd, regions));
118    ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId));
119    htd = getMaster().getTableDescriptors().get(tableName);
120    assertEquals(StoreFileTrackerFactory.Trackers.FILE.name(), htd.getValue(TRACKER_IMPL));
121  }
122
123  @Test
124  public void testCreateWithoutColumnFamily() throws Exception {
125    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
126    final TableName tableName = TableName.valueOf(name.getMethodName());
127    // create table with 0 families will fail
128    final TableDescriptorBuilder builder =
129      TableDescriptorBuilder.newBuilder(MasterProcedureTestingUtility.createHTD(tableName));
130
131    // disable sanity check
132    builder.setValue(TableDescriptorChecker.TABLE_SANITY_CHECKS, Boolean.FALSE.toString());
133    TableDescriptor htd = builder.build();
134    final RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, null);
135
136    long procId = ProcedureTestingUtility.submitAndWait(procExec,
137      new CreateTableProcedure(procExec.getEnvironment(), htd, regions));
138    final Procedure<?> result = procExec.getResult(procId);
139    assertEquals(true, result.isFailed());
140    Throwable cause = ProcedureTestingUtility.getExceptionCause(result);
141    assertTrue("expected DoNotRetryIOException, got " + cause,
142      cause instanceof DoNotRetryIOException);
143  }
144
145  @Test(expected = TableExistsException.class)
146  public void testCreateExisting() throws Exception {
147    final TableName tableName = TableName.valueOf(name.getMethodName());
148    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
149    final TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, "f");
150    final RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, null);
151
152    // create the table
153    long procId1 =
154      procExec.submitProcedure(new CreateTableProcedure(procExec.getEnvironment(), htd, regions));
155
156    // create another with the same name
157    ProcedurePrepareLatch latch2 = new ProcedurePrepareLatch.CompatibilityLatch();
158    long procId2 = procExec
159      .submitProcedure(new CreateTableProcedure(procExec.getEnvironment(), htd, regions, latch2));
160
161    ProcedureTestingUtility.waitProcedure(procExec, procId1);
162    ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId1));
163
164    ProcedureTestingUtility.waitProcedure(procExec, procId2);
165    latch2.await();
166  }
167
168  @Test
169  public void testRecoveryAndDoubleExecution() throws Exception {
170    final TableName tableName = TableName.valueOf(name.getMethodName());
171
172    // create the table
173    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
174    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
175
176    // Start the Create procedure && kill the executor
177    byte[][] splitKeys = null;
178    TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, "f1", "f2");
179    RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, splitKeys);
180    long procId =
181      procExec.submitProcedure(new CreateTableProcedure(procExec.getEnvironment(), htd, regions));
182
183    // Restart the executor and execute the step twice
184    MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId);
185    MasterProcedureTestingUtility.validateTableCreation(getMaster(), tableName, regions, F1, F2);
186  }
187
188  @Test
189  public void testRollbackAndDoubleExecution() throws Exception {
190    final TableName tableName = TableName.valueOf(name.getMethodName());
191    testRollbackAndDoubleExecution(TableDescriptorBuilder
192      .newBuilder(MasterProcedureTestingUtility.createHTD(tableName, F1, F2)));
193  }
194
195  @Test
196  public void testRollbackAndDoubleExecutionOnMobTable() throws Exception {
197    final TableName tableName = TableName.valueOf(name.getMethodName());
198    TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, F1, F2);
199    TableDescriptorBuilder builder =
200      TableDescriptorBuilder.newBuilder(htd).modifyColumnFamily(ColumnFamilyDescriptorBuilder
201        .newBuilder(htd.getColumnFamily(Bytes.toBytes(F1))).setMobEnabled(true).build());
202    testRollbackAndDoubleExecution(builder);
203  }
204
205  private void testRollbackAndDoubleExecution(TableDescriptorBuilder builder) throws Exception {
206    // create the table
207    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
208    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
209
210    // Start the Create procedure && kill the executor
211    final byte[][] splitKeys =
212      new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c") };
213    builder.setRegionReplication(3);
214    TableDescriptor htd = builder.build();
215    RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, splitKeys);
216    long procId =
217      procExec.submitProcedure(new CreateTableProcedure(procExec.getEnvironment(), htd, regions));
218
219    int lastStep = 2; // failing before CREATE_TABLE_WRITE_FS_LAYOUT
220    MasterProcedureTestingUtility.testRollbackAndDoubleExecution(procExec, procId, lastStep);
221
222    TableName tableName = htd.getTableName();
223    MasterProcedureTestingUtility.validateTableDeletion(getMaster(), tableName);
224
225    // are we able to create the table after a rollback?
226    resetProcExecutorTestingKillFlag();
227    testSimpleCreate(tableName, splitKeys);
228  }
229
230  public static class CreateTableProcedureOnHDFSFailure extends CreateTableProcedure {
231    private boolean failOnce = false;
232
233    public CreateTableProcedureOnHDFSFailure() {
234      // Required by the Procedure framework to create the procedure on replay
235      super();
236    }
237
238    public CreateTableProcedureOnHDFSFailure(final MasterProcedureEnv env,
239      final TableDescriptor tableDescriptor, final RegionInfo[] newRegions)
240      throws HBaseIOException {
241      super(env, tableDescriptor, newRegions);
242    }
243
244    @Override
245    protected Flow executeFromState(MasterProcedureEnv env,
246      MasterProcedureProtos.CreateTableState state) throws InterruptedException {
247
248      if (
249        !failOnce && state == MasterProcedureProtos.CreateTableState.CREATE_TABLE_WRITE_FS_LAYOUT
250      ) {
251        try {
252          // To emulate an HDFS failure, create only the first region directory
253          RegionInfo regionInfo = getFirstRegionInfo();
254          Configuration conf = env.getMasterConfiguration();
255          MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
256          Path tempdir = mfs.getTempDir();
257          Path tableDir = CommonFSUtils.getTableDir(tempdir, regionInfo.getTable());
258          Path regionDir = FSUtils.getRegionDirFromTableDir(tableDir, regionInfo);
259          FileSystem fs = FileSystem.get(conf);
260          fs.mkdirs(regionDir);
261
262          failOnce = true;
263          return Flow.HAS_MORE_STATE;
264        } catch (IOException e) {
265          fail("failed to create a region directory: " + e);
266        }
267      }
268
269      return super.executeFromState(env, state);
270    }
271  }
272
273  @Test
274  public void testOnHDFSFailure() throws Exception {
275    final TableName tableName = TableName.valueOf(name.getMethodName());
276
277    // create the table
278    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
279    final byte[][] splitKeys =
280      new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c") };
281    TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, "f1", "f2");
282    RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, splitKeys);
283    long procId = ProcedureTestingUtility.submitAndWait(procExec,
284      new CreateTableProcedureOnHDFSFailure(procExec.getEnvironment(), htd, regions));
285    ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
286  }
287}