001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.procedure;
019
020import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACKER_IMPL;
021import static org.junit.Assert.assertEquals;
022import static org.junit.Assert.assertFalse;
023import static org.junit.Assert.assertTrue;
024import static org.junit.Assert.fail;
025
026import java.io.IOException;
027import org.apache.hadoop.conf.Configuration;
028import org.apache.hadoop.fs.FileSystem;
029import org.apache.hadoop.fs.Path;
030import org.apache.hadoop.hbase.DoNotRetryIOException;
031import org.apache.hadoop.hbase.HBaseClassTestRule;
032import org.apache.hadoop.hbase.HBaseIOException;
033import org.apache.hadoop.hbase.TableExistsException;
034import org.apache.hadoop.hbase.TableName;
035import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
036import org.apache.hadoop.hbase.client.RegionInfo;
037import org.apache.hadoop.hbase.client.TableDescriptor;
038import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
039import org.apache.hadoop.hbase.master.MasterFileSystem;
040import org.apache.hadoop.hbase.procedure2.Procedure;
041import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
042import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
043import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
044import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerForTest;
045import org.apache.hadoop.hbase.testclassification.MasterTests;
046import org.apache.hadoop.hbase.testclassification.MediumTests;
047import org.apache.hadoop.hbase.util.Bytes;
048import org.apache.hadoop.hbase.util.CommonFSUtils;
049import org.apache.hadoop.hbase.util.FSUtils;
050import org.apache.hadoop.hbase.util.ModifyRegionUtils;
051import org.apache.hadoop.hbase.util.TableDescriptorChecker;
052import org.junit.ClassRule;
053import org.junit.Rule;
054import org.junit.Test;
055import org.junit.experimental.categories.Category;
056import org.junit.rules.TestName;
057
058import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos;
059
060@Category({ MasterTests.class, MediumTests.class })
061public class TestCreateTableProcedure extends TestTableDDLProcedureBase {
062
063  @ClassRule
064  public static final HBaseClassTestRule CLASS_RULE =
065    HBaseClassTestRule.forClass(TestCreateTableProcedure.class);
066
067  private static final String F1 = "f1";
068  private static final String F2 = "f2";
069
070  @Rule
071  public TestName name = new TestName();
072
073  @Test
074  public void testSimpleCreate() throws Exception {
075    final TableName tableName = TableName.valueOf(name.getMethodName());
076    final byte[][] splitKeys = null;
077    testSimpleCreate(tableName, splitKeys);
078  }
079
080  @Test
081  public void testSimpleCreateWithSplits() throws Exception {
082    final TableName tableName = TableName.valueOf(name.getMethodName());
083    final byte[][] splitKeys =
084      new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c") };
085    testSimpleCreate(tableName, splitKeys);
086  }
087
088  private void testSimpleCreate(final TableName tableName, byte[][] splitKeys) throws Exception {
089    RegionInfo[] regions = MasterProcedureTestingUtility.createTable(getMasterProcedureExecutor(),
090      tableName, splitKeys, F1, F2);
091    MasterProcedureTestingUtility.validateTableCreation(getMaster(), tableName, regions, F1, F2);
092  }
093
094  @Test
095  public void testCreateWithTrackImpl() throws Exception {
096    final TableName tableName = TableName.valueOf(name.getMethodName());
097    ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
098    TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, F1);
099    String trackerName = StoreFileTrackerForTest.class.getName();
100    htd = TableDescriptorBuilder.newBuilder(htd).setValue(TRACKER_IMPL, trackerName).build();
101    RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, null);
102    long procId = ProcedureTestingUtility.submitAndWait(procExec,
103      new CreateTableProcedure(procExec.getEnvironment(), htd, regions));
104    ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId));
105    htd = getMaster().getTableDescriptors().get(tableName);
106    assertEquals(trackerName, htd.getValue(TRACKER_IMPL));
107  }
108
109  @Test
110  public void testCreateWithFileBasedStoreTrackerImpl() throws Exception {
111    ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
112    procExec.getEnvironment().getMasterConfiguration().set(StoreFileTrackerFactory.TRACKER_IMPL,
113      StoreFileTrackerFactory.Trackers.FILE.name());
114    final TableName tableName = TableName.valueOf(name.getMethodName());
115    TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, F1);
116    RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, null);
117    long procId = ProcedureTestingUtility.submitAndWait(procExec,
118      new CreateTableProcedure(procExec.getEnvironment(), htd, regions));
119    ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId));
120    htd = getMaster().getTableDescriptors().get(tableName);
121    assertEquals(StoreFileTrackerFactory.Trackers.FILE.name(), htd.getValue(TRACKER_IMPL));
122  }
123
124  @Test
125  public void testCreateWithoutColumnFamily() throws Exception {
126    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
127    final TableName tableName = TableName.valueOf(name.getMethodName());
128    // create table with 0 families will fail
129    final TableDescriptorBuilder builder =
130      TableDescriptorBuilder.newBuilder(MasterProcedureTestingUtility.createHTD(tableName));
131
132    // disable sanity check
133    builder.setValue(TableDescriptorChecker.TABLE_SANITY_CHECKS, Boolean.FALSE.toString());
134    TableDescriptor htd = builder.build();
135    final RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, null);
136
137    long procId = ProcedureTestingUtility.submitAndWait(procExec,
138      new CreateTableProcedure(procExec.getEnvironment(), htd, regions));
139    final Procedure<?> result = procExec.getResult(procId);
140    assertEquals(true, result.isFailed());
141    Throwable cause = ProcedureTestingUtility.getExceptionCause(result);
142    assertTrue("expected DoNotRetryIOException, got " + cause,
143      cause instanceof DoNotRetryIOException);
144  }
145
146  @Test(expected = TableExistsException.class)
147  public void testCreateExisting() throws Exception {
148    final TableName tableName = TableName.valueOf(name.getMethodName());
149    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
150    final TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, "f");
151    final RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, null);
152
153    // create the table
154    long procId1 =
155      procExec.submitProcedure(new CreateTableProcedure(procExec.getEnvironment(), htd, regions));
156
157    // create another with the same name
158    ProcedurePrepareLatch latch2 = new ProcedurePrepareLatch.CompatibilityLatch();
159    long procId2 = procExec
160      .submitProcedure(new CreateTableProcedure(procExec.getEnvironment(), htd, regions, latch2));
161
162    ProcedureTestingUtility.waitProcedure(procExec, procId1);
163    ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId1));
164
165    ProcedureTestingUtility.waitProcedure(procExec, procId2);
166    latch2.await();
167  }
168
169  @Test
170  public void testRecoveryAndDoubleExecution() throws Exception {
171    final TableName tableName = TableName.valueOf(name.getMethodName());
172
173    // create the table
174    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
175    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
176
177    // Start the Create procedure && kill the executor
178    byte[][] splitKeys = null;
179    TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, "f1", "f2");
180    RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, splitKeys);
181    long procId =
182      procExec.submitProcedure(new CreateTableProcedure(procExec.getEnvironment(), htd, regions));
183
184    // Restart the executor and execute the step twice
185    MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId);
186    MasterProcedureTestingUtility.validateTableCreation(getMaster(), tableName, regions, F1, F2);
187  }
188
189  @Test
190  public void testRollbackAndDoubleExecution() throws Exception {
191    final TableName tableName = TableName.valueOf(name.getMethodName());
192    testRollbackAndDoubleExecution(TableDescriptorBuilder
193      .newBuilder(MasterProcedureTestingUtility.createHTD(tableName, F1, F2)));
194  }
195
196  @Test
197  public void testRollbackAndDoubleExecutionOnMobTable() throws Exception {
198    final TableName tableName = TableName.valueOf(name.getMethodName());
199    TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, F1, F2);
200    TableDescriptorBuilder builder =
201      TableDescriptorBuilder.newBuilder(htd).modifyColumnFamily(ColumnFamilyDescriptorBuilder
202        .newBuilder(htd.getColumnFamily(Bytes.toBytes(F1))).setMobEnabled(true).build());
203    testRollbackAndDoubleExecution(builder);
204  }
205
206  private void testRollbackAndDoubleExecution(TableDescriptorBuilder builder) throws Exception {
207    // create the table
208    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
209    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
210
211    // Start the Create procedure && kill the executor
212    final byte[][] splitKeys =
213      new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c") };
214    builder.setRegionReplication(3);
215    TableDescriptor htd = builder.build();
216    RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, splitKeys);
217    long procId =
218      procExec.submitProcedure(new CreateTableProcedure(procExec.getEnvironment(), htd, regions));
219
220    int lastStep = 2; // failing before CREATE_TABLE_WRITE_FS_LAYOUT
221    MasterProcedureTestingUtility.testRollbackAndDoubleExecution(procExec, procId, lastStep);
222
223    TableName tableName = htd.getTableName();
224    MasterProcedureTestingUtility.validateTableDeletion(getMaster(), tableName);
225
226    // are we able to create the table after a rollback?
227    resetProcExecutorTestingKillFlag();
228    testSimpleCreate(tableName, splitKeys);
229  }
230
231  public static class CreateTableProcedureOnHDFSFailure extends CreateTableProcedure {
232    private boolean failOnce = false;
233
234    public CreateTableProcedureOnHDFSFailure() {
235      // Required by the Procedure framework to create the procedure on replay
236      super();
237    }
238
239    public CreateTableProcedureOnHDFSFailure(final MasterProcedureEnv env,
240      final TableDescriptor tableDescriptor, final RegionInfo[] newRegions)
241      throws HBaseIOException {
242      super(env, tableDescriptor, newRegions);
243    }
244
245    @Override
246    protected Flow executeFromState(MasterProcedureEnv env,
247      MasterProcedureProtos.CreateTableState state) throws InterruptedException {
248
249      if (
250        !failOnce && state == MasterProcedureProtos.CreateTableState.CREATE_TABLE_WRITE_FS_LAYOUT
251      ) {
252        try {
253          // To emulate an HDFS failure, create only the first region directory
254          RegionInfo regionInfo = getFirstRegionInfo();
255          Configuration conf = env.getMasterConfiguration();
256          MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
257          Path tempdir = mfs.getTempDir();
258          Path tableDir = CommonFSUtils.getTableDir(tempdir, regionInfo.getTable());
259          Path regionDir = FSUtils.getRegionDirFromTableDir(tableDir, regionInfo);
260          FileSystem fs = FileSystem.get(conf);
261          fs.mkdirs(regionDir);
262
263          failOnce = true;
264          return Flow.HAS_MORE_STATE;
265        } catch (IOException e) {
266          fail("failed to create a region directory: " + e);
267        }
268      }
269
270      return super.executeFromState(env, state);
271    }
272  }
273
274  @Test
275  public void testOnHDFSFailure() throws Exception {
276    final TableName tableName = TableName.valueOf(name.getMethodName());
277
278    // create the table
279    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
280    final byte[][] splitKeys =
281      new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c") };
282    TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, "f1", "f2");
283    RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, splitKeys);
284    long procId = ProcedureTestingUtility.submitAndWait(procExec,
285      new CreateTableProcedureOnHDFSFailure(procExec.getEnvironment(), htd, regions));
286    ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
287  }
288
289  @Test
290  public void testCreateTableWithManyRegionReplication() throws IOException {
291    final int EXCEED_MAX_REGION_REPLICATION = 0x10001;
292    TableName tableName = TableName.valueOf(name.getMethodName());
293    ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
294
295    TableDescriptor tableWithManyRegionReplication = TableDescriptorBuilder.newBuilder(tableName)
296      .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("f1")).build())
297      .setRegionReplication(EXCEED_MAX_REGION_REPLICATION).build();
298    RegionInfo[] regions01 =
299      ModifyRegionUtils.createRegionInfos(tableWithManyRegionReplication, null);
300    long procId01 = ProcedureTestingUtility.submitAndWait(procExec, new CreateTableProcedure(
301      procExec.getEnvironment(), tableWithManyRegionReplication, regions01));
302    Procedure<?> result01 = procExec.getResult(procId01);
303    assertTrue(result01.getException().getCause() instanceof IllegalArgumentException);
304    assertFalse(UTIL.getAdmin().tableExists(tableName));
305
306    TableDescriptor tdesc = TableDescriptorBuilder.newBuilder(tableName)
307      .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("f1")).build())
308      .build();
309    RegionInfo[] regions02 = ModifyRegionUtils.createRegionInfos(tdesc, null);
310    long procId02 = ProcedureTestingUtility.submitAndWait(procExec,
311      new CreateTableProcedure(procExec.getEnvironment(), tdesc, regions02));
312    Procedure<?> result02 = procExec.getResult(procId02);
313    assertTrue(result02.isSuccess());
314    assertTrue(UTIL.getAdmin().tableExists(tableName));
315  }
316}