001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.procedure;
019
020import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACKER_IMPL;
021import static org.junit.Assert.assertEquals;
022import static org.junit.Assert.assertFalse;
023import static org.junit.Assert.assertTrue;
024import static org.junit.Assert.fail;
025
026import java.io.IOException;
027import org.apache.hadoop.conf.Configuration;
028import org.apache.hadoop.fs.FileSystem;
029import org.apache.hadoop.fs.Path;
030import org.apache.hadoop.hbase.DoNotRetryIOException;
031import org.apache.hadoop.hbase.HBaseClassTestRule;
032import org.apache.hadoop.hbase.HBaseIOException;
033import org.apache.hadoop.hbase.TableExistsException;
034import org.apache.hadoop.hbase.TableName;
035import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
036import org.apache.hadoop.hbase.client.RegionInfo;
037import org.apache.hadoop.hbase.client.TableDescriptor;
038import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
039import org.apache.hadoop.hbase.master.MasterFileSystem;
040import org.apache.hadoop.hbase.procedure2.Procedure;
041import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
042import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException;
043import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
044import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
045import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerForTest;
046import org.apache.hadoop.hbase.testclassification.MasterTests;
047import org.apache.hadoop.hbase.testclassification.MediumTests;
048import org.apache.hadoop.hbase.util.Bytes;
049import org.apache.hadoop.hbase.util.CommonFSUtils;
050import org.apache.hadoop.hbase.util.FSUtils;
051import org.apache.hadoop.hbase.util.ModifyRegionUtils;
052import org.apache.hadoop.hbase.util.TableDescriptorChecker;
053import org.junit.ClassRule;
054import org.junit.Rule;
055import org.junit.Test;
056import org.junit.experimental.categories.Category;
057import org.junit.rules.TestName;
058
059import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos;
060
061@Category({ MasterTests.class, MediumTests.class })
062public class TestCreateTableProcedure extends TestTableDDLProcedureBase {
063
064  @ClassRule
065  public static final HBaseClassTestRule CLASS_RULE =
066    HBaseClassTestRule.forClass(TestCreateTableProcedure.class);
067
068  private static final String F1 = "f1";
069  private static final String F2 = "f2";
070
071  @Rule
072  public TestName name = new TestName();
073
074  @Test
075  public void testSimpleCreate() throws Exception {
076    final TableName tableName = TableName.valueOf(name.getMethodName());
077    final byte[][] splitKeys = null;
078    testSimpleCreate(tableName, splitKeys);
079  }
080
081  @Test
082  public void testSimpleCreateWithSplits() throws Exception {
083    final TableName tableName = TableName.valueOf(name.getMethodName());
084    final byte[][] splitKeys =
085      new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c") };
086    testSimpleCreate(tableName, splitKeys);
087  }
088
089  private void testSimpleCreate(final TableName tableName, byte[][] splitKeys) throws Exception {
090    RegionInfo[] regions = MasterProcedureTestingUtility.createTable(getMasterProcedureExecutor(),
091      tableName, splitKeys, F1, F2);
092    MasterProcedureTestingUtility.validateTableCreation(getMaster(), tableName, regions, F1, F2);
093  }
094
095  @Test
096  public void testCreateWithTrackImpl() throws Exception {
097    final TableName tableName = TableName.valueOf(name.getMethodName());
098    ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
099    TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, F1);
100    String trackerName = StoreFileTrackerForTest.class.getName();
101    htd = TableDescriptorBuilder.newBuilder(htd).setValue(TRACKER_IMPL, trackerName).build();
102    RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, null);
103    long procId = ProcedureTestingUtility.submitAndWait(procExec,
104      new CreateTableProcedure(procExec.getEnvironment(), htd, regions));
105    ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId));
106    htd = getMaster().getTableDescriptors().get(tableName);
107    assertEquals(trackerName, htd.getValue(TRACKER_IMPL));
108  }
109
110  @Test
111  public void testCreateWithFileBasedStoreTrackerImpl() throws Exception {
112    ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
113    procExec.getEnvironment().getMasterConfiguration().set(StoreFileTrackerFactory.TRACKER_IMPL,
114      StoreFileTrackerFactory.Trackers.FILE.name());
115    final TableName tableName = TableName.valueOf(name.getMethodName());
116    TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, F1);
117    RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, null);
118    long procId = ProcedureTestingUtility.submitAndWait(procExec,
119      new CreateTableProcedure(procExec.getEnvironment(), htd, regions));
120    ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId));
121    htd = getMaster().getTableDescriptors().get(tableName);
122    assertEquals(StoreFileTrackerFactory.Trackers.FILE.name(), htd.getValue(TRACKER_IMPL));
123  }
124
125  @Test
126  public void testCreateWithoutColumnFamily() throws Exception {
127    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
128    final TableName tableName = TableName.valueOf(name.getMethodName());
129    // create table with 0 families will fail
130    final TableDescriptorBuilder builder =
131      TableDescriptorBuilder.newBuilder(MasterProcedureTestingUtility.createHTD(tableName));
132
133    // disable sanity check
134    builder.setValue(TableDescriptorChecker.TABLE_SANITY_CHECKS, Boolean.FALSE.toString());
135    TableDescriptor htd = builder.build();
136    final RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, null);
137
138    long procId = ProcedureTestingUtility.submitAndWait(procExec,
139      new CreateTableProcedure(procExec.getEnvironment(), htd, regions));
140    final Procedure<?> result = procExec.getResult(procId);
141    assertEquals(true, result.isFailed());
142    Throwable cause = ProcedureTestingUtility.getExceptionCause(result);
143    assertTrue("expected DoNotRetryIOException, got " + cause,
144      cause instanceof DoNotRetryIOException);
145  }
146
147  @Test(expected = TableExistsException.class)
148  public void testCreateExisting() throws Exception {
149    final TableName tableName = TableName.valueOf(name.getMethodName());
150    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
151    final TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, "f");
152    final RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, null);
153
154    // create the table
155    long procId1 =
156      procExec.submitProcedure(new CreateTableProcedure(procExec.getEnvironment(), htd, regions));
157
158    // create another with the same name
159    ProcedurePrepareLatch latch2 = new ProcedurePrepareLatch.CompatibilityLatch();
160    long procId2 = procExec
161      .submitProcedure(new CreateTableProcedure(procExec.getEnvironment(), htd, regions, latch2));
162
163    ProcedureTestingUtility.waitProcedure(procExec, procId1);
164    ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId1));
165
166    ProcedureTestingUtility.waitProcedure(procExec, procId2);
167    latch2.await();
168  }
169
170  @Test
171  public void testRecoveryAndDoubleExecution() throws Exception {
172    final TableName tableName = TableName.valueOf(name.getMethodName());
173
174    // create the table
175    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
176    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
177
178    // Start the Create procedure && kill the executor
179    byte[][] splitKeys = null;
180    TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, "f1", "f2");
181    RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, splitKeys);
182    long procId =
183      procExec.submitProcedure(new CreateTableProcedure(procExec.getEnvironment(), htd, regions));
184
185    // Restart the executor and execute the step twice
186    MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId);
187    MasterProcedureTestingUtility.validateTableCreation(getMaster(), tableName, regions, F1, F2);
188  }
189
190  @Test
191  public void testRollbackAndDoubleExecution() throws Exception {
192    final TableName tableName = TableName.valueOf(name.getMethodName());
193    testRollbackAndDoubleExecution(TableDescriptorBuilder
194      .newBuilder(MasterProcedureTestingUtility.createHTD(tableName, F1, F2)));
195  }
196
197  @Test
198  public void testRollbackAndDoubleExecutionOnMobTable() throws Exception {
199    final TableName tableName = TableName.valueOf(name.getMethodName());
200    TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, F1, F2);
201    TableDescriptorBuilder builder =
202      TableDescriptorBuilder.newBuilder(htd).modifyColumnFamily(ColumnFamilyDescriptorBuilder
203        .newBuilder(htd.getColumnFamily(Bytes.toBytes(F1))).setMobEnabled(true).build());
204    testRollbackAndDoubleExecution(builder);
205  }
206
207  private void testRollbackAndDoubleExecution(TableDescriptorBuilder builder) throws Exception {
208    // create the table
209    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
210    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
211
212    // Start the Create procedure && kill the executor
213    final byte[][] splitKeys =
214      new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c") };
215    builder.setRegionReplication(3);
216    TableDescriptor htd = builder.build();
217    RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, splitKeys);
218    long procId =
219      procExec.submitProcedure(new CreateTableProcedure(procExec.getEnvironment(), htd, regions));
220
221    int lastStep = 2; // failing before CREATE_TABLE_WRITE_FS_LAYOUT
222    MasterProcedureTestingUtility.testRollbackAndDoubleExecution(procExec, procId, lastStep);
223
224    TableName tableName = htd.getTableName();
225    MasterProcedureTestingUtility.validateTableDeletion(getMaster(), tableName);
226
227    // are we able to create the table after a rollback?
228    resetProcExecutorTestingKillFlag();
229    testSimpleCreate(tableName, splitKeys);
230  }
231
232  public static class CreateTableProcedureOnHDFSFailure extends CreateTableProcedure {
233    private boolean failOnce = false;
234
235    public CreateTableProcedureOnHDFSFailure() {
236      // Required by the Procedure framework to create the procedure on replay
237      super();
238    }
239
240    public CreateTableProcedureOnHDFSFailure(final MasterProcedureEnv env,
241      final TableDescriptor tableDescriptor, final RegionInfo[] newRegions)
242      throws HBaseIOException {
243      super(env, tableDescriptor, newRegions);
244    }
245
246    @Override
247    protected Flow executeFromState(MasterProcedureEnv env,
248      MasterProcedureProtos.CreateTableState state)
249      throws InterruptedException, ProcedureSuspendedException {
250
251      if (
252        !failOnce && state == MasterProcedureProtos.CreateTableState.CREATE_TABLE_WRITE_FS_LAYOUT
253      ) {
254        try {
255          // To emulate an HDFS failure, create only the first region directory
256          RegionInfo regionInfo = getFirstRegionInfo();
257          Configuration conf = env.getMasterConfiguration();
258          MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
259          Path tempdir = mfs.getTempDir();
260          Path tableDir = CommonFSUtils.getTableDir(tempdir, regionInfo.getTable());
261          Path regionDir = FSUtils.getRegionDirFromTableDir(tableDir, regionInfo);
262          FileSystem fs = FileSystem.get(conf);
263          fs.mkdirs(regionDir);
264
265          failOnce = true;
266          return Flow.HAS_MORE_STATE;
267        } catch (IOException e) {
268          fail("failed to create a region directory: " + e);
269        }
270      }
271
272      return super.executeFromState(env, state);
273    }
274  }
275
276  @Test
277  public void testOnHDFSFailure() throws Exception {
278    final TableName tableName = TableName.valueOf(name.getMethodName());
279
280    // create the table
281    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
282    final byte[][] splitKeys =
283      new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c") };
284    TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, "f1", "f2");
285    RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, splitKeys);
286    long procId = ProcedureTestingUtility.submitAndWait(procExec,
287      new CreateTableProcedureOnHDFSFailure(procExec.getEnvironment(), htd, regions));
288    ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
289  }
290
291  @Test
292  public void testCreateTableWithManyRegionReplication() throws IOException {
293    final int EXCEED_MAX_REGION_REPLICATION = 0x10001;
294    TableName tableName = TableName.valueOf(name.getMethodName());
295    ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
296
297    TableDescriptor tableWithManyRegionReplication = TableDescriptorBuilder.newBuilder(tableName)
298      .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("f1")).build())
299      .setRegionReplication(EXCEED_MAX_REGION_REPLICATION).build();
300    RegionInfo[] regions01 =
301      ModifyRegionUtils.createRegionInfos(tableWithManyRegionReplication, null);
302    long procId01 = ProcedureTestingUtility.submitAndWait(procExec, new CreateTableProcedure(
303      procExec.getEnvironment(), tableWithManyRegionReplication, regions01));
304    Procedure<?> result01 = procExec.getResult(procId01);
305    assertTrue(result01.getException().getCause() instanceof IllegalArgumentException);
306    assertFalse(UTIL.getAdmin().tableExists(tableName));
307
308    TableDescriptor tdesc = TableDescriptorBuilder.newBuilder(tableName)
309      .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("f1")).build())
310      .build();
311    RegionInfo[] regions02 = ModifyRegionUtils.createRegionInfos(tdesc, null);
312    long procId02 = ProcedureTestingUtility.submitAndWait(procExec,
313      new CreateTableProcedure(procExec.getEnvironment(), tdesc, regions02));
314    Procedure<?> result02 = procExec.getResult(procId02);
315    assertTrue(result02.isSuccess());
316    assertTrue(UTIL.getAdmin().tableExists(tableName));
317  }
318}