Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.procedure;
019
020import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACKER_IMPL;
021import static org.junit.jupiter.api.Assertions.assertEquals;
022import static org.junit.jupiter.api.Assertions.assertFalse;
023import static org.junit.jupiter.api.Assertions.assertThrows;
024import static org.junit.jupiter.api.Assertions.assertTrue;
025import static org.junit.jupiter.api.Assertions.fail;
026
027import java.io.IOException;
028import org.apache.hadoop.conf.Configuration;
029import org.apache.hadoop.fs.FileSystem;
030import org.apache.hadoop.fs.Path;
031import org.apache.hadoop.hbase.DoNotRetryIOException;
032import org.apache.hadoop.hbase.HBaseIOException;
033import org.apache.hadoop.hbase.TableExistsException;
034import org.apache.hadoop.hbase.TableName;
035import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
036import org.apache.hadoop.hbase.client.RegionInfo;
037import org.apache.hadoop.hbase.client.TableDescriptor;
038import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
039import org.apache.hadoop.hbase.master.MasterFileSystem;
040import org.apache.hadoop.hbase.procedure2.Procedure;
041import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
042import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException;
043import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
044import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
045import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerForTest;
046import org.apache.hadoop.hbase.testclassification.MasterTests;
047import org.apache.hadoop.hbase.testclassification.MediumTests;
048import org.apache.hadoop.hbase.util.Bytes;
049import org.apache.hadoop.hbase.util.CommonFSUtils;
050import org.apache.hadoop.hbase.util.FSUtils;
051import org.apache.hadoop.hbase.util.ModifyRegionUtils;
052import org.apache.hadoop.hbase.util.TableDescriptorChecker;
053import org.junit.jupiter.api.AfterAll;
054import org.junit.jupiter.api.BeforeAll;
055import org.junit.jupiter.api.BeforeEach;
056import org.junit.jupiter.api.Tag;
057import org.junit.jupiter.api.Test;
058import org.junit.jupiter.api.TestInfo;
059
060import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos;
061
062@Tag(MasterTests.TAG)
063@Tag(MediumTests.TAG)
064public class TestCreateTableProcedure extends TestTableDDLProcedureBase {
065
066  private static final String F1 = "f1";
067  private static final String F2 = "f2";
068  private String testMethodName;
069
070  @BeforeAll
071  public static void setupCluster() throws Exception {
072    TestTableDDLProcedureBase.setupCluster();
073  }
074
075  @AfterAll
076  public static void cleanupTest() throws Exception {
077    TestTableDDLProcedureBase.cleanupTest();
078  }
079
080  @BeforeEach
081  public void setTestMethod(TestInfo testInfo) {
082    testMethodName = testInfo.getTestMethod().get().getName();
083  }
084
085  @Test
086  public void testSimpleCreate() throws Exception {
087    final TableName tableName = TableName.valueOf(testMethodName);
088    final byte[][] splitKeys = null;
089    testSimpleCreate(tableName, splitKeys);
090  }
091
092  @Test
093  public void testSimpleCreateWithSplits() throws Exception {
094    final TableName tableName = TableName.valueOf(testMethodName);
095    final byte[][] splitKeys =
096      new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c") };
097    testSimpleCreate(tableName, splitKeys);
098  }
099
100  private void testSimpleCreate(final TableName tableName, byte[][] splitKeys) throws Exception {
101    RegionInfo[] regions = MasterProcedureTestingUtility.createTable(getMasterProcedureExecutor(),
102      tableName, splitKeys, F1, F2);
103    MasterProcedureTestingUtility.validateTableCreation(getMaster(), tableName, regions, F1, F2);
104  }
105
106  @Test
107  public void testCreateWithTrackImpl() throws Exception {
108    final TableName tableName = TableName.valueOf(testMethodName);
109    ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
110    TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, F1);
111    String trackerName = StoreFileTrackerForTest.class.getName();
112    htd = TableDescriptorBuilder.newBuilder(htd).setValue(TRACKER_IMPL, trackerName).build();
113    RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, null);
114    long procId = ProcedureTestingUtility.submitAndWait(procExec,
115      new CreateTableProcedure(procExec.getEnvironment(), htd, regions));
116    ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId));
117    htd = getMaster().getTableDescriptors().get(tableName);
118    assertEquals(trackerName, htd.getValue(TRACKER_IMPL));
119  }
120
121  @Test
122  public void testCreateWithFileBasedStoreTrackerImpl() throws Exception {
123    ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
124    procExec.getEnvironment().getMasterConfiguration().set(StoreFileTrackerFactory.TRACKER_IMPL,
125      StoreFileTrackerFactory.Trackers.FILE.name());
126    final TableName tableName = TableName.valueOf(testMethodName);
127    TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, F1);
128    RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, null);
129    long procId = ProcedureTestingUtility.submitAndWait(procExec,
130      new CreateTableProcedure(procExec.getEnvironment(), htd, regions));
131    ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId));
132    htd = getMaster().getTableDescriptors().get(tableName);
133    assertEquals(StoreFileTrackerFactory.Trackers.FILE.name(), htd.getValue(TRACKER_IMPL));
134  }
135
136  @Test
137  public void testCreateWithoutColumnFamily() throws Exception {
138    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
139    final TableName tableName = TableName.valueOf(testMethodName);
140    // create table with 0 families will fail
141    final TableDescriptorBuilder builder =
142      TableDescriptorBuilder.newBuilder(MasterProcedureTestingUtility.createHTD(tableName));
143
144    // disable sanity check
145    builder.setValue(TableDescriptorChecker.TABLE_SANITY_CHECKS, Boolean.FALSE.toString());
146    TableDescriptor htd = builder.build();
147    final RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, null);
148
149    long procId = ProcedureTestingUtility.submitAndWait(procExec,
150      new CreateTableProcedure(procExec.getEnvironment(), htd, regions));
151    final Procedure<?> result = procExec.getResult(procId);
152    assertEquals(true, result.isFailed());
153    Throwable cause = ProcedureTestingUtility.getExceptionCause(result);
154    assertTrue(cause instanceof DoNotRetryIOException,
155      "expected DoNotRetryIOException, got " + cause);
156  }
157
158  @Test
159  public void testCreateExisting() throws Exception {
160    final TableName tableName = TableName.valueOf(testMethodName);
161    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
162    final TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, "f");
163    final RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, null);
164
165    // create the table
166    long procId1 =
167      procExec.submitProcedure(new CreateTableProcedure(procExec.getEnvironment(), htd, regions));
168
169    // create another with the same name
170    ProcedurePrepareLatch latch2 = new ProcedurePrepareLatch.CompatibilityLatch();
171    long procId2 = procExec
172      .submitProcedure(new CreateTableProcedure(procExec.getEnvironment(), htd, regions, latch2));
173
174    ProcedureTestingUtility.waitProcedure(procExec, procId1);
175    ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId1));
176
177    ProcedureTestingUtility.waitProcedure(procExec, procId2);
178    assertThrows(TableExistsException.class, latch2::await);
179  }
180
181  @Test
182  public void testRecoveryAndDoubleExecution() throws Exception {
183    final TableName tableName = TableName.valueOf(testMethodName);
184
185    // create the table
186    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
187    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
188
189    // Start the Create procedure && kill the executor
190    byte[][] splitKeys = null;
191    TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, "f1", "f2");
192    RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, splitKeys);
193    long procId =
194      procExec.submitProcedure(new CreateTableProcedure(procExec.getEnvironment(), htd, regions));
195
196    // Restart the executor and execute the step twice
197    MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId);
198    MasterProcedureTestingUtility.validateTableCreation(getMaster(), tableName, regions, F1, F2);
199  }
200
201  @Test
202  public void testRollbackAndDoubleExecution() throws Exception {
203    final TableName tableName = TableName.valueOf(testMethodName);
204    testRollbackAndDoubleExecution(TableDescriptorBuilder
205      .newBuilder(MasterProcedureTestingUtility.createHTD(tableName, F1, F2)));
206  }
207
208  @Test
209  public void testRollbackAndDoubleExecutionOnMobTable() throws Exception {
210    final TableName tableName = TableName.valueOf(testMethodName);
211    TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, F1, F2);
212    TableDescriptorBuilder builder =
213      TableDescriptorBuilder.newBuilder(htd).modifyColumnFamily(ColumnFamilyDescriptorBuilder
214        .newBuilder(htd.getColumnFamily(Bytes.toBytes(F1))).setMobEnabled(true).build());
215    testRollbackAndDoubleExecution(builder);
216  }
217
218  private void testRollbackAndDoubleExecution(TableDescriptorBuilder builder) throws Exception {
219    // create the table
220    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
221    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
222
223    // Start the Create procedure && kill the executor
224    final byte[][] splitKeys =
225      new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c") };
226    builder.setRegionReplication(3);
227    TableDescriptor htd = builder.build();
228    RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, splitKeys);
229    long procId =
230      procExec.submitProcedure(new CreateTableProcedure(procExec.getEnvironment(), htd, regions));
231
232    int lastStep = 2; // failing before CREATE_TABLE_WRITE_FS_LAYOUT
233    MasterProcedureTestingUtility.testRollbackAndDoubleExecution(procExec, procId, lastStep);
234
235    TableName tableName = htd.getTableName();
236    MasterProcedureTestingUtility.validateTableDeletion(getMaster(), tableName);
237
238    // are we able to create the table after a rollback?
239    resetProcExecutorTestingKillFlag();
240    testSimpleCreate(tableName, splitKeys);
241  }
242
243  public static class CreateTableProcedureOnHDFSFailure extends CreateTableProcedure {
244    private boolean failOnce = false;
245
246    public CreateTableProcedureOnHDFSFailure() {
247      // Required by the Procedure framework to create the procedure on replay
248      super();
249    }
250
251    public CreateTableProcedureOnHDFSFailure(final MasterProcedureEnv env,
252      final TableDescriptor tableDescriptor, final RegionInfo[] newRegions)
253      throws HBaseIOException {
254      super(env, tableDescriptor, newRegions);
255    }
256
257    @Override
258    protected Flow executeFromState(MasterProcedureEnv env,
259      MasterProcedureProtos.CreateTableState state)
260      throws InterruptedException, ProcedureSuspendedException {
261
262      if (
263        !failOnce && state == MasterProcedureProtos.CreateTableState.CREATE_TABLE_WRITE_FS_LAYOUT
264      ) {
265        try {
266          // To emulate an HDFS failure, create only the first region directory
267          RegionInfo regionInfo = getFirstRegionInfo();
268          Configuration conf = env.getMasterConfiguration();
269          MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
270          Path tempdir = mfs.getTempDir();
271          Path tableDir = CommonFSUtils.getTableDir(tempdir, regionInfo.getTable());
272          Path regionDir = FSUtils.getRegionDirFromTableDir(tableDir, regionInfo);
273          FileSystem fs = FileSystem.get(conf);
274          fs.mkdirs(regionDir);
275
276          failOnce = true;
277          return Flow.HAS_MORE_STATE;
278        } catch (IOException e) {
279          fail("failed to create a region directory: " + e);
280        }
281      }
282
283      return super.executeFromState(env, state);
284    }
285  }
286
287  @Test
288  public void testOnHDFSFailure() throws Exception {
289    final TableName tableName = TableName.valueOf(testMethodName);
290
291    // create the table
292    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
293    final byte[][] splitKeys =
294      new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c") };
295    TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, "f1", "f2");
296    RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, splitKeys);
297    long procId = ProcedureTestingUtility.submitAndWait(procExec,
298      new CreateTableProcedureOnHDFSFailure(procExec.getEnvironment(), htd, regions));
299    ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
300  }
301
302  @Test
303  public void testCreateTableWithManyRegionReplication() throws IOException {
304    final int EXCEED_MAX_REGION_REPLICATION = 0x10001;
305    TableName tableName = TableName.valueOf(testMethodName);
306    ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
307
308    TableDescriptor tableWithManyRegionReplication = TableDescriptorBuilder.newBuilder(tableName)
309      .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("f1")).build())
310      .setRegionReplication(EXCEED_MAX_REGION_REPLICATION).build();
311    RegionInfo[] regions01 =
312      ModifyRegionUtils.createRegionInfos(tableWithManyRegionReplication, null);
313    long procId01 = ProcedureTestingUtility.submitAndWait(procExec, new CreateTableProcedure(
314      procExec.getEnvironment(), tableWithManyRegionReplication, regions01));
315    Procedure<?> result01 = procExec.getResult(procId01);
316    assertTrue(result01.getException().getCause() instanceof IllegalArgumentException);
317    assertFalse(UTIL.getAdmin().tableExists(tableName));
318
319    TableDescriptor tdesc = TableDescriptorBuilder.newBuilder(tableName)
320      .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("f1")).build())
321      .build();
322    RegionInfo[] regions02 = ModifyRegionUtils.createRegionInfos(tdesc, null);
323    long procId02 = ProcedureTestingUtility.submitAndWait(procExec,
324      new CreateTableProcedure(procExec.getEnvironment(), tdesc, regions02));
325    Procedure<?> result02 = procExec.getResult(procId02);
326    assertTrue(result02.isSuccess());
327    assertTrue(UTIL.getAdmin().tableExists(tableName));
328  }
329}