001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.master.procedure; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertTrue; 022import static org.junit.Assert.fail; 023 024import java.io.IOException; 025import org.apache.hadoop.conf.Configuration; 026import org.apache.hadoop.fs.FileSystem; 027import org.apache.hadoop.fs.Path; 028import org.apache.hadoop.hbase.DoNotRetryIOException; 029import org.apache.hadoop.hbase.HBaseClassTestRule; 030import org.apache.hadoop.hbase.HBaseIOException; 031import org.apache.hadoop.hbase.TableExistsException; 032import org.apache.hadoop.hbase.TableName; 033import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; 034import org.apache.hadoop.hbase.client.RegionInfo; 035import org.apache.hadoop.hbase.client.TableDescriptor; 036import org.apache.hadoop.hbase.client.TableDescriptorBuilder; 037import org.apache.hadoop.hbase.master.MasterFileSystem; 038import org.apache.hadoop.hbase.procedure2.Procedure; 039import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; 040import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; 041import org.apache.hadoop.hbase.testclassification.MasterTests; 042import org.apache.hadoop.hbase.testclassification.MediumTests; 043import org.apache.hadoop.hbase.util.Bytes; 044import org.apache.hadoop.hbase.util.CommonFSUtils; 045import org.apache.hadoop.hbase.util.FSUtils; 046import org.apache.hadoop.hbase.util.ModifyRegionUtils; 047import org.apache.hadoop.hbase.util.TableDescriptorChecker; 048import org.junit.ClassRule; 049import org.junit.Rule; 050import org.junit.Test; 051import org.junit.experimental.categories.Category; 052import org.junit.rules.TestName; 053 054import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos; 055 056@Category({MasterTests.class, MediumTests.class}) 057public class TestCreateTableProcedure extends TestTableDDLProcedureBase { 058 059 @ClassRule 060 public static final HBaseClassTestRule CLASS_RULE = 061 HBaseClassTestRule.forClass(TestCreateTableProcedure.class); 062 063 private static final String F1 = "f1"; 064 private static final String F2 = "f2"; 065 066 @Rule 067 public TestName name = new TestName(); 068 069 @Test 070 public void testSimpleCreate() throws Exception { 071 final TableName tableName = TableName.valueOf(name.getMethodName()); 072 final byte[][] splitKeys = null; 073 testSimpleCreate(tableName, splitKeys); 074 } 075 076 @Test 077 public void testSimpleCreateWithSplits() throws Exception { 078 final TableName tableName = TableName.valueOf(name.getMethodName()); 079 final byte[][] splitKeys = new byte[][] { 080 Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c") 081 }; 082 testSimpleCreate(tableName, splitKeys); 083 } 084 085 private void testSimpleCreate(final TableName tableName, byte[][] splitKeys) throws Exception { 086 RegionInfo[] regions = MasterProcedureTestingUtility.createTable( 087 getMasterProcedureExecutor(), tableName, splitKeys, F1, F2); 088 MasterProcedureTestingUtility.validateTableCreation(getMaster(), tableName, regions, F1, F2); 089 } 090 091 @Test 092 public void testCreateWithoutColumnFamily() throws Exception { 093 final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); 094 final TableName tableName = TableName.valueOf(name.getMethodName()); 095 // create table with 0 families will fail 096 final TableDescriptorBuilder builder = 097 TableDescriptorBuilder.newBuilder(MasterProcedureTestingUtility.createHTD(tableName)); 098 099 // disable sanity check 100 builder.setValue(TableDescriptorChecker.TABLE_SANITY_CHECKS, Boolean.FALSE.toString()); 101 TableDescriptor htd = builder.build(); 102 final RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, null); 103 104 long procId = 105 ProcedureTestingUtility.submitAndWait(procExec, 106 new CreateTableProcedure(procExec.getEnvironment(), htd, regions)); 107 final Procedure<?> result = procExec.getResult(procId); 108 assertEquals(true, result.isFailed()); 109 Throwable cause = ProcedureTestingUtility.getExceptionCause(result); 110 assertTrue("expected DoNotRetryIOException, got " + cause, 111 cause instanceof DoNotRetryIOException); 112 } 113 114 @Test(expected=TableExistsException.class) 115 public void testCreateExisting() throws Exception { 116 final TableName tableName = TableName.valueOf(name.getMethodName()); 117 final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); 118 final TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, "f"); 119 final RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, null); 120 121 // create the table 122 long procId1 = procExec.submitProcedure( 123 new CreateTableProcedure(procExec.getEnvironment(), htd, regions)); 124 125 // create another with the same name 126 ProcedurePrepareLatch latch2 = new ProcedurePrepareLatch.CompatibilityLatch(); 127 long procId2 = procExec.submitProcedure( 128 new CreateTableProcedure(procExec.getEnvironment(), htd, regions, latch2)); 129 130 ProcedureTestingUtility.waitProcedure(procExec, procId1); 131 ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId1)); 132 133 ProcedureTestingUtility.waitProcedure(procExec, procId2); 134 latch2.await(); 135 } 136 137 @Test 138 public void testRecoveryAndDoubleExecution() throws Exception { 139 final TableName tableName = TableName.valueOf(name.getMethodName()); 140 141 // create the table 142 final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); 143 ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); 144 145 // Start the Create procedure && kill the executor 146 byte[][] splitKeys = null; 147 TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, "f1", "f2"); 148 RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, splitKeys); 149 long procId = procExec.submitProcedure( 150 new CreateTableProcedure(procExec.getEnvironment(), htd, regions)); 151 152 // Restart the executor and execute the step twice 153 MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId); 154 MasterProcedureTestingUtility.validateTableCreation(getMaster(), tableName, regions, F1, F2); 155 } 156 157 @Test 158 public void testRollbackAndDoubleExecution() throws Exception { 159 final TableName tableName = TableName.valueOf(name.getMethodName()); 160 testRollbackAndDoubleExecution(TableDescriptorBuilder 161 .newBuilder(MasterProcedureTestingUtility.createHTD(tableName, F1, F2))); 162 } 163 164 @Test 165 public void testRollbackAndDoubleExecutionOnMobTable() throws Exception { 166 final TableName tableName = TableName.valueOf(name.getMethodName()); 167 TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, F1, F2); 168 TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(htd) 169 .modifyColumnFamily(ColumnFamilyDescriptorBuilder 170 .newBuilder(htd.getColumnFamily(Bytes.toBytes(F1))) 171 .setMobEnabled(true) 172 .build()); 173 testRollbackAndDoubleExecution(builder); 174 } 175 176 private void testRollbackAndDoubleExecution(TableDescriptorBuilder builder) throws Exception { 177 // create the table 178 final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); 179 ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); 180 181 // Start the Create procedure && kill the executor 182 final byte[][] splitKeys = new byte[][] { 183 Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c") 184 }; 185 builder.setRegionReplication(3); 186 TableDescriptor htd = builder.build(); 187 RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, splitKeys); 188 long procId = procExec.submitProcedure( 189 new CreateTableProcedure(procExec.getEnvironment(), htd, regions)); 190 191 int lastStep = 2; // failing before CREATE_TABLE_WRITE_FS_LAYOUT 192 MasterProcedureTestingUtility.testRollbackAndDoubleExecution(procExec, procId, lastStep); 193 194 TableName tableName = htd.getTableName(); 195 MasterProcedureTestingUtility.validateTableDeletion(getMaster(), tableName); 196 197 // are we able to create the table after a rollback? 198 resetProcExecutorTestingKillFlag(); 199 testSimpleCreate(tableName, splitKeys); 200 } 201 202 public static class CreateTableProcedureOnHDFSFailure extends CreateTableProcedure { 203 private boolean failOnce = false; 204 205 public CreateTableProcedureOnHDFSFailure() { 206 // Required by the Procedure framework to create the procedure on replay 207 super(); 208 } 209 210 public CreateTableProcedureOnHDFSFailure(final MasterProcedureEnv env, 211 final TableDescriptor tableDescriptor, final RegionInfo[] newRegions) 212 throws HBaseIOException { 213 super(env, tableDescriptor, newRegions); 214 } 215 216 @Override 217 protected Flow executeFromState(MasterProcedureEnv env, 218 MasterProcedureProtos.CreateTableState state) throws InterruptedException { 219 220 if (!failOnce && 221 state == MasterProcedureProtos.CreateTableState.CREATE_TABLE_WRITE_FS_LAYOUT) { 222 try { 223 // To emulate an HDFS failure, create only the first region directory 224 RegionInfo regionInfo = getFirstRegionInfo(); 225 Configuration conf = env.getMasterConfiguration(); 226 MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); 227 Path tempdir = mfs.getTempDir(); 228 Path tableDir = CommonFSUtils.getTableDir(tempdir, regionInfo.getTable()); 229 Path regionDir = FSUtils.getRegionDirFromTableDir(tableDir, regionInfo); 230 FileSystem fs = FileSystem.get(conf); 231 fs.mkdirs(regionDir); 232 233 failOnce = true; 234 return Flow.HAS_MORE_STATE; 235 } catch (IOException e) { 236 fail("failed to create a region directory: " + e); 237 } 238 } 239 240 return super.executeFromState(env, state); 241 } 242 } 243 244 @Test 245 public void testOnHDFSFailure() throws Exception { 246 final TableName tableName = TableName.valueOf(name.getMethodName()); 247 248 // create the table 249 final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); 250 final byte[][] splitKeys = new byte[][] { 251 Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c") 252 }; 253 TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, "f1", "f2"); 254 RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, splitKeys); 255 long procId = ProcedureTestingUtility.submitAndWait(procExec, 256 new CreateTableProcedureOnHDFSFailure(procExec.getEnvironment(), htd, regions)); 257 ProcedureTestingUtility.assertProcNotFailed(procExec, procId); 258 } 259}