001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.master.procedure; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertTrue; 022import static org.junit.Assert.fail; 023 024import java.io.IOException; 025import org.apache.hadoop.conf.Configuration; 026import org.apache.hadoop.fs.FileSystem; 027import org.apache.hadoop.fs.Path; 028import org.apache.hadoop.hbase.DoNotRetryIOException; 029import org.apache.hadoop.hbase.HBaseClassTestRule; 030import org.apache.hadoop.hbase.HBaseIOException; 031import org.apache.hadoop.hbase.TableExistsException; 032import org.apache.hadoop.hbase.TableName; 033import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; 034import org.apache.hadoop.hbase.client.RegionInfo; 035import org.apache.hadoop.hbase.client.TableDescriptor; 036import org.apache.hadoop.hbase.client.TableDescriptorBuilder; 037import org.apache.hadoop.hbase.master.MasterFileSystem; 038import org.apache.hadoop.hbase.procedure2.Procedure; 039import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; 040import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; 041import org.apache.hadoop.hbase.testclassification.MasterTests; 042import org.apache.hadoop.hbase.testclassification.MediumTests; 043import org.apache.hadoop.hbase.util.Bytes; 044import org.apache.hadoop.hbase.util.FSUtils; 045import org.apache.hadoop.hbase.util.ModifyRegionUtils; 046import org.apache.hadoop.hbase.util.TableDescriptorChecker; 047import org.junit.ClassRule; 048import org.junit.Rule; 049import org.junit.Test; 050import org.junit.experimental.categories.Category; 051import org.junit.rules.TestName; 052 053import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos; 054 055@Category({MasterTests.class, MediumTests.class}) 056public class TestCreateTableProcedure extends TestTableDDLProcedureBase { 057 058 @ClassRule 059 public static final HBaseClassTestRule CLASS_RULE = 060 HBaseClassTestRule.forClass(TestCreateTableProcedure.class); 061 062 private static final String F1 = "f1"; 063 private static final String F2 = "f2"; 064 065 @Rule 066 public TestName name = new TestName(); 067 068 @Test 069 public void testSimpleCreate() throws Exception { 070 final TableName tableName = TableName.valueOf(name.getMethodName()); 071 final byte[][] splitKeys = null; 072 testSimpleCreate(tableName, splitKeys); 073 } 074 075 @Test 076 public void testSimpleCreateWithSplits() throws Exception { 077 final TableName tableName = TableName.valueOf(name.getMethodName()); 078 final byte[][] splitKeys = new byte[][] { 079 Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c") 080 }; 081 testSimpleCreate(tableName, splitKeys); 082 } 083 084 private void testSimpleCreate(final TableName tableName, byte[][] splitKeys) throws Exception { 085 RegionInfo[] regions = MasterProcedureTestingUtility.createTable( 086 getMasterProcedureExecutor(), tableName, splitKeys, F1, F2); 087 MasterProcedureTestingUtility.validateTableCreation(getMaster(), tableName, regions, F1, F2); 088 } 089 090 @Test 091 public void testCreateWithoutColumnFamily() throws Exception { 092 final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); 093 final TableName tableName = TableName.valueOf(name.getMethodName()); 094 // create table with 0 families will fail 095 final TableDescriptorBuilder builder = 096 TableDescriptorBuilder.newBuilder(MasterProcedureTestingUtility.createHTD(tableName)); 097 098 // disable sanity check 099 builder.setValue(TableDescriptorChecker.TABLE_SANITY_CHECKS, Boolean.FALSE.toString()); 100 TableDescriptor htd = builder.build(); 101 final RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, null); 102 103 long procId = 104 ProcedureTestingUtility.submitAndWait(procExec, 105 new CreateTableProcedure(procExec.getEnvironment(), htd, regions)); 106 final Procedure<?> result = procExec.getResult(procId); 107 assertEquals(true, result.isFailed()); 108 Throwable cause = ProcedureTestingUtility.getExceptionCause(result); 109 assertTrue("expected DoNotRetryIOException, got " + cause, 110 cause instanceof DoNotRetryIOException); 111 } 112 113 @Test(expected=TableExistsException.class) 114 public void testCreateExisting() throws Exception { 115 final TableName tableName = TableName.valueOf(name.getMethodName()); 116 final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); 117 final TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, "f"); 118 final RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, null); 119 120 // create the table 121 long procId1 = procExec.submitProcedure( 122 new CreateTableProcedure(procExec.getEnvironment(), htd, regions)); 123 124 // create another with the same name 125 ProcedurePrepareLatch latch2 = new ProcedurePrepareLatch.CompatibilityLatch(); 126 long procId2 = procExec.submitProcedure( 127 new CreateTableProcedure(procExec.getEnvironment(), htd, regions, latch2)); 128 129 ProcedureTestingUtility.waitProcedure(procExec, procId1); 130 ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId1)); 131 132 ProcedureTestingUtility.waitProcedure(procExec, procId2); 133 latch2.await(); 134 } 135 136 @Test 137 public void testRecoveryAndDoubleExecution() throws Exception { 138 final TableName tableName = TableName.valueOf(name.getMethodName()); 139 140 // create the table 141 final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); 142 ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); 143 144 // Start the Create procedure && kill the executor 145 byte[][] splitKeys = null; 146 TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, "f1", "f2"); 147 RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, splitKeys); 148 long procId = procExec.submitProcedure( 149 new CreateTableProcedure(procExec.getEnvironment(), htd, regions)); 150 151 // Restart the executor and execute the step twice 152 MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId); 153 MasterProcedureTestingUtility.validateTableCreation(getMaster(), tableName, regions, F1, F2); 154 } 155 156 @Test 157 public void testRollbackAndDoubleExecution() throws Exception { 158 final TableName tableName = TableName.valueOf(name.getMethodName()); 159 testRollbackAndDoubleExecution(TableDescriptorBuilder 160 .newBuilder(MasterProcedureTestingUtility.createHTD(tableName, F1, F2))); 161 } 162 163 @Test 164 public void testRollbackAndDoubleExecutionOnMobTable() throws Exception { 165 final TableName tableName = TableName.valueOf(name.getMethodName()); 166 TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, F1, F2); 167 TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(htd) 168 .modifyColumnFamily(ColumnFamilyDescriptorBuilder 169 .newBuilder(htd.getColumnFamily(Bytes.toBytes(F1))) 170 .setMobEnabled(true) 171 .build()); 172 testRollbackAndDoubleExecution(builder); 173 } 174 175 private void testRollbackAndDoubleExecution(TableDescriptorBuilder builder) throws Exception { 176 // create the table 177 final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); 178 ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); 179 180 // Start the Create procedure && kill the executor 181 final byte[][] splitKeys = new byte[][] { 182 Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c") 183 }; 184 builder.setRegionReplication(3); 185 TableDescriptor htd = builder.build(); 186 RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, splitKeys); 187 long procId = procExec.submitProcedure( 188 new CreateTableProcedure(procExec.getEnvironment(), htd, regions)); 189 190 int lastStep = 2; // failing before CREATE_TABLE_WRITE_FS_LAYOUT 191 MasterProcedureTestingUtility.testRollbackAndDoubleExecution(procExec, procId, lastStep); 192 193 TableName tableName = htd.getTableName(); 194 MasterProcedureTestingUtility.validateTableDeletion(getMaster(), tableName); 195 196 // are we able to create the table after a rollback? 197 resetProcExecutorTestingKillFlag(); 198 testSimpleCreate(tableName, splitKeys); 199 } 200 201 public static class CreateTableProcedureOnHDFSFailure extends CreateTableProcedure { 202 private boolean failOnce = false; 203 204 public CreateTableProcedureOnHDFSFailure() { 205 // Required by the Procedure framework to create the procedure on replay 206 super(); 207 } 208 209 public CreateTableProcedureOnHDFSFailure(final MasterProcedureEnv env, 210 final TableDescriptor tableDescriptor, final RegionInfo[] newRegions) 211 throws HBaseIOException { 212 super(env, tableDescriptor, newRegions); 213 } 214 215 @Override 216 protected Flow executeFromState(MasterProcedureEnv env, 217 MasterProcedureProtos.CreateTableState state) throws InterruptedException { 218 219 if (!failOnce && 220 state == MasterProcedureProtos.CreateTableState.CREATE_TABLE_WRITE_FS_LAYOUT) { 221 try { 222 // To emulate an HDFS failure, create only the first region directory 223 RegionInfo regionInfo = getFirstRegionInfo(); 224 Configuration conf = env.getMasterConfiguration(); 225 MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); 226 Path tempdir = mfs.getTempDir(); 227 Path tableDir = FSUtils.getTableDir(tempdir, regionInfo.getTable()); 228 Path regionDir = FSUtils.getRegionDirFromTableDir(tableDir, regionInfo); 229 FileSystem fs = FileSystem.get(conf); 230 fs.mkdirs(regionDir); 231 232 failOnce = true; 233 return Flow.HAS_MORE_STATE; 234 } catch (IOException e) { 235 fail("failed to create a region directory: " + e); 236 } 237 } 238 239 return super.executeFromState(env, state); 240 } 241 } 242 243 @Test 244 public void testOnHDFSFailure() throws Exception { 245 final TableName tableName = TableName.valueOf(name.getMethodName()); 246 247 // create the table 248 final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); 249 final byte[][] splitKeys = new byte[][] { 250 Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c") 251 }; 252 TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, "f1", "f2"); 253 RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, splitKeys); 254 long procId = ProcedureTestingUtility.submitAndWait(procExec, 255 new CreateTableProcedureOnHDFSFailure(procExec.getEnvironment(), htd, regions)); 256 ProcedureTestingUtility.assertProcNotFailed(procExec, procId); 257 } 258}