001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.master.procedure; 019 020import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACKER_IMPL; 021import static org.junit.Assert.assertEquals; 022import static org.junit.Assert.assertFalse; 023import static org.junit.Assert.assertTrue; 024import static org.junit.Assert.fail; 025 026import java.io.IOException; 027import org.apache.hadoop.conf.Configuration; 028import org.apache.hadoop.fs.FileSystem; 029import org.apache.hadoop.fs.Path; 030import org.apache.hadoop.hbase.DoNotRetryIOException; 031import org.apache.hadoop.hbase.HBaseClassTestRule; 032import org.apache.hadoop.hbase.HBaseIOException; 033import org.apache.hadoop.hbase.TableExistsException; 034import org.apache.hadoop.hbase.TableName; 035import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; 036import org.apache.hadoop.hbase.client.RegionInfo; 037import org.apache.hadoop.hbase.client.TableDescriptor; 038import org.apache.hadoop.hbase.client.TableDescriptorBuilder; 039import org.apache.hadoop.hbase.master.MasterFileSystem; 040import org.apache.hadoop.hbase.procedure2.Procedure; 041import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; 042import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; 043import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; 044import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerForTest; 045import org.apache.hadoop.hbase.testclassification.MasterTests; 046import org.apache.hadoop.hbase.testclassification.MediumTests; 047import org.apache.hadoop.hbase.util.Bytes; 048import org.apache.hadoop.hbase.util.CommonFSUtils; 049import org.apache.hadoop.hbase.util.FSUtils; 050import org.apache.hadoop.hbase.util.ModifyRegionUtils; 051import org.apache.hadoop.hbase.util.TableDescriptorChecker; 052import org.junit.ClassRule; 053import org.junit.Rule; 054import org.junit.Test; 055import org.junit.experimental.categories.Category; 056import org.junit.rules.TestName; 057 058import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos; 059 060@Category({ MasterTests.class, MediumTests.class }) 061public class TestCreateTableProcedure extends TestTableDDLProcedureBase { 062 063 @ClassRule 064 public static final HBaseClassTestRule CLASS_RULE = 065 HBaseClassTestRule.forClass(TestCreateTableProcedure.class); 066 067 private static final String F1 = "f1"; 068 private static final String F2 = "f2"; 069 070 @Rule 071 public TestName name = new TestName(); 072 073 @Test 074 public void testSimpleCreate() throws Exception { 075 final TableName tableName = TableName.valueOf(name.getMethodName()); 076 final byte[][] splitKeys = null; 077 testSimpleCreate(tableName, splitKeys); 078 } 079 080 @Test 081 public void testSimpleCreateWithSplits() throws Exception { 082 final TableName tableName = TableName.valueOf(name.getMethodName()); 083 final byte[][] splitKeys = 084 new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c") }; 085 testSimpleCreate(tableName, splitKeys); 086 } 087 088 private void testSimpleCreate(final TableName tableName, byte[][] splitKeys) throws Exception { 089 RegionInfo[] regions = MasterProcedureTestingUtility.createTable(getMasterProcedureExecutor(), 090 tableName, splitKeys, F1, F2); 091 MasterProcedureTestingUtility.validateTableCreation(getMaster(), tableName, regions, F1, F2); 092 } 093 094 @Test 095 public void testCreateWithTrackImpl() throws Exception { 096 final TableName tableName = TableName.valueOf(name.getMethodName()); 097 ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); 098 TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, F1); 099 String trackerName = StoreFileTrackerForTest.class.getName(); 100 htd = TableDescriptorBuilder.newBuilder(htd).setValue(TRACKER_IMPL, trackerName).build(); 101 RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, null); 102 long procId = ProcedureTestingUtility.submitAndWait(procExec, 103 new CreateTableProcedure(procExec.getEnvironment(), htd, regions)); 104 ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId)); 105 htd = getMaster().getTableDescriptors().get(tableName); 106 assertEquals(trackerName, htd.getValue(TRACKER_IMPL)); 107 } 108 109 @Test 110 public void testCreateWithFileBasedStoreTrackerImpl() throws Exception { 111 ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); 112 procExec.getEnvironment().getMasterConfiguration().set(StoreFileTrackerFactory.TRACKER_IMPL, 113 StoreFileTrackerFactory.Trackers.FILE.name()); 114 final TableName tableName = TableName.valueOf(name.getMethodName()); 115 TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, F1); 116 RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, null); 117 long procId = ProcedureTestingUtility.submitAndWait(procExec, 118 new CreateTableProcedure(procExec.getEnvironment(), htd, regions)); 119 ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId)); 120 htd = getMaster().getTableDescriptors().get(tableName); 121 assertEquals(StoreFileTrackerFactory.Trackers.FILE.name(), htd.getValue(TRACKER_IMPL)); 122 } 123 124 @Test 125 public void testCreateWithoutColumnFamily() throws Exception { 126 final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); 127 final TableName tableName = TableName.valueOf(name.getMethodName()); 128 // create table with 0 families will fail 129 final TableDescriptorBuilder builder = 130 TableDescriptorBuilder.newBuilder(MasterProcedureTestingUtility.createHTD(tableName)); 131 132 // disable sanity check 133 builder.setValue(TableDescriptorChecker.TABLE_SANITY_CHECKS, Boolean.FALSE.toString()); 134 TableDescriptor htd = builder.build(); 135 final RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, null); 136 137 long procId = ProcedureTestingUtility.submitAndWait(procExec, 138 new CreateTableProcedure(procExec.getEnvironment(), htd, regions)); 139 final Procedure<?> result = procExec.getResult(procId); 140 assertEquals(true, result.isFailed()); 141 Throwable cause = ProcedureTestingUtility.getExceptionCause(result); 142 assertTrue("expected DoNotRetryIOException, got " + cause, 143 cause instanceof DoNotRetryIOException); 144 } 145 146 @Test(expected = TableExistsException.class) 147 public void testCreateExisting() throws Exception { 148 final TableName tableName = TableName.valueOf(name.getMethodName()); 149 final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); 150 final TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, "f"); 151 final RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, null); 152 153 // create the table 154 long procId1 = 155 procExec.submitProcedure(new CreateTableProcedure(procExec.getEnvironment(), htd, regions)); 156 157 // create another with the same name 158 ProcedurePrepareLatch latch2 = new ProcedurePrepareLatch.CompatibilityLatch(); 159 long procId2 = procExec 160 .submitProcedure(new CreateTableProcedure(procExec.getEnvironment(), htd, regions, latch2)); 161 162 ProcedureTestingUtility.waitProcedure(procExec, procId1); 163 ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId1)); 164 165 ProcedureTestingUtility.waitProcedure(procExec, procId2); 166 latch2.await(); 167 } 168 169 @Test 170 public void testRecoveryAndDoubleExecution() throws Exception { 171 final TableName tableName = TableName.valueOf(name.getMethodName()); 172 173 // create the table 174 final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); 175 ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); 176 177 // Start the Create procedure && kill the executor 178 byte[][] splitKeys = null; 179 TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, "f1", "f2"); 180 RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, splitKeys); 181 long procId = 182 procExec.submitProcedure(new CreateTableProcedure(procExec.getEnvironment(), htd, regions)); 183 184 // Restart the executor and execute the step twice 185 MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId); 186 MasterProcedureTestingUtility.validateTableCreation(getMaster(), tableName, regions, F1, F2); 187 } 188 189 @Test 190 public void testRollbackAndDoubleExecution() throws Exception { 191 final TableName tableName = TableName.valueOf(name.getMethodName()); 192 testRollbackAndDoubleExecution(TableDescriptorBuilder 193 .newBuilder(MasterProcedureTestingUtility.createHTD(tableName, F1, F2))); 194 } 195 196 @Test 197 public void testRollbackAndDoubleExecutionOnMobTable() throws Exception { 198 final TableName tableName = TableName.valueOf(name.getMethodName()); 199 TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, F1, F2); 200 TableDescriptorBuilder builder = 201 TableDescriptorBuilder.newBuilder(htd).modifyColumnFamily(ColumnFamilyDescriptorBuilder 202 .newBuilder(htd.getColumnFamily(Bytes.toBytes(F1))).setMobEnabled(true).build()); 203 testRollbackAndDoubleExecution(builder); 204 } 205 206 private void testRollbackAndDoubleExecution(TableDescriptorBuilder builder) throws Exception { 207 // create the table 208 final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); 209 ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); 210 211 // Start the Create procedure && kill the executor 212 final byte[][] splitKeys = 213 new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c") }; 214 builder.setRegionReplication(3); 215 TableDescriptor htd = builder.build(); 216 RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, splitKeys); 217 long procId = 218 procExec.submitProcedure(new CreateTableProcedure(procExec.getEnvironment(), htd, regions)); 219 220 int lastStep = 2; // failing before CREATE_TABLE_WRITE_FS_LAYOUT 221 MasterProcedureTestingUtility.testRollbackAndDoubleExecution(procExec, procId, lastStep); 222 223 TableName tableName = htd.getTableName(); 224 MasterProcedureTestingUtility.validateTableDeletion(getMaster(), tableName); 225 226 // are we able to create the table after a rollback? 227 resetProcExecutorTestingKillFlag(); 228 testSimpleCreate(tableName, splitKeys); 229 } 230 231 public static class CreateTableProcedureOnHDFSFailure extends CreateTableProcedure { 232 private boolean failOnce = false; 233 234 public CreateTableProcedureOnHDFSFailure() { 235 // Required by the Procedure framework to create the procedure on replay 236 super(); 237 } 238 239 public CreateTableProcedureOnHDFSFailure(final MasterProcedureEnv env, 240 final TableDescriptor tableDescriptor, final RegionInfo[] newRegions) 241 throws HBaseIOException { 242 super(env, tableDescriptor, newRegions); 243 } 244 245 @Override 246 protected Flow executeFromState(MasterProcedureEnv env, 247 MasterProcedureProtos.CreateTableState state) throws InterruptedException { 248 249 if ( 250 !failOnce && state == MasterProcedureProtos.CreateTableState.CREATE_TABLE_WRITE_FS_LAYOUT 251 ) { 252 try { 253 // To emulate an HDFS failure, create only the first region directory 254 RegionInfo regionInfo = getFirstRegionInfo(); 255 Configuration conf = env.getMasterConfiguration(); 256 MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); 257 Path tempdir = mfs.getTempDir(); 258 Path tableDir = CommonFSUtils.getTableDir(tempdir, regionInfo.getTable()); 259 Path regionDir = FSUtils.getRegionDirFromTableDir(tableDir, regionInfo); 260 FileSystem fs = FileSystem.get(conf); 261 fs.mkdirs(regionDir); 262 263 failOnce = true; 264 return Flow.HAS_MORE_STATE; 265 } catch (IOException e) { 266 fail("failed to create a region directory: " + e); 267 } 268 } 269 270 return super.executeFromState(env, state); 271 } 272 } 273 274 @Test 275 public void testOnHDFSFailure() throws Exception { 276 final TableName tableName = TableName.valueOf(name.getMethodName()); 277 278 // create the table 279 final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); 280 final byte[][] splitKeys = 281 new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c") }; 282 TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, "f1", "f2"); 283 RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, splitKeys); 284 long procId = ProcedureTestingUtility.submitAndWait(procExec, 285 new CreateTableProcedureOnHDFSFailure(procExec.getEnvironment(), htd, regions)); 286 ProcedureTestingUtility.assertProcNotFailed(procExec, procId); 287 } 288 289 @Test 290 public void testCreateTableWithManyRegionReplication() throws IOException { 291 final int EXCEED_MAX_REGION_REPLICATION = 0x10001; 292 TableName tableName = TableName.valueOf(name.getMethodName()); 293 ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); 294 295 TableDescriptor tableWithManyRegionReplication = TableDescriptorBuilder.newBuilder(tableName) 296 .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("f1")).build()) 297 .setRegionReplication(EXCEED_MAX_REGION_REPLICATION).build(); 298 RegionInfo[] regions01 = 299 ModifyRegionUtils.createRegionInfos(tableWithManyRegionReplication, null); 300 long procId01 = ProcedureTestingUtility.submitAndWait(procExec, new CreateTableProcedure( 301 procExec.getEnvironment(), tableWithManyRegionReplication, regions01)); 302 Procedure<?> result01 = procExec.getResult(procId01); 303 assertTrue(result01.getException().getCause() instanceof IllegalArgumentException); 304 assertFalse(UTIL.getAdmin().tableExists(tableName)); 305 306 TableDescriptor tdesc = TableDescriptorBuilder.newBuilder(tableName) 307 .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("f1")).build()) 308 .build(); 309 RegionInfo[] regions02 = ModifyRegionUtils.createRegionInfos(tdesc, null); 310 long procId02 = ProcedureTestingUtility.submitAndWait(procExec, 311 new CreateTableProcedure(procExec.getEnvironment(), tdesc, regions02)); 312 Procedure<?> result02 = procExec.getResult(procId02); 313 assertTrue(result02.isSuccess()); 314 assertTrue(UTIL.getAdmin().tableExists(tableName)); 315 } 316}