001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.master.procedure; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertFalse; 022import static org.junit.Assert.assertTrue; 023 024import java.io.IOException; 025import org.apache.hadoop.hbase.DoNotRetryIOException; 026import org.apache.hadoop.hbase.HBaseClassTestRule; 027import org.apache.hadoop.hbase.HColumnDescriptor; 028import org.apache.hadoop.hbase.HTableDescriptor; 029import org.apache.hadoop.hbase.InvalidFamilyOperationException; 030import org.apache.hadoop.hbase.TableName; 031import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; 032import org.apache.hadoop.hbase.client.PerClientRandomNonceGenerator; 033import org.apache.hadoop.hbase.client.RegionInfo; 034import org.apache.hadoop.hbase.client.TableDescriptor; 035import org.apache.hadoop.hbase.client.TableDescriptorBuilder; 036import org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.StepHook; 037import org.apache.hadoop.hbase.procedure2.Procedure; 038import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; 039import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; 040import org.apache.hadoop.hbase.testclassification.MasterTests; 041import org.apache.hadoop.hbase.testclassification.MediumTests; 042import org.apache.hadoop.hbase.util.Bytes; 043import org.apache.hadoop.hbase.util.NonceKey; 044import org.junit.Assert; 045import org.junit.ClassRule; 046import org.junit.Rule; 047import org.junit.Test; 048import org.junit.experimental.categories.Category; 049import org.junit.rules.TestName; 050 051@Category({MasterTests.class, MediumTests.class}) 052public class TestModifyTableProcedure extends TestTableDDLProcedureBase { 053 054 @ClassRule 055 public static final HBaseClassTestRule CLASS_RULE = 056 HBaseClassTestRule.forClass(TestModifyTableProcedure.class); 057 058 @Rule public TestName name = new TestName(); 059 060 @Test 061 public void testModifyTable() throws Exception { 062 final TableName tableName = TableName.valueOf(name.getMethodName()); 063 final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); 064 065 MasterProcedureTestingUtility.createTable(procExec, tableName, null, "cf"); 066 UTIL.getAdmin().disableTable(tableName); 067 068 // Modify the table descriptor 069 HTableDescriptor htd = new HTableDescriptor(UTIL.getAdmin().getTableDescriptor(tableName)); 070 071 // Test 1: Modify 1 property 072 long newMaxFileSize = htd.getMaxFileSize() * 2; 073 htd.setMaxFileSize(newMaxFileSize); 074 htd.setRegionReplication(3); 075 076 long procId1 = ProcedureTestingUtility.submitAndWait( 077 procExec, new ModifyTableProcedure(procExec.getEnvironment(), htd)); 078 ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId1)); 079 080 HTableDescriptor currentHtd = UTIL.getAdmin().getTableDescriptor(tableName); 081 assertEquals(newMaxFileSize, currentHtd.getMaxFileSize()); 082 083 // Test 2: Modify multiple properties 084 boolean newReadOnlyOption = htd.isReadOnly() ? false : true; 085 long newMemStoreFlushSize = htd.getMemStoreFlushSize() * 2; 086 htd.setReadOnly(newReadOnlyOption); 087 htd.setMemStoreFlushSize(newMemStoreFlushSize); 088 089 long procId2 = ProcedureTestingUtility.submitAndWait( 090 procExec, new ModifyTableProcedure(procExec.getEnvironment(), htd)); 091 ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId2)); 092 093 currentHtd = UTIL.getAdmin().getTableDescriptor(tableName); 094 assertEquals(newReadOnlyOption, currentHtd.isReadOnly()); 095 assertEquals(newMemStoreFlushSize, currentHtd.getMemStoreFlushSize()); 096 } 097 098 @Test 099 public void testModifyTableAddCF() throws Exception { 100 final TableName tableName = TableName.valueOf(name.getMethodName()); 101 final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); 102 103 MasterProcedureTestingUtility.createTable(procExec, tableName, null, "cf1"); 104 HTableDescriptor currentHtd = UTIL.getAdmin().getTableDescriptor(tableName); 105 assertEquals(1, currentHtd.getFamiliesKeys().size()); 106 107 // Test 1: Modify the table descriptor online 108 String cf2 = "cf2"; 109 HTableDescriptor htd = new HTableDescriptor(UTIL.getAdmin().getTableDescriptor(tableName)); 110 htd.addFamily(new HColumnDescriptor(cf2)); 111 112 long procId = ProcedureTestingUtility.submitAndWait( 113 procExec, new ModifyTableProcedure(procExec.getEnvironment(), htd)); 114 ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId)); 115 116 currentHtd = UTIL.getAdmin().getTableDescriptor(tableName); 117 assertEquals(2, currentHtd.getFamiliesKeys().size()); 118 assertTrue(currentHtd.hasFamily(Bytes.toBytes(cf2))); 119 120 // Test 2: Modify the table descriptor offline 121 UTIL.getAdmin().disableTable(tableName); 122 ProcedureTestingUtility.waitNoProcedureRunning(procExec); 123 String cf3 = "cf3"; 124 HTableDescriptor htd2 = 125 new HTableDescriptor(UTIL.getAdmin().getTableDescriptor(tableName)); 126 htd2.addFamily(new HColumnDescriptor(cf3)); 127 128 long procId2 = 129 ProcedureTestingUtility.submitAndWait(procExec, 130 new ModifyTableProcedure(procExec.getEnvironment(), htd2)); 131 ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId2)); 132 133 currentHtd = UTIL.getAdmin().getTableDescriptor(tableName); 134 assertTrue(currentHtd.hasFamily(Bytes.toBytes(cf3))); 135 assertEquals(3, currentHtd.getFamiliesKeys().size()); 136 } 137 138 @Test 139 public void testModifyTableDeleteCF() throws Exception { 140 final TableName tableName = TableName.valueOf(name.getMethodName()); 141 final String cf1 = "cf1"; 142 final String cf2 = "cf2"; 143 final String cf3 = "cf3"; 144 final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); 145 146 MasterProcedureTestingUtility.createTable(procExec, tableName, null, cf1, cf2, cf3); 147 HTableDescriptor currentHtd = UTIL.getAdmin().getTableDescriptor(tableName); 148 assertEquals(3, currentHtd.getFamiliesKeys().size()); 149 150 // Test 1: Modify the table descriptor 151 HTableDescriptor htd = new HTableDescriptor(UTIL.getAdmin().getTableDescriptor(tableName)); 152 htd.removeFamily(Bytes.toBytes(cf2)); 153 154 long procId = ProcedureTestingUtility.submitAndWait( 155 procExec, new ModifyTableProcedure(procExec.getEnvironment(), htd)); 156 ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId)); 157 158 currentHtd = UTIL.getAdmin().getTableDescriptor(tableName); 159 assertEquals(2, currentHtd.getFamiliesKeys().size()); 160 assertFalse(currentHtd.hasFamily(Bytes.toBytes(cf2))); 161 162 // Test 2: Modify the table descriptor offline 163 UTIL.getAdmin().disableTable(tableName); 164 ProcedureTestingUtility.waitNoProcedureRunning(procExec); 165 166 HTableDescriptor htd2 = 167 new HTableDescriptor(UTIL.getAdmin().getTableDescriptor(tableName)); 168 htd2.removeFamily(Bytes.toBytes(cf3)); 169 // Disable Sanity check 170 htd2.setConfiguration("hbase.table.sanity.checks", Boolean.FALSE.toString()); 171 172 long procId2 = 173 ProcedureTestingUtility.submitAndWait(procExec, 174 new ModifyTableProcedure(procExec.getEnvironment(), htd2)); 175 ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId2)); 176 177 currentHtd = UTIL.getAdmin().getTableDescriptor(tableName); 178 assertEquals(1, currentHtd.getFamiliesKeys().size()); 179 assertFalse(currentHtd.hasFamily(Bytes.toBytes(cf3))); 180 181 //Removing the last family will fail 182 HTableDescriptor htd3 = 183 new HTableDescriptor(UTIL.getAdmin().getTableDescriptor(tableName)); 184 htd3.removeFamily(Bytes.toBytes(cf1)); 185 long procId3 = 186 ProcedureTestingUtility.submitAndWait(procExec, 187 new ModifyTableProcedure(procExec.getEnvironment(), htd3)); 188 final Procedure<?> result = procExec.getResult(procId3); 189 assertEquals(true, result.isFailed()); 190 Throwable cause = ProcedureTestingUtility.getExceptionCause(result); 191 assertTrue("expected DoNotRetryIOException, got " + cause, 192 cause instanceof DoNotRetryIOException); 193 assertEquals(1, currentHtd.getFamiliesKeys().size()); 194 assertTrue(currentHtd.hasFamily(Bytes.toBytes(cf1))); 195 } 196 197 @Test 198 public void testRecoveryAndDoubleExecutionOffline() throws Exception { 199 final TableName tableName = TableName.valueOf(name.getMethodName()); 200 final String cf2 = "cf2"; 201 final String cf3 = "cf3"; 202 final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); 203 204 // create the table 205 RegionInfo[] regions = MasterProcedureTestingUtility.createTable( 206 procExec, tableName, null, "cf1", cf3); 207 UTIL.getAdmin().disableTable(tableName); 208 209 ProcedureTestingUtility.waitNoProcedureRunning(procExec); 210 ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); 211 212 // Modify multiple properties of the table. 213 TableDescriptor oldDescriptor = UTIL.getAdmin().getDescriptor(tableName); 214 TableDescriptor newDescriptor = TableDescriptorBuilder.newBuilder(oldDescriptor) 215 .setCompactionEnabled(!oldDescriptor.isCompactionEnabled()) 216 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(cf2)) 217 .removeColumnFamily(Bytes.toBytes(cf3)) 218 .setRegionReplication(3) 219 .build(); 220 221 // Start the Modify procedure && kill the executor 222 long procId = procExec.submitProcedure( 223 new ModifyTableProcedure(procExec.getEnvironment(), newDescriptor)); 224 225 // Restart the executor and execute the step twice 226 MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId); 227 228 // Validate descriptor 229 TableDescriptor currentDescriptor = UTIL.getAdmin().getDescriptor(tableName); 230 assertEquals(newDescriptor.isCompactionEnabled(), currentDescriptor.isCompactionEnabled()); 231 assertEquals(2, newDescriptor.getColumnFamilyNames().size()); 232 233 // cf2 should be added cf3 should be removed 234 MasterProcedureTestingUtility.validateTableCreation(UTIL.getHBaseCluster().getMaster(), 235 tableName, regions, false, "cf1", cf2); 236 } 237 238 @Test 239 public void testRecoveryAndDoubleExecutionOnline() throws Exception { 240 final TableName tableName = TableName.valueOf(name.getMethodName()); 241 final String cf2 = "cf2"; 242 final String cf3 = "cf3"; 243 final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); 244 245 // create the table 246 RegionInfo[] regions = MasterProcedureTestingUtility.createTable( 247 procExec, tableName, null, "cf1", cf3); 248 249 ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); 250 251 // Modify multiple properties of the table. 252 HTableDescriptor htd = new HTableDescriptor(UTIL.getAdmin().getTableDescriptor(tableName)); 253 boolean newCompactionEnableOption = htd.isCompactionEnabled() ? false : true; 254 htd.setCompactionEnabled(newCompactionEnableOption); 255 htd.addFamily(new HColumnDescriptor(cf2)); 256 htd.removeFamily(Bytes.toBytes(cf3)); 257 258 // Start the Modify procedure && kill the executor 259 long procId = procExec.submitProcedure( 260 new ModifyTableProcedure(procExec.getEnvironment(), htd)); 261 262 // Restart the executor and execute the step twice 263 MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId); 264 265 // Validate descriptor 266 HTableDescriptor currentHtd = UTIL.getAdmin().getTableDescriptor(tableName); 267 assertEquals(newCompactionEnableOption, currentHtd.isCompactionEnabled()); 268 assertEquals(2, currentHtd.getFamiliesKeys().size()); 269 assertTrue(currentHtd.hasFamily(Bytes.toBytes(cf2))); 270 assertFalse(currentHtd.hasFamily(Bytes.toBytes(cf3))); 271 272 // cf2 should be added cf3 should be removed 273 MasterProcedureTestingUtility.validateTableCreation(UTIL.getHBaseCluster().getMaster(), 274 tableName, regions, "cf1", cf2); 275 } 276 277 @Test 278 public void testColumnFamilyAdditionTwiceWithNonce() throws Exception { 279 final TableName tableName = TableName.valueOf(name.getMethodName()); 280 final String cf2 = "cf2"; 281 final String cf3 = "cf3"; 282 final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); 283 284 // create the table 285 RegionInfo[] regions = 286 MasterProcedureTestingUtility.createTable(procExec, tableName, null, "cf1", cf3); 287 288 ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); 289 // Modify multiple properties of the table. 290 TableDescriptor td = UTIL.getAdmin().getDescriptor(tableName); 291 TableDescriptor newTd = 292 TableDescriptorBuilder.newBuilder(td).setCompactionEnabled(!td.isCompactionEnabled()) 293 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(cf2)).build(); 294 295 PerClientRandomNonceGenerator nonceGenerator = PerClientRandomNonceGenerator.get(); 296 long nonceGroup = nonceGenerator.getNonceGroup(); 297 long newNonce = nonceGenerator.newNonce(); 298 NonceKey nonceKey = new NonceKey(nonceGroup, newNonce); 299 procExec.registerNonce(nonceKey); 300 301 // Start the Modify procedure && kill the executor 302 final long procId = procExec 303 .submitProcedure(new ModifyTableProcedure(procExec.getEnvironment(), newTd), nonceKey); 304 305 // Restart the executor after MODIFY_TABLE_UPDATE_TABLE_DESCRIPTOR and try to add column family 306 // as nonce are there , we should not fail 307 MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId, new StepHook() { 308 @Override 309 public boolean execute(int step) throws IOException { 310 if (step == 3) { 311 return procId == UTIL.getHBaseCluster().getMaster().addColumn(tableName, 312 ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(cf2)).build(), nonceGroup, 313 newNonce); 314 } 315 return true; 316 } 317 }); 318 319 //Try with different nonce, now it should fail the checks 320 try { 321 UTIL.getHBaseCluster().getMaster().addColumn(tableName, 322 ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(cf2)).build(), nonceGroup, 323 nonceGenerator.newNonce()); 324 Assert.fail(); 325 } catch (InvalidFamilyOperationException e) { 326 } 327 328 // Validate descriptor 329 TableDescriptor currentHtd = UTIL.getAdmin().getDescriptor(tableName); 330 assertEquals(!td.isCompactionEnabled(), currentHtd.isCompactionEnabled()); 331 assertEquals(3, currentHtd.getColumnFamilyCount()); 332 assertTrue(currentHtd.hasColumnFamily(Bytes.toBytes(cf2))); 333 assertTrue(currentHtd.hasColumnFamily(Bytes.toBytes(cf3))); 334 335 // cf2 should be added 336 MasterProcedureTestingUtility.validateTableCreation(UTIL.getHBaseCluster().getMaster(), 337 tableName, regions, "cf1", cf2, cf3); 338 } 339 340 @Test 341 public void testRollbackAndDoubleExecutionOnline() throws Exception { 342 final TableName tableName = TableName.valueOf(name.getMethodName()); 343 final String familyName = "cf2"; 344 final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); 345 346 // create the table 347 RegionInfo[] regions = MasterProcedureTestingUtility.createTable( 348 procExec, tableName, null, "cf1"); 349 350 ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); 351 352 TableDescriptor td = UTIL.getAdmin().getDescriptor(tableName); 353 TableDescriptor newTd = 354 TableDescriptorBuilder.newBuilder(td).setCompactionEnabled(!td.isCompactionEnabled()) 355 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(familyName)).build(); 356 357 // Start the Modify procedure && kill the executor 358 long procId = 359 procExec.submitProcedure(new ModifyTableProcedure(procExec.getEnvironment(), newTd)); 360 361 int lastStep = 3; // failing before MODIFY_TABLE_UPDATE_TABLE_DESCRIPTOR 362 MasterProcedureTestingUtility.testRollbackAndDoubleExecution(procExec, procId, lastStep); 363 364 // cf2 should not be present 365 MasterProcedureTestingUtility.validateTableCreation(UTIL.getHBaseCluster().getMaster(), 366 tableName, regions, "cf1"); 367 } 368 369 @Test 370 public void testRollbackAndDoubleExecutionOffline() throws Exception { 371 final TableName tableName = TableName.valueOf(name.getMethodName()); 372 final String familyName = "cf2"; 373 final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); 374 375 // create the table 376 RegionInfo[] regions = MasterProcedureTestingUtility.createTable( 377 procExec, tableName, null, "cf1"); 378 UTIL.getAdmin().disableTable(tableName); 379 380 ProcedureTestingUtility.waitNoProcedureRunning(procExec); 381 ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); 382 383 TableDescriptor td = UTIL.getAdmin().getDescriptor(tableName); 384 TableDescriptor newTd = 385 TableDescriptorBuilder.newBuilder(td).setCompactionEnabled(!td.isCompactionEnabled()) 386 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(familyName)).setRegionReplication(3) 387 .build(); 388 389 // Start the Modify procedure && kill the executor 390 long procId = procExec.submitProcedure( 391 new ModifyTableProcedure(procExec.getEnvironment(), newTd)); 392 393 // Restart the executor and rollback the step twice 394 int lastStep = 3; // failing before MODIFY_TABLE_UPDATE_TABLE_DESCRIPTOR 395 MasterProcedureTestingUtility.testRollbackAndDoubleExecution(procExec, procId, lastStep); 396 397 // cf2 should not be present 398 MasterProcedureTestingUtility.validateTableCreation(UTIL.getHBaseCluster().getMaster(), 399 tableName, regions, "cf1"); 400 } 401}