001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.procedure;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertTrue;
023
024import java.io.IOException;
025import org.apache.hadoop.hbase.DoNotRetryIOException;
026import org.apache.hadoop.hbase.HBaseClassTestRule;
027import org.apache.hadoop.hbase.HColumnDescriptor;
028import org.apache.hadoop.hbase.HTableDescriptor;
029import org.apache.hadoop.hbase.InvalidFamilyOperationException;
030import org.apache.hadoop.hbase.TableName;
031import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
032import org.apache.hadoop.hbase.client.PerClientRandomNonceGenerator;
033import org.apache.hadoop.hbase.client.RegionInfo;
034import org.apache.hadoop.hbase.client.TableDescriptor;
035import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
036import org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.StepHook;
037import org.apache.hadoop.hbase.procedure2.Procedure;
038import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
039import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
040import org.apache.hadoop.hbase.testclassification.MasterTests;
041import org.apache.hadoop.hbase.testclassification.MediumTests;
042import org.apache.hadoop.hbase.util.Bytes;
043import org.apache.hadoop.hbase.util.NonceKey;
044import org.junit.Assert;
045import org.junit.ClassRule;
046import org.junit.Rule;
047import org.junit.Test;
048import org.junit.experimental.categories.Category;
049import org.junit.rules.TestName;
050
051@Category({MasterTests.class, MediumTests.class})
052public class TestModifyTableProcedure extends TestTableDDLProcedureBase {
053
054  @ClassRule
055  public static final HBaseClassTestRule CLASS_RULE =
056      HBaseClassTestRule.forClass(TestModifyTableProcedure.class);
057
058  @Rule public TestName name = new TestName();
059
060  @Test
061  public void testModifyTable() throws Exception {
062    final TableName tableName = TableName.valueOf(name.getMethodName());
063    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
064
065    MasterProcedureTestingUtility.createTable(procExec, tableName, null, "cf");
066    UTIL.getAdmin().disableTable(tableName);
067
068    // Modify the table descriptor
069    HTableDescriptor htd = new HTableDescriptor(UTIL.getAdmin().getTableDescriptor(tableName));
070
071    // Test 1: Modify 1 property
072    long newMaxFileSize = htd.getMaxFileSize() * 2;
073    htd.setMaxFileSize(newMaxFileSize);
074    htd.setRegionReplication(3);
075
076    long procId1 = ProcedureTestingUtility.submitAndWait(
077        procExec, new ModifyTableProcedure(procExec.getEnvironment(), htd));
078    ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId1));
079
080    HTableDescriptor currentHtd = UTIL.getAdmin().getTableDescriptor(tableName);
081    assertEquals(newMaxFileSize, currentHtd.getMaxFileSize());
082
083    // Test 2: Modify multiple properties
084    boolean newReadOnlyOption = htd.isReadOnly() ? false : true;
085    long newMemStoreFlushSize = htd.getMemStoreFlushSize() * 2;
086    htd.setReadOnly(newReadOnlyOption);
087    htd.setMemStoreFlushSize(newMemStoreFlushSize);
088
089    long procId2 = ProcedureTestingUtility.submitAndWait(
090        procExec, new ModifyTableProcedure(procExec.getEnvironment(), htd));
091    ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId2));
092
093    currentHtd = UTIL.getAdmin().getTableDescriptor(tableName);
094    assertEquals(newReadOnlyOption, currentHtd.isReadOnly());
095    assertEquals(newMemStoreFlushSize, currentHtd.getMemStoreFlushSize());
096  }
097
098  @Test
099  public void testModifyTableAddCF() throws Exception {
100    final TableName tableName = TableName.valueOf(name.getMethodName());
101    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
102
103    MasterProcedureTestingUtility.createTable(procExec, tableName, null, "cf1");
104    HTableDescriptor currentHtd = UTIL.getAdmin().getTableDescriptor(tableName);
105    assertEquals(1, currentHtd.getFamiliesKeys().size());
106
107    // Test 1: Modify the table descriptor online
108    String cf2 = "cf2";
109    HTableDescriptor htd = new HTableDescriptor(UTIL.getAdmin().getTableDescriptor(tableName));
110    htd.addFamily(new HColumnDescriptor(cf2));
111
112    long procId = ProcedureTestingUtility.submitAndWait(
113        procExec, new ModifyTableProcedure(procExec.getEnvironment(), htd));
114    ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId));
115
116    currentHtd = UTIL.getAdmin().getTableDescriptor(tableName);
117    assertEquals(2, currentHtd.getFamiliesKeys().size());
118    assertTrue(currentHtd.hasFamily(Bytes.toBytes(cf2)));
119
120    // Test 2: Modify the table descriptor offline
121    UTIL.getAdmin().disableTable(tableName);
122    ProcedureTestingUtility.waitNoProcedureRunning(procExec);
123    String cf3 = "cf3";
124    HTableDescriptor htd2 =
125        new HTableDescriptor(UTIL.getAdmin().getTableDescriptor(tableName));
126    htd2.addFamily(new HColumnDescriptor(cf3));
127
128    long procId2 =
129        ProcedureTestingUtility.submitAndWait(procExec,
130          new ModifyTableProcedure(procExec.getEnvironment(), htd2));
131    ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId2));
132
133    currentHtd = UTIL.getAdmin().getTableDescriptor(tableName);
134    assertTrue(currentHtd.hasFamily(Bytes.toBytes(cf3)));
135    assertEquals(3, currentHtd.getFamiliesKeys().size());
136  }
137
138  @Test
139  public void testModifyTableDeleteCF() throws Exception {
140    final TableName tableName = TableName.valueOf(name.getMethodName());
141    final String cf1 = "cf1";
142    final String cf2 = "cf2";
143    final String cf3 = "cf3";
144    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
145
146    MasterProcedureTestingUtility.createTable(procExec, tableName, null, cf1, cf2, cf3);
147    HTableDescriptor currentHtd = UTIL.getAdmin().getTableDescriptor(tableName);
148    assertEquals(3, currentHtd.getFamiliesKeys().size());
149
150    // Test 1: Modify the table descriptor
151    HTableDescriptor htd = new HTableDescriptor(UTIL.getAdmin().getTableDescriptor(tableName));
152    htd.removeFamily(Bytes.toBytes(cf2));
153
154    long procId = ProcedureTestingUtility.submitAndWait(
155        procExec, new ModifyTableProcedure(procExec.getEnvironment(), htd));
156    ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId));
157
158    currentHtd = UTIL.getAdmin().getTableDescriptor(tableName);
159    assertEquals(2, currentHtd.getFamiliesKeys().size());
160    assertFalse(currentHtd.hasFamily(Bytes.toBytes(cf2)));
161
162    // Test 2: Modify the table descriptor offline
163    UTIL.getAdmin().disableTable(tableName);
164    ProcedureTestingUtility.waitNoProcedureRunning(procExec);
165
166    HTableDescriptor htd2 =
167        new HTableDescriptor(UTIL.getAdmin().getTableDescriptor(tableName));
168    htd2.removeFamily(Bytes.toBytes(cf3));
169    // Disable Sanity check
170    htd2.setConfiguration("hbase.table.sanity.checks", Boolean.FALSE.toString());
171
172    long procId2 =
173        ProcedureTestingUtility.submitAndWait(procExec,
174          new ModifyTableProcedure(procExec.getEnvironment(), htd2));
175    ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId2));
176
177    currentHtd = UTIL.getAdmin().getTableDescriptor(tableName);
178    assertEquals(1, currentHtd.getFamiliesKeys().size());
179    assertFalse(currentHtd.hasFamily(Bytes.toBytes(cf3)));
180
181    //Removing the last family will fail
182    HTableDescriptor htd3 =
183        new HTableDescriptor(UTIL.getAdmin().getTableDescriptor(tableName));
184    htd3.removeFamily(Bytes.toBytes(cf1));
185    long procId3 =
186        ProcedureTestingUtility.submitAndWait(procExec,
187            new ModifyTableProcedure(procExec.getEnvironment(), htd3));
188    final Procedure<?> result = procExec.getResult(procId3);
189    assertEquals(true, result.isFailed());
190    Throwable cause = ProcedureTestingUtility.getExceptionCause(result);
191    assertTrue("expected DoNotRetryIOException, got " + cause,
192        cause instanceof DoNotRetryIOException);
193    assertEquals(1, currentHtd.getFamiliesKeys().size());
194    assertTrue(currentHtd.hasFamily(Bytes.toBytes(cf1)));
195  }
196
197  @Test
198  public void testRecoveryAndDoubleExecutionOffline() throws Exception {
199    final TableName tableName = TableName.valueOf(name.getMethodName());
200    final String cf2 = "cf2";
201    final String cf3 = "cf3";
202    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
203
204    // create the table
205    RegionInfo[] regions = MasterProcedureTestingUtility.createTable(
206      procExec, tableName, null, "cf1", cf3);
207    UTIL.getAdmin().disableTable(tableName);
208
209    ProcedureTestingUtility.waitNoProcedureRunning(procExec);
210    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
211
212    // Modify multiple properties of the table.
213    TableDescriptor oldDescriptor = UTIL.getAdmin().getDescriptor(tableName);
214    TableDescriptor newDescriptor = TableDescriptorBuilder.newBuilder(oldDescriptor)
215        .setCompactionEnabled(!oldDescriptor.isCompactionEnabled())
216        .setColumnFamily(ColumnFamilyDescriptorBuilder.of(cf2))
217        .removeColumnFamily(Bytes.toBytes(cf3))
218        .setRegionReplication(3)
219        .build();
220
221    // Start the Modify procedure && kill the executor
222    long procId = procExec.submitProcedure(
223      new ModifyTableProcedure(procExec.getEnvironment(), newDescriptor));
224
225    // Restart the executor and execute the step twice
226    MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId);
227
228    // Validate descriptor
229    TableDescriptor currentDescriptor = UTIL.getAdmin().getDescriptor(tableName);
230    assertEquals(newDescriptor.isCompactionEnabled(), currentDescriptor.isCompactionEnabled());
231    assertEquals(2, newDescriptor.getColumnFamilyNames().size());
232
233    // cf2 should be added cf3 should be removed
234    MasterProcedureTestingUtility.validateTableCreation(UTIL.getHBaseCluster().getMaster(),
235      tableName, regions, false, "cf1", cf2);
236  }
237
238  @Test
239  public void testRecoveryAndDoubleExecutionOnline() throws Exception {
240    final TableName tableName = TableName.valueOf(name.getMethodName());
241    final String cf2 = "cf2";
242    final String cf3 = "cf3";
243    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
244
245    // create the table
246    RegionInfo[] regions = MasterProcedureTestingUtility.createTable(
247      procExec, tableName, null, "cf1", cf3);
248
249    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
250
251    // Modify multiple properties of the table.
252    HTableDescriptor htd = new HTableDescriptor(UTIL.getAdmin().getTableDescriptor(tableName));
253    boolean newCompactionEnableOption = htd.isCompactionEnabled() ? false : true;
254    htd.setCompactionEnabled(newCompactionEnableOption);
255    htd.addFamily(new HColumnDescriptor(cf2));
256    htd.removeFamily(Bytes.toBytes(cf3));
257
258    // Start the Modify procedure && kill the executor
259    long procId = procExec.submitProcedure(
260      new ModifyTableProcedure(procExec.getEnvironment(), htd));
261
262    // Restart the executor and execute the step twice
263    MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId);
264
265    // Validate descriptor
266    HTableDescriptor currentHtd = UTIL.getAdmin().getTableDescriptor(tableName);
267    assertEquals(newCompactionEnableOption, currentHtd.isCompactionEnabled());
268    assertEquals(2, currentHtd.getFamiliesKeys().size());
269    assertTrue(currentHtd.hasFamily(Bytes.toBytes(cf2)));
270    assertFalse(currentHtd.hasFamily(Bytes.toBytes(cf3)));
271
272    // cf2 should be added cf3 should be removed
273    MasterProcedureTestingUtility.validateTableCreation(UTIL.getHBaseCluster().getMaster(),
274      tableName, regions, "cf1", cf2);
275  }
276
277  @Test
278  public void testColumnFamilyAdditionTwiceWithNonce() throws Exception {
279    final TableName tableName = TableName.valueOf(name.getMethodName());
280    final String cf2 = "cf2";
281    final String cf3 = "cf3";
282    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
283
284    // create the table
285    RegionInfo[] regions =
286        MasterProcedureTestingUtility.createTable(procExec, tableName, null, "cf1", cf3);
287
288    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
289    // Modify multiple properties of the table.
290    TableDescriptor td = UTIL.getAdmin().getDescriptor(tableName);
291    TableDescriptor newTd =
292        TableDescriptorBuilder.newBuilder(td).setCompactionEnabled(!td.isCompactionEnabled())
293          .setColumnFamily(ColumnFamilyDescriptorBuilder.of(cf2)).build();
294
295    PerClientRandomNonceGenerator nonceGenerator = PerClientRandomNonceGenerator.get();
296    long nonceGroup = nonceGenerator.getNonceGroup();
297    long newNonce = nonceGenerator.newNonce();
298    NonceKey nonceKey = new NonceKey(nonceGroup, newNonce);
299    procExec.registerNonce(nonceKey);
300
301    // Start the Modify procedure && kill the executor
302    final long procId = procExec
303        .submitProcedure(new ModifyTableProcedure(procExec.getEnvironment(), newTd), nonceKey);
304
305    // Restart the executor after MODIFY_TABLE_UPDATE_TABLE_DESCRIPTOR and try to add column family
306    // as nonce are there , we should not fail
307    MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId, new StepHook() {
308      @Override
309      public boolean execute(int step) throws IOException {
310        if (step == 3) {
311          return procId == UTIL.getHBaseCluster().getMaster().addColumn(tableName,
312            ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(cf2)).build(), nonceGroup,
313            newNonce);
314        }
315        return true;
316      }
317    });
318
319    //Try with different nonce, now it should fail the checks
320    try {
321      UTIL.getHBaseCluster().getMaster().addColumn(tableName,
322        ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(cf2)).build(), nonceGroup,
323        nonceGenerator.newNonce());
324      Assert.fail();
325    } catch (InvalidFamilyOperationException e) {
326    }
327
328    // Validate descriptor
329    TableDescriptor currentHtd = UTIL.getAdmin().getDescriptor(tableName);
330    assertEquals(!td.isCompactionEnabled(), currentHtd.isCompactionEnabled());
331    assertEquals(3, currentHtd.getColumnFamilyCount());
332    assertTrue(currentHtd.hasColumnFamily(Bytes.toBytes(cf2)));
333    assertTrue(currentHtd.hasColumnFamily(Bytes.toBytes(cf3)));
334
335    // cf2 should be added
336    MasterProcedureTestingUtility.validateTableCreation(UTIL.getHBaseCluster().getMaster(),
337      tableName, regions, "cf1", cf2, cf3);
338  }
339
340  @Test
341  public void testRollbackAndDoubleExecutionOnline() throws Exception {
342    final TableName tableName = TableName.valueOf(name.getMethodName());
343    final String familyName = "cf2";
344    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
345
346    // create the table
347    RegionInfo[] regions = MasterProcedureTestingUtility.createTable(
348      procExec, tableName, null, "cf1");
349
350    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
351
352    TableDescriptor td = UTIL.getAdmin().getDescriptor(tableName);
353    TableDescriptor newTd =
354      TableDescriptorBuilder.newBuilder(td).setCompactionEnabled(!td.isCompactionEnabled())
355        .setColumnFamily(ColumnFamilyDescriptorBuilder.of(familyName)).build();
356
357    // Start the Modify procedure && kill the executor
358    long procId =
359      procExec.submitProcedure(new ModifyTableProcedure(procExec.getEnvironment(), newTd));
360
361    int lastStep = 3; // failing before MODIFY_TABLE_UPDATE_TABLE_DESCRIPTOR
362    MasterProcedureTestingUtility.testRollbackAndDoubleExecution(procExec, procId, lastStep);
363
364    // cf2 should not be present
365    MasterProcedureTestingUtility.validateTableCreation(UTIL.getHBaseCluster().getMaster(),
366      tableName, regions, "cf1");
367  }
368
369  @Test
370  public void testRollbackAndDoubleExecutionOffline() throws Exception {
371    final TableName tableName = TableName.valueOf(name.getMethodName());
372    final String familyName = "cf2";
373    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
374
375    // create the table
376    RegionInfo[] regions = MasterProcedureTestingUtility.createTable(
377      procExec, tableName, null, "cf1");
378    UTIL.getAdmin().disableTable(tableName);
379
380    ProcedureTestingUtility.waitNoProcedureRunning(procExec);
381    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
382
383    TableDescriptor td = UTIL.getAdmin().getDescriptor(tableName);
384    TableDescriptor newTd =
385      TableDescriptorBuilder.newBuilder(td).setCompactionEnabled(!td.isCompactionEnabled())
386        .setColumnFamily(ColumnFamilyDescriptorBuilder.of(familyName)).setRegionReplication(3)
387        .build();
388
389    // Start the Modify procedure && kill the executor
390    long procId = procExec.submitProcedure(
391      new ModifyTableProcedure(procExec.getEnvironment(), newTd));
392
393    // Restart the executor and rollback the step twice
394    int lastStep = 3; // failing before MODIFY_TABLE_UPDATE_TABLE_DESCRIPTOR
395    MasterProcedureTestingUtility.testRollbackAndDoubleExecution(procExec, procId, lastStep);
396
397    // cf2 should not be present
398    MasterProcedureTestingUtility.validateTableCreation(UTIL.getHBaseCluster().getMaster(),
399      tableName, regions, "cf1");
400  }
401}