001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019package org.apache.hadoop.hbase;
020
021import java.io.IOException;
022import java.util.ArrayList;
023import java.util.List;
024import java.util.Set;
025import java.util.concurrent.ConcurrentHashMap;
026import java.util.concurrent.atomic.AtomicBoolean;
027import java.util.regex.Pattern;
028import org.apache.commons.lang3.RandomStringUtils;
029import org.apache.commons.lang3.RandomUtils;
030import org.apache.hadoop.conf.Configuration;
031import org.apache.hadoop.hbase.client.Admin;
032import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
033import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
034import org.apache.hadoop.hbase.client.Connection;
035import org.apache.hadoop.hbase.client.ConnectionFactory;
036import org.apache.hadoop.hbase.client.Put;
037import org.apache.hadoop.hbase.client.RegionInfo;
038import org.apache.hadoop.hbase.client.Table;
039import org.apache.hadoop.hbase.client.TableDescriptor;
040import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
041import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
042import org.apache.hadoop.hbase.log.HBaseMarkers;
043import org.apache.hadoop.hbase.testclassification.IntegrationTests;
044import org.apache.hadoop.hbase.util.Bytes;
045import org.apache.hadoop.hbase.util.HBaseFsck;
046import org.apache.hadoop.hbase.util.Threads;
047import org.apache.hadoop.hbase.util.hbck.HbckTestingUtil;
048import org.apache.hadoop.util.ToolRunner;
049import org.junit.Assert;
050import org.junit.Test;
051import org.junit.experimental.categories.Category;
052import org.slf4j.Logger;
053import org.slf4j.LoggerFactory;
054
055/**
056 *
057 * Integration test that verifies Procedure V2.
058 *
059 * DDL operations should go through (rollforward or rollback) when primary master is killed by
060 * ChaosMonkey (default MASTER_KILLING).
061 *
062 * <p></p>Multiple Worker threads are started to randomly do the following Actions in loops:
063 * Actions generating and populating tables:
064 * <ul>
065 *     <li>CreateTableAction</li>
066 *     <li>DisableTableAction</li>
067 *     <li>EnableTableAction</li>
068 *     <li>DeleteTableAction</li>
069 *     <li>AddRowAction</li>
070 * </ul>
071 * Actions performing column family DDL operations:
072 * <ul>
073 *     <li>AddColumnFamilyAction</li>
074 *     <li>AlterColumnFamilyVersionsAction</li>
075 *     <li>AlterColumnFamilyEncodingAction</li>
076 *     <li>DeleteColumnFamilyAction</li>
077 * </ul>
078 * Actions performing namespace DDL operations:
079 * <ul>
080 *     <li>AddNamespaceAction</li>
081 *     <li>AlterNamespaceAction</li>
082 *     <li>DeleteNamespaceAction</li>
083 * </ul>
084 * <br/>
085 *
086 * The threads run for a period of time (default 20 minutes) then are stopped at the end of
087 * runtime. Verification is performed towards those checkpoints:
088 * <ol>
089 *     <li>No Actions throw Exceptions.</li>
090 *     <li>No inconsistencies are detected in hbck.</li>
091 * </ol>
092 *
093 * <p>
094 * This test should be run by the hbase user since it invokes hbck at the end
095 * </p><p>
096 * Usage:
097 *  hbase org.apache.hadoop.hbase.IntegrationTestDDLMasterFailover
098 *    -Dhbase.IntegrationTestDDLMasterFailover.runtime=1200000
099 *    -Dhbase.IntegrationTestDDLMasterFailover.numThreads=20
100 *    -Dhbase.IntegrationTestDDLMasterFailover.numRegions=50 --monkey masterKilling
101 */
102
103@Category(IntegrationTests.class)
104public class IntegrationTestDDLMasterFailover extends IntegrationTestBase {
105
106  private static final Logger LOG = LoggerFactory.getLogger(IntegrationTestDDLMasterFailover.class);
107
108  private static final int SERVER_COUNT = 1; // number of slaves for the smallest cluster
109
110  protected static final long DEFAULT_RUN_TIME = 20 * 60 * 1000;
111
112  protected static final int DEFAULT_NUM_THREADS = 20;
113
114  protected static final int DEFAULT_NUM_REGIONS = 50; // number of regions in pre-split tables
115
116  private boolean keepObjectsAtTheEnd = false;
117  protected HBaseClusterInterface cluster;
118
119  protected Connection connection;
120
121  /**
122   * A soft limit on how long we should run
123   */
124  protected static final String RUN_TIME_KEY = "hbase.%s.runtime";
125  protected static final String NUM_THREADS_KEY = "hbase.%s.numThreads";
126  protected static final String NUM_REGIONS_KEY = "hbase.%s.numRegions";
127
128  protected AtomicBoolean running = new AtomicBoolean(true);
129
130  protected AtomicBoolean create_table = new AtomicBoolean(true);
131
132  protected int numThreads, numRegions;
133
134  ConcurrentHashMap<String, NamespaceDescriptor> namespaceMap = new ConcurrentHashMap<>();
135
136  ConcurrentHashMap<TableName, TableDescriptor> enabledTables = new ConcurrentHashMap<>();
137
138  ConcurrentHashMap<TableName, TableDescriptor> disabledTables = new ConcurrentHashMap<>();
139
140  ConcurrentHashMap<TableName, TableDescriptor> deletedTables = new ConcurrentHashMap<>();
141
142  @Override
143  public void setUpCluster() throws Exception {
144    util = getTestingUtil(getConf());
145    LOG.debug("Initializing/checking cluster has " + SERVER_COUNT + " servers");
146    util.initializeCluster(getMinServerCount());
147    LOG.debug("Done initializing/checking cluster");
148    cluster = util.getHBaseClusterInterface();
149  }
150
151  @Override
152  public void cleanUpCluster() throws Exception {
153    if (!keepObjectsAtTheEnd) {
154      Admin admin = util.getAdmin();
155      for (TableName tableName: admin.listTableNames(Pattern.compile("ittable-\\d+"))) {
156        admin.disableTable(tableName);
157        admin.deleteTable(tableName);
158      }
159      NamespaceDescriptor [] nsds = admin.listNamespaceDescriptors();
160      for(NamespaceDescriptor nsd: nsds) {
161        if(nsd.getName().matches("itnamespace\\d+")) {
162          LOG.info("Removing namespace="+nsd.getName());
163          admin.deleteNamespace(nsd.getName());
164        }
165      }
166    }
167
168    enabledTables.clear();
169    disabledTables.clear();
170    deletedTables.clear();
171    namespaceMap.clear();
172
173    Connection connection = getConnection();
174    connection.close();
175    super.cleanUpCluster();
176  }
177
178  protected int getMinServerCount() {
179    return SERVER_COUNT;
180  }
181
182  protected synchronized void setConnection(Connection connection){
183    this.connection = connection;
184  }
185
186  protected synchronized Connection getConnection(){
187    if (this.connection == null) {
188      try {
189        Connection connection = ConnectionFactory.createConnection(getConf());
190        setConnection(connection);
191      } catch (IOException e) {
192        LOG.error(HBaseMarkers.FATAL, "Failed to establish connection.", e);
193      }
194    }
195    return connection;
196  }
197
198  protected void verifyNamespaces() throws  IOException{
199    Connection connection = getConnection();
200    Admin admin = connection.getAdmin();
201    // iterating concurrent map
202    for (String nsName : namespaceMap.keySet()){
203      try {
204        Assert.assertTrue(
205          "Namespace: " + nsName + " in namespaceMap does not exist",
206          admin.getNamespaceDescriptor(nsName) != null);
207      } catch (NamespaceNotFoundException nsnfe) {
208        Assert.fail(
209          "Namespace: " + nsName + " in namespaceMap does not exist: " + nsnfe.getMessage());
210      }
211    }
212    admin.close();
213  }
214
215  protected void verifyTables() throws  IOException{
216    Connection connection = getConnection();
217    Admin admin = connection.getAdmin();
218    // iterating concurrent map
219    for (TableName tableName : enabledTables.keySet()){
220      Assert.assertTrue("Table: " + tableName + " in enabledTables is not enabled",
221          admin.isTableEnabled(tableName));
222    }
223    for (TableName tableName : disabledTables.keySet()){
224      Assert.assertTrue("Table: " + tableName + " in disabledTables is not disabled",
225          admin.isTableDisabled(tableName));
226    }
227    for (TableName tableName : deletedTables.keySet()){
228      Assert.assertFalse("Table: " + tableName + " in deletedTables is not deleted",
229          admin.tableExists(tableName));
230    }
231    admin.close();
232  }
233
234  @Test
235  public void testAsUnitTest() throws Exception {
236    runTest();
237  }
238
239  @Override
240  public int runTestFromCommandLine() throws Exception {
241    int ret = runTest();
242    return ret;
243  }
244
245  private abstract class MasterAction{
246    Connection connection = getConnection();
247
248    abstract void perform() throws IOException;
249  }
250
251  private abstract class NamespaceAction extends MasterAction {
252    final String nsTestConfigKey = "hbase.namespace.testKey";
253
254    // NamespaceAction has implemented selectNamespace() shared by multiple namespace Actions
255    protected NamespaceDescriptor selectNamespace(
256        ConcurrentHashMap<String, NamespaceDescriptor> namespaceMap) {
257      // synchronization to prevent removal from multiple threads
258      synchronized (namespaceMap) {
259        // randomly select namespace from namespaceMap
260        if (namespaceMap.isEmpty()) {
261          return null;
262        }
263        ArrayList<String> namespaceList = new ArrayList<>(namespaceMap.keySet());
264        String randomKey = namespaceList.get(RandomUtils.nextInt(0, namespaceList.size()));
265        NamespaceDescriptor randomNsd = namespaceMap.get(randomKey);
266        // remove from namespaceMap
267        namespaceMap.remove(randomKey);
268        return randomNsd;
269      }
270    }
271  }
272
273  private class CreateNamespaceAction extends NamespaceAction {
274    @Override
275    void perform() throws IOException {
276      Admin admin = connection.getAdmin();
277      try {
278        NamespaceDescriptor nsd;
279        while (true) {
280          nsd = createNamespaceDesc();
281          try {
282            if (admin.getNamespaceDescriptor(nsd.getName()) != null) {
283              // the namespace has already existed.
284              continue;
285            } else {
286              // currently, the code never return null - always throws exception if
287              // namespace is not found - this just a defensive programming to make
288              // sure null situation is handled in case the method changes in the
289              // future.
290              break;
291            }
292          } catch (NamespaceNotFoundException nsnfe) {
293            // This is expected for a random generated NamespaceDescriptor
294            break;
295          }
296        }
297        LOG.info("Creating namespace:" + nsd);
298        admin.createNamespace(nsd);
299        NamespaceDescriptor freshNamespaceDesc = admin.getNamespaceDescriptor(nsd.getName());
300        Assert.assertTrue("Namespace: " + nsd + " was not created", freshNamespaceDesc != null);
301        namespaceMap.put(nsd.getName(), freshNamespaceDesc);
302        LOG.info("Created namespace:" + freshNamespaceDesc);
303      } catch (Exception e){
304        LOG.warn("Caught exception in action: " + this.getClass());
305        throw e;
306      } finally {
307        admin.close();
308      }
309    }
310
311    private NamespaceDescriptor createNamespaceDesc() {
312      String namespaceName = "itnamespace" + String.format("%010d",
313        RandomUtils.nextInt());
314      NamespaceDescriptor nsd = NamespaceDescriptor.create(namespaceName).build();
315
316      nsd.setConfiguration(
317        nsTestConfigKey,
318        String.format("%010d", RandomUtils.nextInt()));
319      return nsd;
320    }
321  }
322
323  private class ModifyNamespaceAction extends NamespaceAction {
324    @Override
325    void perform() throws IOException {
326      NamespaceDescriptor selected = selectNamespace(namespaceMap);
327      if (selected == null) {
328        return;
329      }
330
331      Admin admin = connection.getAdmin();
332      try {
333        String namespaceName = selected.getName();
334        LOG.info("Modifying namespace :" + selected);
335        NamespaceDescriptor modifiedNsd = NamespaceDescriptor.create(namespaceName).build();
336        String nsValueNew;
337        do {
338          nsValueNew = String.format("%010d", RandomUtils.nextInt());
339        } while (selected.getConfigurationValue(nsTestConfigKey).equals(nsValueNew));
340        modifiedNsd.setConfiguration(nsTestConfigKey, nsValueNew);
341        admin.modifyNamespace(modifiedNsd);
342        NamespaceDescriptor freshNamespaceDesc = admin.getNamespaceDescriptor(namespaceName);
343        Assert.assertTrue(
344          "Namespace: " + selected + " was not modified",
345          freshNamespaceDesc.getConfigurationValue(nsTestConfigKey).equals(nsValueNew));
346        Assert.assertTrue(
347          "Namespace: " + namespaceName + " does not exist",
348          admin.getNamespaceDescriptor(namespaceName) != null);
349        namespaceMap.put(namespaceName, freshNamespaceDesc);
350        LOG.info("Modified namespace :" + freshNamespaceDesc);
351      } catch (Exception e){
352        LOG.warn("Caught exception in action: " + this.getClass());
353        throw e;
354      } finally {
355        admin.close();
356      }
357    }
358  }
359
360  private class DeleteNamespaceAction extends NamespaceAction {
361    @Override
362    void perform() throws IOException {
363      NamespaceDescriptor selected = selectNamespace(namespaceMap);
364      if (selected == null) {
365        return;
366      }
367
368      Admin admin = connection.getAdmin();
369      try {
370        String namespaceName = selected.getName();
371        LOG.info("Deleting namespace :" + selected);
372        admin.deleteNamespace(namespaceName);
373        try {
374          if (admin.getNamespaceDescriptor(namespaceName) != null) {
375            // the namespace still exists.
376            Assert.assertTrue("Namespace: " + selected + " was not deleted", false);
377          } else {
378            LOG.info("Deleted namespace :" + selected);
379          }
380        } catch (NamespaceNotFoundException nsnfe) {
381          // This is expected result
382          LOG.info("Deleted namespace :" + selected);
383        }
384      } catch (Exception e){
385        LOG.warn("Caught exception in action: " + this.getClass());
386        throw e;
387      } finally {
388        admin.close();
389      }
390    }
391  }
392
393  private abstract class TableAction extends  MasterAction{
394    // TableAction has implemented selectTable() shared by multiple table Actions
395    protected TableDescriptor selectTable(ConcurrentHashMap<TableName, TableDescriptor> tableMap)
396    {
397      // synchronization to prevent removal from multiple threads
398      synchronized (tableMap){
399        // randomly select table from tableMap
400        if (tableMap.isEmpty()) {
401          return null;
402        }
403        ArrayList<TableName> tableList = new ArrayList<>(tableMap.keySet());
404        TableName randomKey = tableList.get(RandomUtils.nextInt(0, tableList.size()));
405        TableDescriptor randomTd = tableMap.remove(randomKey);
406        return randomTd;
407      }
408    }
409  }
410
411  private class CreateTableAction extends TableAction {
412
413    @Override
414    void perform() throws IOException {
415      Admin admin = connection.getAdmin();
416      try {
417        TableDescriptor td = createTableDesc();
418        TableName tableName = td.getTableName();
419        if ( admin.tableExists(tableName)){
420          return;
421        }
422        String numRegionKey = String.format(NUM_REGIONS_KEY, this.getClass().getSimpleName());
423        numRegions = getConf().getInt(numRegionKey, DEFAULT_NUM_REGIONS);
424        byte[] startKey = Bytes.toBytes("row-0000000000");
425        byte[] endKey = Bytes.toBytes("row-" + Integer.MAX_VALUE);
426        LOG.info("Creating table:" + td);
427        admin.createTable(td, startKey, endKey, numRegions);
428        Assert.assertTrue("Table: " + td + " was not created", admin.tableExists(tableName));
429        TableDescriptor freshTableDesc = admin.getDescriptor(tableName);
430        Assert.assertTrue(
431          "After create, Table: " + tableName + " in not enabled", admin.isTableEnabled(tableName));
432        enabledTables.put(tableName, freshTableDesc);
433        LOG.info("Created table:" + freshTableDesc);
434      } catch (Exception e) {
435        LOG.warn("Caught exception in action: " + this.getClass());
436        throw e;
437      } finally {
438        admin.close();
439      }
440    }
441
442    private TableDescriptor createTableDesc() {
443      String tableName = String.format("ittable-%010d", RandomUtils.nextInt());
444      String familyName = "cf-" + Math.abs(RandomUtils.nextInt());
445      return TableDescriptorBuilder.newBuilder(TableName.valueOf(tableName))
446          .setColumnFamily(ColumnFamilyDescriptorBuilder.of(familyName))
447          .build();
448    }
449  }
450
451  private class DisableTableAction extends TableAction {
452
453    @Override
454    void perform() throws IOException {
455
456      TableDescriptor selected = selectTable(enabledTables);
457      if (selected == null) {
458        return;
459      }
460
461      Admin admin = connection.getAdmin();
462      try {
463        TableName tableName = selected.getTableName();
464        LOG.info("Disabling table :" + selected);
465        admin.disableTable(tableName);
466        Assert.assertTrue("Table: " + selected + " was not disabled",
467            admin.isTableDisabled(tableName));
468        TableDescriptor freshTableDesc = admin.getDescriptor(tableName);
469        Assert.assertTrue(
470          "After disable, Table: " + tableName + " is not disabled",
471          admin.isTableDisabled(tableName));
472        disabledTables.put(tableName, freshTableDesc);
473        LOG.info("Disabled table :" + freshTableDesc);
474      } catch (Exception e){
475        LOG.warn("Caught exception in action: " + this.getClass());
476        // TODO workaround
477        // loose restriction for TableNotDisabledException/TableNotEnabledException thrown in sync
478        // operations
479        // 1) when enable/disable starts, the table state is changed to ENABLING/DISABLING (ZK node
480        // in 1.x), which will be further changed to ENABLED/DISABLED once the operation completes
481        // 2) if master failover happens in the middle of the enable/disable operation, the new
482        // master will try to recover the tables in ENABLING/DISABLING state, as programmed in
483        // AssignmentManager#recoverTableInEnablingState() and
484        // AssignmentManager#recoverTableInDisablingState()
485        // 3) after the new master initialization completes, the procedure tries to re-do the
486        // enable/disable operation, which was already done. Ignore those exceptions before change
487        // of behaviors of AssignmentManager in presence of PV2
488        if (e instanceof TableNotEnabledException) {
489          LOG.warn("Caught TableNotEnabledException in action: " + this.getClass());
490          e.printStackTrace();
491        } else {
492          throw e;
493        }
494      } finally {
495        admin.close();
496      }
497    }
498  }
499
500  private class EnableTableAction extends TableAction {
501
502    @Override
503    void perform() throws IOException {
504
505      TableDescriptor selected = selectTable(disabledTables);
506      if (selected == null ) {
507        return;
508      }
509
510      Admin admin = connection.getAdmin();
511      try {
512        TableName tableName = selected.getTableName();
513        LOG.info("Enabling table :" + selected);
514        admin.enableTable(tableName);
515        Assert.assertTrue("Table: " + selected + " was not enabled",
516            admin.isTableEnabled(tableName));
517        TableDescriptor freshTableDesc = admin.getDescriptor(tableName);
518        Assert.assertTrue(
519          "After enable, Table: " + tableName + " in not enabled", admin.isTableEnabled(tableName));
520        enabledTables.put(tableName, freshTableDesc);
521        LOG.info("Enabled table :" + freshTableDesc);
522      } catch (Exception e){
523        LOG.warn("Caught exception in action: " + this.getClass());
524        // TODO workaround
525        // loose restriction for TableNotDisabledException/TableNotEnabledException thrown in sync
526        // operations 1) when enable/disable starts, the table state is changed to
527        // ENABLING/DISABLING (ZK node in 1.x), which will be further changed to ENABLED/DISABLED
528        // once the operation completes 2) if master failover happens in the middle of the
529        // enable/disable operation, the new master will try to recover the tables in
530        // ENABLING/DISABLING state, as programmed in
531        // AssignmentManager#recoverTableInEnablingState() and
532        // AssignmentManager#recoverTableInDisablingState()
533        // 3) after the new master initialization completes, the procedure tries to re-do the
534        // enable/disable operation, which was already done. Ignore those exceptions before
535        // change of behaviors of AssignmentManager in presence of PV2
536        if (e instanceof TableNotDisabledException) {
537          LOG.warn("Caught TableNotDisabledException in action: " + this.getClass());
538          e.printStackTrace();
539        } else {
540          throw e;
541        }
542      } finally {
543        admin.close();
544      }
545    }
546  }
547
548  private class DeleteTableAction extends TableAction {
549
550    @Override
551    void perform() throws IOException {
552
553      TableDescriptor selected = selectTable(disabledTables);
554      if (selected == null) {
555        return;
556      }
557
558      Admin admin = connection.getAdmin();
559      try {
560        TableName tableName = selected.getTableName();
561        LOG.info("Deleting table :" + selected);
562        admin.deleteTable(tableName);
563        Assert.assertFalse("Table: " + selected + " was not deleted",
564                admin.tableExists(tableName));
565        deletedTables.put(tableName, selected);
566        LOG.info("Deleted table :" + selected);
567      } catch (Exception e) {
568        LOG.warn("Caught exception in action: " + this.getClass());
569        throw e;
570      } finally {
571        admin.close();
572      }
573    }
574  }
575
576
577  private abstract class ColumnAction extends TableAction{
578    // ColumnAction has implemented selectFamily() shared by multiple family Actions
579    protected ColumnFamilyDescriptor selectFamily(TableDescriptor td) {
580      if (td == null) {
581        return null;
582      }
583      ColumnFamilyDescriptor[] families = td.getColumnFamilies();
584      if (families.length == 0){
585        LOG.info("No column families in table: " + td);
586        return null;
587      }
588      ColumnFamilyDescriptor randomCfd = families[RandomUtils.nextInt(0, families.length)];
589      return randomCfd;
590    }
591  }
592
593  private class AddColumnFamilyAction extends ColumnAction {
594
595    @Override
596    void perform() throws IOException {
597      TableDescriptor selected = selectTable(disabledTables);
598      if (selected == null) {
599        return;
600      }
601
602      Admin admin = connection.getAdmin();
603      try {
604        ColumnFamilyDescriptor cfd = createFamilyDesc();
605        if (selected.hasColumnFamily(cfd.getName())){
606          LOG.info(new String(cfd.getName()) + " already exists in table "
607              + selected.getTableName());
608          return;
609        }
610        TableName tableName = selected.getTableName();
611        LOG.info("Adding column family: " + cfd + " to table: " + tableName);
612        admin.addColumnFamily(tableName, cfd);
613        // assertion
614        TableDescriptor freshTableDesc = admin.getDescriptor(tableName);
615        Assert.assertTrue("Column family: " + cfd + " was not added",
616            freshTableDesc.hasColumnFamily(cfd.getName()));
617        Assert.assertTrue(
618          "After add column family, Table: " + tableName + " is not disabled",
619          admin.isTableDisabled(tableName));
620        disabledTables.put(tableName, freshTableDesc);
621        LOG.info("Added column family: " + cfd + " to table: " + tableName);
622      } catch (Exception e) {
623        LOG.warn("Caught exception in action: " + this.getClass());
624        throw e;
625      } finally {
626        admin.close();
627      }
628    }
629
630    private ColumnFamilyDescriptor createFamilyDesc() {
631      String familyName = String.format("cf-%010d", RandomUtils.nextInt());
632      return ColumnFamilyDescriptorBuilder.of(familyName);
633    }
634  }
635
636  private class AlterFamilyVersionsAction extends ColumnAction {
637
638    @Override
639    void perform() throws IOException {
640      TableDescriptor selected = selectTable(disabledTables);
641      if (selected == null) {
642        return;
643      }
644      ColumnFamilyDescriptor columnDesc = selectFamily(selected);
645      if (columnDesc == null){
646        return;
647      }
648
649      Admin admin = connection.getAdmin();
650      int versions = RandomUtils.nextInt(0, 10) + 3;
651      try {
652        TableName tableName = selected.getTableName();
653        LOG.info("Altering versions of column family: " + columnDesc + " to: " + versions +
654            " in table: " + tableName);
655
656        ColumnFamilyDescriptor cfd = ColumnFamilyDescriptorBuilder.newBuilder(columnDesc)
657            .setMinVersions(versions)
658            .setMaxVersions(versions)
659            .build();
660        TableDescriptor td = TableDescriptorBuilder.newBuilder(selected)
661            .modifyColumnFamily(cfd)
662            .build();
663        admin.modifyTable(td);
664
665        // assertion
666        TableDescriptor freshTableDesc = admin.getDescriptor(tableName);
667        ColumnFamilyDescriptor freshColumnDesc = freshTableDesc.getColumnFamily(columnDesc.getName());
668        Assert.assertEquals("Column family: " + columnDesc + " was not altered",
669            freshColumnDesc.getMaxVersions(), versions);
670        Assert.assertEquals("Column family: " + freshColumnDesc + " was not altered",
671            freshColumnDesc.getMinVersions(), versions);
672        Assert.assertTrue(
673          "After alter versions of column family, Table: " + tableName + " is not disabled",
674          admin.isTableDisabled(tableName));
675        disabledTables.put(tableName, freshTableDesc);
676        LOG.info("Altered versions of column family: " + columnDesc + " to: " + versions +
677          " in table: " + tableName);
678      } catch (Exception e) {
679        LOG.warn("Caught exception in action: " + this.getClass());
680        throw e;
681      } finally {
682        admin.close();
683      }
684    }
685  }
686
687  private class AlterFamilyEncodingAction extends ColumnAction {
688
689    @Override
690    void perform() throws IOException {
691      TableDescriptor selected = selectTable(disabledTables);
692      if (selected == null) {
693        return;
694      }
695      ColumnFamilyDescriptor columnDesc = selectFamily(selected);
696      if (columnDesc == null){
697        return;
698      }
699
700      Admin admin = connection.getAdmin();
701      try {
702        TableName tableName = selected.getTableName();
703        // possible DataBlockEncoding ids
704        DataBlockEncoding[] possibleIds = {DataBlockEncoding.NONE, DataBlockEncoding.PREFIX,
705                DataBlockEncoding.DIFF, DataBlockEncoding.FAST_DIFF, DataBlockEncoding.ROW_INDEX_V1};
706        short id = possibleIds[RandomUtils.nextInt(0, possibleIds.length)].getId();
707        LOG.info("Altering encoding of column family: " + columnDesc + " to: " + id +
708            " in table: " + tableName);
709
710        ColumnFamilyDescriptor cfd = ColumnFamilyDescriptorBuilder.newBuilder(columnDesc)
711            .setDataBlockEncoding(DataBlockEncoding.getEncodingById(id))
712            .build();
713        TableDescriptor td = TableDescriptorBuilder.newBuilder(selected)
714            .modifyColumnFamily(cfd)
715            .build();
716        admin.modifyTable(td);
717
718        // assertion
719        TableDescriptor freshTableDesc = admin.getDescriptor(tableName);
720        ColumnFamilyDescriptor freshColumnDesc = freshTableDesc.getColumnFamily(columnDesc.getName());
721        Assert.assertEquals("Encoding of column family: " + columnDesc + " was not altered",
722            freshColumnDesc.getDataBlockEncoding().getId(), id);
723        Assert.assertTrue(
724          "After alter encoding of column family, Table: " + tableName + " is not disabled",
725          admin.isTableDisabled(tableName));
726        disabledTables.put(tableName, freshTableDesc);
727        LOG.info("Altered encoding of column family: " + freshColumnDesc + " to: " + id +
728          " in table: " + tableName);
729      } catch (Exception e) {
730        LOG.warn("Caught exception in action: " + this.getClass());
731        throw e;
732      } finally {
733        admin.close();
734      }
735    }
736  }
737
738  private class DeleteColumnFamilyAction extends ColumnAction {
739
740    @Override
741    void perform() throws IOException {
742      TableDescriptor selected = selectTable(disabledTables);
743      ColumnFamilyDescriptor cfd = selectFamily(selected);
744      if (selected == null || cfd == null) {
745        return;
746      }
747
748      Admin admin = connection.getAdmin();
749      try {
750        if (selected.getColumnFamilyCount() < 2) {
751          LOG.info("No enough column families to delete in table " + selected.getTableName());
752          return;
753        }
754        TableName tableName = selected.getTableName();
755        LOG.info("Deleting column family: " + cfd + " from table: " + tableName);
756        admin.deleteColumnFamily(tableName, cfd.getName());
757        // assertion
758        TableDescriptor freshTableDesc = admin.getDescriptor(tableName);
759        Assert.assertFalse("Column family: " + cfd + " was not added",
760            freshTableDesc.hasColumnFamily(cfd.getName()));
761        Assert.assertTrue(
762          "After delete column family, Table: " + tableName + " is not disabled",
763          admin.isTableDisabled(tableName));
764        disabledTables.put(tableName, freshTableDesc);
765        LOG.info("Deleted column family: " + cfd + " from table: " + tableName);
766      } catch (Exception e) {
767        LOG.warn("Caught exception in action: " + this.getClass());
768        throw e;
769      } finally {
770        admin.close();
771      }
772    }
773  }
774
775  private class AddRowAction extends ColumnAction {
776    // populate tables
777    @Override
778    void perform() throws IOException {
779      TableDescriptor selected = selectTable(enabledTables);
780      if (selected == null ) {
781        return;
782      }
783
784      Admin admin = connection.getAdmin();
785      TableName tableName = selected.getTableName();
786      try (Table table = connection.getTable(tableName)){
787        ArrayList<RegionInfo> regionInfos = new ArrayList<>(admin.getRegions(
788            selected.getTableName()));
789        int numRegions = regionInfos.size();
790        // average number of rows to be added per action to each region
791        int average_rows = 1;
792        int numRows = average_rows * numRegions;
793        LOG.info("Adding " + numRows + " rows to table: " + selected);
794        for (int i = 0; i < numRows; i++){
795          // nextInt(Integer.MAX_VALUE)) to return positive numbers only
796          byte[] rowKey = Bytes.toBytes(
797              "row-" + String.format("%010d", RandomUtils.nextInt()));
798          ColumnFamilyDescriptor cfd = selectFamily(selected);
799          if (cfd == null){
800            return;
801          }
802          byte[] family = cfd.getName();
803          byte[] qualifier = Bytes.toBytes("col-" + RandomUtils.nextInt() % 10);
804          byte[] value = Bytes.toBytes("val-" + RandomStringUtils.randomAlphanumeric(10));
805          Put put = new Put(rowKey);
806          put.addColumn(family, qualifier, value);
807          table.put(put);
808        }
809        TableDescriptor freshTableDesc = admin.getDescriptor(tableName);
810        Assert.assertTrue(
811          "After insert, Table: " + tableName + " in not enabled", admin.isTableEnabled(tableName));
812        enabledTables.put(tableName, freshTableDesc);
813        LOG.info("Added " + numRows + " rows to table: " + selected);
814      } catch (Exception e) {
815        LOG.warn("Caught exception in action: " + this.getClass());
816        throw e;
817      } finally {
818        admin.close();
819      }
820    }
821  }
822
823  private enum ACTION {
824    CREATE_NAMESPACE,
825    MODIFY_NAMESPACE,
826    DELETE_NAMESPACE,
827    CREATE_TABLE,
828    DISABLE_TABLE,
829    ENABLE_TABLE,
830    DELETE_TABLE,
831    ADD_COLUMNFAMILY,
832    DELETE_COLUMNFAMILY,
833    ALTER_FAMILYVERSIONS,
834    ALTER_FAMILYENCODING,
835    ADD_ROW
836  }
837
838  private class Worker extends Thread {
839
840    private Exception savedException;
841
842    private ACTION action;
843
844    @Override
845    public void run() {
846      while (running.get()) {
847        // select random action
848        ACTION selectedAction = ACTION.values()[RandomUtils.nextInt() % ACTION.values().length];
849        this.action = selectedAction;
850        LOG.info("Performing Action: " + selectedAction);
851
852        try {
853          switch (selectedAction) {
854          case CREATE_NAMESPACE:
855            new CreateNamespaceAction().perform();
856            break;
857          case MODIFY_NAMESPACE:
858            new ModifyNamespaceAction().perform();
859            break;
860          case DELETE_NAMESPACE:
861            new DeleteNamespaceAction().perform();
862            break;
863          case CREATE_TABLE:
864            // stop creating new tables in the later stage of the test to avoid too many empty
865            // tables
866            if (create_table.get()) {
867              new CreateTableAction().perform();
868            }
869            break;
870          case ADD_ROW:
871            new AddRowAction().perform();
872            break;
873          case DISABLE_TABLE:
874            new DisableTableAction().perform();
875            break;
876          case ENABLE_TABLE:
877            new EnableTableAction().perform();
878            break;
879          case DELETE_TABLE:
880            // reduce probability of deleting table to 20%
881            if (RandomUtils.nextInt(0, 100) < 20) {
882              new DeleteTableAction().perform();
883            }
884            break;
885          case ADD_COLUMNFAMILY:
886            new AddColumnFamilyAction().perform();
887            break;
888          case DELETE_COLUMNFAMILY:
889            // reduce probability of deleting column family to 20%
890            if (RandomUtils.nextInt(0, 100) < 20) {
891              new DeleteColumnFamilyAction().perform();
892            }
893            break;
894          case ALTER_FAMILYVERSIONS:
895            new AlterFamilyVersionsAction().perform();
896            break;
897          case ALTER_FAMILYENCODING:
898            new AlterFamilyEncodingAction().perform();
899            break;
900          }
901        } catch (Exception ex) {
902          this.savedException = ex;
903          return;
904        }
905      }
906      LOG.info(this.getName() + " stopped");
907    }
908
909    public Exception getSavedException(){
910      return this.savedException;
911    }
912
913    public ACTION getAction(){
914      return this.action;
915    }
916  }
917
918  private void checkException(List<Worker> workers){
919    if(workers == null || workers.isEmpty())
920      return;
921    for (Worker worker : workers){
922      Exception e = worker.getSavedException();
923      if (e != null) {
924        LOG.error("Found exception in thread: " + worker.getName());
925        e.printStackTrace();
926      }
927      Assert.assertNull("Action failed: " + worker.getAction() + " in thread: "
928          + worker.getName(), e);
929    }
930  }
931
932  private int runTest() throws Exception {
933    LOG.info("Starting the test");
934
935    String runtimeKey = String.format(RUN_TIME_KEY, this.getClass().getSimpleName());
936    long runtime = util.getConfiguration().getLong(runtimeKey, DEFAULT_RUN_TIME);
937
938    String numThreadKey = String.format(NUM_THREADS_KEY, this.getClass().getSimpleName());
939    numThreads = util.getConfiguration().getInt(numThreadKey, DEFAULT_NUM_THREADS);
940
941    ArrayList<Worker> workers = new ArrayList<>(numThreads);
942    for (int i = 0; i < numThreads; i++) {
943      checkException(workers);
944      Worker worker = new Worker();
945      LOG.info("Launching worker thread " + worker.getName());
946      workers.add(worker);
947      worker.start();
948    }
949
950    Threads.sleep(runtime / 2);
951    LOG.info("Stopping creating new tables");
952    create_table.set(false);
953    Threads.sleep(runtime / 2);
954    LOG.info("Runtime is up");
955    running.set(false);
956
957    checkException(workers);
958
959    for (Worker worker : workers) {
960      worker.join();
961    }
962    LOG.info("All Worker threads stopped");
963
964    // verify
965    LOG.info("Verify actions of all threads succeeded");
966    checkException(workers);
967    LOG.info("Verify namespaces");
968    verifyNamespaces();
969    LOG.info("Verify states of all tables");
970    verifyTables();
971
972    // RUN HBCK
973
974    HBaseFsck hbck = null;
975    try {
976      LOG.info("Running hbck");
977      hbck = HbckTestingUtil.doFsck(util.getConfiguration(), false);
978      if (HbckTestingUtil.inconsistencyFound(hbck)) {
979        // Find the inconsistency during HBCK. Leave table and namespace undropped so that
980        // we can check outside the test.
981        keepObjectsAtTheEnd = true;
982      }
983      HbckTestingUtil.assertNoErrors(hbck);
984      LOG.info("Finished hbck");
985    } finally {
986      if (hbck != null) {
987        hbck.close();
988      }
989    }
990     return 0;
991  }
992
993  @Override
994  public TableName getTablename() {
995    return null; // This test is not inteded to run with stock Chaos Monkey
996  }
997
998  @Override
999  protected Set<String> getColumnFamilies() {
1000    return null; // This test is not inteded to run with stock Chaos Monkey
1001  }
1002
1003  public static void main(String[] args) throws Exception {
1004    Configuration conf = HBaseConfiguration.create();
1005    IntegrationTestingUtility.setUseDistributedCluster(conf);
1006    IntegrationTestDDLMasterFailover masterFailover = new IntegrationTestDDLMasterFailover();
1007    Connection connection = null;
1008    int ret = 1;
1009    try {
1010      // Initialize connection once, then pass to Actions
1011      LOG.debug("Setting up connection ...");
1012      connection = ConnectionFactory.createConnection(conf);
1013      masterFailover.setConnection(connection);
1014      ret = ToolRunner.run(conf, masterFailover, args);
1015    } catch (IOException e){
1016      LOG.error(HBaseMarkers.FATAL, "Failed to establish connection. Aborting test ...", e);
1017    } finally {
1018      connection = masterFailover.getConnection();
1019      if (connection != null){
1020        connection.close();
1021      }
1022      System.exit(ret);
1023    }
1024  }
1025}