001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.procedure;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertTrue;
022
023import java.io.IOException;
024
025import org.apache.hadoop.conf.Configuration;
026import org.apache.hadoop.hbase.HBaseTestingUtility;
027import org.apache.hadoop.hbase.HConstants;
028import org.apache.hadoop.hbase.MiniHBaseCluster;
029import org.apache.hadoop.hbase.ServerName;
030import org.apache.hadoop.hbase.TableName;
031import org.apache.hadoop.hbase.client.RegionInfo;
032import org.apache.hadoop.hbase.client.RegionReplicaTestHelper;
033import org.apache.hadoop.hbase.client.Table;
034import org.apache.hadoop.hbase.master.HMaster;
035import org.apache.hadoop.hbase.master.assignment.AssignmentTestingUtil;
036import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
037import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
038import org.junit.After;
039import org.junit.Before;
040import org.slf4j.Logger;
041import org.slf4j.LoggerFactory;
042
043public class TestSCPBase {
044  private static final Logger LOG = LoggerFactory.getLogger(TestSCPBase.class);
045  static final int RS_COUNT = 3;
046
047  protected HBaseTestingUtility util;
048
049  protected void setupConf(Configuration conf) {
050    conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1);
051    conf.set("hbase.balancer.tablesOnMaster", "none");
052    conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, RS_COUNT);
053    conf.setInt(HConstants.HBASE_CLIENT_SERVERSIDE_RETRIES_MULTIPLIER, 3);
054    conf.setBoolean("hbase.split.writer.creation.bounded", true);
055    conf.setInt("hbase.regionserver.hlog.splitlog.writer.threads", 8);
056    conf.setBoolean(HConstants.HBASE_SPLIT_WAL_COORDINATED_BY_ZK, true);
057  }
058
059  @Before
060  public void setup() throws Exception {
061    this.util = new HBaseTestingUtility();
062    setupConf(this.util.getConfiguration());
063    startMiniCluster();
064    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(
065      this.util.getHBaseCluster().getMaster().getMasterProcedureExecutor(), false);
066  }
067
068  protected void startMiniCluster() throws Exception {
069    this.util.startMiniCluster(3);
070  }
071
072  @After
073  public void tearDown() throws Exception {
074    MiniHBaseCluster cluster = this.util.getHBaseCluster();
075    HMaster master = cluster == null ? null : cluster.getMaster();
076    if (master != null && master.getMasterProcedureExecutor() != null) {
077      ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(master.getMasterProcedureExecutor(),
078        false);
079    }
080    this.util.shutdownMiniCluster();
081  }
082
083  /**
084   * Run server crash procedure steps twice to test idempotency and that we are persisting all
085   * needed state.
086   */
087  protected void testRecoveryAndDoubleExecution(boolean carryingMeta, boolean doubleExecution)
088      throws Exception {
089    final TableName tableName = TableName.valueOf("testRecoveryAndDoubleExecution-carryingMeta-" +
090      carryingMeta + "-doubleExecution-" + doubleExecution);
091    try (Table t = createTable(tableName)) {
092      // Load the table with a bit of data so some logs to split and some edits in each region.
093      this.util.loadTable(t, HBaseTestingUtility.COLUMNS[0]);
094      final int count = util.countRows(t);
095      assertTrue("expected some rows", count > 0);
096      final String checksum = util.checksumRows(t);
097      // Run the procedure executor outside the master so we can mess with it. Need to disable
098      // Master's running of the server crash processing.
099      final HMaster master = this.util.getHBaseCluster().getMaster();
100      final ProcedureExecutor<MasterProcedureEnv> procExec = master.getMasterProcedureExecutor();
101      // find the first server that match the request and executes the test
102      ServerName rsToKill = null;
103      for (RegionInfo hri : util.getAdmin().getRegions(tableName)) {
104        final ServerName serverName = AssignmentTestingUtil.getServerHoldingRegion(util, hri);
105        if (AssignmentTestingUtil.isServerHoldingMeta(util, serverName) == carryingMeta) {
106          rsToKill = serverName;
107          break;
108        }
109      }
110      // Enable test flags and then queue the crash procedure.
111      ProcedureTestingUtility.waitNoProcedureRunning(procExec);
112      if (doubleExecution) {
113        // For SCP, if you enable this then we will enter an infinite loop, as we will crash between
114        // queue and open for TRSP, and then going back to queue, as we will use the crash rs as the
115        // target server since it is recored in hbase:meta.
116        ProcedureTestingUtility.setKillIfHasParent(procExec, false);
117        ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
118        // kill the RS
119        AssignmentTestingUtil.killRs(util, rsToKill);
120        long procId = getSCPProcId(procExec);
121        // Now run through the procedure twice crashing the executor on each step...
122        MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId);
123      } else {
124        // kill the RS
125        AssignmentTestingUtil.killRs(util, rsToKill);
126        long procId = getSCPProcId(procExec);
127        ProcedureTestingUtility.waitProcedure(procExec, procId);
128      }
129      RegionReplicaTestHelper.assertReplicaDistributed(util, t);
130      assertEquals(count, util.countRows(t));
131      assertEquals(checksum, util.checksumRows(t));
132    }
133  }
134
135  protected long getSCPProcId(ProcedureExecutor<?> procExec) {
136    util.waitFor(30000, () -> !procExec.getProcedures().isEmpty());
137    return procExec.getActiveProcIds().stream().mapToLong(Long::longValue).min().getAsLong();
138  }
139
140  protected Table createTable(final TableName tableName) throws IOException {
141    final Table t = this.util.createTable(tableName, HBaseTestingUtility.COLUMNS,
142      HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE, getRegionReplication());
143    return t;
144  }
145
146  protected int getRegionReplication() {
147    return 1;
148  }
149}