001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.procedure;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertTrue;
022
023import java.io.IOException;
024import org.apache.hadoop.conf.Configuration;
025import org.apache.hadoop.hbase.HBaseTestingUtil;
026import org.apache.hadoop.hbase.ServerName;
027import org.apache.hadoop.hbase.SingleProcessHBaseCluster;
028import org.apache.hadoop.hbase.TableName;
029import org.apache.hadoop.hbase.client.RegionInfo;
030import org.apache.hadoop.hbase.client.RegionReplicaTestHelper;
031import org.apache.hadoop.hbase.client.Table;
032import org.apache.hadoop.hbase.master.HMaster;
033import org.apache.hadoop.hbase.master.assignment.AssignmentTestingUtil;
034import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
035import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
036import org.junit.After;
037import org.junit.Before;
038import org.slf4j.Logger;
039import org.slf4j.LoggerFactory;
040
041public class TestSCPBase {
042  private static final Logger LOG = LoggerFactory.getLogger(TestSCPBase.class);
043  static final int RS_COUNT = 3;
044
045  protected HBaseTestingUtil util;
046
047  protected void setupConf(Configuration conf) {
048    conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1);
049  }
050
051  @Before
052  public void setup() throws Exception {
053    this.util = new HBaseTestingUtil();
054    setupConf(this.util.getConfiguration());
055    startMiniCluster();
056    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(
057      this.util.getHBaseCluster().getMaster().getMasterProcedureExecutor(), false);
058  }
059
060  protected void startMiniCluster() throws Exception {
061    this.util.startMiniCluster(RS_COUNT);
062  }
063
064  @After
065  public void tearDown() throws Exception {
066    SingleProcessHBaseCluster cluster = this.util.getHBaseCluster();
067    HMaster master = cluster == null ? null : cluster.getMaster();
068    if (master != null && master.getMasterProcedureExecutor() != null) {
069      ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(master.getMasterProcedureExecutor(),
070        false);
071    }
072    this.util.shutdownMiniCluster();
073  }
074
075  /**
076   * Run server crash procedure steps twice to test idempotency and that we are persisting all
077   * needed state.
078   */
079  protected void testRecoveryAndDoubleExecution(boolean carryingMeta, boolean doubleExecution)
080    throws Exception {
081    final TableName tableName = TableName.valueOf("testRecoveryAndDoubleExecution-carryingMeta-"
082      + carryingMeta + "-doubleExecution-" + doubleExecution);
083    try (Table t = createTable(tableName)) {
084      // Load the table with a bit of data so some logs to split and some edits in each region.
085      this.util.loadTable(t, HBaseTestingUtil.COLUMNS[0]);
086      final int count = HBaseTestingUtil.countRows(t);
087      assertTrue("expected some rows", count > 0);
088      final String checksum = util.checksumRows(t);
089      // Run the procedure executor outside the master so we can mess with it. Need to disable
090      // Master's running of the server crash processing.
091      final HMaster master = this.util.getHBaseCluster().getMaster();
092      final ProcedureExecutor<MasterProcedureEnv> procExec = master.getMasterProcedureExecutor();
093      // find the first server that match the request and executes the test
094      ServerName rsToKill = null;
095      for (RegionInfo hri : util.getAdmin().getRegions(tableName)) {
096        final ServerName serverName = AssignmentTestingUtil.getServerHoldingRegion(util, hri);
097        if (AssignmentTestingUtil.isServerHoldingMeta(util, serverName) == carryingMeta) {
098          rsToKill = serverName;
099          break;
100        }
101      }
102      // Enable test flags and then queue the crash procedure.
103      ProcedureTestingUtility.waitNoProcedureRunning(procExec);
104      if (doubleExecution) {
105        // For SCP, if you enable this then we will enter an infinite loop, as we will crash between
106        // queue and open for TRSP, and then going back to queue, as we will use the crash rs as the
107        // target server since it is recored in hbase:meta.
108        ProcedureTestingUtility.setKillIfHasParent(procExec, false);
109        ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
110        // kill the RS
111        AssignmentTestingUtil.killRs(util, rsToKill);
112        long procId = getSCPProcId(procExec);
113        // Now run through the procedure twice crashing the executor on each step...
114        MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId);
115      } else {
116        // kill the RS
117        AssignmentTestingUtil.killRs(util, rsToKill);
118        long procId = getSCPProcId(procExec);
119        ProcedureTestingUtility.waitProcedure(procExec, procId);
120      }
121      RegionReplicaTestHelper.assertReplicaDistributed(util, t);
122      assertEquals(count, HBaseTestingUtil.countRows(t));
123      assertEquals(checksum, util.checksumRows(t));
124    }
125  }
126
127  protected long getSCPProcId(ProcedureExecutor<?> procExec) {
128    util.waitFor(30000, () -> !procExec.getProcedures().isEmpty());
129    return procExec.getActiveProcIds().stream().mapToLong(Long::longValue).min().getAsLong();
130  }
131
132  protected Table createTable(final TableName tableName) throws IOException {
133    final Table t = this.util.createTable(tableName, HBaseTestingUtil.COLUMNS,
134      HBaseTestingUtil.KEYS_FOR_HBA_CREATE_TABLE, getRegionReplication());
135    return t;
136  }
137
138  protected int getRegionReplication() {
139    return 1;
140  }
141}