001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.procedure;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertTrue;
022import java.io.IOException;
023import org.apache.hadoop.conf.Configuration;
024import org.apache.hadoop.hbase.HBaseTestingUtility;
025import org.apache.hadoop.hbase.MiniHBaseCluster;
026import org.apache.hadoop.hbase.ServerName;
027import org.apache.hadoop.hbase.TableName;
028import org.apache.hadoop.hbase.client.RegionInfo;
029import org.apache.hadoop.hbase.client.RegionReplicaTestHelper;
030import org.apache.hadoop.hbase.client.Table;
031import org.apache.hadoop.hbase.master.HMaster;
032import org.apache.hadoop.hbase.master.assignment.AssignmentTestingUtil;
033import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
034import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
035import org.junit.After;
036import org.junit.Before;
037import org.slf4j.Logger;
038import org.slf4j.LoggerFactory;
039
040public class TestSCPBase {
041  private static final Logger LOG = LoggerFactory.getLogger(TestSCPBase.class);
042  static final int RS_COUNT = 3;
043
044  protected HBaseTestingUtility util;
045
046  protected void setupConf(Configuration conf) {
047    conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1);
048  }
049
050  @Before
051  public void setup() throws Exception {
052    this.util = new HBaseTestingUtility();
053    setupConf(this.util.getConfiguration());
054    startMiniCluster();
055    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(
056      this.util.getHBaseCluster().getMaster().getMasterProcedureExecutor(), false);
057  }
058
059  protected void startMiniCluster() throws Exception {
060    this.util.startMiniCluster(RS_COUNT);
061  }
062
063  @After
064  public void tearDown() throws Exception {
065    MiniHBaseCluster cluster = this.util.getHBaseCluster();
066    HMaster master = cluster == null ? null : cluster.getMaster();
067    if (master != null && master.getMasterProcedureExecutor() != null) {
068      ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(master.getMasterProcedureExecutor(),
069        false);
070    }
071    this.util.shutdownMiniCluster();
072  }
073
074  /**
075   * Run server crash procedure steps twice to test idempotency and that we are persisting all
076   * needed state.
077   */
078  protected void testRecoveryAndDoubleExecution(boolean carryingMeta, boolean doubleExecution)
079      throws Exception {
080    final TableName tableName = TableName.valueOf("testRecoveryAndDoubleExecution-carryingMeta-" +
081      carryingMeta + "-doubleExecution-" + doubleExecution);
082    try (Table t = createTable(tableName)) {
083      // Load the table with a bit of data so some logs to split and some edits in each region.
084      this.util.loadTable(t, HBaseTestingUtility.COLUMNS[0]);
085      final int count = util.countRows(t);
086      assertTrue("expected some rows", count > 0);
087      final String checksum = util.checksumRows(t);
088      // Run the procedure executor outside the master so we can mess with it. Need to disable
089      // Master's running of the server crash processing.
090      final HMaster master = this.util.getHBaseCluster().getMaster();
091      final ProcedureExecutor<MasterProcedureEnv> procExec = master.getMasterProcedureExecutor();
092      // find the first server that match the request and executes the test
093      ServerName rsToKill = null;
094      for (RegionInfo hri : util.getAdmin().getRegions(tableName)) {
095        final ServerName serverName = AssignmentTestingUtil.getServerHoldingRegion(util, hri);
096        if (AssignmentTestingUtil.isServerHoldingMeta(util, serverName) == carryingMeta) {
097          rsToKill = serverName;
098          break;
099        }
100      }
101      // Enable test flags and then queue the crash procedure.
102      ProcedureTestingUtility.waitNoProcedureRunning(procExec);
103      if (doubleExecution) {
104        // For SCP, if you enable this then we will enter an infinite loop, as we will crash between
105        // queue and open for TRSP, and then going back to queue, as we will use the crash rs as the
106        // target server since it is recored in hbase:meta.
107        ProcedureTestingUtility.setKillIfHasParent(procExec, false);
108        ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
109        // kill the RS
110        AssignmentTestingUtil.killRs(util, rsToKill);
111        long procId = getSCPProcId(procExec);
112        // Now run through the procedure twice crashing the executor on each step...
113        MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId);
114      } else {
115        // kill the RS
116        AssignmentTestingUtil.killRs(util, rsToKill);
117        long procId = getSCPProcId(procExec);
118        ProcedureTestingUtility.waitProcedure(procExec, procId);
119      }
120      RegionReplicaTestHelper.assertReplicaDistributed(util, t);
121      assertEquals(count, util.countRows(t));
122      assertEquals(checksum, util.checksumRows(t));
123    }
124  }
125
126  protected long getSCPProcId(ProcedureExecutor<?> procExec) {
127    util.waitFor(30000, () -> !procExec.getProcedures().isEmpty());
128    return procExec.getActiveProcIds().stream().mapToLong(Long::longValue).min().getAsLong();
129  }
130
131  protected Table createTable(final TableName tableName) throws IOException {
132    final Table t = this.util.createTable(tableName, HBaseTestingUtility.COLUMNS,
133      HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE, getRegionReplication());
134    return t;
135  }
136
137  protected int getRegionReplication() {
138    return 1;
139  }
140}