001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.master.procedure; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertTrue; 022 023import java.io.IOException; 024 025import org.apache.hadoop.conf.Configuration; 026import org.apache.hadoop.hbase.HBaseTestingUtility; 027import org.apache.hadoop.hbase.HConstants; 028import org.apache.hadoop.hbase.MiniHBaseCluster; 029import org.apache.hadoop.hbase.ServerName; 030import org.apache.hadoop.hbase.TableName; 031import org.apache.hadoop.hbase.client.RegionInfo; 032import org.apache.hadoop.hbase.client.RegionReplicaTestHelper; 033import org.apache.hadoop.hbase.client.Table; 034import org.apache.hadoop.hbase.master.HMaster; 035import org.apache.hadoop.hbase.master.assignment.AssignmentTestingUtil; 036import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; 037import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; 038import org.junit.After; 039import org.junit.Before; 040import org.slf4j.Logger; 041import org.slf4j.LoggerFactory; 042 043public class TestSCPBase { 044 private static final Logger LOG = LoggerFactory.getLogger(TestSCPBase.class); 045 static final int RS_COUNT = 3; 046 047 protected HBaseTestingUtility util; 048 049 protected void setupConf(Configuration conf) { 050 conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1); 051 conf.set("hbase.balancer.tablesOnMaster", "none"); 052 conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, RS_COUNT); 053 conf.setInt(HConstants.HBASE_CLIENT_SERVERSIDE_RETRIES_MULTIPLIER, 3); 054 conf.setBoolean("hbase.split.writer.creation.bounded", true); 055 conf.setInt("hbase.regionserver.hlog.splitlog.writer.threads", 8); 056 conf.setBoolean(HConstants.HBASE_SPLIT_WAL_COORDINATED_BY_ZK, true); 057 } 058 059 @Before 060 public void setup() throws Exception { 061 this.util = new HBaseTestingUtility(); 062 setupConf(this.util.getConfiguration()); 063 startMiniCluster(); 064 ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate( 065 this.util.getHBaseCluster().getMaster().getMasterProcedureExecutor(), false); 066 } 067 068 protected void startMiniCluster() throws Exception { 069 this.util.startMiniCluster(3); 070 } 071 072 @After 073 public void tearDown() throws Exception { 074 MiniHBaseCluster cluster = this.util.getHBaseCluster(); 075 HMaster master = cluster == null ? null : cluster.getMaster(); 076 if (master != null && master.getMasterProcedureExecutor() != null) { 077 ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(master.getMasterProcedureExecutor(), 078 false); 079 } 080 this.util.shutdownMiniCluster(); 081 } 082 083 /** 084 * Run server crash procedure steps twice to test idempotency and that we are persisting all 085 * needed state. 086 */ 087 protected void testRecoveryAndDoubleExecution(boolean carryingMeta, boolean doubleExecution) 088 throws Exception { 089 final TableName tableName = TableName.valueOf("testRecoveryAndDoubleExecution-carryingMeta-" + 090 carryingMeta + "-doubleExecution-" + doubleExecution); 091 try (Table t = createTable(tableName)) { 092 // Load the table with a bit of data so some logs to split and some edits in each region. 093 this.util.loadTable(t, HBaseTestingUtility.COLUMNS[0]); 094 final int count = util.countRows(t); 095 assertTrue("expected some rows", count > 0); 096 final String checksum = util.checksumRows(t); 097 // Run the procedure executor outside the master so we can mess with it. Need to disable 098 // Master's running of the server crash processing. 099 final HMaster master = this.util.getHBaseCluster().getMaster(); 100 final ProcedureExecutor<MasterProcedureEnv> procExec = master.getMasterProcedureExecutor(); 101 // find the first server that match the request and executes the test 102 ServerName rsToKill = null; 103 for (RegionInfo hri : util.getAdmin().getRegions(tableName)) { 104 final ServerName serverName = AssignmentTestingUtil.getServerHoldingRegion(util, hri); 105 if (AssignmentTestingUtil.isServerHoldingMeta(util, serverName) == carryingMeta) { 106 rsToKill = serverName; 107 break; 108 } 109 } 110 // Enable test flags and then queue the crash procedure. 111 ProcedureTestingUtility.waitNoProcedureRunning(procExec); 112 if (doubleExecution) { 113 // For SCP, if you enable this then we will enter an infinite loop, as we will crash between 114 // queue and open for TRSP, and then going back to queue, as we will use the crash rs as the 115 // target server since it is recored in hbase:meta. 116 ProcedureTestingUtility.setKillIfHasParent(procExec, false); 117 ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); 118 // kill the RS 119 AssignmentTestingUtil.killRs(util, rsToKill); 120 long procId = getSCPProcId(procExec); 121 // Now run through the procedure twice crashing the executor on each step... 122 MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId); 123 } else { 124 // kill the RS 125 AssignmentTestingUtil.killRs(util, rsToKill); 126 long procId = getSCPProcId(procExec); 127 ProcedureTestingUtility.waitProcedure(procExec, procId); 128 } 129 RegionReplicaTestHelper.assertReplicaDistributed(util, t); 130 assertEquals(count, util.countRows(t)); 131 assertEquals(checksum, util.checksumRows(t)); 132 } 133 } 134 135 protected long getSCPProcId(ProcedureExecutor<?> procExec) { 136 util.waitFor(30000, () -> !procExec.getProcedures().isEmpty()); 137 return procExec.getActiveProcIds().stream().mapToLong(Long::longValue).min().getAsLong(); 138 } 139 140 protected Table createTable(final TableName tableName) throws IOException { 141 final Table t = this.util.createTable(tableName, HBaseTestingUtility.COLUMNS, 142 HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE, getRegionReplication()); 143 return t; 144 } 145 146 protected int getRegionReplication() { 147 return 1; 148 } 149}