001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.master.assignment; 019 020import static org.junit.Assert.assertNotNull; 021import static org.junit.Assert.assertNull; 022 023import java.io.IOException; 024import java.util.ArrayList; 025import java.util.List; 026import java.util.concurrent.CountDownLatch; 027import java.util.concurrent.Future; 028import org.apache.hadoop.conf.Configuration; 029import org.apache.hadoop.hbase.HBaseClassTestRule; 030import org.apache.hadoop.hbase.HBaseTestingUtility; 031import org.apache.hadoop.hbase.PleaseHoldException; 032import org.apache.hadoop.hbase.ServerName; 033import org.apache.hadoop.hbase.StartMiniClusterOption; 034import org.apache.hadoop.hbase.TableName; 035import org.apache.hadoop.hbase.client.RegionInfo; 036import org.apache.hadoop.hbase.master.HMaster; 037import org.apache.hadoop.hbase.master.MasterServices; 038import org.apache.hadoop.hbase.master.RegionPlan; 039import org.apache.hadoop.hbase.master.ServerManager; 040import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure; 041import org.apache.hadoop.hbase.regionserver.HRegionServer; 042import org.apache.hadoop.hbase.testclassification.MasterTests; 043import org.apache.hadoop.hbase.testclassification.MediumTests; 044import org.apache.hadoop.hbase.util.Bytes; 045import org.apache.hadoop.hbase.util.IdLock; 046import org.apache.zookeeper.KeeperException; 047import org.junit.AfterClass; 048import org.junit.BeforeClass; 049import org.junit.ClassRule; 050import org.junit.Test; 051import org.junit.experimental.categories.Category; 052 053import org.apache.hbase.thirdparty.com.google.common.collect.Iterables; 054 055import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode; 056import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionRequest; 057import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionResponse; 058 059/** 060 * Testcase for HBASE-22365. 061 */ 062@Category({ MasterTests.class, MediumTests.class }) 063public class TestSCPGetRegionsRace { 064 065 @ClassRule 066 public static final HBaseClassTestRule CLASS_RULE = 067 HBaseClassTestRule.forClass(TestSCPGetRegionsRace.class); 068 069 private static final List<ServerName> EXCLUDE_SERVERS = new ArrayList<>(); 070 071 private static final class ServerManagerForTest extends ServerManager { 072 073 public ServerManagerForTest(MasterServices master) { 074 super(master); 075 } 076 077 @Override 078 public List<ServerName> createDestinationServersList() { 079 return super.createDestinationServersList(EXCLUDE_SERVERS); 080 } 081 } 082 083 private static CountDownLatch ARRIVE_REPORT; 084 085 private static CountDownLatch RESUME_REPORT; 086 087 private static CountDownLatch ARRIVE_GET; 088 089 private static CountDownLatch RESUME_GET; 090 091 private static final class AssignmentManagerForTest extends AssignmentManager { 092 093 public AssignmentManagerForTest(MasterServices master) { 094 super(master); 095 } 096 097 @Override 098 public ReportRegionStateTransitionResponse reportRegionStateTransition( 099 ReportRegionStateTransitionRequest req) throws PleaseHoldException { 100 if (req.getTransition(0).getTransitionCode() == TransitionCode.CLOSED) { 101 if (ARRIVE_REPORT != null) { 102 ARRIVE_REPORT.countDown(); 103 try { 104 RESUME_REPORT.await(); 105 RESUME_REPORT = null; 106 } catch (InterruptedException e) { 107 throw new RuntimeException(e); 108 } 109 } 110 } 111 return super.reportRegionStateTransition(req); 112 } 113 114 @Override 115 public List<RegionInfo> getRegionsOnServer(ServerName serverName) { 116 List<RegionInfo> regions = super.getRegionsOnServer(serverName); 117 if (ARRIVE_GET != null) { 118 ARRIVE_GET.countDown(); 119 try { 120 RESUME_GET.await(); 121 RESUME_GET = null; 122 } catch (InterruptedException e) { 123 throw new RuntimeException(e); 124 } 125 } 126 return regions; 127 } 128 129 } 130 131 public static final class HMasterForTest extends HMaster { 132 133 public HMasterForTest(Configuration conf) throws IOException, KeeperException { 134 super(conf); 135 } 136 137 @Override 138 protected AssignmentManager createAssignmentManager(MasterServices master) { 139 return new AssignmentManagerForTest(master); 140 } 141 142 @Override 143 protected ServerManager createServerManager(MasterServices master) throws IOException { 144 setupClusterConnection(); 145 return new ServerManagerForTest(master); 146 } 147 } 148 149 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); 150 151 private static TableName NAME = TableName.valueOf("Assign"); 152 153 private static byte[] CF = Bytes.toBytes("cf"); 154 155 @BeforeClass 156 public static void setUp() throws Exception { 157 UTIL.startMiniCluster(StartMiniClusterOption.builder().masterClass(HMasterForTest.class) 158 .numMasters(1).numRegionServers(3).build()); 159 UTIL.createTable(NAME, CF); 160 UTIL.waitTableAvailable(NAME); 161 UTIL.getAdmin().balancerSwitch(false, true); 162 } 163 164 @AfterClass 165 public static void tearDown() throws Exception { 166 UTIL.shutdownMiniCluster(); 167 } 168 169 @Test 170 public void test() throws Exception { 171 RegionInfo region = 172 Iterables.getOnlyElement(UTIL.getMiniHBaseCluster().getRegions(NAME)).getRegionInfo(); 173 HMaster master = UTIL.getMiniHBaseCluster().getMaster(); 174 AssignmentManager am = master.getAssignmentManager(); 175 RegionStateNode rsn = am.getRegionStates().getRegionStateNode(region); 176 ServerName source = rsn.getRegionLocation(); 177 ServerName dest = 178 UTIL.getAdmin().getRegionServers().stream().filter(sn -> !sn.equals(source)).findAny().get(); 179 180 ARRIVE_REPORT = new CountDownLatch(1); 181 RESUME_REPORT = new CountDownLatch(1); 182 183 Future<?> future = am.moveAsync(new RegionPlan(region, source, dest)); 184 185 ARRIVE_REPORT.await(); 186 ARRIVE_REPORT = null; 187 // let's get procedure lock to stop the TRSP 188 IdLock procExecutionLock = master.getMasterProcedureExecutor().getProcExecutionLock(); 189 long procId = master.getProcedures().stream() 190 .filter(p -> p instanceof RegionRemoteProcedureBase).findAny().get().getProcId(); 191 IdLock.Entry lockEntry = procExecutionLock.getLockEntry(procId); 192 RESUME_REPORT.countDown(); 193 194 // kill the source region server 195 ARRIVE_GET = new CountDownLatch(1); 196 RESUME_GET = new CountDownLatch(1); 197 UTIL.getMiniHBaseCluster().killRegionServer(source); 198 199 // wait until we try to get the region list of the region server 200 ARRIVE_GET.await(); 201 ARRIVE_GET = null; 202 // release the procedure lock and let the TRSP to finish 203 procExecutionLock.releaseLockEntry(lockEntry); 204 future.get(); 205 206 // resume the SCP 207 EXCLUDE_SERVERS.add(dest); 208 RESUME_GET.countDown(); 209 // wait until there are no SCPs and TRSPs 210 UTIL.waitFor(60000, () -> master.getProcedures().stream().allMatch(p -> p.isFinished() || 211 (!(p instanceof ServerCrashProcedure) && !(p instanceof TransitRegionStateProcedure)))); 212 213 // assert the region is only on the dest server. 214 HRegionServer rs = UTIL.getMiniHBaseCluster().getRegionServer(dest); 215 assertNotNull(rs.getRegion(region.getEncodedName())); 216 assertNull(UTIL.getOtherRegionServer(rs).getRegion(region.getEncodedName())); 217 } 218}