001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.assignment;
019
020import static org.junit.Assert.assertNotNull;
021import static org.junit.Assert.assertNull;
022
023import java.io.IOException;
024import java.util.ArrayList;
025import java.util.List;
026import java.util.concurrent.CountDownLatch;
027import java.util.concurrent.Future;
028import org.apache.hadoop.conf.Configuration;
029import org.apache.hadoop.hbase.HBaseClassTestRule;
030import org.apache.hadoop.hbase.HBaseTestingUtility;
031import org.apache.hadoop.hbase.PleaseHoldException;
032import org.apache.hadoop.hbase.ServerName;
033import org.apache.hadoop.hbase.StartMiniClusterOption;
034import org.apache.hadoop.hbase.TableName;
035import org.apache.hadoop.hbase.client.RegionInfo;
036import org.apache.hadoop.hbase.master.HMaster;
037import org.apache.hadoop.hbase.master.MasterServices;
038import org.apache.hadoop.hbase.master.RegionPlan;
039import org.apache.hadoop.hbase.master.ServerManager;
040import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure;
041import org.apache.hadoop.hbase.regionserver.HRegionServer;
042import org.apache.hadoop.hbase.testclassification.MasterTests;
043import org.apache.hadoop.hbase.testclassification.MediumTests;
044import org.apache.hadoop.hbase.util.Bytes;
045import org.apache.hadoop.hbase.util.IdLock;
046import org.apache.zookeeper.KeeperException;
047import org.junit.AfterClass;
048import org.junit.BeforeClass;
049import org.junit.ClassRule;
050import org.junit.Test;
051import org.junit.experimental.categories.Category;
052
053import org.apache.hbase.thirdparty.com.google.common.collect.Iterables;
054
055import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode;
056import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionRequest;
057import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionResponse;
058
059/**
060 * Testcase for HBASE-22365.
061 */
062@Category({ MasterTests.class, MediumTests.class })
063public class TestSCPGetRegionsRace {
064
065  @ClassRule
066  public static final HBaseClassTestRule CLASS_RULE =
067    HBaseClassTestRule.forClass(TestSCPGetRegionsRace.class);
068
069  private static final List<ServerName> EXCLUDE_SERVERS = new ArrayList<>();
070
071  private static final class ServerManagerForTest extends ServerManager {
072
073    public ServerManagerForTest(MasterServices master) {
074      super(master);
075    }
076
077    @Override
078    public List<ServerName> createDestinationServersList() {
079      return super.createDestinationServersList(EXCLUDE_SERVERS);
080    }
081  }
082
083  private static CountDownLatch ARRIVE_REPORT;
084
085  private static CountDownLatch RESUME_REPORT;
086
087  private static CountDownLatch ARRIVE_GET;
088
089  private static CountDownLatch RESUME_GET;
090
091  private static final class AssignmentManagerForTest extends AssignmentManager {
092
093    public AssignmentManagerForTest(MasterServices master) {
094      super(master);
095    }
096
097    @Override
098    public ReportRegionStateTransitionResponse reportRegionStateTransition(
099        ReportRegionStateTransitionRequest req) throws PleaseHoldException {
100      if (req.getTransition(0).getTransitionCode() == TransitionCode.CLOSED) {
101        if (ARRIVE_REPORT != null) {
102          ARRIVE_REPORT.countDown();
103          try {
104            RESUME_REPORT.await();
105            RESUME_REPORT = null;
106          } catch (InterruptedException e) {
107            throw new RuntimeException(e);
108          }
109        }
110      }
111      return super.reportRegionStateTransition(req);
112    }
113
114    @Override
115    public List<RegionInfo> getRegionsOnServer(ServerName serverName) {
116      List<RegionInfo> regions = super.getRegionsOnServer(serverName);
117      if (ARRIVE_GET != null) {
118        ARRIVE_GET.countDown();
119        try {
120          RESUME_GET.await();
121          RESUME_GET = null;
122        } catch (InterruptedException e) {
123          throw new RuntimeException(e);
124        }
125      }
126      return regions;
127    }
128
129  }
130
131  public static final class HMasterForTest extends HMaster {
132
133    public HMasterForTest(Configuration conf) throws IOException, KeeperException {
134      super(conf);
135    }
136
137    @Override
138    protected AssignmentManager createAssignmentManager(MasterServices master) {
139      return new AssignmentManagerForTest(master);
140    }
141
142    @Override
143    protected ServerManager createServerManager(MasterServices master) throws IOException {
144      setupClusterConnection();
145      return new ServerManagerForTest(master);
146    }
147  }
148
149  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
150
151  private static TableName NAME = TableName.valueOf("Assign");
152
153  private static byte[] CF = Bytes.toBytes("cf");
154
155  @BeforeClass
156  public static void setUp() throws Exception {
157    UTIL.startMiniCluster(StartMiniClusterOption.builder().masterClass(HMasterForTest.class)
158      .numMasters(1).numRegionServers(3).build());
159    UTIL.createTable(NAME, CF);
160    UTIL.waitTableAvailable(NAME);
161    UTIL.getAdmin().balancerSwitch(false, true);
162  }
163
164  @AfterClass
165  public static void tearDown() throws Exception {
166    UTIL.shutdownMiniCluster();
167  }
168
169  @Test
170  public void test() throws Exception {
171    RegionInfo region =
172      Iterables.getOnlyElement(UTIL.getMiniHBaseCluster().getRegions(NAME)).getRegionInfo();
173    HMaster master = UTIL.getMiniHBaseCluster().getMaster();
174    AssignmentManager am = master.getAssignmentManager();
175    RegionStateNode rsn = am.getRegionStates().getRegionStateNode(region);
176    ServerName source = rsn.getRegionLocation();
177    ServerName dest =
178      UTIL.getAdmin().getRegionServers().stream().filter(sn -> !sn.equals(source)).findAny().get();
179
180    ARRIVE_REPORT = new CountDownLatch(1);
181    RESUME_REPORT = new CountDownLatch(1);
182
183    Future<?> future = am.moveAsync(new RegionPlan(region, source, dest));
184
185    ARRIVE_REPORT.await();
186    ARRIVE_REPORT = null;
187    // let's get procedure lock to stop the TRSP
188    IdLock procExecutionLock = master.getMasterProcedureExecutor().getProcExecutionLock();
189    long procId = master.getProcedures().stream()
190      .filter(p -> p instanceof RegionRemoteProcedureBase).findAny().get().getProcId();
191    IdLock.Entry lockEntry = procExecutionLock.getLockEntry(procId);
192    RESUME_REPORT.countDown();
193
194    // kill the source region server
195    ARRIVE_GET = new CountDownLatch(1);
196    RESUME_GET = new CountDownLatch(1);
197    UTIL.getMiniHBaseCluster().killRegionServer(source);
198
199    // wait until we try to get the region list of the region server
200    ARRIVE_GET.await();
201    ARRIVE_GET = null;
202    // release the procedure lock and let the TRSP to finish
203    procExecutionLock.releaseLockEntry(lockEntry);
204    future.get();
205
206    // resume the SCP
207    EXCLUDE_SERVERS.add(dest);
208    RESUME_GET.countDown();
209    // wait until there are no SCPs and TRSPs
210    UTIL.waitFor(60000, () -> master.getProcedures().stream().allMatch(p -> p.isFinished() ||
211      (!(p instanceof ServerCrashProcedure) && !(p instanceof TransitRegionStateProcedure))));
212
213    // assert the region is only on the dest server.
214    HRegionServer rs = UTIL.getMiniHBaseCluster().getRegionServer(dest);
215    assertNotNull(rs.getRegion(region.getEncodedName()));
216    assertNull(UTIL.getOtherRegionServer(rs).getRegion(region.getEncodedName()));
217  }
218}