001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.assignment;
019
020import static org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.RegionStateTransitionState.REGION_STATE_TRANSITION_CONFIRM_OPENED_VALUE;
021import static org.junit.Assert.assertEquals;
022
023import java.io.IOException;
024import java.util.Set;
025import java.util.concurrent.CountDownLatch;
026import java.util.concurrent.Future;
027import org.apache.hadoop.conf.Configuration;
028import org.apache.hadoop.hbase.HBaseClassTestRule;
029import org.apache.hadoop.hbase.HBaseTestingUtility;
030import org.apache.hadoop.hbase.HConstants;
031import org.apache.hadoop.hbase.PleaseHoldException;
032import org.apache.hadoop.hbase.ServerName;
033import org.apache.hadoop.hbase.TableName;
034import org.apache.hadoop.hbase.client.Put;
035import org.apache.hadoop.hbase.client.RegionInfo;
036import org.apache.hadoop.hbase.client.Table;
037import org.apache.hadoop.hbase.master.HMaster;
038import org.apache.hadoop.hbase.master.MasterServices;
039import org.apache.hadoop.hbase.master.RegionPlan;
040import org.apache.hadoop.hbase.master.RegionState;
041import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
042import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
043import org.apache.hadoop.hbase.testclassification.MasterTests;
044import org.apache.hadoop.hbase.testclassification.MediumTests;
045import org.apache.hadoop.hbase.util.Bytes;
046import org.apache.hadoop.hbase.util.IdLock;
047import org.junit.AfterClass;
048import org.junit.BeforeClass;
049import org.junit.ClassRule;
050import org.junit.Test;
051import org.junit.experimental.categories.Category;
052
053import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionRequest;
054import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionResponse;
055
056@Category({ MasterTests.class, MediumTests.class })
057public class TestReportOnlineRegionsRace {
058
059  @ClassRule
060  public static final HBaseClassTestRule CLASS_RULE =
061    HBaseClassTestRule.forClass(TestReportOnlineRegionsRace.class);
062
063  private static volatile CountDownLatch ARRIVE_RS_REPORT;
064  private static volatile CountDownLatch RESUME_RS_REPORT;
065  private static volatile CountDownLatch FINISH_RS_REPORT;
066
067  private static volatile CountDownLatch RESUME_REPORT_STATE;
068
069  private static final class AssignmentManagerForTest extends AssignmentManager {
070
071    public AssignmentManagerForTest(MasterServices master) {
072      super(master);
073    }
074
075    @Override
076    public void reportOnlineRegions(ServerName serverName, Set<byte[]> regionNames) {
077      if (ARRIVE_RS_REPORT != null) {
078        ARRIVE_RS_REPORT.countDown();
079        try {
080          RESUME_RS_REPORT.await();
081        } catch (InterruptedException e) {
082          throw new RuntimeException(e);
083        }
084      }
085      super.reportOnlineRegions(serverName, regionNames);
086      if (FINISH_RS_REPORT != null) {
087        FINISH_RS_REPORT.countDown();
088      }
089    }
090
091    @Override
092    public ReportRegionStateTransitionResponse reportRegionStateTransition(
093        ReportRegionStateTransitionRequest req) throws PleaseHoldException {
094      if (RESUME_REPORT_STATE != null) {
095        try {
096          RESUME_REPORT_STATE.await();
097        } catch (InterruptedException e) {
098          throw new RuntimeException(e);
099        }
100      }
101      return super.reportRegionStateTransition(req);
102    }
103
104  }
105
106  public static final class HMasterForTest extends HMaster {
107
108    public HMasterForTest(Configuration conf) throws IOException {
109      super(conf);
110    }
111
112    @Override
113    protected AssignmentManager createAssignmentManager(MasterServices master) {
114      return new AssignmentManagerForTest(master);
115    }
116  }
117
118  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
119
120  private static TableName NAME = TableName.valueOf("Race");
121
122  private static byte[] CF = Bytes.toBytes("cf");
123
124  @BeforeClass
125  public static void setUp() throws Exception {
126    UTIL.getConfiguration().setClass(HConstants.MASTER_IMPL, HMasterForTest.class, HMaster.class);
127    UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 1000);
128    UTIL.getConfiguration().setInt(HConstants.REGION_SERVER_HIGH_PRIORITY_HANDLER_COUNT,
129        HConstants.DEFAULT_REGION_SERVER_HIGH_PRIORITY_HANDLER_COUNT);
130    UTIL.startMiniCluster(1);
131    UTIL.createTable(NAME, CF);
132    UTIL.waitTableAvailable(NAME);
133  }
134
135  @AfterClass
136  public static void tearDown() throws Exception {
137    UTIL.shutdownMiniCluster();
138  }
139
140  @Test
141  public void testRace() throws Exception {
142    RegionInfo region = UTIL.getMiniHBaseCluster().getRegions(NAME).get(0).getRegionInfo();
143    ProcedureExecutor<MasterProcedureEnv> procExec =
144      UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor();
145    AssignmentManager am = UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager();
146    RegionStateNode rsn = am.getRegionStates().getRegionStateNode(region);
147
148    // halt a regionServerReport
149    RESUME_RS_REPORT = new CountDownLatch(1);
150    ARRIVE_RS_REPORT = new CountDownLatch(1);
151    FINISH_RS_REPORT = new CountDownLatch(1);
152
153    ARRIVE_RS_REPORT.await();
154
155    // schedule a TRSP to REOPEN the region
156    RESUME_REPORT_STATE = new CountDownLatch(1);
157    Future<byte[]> future =
158      am.moveAsync(new RegionPlan(region, rsn.getRegionLocation(), rsn.getRegionLocation()));
159    TransitRegionStateProcedure proc =
160      procExec.getProcedures().stream().filter(p -> p instanceof TransitRegionStateProcedure)
161        .filter(p -> !p.isFinished()).map(p -> (TransitRegionStateProcedure) p).findAny().get();
162    IdLock procExecLock = procExec.getProcExecutionLock();
163    // a CloseRegionProcedure and then the OpenRegionProcedure we want to block
164    IdLock.Entry lockEntry = procExecLock.getLockEntry(proc.getProcId() + 2);
165    // resume the reportRegionStateTransition to finish the CloseRegionProcedure
166    RESUME_REPORT_STATE.countDown();
167    // wait until we schedule the OpenRegionProcedure
168    UTIL.waitFor(10000,
169      () -> proc.getCurrentStateId() == REGION_STATE_TRANSITION_CONFIRM_OPENED_VALUE);
170    // the region should be in OPENING state
171    assertEquals(RegionState.State.OPENING, rsn.getState());
172    // resume the region server report
173    RESUME_RS_REPORT.countDown();
174    // wait until it finishes, it will find that the region is opened on the rs
175    FINISH_RS_REPORT.await();
176    // let the OpenRegionProcedure go
177    procExecLock.releaseLockEntry(lockEntry);
178    // wait until the TRSP is done
179    future.get();
180
181    // confirm that the region can still be write, i.e, the regionServerReport method should not
182    // change the region state to OPEN
183    try (Table table = UTIL.getConnection().getTableBuilder(NAME, null).setWriteRpcTimeout(1000)
184      .setOperationTimeout(2000).build()) {
185      table.put(
186        new Put(Bytes.toBytes("key")).addColumn(CF, Bytes.toBytes("cq"), Bytes.toBytes("val")));
187    }
188  }
189}