001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.assignment;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertTrue;
022
023import java.io.IOException;
024import java.util.concurrent.CountDownLatch;
025import java.util.concurrent.Future;
026import java.util.concurrent.TimeUnit;
027import java.util.concurrent.atomic.AtomicReference;
028import org.apache.hadoop.conf.Configuration;
029import org.apache.hadoop.hbase.HBaseTestingUtil;
030import org.apache.hadoop.hbase.HConstants;
031import org.apache.hadoop.hbase.PleaseHoldException;
032import org.apache.hadoop.hbase.TableName;
033import org.apache.hadoop.hbase.client.RegionInfo;
034import org.apache.hadoop.hbase.master.HMaster;
035import org.apache.hadoop.hbase.master.MasterServices;
036import org.apache.hadoop.hbase.master.RegionPlan;
037import org.apache.hadoop.hbase.master.RegionState;
038import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
039import org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility;
040import org.apache.hadoop.hbase.master.region.MasterRegion;
041import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
042import org.apache.hadoop.hbase.testclassification.MasterTests;
043import org.apache.hadoop.hbase.testclassification.MediumTests;
044import org.apache.hadoop.hbase.util.Bytes;
045import org.junit.jupiter.api.AfterAll;
046import org.junit.jupiter.api.BeforeAll;
047import org.junit.jupiter.api.Tag;
048import org.junit.jupiter.api.Test;
049
050import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
051import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition;
052import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode;
053import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionRequest;
054import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionResponse;
055
056/**
057 * HBASE-30357: OpenRegionProcedure#restoreSucceedState() must not force the region state to OPEN on
058 * master-failover restore when the persisted transition code is actually FAILED_OPEN.
059 * <p/>
060 * To reproduce the exact crash window without racing a genuinely reporting RS, we intercept the
061 * RS's real OPENED report on the master side and rewrite it to FAILED_OPEN before it is persisted.
062 * While still on the RPC handler thread (i.e. before the woken child OpenRegionProcedure gets a
063 * chance to run its own execute() and persist anything to meta), we lock the RegionStateNode and
064 * perform a genuine restart of the master's ProcedureExecutor/AssignmentManager, forcing a real
065 * reload from the WALProcedureStore and hbase:meta - exactly the mechanism restoreSucceedState() is
066 * meant to handle.
067 */
068@Tag(MasterTests.TAG)
069@Tag(MediumTests.TAG)
070public class TestOpenRegionProcedureRestoreFailedOpen {
071
072  private static final long AWAIT_TIMEOUT_SECONDS = 30;
073
074  private static final AtomicReference<CountDownLatch> ARRIVE = new AtomicReference<>();
075
076  private static final AtomicReference<CountDownLatch> PROCEED = new AtomicReference<>();
077
078  private static final class AssignmentManagerForTest extends AssignmentManager {
079
080    public AssignmentManagerForTest(MasterServices master, MasterRegion masterRegion) {
081      super(master, masterRegion);
082    }
083
084    @Override
085    public ReportRegionStateTransitionResponse reportRegionStateTransition(
086      ReportRegionStateTransitionRequest req) throws PleaseHoldException {
087      RegionStateTransition transition = req.getTransition(0);
088      RegionInfo hri = ProtobufUtil.toRegionInfo(transition.getRegionInfo(0));
089      if (transition.getTransitionCode() != TransitionCode.OPENED || !hri.getTable().equals(NAME)) {
090        return super.reportRegionStateTransition(req);
091      }
092      CountDownLatch arrive = ARRIVE.getAndSet(null);
093      if (arrive == null) {
094        return super.reportRegionStateTransition(req);
095      }
096      ReportRegionStateTransitionRequest failedOpenReq = req.toBuilder()
097        .setTransition(0, transition.toBuilder().setTransitionCode(TransitionCode.FAILED_OPEN)
098          .setOpenSeqNum(HConstants.NO_SEQNUM).build())
099        .build();
100      RegionStateNode regionNode = getRegionStates().getRegionStateNode(hri);
101      // AssignmentManager#updateRegionTransition() (called from super.reportRegionStateTransition
102      // below) also locks this same RegionStateNode; that only works here because the lock is
103      // reentrant for the same thread (see RegionStateNodeLock#lock0).
104      regionNode.lock();
105      try {
106        // persists REPORT_SUCCEED/FAILED_OPEN to the real WALProcedureStore and wakes the child
107        // OpenRegionProcedure, but since we still hold the RegionStateNode lock here (reentrant,
108        // same thread), the woken child can not resume and complete its own meta update - this is
109        // exactly the window a real master crash would leave us in.
110        ReportRegionStateTransitionResponse resp = super.reportRegionStateTransition(failedOpenReq);
111        arrive.countDown();
112        CountDownLatch proceed = PROCEED.get();
113        if (!proceed.await(AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)) {
114          throw new RuntimeException("Timed out waiting for PROCEED");
115        }
116        return resp;
117      } catch (InterruptedException e) {
118        throw new RuntimeException(e);
119      } finally {
120        regionNode.unlock();
121      }
122    }
123  }
124
125  public static final class HMasterForTest extends HMaster {
126
127    public HMasterForTest(Configuration conf) throws IOException {
128      super(conf);
129    }
130
131    @Override
132    protected AssignmentManager createAssignmentManager(MasterServices master,
133      MasterRegion masterRegion) {
134      return new AssignmentManagerForTest(master, masterRegion);
135    }
136  }
137
138  private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
139
140  private static final TableName NAME =
141    TableName.valueOf("TestOpenRegionProcedureRestoreFailedOpen");
142
143  private static final byte[] CF = Bytes.toBytes("cf");
144
145  @BeforeAll
146  public static void setUpBeforeClass() throws Exception {
147    UTIL.getConfiguration().setClass(HConstants.MASTER_IMPL, HMasterForTest.class, HMaster.class);
148    UTIL.startMiniCluster(1);
149    UTIL.createTable(NAME, CF);
150    UTIL.waitTableAvailable(NAME);
151  }
152
153  @AfterAll
154  public static void tearDownAfterClass() throws Exception {
155    UTIL.shutdownMiniCluster();
156  }
157
158  @Test
159  public void testRestoreDoesNotForceOpenAfterFailedOpen() throws Exception {
160    HMaster master = UTIL.getMiniHBaseCluster().getMaster();
161    ProcedureExecutor<MasterProcedureEnv> procExec = master.getMasterProcedureExecutor();
162    AssignmentManager am = master.getAssignmentManager();
163    RegionInfo region = UTIL.getAdmin().getRegions(NAME).get(0);
164    RegionStateNode regionNode = am.getRegionStates().getRegionStateNode(region);
165
166    CountDownLatch arrive = new CountDownLatch(1);
167    CountDownLatch proceed = new CountDownLatch(1);
168    ARRIVE.set(arrive);
169    PROCEED.set(proceed);
170    Future<byte[]> future = am.moveAsync(
171      new RegionPlan(region, regionNode.getRegionLocation(), regionNode.getRegionLocation()));
172    // arrive counts down only after the RS's OPENED report is intercepted, rewritten to
173    // FAILED_OPEN, and persisted, so waiting on it alone is sufficient synchronization
174    assertTrue(arrive.await(AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS));
175
176    MasterProcedureTestingUtility.restartMasterProcedureExecutor(procExec);
177    RegionStateNode reloaded = am.getRegionStates().getRegionStateNode(region);
178    // still OPENING: restarting the ProcedureExecutor re-triggers AssignmentManager#joinCluster's
179    // meta scan, which reloads the state from the persisted hbase:meta column before
180    // restoreSucceedState() runs; regionFailedOpen(regionNode, false) then only detaches the
181    // region from its (now-defunct) server, it does not change the state.
182    // TransitRegionStateProcedure is the one that decides to give up
183    // (and thus set FAILED_OPEN) or retry the open, and with the
184    // default (effectively unbounded) hbase.assignment.maximum.attempts, it never gives up here.
185    assertEquals(RegionState.State.OPENING, reloaded.getState());
186
187    proceed.countDown();
188    future.get(AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS);
189  }
190}