001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.procedure;
019
020import static org.junit.jupiter.api.Assertions.assertTrue;
021
022import java.io.IOException;
023import org.apache.hadoop.hbase.HBaseTestingUtil;
024import org.apache.hadoop.hbase.TableName;
025import org.apache.hadoop.hbase.client.RegionInfo;
026import org.apache.hadoop.hbase.master.RegionState.State;
027import org.apache.hadoop.hbase.master.ServerManager;
028import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
029import org.apache.hadoop.hbase.master.assignment.RegionStateNode;
030import org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure;
031import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
032import org.apache.hadoop.hbase.testclassification.MasterTests;
033import org.apache.hadoop.hbase.testclassification.MediumTests;
034import org.apache.hadoop.hbase.util.Bytes;
035import org.junit.jupiter.api.AfterAll;
036import org.junit.jupiter.api.BeforeAll;
037import org.junit.jupiter.api.Tag;
038import org.junit.jupiter.api.Test;
039import org.slf4j.Logger;
040import org.slf4j.LoggerFactory;
041
042import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureState;
043
044/**
045 * Confirm that we will do backoff when retrying on reopening table regions, to avoid consuming all
046 * the CPUs.
047 */
048@Tag(MasterTests.TAG)
049@Tag(MediumTests.TAG)
050public class TestReopenTableRegionsProcedureBackoff {
051
052  private static final Logger LOG =
053    LoggerFactory.getLogger(TestReopenTableRegionsProcedureBackoff.class);
054
055  private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
056
057  private static TableName TABLE_NAME = TableName.valueOf("Backoff");
058
059  private static byte[] CF = Bytes.toBytes("cf");
060
061  @BeforeAll
062  public static void setUp() throws Exception {
063    UTIL.getConfiguration().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 1);
064    UTIL.startMiniCluster(1);
065    UTIL.createTable(TABLE_NAME, CF);
066  }
067
068  @AfterAll
069  public static void tearDown() throws Exception {
070    UTIL.shutdownMiniCluster();
071  }
072
073  @Test
074  public void testRetryBackoff() throws IOException, InterruptedException {
075    AssignmentManager am = UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager();
076    ProcedureExecutor<MasterProcedureEnv> procExec =
077      UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor();
078    RegionInfo regionInfo = UTIL.getAdmin().getRegions(TABLE_NAME).get(0);
079    RegionStateNode regionNode = am.getRegionStates().getRegionStateNode(regionInfo);
080    // just a dummy one
081    TransitRegionStateProcedure trsp =
082      TransitRegionStateProcedure.unassign(procExec.getEnvironment(), regionInfo);
083    long openSeqNum;
084    regionNode.lock();
085    try {
086      openSeqNum = regionNode.getOpenSeqNum();
087      // make a fake state to let the procedure wait.
088      regionNode.setState(State.OPENING);
089      regionNode.setOpenSeqNum(-1L);
090      regionNode.setProcedure(trsp);
091    } finally {
092      regionNode.unlock();
093    }
094    ReopenTableRegionsProcedure proc = new ReopenTableRegionsProcedure(TABLE_NAME);
095    procExec.submitProcedure(proc);
096    UTIL.waitFor(10000, () -> proc.getState() == ProcedureState.WAITING_TIMEOUT);
097    long oldTimeout = 0;
098    int timeoutIncrements = 0;
099    for (;;) {
100      long timeout = proc.getTimeout();
101      if (timeout > oldTimeout) {
102        LOG.info("Timeout incremented, was {}, now is {}, increments={}", timeout, oldTimeout,
103          timeoutIncrements);
104        oldTimeout = timeout;
105        timeoutIncrements++;
106        if (timeoutIncrements > 3) {
107          // If we incremented at least twice, break; the backoff is working.
108          break;
109        }
110      }
111      Thread.sleep(1000);
112    }
113    regionNode.lock();
114    try {
115      // reset to the correct state
116      regionNode.setState(State.OPEN);
117      regionNode.setOpenSeqNum(openSeqNum);
118      regionNode.unsetProcedure(trsp);
119    } finally {
120      regionNode.unlock();
121    }
122    ProcedureSyncWait.waitForProcedureToComplete(procExec, proc, 60000);
123    assertTrue(regionNode.getOpenSeqNum() > openSeqNum);
124  }
125}