001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.replication;
019
020import java.io.IOException;
021import org.apache.hadoop.hbase.HBaseTestingUtil;
022import org.apache.hadoop.hbase.ProcedureTestUtil;
023import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
024import org.apache.hadoop.hbase.procedure2.Procedure;
025import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
026import org.apache.hadoop.hbase.replication.ReplicationException;
027import org.apache.hadoop.hbase.testclassification.LargeTests;
028import org.apache.hadoop.hbase.testclassification.MasterTests;
029import org.junit.jupiter.api.AfterAll;
030import org.junit.jupiter.api.BeforeAll;
031import org.junit.jupiter.api.Tag;
032import org.junit.jupiter.api.Test;
033
034import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.PeerModificationState;
035
036@Tag(MasterTests.TAG)
037@Tag(LargeTests.TAG)
038public class TestModifyPeerProcedureRetryBackoff {
039
040  private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
041
042  private static boolean FAIL = true;
043
044  public static class TestModifyPeerProcedure extends ModifyPeerProcedure {
045
046    public TestModifyPeerProcedure() {
047    }
048
049    public TestModifyPeerProcedure(String peerId) {
050      super(peerId);
051    }
052
053    @Override
054    public PeerOperationType getPeerOperationType() {
055      return PeerOperationType.ADD;
056    }
057
058    private void tryFail() throws ReplicationException {
059      synchronized (TestModifyPeerProcedureRetryBackoff.class) {
060        if (FAIL) {
061          throw new ReplicationException("Inject error");
062        }
063        FAIL = true;
064      }
065    }
066
067    @Override
068    protected <T extends Procedure<MasterProcedureEnv>> void
069      addChildProcedure(@SuppressWarnings("unchecked") T... subProcedure) {
070      // Make it a no-op
071    }
072
073    @Override
074    protected PeerModificationState nextStateAfterRefresh() {
075      return PeerModificationState.SERIAL_PEER_REOPEN_REGIONS;
076    }
077
078    @Override
079    protected boolean enablePeerBeforeFinish() {
080      return true;
081    }
082
083    @Override
084    protected void updateLastPushedSequenceIdForSerialPeer(MasterProcedureEnv env)
085      throws IOException, ReplicationException {
086      tryFail();
087    }
088
089    @Override
090    protected void reopenRegions(MasterProcedureEnv env) throws IOException {
091      try {
092        tryFail();
093      } catch (ReplicationException e) {
094        throw new IOException(e);
095      }
096    }
097
098    @Override
099    protected void enablePeer(MasterProcedureEnv env) throws ReplicationException {
100      tryFail();
101    }
102
103    @Override
104    protected void prePeerModification(MasterProcedureEnv env)
105      throws IOException, ReplicationException {
106      tryFail();
107    }
108
109    @Override
110    protected void updatePeerStorage(MasterProcedureEnv env) throws ReplicationException {
111      tryFail();
112    }
113
114    @Override
115    protected void postPeerModification(MasterProcedureEnv env)
116      throws IOException, ReplicationException {
117      tryFail();
118    }
119  }
120
121  @BeforeAll
122  public static void setUp() throws Exception {
123    UTIL.startMiniCluster(1);
124  }
125
126  @AfterAll
127  public static void tearDown() throws Exception {
128    UTIL.shutdownMiniCluster();
129  }
130
131  private void assertBackoffIncrease() throws IOException, InterruptedException {
132    ProcedureTestUtil.waitUntilProcedureWaitingTimeout(UTIL, TestModifyPeerProcedure.class, 30000);
133    ProcedureTestUtil.waitUntilProcedureTimeoutIncrease(UTIL, TestModifyPeerProcedure.class, 2);
134    synchronized (TestModifyPeerProcedureRetryBackoff.class) {
135      FAIL = false;
136    }
137    UTIL.waitFor(30000, () -> FAIL);
138  }
139
140  @Test
141  public void test() throws IOException, InterruptedException {
142    ProcedureExecutor<MasterProcedureEnv> procExec =
143      UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor();
144    long procId = procExec.submitProcedure(new TestModifyPeerProcedure("1"));
145    // PRE_PEER_MODIFICATION
146    assertBackoffIncrease();
147    // UPDATE_PEER_STORAGE
148    assertBackoffIncrease();
149    // No retry for REFRESH_PEER_ON_RS
150    // SERIAL_PEER_REOPEN_REGIONS
151    assertBackoffIncrease();
152    // SERIAL_PEER_UPDATE_LAST_PUSHED_SEQ_ID
153    assertBackoffIncrease();
154    // SERIAL_PEER_SET_PEER_ENABLED
155    assertBackoffIncrease();
156    // No retry for SERIAL_PEER_ENABLE_PEER_REFRESH_PEER_ON_RS
157    // POST_PEER_MODIFICATION
158    assertBackoffIncrease();
159    UTIL.waitFor(30000, () -> procExec.isFinished(procId));
160  }
161}