001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master;
019
020import static org.junit.jupiter.api.Assertions.assertFalse;
021
022import java.util.concurrent.CountDownLatch;
023import org.apache.hadoop.hbase.HBaseTestingUtil;
024import org.apache.hadoop.hbase.TableName;
025import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
026import org.apache.hadoop.hbase.master.procedure.TableProcedureInterface;
027import org.apache.hadoop.hbase.procedure2.Procedure;
028import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
029import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException;
030import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.NoopProcedure;
031import org.apache.hadoop.hbase.procedure2.ProcedureYieldException;
032import org.apache.hadoop.hbase.testclassification.MasterTests;
033import org.apache.hadoop.hbase.testclassification.MediumTests;
034import org.junit.jupiter.api.AfterAll;
035import org.junit.jupiter.api.BeforeAll;
036import org.junit.jupiter.api.Tag;
037import org.junit.jupiter.api.Test;
038
039import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureState;
040
041/**
042 * Testcase for HBASE-21490.
043 */
044@Tag(MasterTests.TAG)
045@Tag(MediumTests.TAG)
046public class TestLoadProcedureError {
047
048  private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
049
050  private static TableName NAME = TableName.valueOf("Load");
051
052  private static volatile CountDownLatch ARRIVE;
053
054  private static volatile boolean FINISH_PROC;
055
056  private static volatile boolean FAIL_LOAD;
057
058  public static final class TestProcedure extends NoopProcedure<MasterProcedureEnv>
059    implements TableProcedureInterface {
060
061    @Override
062    protected Procedure<MasterProcedureEnv>[] execute(MasterProcedureEnv env)
063      throws ProcedureYieldException, ProcedureSuspendedException, InterruptedException {
064      if (ARRIVE != null) {
065        ARRIVE.countDown();
066        ARRIVE = null;
067      }
068      if (FINISH_PROC) {
069        return null;
070      }
071      setTimeout(1000);
072      setState(ProcedureState.WAITING_TIMEOUT);
073      throw new ProcedureSuspendedException();
074    }
075
076    @Override
077    protected synchronized boolean setTimeoutFailure(MasterProcedureEnv env) {
078      setState(ProcedureState.RUNNABLE);
079      env.getProcedureScheduler().addBack(this);
080      return false;
081    }
082
083    @Override
084    protected void afterReplay(MasterProcedureEnv env) {
085      if (FAIL_LOAD) {
086        throw new RuntimeException("Inject error");
087      }
088    }
089
090    @Override
091    public TableName getTableName() {
092      return NAME;
093    }
094
095    @Override
096    public TableOperationType getTableOperationType() {
097      return TableOperationType.READ;
098    }
099  }
100
101  @BeforeAll
102  public static void setUp() throws Exception {
103    UTIL.startMiniCluster(1);
104  }
105
106  @AfterAll
107  public static void tearDown() throws Exception {
108    UTIL.shutdownMiniCluster();
109  }
110
111  private void waitNoMaster() {
112    UTIL.waitFor(30000, () -> UTIL.getMiniHBaseCluster().getLiveMasterThreads().isEmpty());
113  }
114
115  @Test
116  public void testLoadError() throws Exception {
117    ProcedureExecutor<MasterProcedureEnv> procExec =
118      UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor();
119    ARRIVE = new CountDownLatch(1);
120    long procId = procExec.submitProcedure(new TestProcedure());
121    ARRIVE.await();
122    FAIL_LOAD = true;
123    // do not persist the store tracker
124    UTIL.getMiniHBaseCluster().getMaster().getProcedureStore().stop(true);
125    UTIL.getMiniHBaseCluster().getMaster().abort("for testing");
126    waitNoMaster();
127    // restart twice, and should fail twice, as we will throw an exception in the afterReplay above
128    // in order to reproduce the problem in HBASE-21490 stably, here we will wait until a master is
129    // fully done, before starting the new master, otherwise the new master may start too early and
130    // call recoverLease on the proc wal files and cause we fail to persist the store tracker when
131    // shutting down
132    UTIL.getMiniHBaseCluster().startMaster();
133    waitNoMaster();
134    UTIL.getMiniHBaseCluster().startMaster();
135    waitNoMaster();
136    FAIL_LOAD = false;
137    HMaster master = UTIL.getMiniHBaseCluster().startMaster().getMaster();
138    UTIL.waitFor(30000, () -> master.isActiveMaster() && master.isInitialized());
139    // assert the procedure is still there and not finished yet
140    TestProcedure proc = (TestProcedure) master.getMasterProcedureExecutor().getProcedure(procId);
141    assertFalse(proc.isFinished());
142    FINISH_PROC = true;
143    UTIL.waitFor(30000, () -> proc.isFinished());
144  }
145}