001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.client;
019
020import static org.junit.Assert.assertFalse;
021import static org.junit.Assert.assertNull;
022import static org.junit.Assert.assertTrue;
023
024import java.io.IOException;
025import java.util.ArrayList;
026import java.util.Collections;
027import java.util.List;
028import org.apache.hadoop.conf.Configuration;
029import org.apache.hadoop.hbase.HBaseClassTestRule;
030import org.apache.hadoop.hbase.HBaseTestingUtility;
031import org.apache.hadoop.hbase.HConstants;
032import org.apache.hadoop.hbase.StartMiniClusterOption;
033import org.apache.hadoop.hbase.master.HMaster;
034import org.apache.hadoop.hbase.master.MasterServices;
035import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
036import org.apache.hadoop.hbase.master.assignment.RegionStateNode;
037import org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure;
038import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
039import org.apache.hadoop.hbase.procedure2.Procedure;
040import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException;
041import org.apache.hadoop.hbase.procedure2.ProcedureYieldException;
042import org.apache.hadoop.hbase.testclassification.MediumTests;
043import org.apache.hadoop.hbase.testclassification.MiscTests;
044import org.junit.AfterClass;
045import org.junit.BeforeClass;
046import org.junit.ClassRule;
047import org.junit.Test;
048import org.junit.experimental.categories.Category;
049
050@Category({ MiscTests.class, MediumTests.class })
051public class TestFailedMetaReplicaAssigment {
052
053  @ClassRule
054  public static final HBaseClassTestRule CLASS_RULE =
055    HBaseClassTestRule.forClass(TestFailedMetaReplicaAssigment.class);
056
057  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
058
059  @BeforeClass
060  public static void setUp() throws Exception {
061    // using our rigged master, to force a failed meta replica assignment when start up master
062    // this test can be removed once we remove the HConstants.META_REPLICAS_NUM config.
063    Configuration conf = TEST_UTIL.getConfiguration();
064    conf.setInt(HConstants.META_REPLICAS_NUM, 3);
065    StartMiniClusterOption option = StartMiniClusterOption.builder().numAlwaysStandByMasters(1)
066      .numMasters(1).numRegionServers(1).masterClass(BrokenMetaReplicaMaster.class).build();
067    TEST_UTIL.startMiniCluster(option);
068  }
069
070  @AfterClass
071  public static void tearDown() throws IOException {
072    TEST_UTIL.shutdownMiniCluster();
073  }
074
075  @Test
076  public void testFailedReplicaAssignment() throws InterruptedException {
077    HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster();
078    // waiting for master to come up
079    TEST_UTIL.waitFor(30000, () -> master.isInitialized());
080
081    AssignmentManager am = master.getAssignmentManager();
082    // showing one of the replicas got assigned
083    RegionInfo metaReplicaHri =
084      RegionReplicaUtil.getRegionInfoForReplica(RegionInfoBuilder.FIRST_META_REGIONINFO, 1);
085    // we use assignAsync so we need to wait a bit
086    TEST_UTIL.waitFor(30000, () -> {
087      RegionStateNode metaReplicaRegionNode =
088        am.getRegionStates().getOrCreateRegionStateNode(metaReplicaHri);
089      return metaReplicaRegionNode.getRegionLocation() != null;
090    });
091    // showing one of the replicas failed to be assigned
092    RegionInfo metaReplicaHri2 =
093      RegionReplicaUtil.getRegionInfoForReplica(RegionInfoBuilder.FIRST_META_REGIONINFO, 2);
094    RegionStateNode metaReplicaRegionNode2 =
095      am.getRegionStates().getOrCreateRegionStateNode(metaReplicaHri2);
096    // wait for several seconds to make sure that it is not assigned
097    for (int i = 0; i < 3; i++) {
098      Thread.sleep(2000);
099      assertNull(metaReplicaRegionNode2.getRegionLocation());
100    }
101
102    // showing master is active and running
103    assertFalse(master.isStopping());
104    assertFalse(master.isStopped());
105    assertTrue(master.isActiveMaster());
106  }
107
108  public static class BrokenTransitRegionStateProcedure extends TransitRegionStateProcedure {
109
110    public BrokenTransitRegionStateProcedure() {
111      super(null, null, null, false, TransitionType.ASSIGN);
112    }
113
114    public BrokenTransitRegionStateProcedure(MasterProcedureEnv env, RegionInfo hri) {
115      super(env, hri, null, false, TransitionType.ASSIGN);
116    }
117
118    @Override
119    protected Procedure[] execute(MasterProcedureEnv env)
120      throws ProcedureSuspendedException, ProcedureYieldException, InterruptedException {
121      throw new ProcedureSuspendedException("Never end procedure!");
122    }
123  }
124
125  public static class BrokenMetaReplicaMaster extends HMaster {
126    public BrokenMetaReplicaMaster(final Configuration conf) throws IOException {
127      super(conf);
128    }
129
130    @Override
131    public AssignmentManager createAssignmentManager(MasterServices master) {
132      return new BrokenMasterMetaAssignmentManager(master);
133    }
134  }
135
136  public static class BrokenMasterMetaAssignmentManager extends AssignmentManager {
137    MasterServices master;
138
139    public BrokenMasterMetaAssignmentManager(final MasterServices master) {
140      super(master);
141      this.master = master;
142    }
143
144    @Override
145    public TransitRegionStateProcedure[] createAssignProcedures(List<RegionInfo> hris) {
146      List<TransitRegionStateProcedure> procs = new ArrayList<>();
147      for (RegionInfo hri : hris) {
148        if (hri.isMetaRegion() && hri.getReplicaId() == 2) {
149          RegionStateNode regionNode = getRegionStates().getOrCreateRegionStateNode(hri);
150          regionNode.lock();
151          try {
152            procs.add(regionNode.setProcedure(new BrokenTransitRegionStateProcedure(
153              master.getMasterProcedureExecutor().getEnvironment(), hri)));
154          } finally {
155            regionNode.unlock();
156          }
157        } else {
158          procs.add(super.createAssignProcedures(Collections.singletonList(hri))[0]);
159        }
160      }
161      return procs.toArray(new TransitRegionStateProcedure[0]);
162    }
163  }
164}