001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertFalse;
022import static org.junit.jupiter.api.Assertions.assertTrue;
023
024import java.io.IOException;
025import java.util.ArrayList;
026import java.util.Collection;
027import org.apache.hadoop.conf.Configuration;
028import org.apache.hadoop.hbase.HBaseTestingUtil;
029import org.apache.hadoop.hbase.HConstants;
030import org.apache.hadoop.hbase.ServerName;
031import org.apache.hadoop.hbase.TableName;
032import org.apache.hadoop.hbase.client.RegionInfo;
033import org.apache.hadoop.hbase.client.RegionReplicaUtil;
034import org.apache.hadoop.hbase.client.Table;
035import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
036import org.apache.hadoop.hbase.testclassification.MediumTests;
037import org.apache.hadoop.hbase.testclassification.RegionServerTests;
038import org.apache.hadoop.hbase.util.Bytes;
039import org.apache.hadoop.hbase.util.RegionSplitter;
040import org.junit.jupiter.api.AfterAll;
041import org.junit.jupiter.api.AfterEach;
042import org.junit.jupiter.api.BeforeAll;
043import org.junit.jupiter.api.BeforeEach;
044import org.junit.jupiter.api.Tag;
045import org.junit.jupiter.api.Test;
046import org.junit.jupiter.api.TestInfo;
047import org.slf4j.Logger;
048import org.slf4j.LoggerFactory;
049
050@Tag(RegionServerTests.TAG)
051@Tag(MediumTests.TAG)
052public class TestRegionReplicasWithRestartScenarios {
053  private static final Logger LOG =
054    LoggerFactory.getLogger(TestRegionReplicasWithRestartScenarios.class);
055
056  private static final int NB_SERVERS = 3;
057  private Table table;
058  private TableName tableName;
059
060  private static final HBaseTestingUtil HTU = new HBaseTestingUtil();
061  private static final byte[] f = HConstants.CATALOG_FAMILY;
062
063  @BeforeAll
064  public static void beforeClass() throws Exception {
065    HTU.getConfiguration().setInt("hbase.master.wait.on.regionservers.mintostart", NB_SERVERS);
066    HTU.startMiniCluster(NB_SERVERS);
067  }
068
069  @BeforeEach
070  public void before(TestInfo testInfo) throws IOException {
071    this.tableName = TableName.valueOf(testInfo.getTestMethod().get().getName());
072    this.table = createTableDirectlyFromHTD(this.tableName);
073  }
074
075  @AfterEach
076  public void after() throws IOException {
077    this.table.close();
078    HTU.deleteTable(this.tableName);
079  }
080
081  private static Table createTableDirectlyFromHTD(final TableName tableName) throws IOException {
082    TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName);
083    builder.setRegionReplication(3);
084    return HTU.createTable(builder.build(), new byte[][] { f }, getSplits(20),
085      new Configuration(HTU.getConfiguration()));
086  }
087
088  private static byte[][] getSplits(int numRegions) {
089    RegionSplitter.UniformSplit split = new RegionSplitter.UniformSplit();
090    split.setFirstRow(Bytes.toBytes(0L));
091    split.setLastRow(Bytes.toBytes(Long.MAX_VALUE));
092    return split.split(numRegions);
093  }
094
095  @AfterAll
096  public static void afterClass() throws Exception {
097    HTU.shutdownMiniCluster();
098  }
099
100  private HRegionServer getRS() {
101    return HTU.getMiniHBaseCluster().getRegionServer(0);
102  }
103
104  private HRegionServer getSecondaryRS() {
105    return HTU.getMiniHBaseCluster().getRegionServer(1);
106  }
107
108  private HRegionServer getTertiaryRS() {
109    return HTU.getMiniHBaseCluster().getRegionServer(2);
110  }
111
112  @Test
113  public void testRegionReplicasCreated() throws Exception {
114    assertReplicaDistributed();
115  }
116
117  @Test
118  public void testWhenRestart() throws Exception {
119    // Start Region before stopping other so SCP has three servers to play with when it goes
120    // about assigning instead of two, depending on sequencing of SCP and RS stop/start.
121    // If two only, then it'll be forced to assign replicas alongside primaries.
122    HTU.getHBaseCluster().startRegionServerAndWait(60000).getRegionServer();
123    HRegionServer stopRegionServer = getRS();
124    ServerName serverName = stopRegionServer.getServerName();
125    // Make a copy because this is actual instance from HRegionServer
126    Collection<HRegion> regionsOnStoppedServer =
127      new ArrayList<HRegion>(stopRegionServer.getOnlineRegionsLocalContext());
128    HTU.getHBaseCluster().stopRegionServer(serverName);
129    HTU.getHBaseCluster().waitForRegionServerToStop(serverName, 60000);
130    HTU.waitTableAvailable(this.tableName);
131    assertReplicaDistributed(regionsOnStoppedServer);
132  }
133
134  private void assertReplicaDistributed() throws Exception {
135    assertReplicaDistributed(getRS().getOnlineRegionsLocalContext());
136  }
137
138  private void assertReplicaDistributed(Collection<HRegion> onlineRegions) throws Exception {
139    LOG.info("ASSERT DISTRIBUTED {}", onlineRegions);
140    boolean res = checkDuplicates(onlineRegions);
141    assertFalse(res);
142    Collection<HRegion> onlineRegions2 = getSecondaryRS().getOnlineRegionsLocalContext();
143    res = checkDuplicates(onlineRegions2);
144    assertFalse(res);
145    Collection<HRegion> onlineRegions3 = getTertiaryRS().getOnlineRegionsLocalContext();
146    checkDuplicates(onlineRegions3);
147    assertFalse(res);
148    int totalRegions = HTU.getMiniHBaseCluster().getLiveRegionServerThreads().stream()
149      .mapToInt(l -> l.getRegionServer().getOnlineRegions().size()).sum();
150    assertEquals(61, totalRegions);
151  }
152
153  private boolean checkDuplicates(Collection<HRegion> onlineRegions3) throws Exception {
154    ArrayList<Region> copyOfRegion = new ArrayList<Region>(onlineRegions3);
155    for (Region region : copyOfRegion) {
156      RegionInfo regionInfo = region.getRegionInfo();
157      RegionInfo regionInfoForReplica =
158        RegionReplicaUtil.getRegionInfoForDefaultReplica(regionInfo);
159      int i = 0;
160      for (Region actualRegion : onlineRegions3) {
161        if (
162          regionInfoForReplica
163            .equals(RegionReplicaUtil.getRegionInfoForDefaultReplica(actualRegion.getRegionInfo()))
164        ) {
165          i++;
166          if (i > 1) {
167            LOG.warn("Duplicate found {} and {}", actualRegion.getRegionInfo(),
168              region.getRegionInfo());
169            assertTrue(Bytes.equals(region.getRegionInfo().getStartKey(),
170              actualRegion.getRegionInfo().getStartKey()));
171            assertTrue(Bytes.equals(region.getRegionInfo().getEndKey(),
172              actualRegion.getRegionInfo().getEndKey()));
173            return true;
174          }
175        }
176      }
177    }
178    return false;
179  }
180}