001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.client;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNotNull;
022import static org.junit.Assert.fail;
023
024import java.io.IOException;
025import java.util.ArrayList;
026import java.util.List;
027import java.util.Optional;
028import org.apache.hadoop.conf.Configuration;
029import org.apache.hadoop.hbase.HBaseTestingUtility;
030import org.apache.hadoop.hbase.HRegionLocation;
031import org.apache.hadoop.hbase.NotServingRegionException;
032import org.apache.hadoop.hbase.RegionLocations;
033import org.apache.hadoop.hbase.ServerName;
034import org.apache.hadoop.hbase.TableName;
035import org.apache.hadoop.hbase.Waiter;
036import org.apache.hadoop.hbase.Waiter.ExplainingPredicate;
037import org.apache.hadoop.hbase.regionserver.Region;
038import org.apache.hadoop.hbase.util.JVMClusterUtil;
039
040public final class RegionReplicaTestHelper {
041
042  private RegionReplicaTestHelper() {
043  }
044
045  // waits for all replicas to have region location
046  static void waitUntilAllMetaReplicasAreReady(HBaseTestingUtility util,
047    ConnectionRegistry registry) throws IOException {
048    Configuration conf = util.getConfiguration();
049    int regionReplicaCount =
050      util.getAdmin().getDescriptor(TableName.META_TABLE_NAME).getRegionReplication();
051    Waiter.waitFor(conf, conf.getLong("hbase.client.sync.wait.timeout.msec", 60000), 200, true,
052      new ExplainingPredicate<IOException>() {
053        @Override
054        public String explainFailure() {
055          return "Not all meta replicas get assigned";
056        }
057
058        @Override
059        public boolean evaluate() {
060          try {
061            RegionLocations locs = registry.getMetaRegionLocations().get();
062            if (locs.size() < regionReplicaCount) {
063              return false;
064            }
065            for (int i = 0; i < regionReplicaCount; i++) {
066              HRegionLocation loc = locs.getRegionLocation(i);
067              // Wait until the replica is served by a region server. There could be delay between
068              // the replica being available to the connection and region server opening it.
069              Optional<ServerName> rsCarryingReplica =
070                getRSCarryingReplica(util, loc.getRegion().getTable(), i);
071              if (!rsCarryingReplica.isPresent()) {
072                return false;
073              }
074            }
075            return true;
076          } catch (Exception e) {
077            TestZKConnectionRegistry.LOG.warn("Failed to get meta region locations", e);
078            return false;
079          }
080        }
081      });
082  }
083
084  static Optional<ServerName> getRSCarryingReplica(HBaseTestingUtility util, TableName tableName,
085    int replicaId) {
086    return util.getHBaseCluster().getRegionServerThreads().stream().map(t -> t.getRegionServer())
087      .filter(rs -> rs.getRegions(tableName).stream()
088        .anyMatch(r -> r.getRegionInfo().getReplicaId() == replicaId))
089      .findAny().map(rs -> rs.getServerName());
090  }
091
092  /**
093   * Return the new location.
094   */
095  static ServerName moveRegion(HBaseTestingUtility util, HRegionLocation currentLoc)
096    throws Exception {
097    ServerName serverName = currentLoc.getServerName();
098    RegionInfo regionInfo = currentLoc.getRegion();
099    TableName tableName = regionInfo.getTable();
100    int replicaId = regionInfo.getReplicaId();
101    ServerName newServerName = util.getHBaseCluster().getRegionServerThreads().stream()
102      .map(t -> t.getRegionServer().getServerName()).filter(sn -> !sn.equals(serverName)).findAny()
103      .get();
104    util.getAdmin().move(regionInfo.getEncodedNameAsBytes(), newServerName);
105    util.waitFor(30000, new ExplainingPredicate<Exception>() {
106
107      @Override
108      public boolean evaluate() throws Exception {
109        Optional<ServerName> newServerName = getRSCarryingReplica(util, tableName, replicaId);
110        return newServerName.isPresent() && !newServerName.get().equals(serverName);
111      }
112
113      @Override
114      public String explainFailure() throws Exception {
115        return regionInfo.getRegionNameAsString() + " is still on " + serverName;
116      }
117    });
118    return newServerName;
119  }
120
121  interface Locator {
122    RegionLocations getRegionLocations(TableName tableName, int replicaId, boolean reload)
123      throws Exception;
124
125    void updateCachedLocationOnError(HRegionLocation loc, Throwable error) throws Exception;
126  }
127
128  static void testLocator(HBaseTestingUtility util, TableName tableName, Locator locator)
129    throws Exception {
130    RegionLocations locs =
131      locator.getRegionLocations(tableName, RegionReplicaUtil.DEFAULT_REPLICA_ID, false);
132    assertEquals(3, locs.size());
133    for (int i = 0; i < 3; i++) {
134      HRegionLocation loc = locs.getRegionLocation(i);
135      assertNotNull(loc);
136      ServerName serverName = getRSCarryingReplica(util, tableName, i).get();
137      assertEquals(serverName, loc.getServerName());
138    }
139    ServerName newServerName = moveRegion(util, locs.getDefaultRegionLocation());
140    // The cached location should not be changed
141    assertEquals(locs.getDefaultRegionLocation().getServerName(),
142      locator.getRegionLocations(tableName, RegionReplicaUtil.DEFAULT_REPLICA_ID, false)
143        .getDefaultRegionLocation().getServerName());
144    // should get the new location when reload = true
145    // when meta replica LoadBalance mode is enabled, it may delay a bit.
146    util.waitFor(3000, new ExplainingPredicate<Exception>() {
147      @Override
148      public boolean evaluate() throws Exception {
149        ServerName sn =
150          locator.getRegionLocations(tableName, RegionReplicaUtil.DEFAULT_REPLICA_ID, true)
151            .getDefaultRegionLocation().getServerName();
152        return newServerName.equals(sn);
153      }
154
155      @Override
156      public String explainFailure() throws Exception {
157        return "New location does not show up in meta (replica) region";
158      }
159    });
160
161    assertEquals(newServerName,
162      locator.getRegionLocations(tableName, RegionReplicaUtil.DEFAULT_REPLICA_ID, true)
163        .getDefaultRegionLocation().getServerName());
164    // the cached location should be replaced
165    assertEquals(newServerName,
166      locator.getRegionLocations(tableName, RegionReplicaUtil.DEFAULT_REPLICA_ID, false)
167        .getDefaultRegionLocation().getServerName());
168
169    ServerName newServerName1 = moveRegion(util, locs.getRegionLocation(1));
170    ServerName newServerName2 = moveRegion(util, locs.getRegionLocation(2));
171
172    // The cached location should not be change
173    assertEquals(locs.getRegionLocation(1).getServerName(),
174      locator.getRegionLocations(tableName, 1, false).getRegionLocation(1).getServerName());
175    // clear the cached location for replica 1
176    locator.updateCachedLocationOnError(locs.getRegionLocation(1), new NotServingRegionException());
177    // the cached location for replica 2 should not be changed
178    assertEquals(locs.getRegionLocation(2).getServerName(),
179      locator.getRegionLocations(tableName, 2, false).getRegionLocation(2).getServerName());
180    // should get the new location as we have cleared the old location
181    assertEquals(newServerName1,
182      locator.getRegionLocations(tableName, 1, false).getRegionLocation(1).getServerName());
183    // as we will get the new location for replica 2 at once, we should also get the new location
184    // for replica 2
185    assertEquals(newServerName2,
186      locator.getRegionLocations(tableName, 2, false).getRegionLocation(2).getServerName());
187  }
188
189  public static void assertReplicaDistributed(HBaseTestingUtility util, Table t)
190    throws IOException {
191    if (t.getDescriptor().getRegionReplication() <= 1) {
192      return;
193    }
194    List<RegionInfo> regionInfos = new ArrayList<>();
195    for (JVMClusterUtil.RegionServerThread rs : util.getMiniHBaseCluster()
196      .getRegionServerThreads()) {
197      regionInfos.clear();
198      for (Region r : rs.getRegionServer().getRegions(t.getName())) {
199        if (contains(regionInfos, r.getRegionInfo())) {
200          fail("Replica regions should be assigned to different region servers");
201        } else {
202          regionInfos.add(r.getRegionInfo());
203        }
204      }
205    }
206  }
207
208  private static boolean contains(List<RegionInfo> regionInfos, RegionInfo regionInfo) {
209    for (RegionInfo info : regionInfos) {
210      if (RegionReplicaUtil.isReplicasForSameRegion(info, regionInfo)) {
211        return true;
212      }
213    }
214    return false;
215  }
216}