001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.balancer;
019
020import static org.junit.Assert.assertNotNull;
021import static org.junit.Assert.assertNull;
022import static org.junit.Assert.assertTrue;
023
024import java.util.ArrayList;
025import java.util.HashMap;
026import java.util.HashSet;
027import java.util.LinkedList;
028import java.util.List;
029import java.util.Map;
030import java.util.Map.Entry;
031import java.util.Queue;
032import java.util.Random;
033import java.util.Set;
034import java.util.SortedSet;
035import java.util.TreeMap;
036import java.util.TreeSet;
037import java.util.concurrent.ThreadLocalRandom;
038import java.util.stream.Collectors;
039import java.util.stream.Stream;
040import org.apache.hadoop.conf.Configuration;
041import org.apache.hadoop.hbase.HBaseConfiguration;
042import org.apache.hadoop.hbase.ServerName;
043import org.apache.hadoop.hbase.TableName;
044import org.apache.hadoop.hbase.client.RegionInfo;
045import org.apache.hadoop.hbase.client.RegionInfoBuilder;
046import org.apache.hadoop.hbase.client.RegionReplicaUtil;
047import org.apache.hadoop.hbase.master.RackManager;
048import org.apache.hadoop.hbase.master.RegionPlan;
049import org.apache.hadoop.hbase.util.Bytes;
050import org.apache.hadoop.net.DNSToSwitchMapping;
051import org.junit.Assert;
052import org.junit.BeforeClass;
053import org.slf4j.Logger;
054import org.slf4j.LoggerFactory;
055
056/**
057 * Class used to be the base of unit tests on load balancers. It gives helper
058 * methods to create maps of {@link ServerName} to lists of {@link RegionInfo}
059 * and to check list of region plans.
060 *
061 */
062public class BalancerTestBase {
063  private static final Logger LOG = LoggerFactory.getLogger(BalancerTestBase.class);
064  static int regionId = 0;
065  protected static Configuration conf;
066  protected static StochasticLoadBalancer loadBalancer;
067
068  @BeforeClass
069  public static void beforeAllTests() throws Exception {
070    conf = HBaseConfiguration.create();
071    conf.setClass("hbase.util.ip.to.rack.determiner", MockMapping.class, DNSToSwitchMapping.class);
072    conf.setFloat("hbase.master.balancer.stochastic.maxMovePercent", 0.75f);
073    conf.setFloat("hbase.regions.slop", 0.0f);
074    conf.setFloat("hbase.master.balancer.stochastic.localityCost", 0);
075    loadBalancer = new StochasticLoadBalancer();
076    loadBalancer.setConf(conf);
077  }
078
079  protected int[] largeCluster = new int[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
080      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
081      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
082      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
083      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
084      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
085      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
086      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
087      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
088      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
089      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
090      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
091      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
092      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56 };
093
094  // int[testnum][servernumber] -> numregions
095  protected int[][] clusterStateMocks = new int[][]{
096      // 1 node
097      new int[]{0},
098      new int[]{1},
099      new int[]{10},
100      // 2 node
101      new int[]{0, 0},
102      new int[]{2, 0},
103      new int[]{2, 1},
104      new int[]{2, 2},
105      new int[]{2, 3},
106      new int[]{2, 4},
107      new int[]{1, 1},
108      new int[]{0, 1},
109      new int[]{10, 1},
110      new int[]{514, 1432},
111      new int[]{48, 53},
112      // 3 node
113      new int[]{0, 1, 2},
114      new int[]{1, 2, 3},
115      new int[]{0, 2, 2},
116      new int[]{0, 3, 0},
117      new int[]{0, 4, 0},
118      new int[]{20, 20, 0},
119      // 4 node
120      new int[]{0, 1, 2, 3},
121      new int[]{4, 0, 0, 0},
122      new int[]{5, 0, 0, 0},
123      new int[]{6, 6, 0, 0},
124      new int[]{6, 2, 0, 0},
125      new int[]{6, 1, 0, 0},
126      new int[]{6, 0, 0, 0},
127      new int[]{4, 4, 4, 7},
128      new int[]{4, 4, 4, 8},
129      new int[]{0, 0, 0, 7},
130      // 5 node
131      new int[]{1, 1, 1, 1, 4},
132      // 6 nodes
133      new int[]{1500, 500, 500, 500, 10, 0},
134      new int[]{1500, 500, 500, 500, 500, 0},
135      // more nodes
136      new int[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},
137      new int[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 10},
138      new int[]{6, 6, 5, 6, 6, 6, 6, 6, 6, 1},
139      new int[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 54},
140      new int[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 55},
141      new int[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 56},
142      new int[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 16},
143      new int[]{1, 1, 1, 1, 1, 1, 1, 1, 1, 8},
144      new int[]{1, 1, 1, 1, 1, 1, 1, 1, 1, 9},
145      new int[]{1, 1, 1, 1, 1, 1, 1, 1, 1, 10},
146      new int[]{1, 1, 1, 1, 1, 1, 1, 1, 1, 123},
147      new int[]{1, 1, 1, 1, 1, 1, 1, 1, 1, 155},
148      new int[]{10, 7, 12, 8, 11, 10, 9, 14},
149      new int[]{13, 14, 6, 10, 10, 10, 8, 10},
150      new int[]{130, 14, 60, 10, 100, 10, 80, 10},
151      new int[]{130, 140, 60, 100, 100, 100, 80, 100},
152      new int[]{0, 5 , 5, 5, 5},
153      largeCluster,
154
155  };
156
157
158  // This class is introduced because IP to rack resolution can be lengthy.
159  public static class MockMapping implements DNSToSwitchMapping {
160    public MockMapping(Configuration conf) {
161    }
162
163    @Override
164    public List<String> resolve(List<String> names) {
165      return Stream.generate(() -> "rack").limit(names.size()).collect(Collectors.toList());
166    }
167
168    // do not add @Override annotations here. It mighty break compilation with earlier Hadoops
169    public void reloadCachedMappings() {
170    }
171
172    // do not add @Override annotations here. It mighty break compilation with earlier Hadoops
173    public void reloadCachedMappings(List<String> arg0) {
174    }
175  }
176
177  /**
178   * Invariant is that all servers have between floor(avg) and ceiling(avg)
179   * number of regions.
180   */
181  public void assertClusterAsBalanced(List<ServerAndLoad> servers) {
182    int numServers = servers.size();
183    int numRegions = 0;
184    int maxRegions = 0;
185    int minRegions = Integer.MAX_VALUE;
186    for (ServerAndLoad server : servers) {
187      int nr = server.getLoad();
188      if (nr > maxRegions) {
189        maxRegions = nr;
190      }
191      if (nr < minRegions) {
192        minRegions = nr;
193      }
194      numRegions += nr;
195    }
196    if (maxRegions - minRegions < 2) {
197      // less than 2 between max and min, can't balance
198      return;
199    }
200    int min = numRegions / numServers;
201    int max = numRegions % numServers == 0 ? min : min + 1;
202
203    for (ServerAndLoad server : servers) {
204      assertTrue(server.getLoad() >= 0);
205      assertTrue(server.getLoad() <= max);
206      assertTrue(server.getLoad() >= min);
207    }
208  }
209
210  /**
211   * Invariant is that all servers have between acceptable range
212   * number of regions.
213   */
214  public boolean assertClusterOverallAsBalanced(List<ServerAndLoad> servers, int tablenum) {
215    int numServers = servers.size();
216    int numRegions = 0;
217    int maxRegions = 0;
218    int minRegions = Integer.MAX_VALUE;
219    for (ServerAndLoad server : servers) {
220      int nr = server.getLoad();
221      if (nr > maxRegions) {
222        maxRegions = nr;
223      }
224      if (nr < minRegions) {
225        minRegions = nr;
226      }
227      numRegions += nr;
228    }
229    if (maxRegions - minRegions < 2) {
230      // less than 2 between max and min, can't balance
231      return true;
232    }
233    int min = numRegions / numServers;
234    int max = numRegions % numServers == 0 ? min : min + 1;
235
236    for (ServerAndLoad server : servers) {
237      if (server.getLoad() < 0 || server.getLoad() > max + tablenum/2 + 1  || server.getLoad() < min - tablenum/2 - 1)
238        return false;
239    }
240    return true;
241  }
242
243  /**
244   * Checks whether region replicas are not hosted on the same host.
245   */
246  public void assertRegionReplicaPlacement(Map<ServerName, List<RegionInfo>> serverMap, RackManager rackManager) {
247    TreeMap<String, Set<RegionInfo>> regionsPerHost = new TreeMap<>();
248    TreeMap<String, Set<RegionInfo>> regionsPerRack = new TreeMap<>();
249
250    for (Entry<ServerName, List<RegionInfo>> entry : serverMap.entrySet()) {
251      String hostname = entry.getKey().getHostname();
252      Set<RegionInfo> infos = regionsPerHost.get(hostname);
253      if (infos == null) {
254        infos = new HashSet<>();
255        regionsPerHost.put(hostname, infos);
256      }
257
258      for (RegionInfo info : entry.getValue()) {
259        RegionInfo primaryInfo = RegionReplicaUtil.getRegionInfoForDefaultReplica(info);
260        if (!infos.add(primaryInfo)) {
261          Assert.fail("Two or more region replicas are hosted on the same host after balance");
262        }
263      }
264    }
265
266    if (rackManager == null) {
267      return;
268    }
269
270    for (Entry<ServerName, List<RegionInfo>> entry : serverMap.entrySet()) {
271      String rack = rackManager.getRack(entry.getKey());
272      Set<RegionInfo> infos = regionsPerRack.get(rack);
273      if (infos == null) {
274        infos = new HashSet<>();
275        regionsPerRack.put(rack, infos);
276      }
277
278      for (RegionInfo info : entry.getValue()) {
279        RegionInfo primaryInfo = RegionReplicaUtil.getRegionInfoForDefaultReplica(info);
280        if (!infos.add(primaryInfo)) {
281          Assert.fail("Two or more region replicas are hosted on the same rack after balance");
282        }
283      }
284    }
285  }
286
287  protected String printStats(List<ServerAndLoad> servers) {
288    int numServers = servers.size();
289    int totalRegions = 0;
290    for (ServerAndLoad server : servers) {
291      totalRegions += server.getLoad();
292    }
293    float average = (float) totalRegions / numServers;
294    int max = (int) Math.ceil(average);
295    int min = (int) Math.floor(average);
296    return "[srvr=" + numServers + " rgns=" + totalRegions + " avg=" + average + " max=" + max
297        + " min=" + min + "]";
298  }
299
300  protected List<ServerAndLoad> convertToList(final Map<ServerName, List<RegionInfo>> servers) {
301    List<ServerAndLoad> list = new ArrayList<>(servers.size());
302    for (Map.Entry<ServerName, List<RegionInfo>> e : servers.entrySet()) {
303      list.add(new ServerAndLoad(e.getKey(), e.getValue().size()));
304    }
305    return list;
306  }
307
308  protected String printMock(List<ServerAndLoad> balancedCluster) {
309    SortedSet<ServerAndLoad> sorted = new TreeSet<>(balancedCluster);
310    ServerAndLoad[] arr = sorted.toArray(new ServerAndLoad[sorted.size()]);
311    StringBuilder sb = new StringBuilder(sorted.size() * 4 + 4);
312    sb.append("{ ");
313    for (int i = 0; i < arr.length; i++) {
314      if (i != 0) {
315        sb.append(" , ");
316      }
317      sb.append(arr[i].getServerName().getHostname());
318      sb.append(":");
319      sb.append(arr[i].getLoad());
320    }
321    sb.append(" }");
322    return sb.toString();
323  }
324
325  /**
326   * This assumes the RegionPlan HSI instances are the same ones in the map, so
327   * actually no need to even pass in the map, but I think it's clearer.
328   *
329   * @param list
330   * @param plans
331   * @return a list of all added {@link ServerAndLoad} values.
332   */
333  protected List<ServerAndLoad> reconcile(List<ServerAndLoad> list,
334                                          List<RegionPlan> plans,
335                                          Map<ServerName, List<RegionInfo>> servers) {
336    List<ServerAndLoad> result = new ArrayList<>(list.size());
337
338    Map<ServerName, ServerAndLoad> map = new HashMap<>(list.size());
339    for (ServerAndLoad sl : list) {
340      map.put(sl.getServerName(), sl);
341    }
342    if (plans != null) {
343      for (RegionPlan plan : plans) {
344        ServerName source = plan.getSource();
345
346        updateLoad(map, source, -1);
347        ServerName destination = plan.getDestination();
348        updateLoad(map, destination, +1);
349
350        servers.get(source).remove(plan.getRegionInfo());
351        servers.get(destination).add(plan.getRegionInfo());
352      }
353    }
354    result.clear();
355    result.addAll(map.values());
356    return result;
357  }
358
359  protected void updateLoad(final Map<ServerName, ServerAndLoad> map,
360                            final ServerName sn,
361                            final int diff) {
362    ServerAndLoad sal = map.get(sn);
363    if (sal == null) sal = new ServerAndLoad(sn, 0);
364    sal = new ServerAndLoad(sn, sal.getLoad() + diff);
365    map.put(sn, sal);
366  }
367
368  protected TreeMap<ServerName, List<RegionInfo>> mockClusterServers(int[] mockCluster) {
369    return mockClusterServers(mockCluster, -1);
370  }
371
372  protected BaseLoadBalancer.Cluster mockCluster(int[] mockCluster) {
373    return new BaseLoadBalancer.Cluster(
374      mockClusterServers(mockCluster, -1), null, null, null);
375  }
376
377  protected TreeMap<ServerName, List<RegionInfo>> mockClusterServers(int[] mockCluster, int numTables) {
378    int numServers = mockCluster.length;
379    TreeMap<ServerName, List<RegionInfo>> servers = new TreeMap<>();
380    for (int i = 0; i < numServers; i++) {
381      int numRegions = mockCluster[i];
382      ServerAndLoad sal = randomServer(0);
383      List<RegionInfo> regions = randomRegions(numRegions, numTables);
384      servers.put(sal.getServerName(), regions);
385    }
386    return servers;
387  }
388
389  protected TreeMap<ServerName, List<RegionInfo>> mockUniformClusterServers(int[] mockCluster) {
390    int numServers = mockCluster.length;
391    TreeMap<ServerName, List<RegionInfo>> servers = new TreeMap<>();
392    for (int i = 0; i < numServers; i++) {
393      int numRegions = mockCluster[i];
394      ServerAndLoad sal = randomServer(0);
395      List<RegionInfo> regions = uniformRegions(numRegions);
396      servers.put(sal.getServerName(), regions);
397    }
398    return servers;
399  }
400
401  protected HashMap<TableName, TreeMap<ServerName, List<RegionInfo>>> mockClusterServersWithTables(Map<ServerName, List<RegionInfo>> clusterServers) {
402    HashMap<TableName, TreeMap<ServerName, List<RegionInfo>>> result = new HashMap<>();
403    for (Map.Entry<ServerName, List<RegionInfo>> entry : clusterServers.entrySet()) {
404      ServerName sal = entry.getKey();
405      List<RegionInfo> regions = entry.getValue();
406      for (RegionInfo hri : regions){
407        TreeMap<ServerName, List<RegionInfo>> servers = result.get(hri.getTable());
408        if (servers == null) {
409          servers = new TreeMap<>();
410          result.put(hri.getTable(), servers);
411        }
412        List<RegionInfo> hrilist = servers.get(sal);
413        if (hrilist == null) {
414          hrilist = new ArrayList<>();
415          servers.put(sal, hrilist);
416        }
417        hrilist.add(hri);
418      }
419    }
420    for(Map.Entry<TableName, TreeMap<ServerName, List<RegionInfo>>> entry : result.entrySet()){
421      for(ServerName srn : clusterServers.keySet()){
422        if (!entry.getValue().containsKey(srn)) entry.getValue().put(srn, new ArrayList<>());
423      }
424    }
425    return result;
426  }
427
428  private Queue<RegionInfo> regionQueue = new LinkedList<>();
429
430  protected List<RegionInfo> randomRegions(int numRegions) {
431    return randomRegions(numRegions, -1);
432  }
433
434  protected List<RegionInfo> randomRegions(int numRegions, int numTables) {
435    List<RegionInfo> regions = new ArrayList<>(numRegions);
436    byte[] start = new byte[16];
437    byte[] end = new byte[16];
438    Random rand = ThreadLocalRandom.current();
439    rand.nextBytes(start);
440    rand.nextBytes(end);
441    for (int i = 0; i < numRegions; i++) {
442      if (!regionQueue.isEmpty()) {
443        regions.add(regionQueue.poll());
444        continue;
445      }
446      Bytes.putInt(start, 0, numRegions << 1);
447      Bytes.putInt(end, 0, (numRegions << 1) + 1);
448      TableName tableName =
449          TableName.valueOf("table" + (numTables > 0 ? rand.nextInt(numTables) : i));
450      RegionInfo hri = RegionInfoBuilder.newBuilder(tableName)
451          .setStartKey(start)
452          .setEndKey(end)
453          .setSplit(false)
454          .setRegionId(regionId++)
455          .build();
456      regions.add(hri);
457    }
458    return regions;
459  }
460
461  protected List<RegionInfo> uniformRegions(int numRegions) {
462    List<RegionInfo> regions = new ArrayList<>(numRegions);
463    byte[] start = new byte[16];
464    byte[] end = new byte[16];
465    Random rand = ThreadLocalRandom.current();
466    rand.nextBytes(start);
467    rand.nextBytes(end);
468    for (int i = 0; i < numRegions; i++) {
469      Bytes.putInt(start, 0, numRegions << 1);
470      Bytes.putInt(end, 0, (numRegions << 1) + 1);
471      TableName tableName =
472              TableName.valueOf("table" + i);
473      RegionInfo hri = RegionInfoBuilder.newBuilder(tableName)
474          .setStartKey(start)
475          .setEndKey(end)
476          .setSplit(false)
477          .build();
478      regions.add(hri);
479    }
480    return regions;
481  }
482
483  protected void returnRegions(List<RegionInfo> regions) {
484    regionQueue.addAll(regions);
485  }
486
487  private Queue<ServerName> serverQueue = new LinkedList<>();
488
489  protected ServerAndLoad randomServer(final int numRegionsPerServer) {
490    if (!this.serverQueue.isEmpty()) {
491      ServerName sn = this.serverQueue.poll();
492      return new ServerAndLoad(sn, numRegionsPerServer);
493    }
494    Random rand = ThreadLocalRandom.current();
495    String host = "srv" + rand.nextInt(Integer.MAX_VALUE);
496    int port = rand.nextInt(60000);
497    long startCode = rand.nextLong();
498    ServerName sn = ServerName.valueOf(host, port, startCode);
499    return new ServerAndLoad(sn, numRegionsPerServer);
500  }
501
502  protected List<ServerAndLoad> randomServers(int numServers, int numRegionsPerServer) {
503    List<ServerAndLoad> servers = new ArrayList<>(numServers);
504    for (int i = 0; i < numServers; i++) {
505      servers.add(randomServer(numRegionsPerServer));
506    }
507    return servers;
508  }
509
510  protected void returnServer(ServerName server) {
511    serverQueue.add(server);
512  }
513
514  protected void returnServers(List<ServerName> servers) {
515    this.serverQueue.addAll(servers);
516  }
517
518  protected void testWithCluster(int numNodes,
519      int numRegions,
520      int numRegionsPerServer,
521      int replication,
522      int numTables,
523      boolean assertFullyBalanced, boolean assertFullyBalancedForReplicas) {
524    Map<ServerName, List<RegionInfo>> serverMap =
525        createServerMap(numNodes, numRegions, numRegionsPerServer, replication, numTables);
526    testWithCluster(serverMap, null, assertFullyBalanced, assertFullyBalancedForReplicas);
527  }
528
529  protected void testWithCluster(Map<ServerName, List<RegionInfo>> serverMap,
530      RackManager rackManager, boolean assertFullyBalanced, boolean assertFullyBalancedForReplicas) {
531    List<ServerAndLoad> list = convertToList(serverMap);
532    LOG.info("Mock Cluster : " + printMock(list) + " " + printStats(list));
533
534    loadBalancer.setRackManager(rackManager);
535    // Run the balancer.
536    List<RegionPlan> plans = loadBalancer.balanceCluster(serverMap);
537    assertNotNull(plans);
538
539    // Check to see that this actually got to a stable place.
540    if (assertFullyBalanced || assertFullyBalancedForReplicas) {
541      // Apply the plan to the mock cluster.
542      List<ServerAndLoad> balancedCluster = reconcile(list, plans, serverMap);
543
544      // Print out the cluster loads to make debugging easier.
545      LOG.info("Mock Balance : " + printMock(balancedCluster));
546
547      if (assertFullyBalanced) {
548        assertClusterAsBalanced(balancedCluster);
549        List<RegionPlan> secondPlans =  loadBalancer.balanceCluster(serverMap);
550        assertNull(secondPlans);
551      }
552
553      if (assertFullyBalancedForReplicas) {
554        assertRegionReplicaPlacement(serverMap, rackManager);
555      }
556    }
557  }
558
559  protected Map<ServerName, List<RegionInfo>> createServerMap(int numNodes,
560                                                             int numRegions,
561                                                             int numRegionsPerServer,
562                                                             int replication,
563                                                             int numTables) {
564    //construct a cluster of numNodes, having  a total of numRegions. Each RS will hold
565    //numRegionsPerServer many regions except for the last one, which will host all the
566    //remaining regions
567    int[] cluster = new int[numNodes];
568    for (int i =0; i < numNodes; i++) {
569      cluster[i] = numRegionsPerServer;
570    }
571    cluster[cluster.length - 1] = numRegions - ((cluster.length - 1) * numRegionsPerServer);
572    Map<ServerName, List<RegionInfo>> clusterState = mockClusterServers(cluster, numTables);
573    if (replication > 0) {
574      // replicate the regions to the same servers
575      for (List<RegionInfo> regions : clusterState.values()) {
576        int length = regions.size();
577        for (int i = 0; i < length; i++) {
578          for (int r = 1; r < replication ; r++) {
579            regions.add(RegionReplicaUtil.getRegionInfoForReplica(regions.get(i), r));
580          }
581        }
582      }
583    }
584
585    return clusterState;
586  }
587
588}