001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.balancer;
019
020import static org.junit.Assert.assertTrue;
021
022import java.util.ArrayDeque;
023import java.util.ArrayList;
024import java.util.HashMap;
025import java.util.HashSet;
026import java.util.List;
027import java.util.Map;
028import java.util.NavigableSet;
029import java.util.Queue;
030import java.util.Random;
031import java.util.Set;
032import java.util.TreeMap;
033import java.util.TreeSet;
034import java.util.concurrent.ThreadLocalRandom;
035import java.util.stream.Collectors;
036import java.util.stream.Stream;
037import org.apache.hadoop.conf.Configuration;
038import org.apache.hadoop.hbase.ServerName;
039import org.apache.hadoop.hbase.TableName;
040import org.apache.hadoop.hbase.client.RegionInfo;
041import org.apache.hadoop.hbase.client.RegionInfoBuilder;
042import org.apache.hadoop.hbase.client.RegionReplicaUtil;
043import org.apache.hadoop.hbase.master.RackManager;
044import org.apache.hadoop.hbase.master.RegionPlan;
045import org.apache.hadoop.hbase.util.Bytes;
046import org.apache.hadoop.net.DNSToSwitchMapping;
047import org.junit.Assert;
048import org.slf4j.Logger;
049import org.slf4j.LoggerFactory;
050
051/**
052 * Class used to be the base of unit tests on load balancers. It gives helper methods to create maps
053 * of {@link ServerName} to lists of {@link RegionInfo} and to check list of region plans.
054 */
055public class BalancerTestBase {
056  private static final Logger LOG = LoggerFactory.getLogger(BalancerTestBase.class);
057  static int regionId = 0;
058  protected static Configuration conf;
059
060  protected int[] largeCluster = new int[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
061    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
062    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
063    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
064    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
065    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
066    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
067    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
068    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
069    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
070    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
071    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
072    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56 };
073
074  // int[testnum][servernumber] -> numregions
075  protected int[][] clusterStateMocks = new int[][] {
076    // 1 node
077    new int[] { 0 }, new int[] { 1 }, new int[] { 10 },
078    // 2 node
079    new int[] { 0, 0 }, new int[] { 2, 0 }, new int[] { 2, 1 }, new int[] { 2, 2 },
080    new int[] { 2, 3 }, new int[] { 2, 4 }, new int[] { 1, 1 }, new int[] { 0, 1 },
081    new int[] { 10, 1 }, new int[] { 514, 1432 }, new int[] { 48, 53 },
082    // 3 node
083    new int[] { 0, 1, 2 }, new int[] { 1, 2, 3 }, new int[] { 0, 2, 2 }, new int[] { 0, 3, 0 },
084    new int[] { 0, 4, 0 }, new int[] { 20, 20, 0 },
085    // 4 node
086    new int[] { 0, 1, 2, 3 }, new int[] { 4, 0, 0, 0 }, new int[] { 5, 0, 0, 0 },
087    new int[] { 6, 6, 0, 0 }, new int[] { 6, 2, 0, 0 }, new int[] { 6, 1, 0, 0 },
088    new int[] { 6, 0, 0, 0 }, new int[] { 4, 4, 4, 7 }, new int[] { 4, 4, 4, 8 },
089    new int[] { 0, 0, 0, 7 },
090    // 5 node
091    new int[] { 1, 1, 1, 1, 4 },
092    // 6 nodes
093    new int[] { 1500, 500, 500, 500, 10, 0 }, new int[] { 1500, 500, 500, 500, 500, 0 },
094    // more nodes
095    new int[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 },
096    new int[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 10 }, new int[] { 6, 6, 5, 6, 6, 6, 6, 6, 6, 1 },
097    new int[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 54 }, new int[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 55 },
098    new int[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 56 }, new int[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 16 },
099    new int[] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 8 }, new int[] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 9 },
100    new int[] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 10 }, new int[] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 123 },
101    new int[] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 155 }, new int[] { 10, 7, 12, 8, 11, 10, 9, 14 },
102    new int[] { 13, 14, 6, 10, 10, 10, 8, 10 }, new int[] { 130, 14, 60, 10, 100, 10, 80, 10 },
103    new int[] { 130, 140, 60, 100, 100, 100, 80, 100 }, new int[] { 0, 5, 5, 5, 5 }, largeCluster,
104
105  };
106
107  // int[testnum][servernumber] -> numregions
108  protected int[][] clusterStateMocksWithNoSlop = new int[][] {
109    // 1 node
110    new int[] { 0 }, new int[] { 1 }, new int[] { 10 },
111    // 2 node
112    new int[] { 0, 0 }, new int[] { 2, 1 }, new int[] { 2, 2 }, new int[] { 2, 3 },
113    new int[] { 1, 1 }, new int[] { 80, 120 }, new int[] { 1428, 1432 },
114    // more nodes
115    new int[] { 100, 90, 120, 90, 110, 100, 90, 120 }, };
116
117  // int[testnum][servernumber] -> numregions
118  protected int[][] clusterStateMocksWithSlop = new int[][] {
119    // 2 node
120    new int[] { 1, 4 }, new int[] { 10, 20 }, new int[] { 80, 123 },
121    // more nodes
122    new int[] { 100, 100, 100, 100, 100, 100, 100, 100, 100, 200 },
123    new int[] { 10, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
124      5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
125      5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
126      5, 5, 5, 5, 5, 5, 5, 5, 5, 5 }, };
127
128  // This class is introduced because IP to rack resolution can be lengthy.
129  public static class MockMapping implements DNSToSwitchMapping {
130    public MockMapping(Configuration conf) {
131    }
132
133    @Override
134    public List<String> resolve(List<String> names) {
135      return Stream.generate(() -> "rack").limit(names.size()).collect(Collectors.toList());
136    }
137
138    @Override
139    public void reloadCachedMappings() {
140    }
141
142    @Override
143    public void reloadCachedMappings(List<String> arg0) {
144    }
145  }
146
147  /**
148   * Invariant is that all servers have between floor(avg) and ceiling(avg) number of regions.
149   */
150  public void assertClusterAsBalanced(List<ServerAndLoad> servers) {
151    int numServers = servers.size();
152    int numRegions = 0;
153    int maxRegions = 0;
154    int minRegions = Integer.MAX_VALUE;
155    for (ServerAndLoad server : servers) {
156      int nr = server.getLoad();
157      if (nr > maxRegions) {
158        maxRegions = nr;
159      }
160      if (nr < minRegions) {
161        minRegions = nr;
162      }
163      numRegions += nr;
164    }
165    if (maxRegions - minRegions < 2) {
166      // less than 2 between max and min, can't balance
167      return;
168    }
169    int min = numRegions / numServers;
170    int max = numRegions % numServers == 0 ? min : min + 1;
171
172    for (ServerAndLoad server : servers) {
173      assertTrue("All servers should have a positive load. " + server, server.getLoad() >= 0);
174      assertTrue("All servers should have load no more than " + max + ". " + server,
175        server.getLoad() <= max);
176      assertTrue("All servers should have load no less than " + min + ". " + server,
177        server.getLoad() >= min);
178    }
179  }
180
181  /**
182   * Invariant is that all servers have between acceptable range number of regions.
183   */
184  public boolean assertClusterOverallAsBalanced(List<ServerAndLoad> servers, int tablenum) {
185    int numServers = servers.size();
186    int numRegions = 0;
187    int maxRegions = 0;
188    int minRegions = Integer.MAX_VALUE;
189    for (ServerAndLoad server : servers) {
190      int nr = server.getLoad();
191      if (nr > maxRegions) {
192        maxRegions = nr;
193      }
194      if (nr < minRegions) {
195        minRegions = nr;
196      }
197      numRegions += nr;
198    }
199    if (maxRegions - minRegions < 2) {
200      // less than 2 between max and min, can't balance
201      return true;
202    }
203    int min = numRegions / numServers;
204    int max = numRegions % numServers == 0 ? min : min + 1;
205
206    for (ServerAndLoad server : servers) {
207      // The '5' in below is arbitrary.
208      if (
209        server.getLoad() < 0 || server.getLoad() > max + (tablenum / 2 + 5)
210          || server.getLoad() < (min - tablenum / 2 - 5)
211      ) {
212        LOG.warn("server={}, load={}, max={}, tablenum={}, min={}", server.getServerName(),
213          server.getLoad(), max, tablenum, min);
214        return false;
215      }
216    }
217    return true;
218  }
219
220  /**
221   * Checks whether region replicas are not hosted on the same host.
222   */
223  public void assertRegionReplicaPlacement(Map<ServerName, List<RegionInfo>> serverMap,
224    RackManager rackManager) {
225    TreeMap<String, Set<RegionInfo>> regionsPerHost = new TreeMap<>();
226    TreeMap<String, Set<RegionInfo>> regionsPerRack = new TreeMap<>();
227
228    for (Map.Entry<ServerName, List<RegionInfo>> entry : serverMap.entrySet()) {
229      String hostname = entry.getKey().getHostname();
230      Set<RegionInfo> infos = regionsPerHost.get(hostname);
231      if (infos == null) {
232        infos = new HashSet<>();
233        regionsPerHost.put(hostname, infos);
234      }
235
236      for (RegionInfo info : entry.getValue()) {
237        RegionInfo primaryInfo = RegionReplicaUtil.getRegionInfoForDefaultReplica(info);
238        if (!infos.add(primaryInfo)) {
239          Assert.fail("Two or more region replicas are hosted on the same host after balance");
240        }
241      }
242    }
243
244    if (rackManager == null) {
245      return;
246    }
247
248    for (Map.Entry<ServerName, List<RegionInfo>> entry : serverMap.entrySet()) {
249      String rack = rackManager.getRack(entry.getKey());
250      Set<RegionInfo> infos = regionsPerRack.get(rack);
251      if (infos == null) {
252        infos = new HashSet<>();
253        regionsPerRack.put(rack, infos);
254      }
255
256      for (RegionInfo info : entry.getValue()) {
257        RegionInfo primaryInfo = RegionReplicaUtil.getRegionInfoForDefaultReplica(info);
258        if (!infos.add(primaryInfo)) {
259          Assert.fail("Two or more region replicas are hosted on the same rack after balance");
260        }
261      }
262    }
263  }
264
265  protected String printStats(List<ServerAndLoad> servers) {
266    int numServers = servers.size();
267    int totalRegions = 0;
268    for (ServerAndLoad server : servers) {
269      totalRegions += server.getLoad();
270    }
271    float average = (float) totalRegions / numServers;
272    int max = (int) Math.ceil(average);
273    int min = (int) Math.floor(average);
274    return "[srvr=" + numServers + " rgns=" + totalRegions + " avg=" + average + " max=" + max
275      + " min=" + min + "]";
276  }
277
278  protected List<ServerAndLoad> convertToList(final Map<ServerName, List<RegionInfo>> servers) {
279    List<ServerAndLoad> list = new ArrayList<>(servers.size());
280    for (Map.Entry<ServerName, List<RegionInfo>> e : servers.entrySet()) {
281      list.add(new ServerAndLoad(e.getKey(), e.getValue().size()));
282    }
283    return list;
284  }
285
286  protected String printMock(List<ServerAndLoad> balancedCluster) {
287    NavigableSet<ServerAndLoad> sorted = new TreeSet<>(balancedCluster);
288    ServerAndLoad[] arr = sorted.toArray(new ServerAndLoad[sorted.size()]);
289    StringBuilder sb = new StringBuilder(sorted.size() * 4 + 4);
290    sb.append("{ ");
291    for (int i = 0; i < arr.length; i++) {
292      if (i != 0) {
293        sb.append(" , ");
294      }
295      sb.append(arr[i].getServerName().getHostname());
296      sb.append(":");
297      sb.append(arr[i].getLoad());
298    }
299    sb.append(" }");
300    return sb.toString();
301  }
302
303  /**
304   * This assumes the RegionPlan HSI instances are the same ones in the map, so actually no need to
305   * even pass in the map, but I think it's clearer.
306   * @return a list of all added {@link ServerAndLoad} values.
307   */
308  protected List<ServerAndLoad> reconcile(List<ServerAndLoad> list, List<RegionPlan> plans,
309    Map<ServerName, List<RegionInfo>> servers) {
310    List<ServerAndLoad> result = new ArrayList<>(list.size());
311
312    Map<ServerName, ServerAndLoad> map = new HashMap<>(list.size());
313    for (ServerAndLoad sl : list) {
314      map.put(sl.getServerName(), sl);
315    }
316    if (plans != null) {
317      for (RegionPlan plan : plans) {
318        ServerName source = plan.getSource();
319
320        updateLoad(map, source, -1);
321        ServerName destination = plan.getDestination();
322        updateLoad(map, destination, +1);
323
324        servers.get(source).remove(plan.getRegionInfo());
325        servers.get(destination).add(plan.getRegionInfo());
326      }
327    }
328    result.clear();
329    result.addAll(map.values());
330    return result;
331  }
332
333  protected void updateLoad(final Map<ServerName, ServerAndLoad> map, final ServerName sn,
334    final int diff) {
335    ServerAndLoad sal = map.get(sn);
336    if (sal == null) sal = new ServerAndLoad(sn, 0);
337    sal = new ServerAndLoad(sn, sal.getLoad() + diff);
338    map.put(sn, sal);
339  }
340
341  protected TreeMap<ServerName, List<RegionInfo>> mockClusterServers(int[][] mockCluster) {
342    // dimension1: table, dimension2: regions per server
343    int numTables = mockCluster.length;
344    TreeMap<ServerName, List<RegionInfo>> servers = new TreeMap<>();
345    for (int i = 0; i < numTables; i++) {
346      TableName tableName = TableName.valueOf("table" + i);
347      for (int j = 0; j < mockCluster[i].length; j++) {
348        ServerName serverName = ServerName.valueOf("server" + j, 1000, -1);
349        int numRegions = mockCluster[i][j];
350        List<RegionInfo> regions = createRegions(numRegions, tableName);
351        servers.putIfAbsent(serverName, new ArrayList<>());
352        servers.get(serverName).addAll(regions);
353      }
354    }
355    return servers;
356  }
357
358  protected TreeMap<ServerName, List<RegionInfo>> mockClusterServers(int[] mockCluster) {
359    return mockClusterServers(mockCluster, -1);
360  }
361
362  protected BalancerClusterState mockCluster(int[] mockCluster) {
363    return new BalancerClusterState(mockClusterServers(mockCluster, -1), null, null, null);
364  }
365
366  protected TreeMap<ServerName, List<RegionInfo>> mockClusterServers(int[] mockCluster,
367    int numTables) {
368    int numServers = mockCluster.length;
369    TreeMap<ServerName, List<RegionInfo>> servers = new TreeMap<>();
370    for (int i = 0; i < numServers; i++) {
371      int numRegions = mockCluster[i];
372      ServerAndLoad sal = randomServer(0);
373      List<RegionInfo> regions = randomRegions(numRegions, numTables);
374      servers.put(sal.getServerName(), regions);
375    }
376    return servers;
377  }
378
379  protected TreeMap<ServerName, List<RegionInfo>> mockUniformClusterServers(int[] mockCluster) {
380    int numServers = mockCluster.length;
381    TreeMap<ServerName, List<RegionInfo>> servers = new TreeMap<>();
382    for (int i = 0; i < numServers; i++) {
383      int numRegions = mockCluster[i];
384      ServerAndLoad sal = randomServer(0);
385      List<RegionInfo> regions = uniformRegions(numRegions);
386      servers.put(sal.getServerName(), regions);
387    }
388    return servers;
389  }
390
391  protected HashMap<TableName, TreeMap<ServerName, List<RegionInfo>>>
392    mockClusterServersWithTables(Map<ServerName, List<RegionInfo>> clusterServers) {
393    HashMap<TableName, TreeMap<ServerName, List<RegionInfo>>> result = new HashMap<>();
394    for (Map.Entry<ServerName, List<RegionInfo>> entry : clusterServers.entrySet()) {
395      ServerName sal = entry.getKey();
396      List<RegionInfo> regions = entry.getValue();
397      for (RegionInfo hri : regions) {
398        TreeMap<ServerName, List<RegionInfo>> servers = result.get(hri.getTable());
399        if (servers == null) {
400          servers = new TreeMap<>();
401          result.put(hri.getTable(), servers);
402        }
403        List<RegionInfo> hrilist = servers.get(sal);
404        if (hrilist == null) {
405          hrilist = new ArrayList<>();
406          servers.put(sal, hrilist);
407        }
408        hrilist.add(hri);
409      }
410    }
411    for (Map.Entry<TableName, TreeMap<ServerName, List<RegionInfo>>> entry : result.entrySet()) {
412      for (ServerName srn : clusterServers.keySet()) {
413        if (!entry.getValue().containsKey(srn)) entry.getValue().put(srn, new ArrayList<>());
414      }
415    }
416    return result;
417  }
418
419  private Queue<RegionInfo> regionQueue = new ArrayDeque<>();
420
421  protected List<RegionInfo> randomRegions(int numRegions) {
422    return randomRegions(numRegions, -1);
423  }
424
425  protected List<RegionInfo> createRegions(int numRegions, TableName tableName) {
426    List<RegionInfo> regions = new ArrayList<>(numRegions);
427    byte[] start = new byte[16];
428    Bytes.random(start);
429    byte[] end = new byte[16];
430    Bytes.random(end);
431    for (int i = 0; i < numRegions; i++) {
432      Bytes.putInt(start, 0, numRegions << 1);
433      Bytes.putInt(end, 0, (numRegions << 1) + 1);
434      RegionInfo hri = RegionInfoBuilder.newBuilder(tableName).setStartKey(start).setEndKey(end)
435        .setSplit(false).build();
436      regions.add(hri);
437    }
438    return regions;
439  }
440
441  protected List<RegionInfo> randomRegions(int numRegions, int numTables) {
442    List<RegionInfo> regions = new ArrayList<>(numRegions);
443    byte[] start = new byte[16];
444    Bytes.random(start);
445    byte[] end = new byte[16];
446    Bytes.random(end);
447    for (int i = 0; i < numRegions; i++) {
448      if (!regionQueue.isEmpty()) {
449        regions.add(regionQueue.poll());
450        continue;
451      }
452      Bytes.putInt(start, 0, numRegions << 1);
453      Bytes.putInt(end, 0, (numRegions << 1) + 1);
454      TableName tableName = TableName
455        .valueOf("table" + (numTables > 0 ? ThreadLocalRandom.current().nextInt(numTables) : i));
456      RegionInfo hri = RegionInfoBuilder.newBuilder(tableName).setStartKey(start).setEndKey(end)
457        .setSplit(false).setRegionId(regionId++).build();
458      regions.add(hri);
459    }
460    return regions;
461  }
462
463  protected List<RegionInfo> uniformRegions(int numRegions) {
464    List<RegionInfo> regions = new ArrayList<>(numRegions);
465    byte[] start = new byte[16];
466    Bytes.random(start);
467    byte[] end = new byte[16];
468    Bytes.random(end);
469    for (int i = 0; i < numRegions; i++) {
470      Bytes.putInt(start, 0, numRegions << 1);
471      Bytes.putInt(end, 0, (numRegions << 1) + 1);
472      TableName tableName = TableName.valueOf("table" + i);
473      RegionInfo hri = RegionInfoBuilder.newBuilder(tableName).setStartKey(start).setEndKey(end)
474        .setSplit(false).build();
475      regions.add(hri);
476    }
477    return regions;
478  }
479
480  protected void returnRegions(List<RegionInfo> regions) {
481    regionQueue.addAll(regions);
482  }
483
484  private Queue<ServerName> serverQueue = new ArrayDeque<>();
485
486  protected ServerAndLoad randomServer(final int numRegionsPerServer) {
487    if (!this.serverQueue.isEmpty()) {
488      ServerName sn = this.serverQueue.poll();
489      return new ServerAndLoad(sn, numRegionsPerServer);
490    }
491    Random rand = ThreadLocalRandom.current();
492    String host = "srv" + rand.nextInt(Integer.MAX_VALUE);
493    int port = rand.nextInt(60000);
494    long startCode = rand.nextLong();
495    ServerName sn = ServerName.valueOf(host, port, startCode);
496    return new ServerAndLoad(sn, numRegionsPerServer);
497  }
498
499  protected List<ServerAndLoad> randomServers(int numServers, int numRegionsPerServer) {
500    List<ServerAndLoad> servers = new ArrayList<>(numServers);
501    for (int i = 0; i < numServers; i++) {
502      servers.add(randomServer(numRegionsPerServer));
503    }
504    return servers;
505  }
506
507  protected void returnServer(ServerName server) {
508    serverQueue.add(server);
509  }
510
511  protected void returnServers(List<ServerName> servers) {
512    this.serverQueue.addAll(servers);
513  }
514
515  protected Map<ServerName, List<RegionInfo>> createServerMap(int numNodes, int numRegions,
516    int numRegionsPerServer, int replication, int numTables) {
517    // construct a cluster of numNodes, having a total of numRegions. Each RS will hold
518    // numRegionsPerServer many regions except for the last one, which will host all the
519    // remaining regions
520    int[] cluster = new int[numNodes];
521    for (int i = 0; i < numNodes; i++) {
522      cluster[i] = numRegionsPerServer;
523    }
524    cluster[cluster.length - 1] = numRegions - ((cluster.length - 1) * numRegionsPerServer);
525    Map<ServerName, List<RegionInfo>> clusterState = mockClusterServers(cluster, numTables);
526    if (replication > 0) {
527      // replicate the regions to the same servers
528      for (List<RegionInfo> regions : clusterState.values()) {
529        int length = regions.size();
530        for (int i = 0; i < length; i++) {
531          for (int r = 1; r < replication; r++) {
532            regions.add(RegionReplicaUtil.getRegionInfoForReplica(regions.get(i), r));
533          }
534        }
535      }
536    }
537
538    return clusterState;
539  }
540}