001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.balancer;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertFalse;
022import static org.junit.jupiter.api.Assertions.assertTrue;
023import static org.mockito.Mockito.any;
024import static org.mockito.Mockito.mock;
025import static org.mockito.Mockito.when;
026
027import java.io.FileNotFoundException;
028import java.io.IOException;
029import java.util.ArrayList;
030import java.util.HashMap;
031import java.util.List;
032import java.util.Map;
033import java.util.Optional;
034import java.util.Random;
035import java.util.Set;
036import java.util.TreeMap;
037import java.util.TreeSet;
038import java.util.concurrent.ThreadLocalRandom;
039import org.apache.commons.lang3.StringUtils;
040import org.apache.hadoop.conf.Configuration;
041import org.apache.hadoop.hbase.HBaseConfiguration;
042import org.apache.hadoop.hbase.RegionMetrics;
043import org.apache.hadoop.hbase.ServerMetrics;
044import org.apache.hadoop.hbase.ServerName;
045import org.apache.hadoop.hbase.Size;
046import org.apache.hadoop.hbase.TableDescriptors;
047import org.apache.hadoop.hbase.TableName;
048import org.apache.hadoop.hbase.client.RegionInfo;
049import org.apache.hadoop.hbase.client.RegionInfoBuilder;
050import org.apache.hadoop.hbase.client.TableDescriptor;
051import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
052import org.apache.hadoop.hbase.master.HMaster;
053import org.apache.hadoop.hbase.master.MasterServices;
054import org.apache.hadoop.hbase.master.RegionPlan;
055import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
056import org.apache.hadoop.hbase.net.Address;
057import org.apache.hadoop.hbase.rsgroup.RSGroupInfo;
058import org.apache.hadoop.hbase.rsgroup.RSGroupInfoManager;
059import org.apache.hadoop.hbase.util.Bytes;
060
061import org.apache.hbase.thirdparty.com.google.common.collect.ArrayListMultimap;
062import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
063
064/**
065 * Base UT of RSGroupableBalancer.
066 */
067public class RSGroupableBalancerTestBase extends BalancerTestBase {
068
069  static String[] groups = new String[] { RSGroupInfo.DEFAULT_GROUP, "dg2", "dg3", "dg4" };
070  static TableName table0 = TableName.valueOf("dt0");
071  static TableName[] tables = new TableName[] { TableName.valueOf("dt1"), TableName.valueOf("dt2"),
072    TableName.valueOf("dt3"), TableName.valueOf("dt4") };
073  static List<ServerName> servers;
074  static Map<String, RSGroupInfo> groupMap;
075  static Map<TableName, TableDescriptor> tableDescs;
076  int[] regionAssignment = new int[] { 2, 5, 7, 10, 4, 3, 1 };
077  static int regionId = 0;
078  static Configuration conf = HBaseConfiguration.create();
079
080  /**
081   * Invariant is that all servers of a group have load between floor(avg) and ceiling(avg) number
082   * of regions.
083   */
084  protected void assertClusterAsBalanced(ArrayListMultimap<String, ServerAndLoad> groupLoadMap) {
085    for (String gName : groupLoadMap.keySet()) {
086      List<ServerAndLoad> groupLoad = groupLoadMap.get(gName);
087      int numServers = groupLoad.size();
088      int numRegions = 0;
089      int maxRegions = 0;
090      int minRegions = Integer.MAX_VALUE;
091      for (ServerAndLoad server : groupLoad) {
092        int nr = server.getLoad();
093        if (nr > maxRegions) {
094          maxRegions = nr;
095        }
096        if (nr < minRegions) {
097          minRegions = nr;
098        }
099        numRegions += nr;
100      }
101      if (maxRegions - minRegions < 2) {
102        // less than 2 between max and min, can't balance
103        return;
104      }
105      int min = numRegions / numServers;
106      int max = numRegions % numServers == 0 ? min : min + 1;
107
108      for (ServerAndLoad server : groupLoad) {
109        assertTrue(server.getLoad() <= max);
110        assertTrue(server.getLoad() >= min);
111      }
112    }
113  }
114
115  /**
116   * All regions have an assignment.
117   */
118  protected void assertImmediateAssignment(List<RegionInfo> regions, List<ServerName> servers,
119    Map<RegionInfo, ServerName> assignments) throws IOException {
120    for (RegionInfo region : regions) {
121      assertTrue(assignments.containsKey(region));
122      ServerName server = assignments.get(region);
123      TableName tableName = region.getTable();
124
125      String groupName =
126        tableDescs.get(tableName).getRegionServerGroup().orElse(RSGroupInfo.DEFAULT_GROUP);
127      assertTrue(StringUtils.isNotEmpty(groupName));
128      RSGroupInfo gInfo = getMockedGroupInfoManager().getRSGroup(groupName);
129      assertTrue(gInfo.containsServer(server.getAddress()),
130        "Region is not correctly assigned to group servers.");
131    }
132  }
133
134  /**
135   * Asserts a valid retained assignment plan.
136   * <p>
137   * Must meet the following conditions:
138   * <ul>
139   * <li>Every input region has an assignment, and to an online server
140   * <li>If a region had an existing assignment to a server with the same address a a currently
141   * online server, it will be assigned to it
142   * </ul>
143   */
144  protected void assertRetainedAssignment(Map<RegionInfo, ServerName> existing,
145    List<ServerName> servers, Map<ServerName, List<RegionInfo>> assignment)
146    throws FileNotFoundException, IOException {
147    // Verify condition 1, every region assigned, and to online server
148    Set<ServerName> onlineServerSet = new TreeSet<>(servers);
149    Set<RegionInfo> assignedRegions = new TreeSet<>(RegionInfo.COMPARATOR);
150    for (Map.Entry<ServerName, List<RegionInfo>> a : assignment.entrySet()) {
151      assertTrue(onlineServerSet.contains(a.getKey()),
152        "Region assigned to server that was not listed as online");
153      for (RegionInfo r : a.getValue()) {
154        assignedRegions.add(r);
155      }
156    }
157    assertEquals(existing.size(), assignedRegions.size());
158
159    // Verify condition 2, every region must be assigned to correct server.
160    Set<String> onlineHostNames = new TreeSet<>();
161    for (ServerName s : servers) {
162      onlineHostNames.add(s.getHostname());
163    }
164
165    for (Map.Entry<ServerName, List<RegionInfo>> a : assignment.entrySet()) {
166      ServerName currentServer = a.getKey();
167      for (RegionInfo r : a.getValue()) {
168        ServerName oldAssignedServer = existing.get(r);
169        TableName tableName = r.getTable();
170        String groupName =
171          tableDescs.get(tableName).getRegionServerGroup().orElse(RSGroupInfo.DEFAULT_GROUP);
172        assertTrue(StringUtils.isNotEmpty(groupName));
173        RSGroupInfo gInfo = getMockedGroupInfoManager().getRSGroup(groupName);
174        assertTrue(gInfo.containsServer(currentServer.getAddress()),
175          "Region is not correctly assigned to group servers.");
176        if (
177          oldAssignedServer != null && onlineHostNames.contains(oldAssignedServer.getHostname())
178        ) {
179          // this region was previously assigned somewhere, and that
180          // host is still around, then the host must have been is a
181          // different group.
182          if (!oldAssignedServer.getAddress().equals(currentServer.getAddress())) {
183            assertFalse(gInfo.containsServer(oldAssignedServer.getAddress()));
184          }
185        }
186      }
187    }
188  }
189
190  protected String printStats(ArrayListMultimap<String, ServerAndLoad> groupBasedLoad) {
191    StringBuilder sb = new StringBuilder();
192    sb.append("\n");
193    for (String groupName : groupBasedLoad.keySet()) {
194      sb.append("Stats for group: " + groupName);
195      sb.append("\n");
196      sb.append(groupMap.get(groupName).getServers());
197      sb.append("\n");
198      List<ServerAndLoad> groupLoad = groupBasedLoad.get(groupName);
199      int numServers = groupLoad.size();
200      int totalRegions = 0;
201      sb.append("Per Server Load: \n");
202      for (ServerAndLoad sLoad : groupLoad) {
203        sb.append("Server :" + sLoad.getServerName() + " Load : " + sLoad.getLoad() + "\n");
204        totalRegions += sLoad.getLoad();
205      }
206      sb.append(" Group Statistics : \n");
207      float average = (float) totalRegions / numServers;
208      int max = (int) Math.ceil(average);
209      int min = (int) Math.floor(average);
210      sb.append("[srvr=" + numServers + " rgns=" + totalRegions + " avg=" + average + " max=" + max
211        + " min=" + min + "]");
212      sb.append("\n");
213      sb.append("===============================");
214      sb.append("\n");
215    }
216    return sb.toString();
217  }
218
219  protected ArrayListMultimap<String, ServerAndLoad>
220    convertToGroupBasedMap(final Map<ServerName, List<RegionInfo>> serversMap) throws IOException {
221    ArrayListMultimap<String, ServerAndLoad> loadMap = ArrayListMultimap.create();
222    for (RSGroupInfo gInfo : getMockedGroupInfoManager().listRSGroups()) {
223      Set<Address> groupServers = gInfo.getServers();
224      for (Address hostPort : groupServers) {
225        ServerName actual = null;
226        for (ServerName entry : servers) {
227          if (entry.getAddress().equals(hostPort)) {
228            actual = entry;
229            break;
230          }
231        }
232        List<RegionInfo> regions = serversMap.get(actual);
233        assertTrue(regions != null, "No load for " + actual);
234        loadMap.put(gInfo.getName(), new ServerAndLoad(actual, regions.size()));
235      }
236    }
237    return loadMap;
238  }
239
240  protected ArrayListMultimap<String, ServerAndLoad>
241    reconcile(ArrayListMultimap<String, ServerAndLoad> previousLoad, List<RegionPlan> plans) {
242    ArrayListMultimap<String, ServerAndLoad> result = ArrayListMultimap.create();
243    result.putAll(previousLoad);
244    if (plans != null) {
245      for (RegionPlan plan : plans) {
246        ServerName source = plan.getSource();
247        updateLoad(result, source, -1);
248        ServerName destination = plan.getDestination();
249        updateLoad(result, destination, +1);
250      }
251    }
252    return result;
253  }
254
255  protected void updateLoad(ArrayListMultimap<String, ServerAndLoad> previousLoad,
256    final ServerName sn, final int diff) {
257    for (String groupName : previousLoad.keySet()) {
258      ServerAndLoad newSAL = null;
259      ServerAndLoad oldSAL = null;
260      for (ServerAndLoad sal : previousLoad.get(groupName)) {
261        if (ServerName.isSameAddress(sn, sal.getServerName())) {
262          oldSAL = sal;
263          newSAL = new ServerAndLoad(sn, sal.getLoad() + diff);
264          break;
265        }
266      }
267      if (newSAL != null) {
268        previousLoad.remove(groupName, oldSAL);
269        previousLoad.put(groupName, newSAL);
270        break;
271      }
272    }
273  }
274
275  protected Map<ServerName, List<RegionInfo>> mockClusterServers() throws IOException {
276    assertTrue(servers.size() == regionAssignment.length);
277    Map<ServerName, List<RegionInfo>> assignment = new TreeMap<>();
278    for (int i = 0; i < servers.size(); i++) {
279      int numRegions = regionAssignment[i];
280      List<RegionInfo> regions = assignedRegions(numRegions, servers.get(i));
281      assignment.put(servers.get(i), regions);
282    }
283    return assignment;
284  }
285
286  /**
287   * Generate a list of regions evenly distributed between the tables.
288   * @param numRegions The number of regions to be generated.
289   * @return List of RegionInfo.
290   */
291  protected List<RegionInfo> randomRegions(int numRegions) {
292    List<RegionInfo> regions = new ArrayList<>(numRegions);
293    byte[] start = new byte[16];
294    Bytes.random(start);
295    byte[] end = new byte[16];
296    Bytes.random(end);
297    int regionIdx = ThreadLocalRandom.current().nextInt(tables.length);
298    for (int i = 0; i < numRegions; i++) {
299      Bytes.putInt(start, 0, numRegions << 1);
300      Bytes.putInt(end, 0, (numRegions << 1) + 1);
301      int tableIndex = (i + regionIdx) % tables.length;
302      regions.add(RegionInfoBuilder.newBuilder(tables[tableIndex]).setStartKey(start).setEndKey(end)
303        .setSplit(false).setRegionId(regionId++).build());
304    }
305    return regions;
306  }
307
308  /**
309   * Generate assigned regions to a given server using group information.
310   * @param numRegions the num regions to generate
311   * @param sn         the servername
312   * @return the list of regions
313   * @throws java.io.IOException Signals that an I/O exception has occurred.
314   */
315  protected List<RegionInfo> assignedRegions(int numRegions, ServerName sn) throws IOException {
316    List<RegionInfo> regions = new ArrayList<>(numRegions);
317    byte[] start = new byte[16];
318    byte[] end = new byte[16];
319    Bytes.putInt(start, 0, numRegions << 1);
320    Bytes.putInt(end, 0, (numRegions << 1) + 1);
321    for (int i = 0; i < numRegions; i++) {
322      TableName tableName = getTableName(sn);
323      regions.add(RegionInfoBuilder.newBuilder(tableName).setStartKey(start).setEndKey(end)
324        .setSplit(false).setRegionId(regionId++).build());
325    }
326    return regions;
327  }
328
329  protected static List<ServerName> generateServers(int numServers) {
330    List<ServerName> servers = new ArrayList<>(numServers);
331    Random rand = ThreadLocalRandom.current();
332    for (int i = 0; i < numServers; i++) {
333      String host = "server" + rand.nextInt(100000);
334      int port = rand.nextInt(60000);
335      servers.add(ServerName.valueOf(host, port, -1));
336    }
337    return servers;
338  }
339
340  /**
341   * Construct group info, with each group having at least one server.
342   * @param servers the servers
343   * @param groups  the groups
344   * @return the map
345   */
346  protected static Map<String, RSGroupInfo> constructGroupInfo(List<ServerName> servers,
347    String[] groups) {
348    assertTrue(servers != null);
349    assertTrue(servers.size() >= groups.length);
350    int index = 0;
351    Map<String, RSGroupInfo> groupMap = new HashMap<>();
352    for (String grpName : groups) {
353      RSGroupInfo RSGroupInfo = new RSGroupInfo(grpName);
354      RSGroupInfo.addServer(servers.get(index).getAddress());
355      groupMap.put(grpName, RSGroupInfo);
356      index++;
357    }
358    Random rand = ThreadLocalRandom.current();
359    while (index < servers.size()) {
360      int grpIndex = rand.nextInt(groups.length);
361      groupMap.get(groups[grpIndex]).addServer(servers.get(index).getAddress());
362      index++;
363    }
364    return groupMap;
365  }
366
367  /**
368   * Construct table descriptors evenly distributed between the groups.
369   * @param hasBogusTable there is a table that does not determine the group
370   * @return the list of table descriptors
371   */
372  protected static Map<TableName, TableDescriptor> constructTableDesc(boolean hasBogusTable) {
373    Map<TableName, TableDescriptor> tds = new HashMap<>();
374    Random rand = ThreadLocalRandom.current();
375    int index = rand.nextInt(groups.length);
376    for (int i = 0; i < tables.length; i++) {
377      int grpIndex = (i + index) % groups.length;
378      String groupName = groups[grpIndex];
379      TableDescriptor htd =
380        TableDescriptorBuilder.newBuilder(tables[i]).setRegionServerGroup(groupName).build();
381      tds.put(htd.getTableName(), htd);
382    }
383    if (hasBogusTable) {
384      tds.put(table0, TableDescriptorBuilder.newBuilder(table0).setRegionServerGroup("").build());
385    }
386    return tds;
387  }
388
389  protected static MasterServices getMockedMaster() throws IOException {
390    TableDescriptors tds = mock(TableDescriptors.class);
391    when(tds.get(tables[0])).thenReturn(tableDescs.get(tables[0]));
392    when(tds.get(tables[1])).thenReturn(tableDescs.get(tables[1]));
393    when(tds.get(tables[2])).thenReturn(tableDescs.get(tables[2]));
394    when(tds.get(tables[3])).thenReturn(tableDescs.get(tables[3]));
395    MasterServices services = mock(HMaster.class);
396    when(services.getTableDescriptors()).thenReturn(tds);
397    AssignmentManager am = mock(AssignmentManager.class);
398    when(services.getAssignmentManager()).thenReturn(am);
399    when(services.getConfiguration()).thenReturn(conf);
400    RSGroupInfoManager manager = getMockedGroupInfoManager();
401    when(services.getRSGroupInfoManager()).thenReturn(manager);
402    return services;
403  }
404
405  protected static RSGroupInfoManager getMockedGroupInfoManager() throws IOException {
406    RSGroupInfoManager gm = mock(RSGroupInfoManager.class);
407    when(gm.getRSGroup(any())).thenAnswer(invocation -> {
408      return groupMap.get(invocation.getArgument(0));
409    });
410    when(gm.listRSGroups()).thenReturn(Lists.newLinkedList(groupMap.values()));
411    when(gm.isOnline()).thenReturn(true);
412    return gm;
413  }
414
415  protected TableName getTableName(ServerName sn) throws IOException {
416    TableName tableName = null;
417    RSGroupInfoManager gm = getMockedGroupInfoManager();
418    RSGroupInfo groupOfServer = null;
419    for (RSGroupInfo gInfo : gm.listRSGroups()) {
420      if (gInfo.containsServer(sn.getAddress())) {
421        groupOfServer = gInfo;
422        break;
423      }
424    }
425
426    for (TableDescriptor desc : tableDescs.values()) {
427      Optional<String> optGroupName = desc.getRegionServerGroup();
428      if (optGroupName.isPresent() && optGroupName.get().endsWith(groupOfServer.getName())) {
429        tableName = desc.getTableName();
430      }
431    }
432    return tableName;
433  }
434
435  protected ServerMetrics mockServerMetricsWithRegionCacheInfo(ServerName server,
436    List<RegionInfo> regionsOnServer, float currentCacheRatio, List<RegionInfo> oldRegionCacheInfo,
437    int oldRegionCachedSize, int regionSize) {
438    ServerMetrics serverMetrics = mock(ServerMetrics.class);
439    Map<byte[], RegionMetrics> regionLoadMap = new TreeMap<>(Bytes.BYTES_COMPARATOR);
440    for (RegionInfo info : regionsOnServer) {
441      RegionMetrics rl = mock(RegionMetrics.class);
442      when(rl.getReadRequestCount()).thenReturn(0L);
443      when(rl.getWriteRequestCount()).thenReturn(0L);
444      when(rl.getMemStoreSize()).thenReturn(Size.ZERO);
445      when(rl.getStoreFileSize()).thenReturn(Size.ZERO);
446      when(rl.getCurrentRegionCachedRatio()).thenReturn(currentCacheRatio);
447      when(rl.getRegionSizeMB()).thenReturn(new Size(regionSize, Size.Unit.MEGABYTE));
448      regionLoadMap.put(info.getRegionName(), rl);
449    }
450    when(serverMetrics.getRegionMetrics()).thenReturn(regionLoadMap);
451    Map<String, Integer> oldCacheRatioMap = new HashMap<>();
452    for (RegionInfo info : oldRegionCacheInfo) {
453      oldCacheRatioMap.put(info.getEncodedName(), oldRegionCachedSize);
454    }
455    when(serverMetrics.getRegionCachedInfo()).thenReturn(oldCacheRatioMap);
456    return serverMetrics;
457  }
458}