001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.balancer;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertTrue;
023import static org.mockito.Mockito.mock;
024import static org.mockito.Mockito.when;
025
026import java.io.FileNotFoundException;
027import java.io.IOException;
028import java.util.ArrayList;
029import java.util.HashMap;
030import java.util.List;
031import java.util.Map;
032import java.util.Optional;
033import java.util.Random;
034import java.util.Set;
035import java.util.TreeMap;
036import java.util.TreeSet;
037import java.util.concurrent.ThreadLocalRandom;
038import org.apache.commons.lang3.StringUtils;
039import org.apache.hadoop.conf.Configuration;
040import org.apache.hadoop.hbase.HBaseConfiguration;
041import org.apache.hadoop.hbase.RegionMetrics;
042import org.apache.hadoop.hbase.ServerMetrics;
043import org.apache.hadoop.hbase.ServerName;
044import org.apache.hadoop.hbase.Size;
045import org.apache.hadoop.hbase.TableDescriptors;
046import org.apache.hadoop.hbase.TableName;
047import org.apache.hadoop.hbase.client.RegionInfo;
048import org.apache.hadoop.hbase.client.RegionInfoBuilder;
049import org.apache.hadoop.hbase.client.TableDescriptor;
050import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
051import org.apache.hadoop.hbase.master.HMaster;
052import org.apache.hadoop.hbase.master.MasterServices;
053import org.apache.hadoop.hbase.master.RegionPlan;
054import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
055import org.apache.hadoop.hbase.net.Address;
056import org.apache.hadoop.hbase.rsgroup.RSGroupInfo;
057import org.apache.hadoop.hbase.rsgroup.RSGroupInfoManager;
058import org.apache.hadoop.hbase.util.Bytes;
059import org.mockito.Mockito;
060import org.mockito.invocation.InvocationOnMock;
061import org.mockito.stubbing.Answer;
062
063import org.apache.hbase.thirdparty.com.google.common.collect.ArrayListMultimap;
064import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
065
066/**
067 * Base UT of RSGroupableBalancer.
068 */
069public class RSGroupableBalancerTestBase extends BalancerTestBase {
070
071  static String[] groups = new String[] { RSGroupInfo.DEFAULT_GROUP, "dg2", "dg3", "dg4" };
072  static TableName table0 = TableName.valueOf("dt0");
073  static TableName[] tables = new TableName[] { TableName.valueOf("dt1"), TableName.valueOf("dt2"),
074    TableName.valueOf("dt3"), TableName.valueOf("dt4") };
075  static List<ServerName> servers;
076  static Map<String, RSGroupInfo> groupMap;
077  static Map<TableName, TableDescriptor> tableDescs;
078  int[] regionAssignment = new int[] { 2, 5, 7, 10, 4, 3, 1 };
079  static int regionId = 0;
080  static Configuration conf = HBaseConfiguration.create();
081
082  /**
083   * Invariant is that all servers of a group have load between floor(avg) and ceiling(avg) number
084   * of regions.
085   */
086  protected void assertClusterAsBalanced(ArrayListMultimap<String, ServerAndLoad> groupLoadMap) {
087    for (String gName : groupLoadMap.keySet()) {
088      List<ServerAndLoad> groupLoad = groupLoadMap.get(gName);
089      int numServers = groupLoad.size();
090      int numRegions = 0;
091      int maxRegions = 0;
092      int minRegions = Integer.MAX_VALUE;
093      for (ServerAndLoad server : groupLoad) {
094        int nr = server.getLoad();
095        if (nr > maxRegions) {
096          maxRegions = nr;
097        }
098        if (nr < minRegions) {
099          minRegions = nr;
100        }
101        numRegions += nr;
102      }
103      if (maxRegions - minRegions < 2) {
104        // less than 2 between max and min, can't balance
105        return;
106      }
107      int min = numRegions / numServers;
108      int max = numRegions % numServers == 0 ? min : min + 1;
109
110      for (ServerAndLoad server : groupLoad) {
111        assertTrue(server.getLoad() <= max);
112        assertTrue(server.getLoad() >= min);
113      }
114    }
115  }
116
117  /**
118   * All regions have an assignment.
119   */
120  protected void assertImmediateAssignment(List<RegionInfo> regions, List<ServerName> servers,
121    Map<RegionInfo, ServerName> assignments) throws IOException {
122    for (RegionInfo region : regions) {
123      assertTrue(assignments.containsKey(region));
124      ServerName server = assignments.get(region);
125      TableName tableName = region.getTable();
126
127      String groupName =
128        tableDescs.get(tableName).getRegionServerGroup().orElse(RSGroupInfo.DEFAULT_GROUP);
129      assertTrue(StringUtils.isNotEmpty(groupName));
130      RSGroupInfo gInfo = getMockedGroupInfoManager().getRSGroup(groupName);
131      assertTrue("Region is not correctly assigned to group servers.",
132        gInfo.containsServer(server.getAddress()));
133    }
134  }
135
136  /**
137   * Asserts a valid retained assignment plan.
138   * <p>
139   * Must meet the following conditions:
140   * <ul>
141   * <li>Every input region has an assignment, and to an online server
142   * <li>If a region had an existing assignment to a server with the same address a a currently
143   * online server, it will be assigned to it
144   * </ul>
145   */
146  protected void assertRetainedAssignment(Map<RegionInfo, ServerName> existing,
147    List<ServerName> servers, Map<ServerName, List<RegionInfo>> assignment)
148    throws FileNotFoundException, IOException {
149    // Verify condition 1, every region assigned, and to online server
150    Set<ServerName> onlineServerSet = new TreeSet<>(servers);
151    Set<RegionInfo> assignedRegions = new TreeSet<>(RegionInfo.COMPARATOR);
152    for (Map.Entry<ServerName, List<RegionInfo>> a : assignment.entrySet()) {
153      assertTrue("Region assigned to server that was not listed as online",
154        onlineServerSet.contains(a.getKey()));
155      for (RegionInfo r : a.getValue()) {
156        assignedRegions.add(r);
157      }
158    }
159    assertEquals(existing.size(), assignedRegions.size());
160
161    // Verify condition 2, every region must be assigned to correct server.
162    Set<String> onlineHostNames = new TreeSet<>();
163    for (ServerName s : servers) {
164      onlineHostNames.add(s.getHostname());
165    }
166
167    for (Map.Entry<ServerName, List<RegionInfo>> a : assignment.entrySet()) {
168      ServerName currentServer = a.getKey();
169      for (RegionInfo r : a.getValue()) {
170        ServerName oldAssignedServer = existing.get(r);
171        TableName tableName = r.getTable();
172        String groupName =
173          tableDescs.get(tableName).getRegionServerGroup().orElse(RSGroupInfo.DEFAULT_GROUP);
174        assertTrue(StringUtils.isNotEmpty(groupName));
175        RSGroupInfo gInfo = getMockedGroupInfoManager().getRSGroup(groupName);
176        assertTrue("Region is not correctly assigned to group servers.",
177          gInfo.containsServer(currentServer.getAddress()));
178        if (
179          oldAssignedServer != null && onlineHostNames.contains(oldAssignedServer.getHostname())
180        ) {
181          // this region was previously assigned somewhere, and that
182          // host is still around, then the host must have been is a
183          // different group.
184          if (!oldAssignedServer.getAddress().equals(currentServer.getAddress())) {
185            assertFalse(gInfo.containsServer(oldAssignedServer.getAddress()));
186          }
187        }
188      }
189    }
190  }
191
192  protected String printStats(ArrayListMultimap<String, ServerAndLoad> groupBasedLoad) {
193    StringBuilder sb = new StringBuilder();
194    sb.append("\n");
195    for (String groupName : groupBasedLoad.keySet()) {
196      sb.append("Stats for group: " + groupName);
197      sb.append("\n");
198      sb.append(groupMap.get(groupName).getServers());
199      sb.append("\n");
200      List<ServerAndLoad> groupLoad = groupBasedLoad.get(groupName);
201      int numServers = groupLoad.size();
202      int totalRegions = 0;
203      sb.append("Per Server Load: \n");
204      for (ServerAndLoad sLoad : groupLoad) {
205        sb.append("Server :" + sLoad.getServerName() + " Load : " + sLoad.getLoad() + "\n");
206        totalRegions += sLoad.getLoad();
207      }
208      sb.append(" Group Statistics : \n");
209      float average = (float) totalRegions / numServers;
210      int max = (int) Math.ceil(average);
211      int min = (int) Math.floor(average);
212      sb.append("[srvr=" + numServers + " rgns=" + totalRegions + " avg=" + average + " max=" + max
213        + " min=" + min + "]");
214      sb.append("\n");
215      sb.append("===============================");
216      sb.append("\n");
217    }
218    return sb.toString();
219  }
220
221  protected ArrayListMultimap<String, ServerAndLoad>
222    convertToGroupBasedMap(final Map<ServerName, List<RegionInfo>> serversMap) throws IOException {
223    ArrayListMultimap<String, ServerAndLoad> loadMap = ArrayListMultimap.create();
224    for (RSGroupInfo gInfo : getMockedGroupInfoManager().listRSGroups()) {
225      Set<Address> groupServers = gInfo.getServers();
226      for (Address hostPort : groupServers) {
227        ServerName actual = null;
228        for (ServerName entry : servers) {
229          if (entry.getAddress().equals(hostPort)) {
230            actual = entry;
231            break;
232          }
233        }
234        List<RegionInfo> regions = serversMap.get(actual);
235        assertTrue("No load for " + actual, regions != null);
236        loadMap.put(gInfo.getName(), new ServerAndLoad(actual, regions.size()));
237      }
238    }
239    return loadMap;
240  }
241
242  protected ArrayListMultimap<String, ServerAndLoad>
243    reconcile(ArrayListMultimap<String, ServerAndLoad> previousLoad, List<RegionPlan> plans) {
244    ArrayListMultimap<String, ServerAndLoad> result = ArrayListMultimap.create();
245    result.putAll(previousLoad);
246    if (plans != null) {
247      for (RegionPlan plan : plans) {
248        ServerName source = plan.getSource();
249        updateLoad(result, source, -1);
250        ServerName destination = plan.getDestination();
251        updateLoad(result, destination, +1);
252      }
253    }
254    return result;
255  }
256
257  protected void updateLoad(ArrayListMultimap<String, ServerAndLoad> previousLoad,
258    final ServerName sn, final int diff) {
259    for (String groupName : previousLoad.keySet()) {
260      ServerAndLoad newSAL = null;
261      ServerAndLoad oldSAL = null;
262      for (ServerAndLoad sal : previousLoad.get(groupName)) {
263        if (ServerName.isSameAddress(sn, sal.getServerName())) {
264          oldSAL = sal;
265          newSAL = new ServerAndLoad(sn, sal.getLoad() + diff);
266          break;
267        }
268      }
269      if (newSAL != null) {
270        previousLoad.remove(groupName, oldSAL);
271        previousLoad.put(groupName, newSAL);
272        break;
273      }
274    }
275  }
276
277  protected Map<ServerName, List<RegionInfo>> mockClusterServers() throws IOException {
278    assertTrue(servers.size() == regionAssignment.length);
279    Map<ServerName, List<RegionInfo>> assignment = new TreeMap<>();
280    for (int i = 0; i < servers.size(); i++) {
281      int numRegions = regionAssignment[i];
282      List<RegionInfo> regions = assignedRegions(numRegions, servers.get(i));
283      assignment.put(servers.get(i), regions);
284    }
285    return assignment;
286  }
287
288  /**
289   * Generate a list of regions evenly distributed between the tables.
290   * @param numRegions The number of regions to be generated.
291   * @return List of RegionInfo.
292   */
293  protected List<RegionInfo> randomRegions(int numRegions) {
294    List<RegionInfo> regions = new ArrayList<>(numRegions);
295    byte[] start = new byte[16];
296    Bytes.random(start);
297    byte[] end = new byte[16];
298    Bytes.random(end);
299    int regionIdx = ThreadLocalRandom.current().nextInt(tables.length);
300    for (int i = 0; i < numRegions; i++) {
301      Bytes.putInt(start, 0, numRegions << 1);
302      Bytes.putInt(end, 0, (numRegions << 1) + 1);
303      int tableIndex = (i + regionIdx) % tables.length;
304      regions.add(RegionInfoBuilder.newBuilder(tables[tableIndex]).setStartKey(start).setEndKey(end)
305        .setSplit(false).setRegionId(regionId++).build());
306    }
307    return regions;
308  }
309
310  /**
311   * Generate assigned regions to a given server using group information.
312   * @param numRegions the num regions to generate
313   * @param sn         the servername
314   * @return the list of regions
315   * @throws java.io.IOException Signals that an I/O exception has occurred.
316   */
317  protected List<RegionInfo> assignedRegions(int numRegions, ServerName sn) throws IOException {
318    List<RegionInfo> regions = new ArrayList<>(numRegions);
319    byte[] start = new byte[16];
320    byte[] end = new byte[16];
321    Bytes.putInt(start, 0, numRegions << 1);
322    Bytes.putInt(end, 0, (numRegions << 1) + 1);
323    for (int i = 0; i < numRegions; i++) {
324      TableName tableName = getTableName(sn);
325      regions.add(RegionInfoBuilder.newBuilder(tableName).setStartKey(start).setEndKey(end)
326        .setSplit(false).setRegionId(regionId++).build());
327    }
328    return regions;
329  }
330
331  protected static List<ServerName> generateServers(int numServers) {
332    List<ServerName> servers = new ArrayList<>(numServers);
333    Random rand = ThreadLocalRandom.current();
334    for (int i = 0; i < numServers; i++) {
335      String host = "server" + rand.nextInt(100000);
336      int port = rand.nextInt(60000);
337      servers.add(ServerName.valueOf(host, port, -1));
338    }
339    return servers;
340  }
341
342  /**
343   * Construct group info, with each group having at least one server.
344   * @param servers the servers
345   * @param groups  the groups
346   * @return the map
347   */
348  protected static Map<String, RSGroupInfo> constructGroupInfo(List<ServerName> servers,
349    String[] groups) {
350    assertTrue(servers != null);
351    assertTrue(servers.size() >= groups.length);
352    int index = 0;
353    Map<String, RSGroupInfo> groupMap = new HashMap<>();
354    for (String grpName : groups) {
355      RSGroupInfo RSGroupInfo = new RSGroupInfo(grpName);
356      RSGroupInfo.addServer(servers.get(index).getAddress());
357      groupMap.put(grpName, RSGroupInfo);
358      index++;
359    }
360    Random rand = ThreadLocalRandom.current();
361    while (index < servers.size()) {
362      int grpIndex = rand.nextInt(groups.length);
363      groupMap.get(groups[grpIndex]).addServer(servers.get(index).getAddress());
364      index++;
365    }
366    return groupMap;
367  }
368
369  /**
370   * Construct table descriptors evenly distributed between the groups.
371   * @param hasBogusTable there is a table that does not determine the group
372   * @return the list of table descriptors
373   */
374  protected static Map<TableName, TableDescriptor> constructTableDesc(boolean hasBogusTable) {
375    Map<TableName, TableDescriptor> tds = new HashMap<>();
376    Random rand = ThreadLocalRandom.current();
377    int index = rand.nextInt(groups.length);
378    for (int i = 0; i < tables.length; i++) {
379      int grpIndex = (i + index) % groups.length;
380      String groupName = groups[grpIndex];
381      TableDescriptor htd =
382        TableDescriptorBuilder.newBuilder(tables[i]).setRegionServerGroup(groupName).build();
383      tds.put(htd.getTableName(), htd);
384    }
385    if (hasBogusTable) {
386      tds.put(table0, TableDescriptorBuilder.newBuilder(table0).setRegionServerGroup("").build());
387    }
388    return tds;
389  }
390
391  protected static MasterServices getMockedMaster() throws IOException {
392    TableDescriptors tds = Mockito.mock(TableDescriptors.class);
393    Mockito.when(tds.get(tables[0])).thenReturn(tableDescs.get(tables[0]));
394    Mockito.when(tds.get(tables[1])).thenReturn(tableDescs.get(tables[1]));
395    Mockito.when(tds.get(tables[2])).thenReturn(tableDescs.get(tables[2]));
396    Mockito.when(tds.get(tables[3])).thenReturn(tableDescs.get(tables[3]));
397    MasterServices services = Mockito.mock(HMaster.class);
398    Mockito.when(services.getTableDescriptors()).thenReturn(tds);
399    AssignmentManager am = Mockito.mock(AssignmentManager.class);
400    Mockito.when(services.getAssignmentManager()).thenReturn(am);
401    Mockito.when(services.getConfiguration()).thenReturn(conf);
402    RSGroupInfoManager manager = getMockedGroupInfoManager();
403    Mockito.when(services.getRSGroupInfoManager()).thenReturn(manager);
404    return services;
405  }
406
407  protected static RSGroupInfoManager getMockedGroupInfoManager() throws IOException {
408    RSGroupInfoManager gm = Mockito.mock(RSGroupInfoManager.class);
409    Mockito.when(gm.getRSGroup(Mockito.any())).thenAnswer(new Answer<RSGroupInfo>() {
410      @Override
411      public RSGroupInfo answer(InvocationOnMock invocation) throws Throwable {
412        return groupMap.get(invocation.getArgument(0));
413      }
414    });
415    Mockito.when(gm.listRSGroups()).thenReturn(Lists.newLinkedList(groupMap.values()));
416    Mockito.when(gm.isOnline()).thenReturn(true);
417    return gm;
418  }
419
420  protected TableName getTableName(ServerName sn) throws IOException {
421    TableName tableName = null;
422    RSGroupInfoManager gm = getMockedGroupInfoManager();
423    RSGroupInfo groupOfServer = null;
424    for (RSGroupInfo gInfo : gm.listRSGroups()) {
425      if (gInfo.containsServer(sn.getAddress())) {
426        groupOfServer = gInfo;
427        break;
428      }
429    }
430
431    for (TableDescriptor desc : tableDescs.values()) {
432      Optional<String> optGroupName = desc.getRegionServerGroup();
433      if (optGroupName.isPresent() && optGroupName.get().endsWith(groupOfServer.getName())) {
434        tableName = desc.getTableName();
435      }
436    }
437    return tableName;
438  }
439
440  protected ServerMetrics mockServerMetricsWithRegionCacheInfo(ServerName server,
441    List<RegionInfo> regionsOnServer, float currentCacheRatio, List<RegionInfo> oldRegionCacheInfo,
442    int oldRegionCachedSize, int regionSize) {
443    ServerMetrics serverMetrics = mock(ServerMetrics.class);
444    Map<byte[], RegionMetrics> regionLoadMap = new TreeMap<>(Bytes.BYTES_COMPARATOR);
445    for (RegionInfo info : regionsOnServer) {
446      RegionMetrics rl = mock(RegionMetrics.class);
447      when(rl.getReadRequestCount()).thenReturn(0L);
448      when(rl.getWriteRequestCount()).thenReturn(0L);
449      when(rl.getMemStoreSize()).thenReturn(Size.ZERO);
450      when(rl.getStoreFileSize()).thenReturn(Size.ZERO);
451      when(rl.getCurrentRegionCachedRatio()).thenReturn(currentCacheRatio);
452      when(rl.getRegionSizeMB()).thenReturn(new Size(regionSize, Size.Unit.MEGABYTE));
453      regionLoadMap.put(info.getRegionName(), rl);
454    }
455    when(serverMetrics.getRegionMetrics()).thenReturn(regionLoadMap);
456    Map<String, Integer> oldCacheRatioMap = new HashMap<>();
457    for (RegionInfo info : oldRegionCacheInfo) {
458      oldCacheRatioMap.put(info.getEncodedName(), oldRegionCachedSize);
459    }
460    when(serverMetrics.getRegionCachedInfo()).thenReturn(oldCacheRatioMap);
461    return serverMetrics;
462  }
463}