001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.balancer;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertNotNull;
023import static org.junit.Assert.assertNull;
024import static org.junit.Assert.assertTrue;
025import static org.mockito.Mockito.mock;
026import static org.mockito.Mockito.when;
027
028import java.util.ArrayList;
029import java.util.Arrays;
030import java.util.HashMap;
031import java.util.HashSet;
032import java.util.List;
033import java.util.Map;
034import java.util.Queue;
035import java.util.Set;
036import java.util.TreeMap;
037import org.apache.hadoop.conf.Configuration;
038import org.apache.hadoop.hbase.ClusterMetrics;
039import org.apache.hadoop.hbase.HBaseClassTestRule;
040import org.apache.hadoop.hbase.HBaseConfiguration;
041import org.apache.hadoop.hbase.HConstants;
042import org.apache.hadoop.hbase.RegionMetrics;
043import org.apache.hadoop.hbase.ServerMetrics;
044import org.apache.hadoop.hbase.ServerName;
045import org.apache.hadoop.hbase.Size;
046import org.apache.hadoop.hbase.TableName;
047import org.apache.hadoop.hbase.client.RegionInfo;
048import org.apache.hadoop.hbase.master.RegionPlan;
049import org.apache.hadoop.hbase.testclassification.MasterTests;
050import org.apache.hadoop.hbase.testclassification.MediumTests;
051import org.apache.hadoop.hbase.util.Bytes;
052import org.junit.Before;
053import org.junit.BeforeClass;
054import org.junit.ClassRule;
055import org.junit.Test;
056import org.junit.experimental.categories.Category;
057
058import org.apache.hbase.thirdparty.org.apache.commons.collections4.CollectionUtils;
059
060@Category({ MasterTests.class, MediumTests.class })
061public class TestStochasticLoadBalancer extends StochasticBalancerTestBase {
062
063  @ClassRule
064  public static final HBaseClassTestRule CLASS_RULE =
065    HBaseClassTestRule.forClass(TestStochasticLoadBalancer.class);
066
067  private static final String REGION_KEY = "testRegion";
068
069  // Mapping of locality test -> expected locality
070  private float[] expectedLocalities = { 1.0f, 0.0f, 0.50f, 0.25f, 1.0f };
071  private static Configuration storedConfiguration;
072
073  @BeforeClass
074  public static void saveInitialConfiguration() {
075    storedConfiguration = new Configuration(conf);
076  }
077
078  @Before
079  public void beforeEachTest() {
080    conf = new Configuration(storedConfiguration);
081    loadBalancer.onConfigurationChange(conf);
082  }
083
084  /**
085   * Data set for testLocalityCost: [test][0][0] = mapping of server to number of regions it hosts
086   * [test][region + 1][0] = server that region is hosted on [test][region + 1][server + 1] =
087   * locality for region on server
088   */
089  private int[][][] clusterRegionLocationMocks = new int[][][] {
090    // Test 1: each region is entirely on server that hosts it
091    new int[][] { new int[] { 2, 1, 1 }, new int[] { 2, 0, 0, 100 }, // region 0 is hosted and
092                                                                     // entirely local on server 2
093      new int[] { 0, 100, 0, 0 }, // region 1 is hosted and entirely on server 0
094      new int[] { 0, 100, 0, 0 }, // region 2 is hosted and entirely on server 0
095      new int[] { 1, 0, 100, 0 }, // region 3 is hosted and entirely on server 1
096    },
097
098    // Test 2: each region is 0% local on the server that hosts it
099    new int[][] { new int[] { 1, 2, 1 }, new int[] { 0, 0, 0, 100 }, // region 0 is hosted and
100                                                                     // entirely local on server 2
101      new int[] { 1, 100, 0, 0 }, // region 1 is hosted and entirely on server 0
102      new int[] { 1, 100, 0, 0 }, // region 2 is hosted and entirely on server 0
103      new int[] { 2, 0, 100, 0 }, // region 3 is hosted and entirely on server 1
104    },
105
106    // Test 3: each region is 25% local on the server that hosts it (and 50% locality is possible)
107    new int[][] { new int[] { 1, 2, 1 }, new int[] { 0, 25, 0, 50 }, // region 0 is hosted and
108                                                                     // entirely local on server 2
109      new int[] { 1, 50, 25, 0 }, // region 1 is hosted and entirely on server 0
110      new int[] { 1, 50, 25, 0 }, // region 2 is hosted and entirely on server 0
111      new int[] { 2, 0, 50, 25 }, // region 3 is hosted and entirely on server 1
112    },
113
114    // Test 4: each region is 25% local on the server that hosts it (and 100% locality is possible)
115    new int[][] { new int[] { 1, 2, 1 }, new int[] { 0, 25, 0, 100 }, // region 0 is hosted and
116                                                                      // entirely local on server 2
117      new int[] { 1, 100, 25, 0 }, // region 1 is hosted and entirely on server 0
118      new int[] { 1, 100, 25, 0 }, // region 2 is hosted and entirely on server 0
119      new int[] { 2, 0, 100, 25 }, // region 3 is hosted and entirely on server 1
120    },
121
122    // Test 5: each region is 75% local on the server that hosts it (and 75% locality is possible
123    // everywhere)
124    new int[][] { new int[] { 1, 2, 1 }, new int[] { 0, 75, 75, 75 }, // region 0 is hosted and
125                                                                      // entirely local on server 2
126      new int[] { 1, 75, 75, 75 }, // region 1 is hosted and entirely on server 0
127      new int[] { 1, 75, 75, 75 }, // region 2 is hosted and entirely on server 0
128      new int[] { 2, 75, 75, 75 }, // region 3 is hosted and entirely on server 1
129    }, };
130
131  private ServerMetrics mockServerMetricsWithCpRequests(List<RegionInfo> regionsOnServer,
132    long cpRequestCount) {
133    ServerMetrics serverMetrics = mock(ServerMetrics.class);
134    Map<byte[], RegionMetrics> regionLoadMap = new TreeMap<>(Bytes.BYTES_COMPARATOR);
135    for (RegionInfo info : regionsOnServer) {
136      RegionMetrics rl = mock(RegionMetrics.class);
137      when(rl.getReadRequestCount()).thenReturn(0L);
138      when(rl.getCpRequestCount()).thenReturn(cpRequestCount);
139      when(rl.getWriteRequestCount()).thenReturn(0L);
140      when(rl.getMemStoreSize()).thenReturn(Size.ZERO);
141      when(rl.getStoreFileSize()).thenReturn(Size.ZERO);
142      regionLoadMap.put(info.getRegionName(), rl);
143    }
144    when(serverMetrics.getRegionMetrics()).thenReturn(regionLoadMap);
145    return serverMetrics;
146  }
147
148  @Test
149  public void testCPRequestCost() {
150    // in order to pass needsBalance judgement
151    conf.setFloat("hbase.master.balancer.stochastic.cpRequestCost", 10000f);
152    loadBalancer.onConfigurationChange(conf);
153    // mock cluster State
154    Map<ServerName, List<RegionInfo>> clusterState = new HashMap<ServerName, List<RegionInfo>>();
155    ServerName serverA = randomServer(3).getServerName();
156    ServerName serverB = randomServer(3).getServerName();
157    ServerName serverC = randomServer(3).getServerName();
158    List<RegionInfo> regionsOnServerA = randomRegions(3);
159    List<RegionInfo> regionsOnServerB = randomRegions(3);
160    List<RegionInfo> regionsOnServerC = randomRegions(3);
161    clusterState.put(serverA, regionsOnServerA);
162    clusterState.put(serverB, regionsOnServerB);
163    clusterState.put(serverC, regionsOnServerC);
164    // mock ClusterMetrics
165    Map<ServerName, ServerMetrics> serverMetricsMap = new TreeMap<>();
166    serverMetricsMap.put(serverA, mockServerMetricsWithCpRequests(regionsOnServerA, 0));
167    serverMetricsMap.put(serverB, mockServerMetricsWithCpRequests(regionsOnServerB, 0));
168    serverMetricsMap.put(serverC, mockServerMetricsWithCpRequests(regionsOnServerC, 0));
169    ClusterMetrics clusterStatus = mock(ClusterMetrics.class);
170    when(clusterStatus.getLiveServerMetrics()).thenReturn(serverMetricsMap);
171    loadBalancer.updateClusterMetrics(clusterStatus);
172
173    // CPRequestCostFunction are Rate based, So doing setClusterMetrics again
174    // this time, regions on serverA with more cpRequestCount load
175    // serverA : 1000,1000,1000
176    // serverB : 0,0,0
177    // serverC : 0,0,0
178    // so should move two regions from serverA to serverB & serverC
179    serverMetricsMap = new TreeMap<>();
180    serverMetricsMap.put(serverA, mockServerMetricsWithCpRequests(regionsOnServerA, 1000));
181    serverMetricsMap.put(serverB, mockServerMetricsWithCpRequests(regionsOnServerB, 0));
182    serverMetricsMap.put(serverC, mockServerMetricsWithCpRequests(regionsOnServerC, 0));
183    clusterStatus = mock(ClusterMetrics.class);
184    when(clusterStatus.getLiveServerMetrics()).thenReturn(serverMetricsMap);
185    loadBalancer.updateClusterMetrics(clusterStatus);
186
187    List<RegionPlan> plans =
188      loadBalancer.balanceTable(HConstants.ENSEMBLE_TABLE_NAME, clusterState);
189    Set<RegionInfo> regionsMoveFromServerA = new HashSet<>();
190    Set<ServerName> targetServers = new HashSet<>();
191    for (RegionPlan plan : plans) {
192      if (plan.getSource().equals(serverA)) {
193        regionsMoveFromServerA.add(plan.getRegionInfo());
194        targetServers.add(plan.getDestination());
195      }
196    }
197    // should move 2 regions from serverA, one moves to serverB, the other moves to serverC
198    assertEquals(2, regionsMoveFromServerA.size());
199    assertEquals(2, targetServers.size());
200    assertTrue(regionsOnServerA.containsAll(regionsMoveFromServerA));
201  }
202
203  @Test
204  public void testKeepRegionLoad() throws Exception {
205    ServerName sn = ServerName.valueOf("test:8080", 100);
206    int numClusterStatusToAdd = 20000;
207    for (int i = 0; i < numClusterStatusToAdd; i++) {
208      ServerMetrics sl = mock(ServerMetrics.class);
209
210      RegionMetrics rl = mock(RegionMetrics.class);
211      when(rl.getReadRequestCount()).thenReturn(0L);
212      when(rl.getCpRequestCount()).thenReturn(0L);
213      when(rl.getWriteRequestCount()).thenReturn(0L);
214      when(rl.getMemStoreSize()).thenReturn(Size.ZERO);
215      when(rl.getStoreFileSize()).thenReturn(new Size(i, Size.Unit.MEGABYTE));
216
217      Map<byte[], RegionMetrics> regionLoadMap = new TreeMap<>(Bytes.BYTES_COMPARATOR);
218      regionLoadMap.put(Bytes.toBytes(REGION_KEY), rl);
219      when(sl.getRegionMetrics()).thenReturn(regionLoadMap);
220
221      ClusterMetrics clusterStatus = mock(ClusterMetrics.class);
222      Map<ServerName, ServerMetrics> serverMetricsMap = new TreeMap<>();
223      serverMetricsMap.put(sn, sl);
224      when(clusterStatus.getLiveServerMetrics()).thenReturn(serverMetricsMap);
225
226      loadBalancer.updateClusterMetrics(clusterStatus);
227    }
228
229    String regionNameAsString = RegionInfo.getRegionNameAsString(Bytes.toBytes(REGION_KEY));
230    assertTrue(loadBalancer.loads.get(regionNameAsString) != null);
231    assertTrue(loadBalancer.loads.get(regionNameAsString).size() == 15);
232
233    Queue<BalancerRegionLoad> loads = loadBalancer.loads.get(regionNameAsString);
234    int i = 0;
235    while (loads.size() > 0) {
236      BalancerRegionLoad rl = loads.remove();
237      assertEquals(i + (numClusterStatusToAdd - 15), rl.getStorefileSizeMB());
238      i++;
239    }
240  }
241
242  @Test
243  public void testUpdateBalancerLoadInfo() {
244    int[] cluster = new int[] { 10, 0 };
245    Map<ServerName, List<RegionInfo>> servers = mockClusterServers(cluster);
246    BalancerClusterState clusterState = mockCluster(cluster);
247    Map<TableName, Map<ServerName, List<RegionInfo>>> LoadOfAllTable =
248      (Map) mockClusterServersWithTables(servers);
249    boolean[] perTableBalancerConfigs = { true, false };
250    for (boolean isByTable : perTableBalancerConfigs) {
251      conf.setBoolean(HConstants.HBASE_MASTER_LOADBALANCE_BYTABLE, isByTable);
252      loadBalancer.onConfigurationChange(conf);
253      dummyMetricsStochasticBalancer.clearDummyMetrics();
254      loadBalancer.updateBalancerLoadInfo(LoadOfAllTable);
255      assertTrue("Metrics should be recorded!",
256        dummyMetricsStochasticBalancer.getDummyCostsMap() != null
257          && !dummyMetricsStochasticBalancer.getDummyCostsMap().isEmpty());
258
259      String metricRecordKey;
260      if (isByTable) {
261        metricRecordKey = "table1#" + StochasticLoadBalancer.OVERALL_COST_FUNCTION_NAME;
262      } else {
263        metricRecordKey =
264          HConstants.ENSEMBLE_TABLE_NAME + "#" + StochasticLoadBalancer.OVERALL_COST_FUNCTION_NAME;
265      }
266      double curOverallCost = loadBalancer.computeCost(clusterState, Double.MAX_VALUE);
267      double curOverallCostInMetrics =
268        dummyMetricsStochasticBalancer.getDummyCostsMap().get(metricRecordKey);
269      assertEquals(curOverallCost, curOverallCostInMetrics, 0.001);
270    }
271  }
272
273  @Test
274  public void testUpdateStochasticCosts() {
275    int[] cluster = new int[] { 10, 0 };
276    Map<ServerName, List<RegionInfo>> servers = mockClusterServers(cluster);
277    BalancerClusterState clusterState = mockCluster(cluster);
278    conf.setFloat("hbase.master.balancer.stochastic.minCostNeedBalance", 1.0f);
279    conf.setBoolean(HConstants.HBASE_MASTER_LOADBALANCE_BYTABLE, false);
280    loadBalancer.onConfigurationChange(conf);
281    dummyMetricsStochasticBalancer.clearDummyMetrics();
282    List<RegionPlan> plans =
283      loadBalancer.balanceCluster((Map) mockClusterServersWithTables(servers));
284
285    assertTrue("Balance plan should not be empty!", plans != null && !plans.isEmpty());
286    assertTrue("There should be metrics record in MetricsStochasticBalancer",
287      !dummyMetricsStochasticBalancer.getDummyCostsMap().isEmpty());
288
289    double overallCostOfCluster = loadBalancer.computeCost(clusterState, Double.MAX_VALUE);
290    double overallCostInMetrics = dummyMetricsStochasticBalancer.getDummyCostsMap().get(
291      HConstants.ENSEMBLE_TABLE_NAME + "#" + StochasticLoadBalancer.OVERALL_COST_FUNCTION_NAME);
292    assertEquals(overallCostOfCluster, overallCostInMetrics, 0.001);
293  }
294
295  @Test
296  public void testUpdateStochasticCostsIfBalanceNotRan() {
297    int[] cluster = new int[] { 10, 10 };
298    Map<ServerName, List<RegionInfo>> servers = mockClusterServers(cluster);
299    BalancerClusterState clusterState = mockCluster(cluster);
300    conf.setFloat("hbase.master.balancer.stochastic.minCostNeedBalance", Float.MAX_VALUE);
301    conf.setBoolean(HConstants.HBASE_MASTER_LOADBALANCE_BYTABLE, false);
302    loadBalancer.onConfigurationChange(conf);
303    dummyMetricsStochasticBalancer.clearDummyMetrics();
304    List<RegionPlan> plans =
305      loadBalancer.balanceCluster((Map) mockClusterServersWithTables(servers));
306
307    assertTrue("Balance plan should be empty!", plans == null || plans.isEmpty());
308    assertTrue("There should be metrics record in MetricsStochasticBalancer!",
309      !dummyMetricsStochasticBalancer.getDummyCostsMap().isEmpty());
310
311    double overallCostOfCluster = loadBalancer.computeCost(clusterState, Double.MAX_VALUE);
312    double overallCostInMetrics = dummyMetricsStochasticBalancer.getDummyCostsMap().get(
313      HConstants.ENSEMBLE_TABLE_NAME + "#" + StochasticLoadBalancer.OVERALL_COST_FUNCTION_NAME);
314    assertEquals(overallCostOfCluster, overallCostInMetrics, 0.001);
315  }
316
317  @Test
318  public void testNeedBalance() {
319    conf.setFloat("hbase.master.balancer.stochastic.minCostNeedBalance", 1.0f);
320    conf.setFloat(HConstants.LOAD_BALANCER_SLOP_KEY, -1f);
321    conf.setBoolean("hbase.master.balancer.stochastic.runMaxSteps", false);
322    conf.setLong("hbase.master.balancer.stochastic.maxSteps", 5000L);
323    // Test with/without per table balancer.
324    boolean[] perTableBalancerConfigs = { true, false };
325    for (boolean isByTable : perTableBalancerConfigs) {
326      conf.setBoolean(HConstants.HBASE_MASTER_LOADBALANCE_BYTABLE, isByTable);
327      loadBalancer.onConfigurationChange(conf);
328      for (int[] mockCluster : clusterStateMocks) {
329        assertTrue(hasEmptyBalancerPlans(mockCluster) || needsBalanceIdleRegion(mockCluster));
330      }
331    }
332  }
333
334  @Test
335  public void testBalanceOfSloppyServers() {
336    // We are testing slop checks, so don't "accidentally" balance due to a minCost calculation.
337    // During development, imbalance of a 100 server cluster, with one node having 10 regions
338    // and the rest having 5, is 0.0048. With minCostNeedBalance default of 0.025, test should
339    // validate slop checks without this override. We override just to ensure we will always
340    // validate slop check here, and for small clusters as well.
341    conf.setFloat("hbase.master.balancer.stochastic.minCostNeedBalance", 1.0f);
342    conf.setBoolean("hbase.master.balancer.stochastic.runMaxSteps", false);
343    conf.setLong("hbase.master.balancer.stochastic.maxSteps", 5000L);
344    loadBalancer.onConfigurationChange(conf);
345    for (int[] mockCluster : clusterStateMocksWithNoSlop) {
346      assertTrue(hasEmptyBalancerPlans(mockCluster));
347    }
348    for (int[] mockCluster : clusterStateMocksWithSlop) {
349      assertFalse(hasEmptyBalancerPlans(mockCluster));
350    }
351  }
352
353  @Test
354  public void testSloppyTablesLoadBalanceByTable() {
355    int[][] regionsPerServerPerTable = new int[][] {
356      new int[] { 8, 2, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
357        5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5 },
358      new int[] { 2, 8, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
359        5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5 }, };
360    conf.setFloat("hbase.master.balancer.stochastic.minCostNeedBalance", 1.0f);
361    conf.setBoolean("hbase.master.balancer.stochastic.runMaxSteps", false);
362    conf.setLong("hbase.master.balancer.stochastic.maxSteps", 5000L);
363    conf.setBoolean(HConstants.HBASE_MASTER_LOADBALANCE_BYTABLE, true);
364    loadBalancer.onConfigurationChange(conf);
365    assertFalse(hasEmptyBalancerPlans(regionsPerServerPerTable));
366  }
367
368  private boolean hasEmptyBalancerPlans(int[] mockCluster) {
369    Map<ServerName, List<RegionInfo>> servers = mockClusterServers(mockCluster);
370    return hasEmptyBalancerPlans(servers);
371  }
372
373  private boolean hasEmptyBalancerPlans(int[][] mockCluster) {
374    Map<ServerName, List<RegionInfo>> servers = mockClusterServers(mockCluster);
375    return hasEmptyBalancerPlans(servers);
376  }
377
378  private boolean hasEmptyBalancerPlans(Map<ServerName, List<RegionInfo>> servers) {
379    Map<TableName, Map<ServerName, List<RegionInfo>>> loadOfAllTable =
380      (Map) mockClusterServersWithTables(servers);
381    List<RegionPlan> plans = loadBalancer.balanceCluster(loadOfAllTable);
382    return plans == null || plans.isEmpty();
383  }
384
385  @Test
386  public void testLocalityCost() throws Exception {
387    Configuration conf = HBaseConfiguration.create();
388    CostFunction costFunction = new ServerLocalityCostFunction(conf);
389
390    for (int test = 0; test < clusterRegionLocationMocks.length; test++) {
391      int[][] clusterRegionLocations = clusterRegionLocationMocks[test];
392      MockCluster cluster = new MockCluster(clusterRegionLocations);
393      costFunction.prepare(cluster);
394      double cost = costFunction.cost();
395      double expected = 1 - expectedLocalities[test];
396      assertEquals(expected, cost, 0.001);
397    }
398  }
399
400  @Test
401  public void testMoveCostMultiplier() throws Exception {
402    Configuration conf = HBaseConfiguration.create();
403    ClusterInfoProvider provider = mock(ClusterInfoProvider.class);
404    CostFunction costFunction = new MoveCostFunction(conf, provider);
405    when(provider.isOffPeakHour()).thenReturn(false);
406    BalancerClusterState cluster = mockCluster(clusterStateMocks[0]);
407    costFunction.prepare(cluster);
408    costFunction.cost();
409    assertEquals(MoveCostFunction.DEFAULT_MOVE_COST, costFunction.getMultiplier(), 0.01);
410
411    // In offpeak hours, the multiplier of move cost should be lower
412    when(provider.isOffPeakHour()).thenReturn(true);
413    costFunction.prepare(cluster);
414    costFunction.cost();
415    assertEquals(MoveCostFunction.DEFAULT_MOVE_COST_OFFPEAK, costFunction.getMultiplier(), 0.01);
416  }
417
418  @Test
419  public void testMoveCost() throws Exception {
420    Configuration conf = HBaseConfiguration.create();
421    CostFunction costFunction = new MoveCostFunction(conf, new DummyClusterInfoProvider(conf));
422    for (int[] mockCluster : clusterStateMocks) {
423      BalancerClusterState cluster = mockCluster(mockCluster);
424      costFunction.prepare(cluster);
425      double cost = costFunction.cost();
426      assertEquals(0.0f, cost, 0.001);
427
428      // cluster region number is smaller than maxMoves=600
429      cluster.setNumRegions(200);
430      cluster.setNumMovedRegions(10);
431      cost = costFunction.cost();
432      assertEquals(0.05f, cost, 0.001);
433      cluster.setNumMovedRegions(100);
434      cost = costFunction.cost();
435      assertEquals(0.5f, cost, 0.001);
436      cluster.setNumMovedRegions(200);
437      cost = costFunction.cost();
438      assertEquals(1.0f, cost, 0.001);
439
440      // cluster region number is bigger than maxMoves=2500
441      cluster.setNumRegions(10000);
442      cluster.setNumMovedRegions(250);
443      cost = costFunction.cost();
444      assertEquals(0.025f, cost, 0.001);
445      cluster.setNumMovedRegions(1250);
446      cost = costFunction.cost();
447      assertEquals(0.125f, cost, 0.001);
448      cluster.setNumMovedRegions(2500);
449      cost = costFunction.cost();
450      assertEquals(0.25f, cost, 0.01);
451    }
452  }
453
454  @Test
455  public void testSkewCost() {
456    Configuration conf = HBaseConfiguration.create();
457    CostFunction costFunction = new RegionCountSkewCostFunction(conf);
458    for (int[] mockCluster : clusterStateMocks) {
459      costFunction.prepare(mockCluster(mockCluster));
460      double cost = costFunction.cost();
461      assertTrue(cost >= 0);
462      assertTrue(cost <= 1.01);
463    }
464
465    costFunction.prepare(mockCluster(new int[] { 0, 0, 0, 0, 1 }));
466    assertEquals(0, costFunction.cost(), 0.01);
467    costFunction.prepare(mockCluster(new int[] { 0, 0, 0, 1, 1 }));
468    assertEquals(0, costFunction.cost(), 0.01);
469    costFunction.prepare(mockCluster(new int[] { 0, 0, 1, 1, 1 }));
470    assertEquals(0, costFunction.cost(), 0.01);
471    costFunction.prepare(mockCluster(new int[] { 0, 1, 1, 1, 1 }));
472    assertEquals(0, costFunction.cost(), 0.01);
473    costFunction.prepare(mockCluster(new int[] { 1, 1, 1, 1, 1 }));
474    assertEquals(0, costFunction.cost(), 0.01);
475    costFunction.prepare(mockCluster(new int[] { 10000, 0, 0, 0, 0 }));
476    assertEquals(1, costFunction.cost(), 0.01);
477  }
478
479  @Test
480  public void testCostAfterUndoAction() {
481    final int runs = 10;
482    for (int[] mockCluster : clusterStateMocks) {
483      BalancerClusterState cluster = mockCluster(mockCluster);
484      loadBalancer.initCosts(cluster);
485      for (int i = 0; i != runs; ++i) {
486        final double expectedCost = loadBalancer.computeCost(cluster, Double.MAX_VALUE);
487        BalanceAction action = loadBalancer.nextAction(cluster);
488        cluster.doAction(action);
489        loadBalancer.updateCostsAndWeightsWithAction(cluster, action);
490        BalanceAction undoAction = action.undoAction();
491        cluster.doAction(undoAction);
492        loadBalancer.updateCostsAndWeightsWithAction(cluster, undoAction);
493        final double actualCost = loadBalancer.computeCost(cluster, Double.MAX_VALUE);
494        assertEquals(expectedCost, actualCost, 0);
495      }
496    }
497  }
498
499  @Test
500  public void testTableSkewCost() {
501    Configuration conf = HBaseConfiguration.create();
502    CostFunction costFunction = new TableSkewCostFunction(conf);
503    for (int[] mockCluster : clusterStateMocks) {
504      BalancerClusterState cluster = mockCluster(mockCluster);
505      costFunction.prepare(cluster);
506      double cost = costFunction.cost();
507      assertTrue(cost >= 0);
508      assertTrue(cost <= 1.01);
509    }
510  }
511
512  @Test
513  public void testRegionLoadCost() {
514    List<BalancerRegionLoad> regionLoads = new ArrayList<>();
515    for (int i = 1; i < 5; i++) {
516      BalancerRegionLoad regionLoad = mock(BalancerRegionLoad.class);
517      when(regionLoad.getReadRequestsCount()).thenReturn((long) i);
518      when(regionLoad.getCpRequestsCount()).thenReturn((long) i);
519      when(regionLoad.getStorefileSizeMB()).thenReturn(i);
520      regionLoads.add(regionLoad);
521    }
522
523    Configuration conf = HBaseConfiguration.create();
524    ReadRequestCostFunction readCostFunction = new ReadRequestCostFunction(conf);
525    double rateResult = readCostFunction.getRegionLoadCost(regionLoads);
526    // read requests are treated as a rate so the average rate here is simply 1
527    assertEquals(1, rateResult, 0.01);
528
529    CPRequestCostFunction cpCostFunction = new CPRequestCostFunction(conf);
530    rateResult = cpCostFunction.getRegionLoadCost(regionLoads);
531    // coprocessor requests are treated as a rate so the average rate here is simply 1
532    assertEquals(1, rateResult, 0.01);
533
534    StoreFileCostFunction storeFileCostFunction = new StoreFileCostFunction(conf);
535    double result = storeFileCostFunction.getRegionLoadCost(regionLoads);
536    // storefile size cost is simply an average of it's value over time
537    assertEquals(2.5, result, 0.01);
538  }
539
540  @Test
541  public void testRegionLoadCostWhenDecrease() {
542    List<BalancerRegionLoad> regionLoads = new ArrayList<>();
543    // test region loads of [1,2,1,4]
544    for (int i = 1; i < 5; i++) {
545      int load = i == 3 ? 1 : i;
546      BalancerRegionLoad regionLoad = mock(BalancerRegionLoad.class);
547      when(regionLoad.getReadRequestsCount()).thenReturn((long) load);
548      when(regionLoad.getCpRequestsCount()).thenReturn((long) load);
549      regionLoads.add(regionLoad);
550    }
551
552    Configuration conf = HBaseConfiguration.create();
553    ReadRequestCostFunction readCostFunction = new ReadRequestCostFunction(conf);
554    double rateResult = readCostFunction.getRegionLoadCost(regionLoads);
555    // read requests are treated as a rate so the average rate here is simply 1
556    assertEquals(1.67, rateResult, 0.01);
557
558    CPRequestCostFunction cpCostFunction = new CPRequestCostFunction(conf);
559    rateResult = cpCostFunction.getRegionLoadCost(regionLoads);
560    // coprocessor requests are treated as a rate so the average rate here is simply 1
561    assertEquals(1.67, rateResult, 0.01);
562  }
563
564  @Test
565  public void testLosingRs() throws Exception {
566    int numNodes = 3;
567    int numRegions = 20;
568    int numRegionsPerServer = 3; // all servers except one
569    int replication = 1;
570    int numTables = 2;
571
572    Map<ServerName, List<RegionInfo>> serverMap =
573      createServerMap(numNodes, numRegions, numRegionsPerServer, replication, numTables);
574    List<ServerAndLoad> list = convertToList(serverMap);
575
576    List<RegionPlan> plans = loadBalancer.balanceTable(HConstants.ENSEMBLE_TABLE_NAME, serverMap);
577    assertNotNull(plans);
578
579    // Apply the plan to the mock cluster.
580    List<ServerAndLoad> balancedCluster = reconcile(list, plans, serverMap);
581
582    assertClusterAsBalanced(balancedCluster);
583
584    ServerName sn = serverMap.keySet().toArray(new ServerName[serverMap.size()])[0];
585
586    ServerName deadSn = ServerName.valueOf(sn.getHostname(), sn.getPort(), sn.getStartcode() - 100);
587
588    serverMap.put(deadSn, new ArrayList<>(0));
589
590    plans = loadBalancer.balanceTable(HConstants.ENSEMBLE_TABLE_NAME, serverMap);
591    assertNull(plans);
592  }
593
594  @Test
595  public void testAdditionalCostFunction() {
596    conf.set(StochasticLoadBalancer.COST_FUNCTIONS_COST_FUNCTIONS_KEY,
597      DummyCostFunction.class.getName());
598
599    loadBalancer.onConfigurationChange(conf);
600    assertTrue(Arrays.asList(loadBalancer.getCostFunctionNames())
601      .contains(DummyCostFunction.class.getSimpleName()));
602  }
603
604  @Test
605  public void testDefaultCostFunctionList() {
606    List<String> expected = Arrays.asList(RegionCountSkewCostFunction.class.getSimpleName(),
607      PrimaryRegionCountSkewCostFunction.class.getSimpleName(),
608      MoveCostFunction.class.getSimpleName(), RackLocalityCostFunction.class.getSimpleName(),
609      TableSkewCostFunction.class.getSimpleName(),
610      RegionReplicaHostCostFunction.class.getSimpleName(),
611      RegionReplicaRackCostFunction.class.getSimpleName(),
612      ReadRequestCostFunction.class.getSimpleName(), CPRequestCostFunction.class.getSimpleName(),
613      WriteRequestCostFunction.class.getSimpleName(),
614      MemStoreSizeCostFunction.class.getSimpleName(), StoreFileCostFunction.class.getSimpleName());
615
616    List<String> actual = Arrays.asList(loadBalancer.getCostFunctionNames());
617    assertTrue("ExpectedCostFunctions: " + expected + " ActualCostFunctions: " + actual,
618      CollectionUtils.isEqualCollection(expected, actual));
619  }
620
621  private boolean needsBalanceIdleRegion(int[] cluster) {
622    return Arrays.stream(cluster).anyMatch(x -> x > 1)
623      && Arrays.stream(cluster).anyMatch(x -> x < 1);
624  }
625
626  // This mock allows us to test the LocalityCostFunction
627  private class MockCluster extends BalancerClusterState {
628
629    private int[][] localities = null; // [region][server] = percent of blocks
630
631    public MockCluster(int[][] regions) {
632
633      // regions[0] is an array where index = serverIndex an value = number of regions
634      super(mockClusterServers(regions[0], 1), null, null, null);
635
636      localities = new int[regions.length - 1][];
637      for (int i = 1; i < regions.length; i++) {
638        int regionIndex = i - 1;
639        localities[regionIndex] = new int[regions[i].length - 1];
640        regionIndexToServerIndex[regionIndex] = regions[i][0];
641        for (int j = 1; j < regions[i].length; j++) {
642          int serverIndex = j - 1;
643          localities[regionIndex][serverIndex] =
644            regions[i][j] > 100 ? regions[i][j] % 100 : regions[i][j];
645        }
646      }
647    }
648
649    @Override
650    float getLocalityOfRegion(int region, int server) {
651      // convert the locality percentage to a fraction
652      return localities[region][server] / 100.0f;
653    }
654
655    @Override
656    public int getRegionSizeMB(int region) {
657      return 1;
658    }
659  }
660}