001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.master.balancer; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertFalse; 022import static org.junit.Assert.assertNotNull; 023import static org.junit.Assert.assertNull; 024import static org.junit.Assert.assertTrue; 025import static org.mockito.Mockito.mock; 026import static org.mockito.Mockito.when; 027 028import java.util.ArrayList; 029import java.util.Arrays; 030import java.util.HashMap; 031import java.util.HashSet; 032import java.util.List; 033import java.util.Map; 034import java.util.Queue; 035import java.util.Set; 036import java.util.TreeMap; 037import org.apache.hadoop.conf.Configuration; 038import org.apache.hadoop.hbase.ClusterMetrics; 039import org.apache.hadoop.hbase.HBaseClassTestRule; 040import org.apache.hadoop.hbase.HBaseConfiguration; 041import org.apache.hadoop.hbase.HConstants; 042import org.apache.hadoop.hbase.RegionMetrics; 043import org.apache.hadoop.hbase.ServerMetrics; 044import org.apache.hadoop.hbase.ServerName; 045import org.apache.hadoop.hbase.Size; 046import org.apache.hadoop.hbase.TableName; 047import org.apache.hadoop.hbase.client.RegionInfo; 048import org.apache.hadoop.hbase.master.RegionPlan; 049import org.apache.hadoop.hbase.testclassification.MasterTests; 050import org.apache.hadoop.hbase.testclassification.MediumTests; 051import org.apache.hadoop.hbase.util.Bytes; 052import org.junit.Before; 053import org.junit.BeforeClass; 054import org.junit.ClassRule; 055import org.junit.Test; 056import org.junit.experimental.categories.Category; 057 058import org.apache.hbase.thirdparty.org.apache.commons.collections4.CollectionUtils; 059 060@Category({ MasterTests.class, MediumTests.class }) 061public class TestStochasticLoadBalancer extends StochasticBalancerTestBase { 062 063 @ClassRule 064 public static final HBaseClassTestRule CLASS_RULE = 065 HBaseClassTestRule.forClass(TestStochasticLoadBalancer.class); 066 067 private static final String REGION_KEY = "testRegion"; 068 069 // Mapping of locality test -> expected locality 070 private float[] expectedLocalities = { 1.0f, 0.0f, 0.50f, 0.25f, 1.0f }; 071 private static Configuration storedConfiguration; 072 073 @BeforeClass 074 public static void saveInitialConfiguration() { 075 storedConfiguration = new Configuration(conf); 076 } 077 078 @Before 079 public void beforeEachTest() { 080 conf = new Configuration(storedConfiguration); 081 loadBalancer.onConfigurationChange(conf); 082 } 083 084 /** 085 * Data set for testLocalityCost: [test][0][0] = mapping of server to number of regions it hosts 086 * [test][region + 1][0] = server that region is hosted on [test][region + 1][server + 1] = 087 * locality for region on server 088 */ 089 private int[][][] clusterRegionLocationMocks = new int[][][] { 090 // Test 1: each region is entirely on server that hosts it 091 new int[][] { new int[] { 2, 1, 1 }, new int[] { 2, 0, 0, 100 }, // region 0 is hosted and 092 // entirely local on server 2 093 new int[] { 0, 100, 0, 0 }, // region 1 is hosted and entirely on server 0 094 new int[] { 0, 100, 0, 0 }, // region 2 is hosted and entirely on server 0 095 new int[] { 1, 0, 100, 0 }, // region 3 is hosted and entirely on server 1 096 }, 097 098 // Test 2: each region is 0% local on the server that hosts it 099 new int[][] { new int[] { 1, 2, 1 }, new int[] { 0, 0, 0, 100 }, // region 0 is hosted and 100 // entirely local on server 2 101 new int[] { 1, 100, 0, 0 }, // region 1 is hosted and entirely on server 0 102 new int[] { 1, 100, 0, 0 }, // region 2 is hosted and entirely on server 0 103 new int[] { 2, 0, 100, 0 }, // region 3 is hosted and entirely on server 1 104 }, 105 106 // Test 3: each region is 25% local on the server that hosts it (and 50% locality is possible) 107 new int[][] { new int[] { 1, 2, 1 }, new int[] { 0, 25, 0, 50 }, // region 0 is hosted and 108 // entirely local on server 2 109 new int[] { 1, 50, 25, 0 }, // region 1 is hosted and entirely on server 0 110 new int[] { 1, 50, 25, 0 }, // region 2 is hosted and entirely on server 0 111 new int[] { 2, 0, 50, 25 }, // region 3 is hosted and entirely on server 1 112 }, 113 114 // Test 4: each region is 25% local on the server that hosts it (and 100% locality is possible) 115 new int[][] { new int[] { 1, 2, 1 }, new int[] { 0, 25, 0, 100 }, // region 0 is hosted and 116 // entirely local on server 2 117 new int[] { 1, 100, 25, 0 }, // region 1 is hosted and entirely on server 0 118 new int[] { 1, 100, 25, 0 }, // region 2 is hosted and entirely on server 0 119 new int[] { 2, 0, 100, 25 }, // region 3 is hosted and entirely on server 1 120 }, 121 122 // Test 5: each region is 75% local on the server that hosts it (and 75% locality is possible 123 // everywhere) 124 new int[][] { new int[] { 1, 2, 1 }, new int[] { 0, 75, 75, 75 }, // region 0 is hosted and 125 // entirely local on server 2 126 new int[] { 1, 75, 75, 75 }, // region 1 is hosted and entirely on server 0 127 new int[] { 1, 75, 75, 75 }, // region 2 is hosted and entirely on server 0 128 new int[] { 2, 75, 75, 75 }, // region 3 is hosted and entirely on server 1 129 }, }; 130 131 private ServerMetrics mockServerMetricsWithCpRequests(List<RegionInfo> regionsOnServer, 132 long cpRequestCount) { 133 ServerMetrics serverMetrics = mock(ServerMetrics.class); 134 Map<byte[], RegionMetrics> regionLoadMap = new TreeMap<>(Bytes.BYTES_COMPARATOR); 135 for (RegionInfo info : regionsOnServer) { 136 RegionMetrics rl = mock(RegionMetrics.class); 137 when(rl.getReadRequestCount()).thenReturn(0L); 138 when(rl.getCpRequestCount()).thenReturn(cpRequestCount); 139 when(rl.getWriteRequestCount()).thenReturn(0L); 140 when(rl.getMemStoreSize()).thenReturn(Size.ZERO); 141 when(rl.getStoreFileSize()).thenReturn(Size.ZERO); 142 regionLoadMap.put(info.getRegionName(), rl); 143 } 144 when(serverMetrics.getRegionMetrics()).thenReturn(regionLoadMap); 145 return serverMetrics; 146 } 147 148 @Test 149 public void testCPRequestCost() { 150 // in order to pass needsBalance judgement 151 conf.setFloat("hbase.master.balancer.stochastic.cpRequestCost", 10000f); 152 loadBalancer.onConfigurationChange(conf); 153 // mock cluster State 154 Map<ServerName, List<RegionInfo>> clusterState = new HashMap<ServerName, List<RegionInfo>>(); 155 ServerName serverA = randomServer(3).getServerName(); 156 ServerName serverB = randomServer(3).getServerName(); 157 ServerName serverC = randomServer(3).getServerName(); 158 List<RegionInfo> regionsOnServerA = randomRegions(3); 159 List<RegionInfo> regionsOnServerB = randomRegions(3); 160 List<RegionInfo> regionsOnServerC = randomRegions(3); 161 clusterState.put(serverA, regionsOnServerA); 162 clusterState.put(serverB, regionsOnServerB); 163 clusterState.put(serverC, regionsOnServerC); 164 // mock ClusterMetrics 165 Map<ServerName, ServerMetrics> serverMetricsMap = new TreeMap<>(); 166 serverMetricsMap.put(serverA, mockServerMetricsWithCpRequests(regionsOnServerA, 0)); 167 serverMetricsMap.put(serverB, mockServerMetricsWithCpRequests(regionsOnServerB, 0)); 168 serverMetricsMap.put(serverC, mockServerMetricsWithCpRequests(regionsOnServerC, 0)); 169 ClusterMetrics clusterStatus = mock(ClusterMetrics.class); 170 when(clusterStatus.getLiveServerMetrics()).thenReturn(serverMetricsMap); 171 loadBalancer.updateClusterMetrics(clusterStatus); 172 173 // CPRequestCostFunction are Rate based, So doing setClusterMetrics again 174 // this time, regions on serverA with more cpRequestCount load 175 // serverA : 1000,1000,1000 176 // serverB : 0,0,0 177 // serverC : 0,0,0 178 // so should move two regions from serverA to serverB & serverC 179 serverMetricsMap = new TreeMap<>(); 180 serverMetricsMap.put(serverA, mockServerMetricsWithCpRequests(regionsOnServerA, 1000)); 181 serverMetricsMap.put(serverB, mockServerMetricsWithCpRequests(regionsOnServerB, 0)); 182 serverMetricsMap.put(serverC, mockServerMetricsWithCpRequests(regionsOnServerC, 0)); 183 clusterStatus = mock(ClusterMetrics.class); 184 when(clusterStatus.getLiveServerMetrics()).thenReturn(serverMetricsMap); 185 loadBalancer.updateClusterMetrics(clusterStatus); 186 187 List<RegionPlan> plans = 188 loadBalancer.balanceTable(HConstants.ENSEMBLE_TABLE_NAME, clusterState); 189 Set<RegionInfo> regionsMoveFromServerA = new HashSet<>(); 190 Set<ServerName> targetServers = new HashSet<>(); 191 for (RegionPlan plan : plans) { 192 if (plan.getSource().equals(serverA)) { 193 regionsMoveFromServerA.add(plan.getRegionInfo()); 194 targetServers.add(plan.getDestination()); 195 } 196 } 197 // should move 2 regions from serverA, one moves to serverB, the other moves to serverC 198 assertEquals(2, regionsMoveFromServerA.size()); 199 assertEquals(2, targetServers.size()); 200 assertTrue(regionsOnServerA.containsAll(regionsMoveFromServerA)); 201 } 202 203 @Test 204 public void testKeepRegionLoad() throws Exception { 205 ServerName sn = ServerName.valueOf("test:8080", 100); 206 int numClusterStatusToAdd = 20000; 207 for (int i = 0; i < numClusterStatusToAdd; i++) { 208 ServerMetrics sl = mock(ServerMetrics.class); 209 210 RegionMetrics rl = mock(RegionMetrics.class); 211 when(rl.getReadRequestCount()).thenReturn(0L); 212 when(rl.getCpRequestCount()).thenReturn(0L); 213 when(rl.getWriteRequestCount()).thenReturn(0L); 214 when(rl.getMemStoreSize()).thenReturn(Size.ZERO); 215 when(rl.getStoreFileSize()).thenReturn(new Size(i, Size.Unit.MEGABYTE)); 216 217 Map<byte[], RegionMetrics> regionLoadMap = new TreeMap<>(Bytes.BYTES_COMPARATOR); 218 regionLoadMap.put(Bytes.toBytes(REGION_KEY), rl); 219 when(sl.getRegionMetrics()).thenReturn(regionLoadMap); 220 221 ClusterMetrics clusterStatus = mock(ClusterMetrics.class); 222 Map<ServerName, ServerMetrics> serverMetricsMap = new TreeMap<>(); 223 serverMetricsMap.put(sn, sl); 224 when(clusterStatus.getLiveServerMetrics()).thenReturn(serverMetricsMap); 225 226 loadBalancer.updateClusterMetrics(clusterStatus); 227 } 228 229 String regionNameAsString = RegionInfo.getRegionNameAsString(Bytes.toBytes(REGION_KEY)); 230 assertTrue(loadBalancer.loads.get(regionNameAsString) != null); 231 assertTrue(loadBalancer.loads.get(regionNameAsString).size() == 15); 232 233 Queue<BalancerRegionLoad> loads = loadBalancer.loads.get(regionNameAsString); 234 int i = 0; 235 while (loads.size() > 0) { 236 BalancerRegionLoad rl = loads.remove(); 237 assertEquals(i + (numClusterStatusToAdd - 15), rl.getStorefileSizeMB()); 238 i++; 239 } 240 } 241 242 @Test 243 public void testUpdateBalancerLoadInfo() { 244 int[] cluster = new int[] { 10, 0 }; 245 Map<ServerName, List<RegionInfo>> servers = mockClusterServers(cluster); 246 BalancerClusterState clusterState = mockCluster(cluster); 247 Map<TableName, Map<ServerName, List<RegionInfo>>> LoadOfAllTable = 248 (Map) mockClusterServersWithTables(servers); 249 boolean[] perTableBalancerConfigs = { true, false }; 250 for (boolean isByTable : perTableBalancerConfigs) { 251 conf.setBoolean(HConstants.HBASE_MASTER_LOADBALANCE_BYTABLE, isByTable); 252 loadBalancer.onConfigurationChange(conf); 253 dummyMetricsStochasticBalancer.clearDummyMetrics(); 254 loadBalancer.updateBalancerLoadInfo(LoadOfAllTable); 255 assertTrue("Metrics should be recorded!", 256 dummyMetricsStochasticBalancer.getDummyCostsMap() != null 257 && !dummyMetricsStochasticBalancer.getDummyCostsMap().isEmpty()); 258 259 String metricRecordKey; 260 if (isByTable) { 261 metricRecordKey = "table1#" + StochasticLoadBalancer.OVERALL_COST_FUNCTION_NAME; 262 } else { 263 metricRecordKey = 264 HConstants.ENSEMBLE_TABLE_NAME + "#" + StochasticLoadBalancer.OVERALL_COST_FUNCTION_NAME; 265 } 266 double curOverallCost = loadBalancer.computeCost(clusterState, Double.MAX_VALUE); 267 double curOverallCostInMetrics = 268 dummyMetricsStochasticBalancer.getDummyCostsMap().get(metricRecordKey); 269 assertEquals(curOverallCost, curOverallCostInMetrics, 0.001); 270 } 271 } 272 273 @Test 274 public void testUpdateStochasticCosts() { 275 int[] cluster = new int[] { 10, 0 }; 276 Map<ServerName, List<RegionInfo>> servers = mockClusterServers(cluster); 277 BalancerClusterState clusterState = mockCluster(cluster); 278 conf.setFloat("hbase.master.balancer.stochastic.minCostNeedBalance", 1.0f); 279 conf.setBoolean(HConstants.HBASE_MASTER_LOADBALANCE_BYTABLE, false); 280 loadBalancer.onConfigurationChange(conf); 281 dummyMetricsStochasticBalancer.clearDummyMetrics(); 282 List<RegionPlan> plans = 283 loadBalancer.balanceCluster((Map) mockClusterServersWithTables(servers)); 284 285 assertTrue("Balance plan should not be empty!", plans != null && !plans.isEmpty()); 286 assertTrue("There should be metrics record in MetricsStochasticBalancer", 287 !dummyMetricsStochasticBalancer.getDummyCostsMap().isEmpty()); 288 289 double overallCostOfCluster = loadBalancer.computeCost(clusterState, Double.MAX_VALUE); 290 double overallCostInMetrics = dummyMetricsStochasticBalancer.getDummyCostsMap().get( 291 HConstants.ENSEMBLE_TABLE_NAME + "#" + StochasticLoadBalancer.OVERALL_COST_FUNCTION_NAME); 292 assertEquals(overallCostOfCluster, overallCostInMetrics, 0.001); 293 } 294 295 @Test 296 public void testUpdateStochasticCostsIfBalanceNotRan() { 297 int[] cluster = new int[] { 10, 10 }; 298 Map<ServerName, List<RegionInfo>> servers = mockClusterServers(cluster); 299 BalancerClusterState clusterState = mockCluster(cluster); 300 conf.setFloat("hbase.master.balancer.stochastic.minCostNeedBalance", Float.MAX_VALUE); 301 conf.setBoolean(HConstants.HBASE_MASTER_LOADBALANCE_BYTABLE, false); 302 loadBalancer.onConfigurationChange(conf); 303 dummyMetricsStochasticBalancer.clearDummyMetrics(); 304 List<RegionPlan> plans = 305 loadBalancer.balanceCluster((Map) mockClusterServersWithTables(servers)); 306 307 assertTrue("Balance plan should be empty!", plans == null || plans.isEmpty()); 308 assertTrue("There should be metrics record in MetricsStochasticBalancer!", 309 !dummyMetricsStochasticBalancer.getDummyCostsMap().isEmpty()); 310 311 double overallCostOfCluster = loadBalancer.computeCost(clusterState, Double.MAX_VALUE); 312 double overallCostInMetrics = dummyMetricsStochasticBalancer.getDummyCostsMap().get( 313 HConstants.ENSEMBLE_TABLE_NAME + "#" + StochasticLoadBalancer.OVERALL_COST_FUNCTION_NAME); 314 assertEquals(overallCostOfCluster, overallCostInMetrics, 0.001); 315 } 316 317 @Test 318 public void testNeedBalance() { 319 conf.setFloat("hbase.master.balancer.stochastic.minCostNeedBalance", 1.0f); 320 conf.setFloat(HConstants.LOAD_BALANCER_SLOP_KEY, -1f); 321 conf.setBoolean("hbase.master.balancer.stochastic.runMaxSteps", false); 322 conf.setLong("hbase.master.balancer.stochastic.maxSteps", 5000L); 323 // Test with/without per table balancer. 324 boolean[] perTableBalancerConfigs = { true, false }; 325 for (boolean isByTable : perTableBalancerConfigs) { 326 conf.setBoolean(HConstants.HBASE_MASTER_LOADBALANCE_BYTABLE, isByTable); 327 loadBalancer.onConfigurationChange(conf); 328 for (int[] mockCluster : clusterStateMocks) { 329 assertTrue(hasEmptyBalancerPlans(mockCluster) || needsBalanceIdleRegion(mockCluster)); 330 } 331 } 332 } 333 334 @Test 335 public void testBalanceOfSloppyServers() { 336 // We are testing slop checks, so don't "accidentally" balance due to a minCost calculation. 337 // During development, imbalance of a 100 server cluster, with one node having 10 regions 338 // and the rest having 5, is 0.0048. With minCostNeedBalance default of 0.025, test should 339 // validate slop checks without this override. We override just to ensure we will always 340 // validate slop check here, and for small clusters as well. 341 conf.setFloat("hbase.master.balancer.stochastic.minCostNeedBalance", 1.0f); 342 conf.setBoolean("hbase.master.balancer.stochastic.runMaxSteps", false); 343 conf.setLong("hbase.master.balancer.stochastic.maxSteps", 5000L); 344 loadBalancer.onConfigurationChange(conf); 345 for (int[] mockCluster : clusterStateMocksWithNoSlop) { 346 assertTrue(hasEmptyBalancerPlans(mockCluster)); 347 } 348 for (int[] mockCluster : clusterStateMocksWithSlop) { 349 assertFalse(hasEmptyBalancerPlans(mockCluster)); 350 } 351 } 352 353 @Test 354 public void testSloppyTablesLoadBalanceByTable() { 355 int[][] regionsPerServerPerTable = new int[][] { 356 new int[] { 8, 2, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 357 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5 }, 358 new int[] { 2, 8, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 359 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5 }, }; 360 conf.setFloat("hbase.master.balancer.stochastic.minCostNeedBalance", 1.0f); 361 conf.setBoolean("hbase.master.balancer.stochastic.runMaxSteps", false); 362 conf.setLong("hbase.master.balancer.stochastic.maxSteps", 5000L); 363 conf.setBoolean(HConstants.HBASE_MASTER_LOADBALANCE_BYTABLE, true); 364 loadBalancer.onConfigurationChange(conf); 365 assertFalse(hasEmptyBalancerPlans(regionsPerServerPerTable)); 366 } 367 368 private boolean hasEmptyBalancerPlans(int[] mockCluster) { 369 Map<ServerName, List<RegionInfo>> servers = mockClusterServers(mockCluster); 370 return hasEmptyBalancerPlans(servers); 371 } 372 373 private boolean hasEmptyBalancerPlans(int[][] mockCluster) { 374 Map<ServerName, List<RegionInfo>> servers = mockClusterServers(mockCluster); 375 return hasEmptyBalancerPlans(servers); 376 } 377 378 private boolean hasEmptyBalancerPlans(Map<ServerName, List<RegionInfo>> servers) { 379 Map<TableName, Map<ServerName, List<RegionInfo>>> loadOfAllTable = 380 (Map) mockClusterServersWithTables(servers); 381 List<RegionPlan> plans = loadBalancer.balanceCluster(loadOfAllTable); 382 return plans == null || plans.isEmpty(); 383 } 384 385 @Test 386 public void testLocalityCost() throws Exception { 387 Configuration conf = HBaseConfiguration.create(); 388 CostFunction costFunction = new ServerLocalityCostFunction(conf); 389 390 for (int test = 0; test < clusterRegionLocationMocks.length; test++) { 391 int[][] clusterRegionLocations = clusterRegionLocationMocks[test]; 392 MockCluster cluster = new MockCluster(clusterRegionLocations); 393 costFunction.prepare(cluster); 394 double cost = costFunction.cost(); 395 double expected = 1 - expectedLocalities[test]; 396 assertEquals(expected, cost, 0.001); 397 } 398 } 399 400 @Test 401 public void testMoveCostMultiplier() throws Exception { 402 Configuration conf = HBaseConfiguration.create(); 403 ClusterInfoProvider provider = mock(ClusterInfoProvider.class); 404 CostFunction costFunction = new MoveCostFunction(conf, provider); 405 when(provider.isOffPeakHour()).thenReturn(false); 406 BalancerClusterState cluster = mockCluster(clusterStateMocks[0]); 407 costFunction.prepare(cluster); 408 costFunction.cost(); 409 assertEquals(MoveCostFunction.DEFAULT_MOVE_COST, costFunction.getMultiplier(), 0.01); 410 411 // In offpeak hours, the multiplier of move cost should be lower 412 when(provider.isOffPeakHour()).thenReturn(true); 413 costFunction.prepare(cluster); 414 costFunction.cost(); 415 assertEquals(MoveCostFunction.DEFAULT_MOVE_COST_OFFPEAK, costFunction.getMultiplier(), 0.01); 416 } 417 418 @Test 419 public void testMoveCost() throws Exception { 420 Configuration conf = HBaseConfiguration.create(); 421 CostFunction costFunction = new MoveCostFunction(conf, new DummyClusterInfoProvider(conf)); 422 for (int[] mockCluster : clusterStateMocks) { 423 BalancerClusterState cluster = mockCluster(mockCluster); 424 costFunction.prepare(cluster); 425 double cost = costFunction.cost(); 426 assertEquals(0.0f, cost, 0.001); 427 428 // cluster region number is smaller than maxMoves=600 429 cluster.setNumRegions(200); 430 cluster.setNumMovedRegions(10); 431 cost = costFunction.cost(); 432 assertEquals(0.05f, cost, 0.001); 433 cluster.setNumMovedRegions(100); 434 cost = costFunction.cost(); 435 assertEquals(0.5f, cost, 0.001); 436 cluster.setNumMovedRegions(200); 437 cost = costFunction.cost(); 438 assertEquals(1.0f, cost, 0.001); 439 440 // cluster region number is bigger than maxMoves=2500 441 cluster.setNumRegions(10000); 442 cluster.setNumMovedRegions(250); 443 cost = costFunction.cost(); 444 assertEquals(0.025f, cost, 0.001); 445 cluster.setNumMovedRegions(1250); 446 cost = costFunction.cost(); 447 assertEquals(0.125f, cost, 0.001); 448 cluster.setNumMovedRegions(2500); 449 cost = costFunction.cost(); 450 assertEquals(0.25f, cost, 0.01); 451 } 452 } 453 454 @Test 455 public void testSkewCost() { 456 Configuration conf = HBaseConfiguration.create(); 457 CostFunction costFunction = new RegionCountSkewCostFunction(conf); 458 for (int[] mockCluster : clusterStateMocks) { 459 costFunction.prepare(mockCluster(mockCluster)); 460 double cost = costFunction.cost(); 461 assertTrue(cost >= 0); 462 assertTrue(cost <= 1.01); 463 } 464 465 costFunction.prepare(mockCluster(new int[] { 0, 0, 0, 0, 1 })); 466 assertEquals(0, costFunction.cost(), 0.01); 467 costFunction.prepare(mockCluster(new int[] { 0, 0, 0, 1, 1 })); 468 assertEquals(0, costFunction.cost(), 0.01); 469 costFunction.prepare(mockCluster(new int[] { 0, 0, 1, 1, 1 })); 470 assertEquals(0, costFunction.cost(), 0.01); 471 costFunction.prepare(mockCluster(new int[] { 0, 1, 1, 1, 1 })); 472 assertEquals(0, costFunction.cost(), 0.01); 473 costFunction.prepare(mockCluster(new int[] { 1, 1, 1, 1, 1 })); 474 assertEquals(0, costFunction.cost(), 0.01); 475 costFunction.prepare(mockCluster(new int[] { 10000, 0, 0, 0, 0 })); 476 assertEquals(1, costFunction.cost(), 0.01); 477 } 478 479 @Test 480 public void testCostAfterUndoAction() { 481 final int runs = 10; 482 for (int[] mockCluster : clusterStateMocks) { 483 BalancerClusterState cluster = mockCluster(mockCluster); 484 loadBalancer.initCosts(cluster); 485 for (int i = 0; i != runs; ++i) { 486 final double expectedCost = loadBalancer.computeCost(cluster, Double.MAX_VALUE); 487 BalanceAction action = loadBalancer.nextAction(cluster); 488 cluster.doAction(action); 489 loadBalancer.updateCostsAndWeightsWithAction(cluster, action); 490 BalanceAction undoAction = action.undoAction(); 491 cluster.doAction(undoAction); 492 loadBalancer.updateCostsAndWeightsWithAction(cluster, undoAction); 493 final double actualCost = loadBalancer.computeCost(cluster, Double.MAX_VALUE); 494 assertEquals(expectedCost, actualCost, 0); 495 } 496 } 497 } 498 499 @Test 500 public void testTableSkewCost() { 501 Configuration conf = HBaseConfiguration.create(); 502 CostFunction costFunction = new TableSkewCostFunction(conf); 503 for (int[] mockCluster : clusterStateMocks) { 504 BalancerClusterState cluster = mockCluster(mockCluster); 505 costFunction.prepare(cluster); 506 double cost = costFunction.cost(); 507 assertTrue(cost >= 0); 508 assertTrue(cost <= 1.01); 509 } 510 } 511 512 @Test 513 public void testRegionLoadCost() { 514 List<BalancerRegionLoad> regionLoads = new ArrayList<>(); 515 for (int i = 1; i < 5; i++) { 516 BalancerRegionLoad regionLoad = mock(BalancerRegionLoad.class); 517 when(regionLoad.getReadRequestsCount()).thenReturn((long) i); 518 when(regionLoad.getCpRequestsCount()).thenReturn((long) i); 519 when(regionLoad.getStorefileSizeMB()).thenReturn(i); 520 regionLoads.add(regionLoad); 521 } 522 523 Configuration conf = HBaseConfiguration.create(); 524 ReadRequestCostFunction readCostFunction = new ReadRequestCostFunction(conf); 525 double rateResult = readCostFunction.getRegionLoadCost(regionLoads); 526 // read requests are treated as a rate so the average rate here is simply 1 527 assertEquals(1, rateResult, 0.01); 528 529 CPRequestCostFunction cpCostFunction = new CPRequestCostFunction(conf); 530 rateResult = cpCostFunction.getRegionLoadCost(regionLoads); 531 // coprocessor requests are treated as a rate so the average rate here is simply 1 532 assertEquals(1, rateResult, 0.01); 533 534 StoreFileCostFunction storeFileCostFunction = new StoreFileCostFunction(conf); 535 double result = storeFileCostFunction.getRegionLoadCost(regionLoads); 536 // storefile size cost is simply an average of it's value over time 537 assertEquals(2.5, result, 0.01); 538 } 539 540 @Test 541 public void testRegionLoadCostWhenDecrease() { 542 List<BalancerRegionLoad> regionLoads = new ArrayList<>(); 543 // test region loads of [1,2,1,4] 544 for (int i = 1; i < 5; i++) { 545 int load = i == 3 ? 1 : i; 546 BalancerRegionLoad regionLoad = mock(BalancerRegionLoad.class); 547 when(regionLoad.getReadRequestsCount()).thenReturn((long) load); 548 when(regionLoad.getCpRequestsCount()).thenReturn((long) load); 549 regionLoads.add(regionLoad); 550 } 551 552 Configuration conf = HBaseConfiguration.create(); 553 ReadRequestCostFunction readCostFunction = new ReadRequestCostFunction(conf); 554 double rateResult = readCostFunction.getRegionLoadCost(regionLoads); 555 // read requests are treated as a rate so the average rate here is simply 1 556 assertEquals(1.67, rateResult, 0.01); 557 558 CPRequestCostFunction cpCostFunction = new CPRequestCostFunction(conf); 559 rateResult = cpCostFunction.getRegionLoadCost(regionLoads); 560 // coprocessor requests are treated as a rate so the average rate here is simply 1 561 assertEquals(1.67, rateResult, 0.01); 562 } 563 564 @Test 565 public void testLosingRs() throws Exception { 566 int numNodes = 3; 567 int numRegions = 20; 568 int numRegionsPerServer = 3; // all servers except one 569 int replication = 1; 570 int numTables = 2; 571 572 Map<ServerName, List<RegionInfo>> serverMap = 573 createServerMap(numNodes, numRegions, numRegionsPerServer, replication, numTables); 574 List<ServerAndLoad> list = convertToList(serverMap); 575 576 List<RegionPlan> plans = loadBalancer.balanceTable(HConstants.ENSEMBLE_TABLE_NAME, serverMap); 577 assertNotNull(plans); 578 579 // Apply the plan to the mock cluster. 580 List<ServerAndLoad> balancedCluster = reconcile(list, plans, serverMap); 581 582 assertClusterAsBalanced(balancedCluster); 583 584 ServerName sn = serverMap.keySet().toArray(new ServerName[serverMap.size()])[0]; 585 586 ServerName deadSn = ServerName.valueOf(sn.getHostname(), sn.getPort(), sn.getStartcode() - 100); 587 588 serverMap.put(deadSn, new ArrayList<>(0)); 589 590 plans = loadBalancer.balanceTable(HConstants.ENSEMBLE_TABLE_NAME, serverMap); 591 assertNull(plans); 592 } 593 594 @Test 595 public void testAdditionalCostFunction() { 596 conf.set(StochasticLoadBalancer.COST_FUNCTIONS_COST_FUNCTIONS_KEY, 597 DummyCostFunction.class.getName()); 598 599 loadBalancer.onConfigurationChange(conf); 600 assertTrue(Arrays.asList(loadBalancer.getCostFunctionNames()) 601 .contains(DummyCostFunction.class.getSimpleName())); 602 } 603 604 @Test 605 public void testDefaultCostFunctionList() { 606 List<String> expected = Arrays.asList(RegionCountSkewCostFunction.class.getSimpleName(), 607 PrimaryRegionCountSkewCostFunction.class.getSimpleName(), 608 MoveCostFunction.class.getSimpleName(), RackLocalityCostFunction.class.getSimpleName(), 609 TableSkewCostFunction.class.getSimpleName(), 610 RegionReplicaHostCostFunction.class.getSimpleName(), 611 RegionReplicaRackCostFunction.class.getSimpleName(), 612 ReadRequestCostFunction.class.getSimpleName(), CPRequestCostFunction.class.getSimpleName(), 613 WriteRequestCostFunction.class.getSimpleName(), 614 MemStoreSizeCostFunction.class.getSimpleName(), StoreFileCostFunction.class.getSimpleName()); 615 616 List<String> actual = Arrays.asList(loadBalancer.getCostFunctionNames()); 617 assertTrue("ExpectedCostFunctions: " + expected + " ActualCostFunctions: " + actual, 618 CollectionUtils.isEqualCollection(expected, actual)); 619 } 620 621 private boolean needsBalanceIdleRegion(int[] cluster) { 622 return Arrays.stream(cluster).anyMatch(x -> x > 1) 623 && Arrays.stream(cluster).anyMatch(x -> x < 1); 624 } 625 626 // This mock allows us to test the LocalityCostFunction 627 private class MockCluster extends BalancerClusterState { 628 629 private int[][] localities = null; // [region][server] = percent of blocks 630 631 public MockCluster(int[][] regions) { 632 633 // regions[0] is an array where index = serverIndex an value = number of regions 634 super(mockClusterServers(regions[0], 1), null, null, null); 635 636 localities = new int[regions.length - 1][]; 637 for (int i = 1; i < regions.length; i++) { 638 int regionIndex = i - 1; 639 localities[regionIndex] = new int[regions[i].length - 1]; 640 regionIndexToServerIndex[regionIndex] = regions[i][0]; 641 for (int j = 1; j < regions[i].length; j++) { 642 int serverIndex = j - 1; 643 localities[regionIndex][serverIndex] = 644 regions[i][j] > 100 ? regions[i][j] % 100 : regions[i][j]; 645 } 646 } 647 } 648 649 @Override 650 float getLocalityOfRegion(int region, int server) { 651 // convert the locality percentage to a fraction 652 return localities[region][server] / 100.0f; 653 } 654 655 @Override 656 public int getRegionSizeMB(int region) { 657 return 1; 658 } 659 } 660}