Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.balancer;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertTrue;
022import static org.mockito.Mockito.mock;
023import static org.mockito.Mockito.when;
024
025import java.util.ArrayList;
026import java.util.HashMap;
027import java.util.HashSet;
028import java.util.List;
029import java.util.Map;
030import java.util.Random;
031import java.util.Set;
032import java.util.TreeMap;
033import java.util.concurrent.ThreadLocalRandom;
034import org.apache.hadoop.conf.Configuration;
035import org.apache.hadoop.hbase.ClusterMetrics;
036import org.apache.hadoop.hbase.HBaseClassTestRule;
037import org.apache.hadoop.hbase.HBaseConfiguration;
038import org.apache.hadoop.hbase.HConstants;
039import org.apache.hadoop.hbase.RegionMetrics;
040import org.apache.hadoop.hbase.ServerMetrics;
041import org.apache.hadoop.hbase.ServerName;
042import org.apache.hadoop.hbase.Size;
043import org.apache.hadoop.hbase.TableName;
044import org.apache.hadoop.hbase.client.RegionInfo;
045import org.apache.hadoop.hbase.client.TableDescriptor;
046import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
047import org.apache.hadoop.hbase.master.RegionPlan;
048import org.apache.hadoop.hbase.testclassification.LargeTests;
049import org.apache.hadoop.hbase.util.Bytes;
050import org.junit.BeforeClass;
051import org.junit.ClassRule;
052import org.junit.Test;
053import org.junit.experimental.categories.Category;
054import org.slf4j.Logger;
055import org.slf4j.LoggerFactory;
056
057import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
058
059@Category({ LargeTests.class })
060public class TestCacheAwareLoadBalancer extends BalancerTestBase {
061  @ClassRule
062  public static final HBaseClassTestRule CLASS_RULE =
063    HBaseClassTestRule.forClass(TestCacheAwareLoadBalancer.class);
064
065  private static final Logger LOG = LoggerFactory.getLogger(TestCacheAwareLoadBalancer.class);
066
067  private static CacheAwareLoadBalancer loadBalancer;
068
069  static List<ServerName> servers;
070
071  static List<TableDescriptor> tableDescs;
072
073  static Map<TableName, String> tableMap = new HashMap<>();
074
075  static TableName[] tables = new TableName[] { TableName.valueOf("dt1"), TableName.valueOf("dt2"),
076    TableName.valueOf("dt3"), TableName.valueOf("dt4") };
077
078  private static List<ServerName> generateServers(int numServers) {
079    List<ServerName> servers = new ArrayList<>(numServers);
080    Random rand = ThreadLocalRandom.current();
081    for (int i = 0; i < numServers; i++) {
082      String host = "server" + rand.nextInt(100000);
083      int port = rand.nextInt(60000);
084      servers.add(ServerName.valueOf(host, port, -1));
085    }
086    return servers;
087  }
088
089  private static List<TableDescriptor> constructTableDesc(boolean hasBogusTable) {
090    List<TableDescriptor> tds = Lists.newArrayList();
091    for (int i = 0; i < tables.length; i++) {
092      TableDescriptor htd = TableDescriptorBuilder.newBuilder(tables[i]).build();
093      tds.add(htd);
094    }
095    return tds;
096  }
097
098  private ServerMetrics mockServerMetricsWithRegionCacheInfo(ServerName server,
099    List<RegionInfo> regionsOnServer, float currentCacheRatio, List<RegionInfo> oldRegionCacheInfo,
100    int oldRegionCachedSize, int regionSize) {
101    ServerMetrics serverMetrics = mock(ServerMetrics.class);
102    Map<byte[], RegionMetrics> regionLoadMap = new TreeMap<>(Bytes.BYTES_COMPARATOR);
103    for (RegionInfo info : regionsOnServer) {
104      RegionMetrics rl = mock(RegionMetrics.class);
105      when(rl.getReadRequestCount()).thenReturn(0L);
106      when(rl.getWriteRequestCount()).thenReturn(0L);
107      when(rl.getMemStoreSize()).thenReturn(Size.ZERO);
108      when(rl.getStoreFileSize()).thenReturn(Size.ZERO);
109      when(rl.getCurrentRegionCachedRatio()).thenReturn(currentCacheRatio);
110      when(rl.getRegionSizeMB()).thenReturn(new Size(regionSize, Size.Unit.MEGABYTE));
111      regionLoadMap.put(info.getRegionName(), rl);
112    }
113    when(serverMetrics.getRegionMetrics()).thenReturn(regionLoadMap);
114    Map<String, Integer> oldCacheRatioMap = new HashMap<>();
115    for (RegionInfo info : oldRegionCacheInfo) {
116      oldCacheRatioMap.put(info.getEncodedName(), oldRegionCachedSize);
117    }
118    when(serverMetrics.getRegionCachedInfo()).thenReturn(oldCacheRatioMap);
119    return serverMetrics;
120  }
121
122  @BeforeClass
123  public static void beforeAllTests() throws Exception {
124    servers = generateServers(3);
125    tableDescs = constructTableDesc(false);
126    Configuration conf = HBaseConfiguration.create();
127    conf.set(HConstants.BUCKET_CACHE_PERSISTENT_PATH_KEY, "prefetch_file_list");
128    loadBalancer = new CacheAwareLoadBalancer();
129    loadBalancer.setClusterInfoProvider(new DummyClusterInfoProvider(conf));
130    loadBalancer.loadConf(conf);
131  }
132
133  @Test
134  public void testRegionsNotCachedOnOldServerAndCurrentServer() throws Exception {
135    // The regions are not cached on old server as well as the current server. This causes
136    // skewness in the region allocation which should be fixed by the balancer
137
138    Map<ServerName, List<RegionInfo>> clusterState = new HashMap<>();
139    ServerName server0 = servers.get(0);
140    ServerName server1 = servers.get(1);
141    ServerName server2 = servers.get(2);
142
143    // Simulate that the regions previously hosted by server1 are now hosted on server0
144    List<RegionInfo> regionsOnServer0 = randomRegions(10);
145    List<RegionInfo> regionsOnServer1 = randomRegions(0);
146    List<RegionInfo> regionsOnServer2 = randomRegions(5);
147
148    clusterState.put(server0, regionsOnServer0);
149    clusterState.put(server1, regionsOnServer1);
150    clusterState.put(server2, regionsOnServer2);
151
152    // Mock cluster metrics
153    Map<ServerName, ServerMetrics> serverMetricsMap = new TreeMap<>();
154    serverMetricsMap.put(server0, mockServerMetricsWithRegionCacheInfo(server0, regionsOnServer0,
155      0.0f, new ArrayList<>(), 0, 10));
156    serverMetricsMap.put(server1, mockServerMetricsWithRegionCacheInfo(server1, regionsOnServer1,
157      0.0f, new ArrayList<>(), 0, 10));
158    serverMetricsMap.put(server2, mockServerMetricsWithRegionCacheInfo(server2, regionsOnServer2,
159      0.0f, new ArrayList<>(), 0, 10));
160    ClusterMetrics clusterMetrics = mock(ClusterMetrics.class);
161    when(clusterMetrics.getLiveServerMetrics()).thenReturn(serverMetricsMap);
162    loadBalancer.updateClusterMetrics(clusterMetrics);
163
164    Map<TableName, Map<ServerName, List<RegionInfo>>> LoadOfAllTable =
165      (Map) mockClusterServersWithTables(clusterState);
166    List<RegionPlan> plans = loadBalancer.balanceCluster(LoadOfAllTable);
167    Set<RegionInfo> regionsMovedFromServer0 = new HashSet<>();
168    Map<ServerName, List<RegionInfo>> targetServers = new HashMap<>();
169    for (RegionPlan plan : plans) {
170      if (plan.getSource().equals(server0)) {
171        regionsMovedFromServer0.add(plan.getRegionInfo());
172        if (!targetServers.containsKey(plan.getDestination())) {
173          targetServers.put(plan.getDestination(), new ArrayList<>());
174        }
175        targetServers.get(plan.getDestination()).add(plan.getRegionInfo());
176      }
177    }
178    // should move 5 regions from server0 to server 1
179    assertEquals(5, regionsMovedFromServer0.size());
180    assertEquals(5, targetServers.get(server1).size());
181  }
182
183  @Test
184  public void testRegionsPartiallyCachedOnOldServerAndNotCachedOnCurrentServer() throws Exception {
185    // The regions are partially cached on old server but not cached on the current server
186
187    Map<ServerName, List<RegionInfo>> clusterState = new HashMap<>();
188    ServerName server0 = servers.get(0);
189    ServerName server1 = servers.get(1);
190    ServerName server2 = servers.get(2);
191
192    // Simulate that the regions previously hosted by server1 are now hosted on server0
193    List<RegionInfo> regionsOnServer0 = randomRegions(10);
194    List<RegionInfo> regionsOnServer1 = randomRegions(0);
195    List<RegionInfo> regionsOnServer2 = randomRegions(5);
196
197    clusterState.put(server0, regionsOnServer0);
198    clusterState.put(server1, regionsOnServer1);
199    clusterState.put(server2, regionsOnServer2);
200
201    // Mock cluster metrics
202
203    // Mock 5 regions from server0 were previously hosted on server1
204    List<RegionInfo> oldCachedRegions = regionsOnServer0.subList(5, regionsOnServer0.size() - 1);
205
206    Map<ServerName, ServerMetrics> serverMetricsMap = new TreeMap<>();
207    serverMetricsMap.put(server0, mockServerMetricsWithRegionCacheInfo(server0, regionsOnServer0,
208      0.0f, new ArrayList<>(), 0, 10));
209    serverMetricsMap.put(server1, mockServerMetricsWithRegionCacheInfo(server1, regionsOnServer1,
210      0.0f, oldCachedRegions, 6, 10));
211    serverMetricsMap.put(server2, mockServerMetricsWithRegionCacheInfo(server2, regionsOnServer2,
212      0.0f, new ArrayList<>(), 0, 10));
213    ClusterMetrics clusterMetrics = mock(ClusterMetrics.class);
214    when(clusterMetrics.getLiveServerMetrics()).thenReturn(serverMetricsMap);
215    loadBalancer.updateClusterMetrics(clusterMetrics);
216
217    Map<TableName, Map<ServerName, List<RegionInfo>>> LoadOfAllTable =
218      (Map) mockClusterServersWithTables(clusterState);
219    List<RegionPlan> plans = loadBalancer.balanceCluster(LoadOfAllTable);
220    Set<RegionInfo> regionsMovedFromServer0 = new HashSet<>();
221    Map<ServerName, List<RegionInfo>> targetServers = new HashMap<>();
222    for (RegionPlan plan : plans) {
223      if (plan.getSource().equals(server0)) {
224        regionsMovedFromServer0.add(plan.getRegionInfo());
225        if (!targetServers.containsKey(plan.getDestination())) {
226          targetServers.put(plan.getDestination(), new ArrayList<>());
227        }
228        targetServers.get(plan.getDestination()).add(plan.getRegionInfo());
229      }
230    }
231    // should move 5 regions from server0 to server1
232    assertEquals(5, regionsMovedFromServer0.size());
233    assertEquals(5, targetServers.get(server1).size());
234    assertTrue(targetServers.get(server1).containsAll(oldCachedRegions));
235  }
236
237  @Test
238  public void testRegionsFullyCachedOnOldServerAndNotCachedOnCurrentServers() throws Exception {
239    // The regions are fully cached on old server
240
241    Map<ServerName, List<RegionInfo>> clusterState = new HashMap<>();
242    ServerName server0 = servers.get(0);
243    ServerName server1 = servers.get(1);
244    ServerName server2 = servers.get(2);
245
246    // Simulate that the regions previously hosted by server1 are now hosted on server0
247    List<RegionInfo> regionsOnServer0 = randomRegions(10);
248    List<RegionInfo> regionsOnServer1 = randomRegions(0);
249    List<RegionInfo> regionsOnServer2 = randomRegions(5);
250
251    clusterState.put(server0, regionsOnServer0);
252    clusterState.put(server1, regionsOnServer1);
253    clusterState.put(server2, regionsOnServer2);
254
255    // Mock cluster metrics
256
257    // Mock 5 regions from server0 were previously hosted on server1
258    List<RegionInfo> oldCachedRegions = regionsOnServer0.subList(5, regionsOnServer0.size() - 1);
259
260    Map<ServerName, ServerMetrics> serverMetricsMap = new TreeMap<>();
261    serverMetricsMap.put(server0, mockServerMetricsWithRegionCacheInfo(server0, regionsOnServer0,
262      0.0f, new ArrayList<>(), 0, 10));
263    serverMetricsMap.put(server1, mockServerMetricsWithRegionCacheInfo(server1, regionsOnServer1,
264      0.0f, oldCachedRegions, 10, 10));
265    serverMetricsMap.put(server2, mockServerMetricsWithRegionCacheInfo(server2, regionsOnServer2,
266      0.0f, new ArrayList<>(), 0, 10));
267    ClusterMetrics clusterMetrics = mock(ClusterMetrics.class);
268    when(clusterMetrics.getLiveServerMetrics()).thenReturn(serverMetricsMap);
269    loadBalancer.updateClusterMetrics(clusterMetrics);
270
271    Map<TableName, Map<ServerName, List<RegionInfo>>> LoadOfAllTable =
272      (Map) mockClusterServersWithTables(clusterState);
273    List<RegionPlan> plans = loadBalancer.balanceCluster(LoadOfAllTable);
274    Set<RegionInfo> regionsMovedFromServer0 = new HashSet<>();
275    Map<ServerName, List<RegionInfo>> targetServers = new HashMap<>();
276    for (RegionPlan plan : plans) {
277      if (plan.getSource().equals(server0)) {
278        regionsMovedFromServer0.add(plan.getRegionInfo());
279        if (!targetServers.containsKey(plan.getDestination())) {
280          targetServers.put(plan.getDestination(), new ArrayList<>());
281        }
282        targetServers.get(plan.getDestination()).add(plan.getRegionInfo());
283      }
284    }
285    // should move 5 regions from server0 to server1
286    assertEquals(5, regionsMovedFromServer0.size());
287    assertEquals(5, targetServers.get(server1).size());
288    assertTrue(targetServers.get(server1).containsAll(oldCachedRegions));
289  }
290
291  @Test
292  public void testRegionsFullyCachedOnOldAndCurrentServers() throws Exception {
293    // The regions are fully cached on old server
294
295    Map<ServerName, List<RegionInfo>> clusterState = new HashMap<>();
296    ServerName server0 = servers.get(0);
297    ServerName server1 = servers.get(1);
298    ServerName server2 = servers.get(2);
299
300    // Simulate that the regions previously hosted by server1 are now hosted on server0
301    List<RegionInfo> regionsOnServer0 = randomRegions(10);
302    List<RegionInfo> regionsOnServer1 = randomRegions(0);
303    List<RegionInfo> regionsOnServer2 = randomRegions(5);
304
305    clusterState.put(server0, regionsOnServer0);
306    clusterState.put(server1, regionsOnServer1);
307    clusterState.put(server2, regionsOnServer2);
308
309    // Mock cluster metrics
310
311    // Mock 5 regions from server0 were previously hosted on server1
312    List<RegionInfo> oldCachedRegions = regionsOnServer0.subList(5, regionsOnServer0.size() - 1);
313
314    Map<ServerName, ServerMetrics> serverMetricsMap = new TreeMap<>();
315    serverMetricsMap.put(server0, mockServerMetricsWithRegionCacheInfo(server0, regionsOnServer0,
316      1.0f, new ArrayList<>(), 0, 10));
317    serverMetricsMap.put(server1, mockServerMetricsWithRegionCacheInfo(server1, regionsOnServer1,
318      1.0f, oldCachedRegions, 10, 10));
319    serverMetricsMap.put(server2, mockServerMetricsWithRegionCacheInfo(server2, regionsOnServer2,
320      1.0f, new ArrayList<>(), 0, 10));
321    ClusterMetrics clusterMetrics = mock(ClusterMetrics.class);
322    when(clusterMetrics.getLiveServerMetrics()).thenReturn(serverMetricsMap);
323    loadBalancer.updateClusterMetrics(clusterMetrics);
324
325    Map<TableName, Map<ServerName, List<RegionInfo>>> LoadOfAllTable =
326      (Map) mockClusterServersWithTables(clusterState);
327    List<RegionPlan> plans = loadBalancer.balanceCluster(LoadOfAllTable);
328    Set<RegionInfo> regionsMovedFromServer0 = new HashSet<>();
329    Map<ServerName, List<RegionInfo>> targetServers = new HashMap<>();
330    for (RegionPlan plan : plans) {
331      if (plan.getSource().equals(server0)) {
332        regionsMovedFromServer0.add(plan.getRegionInfo());
333        if (!targetServers.containsKey(plan.getDestination())) {
334          targetServers.put(plan.getDestination(), new ArrayList<>());
335        }
336        targetServers.get(plan.getDestination()).add(plan.getRegionInfo());
337      }
338    }
339    // should move 5 regions from server0 to server1
340    assertEquals(5, regionsMovedFromServer0.size());
341    assertEquals(5, targetServers.get(server1).size());
342    assertTrue(targetServers.get(server1).containsAll(oldCachedRegions));
343  }
344
345  @Test
346  public void testRegionsPartiallyCachedOnOldServerAndCurrentServer() throws Exception {
347    // The regions are partially cached on old server
348
349    Map<ServerName, List<RegionInfo>> clusterState = new HashMap<>();
350    ServerName server0 = servers.get(0);
351    ServerName server1 = servers.get(1);
352    ServerName server2 = servers.get(2);
353
354    // Simulate that the regions previously hosted by server1 are now hosted on server0
355    List<RegionInfo> regionsOnServer0 = randomRegions(10);
356    List<RegionInfo> regionsOnServer1 = randomRegions(0);
357    List<RegionInfo> regionsOnServer2 = randomRegions(5);
358
359    clusterState.put(server0, regionsOnServer0);
360    clusterState.put(server1, regionsOnServer1);
361    clusterState.put(server2, regionsOnServer2);
362
363    // Mock cluster metrics
364
365    // Mock 5 regions from server0 were previously hosted on server1
366    List<RegionInfo> oldCachedRegions = regionsOnServer0.subList(5, regionsOnServer0.size() - 1);
367
368    Map<ServerName, ServerMetrics> serverMetricsMap = new TreeMap<>();
369    serverMetricsMap.put(server0, mockServerMetricsWithRegionCacheInfo(server0, regionsOnServer0,
370      0.2f, new ArrayList<>(), 0, 10));
371    serverMetricsMap.put(server1, mockServerMetricsWithRegionCacheInfo(server1, regionsOnServer1,
372      0.0f, oldCachedRegions, 6, 10));
373    serverMetricsMap.put(server2, mockServerMetricsWithRegionCacheInfo(server2, regionsOnServer2,
374      1.0f, new ArrayList<>(), 0, 10));
375    ClusterMetrics clusterMetrics = mock(ClusterMetrics.class);
376    when(clusterMetrics.getLiveServerMetrics()).thenReturn(serverMetricsMap);
377    loadBalancer.updateClusterMetrics(clusterMetrics);
378
379    Map<TableName, Map<ServerName, List<RegionInfo>>> LoadOfAllTable =
380      (Map) mockClusterServersWithTables(clusterState);
381    List<RegionPlan> plans = loadBalancer.balanceCluster(LoadOfAllTable);
382    Set<RegionInfo> regionsMovedFromServer0 = new HashSet<>();
383    Map<ServerName, List<RegionInfo>> targetServers = new HashMap<>();
384    for (RegionPlan plan : plans) {
385      if (plan.getSource().equals(server0)) {
386        regionsMovedFromServer0.add(plan.getRegionInfo());
387        if (!targetServers.containsKey(plan.getDestination())) {
388          targetServers.put(plan.getDestination(), new ArrayList<>());
389        }
390        targetServers.get(plan.getDestination()).add(plan.getRegionInfo());
391      }
392    }
393    assertEquals(5, regionsMovedFromServer0.size());
394    assertEquals(5, targetServers.get(server1).size());
395    assertTrue(targetServers.get(server1).containsAll(oldCachedRegions));
396  }
397}