Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.balancer;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNotEquals;
022import static org.junit.Assert.assertNotNull;
023import static org.junit.Assert.assertTrue;
024
025import java.io.IOException;
026import java.util.Arrays;
027import java.util.EnumSet;
028import java.util.List;
029import java.util.Map;
030import org.apache.hadoop.conf.Configuration;
031import org.apache.hadoop.hbase.ClusterMetrics;
032import org.apache.hadoop.hbase.ClusterMetrics.Option;
033import org.apache.hadoop.hbase.HBaseClassTestRule;
034import org.apache.hadoop.hbase.HBaseTestingUtility;
035import org.apache.hadoop.hbase.HConstants;
036import org.apache.hadoop.hbase.MiniHBaseCluster;
037import org.apache.hadoop.hbase.ServerName;
038import org.apache.hadoop.hbase.TableName;
039import org.apache.hadoop.hbase.Waiter;
040import org.apache.hadoop.hbase.client.Admin;
041import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
042import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
043import org.apache.hadoop.hbase.client.RegionInfo;
044import org.apache.hadoop.hbase.client.TableDescriptor;
045import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
046import org.apache.hadoop.hbase.favored.FavoredNodesManager;
047import org.apache.hadoop.hbase.master.HMaster;
048import org.apache.hadoop.hbase.master.LoadBalancer;
049import org.apache.hadoop.hbase.master.RackManager;
050import org.apache.hadoop.hbase.master.assignment.RegionStates;
051import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster;
052import org.apache.hadoop.hbase.regionserver.HRegion;
053import org.apache.hadoop.hbase.regionserver.HRegionServer;
054import org.apache.hadoop.hbase.regionserver.Region;
055import org.apache.hadoop.hbase.testclassification.LargeTests;
056import org.apache.hadoop.hbase.util.Bytes;
057import org.apache.hadoop.hbase.util.JVMClusterUtil;
058import org.junit.After;
059import org.junit.Before;
060import org.junit.BeforeClass;
061import org.junit.ClassRule;
062import org.junit.Rule;
063import org.junit.Test;
064import org.junit.experimental.categories.Category;
065import org.junit.rules.TestName;
066import org.slf4j.Logger;
067import org.slf4j.LoggerFactory;
068
069import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
070import org.apache.hbase.thirdparty.com.google.common.collect.Maps;
071
072@Category(LargeTests.class)
073public class TestFavoredStochasticBalancerPickers extends BalancerTestBase {
074
075  @ClassRule
076  public static final HBaseClassTestRule CLASS_RULE =
077      HBaseClassTestRule.forClass(TestFavoredStochasticBalancerPickers.class);
078
079  private static final Logger LOG =
080      LoggerFactory.getLogger(TestFavoredStochasticBalancerPickers.class);
081
082  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
083  private static final int SLAVES = 6;
084  private static final int REGIONS = SLAVES * 3;
085  private static Configuration conf;
086
087  private Admin admin;
088  private MiniHBaseCluster cluster;
089
090  @Rule
091  public TestName name = new TestName();
092
093  @BeforeClass
094  public static void setupBeforeClass() throws Exception {
095    conf = TEST_UTIL.getConfiguration();
096    // Enable favored nodes based load balancer
097    conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
098        LoadOnlyFavoredStochasticBalancer.class, LoadBalancer.class);
099    conf.setLong("hbase.master.balancer.stochastic.maxRunningTime", 30000);
100    conf.setInt("hbase.master.balancer.stochastic.moveCost", 0);
101    conf.setBoolean("hbase.master.balancer.stochastic.execute.maxSteps", true);
102    conf.set(BaseLoadBalancer.TABLES_ON_MASTER, "none");
103  }
104
105  @Before
106  public void startCluster() throws Exception {
107    TEST_UTIL.startMiniCluster(SLAVES);
108    TEST_UTIL.getDFSCluster().waitClusterUp();
109    TEST_UTIL.getHBaseCluster().waitForActiveAndReadyMaster(120*1000);
110    cluster = TEST_UTIL.getMiniHBaseCluster();
111    admin = TEST_UTIL.getAdmin();
112    admin.setBalancerRunning(false, true);
113  }
114
115  @After
116  public void stopCluster() throws Exception {
117    TEST_UTIL.cleanupTestDir();
118    TEST_UTIL.shutdownMiniCluster();
119  }
120
121
122  @Test
123  public void testPickers() throws Exception {
124    TableName tableName = TableName.valueOf(name.getMethodName());
125    ColumnFamilyDescriptor columnFamilyDescriptor =
126        ColumnFamilyDescriptorBuilder.newBuilder(HConstants.CATALOG_FAMILY).build();
127    TableDescriptor desc = TableDescriptorBuilder
128        .newBuilder(tableName)
129        .setColumnFamily(columnFamilyDescriptor)
130        .build();
131    admin.createTable(desc, Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), REGIONS);
132    TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
133    TEST_UTIL.loadTable(admin.getConnection().getTable(tableName), HConstants.CATALOG_FAMILY);
134    admin.flush(tableName);
135
136    HMaster master = cluster.getMaster();
137    FavoredNodesManager fnm = master.getFavoredNodesManager();
138    ServerName masterServerName = master.getServerName();
139    List<ServerName> excludedServers = Lists.newArrayList(masterServerName);
140    final ServerName mostLoadedServer = getRSWithMaxRegions(tableName, excludedServers);
141    assertNotNull(mostLoadedServer);
142    int numRegions = getTableRegionsFromServer(tableName, mostLoadedServer).size();
143    excludedServers.add(mostLoadedServer);
144    // Lets find another server with more regions to calculate number of regions to move
145    ServerName source = getRSWithMaxRegions(tableName, excludedServers);
146    assertNotNull(source);
147    int regionsToMove = getTableRegionsFromServer(tableName, source).size()/2;
148
149    // Since move only works if the target is part of favored nodes of the region, lets get all
150    // regions that are movable to mostLoadedServer
151    List<RegionInfo> hris = getRegionsThatCanBeMoved(tableName, mostLoadedServer);
152    RegionStates rst = master.getAssignmentManager().getRegionStates();
153    for (int i = 0; i < regionsToMove; i++) {
154      final RegionInfo regionInfo = hris.get(i);
155      admin.move(regionInfo.getEncodedNameAsBytes(),
156          Bytes.toBytes(mostLoadedServer.getServerName()));
157      LOG.info("Moving region: " + hris.get(i).getRegionNameAsString() + " to " + mostLoadedServer);
158      TEST_UTIL.waitFor(60000, new Waiter.Predicate<Exception>() {
159        @Override
160        public boolean evaluate() throws Exception {
161          return ServerName.isSameAddress(
162              rst.getRegionServerOfRegion(regionInfo), mostLoadedServer);
163        }
164      });
165    }
166    final int finalRegions = numRegions + regionsToMove;
167    TEST_UTIL.waitUntilNoRegionsInTransition(60000);
168    TEST_UTIL.waitFor(60000, new Waiter.Predicate<Exception>() {
169      @Override
170      public boolean evaluate() throws Exception {
171        int numRegions = getTableRegionsFromServer(tableName, mostLoadedServer).size();
172        return (numRegions == finalRegions);
173      }
174    });
175    TEST_UTIL.getHBaseCluster().startRegionServerAndWait(60000);
176
177    Map<ServerName, List<RegionInfo>> serverAssignments = Maps.newHashMap();
178    ClusterMetrics status = admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS));
179    for (ServerName sn : status.getLiveServerMetrics().keySet()) {
180      if (!ServerName.isSameAddress(sn, masterServerName)) {
181        serverAssignments.put(sn, getTableRegionsFromServer(tableName, sn));
182      }
183    }
184    RegionLocationFinder regionFinder = new RegionLocationFinder();
185    regionFinder.setClusterMetrics(admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)));
186    regionFinder.setConf(conf);
187    regionFinder.setServices(TEST_UTIL.getMiniHBaseCluster().getMaster());
188    Cluster cluster = new Cluster(serverAssignments, null, regionFinder, new RackManager(conf));
189    LoadOnlyFavoredStochasticBalancer balancer = (LoadOnlyFavoredStochasticBalancer) TEST_UTIL
190        .getMiniHBaseCluster().getMaster().getLoadBalancer();
191
192    cluster.sortServersByRegionCount();
193    Integer[] servers = cluster.serverIndicesSortedByRegionCount;
194    LOG.info("Servers sorted by region count:" + Arrays.toString(servers));
195    LOG.info("Cluster dump: " + cluster);
196    if (!mostLoadedServer.equals(cluster.servers[servers[servers.length -1]])) {
197      LOG.error("Most loaded server: " + mostLoadedServer + " does not match: "
198          + cluster.servers[servers[servers.length -1]]);
199    }
200    assertEquals(mostLoadedServer, cluster.servers[servers[servers.length -1]]);
201    FavoredStochasticBalancer.FavoredNodeLoadPicker loadPicker = balancer.new FavoredNodeLoadPicker();
202    boolean userRegionPicked = false;
203    for (int i = 0; i < 100; i++) {
204      if (userRegionPicked) {
205        break;
206      } else {
207        Cluster.Action action = loadPicker.generate(cluster);
208        if (action.type == Cluster.Action.Type.MOVE_REGION) {
209          Cluster.MoveRegionAction moveRegionAction = (Cluster.MoveRegionAction) action;
210          RegionInfo region = cluster.regions[moveRegionAction.region];
211          assertNotEquals(-1, moveRegionAction.toServer);
212          ServerName destinationServer = cluster.servers[moveRegionAction.toServer];
213          assertEquals(cluster.servers[moveRegionAction.fromServer], mostLoadedServer);
214          if (!region.getTable().isSystemTable()) {
215            List<ServerName> favNodes = fnm.getFavoredNodes(region);
216            assertTrue(favNodes.contains(ServerName.valueOf(destinationServer.getHostAndPort(), -1)));
217            userRegionPicked = true;
218          }
219        }
220      }
221    }
222    assertTrue("load picker did not pick expected regions in 100 iterations.", userRegionPicked);
223  }
224
225  /*
226   * A region can only be moved to one of its favored node. Hence this method helps us to
227   * get that list which makes it easy to write non-flaky tests.
228   */
229  private List<RegionInfo> getRegionsThatCanBeMoved(TableName tableName,
230      ServerName serverName) {
231    List<RegionInfo> regions = Lists.newArrayList();
232    RegionStates rst = cluster.getMaster().getAssignmentManager().getRegionStates();
233    FavoredNodesManager fnm = cluster.getMaster().getFavoredNodesManager();
234    for (RegionInfo regionInfo : fnm.getRegionsOfFavoredNode(serverName)) {
235      if (regionInfo.getTable().equals(tableName) &&
236          !ServerName.isSameAddress(rst.getRegionServerOfRegion(regionInfo), serverName)) {
237        regions.add(regionInfo);
238      }
239    }
240    return regions;
241  }
242
243  private List<RegionInfo> getTableRegionsFromServer(TableName tableName, ServerName source)
244      throws IOException {
245    List<RegionInfo> regionInfos = Lists.newArrayList();
246    HRegionServer regionServer = cluster.getRegionServer(source);
247    for (Region region : regionServer.getRegions(tableName)) {
248      regionInfos.add(region.getRegionInfo());
249    }
250    return regionInfos;
251  }
252
253  private ServerName getRSWithMaxRegions(TableName tableName, List<ServerName> excludeNodes)
254      throws IOException {
255
256    int maxRegions = 0;
257    ServerName maxLoadedServer = null;
258
259    for (JVMClusterUtil.RegionServerThread rst : cluster.getLiveRegionServerThreads()) {
260      List<HRegion> regions = rst.getRegionServer().getRegions(tableName);
261      LOG.debug("Server: " + rst.getRegionServer().getServerName() + " regions: " + regions.size());
262      if (regions.size() > maxRegions) {
263        if (excludeNodes == null ||
264            !doesMatchExcludeNodes(excludeNodes, rst.getRegionServer().getServerName())) {
265          maxRegions = regions.size();
266          maxLoadedServer = rst.getRegionServer().getServerName();
267        }
268      }
269    }
270    return maxLoadedServer;
271  }
272
273  private boolean doesMatchExcludeNodes(List<ServerName> excludeNodes, ServerName sn) {
274    for (ServerName excludeSN : excludeNodes) {
275      if (ServerName.isSameAddress(sn, excludeSN)) {
276        return true;
277      }
278    }
279    return false;
280  }
281}