@InterfaceAudience.Private public class FavoredStochasticBalancer extends StochasticLoadBalancer implements FavoredNodesPromoter
LoadBalancer
that
assigns favored nodes for each region. There is a Primary RegionServer that hosts
the region, and then there is Secondary and Tertiary RegionServers. Currently, the
favored nodes information is used in creating HDFS files - the Primary RegionServer
passes the primary, secondary, tertiary node addresses as hints to the
DistributedFileSystem API for creating files on the filesystem. These nodes are
treated as hints by the HDFS to place the blocks of the file. This alleviates the
problem to do with reading from remote nodes (since we can make the Secondary
RegionServer as the new Primary RegionServer) after a region is recovered. This
should help provide consistent read latencies for the regions even when their
primary region servers die. This provides two
StochasticLoadBalancer.CandidateGenerator
Modifier and Type | Class and Description |
---|---|
(package private) class |
FavoredStochasticBalancer.FavoredNodeLoadPicker |
private class |
FavoredStochasticBalancer.FavoredNodeLocalityPicker |
StochasticLoadBalancer.CandidateGenerator, StochasticLoadBalancer.CostFromRegionLoadAsRateFunction, StochasticLoadBalancer.CostFromRegionLoadFunction, StochasticLoadBalancer.CostFunction, StochasticLoadBalancer.LoadCandidateGenerator, StochasticLoadBalancer.LocalityBasedCandidateGenerator, StochasticLoadBalancer.LocalityBasedCostFunction, StochasticLoadBalancer.MemStoreSizeCostFunction, StochasticLoadBalancer.MoveCostFunction, StochasticLoadBalancer.PrimaryRegionCountSkewCostFunction, StochasticLoadBalancer.RackLocalityCostFunction, StochasticLoadBalancer.RandomCandidateGenerator, StochasticLoadBalancer.ReadRequestCostFunction, StochasticLoadBalancer.RegionCountSkewCostFunction, StochasticLoadBalancer.RegionReplicaCandidateGenerator, StochasticLoadBalancer.RegionReplicaHostCostFunction, StochasticLoadBalancer.RegionReplicaRackCandidateGenerator, StochasticLoadBalancer.RegionReplicaRackCostFunction, StochasticLoadBalancer.ServerLocalityCostFunction, StochasticLoadBalancer.StoreFileCostFunction, StochasticLoadBalancer.TableSkewCostFunction, StochasticLoadBalancer.WriteRequestCostFunction
BaseLoadBalancer.Cluster
Modifier and Type | Field and Description |
---|---|
private FavoredNodesManager |
fnm |
private static org.slf4j.Logger |
LOG |
KEEP_REGION_LOADS, loads, MAX_RUNNING_TIME_KEY, MAX_STEPS_KEY, MIN_COST_NEED_BALANCE_KEY, RANDOM, RUN_MAX_STEPS_KEY, STEPS_PER_REGION_KEY
clusterStatus, config, IDLE_SERVER_PREDICATOR, maintenanceMode, masterServerName, metricsBalancer, MIN_SERVER_BALANCE, onlySystemTablesOnMaster, overallSlop, rackManager, regionFinder, services, slop, useRegionFinder
FAVORED_ALWAYS_ASSIGN_REGIONS
BOGUS_SERVER_NAME, SYSTEM_TABLES_ON_MASTER, TABLES_ON_MASTER
Constructor and Description |
---|
FavoredStochasticBalancer() |
Modifier and Type | Method and Description |
---|---|
private void |
addRegionToMap(Map<ServerName,List<RegionInfo>> assignmentMapForFavoredNodes,
RegionInfo region,
ServerName host) |
private void |
assignRegionToAvailableFavoredNode(Map<ServerName,List<RegionInfo>> assignmentMapForFavoredNodes,
RegionInfo region,
ServerName primaryHost,
ServerName secondaryHost,
ServerName tertiaryHost) |
List<RegionPlan> |
balanceCluster(Map<ServerName,List<RegionInfo>> clusterState)
Given the cluster state this will try and approach an optimal balance.
|
protected void |
configureGenerators() |
void |
generateFavoredNodesForDaughter(List<ServerName> servers,
RegionInfo parent,
RegionInfo regionA,
RegionInfo regionB) |
void |
generateFavoredNodesForMergedRegion(RegionInfo merged,
RegionInfo[] mergeParents) |
private Map<ServerName,List<RegionInfo>> |
generateFNForRegionsWithoutFN(FavoredNodeAssignmentHelper helper,
List<RegionInfo> regions) |
List<ServerName> |
getFavoredNodes(RegionInfo regionInfo) |
private Set<ServerName> |
getInheritedFNForDaughter(FavoredNodeAssignmentHelper helper,
List<ServerName> parentFavoredNodes,
FavoredNodesPlan.Position primary,
FavoredNodesPlan.Position secondary) |
private List<ServerName> |
getOnlineFavoredNodes(List<ServerName> onlineServers,
List<ServerName> serversWithoutStartCodes) |
private ServerName |
getServerFromFavoredNode(List<ServerName> servers,
ServerName fn) |
void |
initialize()
Initialize the load balancer.
|
private void |
mergeAssignmentMaps(Map<ServerName,List<RegionInfo>> assignmentMap,
Map<ServerName,List<RegionInfo>> otherAssignments) |
ServerName |
randomAssignment(RegionInfo regionInfo,
List<ServerName> servers)
Used to assign a single region to a random server.
|
Map<ServerName,List<RegionInfo>> |
retainAssignment(Map<RegionInfo,ServerName> regions,
List<ServerName> servers)
Generates a bulk assignment startup plan, attempting to reuse the existing
assignment information from META, but adjusting for the specified list of
available/online servers available for assignment.
|
Map<ServerName,List<RegionInfo>> |
roundRobinAssignment(List<RegionInfo> regions,
List<ServerName> servers)
Generates a bulk assignment plan to be used on cluster startup using a
simple round-robin assignment.
|
private Pair<Map<ServerName,List<RegionInfo>>,List<RegionInfo>> |
segregateRegionsAndAssignRegionsWithFavoredNodes(Collection<RegionInfo> regions,
List<ServerName> onlineServers) |
void |
setMasterServices(MasterServices masterServices)
Set the master service.
|
private void |
updateFavoredNodesForRegion(RegionInfo regionInfo,
List<ServerName> newFavoredNodes) |
areSomeRegionReplicasColocated, balanceCluster, composeAttributeName, computeCost, getCostFunctionNames, initCosts, needsBalance, nextAction, onConfigurationChange, setCandidateGenerators, setClusterMetrics, setConf, setSlop, updateCostsWithAction, updateMetricsSize
assignMasterSystemRegions, balanceMasterRegions, createCluster, getConf, getRegionAssignmentsByServer, isStopped, postMasterStartupInitialize, regionOffline, regionOnline, setClusterLoad, setRackManager, shouldBeOnMaster, stop, updateBalancerStatus
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
isMasterCanHostUserRegions, isSystemTablesOnlyOnMaster, isTablesOnMaster
private static final org.slf4j.Logger LOG
private FavoredNodesManager fnm
public FavoredStochasticBalancer()
public void initialize() throws HBaseIOException
LoadBalancer
initialize
in interface LoadBalancer
initialize
in class BaseLoadBalancer
HBaseIOException
protected void configureGenerators()
public void setMasterServices(MasterServices masterServices)
LoadBalancer
setMasterServices
in interface LoadBalancer
setMasterServices
in class StochasticLoadBalancer
public Map<ServerName,List<RegionInfo>> roundRobinAssignment(List<RegionInfo> regions, List<ServerName> servers) throws HBaseIOException
BaseLoadBalancer
Takes a list of all the regions and all the servers in the cluster and returns a map of each server to the regions that it should be assigned.
Currently implemented as a round-robin assignment. Same invariant as load balancing, all servers holding floor(avg) or ceiling(avg). TODO: Use block locations from HDFS to place regions with their blocks
roundRobinAssignment
in interface LoadBalancer
roundRobinAssignment
in class BaseLoadBalancer
regions
- all regionsservers
- all serversHBaseIOException
private void mergeAssignmentMaps(Map<ServerName,List<RegionInfo>> assignmentMap, Map<ServerName,List<RegionInfo>> otherAssignments)
private Map<ServerName,List<RegionInfo>> generateFNForRegionsWithoutFN(FavoredNodeAssignmentHelper helper, List<RegionInfo> regions) throws IOException
IOException
private Pair<Map<ServerName,List<RegionInfo>>,List<RegionInfo>> segregateRegionsAndAssignRegionsWithFavoredNodes(Collection<RegionInfo> regions, List<ServerName> onlineServers) throws HBaseIOException
HBaseIOException
private void addRegionToMap(Map<ServerName,List<RegionInfo>> assignmentMapForFavoredNodes, RegionInfo region, ServerName host)
private ServerName getServerFromFavoredNode(List<ServerName> servers, ServerName fn)
private void assignRegionToAvailableFavoredNode(Map<ServerName,List<RegionInfo>> assignmentMapForFavoredNodes, RegionInfo region, ServerName primaryHost, ServerName secondaryHost, ServerName tertiaryHost)
public ServerName randomAssignment(RegionInfo regionInfo, List<ServerName> servers) throws HBaseIOException
BaseLoadBalancer
randomAssignment
in interface LoadBalancer
randomAssignment
in class BaseLoadBalancer
regionInfo
- Region for which this selection is being done.HBaseIOException
private void updateFavoredNodesForRegion(RegionInfo regionInfo, List<ServerName> newFavoredNodes) throws IOException
IOException
public Map<ServerName,List<RegionInfo>> retainAssignment(Map<RegionInfo,ServerName> regions, List<ServerName> servers) throws HBaseIOException
BaseLoadBalancer
Takes a map of all regions to their existing assignment from META. Also takes a list of online servers for regions to be assigned to. Attempts to retain all assignment, so in some instances initial assignment will not be completely balanced.
Any leftover regions without an existing server to be assigned to will be assigned randomly to available servers.
retainAssignment
in interface LoadBalancer
retainAssignment
in class BaseLoadBalancer
regions
- regions and existing assignment from metaservers
- available serversHBaseIOException
private List<ServerName> getOnlineFavoredNodes(List<ServerName> onlineServers, List<ServerName> serversWithoutStartCodes)
public List<ServerName> getFavoredNodes(RegionInfo regionInfo)
public void generateFavoredNodesForDaughter(List<ServerName> servers, RegionInfo parent, RegionInfo regionA, RegionInfo regionB) throws IOException
generateFavoredNodesForDaughter
in interface FavoredNodesPromoter
IOException
private Set<ServerName> getInheritedFNForDaughter(FavoredNodeAssignmentHelper helper, List<ServerName> parentFavoredNodes, FavoredNodesPlan.Position primary, FavoredNodesPlan.Position secondary) throws IOException
IOException
public void generateFavoredNodesForMergedRegion(RegionInfo merged, RegionInfo[] mergeParents) throws IOException
generateFavoredNodesForMergedRegion
in interface FavoredNodesPromoter
IOException
public List<RegionPlan> balanceCluster(Map<ServerName,List<RegionInfo>> clusterState)
StochasticLoadBalancer
balanceCluster
in interface LoadBalancer
balanceCluster
in class StochasticLoadBalancer
Copyright © 2007–2020 The Apache Software Foundation. All rights reserved.