001/** 002 * 003 * Licensed to the Apache Software Foundation (ASF) under one 004 * or more contributor license agreements. See the NOTICE file 005 * distributed with this work for additional information 006 * regarding copyright ownership. The ASF licenses this file 007 * to you under the Apache License, Version 2.0 (the 008 * "License"); you may not use this file except in compliance 009 * with the License. You may obtain a copy of the License at 010 * 011 * http://www.apache.org/licenses/LICENSE-2.0 012 * 013 * Unless required by applicable law or agreed to in writing, software 014 * distributed under the License is distributed on an "AS IS" BASIS, 015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 016 * See the License for the specific language governing permissions and 017 * limitations under the License. 018 */ 019package org.apache.hadoop.hbase; 020 021import java.util.ArrayList; 022import java.util.Comparator; 023import java.util.List; 024import java.util.Map; 025import java.util.NavigableSet; 026import java.util.TreeMap; 027import java.util.TreeSet; 028import org.apache.hadoop.fs.StorageType; 029import org.apache.hadoop.hbase.util.DNS; 030import org.apache.yetus.audience.InterfaceAudience; 031 032/** 033 * Data structure to describe the distribution of HDFS blocks among hosts. 034 * 035 * Adding erroneous data will be ignored silently. 036 */ 037@InterfaceAudience.Private 038public class HDFSBlocksDistribution { 039 private Map<String,HostAndWeight> hostAndWeights = null; 040 private long uniqueBlocksTotalWeight = 0; 041 042 /** 043 * Stores the hostname and weight for that hostname. 044 * 045 * This is used when determining the physical locations of the blocks making 046 * up a region. 047 * 048 * To make a prioritized list of the hosts holding the most data of a region, 049 * this class is used to count the total weight for each host. The weight is 050 * currently just the size of the file. 051 */ 052 public static class HostAndWeight { 053 054 private final String host; 055 private long weight; 056 private long weightForSsd; 057 058 /** 059 * Constructor 060 * @param host the host name 061 * @param weight the weight 062 * @param weightForSsd the weight for ssd 063 */ 064 public HostAndWeight(String host, long weight, long weightForSsd) { 065 this.host = host; 066 this.weight = weight; 067 this.weightForSsd = weightForSsd; 068 } 069 070 /** 071 * add weight 072 * @param weight the weight 073 * @param weightForSsd the weight for ssd 074 */ 075 public void addWeight(long weight, long weightForSsd) { 076 this.weight += weight; 077 this.weightForSsd += weightForSsd; 078 } 079 080 /** 081 * @return the host name 082 */ 083 public String getHost() { 084 return host; 085 } 086 087 /** 088 * @return the weight 089 */ 090 public long getWeight() { 091 return weight; 092 } 093 094 /** 095 * @return the weight for ssd 096 */ 097 public long getWeightForSsd() { 098 return weightForSsd; 099 } 100 101 /** 102 * comparator used to sort hosts based on weight 103 */ 104 public static class WeightComparator implements Comparator<HostAndWeight> { 105 @Override 106 public int compare(HostAndWeight l, HostAndWeight r) { 107 if(l.getWeight() == r.getWeight()) { 108 return l.getHost().compareTo(r.getHost()); 109 } 110 return l.getWeight() < r.getWeight() ? -1 : 1; 111 } 112 } 113 } 114 115 /** 116 * Constructor 117 */ 118 public HDFSBlocksDistribution() { 119 this.hostAndWeights = new TreeMap<>(); 120 } 121 122 /** 123 * @see java.lang.Object#toString() 124 */ 125 @Override 126 public synchronized String toString() { 127 return "number of unique hosts in the distribution=" + 128 this.hostAndWeights.size(); 129 } 130 131 /** 132 * add some weight to a list of hosts, update the value of unique block weight 133 * @param hosts the list of the host 134 * @param weight the weight 135 */ 136 public void addHostsAndBlockWeight(String[] hosts, long weight) { 137 addHostsAndBlockWeight(hosts, weight, null); 138 } 139 140 /** 141 * add some weight to a list of hosts, update the value of unique block weight 142 * @param hosts the list of the host 143 * @param weight the weight 144 */ 145 public void addHostsAndBlockWeight(String[] hosts, long weight, StorageType[] storageTypes) { 146 if (hosts == null || hosts.length == 0) { 147 // erroneous data 148 return; 149 } 150 151 addUniqueWeight(weight); 152 if (storageTypes != null && storageTypes.length == hosts.length) { 153 for (int i = 0; i < hosts.length; i++) { 154 long weightForSsd = 0; 155 if (storageTypes[i] == StorageType.SSD) { 156 weightForSsd = weight; 157 } 158 addHostAndBlockWeight(hosts[i], weight, weightForSsd); 159 } 160 } else { 161 for (String hostname : hosts) { 162 addHostAndBlockWeight(hostname, weight, 0); 163 } 164 } 165 } 166 167 /** 168 * add some weight to the total unique weight 169 * @param weight the weight 170 */ 171 private void addUniqueWeight(long weight) { 172 uniqueBlocksTotalWeight += weight; 173 } 174 175 /** 176 * add some weight to a specific host 177 * @param host the host name 178 * @param weight the weight 179 * @param weightForSsd the weight for ssd 180 */ 181 private void addHostAndBlockWeight(String host, long weight, long weightForSsd) { 182 if (host == null) { 183 // erroneous data 184 return; 185 } 186 187 HostAndWeight hostAndWeight = this.hostAndWeights.get(host); 188 if(hostAndWeight == null) { 189 hostAndWeight = new HostAndWeight(host, weight, weightForSsd); 190 this.hostAndWeights.put(host, hostAndWeight); 191 } else { 192 hostAndWeight.addWeight(weight, weightForSsd); 193 } 194 } 195 196 /** 197 * @return the hosts and their weights 198 */ 199 public Map<String,HostAndWeight> getHostAndWeights() { 200 return this.hostAndWeights; 201 } 202 203 /** 204 * return the weight for a specific host, that will be the total bytes of all 205 * blocks on the host 206 * @param host the host name 207 * @return the weight of the given host 208 */ 209 public long getWeight(String host) { 210 long weight = 0; 211 if (host != null) { 212 HostAndWeight hostAndWeight = this.hostAndWeights.get(host); 213 if(hostAndWeight != null) { 214 weight = hostAndWeight.getWeight(); 215 } 216 } 217 return weight; 218 } 219 220 /** 221 * @return the sum of all unique blocks' weight 222 */ 223 public long getUniqueBlocksTotalWeight() { 224 return uniqueBlocksTotalWeight; 225 } 226 227 /** 228 * Implementations 'visit' hostAndWeight. 229 */ 230 public interface Visitor { 231 long visit(final HostAndWeight hostAndWeight); 232 } 233 234 /** 235 * @param host the host name 236 * @return the locality index of the given host 237 */ 238 public float getBlockLocalityIndex(String host) { 239 if (uniqueBlocksTotalWeight == 0) { 240 return 0.0f; 241 } else { 242 return (float) getBlocksLocalityWeightInternal(host, HostAndWeight::getWeight) 243 / (float) uniqueBlocksTotalWeight; 244 } 245 } 246 247 /** 248 * @param host the host name 249 * @return the locality index with ssd of the given host 250 */ 251 public float getBlockLocalityIndexForSsd(String host) { 252 if (uniqueBlocksTotalWeight == 0) { 253 return 0.0f; 254 } else { 255 return (float) getBlocksLocalityWeightInternal(host, HostAndWeight::getWeightForSsd) 256 / (float) uniqueBlocksTotalWeight; 257 } 258 } 259 260 /** 261 * @param host the host name 262 * @return the blocks local weight of the given host 263 */ 264 public long getBlocksLocalWeight(String host) { 265 return getBlocksLocalityWeightInternal(host, HostAndWeight::getWeight); 266 } 267 268 /** 269 * @param host the host name 270 * @return the blocks local with ssd weight of the given host 271 */ 272 public long getBlocksLocalWithSsdWeight(String host) { 273 return getBlocksLocalityWeightInternal(host, HostAndWeight::getWeightForSsd); 274 } 275 276 /** 277 * @param host the host name 278 * @return the locality index of the given host 279 */ 280 private long getBlocksLocalityWeightInternal(String host, Visitor visitor) { 281 long localityIndex = 0; 282 HostAndWeight hostAndWeight = this.hostAndWeights.get(host); 283 // Compatible with local mode, see HBASE-24569 284 if (hostAndWeight == null) { 285 String currentHost = ""; 286 try { 287 currentHost = DNS.getDefaultHost("default", "default"); 288 } catch (Exception e) { 289 // Just ignore, it's ok, avoid too many log info 290 } 291 if (host.equals(currentHost)) { 292 hostAndWeight = this.hostAndWeights.get(HConstants.LOCALHOST); 293 } 294 } 295 if (hostAndWeight != null && uniqueBlocksTotalWeight != 0) { 296 localityIndex = visitor.visit(hostAndWeight); 297 } 298 return localityIndex; 299 } 300 301 /** 302 * This will add the distribution from input to this object 303 * @param otherBlocksDistribution the other hdfs blocks distribution 304 */ 305 public void add(HDFSBlocksDistribution otherBlocksDistribution) { 306 Map<String,HostAndWeight> otherHostAndWeights = 307 otherBlocksDistribution.getHostAndWeights(); 308 for (Map.Entry<String, HostAndWeight> otherHostAndWeight: 309 otherHostAndWeights.entrySet()) { 310 addHostAndBlockWeight(otherHostAndWeight.getValue().host, 311 otherHostAndWeight.getValue().weight, otherHostAndWeight.getValue().weightForSsd); 312 } 313 addUniqueWeight(otherBlocksDistribution.getUniqueBlocksTotalWeight()); 314 } 315 316 /** 317 * return the sorted list of hosts in terms of their weights 318 */ 319 public List<String> getTopHosts() { 320 HostAndWeight[] hostAndWeights = getTopHostsWithWeights(); 321 List<String> topHosts = new ArrayList<>(hostAndWeights.length); 322 for(HostAndWeight haw : hostAndWeights) { 323 topHosts.add(haw.getHost()); 324 } 325 return topHosts; 326 } 327 328 /** 329 * return the sorted list of hosts in terms of their weights 330 */ 331 public HostAndWeight[] getTopHostsWithWeights() { 332 NavigableSet<HostAndWeight> orderedHosts = new TreeSet<>(new HostAndWeight.WeightComparator()); 333 orderedHosts.addAll(this.hostAndWeights.values()); 334 return orderedHosts.descendingSet().toArray(new HostAndWeight[orderedHosts.size()]); 335 } 336 337}