001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase; 019 020import java.util.ArrayList; 021import java.util.Comparator; 022import java.util.List; 023import java.util.Map; 024import java.util.NavigableSet; 025import java.util.TreeMap; 026import java.util.TreeSet; 027import org.apache.hadoop.fs.StorageType; 028import org.apache.hadoop.hbase.util.DNS; 029import org.apache.yetus.audience.InterfaceAudience; 030 031/** 032 * Data structure to describe the distribution of HDFS blocks among hosts. Adding erroneous data 033 * will be ignored silently. 034 */ 035@InterfaceAudience.Private 036public class HDFSBlocksDistribution { 037 private Map<String, HostAndWeight> hostAndWeights = null; 038 private long uniqueBlocksTotalWeight = 0; 039 040 /** 041 * Stores the hostname and weight for that hostname. This is used when determining the physical 042 * locations of the blocks making up a region. To make a prioritized list of the hosts holding the 043 * most data of a region, this class is used to count the total weight for each host. The weight 044 * is currently just the size of the file. 045 */ 046 public static class HostAndWeight { 047 048 private final String host; 049 private long weight; 050 private long weightForSsd; 051 052 /** 053 * Constructor 054 * @param host the host name 055 * @param weight the weight 056 * @param weightForSsd the weight for ssd 057 */ 058 public HostAndWeight(String host, long weight, long weightForSsd) { 059 this.host = host; 060 this.weight = weight; 061 this.weightForSsd = weightForSsd; 062 } 063 064 /** 065 * add weight 066 * @param weight the weight 067 * @param weightForSsd the weight for ssd 068 */ 069 public void addWeight(long weight, long weightForSsd) { 070 this.weight += weight; 071 this.weightForSsd += weightForSsd; 072 } 073 074 /** Returns the host name */ 075 public String getHost() { 076 return host; 077 } 078 079 /** Returns the weight */ 080 public long getWeight() { 081 return weight; 082 } 083 084 /** Returns the weight for ssd */ 085 public long getWeightForSsd() { 086 return weightForSsd; 087 } 088 089 /** Comparator used to sort hosts based on weight */ 090 public static class WeightComparator implements Comparator<HostAndWeight> { 091 @Override 092 public int compare(HostAndWeight l, HostAndWeight r) { 093 if (l.getWeight() == r.getWeight()) { 094 return l.getHost().compareTo(r.getHost()); 095 } 096 return l.getWeight() < r.getWeight() ? -1 : 1; 097 } 098 } 099 } 100 101 public HDFSBlocksDistribution() { 102 this.hostAndWeights = new TreeMap<>(); 103 } 104 105 @Override 106 public synchronized String toString() { 107 return "number of unique hosts in the distribution=" + this.hostAndWeights.size(); 108 } 109 110 /** 111 * add some weight to a list of hosts, update the value of unique block weight 112 * @param hosts the list of the host 113 * @param weight the weight 114 */ 115 public void addHostsAndBlockWeight(String[] hosts, long weight) { 116 addHostsAndBlockWeight(hosts, weight, null); 117 } 118 119 /** 120 * add some weight to a list of hosts, update the value of unique block weight 121 * @param hosts the list of the host 122 * @param weight the weight 123 */ 124 public void addHostsAndBlockWeight(String[] hosts, long weight, StorageType[] storageTypes) { 125 if (hosts == null || hosts.length == 0) { 126 // erroneous data 127 return; 128 } 129 130 addUniqueWeight(weight); 131 if (storageTypes != null && storageTypes.length == hosts.length) { 132 for (int i = 0; i < hosts.length; i++) { 133 long weightForSsd = 0; 134 if (storageTypes[i] == StorageType.SSD) { 135 weightForSsd = weight; 136 } 137 addHostAndBlockWeight(hosts[i], weight, weightForSsd); 138 } 139 } else { 140 for (String hostname : hosts) { 141 addHostAndBlockWeight(hostname, weight, 0); 142 } 143 } 144 } 145 146 /** 147 * add some weight to the total unique weight 148 * @param weight the weight 149 */ 150 private void addUniqueWeight(long weight) { 151 uniqueBlocksTotalWeight += weight; 152 } 153 154 /** 155 * add some weight to a specific host 156 * @param host the host name 157 * @param weight the weight 158 * @param weightForSsd the weight for ssd 159 */ 160 private void addHostAndBlockWeight(String host, long weight, long weightForSsd) { 161 if (host == null) { 162 // erroneous data 163 return; 164 } 165 166 HostAndWeight hostAndWeight = this.hostAndWeights.get(host); 167 if (hostAndWeight == null) { 168 hostAndWeight = new HostAndWeight(host, weight, weightForSsd); 169 this.hostAndWeights.put(host, hostAndWeight); 170 } else { 171 hostAndWeight.addWeight(weight, weightForSsd); 172 } 173 } 174 175 /** Returns the hosts and their weights */ 176 public Map<String, HostAndWeight> getHostAndWeights() { 177 return this.hostAndWeights; 178 } 179 180 /** 181 * return the weight for a specific host, that will be the total bytes of all blocks on the host 182 * @param host the host name 183 * @return the weight of the given host 184 */ 185 public long getWeight(String host) { 186 long weight = 0; 187 if (host != null) { 188 HostAndWeight hostAndWeight = this.hostAndWeights.get(host); 189 if (hostAndWeight != null) { 190 weight = hostAndWeight.getWeight(); 191 } 192 } 193 return weight; 194 } 195 196 /** Returns the sum of all unique blocks' weight */ 197 public long getUniqueBlocksTotalWeight() { 198 return uniqueBlocksTotalWeight; 199 } 200 201 /** Implementations 'visit' hostAndWeight. */ 202 public interface Visitor { 203 long visit(final HostAndWeight hostAndWeight); 204 } 205 206 /** 207 * Get the block locality index for a given host 208 * @param host the host name 209 * @return the locality index of the given host 210 */ 211 public float getBlockLocalityIndex(String host) { 212 if (uniqueBlocksTotalWeight == 0) { 213 return 0.0f; 214 } else { 215 return (float) getBlocksLocalityWeightInternal(host, HostAndWeight::getWeight) 216 / (float) uniqueBlocksTotalWeight; 217 } 218 } 219 220 /** 221 * Get the block locality index for a ssd for a given host 222 * @param host the host name 223 * @return the locality index with ssd of the given host 224 */ 225 public float getBlockLocalityIndexForSsd(String host) { 226 if (uniqueBlocksTotalWeight == 0) { 227 return 0.0f; 228 } else { 229 return (float) getBlocksLocalityWeightInternal(host, HostAndWeight::getWeightForSsd) 230 / (float) uniqueBlocksTotalWeight; 231 } 232 } 233 234 /** 235 * Get the blocks local weight for a given host 236 * @param host the host name 237 * @return the blocks local weight of the given host 238 */ 239 public long getBlocksLocalWeight(String host) { 240 return getBlocksLocalityWeightInternal(host, HostAndWeight::getWeight); 241 } 242 243 /** 244 * Get the blocks local weight with ssd for a given host 245 * @param host the host name 246 * @return the blocks local with ssd weight of the given host 247 */ 248 public long getBlocksLocalWithSsdWeight(String host) { 249 return getBlocksLocalityWeightInternal(host, HostAndWeight::getWeightForSsd); 250 } 251 252 private long getBlocksLocalityWeightInternal(String host, Visitor visitor) { 253 long localityIndex = 0; 254 HostAndWeight hostAndWeight = this.hostAndWeights.get(host); 255 // Compatible with local mode, see HBASE-24569 256 if (hostAndWeight == null) { 257 String currentHost = ""; 258 try { 259 currentHost = DNS.getDefaultHost("default", "default"); 260 } catch (Exception e) { 261 // Just ignore, it's ok, avoid too many log info 262 } 263 if (host.equals(currentHost)) { 264 hostAndWeight = this.hostAndWeights.get(HConstants.LOCALHOST); 265 } 266 } 267 if (hostAndWeight != null && uniqueBlocksTotalWeight != 0) { 268 localityIndex = visitor.visit(hostAndWeight); 269 } 270 return localityIndex; 271 } 272 273 /** 274 * This will add the distribution from input to this object 275 * @param otherBlocksDistribution the other hdfs blocks distribution 276 */ 277 public void add(HDFSBlocksDistribution otherBlocksDistribution) { 278 Map<String, HostAndWeight> otherHostAndWeights = otherBlocksDistribution.getHostAndWeights(); 279 for (Map.Entry<String, HostAndWeight> otherHostAndWeight : otherHostAndWeights.entrySet()) { 280 addHostAndBlockWeight(otherHostAndWeight.getValue().host, 281 otherHostAndWeight.getValue().weight, otherHostAndWeight.getValue().weightForSsd); 282 } 283 addUniqueWeight(otherBlocksDistribution.getUniqueBlocksTotalWeight()); 284 } 285 286 /** Return the sorted list of hosts in terms of their weights */ 287 public List<String> getTopHosts() { 288 HostAndWeight[] hostAndWeights = getTopHostsWithWeights(); 289 List<String> topHosts = new ArrayList<>(hostAndWeights.length); 290 for (HostAndWeight haw : hostAndWeights) { 291 topHosts.add(haw.getHost()); 292 } 293 return topHosts; 294 } 295 296 /** Return the sorted list of hosts in terms of their weights */ 297 public HostAndWeight[] getTopHostsWithWeights() { 298 NavigableSet<HostAndWeight> orderedHosts = new TreeSet<>(new HostAndWeight.WeightComparator()); 299 orderedHosts.addAll(this.hostAndWeights.values()); 300 return orderedHosts.descendingSet().toArray(new HostAndWeight[orderedHosts.size()]); 301 } 302 303}