001/** 002 * 003 * Licensed to the Apache Software Foundation (ASF) under one 004 * or more contributor license agreements. See the NOTICE file 005 * distributed with this work for additional information 006 * regarding copyright ownership. The ASF licenses this file 007 * to you under the Apache License, Version 2.0 (the 008 * "License"); you may not use this file except in compliance 009 * with the License. You may obtain a copy of the License at 010 * 011 * http://www.apache.org/licenses/LICENSE-2.0 012 * 013 * Unless required by applicable law or agreed to in writing, software 014 * distributed under the License is distributed on an "AS IS" BASIS, 015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 016 * See the License for the specific language governing permissions and 017 * limitations under the License. 018 */ 019package org.apache.hadoop.hbase; 020 021import java.util.ArrayList; 022import java.util.Comparator; 023import java.util.List; 024import java.util.Map; 025import java.util.NavigableSet; 026import java.util.TreeMap; 027import java.util.TreeSet; 028 029import org.apache.yetus.audience.InterfaceAudience; 030 031 032/** 033 * Data structure to describe the distribution of HDFS blocks among hosts. 034 * 035 * Adding erroneous data will be ignored silently. 036 */ 037@InterfaceAudience.Private 038public class HDFSBlocksDistribution { 039 private Map<String,HostAndWeight> hostAndWeights = null; 040 private long uniqueBlocksTotalWeight = 0; 041 042 /** 043 * Stores the hostname and weight for that hostname. 044 * 045 * This is used when determining the physical locations of the blocks making 046 * up a region. 047 * 048 * To make a prioritized list of the hosts holding the most data of a region, 049 * this class is used to count the total weight for each host. The weight is 050 * currently just the size of the file. 051 */ 052 public static class HostAndWeight { 053 054 private String host; 055 private long weight; 056 057 /** 058 * Constructor 059 * @param host the host name 060 * @param weight the weight 061 */ 062 public HostAndWeight(String host, long weight) { 063 this.host = host; 064 this.weight = weight; 065 } 066 067 /** 068 * add weight 069 * @param weight the weight 070 */ 071 public void addWeight(long weight) { 072 this.weight += weight; 073 } 074 075 /** 076 * @return the host name 077 */ 078 public String getHost() { 079 return host; 080 } 081 082 /** 083 * @return the weight 084 */ 085 public long getWeight() { 086 return weight; 087 } 088 089 /** 090 * comparator used to sort hosts based on weight 091 */ 092 public static class WeightComparator implements Comparator<HostAndWeight> { 093 @Override 094 public int compare(HostAndWeight l, HostAndWeight r) { 095 if(l.getWeight() == r.getWeight()) { 096 return l.getHost().compareTo(r.getHost()); 097 } 098 return l.getWeight() < r.getWeight() ? -1 : 1; 099 } 100 } 101 } 102 103 /** 104 * Constructor 105 */ 106 public HDFSBlocksDistribution() { 107 this.hostAndWeights = new TreeMap<>(); 108 } 109 110 /** 111 * @see java.lang.Object#toString() 112 */ 113 @Override 114 public synchronized String toString() { 115 return "number of unique hosts in the distribution=" + 116 this.hostAndWeights.size(); 117 } 118 119 /** 120 * add some weight to a list of hosts, update the value of unique block weight 121 * @param hosts the list of the host 122 * @param weight the weight 123 */ 124 public void addHostsAndBlockWeight(String[] hosts, long weight) { 125 if (hosts == null || hosts.length == 0) { 126 // erroneous data 127 return; 128 } 129 130 addUniqueWeight(weight); 131 for (String hostname : hosts) { 132 addHostAndBlockWeight(hostname, weight); 133 } 134 } 135 136 /** 137 * add some weight to the total unique weight 138 * @param weight the weight 139 */ 140 private void addUniqueWeight(long weight) { 141 uniqueBlocksTotalWeight += weight; 142 } 143 144 145 /** 146 * add some weight to a specific host 147 * @param host the host name 148 * @param weight the weight 149 */ 150 private void addHostAndBlockWeight(String host, long weight) { 151 if (host == null) { 152 // erroneous data 153 return; 154 } 155 156 HostAndWeight hostAndWeight = this.hostAndWeights.get(host); 157 if(hostAndWeight == null) { 158 hostAndWeight = new HostAndWeight(host, weight); 159 this.hostAndWeights.put(host, hostAndWeight); 160 } else { 161 hostAndWeight.addWeight(weight); 162 } 163 } 164 165 /** 166 * @return the hosts and their weights 167 */ 168 public Map<String,HostAndWeight> getHostAndWeights() { 169 return this.hostAndWeights; 170 } 171 172 /** 173 * return the weight for a specific host, that will be the total bytes of all 174 * blocks on the host 175 * @param host the host name 176 * @return the weight of the given host 177 */ 178 public long getWeight(String host) { 179 long weight = 0; 180 if (host != null) { 181 HostAndWeight hostAndWeight = this.hostAndWeights.get(host); 182 if(hostAndWeight != null) { 183 weight = hostAndWeight.getWeight(); 184 } 185 } 186 return weight; 187 } 188 189 /** 190 * @return the sum of all unique blocks' weight 191 */ 192 public long getUniqueBlocksTotalWeight() { 193 return uniqueBlocksTotalWeight; 194 } 195 196 /** 197 * return the locality index of a given host 198 * @param host the host name 199 * @return the locality index of the given host 200 */ 201 public float getBlockLocalityIndex(String host) { 202 float localityIndex = 0; 203 HostAndWeight hostAndWeight = this.hostAndWeights.get(host); 204 if (hostAndWeight != null && uniqueBlocksTotalWeight != 0) { 205 localityIndex=(float)hostAndWeight.weight/(float)uniqueBlocksTotalWeight; 206 } 207 return localityIndex; 208 } 209 210 211 /** 212 * This will add the distribution from input to this object 213 * @param otherBlocksDistribution the other hdfs blocks distribution 214 */ 215 public void add(HDFSBlocksDistribution otherBlocksDistribution) { 216 Map<String,HostAndWeight> otherHostAndWeights = 217 otherBlocksDistribution.getHostAndWeights(); 218 for (Map.Entry<String, HostAndWeight> otherHostAndWeight: 219 otherHostAndWeights.entrySet()) { 220 addHostAndBlockWeight(otherHostAndWeight.getValue().host, 221 otherHostAndWeight.getValue().weight); 222 } 223 addUniqueWeight(otherBlocksDistribution.getUniqueBlocksTotalWeight()); 224 } 225 226 /** 227 * return the sorted list of hosts in terms of their weights 228 */ 229 public List<String> getTopHosts() { 230 HostAndWeight[] hostAndWeights = getTopHostsWithWeights(); 231 List<String> topHosts = new ArrayList<>(hostAndWeights.length); 232 for(HostAndWeight haw : hostAndWeights) { 233 topHosts.add(haw.getHost()); 234 } 235 return topHosts; 236 } 237 238 /** 239 * return the sorted list of hosts in terms of their weights 240 */ 241 public HostAndWeight[] getTopHostsWithWeights() { 242 NavigableSet<HostAndWeight> orderedHosts = new TreeSet<>(new HostAndWeight.WeightComparator()); 243 orderedHosts.addAll(this.hostAndWeights.values()); 244 return orderedHosts.descendingSet().toArray(new HostAndWeight[orderedHosts.size()]); 245 } 246 247}