001/**
002 *
003 * Licensed to the Apache Software Foundation (ASF) under one
004 * or more contributor license agreements.  See the NOTICE file
005 * distributed with this work for additional information
006 * regarding copyright ownership.  The ASF licenses this file
007 * to you under the Apache License, Version 2.0 (the
008 * "License"); you may not use this file except in compliance
009 * with the License.  You may obtain a copy of the License at
010 *
011 *     http://www.apache.org/licenses/LICENSE-2.0
012 *
013 * Unless required by applicable law or agreed to in writing, software
014 * distributed under the License is distributed on an "AS IS" BASIS,
015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
016 * See the License for the specific language governing permissions and
017 * limitations under the License.
018 */
019package org.apache.hadoop.hbase;
020
021import java.util.ArrayList;
022import java.util.Comparator;
023import java.util.List;
024import java.util.Map;
025import java.util.NavigableSet;
026import java.util.TreeMap;
027import java.util.TreeSet;
028
029import org.apache.yetus.audience.InterfaceAudience;
030
031
032/**
033 * Data structure to describe the distribution of HDFS blocks among hosts.
034 *
035 * Adding erroneous data will be ignored silently.
036 */
037@InterfaceAudience.Private
038public class HDFSBlocksDistribution {
039  private Map<String,HostAndWeight> hostAndWeights = null;
040  private long uniqueBlocksTotalWeight = 0;
041
042  /**
043   * Stores the hostname and weight for that hostname.
044   *
045   * This is used when determining the physical locations of the blocks making
046   * up a region.
047   *
048   * To make a prioritized list of the hosts holding the most data of a region,
049   * this class is used to count the total weight for each host.  The weight is
050   * currently just the size of the file.
051   */
052  public static class HostAndWeight {
053
054    private String host;
055    private long weight;
056
057    /**
058     * Constructor
059     * @param host the host name
060     * @param weight the weight
061     */
062    public HostAndWeight(String host, long weight) {
063      this.host = host;
064      this.weight = weight;
065    }
066
067    /**
068     * add weight
069     * @param weight the weight
070     */
071    public void addWeight(long weight) {
072      this.weight += weight;
073    }
074
075    /**
076     * @return the host name
077     */
078    public String getHost() {
079      return host;
080    }
081
082    /**
083     * @return the weight
084     */
085    public long getWeight() {
086      return weight;
087    }
088
089    /**
090     * comparator used to sort hosts based on weight
091     */
092    public static class WeightComparator implements Comparator<HostAndWeight> {
093      @Override
094      public int compare(HostAndWeight l, HostAndWeight r) {
095        if(l.getWeight() == r.getWeight()) {
096          return l.getHost().compareTo(r.getHost());
097        }
098        return l.getWeight() < r.getWeight() ? -1 : 1;
099      }
100    }
101  }
102
103  /**
104   * Constructor
105   */
106  public HDFSBlocksDistribution() {
107    this.hostAndWeights = new TreeMap<>();
108  }
109
110  /**
111   * @see java.lang.Object#toString()
112   */
113  @Override
114  public synchronized String toString() {
115    return "number of unique hosts in the distribution=" +
116      this.hostAndWeights.size();
117  }
118
119  /**
120   * add some weight to a list of hosts, update the value of unique block weight
121   * @param hosts the list of the host
122   * @param weight the weight
123   */
124  public void addHostsAndBlockWeight(String[] hosts, long weight) {
125    if (hosts == null || hosts.length == 0) {
126      // erroneous data
127      return;
128    }
129
130    addUniqueWeight(weight);
131    for (String hostname : hosts) {
132      addHostAndBlockWeight(hostname, weight);
133    }
134  }
135
136  /**
137   * add some weight to the total unique weight
138   * @param weight the weight
139   */
140  private void addUniqueWeight(long weight) {
141    uniqueBlocksTotalWeight += weight;
142  }
143
144
145  /**
146   * add some weight to a specific host
147   * @param host the host name
148   * @param weight the weight
149   */
150  private void addHostAndBlockWeight(String host, long weight) {
151    if (host == null) {
152      // erroneous data
153      return;
154    }
155
156    HostAndWeight hostAndWeight = this.hostAndWeights.get(host);
157    if(hostAndWeight == null) {
158      hostAndWeight = new HostAndWeight(host, weight);
159      this.hostAndWeights.put(host, hostAndWeight);
160    } else {
161      hostAndWeight.addWeight(weight);
162    }
163  }
164
165  /**
166   * @return the hosts and their weights
167   */
168  public Map<String,HostAndWeight> getHostAndWeights() {
169    return this.hostAndWeights;
170  }
171
172  /**
173   * return the weight for a specific host, that will be the total bytes of all
174   * blocks on the host
175   * @param host the host name
176   * @return the weight of the given host
177   */
178  public long getWeight(String host) {
179    long weight = 0;
180    if (host != null) {
181      HostAndWeight hostAndWeight = this.hostAndWeights.get(host);
182      if(hostAndWeight != null) {
183        weight = hostAndWeight.getWeight();
184      }
185    }
186    return weight;
187  }
188
189  /**
190   * @return the sum of all unique blocks' weight
191   */
192  public long getUniqueBlocksTotalWeight() {
193    return uniqueBlocksTotalWeight;
194  }
195
196  /**
197   * return the locality index of a given host
198   * @param host the host name
199   * @return the locality index of the given host
200   */
201  public float getBlockLocalityIndex(String host) {
202    float localityIndex = 0;
203    HostAndWeight hostAndWeight = this.hostAndWeights.get(host);
204    if (hostAndWeight != null && uniqueBlocksTotalWeight != 0) {
205      localityIndex=(float)hostAndWeight.weight/(float)uniqueBlocksTotalWeight;
206    }
207    return localityIndex;
208  }
209
210
211  /**
212   * This will add the distribution from input to this object
213   * @param otherBlocksDistribution the other hdfs blocks distribution
214   */
215  public void add(HDFSBlocksDistribution otherBlocksDistribution) {
216    Map<String,HostAndWeight> otherHostAndWeights =
217      otherBlocksDistribution.getHostAndWeights();
218    for (Map.Entry<String, HostAndWeight> otherHostAndWeight:
219      otherHostAndWeights.entrySet()) {
220      addHostAndBlockWeight(otherHostAndWeight.getValue().host,
221        otherHostAndWeight.getValue().weight);
222    }
223    addUniqueWeight(otherBlocksDistribution.getUniqueBlocksTotalWeight());
224  }
225
226  /**
227   * return the sorted list of hosts in terms of their weights
228   */
229  public List<String> getTopHosts() {
230    HostAndWeight[] hostAndWeights = getTopHostsWithWeights();
231    List<String> topHosts = new ArrayList<>(hostAndWeights.length);
232    for(HostAndWeight haw : hostAndWeights) {
233      topHosts.add(haw.getHost());
234    }
235    return topHosts;
236  }
237
238  /**
239   * return the sorted list of hosts in terms of their weights
240   */
241  public HostAndWeight[] getTopHostsWithWeights() {
242    NavigableSet<HostAndWeight> orderedHosts = new TreeSet<>(new HostAndWeight.WeightComparator());
243    orderedHosts.addAll(this.hostAndWeights.values());
244    return orderedHosts.descendingSet().toArray(new HostAndWeight[orderedHosts.size()]);
245  }
246
247}