001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase;
019
020import java.util.ArrayList;
021import java.util.Comparator;
022import java.util.List;
023import java.util.Map;
024import java.util.NavigableSet;
025import java.util.TreeMap;
026import java.util.TreeSet;
027import org.apache.hadoop.fs.StorageType;
028import org.apache.hadoop.hbase.util.DNS;
029import org.apache.yetus.audience.InterfaceAudience;
030
031/**
032 * Data structure to describe the distribution of HDFS blocks among hosts. Adding erroneous data
033 * will be ignored silently.
034 */
035@InterfaceAudience.Private
036public class HDFSBlocksDistribution {
037  private Map<String, HostAndWeight> hostAndWeights = null;
038  private long uniqueBlocksTotalWeight = 0;
039
040  /**
041   * Stores the hostname and weight for that hostname. This is used when determining the physical
042   * locations of the blocks making up a region. To make a prioritized list of the hosts holding the
043   * most data of a region, this class is used to count the total weight for each host. The weight
044   * is currently just the size of the file.
045   */
046  public static class HostAndWeight {
047
048    private final String host;
049    private long weight;
050    private long weightForSsd;
051
052    /**
053     * Constructor
054     * @param host         the host name
055     * @param weight       the weight
056     * @param weightForSsd the weight for ssd
057     */
058    public HostAndWeight(String host, long weight, long weightForSsd) {
059      this.host = host;
060      this.weight = weight;
061      this.weightForSsd = weightForSsd;
062    }
063
064    /**
065     * add weight
066     * @param weight       the weight
067     * @param weightForSsd the weight for ssd
068     */
069    public void addWeight(long weight, long weightForSsd) {
070      this.weight += weight;
071      this.weightForSsd += weightForSsd;
072    }
073
074    /** Returns the host name */
075    public String getHost() {
076      return host;
077    }
078
079    /** Returns the weight */
080    public long getWeight() {
081      return weight;
082    }
083
084    /** Returns the weight for ssd */
085    public long getWeightForSsd() {
086      return weightForSsd;
087    }
088
089    /** Comparator used to sort hosts based on weight */
090    public static class WeightComparator implements Comparator<HostAndWeight> {
091      @Override
092      public int compare(HostAndWeight l, HostAndWeight r) {
093        if (l.getWeight() == r.getWeight()) {
094          return l.getHost().compareTo(r.getHost());
095        }
096        return l.getWeight() < r.getWeight() ? -1 : 1;
097      }
098    }
099  }
100
101  public HDFSBlocksDistribution() {
102    this.hostAndWeights = new TreeMap<>();
103  }
104
105  @Override
106  public synchronized String toString() {
107    return "number of unique hosts in the distribution=" + this.hostAndWeights.size();
108  }
109
110  /**
111   * add some weight to a list of hosts, update the value of unique block weight
112   * @param hosts  the list of the host
113   * @param weight the weight
114   */
115  public void addHostsAndBlockWeight(String[] hosts, long weight) {
116    addHostsAndBlockWeight(hosts, weight, null);
117  }
118
119  /**
120   * add some weight to a list of hosts, update the value of unique block weight
121   * @param hosts  the list of the host
122   * @param weight the weight
123   */
124  public void addHostsAndBlockWeight(String[] hosts, long weight, StorageType[] storageTypes) {
125    if (hosts == null || hosts.length == 0) {
126      // erroneous data
127      return;
128    }
129
130    addUniqueWeight(weight);
131    if (storageTypes != null && storageTypes.length == hosts.length) {
132      for (int i = 0; i < hosts.length; i++) {
133        long weightForSsd = 0;
134        if (storageTypes[i] == StorageType.SSD) {
135          weightForSsd = weight;
136        }
137        addHostAndBlockWeight(hosts[i], weight, weightForSsd);
138      }
139    } else {
140      for (String hostname : hosts) {
141        addHostAndBlockWeight(hostname, weight, 0);
142      }
143    }
144  }
145
146  /**
147   * add some weight to the total unique weight
148   * @param weight the weight
149   */
150  private void addUniqueWeight(long weight) {
151    uniqueBlocksTotalWeight += weight;
152  }
153
154  /**
155   * add some weight to a specific host
156   * @param host         the host name
157   * @param weight       the weight
158   * @param weightForSsd the weight for ssd
159   */
160  private void addHostAndBlockWeight(String host, long weight, long weightForSsd) {
161    if (host == null) {
162      // erroneous data
163      return;
164    }
165
166    HostAndWeight hostAndWeight = this.hostAndWeights.get(host);
167    if (hostAndWeight == null) {
168      hostAndWeight = new HostAndWeight(host, weight, weightForSsd);
169      this.hostAndWeights.put(host, hostAndWeight);
170    } else {
171      hostAndWeight.addWeight(weight, weightForSsd);
172    }
173  }
174
175  /** Returns the hosts and their weights */
176  public Map<String, HostAndWeight> getHostAndWeights() {
177    return this.hostAndWeights;
178  }
179
180  /**
181   * return the weight for a specific host, that will be the total bytes of all blocks on the host
182   * @param host the host name
183   * @return the weight of the given host
184   */
185  public long getWeight(String host) {
186    long weight = 0;
187    if (host != null) {
188      HostAndWeight hostAndWeight = this.hostAndWeights.get(host);
189      if (hostAndWeight != null) {
190        weight = hostAndWeight.getWeight();
191      }
192    }
193    return weight;
194  }
195
196  /** Returns the sum of all unique blocks' weight */
197  public long getUniqueBlocksTotalWeight() {
198    return uniqueBlocksTotalWeight;
199  }
200
201  /** Implementations 'visit' hostAndWeight. */
202  public interface Visitor {
203    long visit(final HostAndWeight hostAndWeight);
204  }
205
206  /**
207   * Get the block locality index for a given host
208   * @param host the host name
209   * @return the locality index of the given host
210   */
211  public float getBlockLocalityIndex(String host) {
212    if (uniqueBlocksTotalWeight == 0) {
213      return 0.0f;
214    } else {
215      return (float) getBlocksLocalityWeightInternal(host, HostAndWeight::getWeight)
216        / (float) uniqueBlocksTotalWeight;
217    }
218  }
219
220  /**
221   * Get the block locality index for a ssd for a given host
222   * @param host the host name
223   * @return the locality index with ssd of the given host
224   */
225  public float getBlockLocalityIndexForSsd(String host) {
226    if (uniqueBlocksTotalWeight == 0) {
227      return 0.0f;
228    } else {
229      return (float) getBlocksLocalityWeightInternal(host, HostAndWeight::getWeightForSsd)
230        / (float) uniqueBlocksTotalWeight;
231    }
232  }
233
234  /**
235   * Get the blocks local weight for a given host
236   * @param host the host name
237   * @return the blocks local weight of the given host
238   */
239  public long getBlocksLocalWeight(String host) {
240    return getBlocksLocalityWeightInternal(host, HostAndWeight::getWeight);
241  }
242
243  /**
244   * Get the blocks local weight with ssd for a given host
245   * @param host the host name
246   * @return the blocks local with ssd weight of the given host
247   */
248  public long getBlocksLocalWithSsdWeight(String host) {
249    return getBlocksLocalityWeightInternal(host, HostAndWeight::getWeightForSsd);
250  }
251
252  private long getBlocksLocalityWeightInternal(String host, Visitor visitor) {
253    long localityIndex = 0;
254    HostAndWeight hostAndWeight = this.hostAndWeights.get(host);
255    // Compatible with local mode, see HBASE-24569
256    if (hostAndWeight == null) {
257      String currentHost = "";
258      try {
259        currentHost = DNS.getDefaultHost("default", "default");
260      } catch (Exception e) {
261        // Just ignore, it's ok, avoid too many log info
262      }
263      if (host.equals(currentHost)) {
264        hostAndWeight = this.hostAndWeights.get(HConstants.LOCALHOST);
265      }
266    }
267    if (hostAndWeight != null && uniqueBlocksTotalWeight != 0) {
268      localityIndex = visitor.visit(hostAndWeight);
269    }
270    return localityIndex;
271  }
272
273  /**
274   * This will add the distribution from input to this object
275   * @param otherBlocksDistribution the other hdfs blocks distribution
276   */
277  public void add(HDFSBlocksDistribution otherBlocksDistribution) {
278    Map<String, HostAndWeight> otherHostAndWeights = otherBlocksDistribution.getHostAndWeights();
279    for (Map.Entry<String, HostAndWeight> otherHostAndWeight : otherHostAndWeights.entrySet()) {
280      addHostAndBlockWeight(otherHostAndWeight.getValue().host,
281        otherHostAndWeight.getValue().weight, otherHostAndWeight.getValue().weightForSsd);
282    }
283    addUniqueWeight(otherBlocksDistribution.getUniqueBlocksTotalWeight());
284  }
285
286  /** Return the sorted list of hosts in terms of their weights */
287  public List<String> getTopHosts() {
288    HostAndWeight[] hostAndWeights = getTopHostsWithWeights();
289    List<String> topHosts = new ArrayList<>(hostAndWeights.length);
290    for (HostAndWeight haw : hostAndWeights) {
291      topHosts.add(haw.getHost());
292    }
293    return topHosts;
294  }
295
296  /** Return the sorted list of hosts in terms of their weights */
297  public HostAndWeight[] getTopHostsWithWeights() {
298    NavigableSet<HostAndWeight> orderedHosts = new TreeSet<>(new HostAndWeight.WeightComparator());
299    orderedHosts.addAll(this.hostAndWeights.values());
300    return orderedHosts.descendingSet().toArray(new HostAndWeight[orderedHosts.size()]);
301  }
302
303}