View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.master.balancer;
19  
20  import java.io.FileNotFoundException;
21  import java.io.IOException;
22  import java.util.ArrayList;
23  import java.util.Collection;
24  import java.util.HashMap;
25  import java.util.LinkedList;
26  import java.util.List;
27  import java.util.concurrent.ExecutionException;
28  import java.util.concurrent.TimeUnit;
29  
30  import org.apache.commons.logging.Log;
31  import org.apache.commons.logging.LogFactory;
32  import org.apache.hadoop.conf.Configuration;
33  import org.apache.hadoop.hbase.ClusterStatus;
34  import org.apache.hadoop.hbase.TableName;
35  import org.apache.hadoop.hbase.HDFSBlocksDistribution;
36  import org.apache.hadoop.hbase.HRegionInfo;
37  import org.apache.hadoop.hbase.HTableDescriptor;
38  import org.apache.hadoop.hbase.ServerName;
39  import org.apache.hadoop.hbase.master.MasterServices;
40  import org.apache.hadoop.hbase.regionserver.HRegion;
41  
42  import com.google.common.cache.CacheBuilder;
43  import com.google.common.cache.CacheLoader;
44  import com.google.common.cache.LoadingCache;
45  
46  /**
47   * This will find where data for a region is located in HDFS. It ranks
48   * {@link ServerName}'s by the size of the store files they are holding for a
49   * given region.
50   *
51   */
52  class RegionLocationFinder {
53  
54    private static Log LOG = LogFactory.getLog(RegionLocationFinder.class);
55  
56    private Configuration conf;
57    private ClusterStatus status;
58    private MasterServices services;
59  
60    private CacheLoader<HRegionInfo, List<ServerName>> loader =
61        new CacheLoader<HRegionInfo, List<ServerName>>() {
62  
63          @Override
64          public List<ServerName> load(HRegionInfo key) throws Exception {
65            List<ServerName> servers = internalGetTopBlockLocation(key);
66            if (servers == null) {
67              return new LinkedList<ServerName>();
68            }
69            return servers;
70          }
71        };
72  
73    // The cache for where regions are located.
74    private LoadingCache<HRegionInfo, List<ServerName>> cache = null;
75  
76    /**
77     * Create a cache for region to list of servers
78     * @param mins Number of mins to cache
79     * @return A new Cache.
80     */
81    private LoadingCache<HRegionInfo, List<ServerName>> createCache(int mins) {
82      return CacheBuilder.newBuilder().expireAfterAccess(mins, TimeUnit.MINUTES).build(loader);
83    }
84  
85    public Configuration getConf() {
86      return conf;
87    }
88  
89    public void setConf(Configuration conf) {
90      this.conf = conf;
91      cache = createCache(conf.getInt("hbase.master.balancer.regionLocationCacheTime", 30));
92    }
93  
94    public void setServices(MasterServices services) {
95      this.services = services;
96    }
97  
98    public void setClusterStatus(ClusterStatus status) {
99      this.status = status;
100   }
101 
102   protected List<ServerName> getTopBlockLocations(HRegionInfo region) {
103     List<ServerName> servers = null;
104     try {
105       servers = cache.get(region);
106     } catch (ExecutionException ex) {
107       servers = new LinkedList<ServerName>();
108     }
109     return servers;
110 
111   }
112 
113   /**
114    * Returns an ordered list of hosts that are hosting the blocks for this
115    * region. The weight of each host is the sum of the block lengths of all
116    * files on that host, so the first host in the list is the server which holds
117    * the most bytes of the given region's HFiles.
118    *
119    * @param region region
120    * @return ordered list of hosts holding blocks of the specified region
121    */
122   protected List<ServerName> internalGetTopBlockLocation(HRegionInfo region) {
123     List<ServerName> topServerNames = null;
124     try {
125       HTableDescriptor tableDescriptor = getTableDescriptor(region.getTable());
126       if (tableDescriptor != null) {
127         HDFSBlocksDistribution blocksDistribution =
128             HRegion.computeHDFSBlocksDistribution(getConf(), tableDescriptor, region);
129         List<String> topHosts = blocksDistribution.getTopHosts();
130         topServerNames = mapHostNameToServerName(topHosts);
131       }
132     } catch (IOException ioe) {
133       LOG.debug("IOException during HDFSBlocksDistribution computation. for " + "region = "
134           + region.getEncodedName(), ioe);
135     }
136 
137     return topServerNames;
138   }
139 
140   /**
141    * return HTableDescriptor for a given tableName
142    *
143    * @param tableName the table name
144    * @return HTableDescriptor
145    * @throws IOException
146    */
147   protected HTableDescriptor getTableDescriptor(TableName tableName) throws IOException {
148     HTableDescriptor tableDescriptor = null;
149     try {
150       if (this.services != null) {
151         tableDescriptor = this.services.getTableDescriptors().get(tableName);
152       }
153     } catch (FileNotFoundException fnfe) {
154       LOG.debug("FileNotFoundException during getTableDescriptors." + " Current table name = "
155           + tableName, fnfe);
156     }
157 
158     return tableDescriptor;
159   }
160 
161   /**
162    * Map hostname to ServerName, The output ServerName list will have the same
163    * order as input hosts.
164    *
165    * @param hosts the list of hosts
166    * @return ServerName list
167    */
168   protected List<ServerName> mapHostNameToServerName(List<String> hosts) {
169     if (hosts == null || status == null) {
170       return null;
171     }
172 
173     List<ServerName> topServerNames = new ArrayList<ServerName>();
174     Collection<ServerName> regionServers = status.getServers();
175 
176     // create a mapping from hostname to ServerName for fast lookup
177     HashMap<String, ServerName> hostToServerName = new HashMap<String, ServerName>();
178     for (ServerName sn : regionServers) {
179       hostToServerName.put(sn.getHostname(), sn);
180     }
181 
182     for (String host : hosts) {
183       ServerName sn = hostToServerName.get(host);
184       // it is possible that HDFS is up ( thus host is valid ),
185       // but RS is down ( thus sn is null )
186       if (sn != null) {
187         topServerNames.add(sn);
188       }
189     }
190     return topServerNames;
191   }
192 }