001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase; 019 020import java.io.IOException; 021import java.lang.reflect.InvocationTargetException; 022import java.lang.reflect.Method; 023import java.util.Map; 024 025import org.apache.hadoop.conf.Configuration; 026import org.apache.hadoop.hbase.util.VersionInfo; 027import org.apache.hadoop.hbase.zookeeper.ZKConfig; 028import org.apache.yetus.audience.InterfaceAudience; 029import org.slf4j.Logger; 030import org.slf4j.LoggerFactory; 031 032/** 033 * Adds HBase configuration files to a Configuration 034 */ 035@InterfaceAudience.Public 036public class HBaseConfiguration extends Configuration { 037 private static final Logger LOG = LoggerFactory.getLogger(HBaseConfiguration.class); 038 039 /** 040 * Instantiating HBaseConfiguration() is deprecated. Please use 041 * HBaseConfiguration#create() to construct a plain Configuration 042 * @deprecated since 0.90.0. Please use {@link #create()} instead. 043 * @see #create() 044 * @see <a href="https://issues.apache.org/jira/browse/HBASE-2036">HBASE-2036</a> 045 */ 046 @Deprecated 047 public HBaseConfiguration() { 048 //TODO:replace with private constructor, HBaseConfiguration should not extend Configuration 049 super(); 050 addHbaseResources(this); 051 LOG.warn("instantiating HBaseConfiguration() is deprecated. Please use" 052 + " HBaseConfiguration#create() to construct a plain Configuration"); 053 } 054 055 /** 056 * Instantiating HBaseConfiguration() is deprecated. Please use 057 * HBaseConfiguration#create(conf) to construct a plain Configuration 058 * @deprecated since 0.90.0. Please use {@link #create(Configuration)} instead. 059 * @see #create(Configuration) 060 * @see <a href="https://issues.apache.org/jira/browse/HBASE-2036">HBASE-2036</a> 061 */ 062 @Deprecated 063 public HBaseConfiguration(final Configuration c) { 064 //TODO:replace with private constructor 065 this(); 066 merge(this, c); 067 } 068 069 private static void checkDefaultsVersion(Configuration conf) { 070 if (conf.getBoolean("hbase.defaults.for.version.skip", Boolean.FALSE)) return; 071 String defaultsVersion = conf.get("hbase.defaults.for.version"); 072 String thisVersion = VersionInfo.getVersion(); 073 if (!thisVersion.equals(defaultsVersion)) { 074 throw new RuntimeException( 075 "hbase-default.xml file seems to be for an older version of HBase (" + 076 defaultsVersion + "), this version is " + thisVersion); 077 } 078 } 079 080 public static Configuration addHbaseResources(Configuration conf) { 081 conf.addResource("hbase-default.xml"); 082 conf.addResource("hbase-site.xml"); 083 084 checkDefaultsVersion(conf); 085 return conf; 086 } 087 088 /** 089 * Creates a Configuration with HBase resources 090 * @return a Configuration with HBase resources 091 */ 092 public static Configuration create() { 093 Configuration conf = new Configuration(); 094 // In case HBaseConfiguration is loaded from a different classloader than 095 // Configuration, conf needs to be set with appropriate class loader to resolve 096 // HBase resources. 097 conf.setClassLoader(HBaseConfiguration.class.getClassLoader()); 098 return addHbaseResources(conf); 099 } 100 101 /** 102 * @param that Configuration to clone. 103 * @return a Configuration created with the hbase-*.xml files plus 104 * the given configuration. 105 */ 106 public static Configuration create(final Configuration that) { 107 Configuration conf = create(); 108 merge(conf, that); 109 return conf; 110 } 111 112 /** 113 * Merge two configurations. 114 * @param destConf the configuration that will be overwritten with items 115 * from the srcConf 116 * @param srcConf the source configuration 117 **/ 118 public static void merge(Configuration destConf, Configuration srcConf) { 119 for (Map.Entry<String, String> e : srcConf) { 120 destConf.set(e.getKey(), e.getValue()); 121 } 122 } 123 124 /** 125 * Returns a subset of the configuration properties, matching the given key prefix. 126 * The prefix is stripped from the return keys, ie. when calling with a prefix of "myprefix", 127 * the entry "myprefix.key1 = value1" would be returned as "key1 = value1". If an entry's 128 * key matches the prefix exactly ("myprefix = value2"), it will <strong>not</strong> be 129 * included in the results, since it would show up as an entry with an empty key. 130 */ 131 public static Configuration subset(Configuration srcConf, String prefix) { 132 Configuration newConf = new Configuration(false); 133 for (Map.Entry<String, String> entry : srcConf) { 134 if (entry.getKey().startsWith(prefix)) { 135 String newKey = entry.getKey().substring(prefix.length()); 136 // avoid entries that would produce an empty key 137 if (!newKey.isEmpty()) { 138 newConf.set(newKey, entry.getValue()); 139 } 140 } 141 } 142 return newConf; 143 } 144 145 /** 146 * Sets all the entries in the provided {@code Map<String, String>} as properties in the 147 * given {@code Configuration}. Each property will have the specified prefix prepended, 148 * so that the configuration entries are keyed by {@code prefix + entry.getKey()}. 149 */ 150 public static void setWithPrefix(Configuration conf, String prefix, 151 Iterable<Map.Entry<String, String>> properties) { 152 for (Map.Entry<String, String> entry : properties) { 153 conf.set(prefix + entry.getKey(), entry.getValue()); 154 } 155 } 156 157 /** 158 * @return whether to show HBase Configuration in servlet 159 */ 160 public static boolean isShowConfInServlet() { 161 boolean isShowConf = false; 162 try { 163 if (Class.forName("org.apache.hadoop.conf.ConfServlet") != null) { 164 isShowConf = true; 165 } 166 } catch (LinkageError e) { 167 // should we handle it more aggressively in addition to log the error? 168 LOG.warn("Error thrown: ", e); 169 } catch (ClassNotFoundException ce) { 170 LOG.debug("ClassNotFound: ConfServlet"); 171 // ignore 172 } 173 return isShowConf; 174 } 175 176 /** 177 * Get the value of the <code>name</code> property as an <code>int</code>, possibly referring to 178 * the deprecated name of the configuration property. If no such property exists, the provided 179 * default value is returned, or if the specified value is not a valid <code>int</code>, then an 180 * error is thrown. 181 * @param name property name. 182 * @param deprecatedName a deprecatedName for the property to use if non-deprecated name is not 183 * used 184 * @param defaultValue default value. 185 * @throws NumberFormatException when the value is invalid 186 * @return property value as an <code>int</code>, or <code>defaultValue</code>. 187 * @deprecated it will be removed in 3.0.0. Use 188 * {@link Configuration#addDeprecation(String, String)} instead. 189 */ 190 @Deprecated 191 public static int getInt(Configuration conf, String name, 192 String deprecatedName, int defaultValue) { 193 if (conf.get(deprecatedName) != null) { 194 LOG.warn(String.format("Config option \"%s\" is deprecated. Instead, use \"%s\"" 195 , deprecatedName, name)); 196 return conf.getInt(deprecatedName, defaultValue); 197 } else { 198 return conf.getInt(name, defaultValue); 199 } 200 } 201 202 /** 203 * Get the password from the Configuration instance using the 204 * getPassword method if it exists. If not, then fall back to the 205 * general get method for configuration elements. 206 * 207 * @param conf configuration instance for accessing the passwords 208 * @param alias the name of the password element 209 * @param defPass the default password 210 * @return String password or default password 211 * @throws IOException 212 */ 213 public static String getPassword(Configuration conf, String alias, 214 String defPass) throws IOException { 215 String passwd = null; 216 try { 217 Method m = Configuration.class.getMethod("getPassword", String.class); 218 char[] p = (char[]) m.invoke(conf, alias); 219 if (p != null) { 220 LOG.debug(String.format("Config option \"%s\" was found through" + 221 " the Configuration getPassword method.", alias)); 222 passwd = new String(p); 223 } else { 224 LOG.debug(String.format( 225 "Config option \"%s\" was not found. Using provided default value", 226 alias)); 227 passwd = defPass; 228 } 229 } catch (NoSuchMethodException e) { 230 // this is a version of Hadoop where the credential 231 //provider API doesn't exist yet 232 LOG.debug(String.format( 233 "Credential.getPassword method is not available." + 234 " Falling back to configuration.")); 235 passwd = conf.get(alias, defPass); 236 } catch (SecurityException e) { 237 throw new IOException(e.getMessage(), e); 238 } catch (IllegalAccessException e) { 239 throw new IOException(e.getMessage(), e); 240 } catch (IllegalArgumentException e) { 241 throw new IOException(e.getMessage(), e); 242 } catch (InvocationTargetException e) { 243 throw new IOException(e.getMessage(), e); 244 } 245 return passwd; 246 } 247 248 /** 249 * Generates a {@link Configuration} instance by applying the ZooKeeper cluster key 250 * to the base Configuration. Note that additional configuration properties may be needed 251 * for a remote cluster, so it is preferable to use 252 * {@link #createClusterConf(Configuration, String, String)}. 253 * 254 * @param baseConf the base configuration to use, containing prefixed override properties 255 * @param clusterKey the ZooKeeper quorum cluster key to apply, or {@code null} if none 256 * 257 * @return the merged configuration with override properties and cluster key applied 258 * 259 * @see #createClusterConf(Configuration, String, String) 260 */ 261 public static Configuration createClusterConf(Configuration baseConf, String clusterKey) 262 throws IOException { 263 return createClusterConf(baseConf, clusterKey, null); 264 } 265 266 /** 267 * Generates a {@link Configuration} instance by applying property overrides prefixed by 268 * a cluster profile key to the base Configuration. Override properties are extracted by 269 * the {@link #subset(Configuration, String)} method, then the merged on top of the base 270 * Configuration and returned. 271 * 272 * @param baseConf the base configuration to use, containing prefixed override properties 273 * @param clusterKey the ZooKeeper quorum cluster key to apply, or {@code null} if none 274 * @param overridePrefix the property key prefix to match for override properties, 275 * or {@code null} if none 276 * @return the merged configuration with override properties and cluster key applied 277 */ 278 public static Configuration createClusterConf(Configuration baseConf, String clusterKey, 279 String overridePrefix) throws IOException { 280 Configuration clusterConf = HBaseConfiguration.create(baseConf); 281 if (clusterKey != null && !clusterKey.isEmpty()) { 282 applyClusterKeyToConf(clusterConf, clusterKey); 283 } 284 285 if (overridePrefix != null && !overridePrefix.isEmpty()) { 286 Configuration clusterSubset = HBaseConfiguration.subset(clusterConf, overridePrefix); 287 HBaseConfiguration.merge(clusterConf, clusterSubset); 288 } 289 return clusterConf; 290 } 291 292 /** 293 * Apply the settings in the given key to the given configuration, this is 294 * used to communicate with distant clusters 295 * @param conf configuration object to configure 296 * @param key string that contains the 3 required configuratins 297 */ 298 private static void applyClusterKeyToConf(Configuration conf, String key) 299 throws IOException { 300 ZKConfig.ZKClusterKey zkClusterKey = ZKConfig.transformClusterKey(key); 301 conf.set(HConstants.ZOOKEEPER_QUORUM, zkClusterKey.getQuorumString()); 302 conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, zkClusterKey.getClientPort()); 303 conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, zkClusterKey.getZnodeParent()); 304 // Without the right registry, the above configs are useless. Also, we don't use setClass() 305 // here because the ConnectionRegistry* classes are not resolvable from this module. 306 // This will be broken if ZkConnectionRegistry class gets renamed or moved. Is there a better 307 // way? 308 LOG.info("Overriding client registry implementation to {}", 309 HConstants.ZK_CONNECTION_REGISTRY_CLASS); 310 conf.set(HConstants.CLIENT_CONNECTION_REGISTRY_IMPL_CONF_KEY, 311 HConstants.ZK_CONNECTION_REGISTRY_CLASS); 312 } 313 314 /** 315 * For debugging. Dump configurations to system output as xml format. 316 * Master and RS configurations can also be dumped using 317 * http services. e.g. "curl http://master:16010/dump" 318 */ 319 public static void main(String[] args) throws Exception { 320 HBaseConfiguration.create().writeXml(System.out); 321 } 322}