001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase; 019 020import java.io.IOException; 021import java.util.Map; 022import org.apache.hadoop.conf.Configuration; 023import org.apache.hadoop.hbase.util.VersionInfo; 024import org.apache.hadoop.hbase.zookeeper.ZKConfig; 025import org.apache.yetus.audience.InterfaceAudience; 026import org.slf4j.Logger; 027import org.slf4j.LoggerFactory; 028 029/** 030 * Adds HBase configuration files to a Configuration 031 */ 032@InterfaceAudience.Public 033public class HBaseConfiguration extends Configuration { 034 private static final Logger LOG = LoggerFactory.getLogger(HBaseConfiguration.class); 035 036 static { 037 addDeprecatedKeys(); 038 } 039 040 private static void checkDefaultsVersion(Configuration conf) { 041 if (conf.getBoolean("hbase.defaults.for.version.skip", Boolean.FALSE)) return; 042 String defaultsVersion = conf.get("hbase.defaults.for.version"); 043 String thisVersion = VersionInfo.getVersion(); 044 if (!thisVersion.equals(defaultsVersion)) { 045 throw new RuntimeException( 046 "hbase-default.xml file seems to be for an older version of HBase (" + defaultsVersion 047 + "), this version is " + thisVersion); 048 } 049 } 050 051 /** 052 * The hbase.ipc.server.reservoir.initial.max and hbase.ipc.server.reservoir.initial.buffer.size 053 * were introduced in HBase2.0.0, while in HBase3.0.0 the two config keys will be replaced by 054 * hbase.server.allocator.max.buffer.count and hbase.server.allocator.buffer.size. Also the 055 * hbase.ipc.server.reservoir.enabled will be replaced by hbase.server.allocator.pool.enabled. 056 * Keep the three old config keys here for HBase2.x compatibility. <br> 057 * HBASE-24667: This config hbase.regionserver.hostname.disable.master.reversedns will be replaced 058 * by hbase.unsafe.regionserver.hostname.disable.master.reversedns. Keep the old config keys here 059 * for backward compatibility. <br> 060 * Note: Before Hadoop-3.3, we must call the addDeprecations method before creating the 061 * Configuration object to work correctly. After this bug is fixed in hadoop-3.3, there will be no 062 * order problem. 063 * @see <a href="https://issues.apache.org/jira/browse/HADOOP-15708">HADOOP-15708</a> 064 */ 065 private static void addDeprecatedKeys() { 066 Configuration.addDeprecations(new DeprecationDelta[] { 067 new DeprecationDelta("hbase.regionserver.hostname", "hbase.unsafe.regionserver.hostname"), 068 new DeprecationDelta("hbase.regionserver.hostname.disable.master.reversedns", 069 "hbase.unsafe.regionserver.hostname.disable.master.reversedns"), 070 new DeprecationDelta("hbase.offheapcache.minblocksize", "hbase.blockcache.minblocksize"), 071 new DeprecationDelta("hbase.ipc.server.reservoir.enabled", 072 "hbase.server.allocator.pool.enabled"), 073 new DeprecationDelta("hbase.ipc.server.reservoir.initial.max", 074 "hbase.server.allocator.max.buffer.count"), 075 new DeprecationDelta("hbase.ipc.server.reservoir.initial.buffer.size", 076 "hbase.server.allocator.buffer.size"), 077 new DeprecationDelta("hlog.bulk.output", "wal.bulk.output"), 078 new DeprecationDelta("hlog.input.tables", "wal.input.tables"), 079 new DeprecationDelta("hlog.input.tablesmap", "wal.input.tablesmap"), 080 new DeprecationDelta("hbase.master.mob.ttl.cleaner.period", 081 "hbase.master.mob.cleaner.period"), 082 new DeprecationDelta("hbase.normalizer.min.region.count", 083 "hbase.normalizer.merge.min.region.count") }); 084 } 085 086 public static Configuration addHbaseResources(Configuration conf) { 087 conf.addResource("hbase-default.xml"); 088 conf.addResource("hbase-site.xml"); 089 090 checkDefaultsVersion(conf); 091 return conf; 092 } 093 094 /** 095 * Creates a Configuration with HBase resources 096 * @return a Configuration with HBase resources 097 */ 098 public static Configuration create() { 099 Configuration conf = new Configuration(); 100 // In case HBaseConfiguration is loaded from a different classloader than 101 // Configuration, conf needs to be set with appropriate class loader to resolve 102 // HBase resources. 103 conf.setClassLoader(HBaseConfiguration.class.getClassLoader()); 104 return addHbaseResources(conf); 105 } 106 107 /** 108 * Creates a Configuration with HBase resources 109 * @param that Configuration to clone. 110 * @return a Configuration created with the hbase-*.xml files plus the given configuration. 111 */ 112 public static Configuration create(final Configuration that) { 113 Configuration conf = create(); 114 merge(conf, that); 115 return conf; 116 } 117 118 /** 119 * Merge two configurations. 120 * @param destConf the configuration that will be overwritten with items from the srcConf 121 * @param srcConf the source configuration 122 **/ 123 public static void merge(Configuration destConf, Configuration srcConf) { 124 for (Map.Entry<String, String> e : srcConf) { 125 destConf.set(e.getKey(), e.getValue()); 126 } 127 } 128 129 /** 130 * Returns a subset of the configuration properties, matching the given key prefix. The prefix is 131 * stripped from the return keys, ie. when calling with a prefix of "myprefix", the entry 132 * "myprefix.key1 = value1" would be returned as "key1 = value1". If an entry's key matches the 133 * prefix exactly ("myprefix = value2"), it will <strong>not</strong> be included in the results, 134 * since it would show up as an entry with an empty key. 135 */ 136 public static Configuration subset(Configuration srcConf, String prefix) { 137 Configuration newConf = new Configuration(false); 138 for (Map.Entry<String, String> entry : srcConf) { 139 if (entry.getKey().startsWith(prefix)) { 140 String newKey = entry.getKey().substring(prefix.length()); 141 // avoid entries that would produce an empty key 142 if (!newKey.isEmpty()) { 143 newConf.set(newKey, entry.getValue()); 144 } 145 } 146 } 147 return newConf; 148 } 149 150 /** 151 * Sets all the entries in the provided {@code Map<String, String>} as properties in the given 152 * {@code Configuration}. Each property will have the specified prefix prepended, so that the 153 * configuration entries are keyed by {@code prefix + entry.getKey()}. 154 */ 155 public static void setWithPrefix(Configuration conf, String prefix, 156 Iterable<Map.Entry<String, String>> properties) { 157 for (Map.Entry<String, String> entry : properties) { 158 conf.set(prefix + entry.getKey(), entry.getValue()); 159 } 160 } 161 162 /** Returns whether to show HBase Configuration in servlet */ 163 public static boolean isShowConfInServlet() { 164 boolean isShowConf = false; 165 try { 166 if (Class.forName("org.apache.hadoop.conf.ConfServlet") != null) { 167 isShowConf = true; 168 } 169 } catch (LinkageError e) { 170 // should we handle it more aggressively in addition to log the error? 171 LOG.warn("Error thrown: ", e); 172 } catch (ClassNotFoundException ce) { 173 LOG.debug("ClassNotFound: ConfServlet"); 174 // ignore 175 } 176 return isShowConf; 177 } 178 179 /** 180 * Get the password from the Configuration instance using the getPassword method if it exists. If 181 * not, then fall back to the general get method for configuration elements. 182 * @param conf configuration instance for accessing the passwords 183 * @param alias the name of the password element 184 * @param defPass the default password 185 * @return String password or default password 186 */ 187 public static String getPassword(Configuration conf, String alias, String defPass) 188 throws IOException { 189 String passwd; 190 char[] p = conf.getPassword(alias); 191 if (p != null) { 192 LOG.debug("Config option {} was found through the Configuration getPassword method.", alias); 193 passwd = new String(p); 194 } else { 195 LOG.debug("Config option {} was not found. Using provided default value", alias); 196 passwd = defPass; 197 } 198 return passwd; 199 } 200 201 /** 202 * Generates a {@link Configuration} instance by applying the ZooKeeper cluster key to the base 203 * Configuration. Note that additional configuration properties may be needed for a remote 204 * cluster, so it is preferable to use {@link #createClusterConf(Configuration, String, String)}. 205 * @param baseConf the base configuration to use, containing prefixed override properties 206 * @param clusterKey the ZooKeeper quorum cluster key to apply, or {@code null} if none 207 * @return the merged configuration with override properties and cluster key applied 208 * @see #createClusterConf(Configuration, String, String) 209 */ 210 public static Configuration createClusterConf(Configuration baseConf, String clusterKey) 211 throws IOException { 212 return createClusterConf(baseConf, clusterKey, null); 213 } 214 215 /** 216 * Generates a {@link Configuration} instance by applying property overrides prefixed by a cluster 217 * profile key to the base Configuration. Override properties are extracted by the 218 * {@link #subset(Configuration, String)} method, then the merged on top of the base Configuration 219 * and returned. 220 * @param baseConf the base configuration to use, containing prefixed override properties 221 * @param clusterKey the ZooKeeper quorum cluster key to apply, or {@code null} if none 222 * @param overridePrefix the property key prefix to match for override properties, or {@code null} 223 * if none 224 * @return the merged configuration with override properties and cluster key applied 225 */ 226 public static Configuration createClusterConf(Configuration baseConf, String clusterKey, 227 String overridePrefix) throws IOException { 228 Configuration clusterConf = HBaseConfiguration.create(baseConf); 229 if (clusterKey != null && !clusterKey.isEmpty()) { 230 applyClusterKeyToConf(clusterConf, clusterKey); 231 } 232 233 if (overridePrefix != null && !overridePrefix.isEmpty()) { 234 Configuration clusterSubset = HBaseConfiguration.subset(clusterConf, overridePrefix); 235 HBaseConfiguration.merge(clusterConf, clusterSubset); 236 } 237 return clusterConf; 238 } 239 240 /** 241 * Apply the settings in the given key to the given configuration, this is used to communicate 242 * with distant clusters 243 * @param conf configuration object to configure 244 * @param key string that contains the 3 required configuratins 245 */ 246 private static void applyClusterKeyToConf(Configuration conf, String key) throws IOException { 247 ZKConfig.ZKClusterKey zkClusterKey = ZKConfig.transformClusterKey(key); 248 conf.set(HConstants.ZOOKEEPER_QUORUM, zkClusterKey.getQuorumString()); 249 conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, zkClusterKey.getClientPort()); 250 conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, zkClusterKey.getZnodeParent()); 251 // Without the right registry, the above configs are useless. Also, we don't use setClass() 252 // here because the ConnectionRegistry* classes are not resolvable from this module. 253 // This will be broken if ZkConnectionRegistry class gets renamed or moved. Is there a better 254 // way? 255 LOG.info("Overriding client registry implementation to {}", 256 HConstants.ZK_CONNECTION_REGISTRY_CLASS); 257 conf.set(HConstants.CLIENT_CONNECTION_REGISTRY_IMPL_CONF_KEY, 258 HConstants.ZK_CONNECTION_REGISTRY_CLASS); 259 } 260 261 /** 262 * For debugging. Dump configurations to system output as xml format. Master and RS configurations 263 * can also be dumped using http services. e.g. "curl http://master:16010/dump" 264 */ 265 public static void main(String[] args) throws Exception { 266 HBaseConfiguration.create().writeXml(System.out); 267 } 268}