001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase;
019
020import java.io.IOException;
021import java.lang.reflect.InvocationTargetException;
022import java.lang.reflect.Method;
023import java.util.Map;
024
025import org.apache.hadoop.conf.Configuration;
026import org.apache.hadoop.hbase.util.VersionInfo;
027import org.apache.hadoop.hbase.zookeeper.ZKConfig;
028import org.apache.yetus.audience.InterfaceAudience;
029import org.slf4j.Logger;
030import org.slf4j.LoggerFactory;
031
032/**
033 * Adds HBase configuration files to a Configuration
034 */
035@InterfaceAudience.Public
036public class HBaseConfiguration extends Configuration {
037  private static final Logger LOG = LoggerFactory.getLogger(HBaseConfiguration.class);
038
039  static {
040    addDeprecatedKeys();
041  }
042
043  private static void checkDefaultsVersion(Configuration conf) {
044    if (conf.getBoolean("hbase.defaults.for.version.skip", Boolean.FALSE)) return;
045    String defaultsVersion = conf.get("hbase.defaults.for.version");
046    String thisVersion = VersionInfo.getVersion();
047    if (!thisVersion.equals(defaultsVersion)) {
048      throw new RuntimeException(
049        "hbase-default.xml file seems to be for an older version of HBase (" +
050        defaultsVersion + "), this version is " + thisVersion);
051    }
052  }
053
054  /**
055   * The hbase.ipc.server.reservoir.initial.max and hbase.ipc.server.reservoir.initial.buffer.size
056   * were introduced in HBase2.0.0, while in HBase3.0.0 the two config keys will be replaced by
057   * hbase.server.allocator.max.buffer.count and hbase.server.allocator.buffer.size.
058   * Also the hbase.ipc.server.reservoir.enabled will be replaced by
059   * hbase.server.allocator.pool.enabled. Keep the three old config keys here for HBase2.x
060   * compatibility.
061   * <br>
062   * HBASE-24667: This config hbase.regionserver.hostname.disable.master.reversedns will be
063   * replaced by hbase.unsafe.regionserver.hostname.disable.master.reversedns. Keep the old config
064   * keys here for backward compatibility.
065   * <br>
066   * Note: Before Hadoop-3.3, we must call the addDeprecations method before creating the
067   * Configuration object to work correctly. After this bug is fixed in hadoop-3.3, there will be
068   * no order problem.
069   * @see <a href="https://issues.apache.org/jira/browse/HADOOP-15708">HADOOP-15708</a>
070   */
071  private static void addDeprecatedKeys() {
072    Configuration.addDeprecations(new DeprecationDelta[]{
073      new DeprecationDelta("hbase.regionserver.hostname", "hbase.unsafe.regionserver.hostname"),
074      new DeprecationDelta("hbase.regionserver.hostname.disable.master.reversedns",
075        "hbase.unsafe.regionserver.hostname.disable.master.reversedns"),
076      new DeprecationDelta("hbase.offheapcache.minblocksize",
077        "hbase.blockcache.minblocksize"),
078      new DeprecationDelta("hbase.ipc.server.reservoir.enabled",
079        "hbase.server.allocator.pool.enabled"),
080      new DeprecationDelta("hbase.ipc.server.reservoir.initial.max",
081        "hbase.server.allocator.max.buffer.count"),
082      new DeprecationDelta("hbase.ipc.server.reservoir.initial.buffer.size",
083        "hbase.server.allocator.buffer.size"),
084      new DeprecationDelta("hlog.bulk.output", "wal.bulk.output"),
085      new DeprecationDelta("hlog.input.tables", "wal.input.tables"),
086      new DeprecationDelta("hlog.input.tablesmap", "wal.input.tablesmap"),
087      new DeprecationDelta("hbase.master.mob.ttl.cleaner.period",
088        "hbase.master.mob.cleaner.period"),
089      new DeprecationDelta("hbase.normalizer.min.region.count",
090        "hbase.normalizer.merge.min.region.count")
091    });
092  }
093
094  public static Configuration addHbaseResources(Configuration conf) {
095    conf.addResource("hbase-default.xml");
096    conf.addResource("hbase-site.xml");
097
098    checkDefaultsVersion(conf);
099    return conf;
100  }
101
102  /**
103   * Creates a Configuration with HBase resources
104   * @return a Configuration with HBase resources
105   */
106  public static Configuration create() {
107    Configuration conf = new Configuration();
108    // In case HBaseConfiguration is loaded from a different classloader than
109    // Configuration, conf needs to be set with appropriate class loader to resolve
110    // HBase resources.
111    conf.setClassLoader(HBaseConfiguration.class.getClassLoader());
112    return addHbaseResources(conf);
113  }
114
115  /**
116   * @param that Configuration to clone.
117   * @return a Configuration created with the hbase-*.xml files plus
118   * the given configuration.
119   */
120  public static Configuration create(final Configuration that) {
121    Configuration conf = create();
122    merge(conf, that);
123    return conf;
124  }
125
126  /**
127   * Merge two configurations.
128   * @param destConf the configuration that will be overwritten with items
129   *                 from the srcConf
130   * @param srcConf the source configuration
131   **/
132  public static void merge(Configuration destConf, Configuration srcConf) {
133    for (Map.Entry<String, String> e : srcConf) {
134      destConf.set(e.getKey(), e.getValue());
135    }
136  }
137
138  /**
139   * Returns a subset of the configuration properties, matching the given key prefix.
140   * The prefix is stripped from the return keys, ie. when calling with a prefix of "myprefix",
141   * the entry "myprefix.key1 = value1" would be returned as "key1 = value1".  If an entry's
142   * key matches the prefix exactly ("myprefix = value2"), it will <strong>not</strong> be
143   * included in the results, since it would show up as an entry with an empty key.
144   */
145  public static Configuration subset(Configuration srcConf, String prefix) {
146    Configuration newConf = new Configuration(false);
147    for (Map.Entry<String, String> entry : srcConf) {
148      if (entry.getKey().startsWith(prefix)) {
149        String newKey = entry.getKey().substring(prefix.length());
150        // avoid entries that would produce an empty key
151        if (!newKey.isEmpty()) {
152          newConf.set(newKey, entry.getValue());
153        }
154      }
155    }
156    return newConf;
157  }
158
159  /**
160   * Sets all the entries in the provided {@code Map<String, String>} as properties in the
161   * given {@code Configuration}.  Each property will have the specified prefix prepended,
162   * so that the configuration entries are keyed by {@code prefix + entry.getKey()}.
163   */
164  public static void setWithPrefix(Configuration conf, String prefix,
165                                   Iterable<Map.Entry<String, String>> properties) {
166    for (Map.Entry<String, String> entry : properties) {
167      conf.set(prefix + entry.getKey(), entry.getValue());
168    }
169  }
170
171  /**
172   * @return whether to show HBase Configuration in servlet
173   */
174  public static boolean isShowConfInServlet() {
175    boolean isShowConf = false;
176    try {
177      if (Class.forName("org.apache.hadoop.conf.ConfServlet") != null) {
178        isShowConf = true;
179      }
180    } catch (LinkageError e) {
181       // should we handle it more aggressively in addition to log the error?
182       LOG.warn("Error thrown: ", e);
183    } catch (ClassNotFoundException ce) {
184      LOG.debug("ClassNotFound: ConfServlet");
185      // ignore
186    }
187    return isShowConf;
188  }
189
190  /**
191   * Get the password from the Configuration instance using the
192   * getPassword method if it exists. If not, then fall back to the
193   * general get method for configuration elements.
194   *
195   * @param conf    configuration instance for accessing the passwords
196   * @param alias   the name of the password element
197   * @param defPass the default password
198   * @return String password or default password
199   * @throws IOException
200   */
201  public static String getPassword(Configuration conf, String alias,
202      String defPass) throws IOException {
203    String passwd;
204    char[] p = conf.getPassword(alias);
205    if (p != null) {
206      LOG.debug("Config option {} was found through the Configuration getPassword method.", alias);
207      passwd = new String(p);
208    } else {
209      LOG.debug("Config option {} was not found. Using provided default value", alias);
210      passwd = defPass;
211    }
212    return passwd;
213  }
214
215  /**
216   * Generates a {@link Configuration} instance by applying the ZooKeeper cluster key
217   * to the base Configuration.  Note that additional configuration properties may be needed
218   * for a remote cluster, so it is preferable to use
219   * {@link #createClusterConf(Configuration, String, String)}.
220   *
221   * @param baseConf the base configuration to use, containing prefixed override properties
222   * @param clusterKey the ZooKeeper quorum cluster key to apply, or {@code null} if none
223   *
224   * @return the merged configuration with override properties and cluster key applied
225   *
226   * @see #createClusterConf(Configuration, String, String)
227   */
228  public static Configuration createClusterConf(Configuration baseConf, String clusterKey)
229      throws IOException {
230    return createClusterConf(baseConf, clusterKey, null);
231  }
232
233  /**
234   * Generates a {@link Configuration} instance by applying property overrides prefixed by
235   * a cluster profile key to the base Configuration.  Override properties are extracted by
236   * the {@link #subset(Configuration, String)} method, then the merged on top of the base
237   * Configuration and returned.
238   *
239   * @param baseConf the base configuration to use, containing prefixed override properties
240   * @param clusterKey the ZooKeeper quorum cluster key to apply, or {@code null} if none
241   * @param overridePrefix the property key prefix to match for override properties,
242   *     or {@code null} if none
243   * @return the merged configuration with override properties and cluster key applied
244   */
245  public static Configuration createClusterConf(Configuration baseConf, String clusterKey,
246      String overridePrefix) throws IOException {
247    Configuration clusterConf = HBaseConfiguration.create(baseConf);
248    if (clusterKey != null && !clusterKey.isEmpty()) {
249      applyClusterKeyToConf(clusterConf, clusterKey);
250    }
251
252    if (overridePrefix != null && !overridePrefix.isEmpty()) {
253      Configuration clusterSubset = HBaseConfiguration.subset(clusterConf, overridePrefix);
254      HBaseConfiguration.merge(clusterConf, clusterSubset);
255    }
256    return clusterConf;
257  }
258
259  /**
260   * Apply the settings in the given key to the given configuration, this is
261   * used to communicate with distant clusters
262   * @param conf configuration object to configure
263   * @param key string that contains the 3 required configuratins
264   */
265  private static void applyClusterKeyToConf(Configuration conf, String key)
266      throws IOException {
267    ZKConfig.ZKClusterKey zkClusterKey = ZKConfig.transformClusterKey(key);
268    conf.set(HConstants.ZOOKEEPER_QUORUM, zkClusterKey.getQuorumString());
269    conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, zkClusterKey.getClientPort());
270    conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, zkClusterKey.getZnodeParent());
271    // Without the right registry, the above configs are useless. Also, we don't use setClass()
272    // here because the ConnectionRegistry* classes are not resolvable from this module.
273    // This will be broken if ZkConnectionRegistry class gets renamed or moved. Is there a better
274    // way?
275    LOG.info("Overriding client registry implementation to {}",
276        HConstants.ZK_CONNECTION_REGISTRY_CLASS);
277    conf.set(HConstants.CLIENT_CONNECTION_REGISTRY_IMPL_CONF_KEY,
278        HConstants.ZK_CONNECTION_REGISTRY_CLASS);
279  }
280
281  /**
282   * For debugging.  Dump configurations to system output as xml format.
283   * Master and RS configurations can also be dumped using
284   * http services. e.g. "curl http://master:16010/dump"
285   */
286  public static void main(String[] args) throws Exception {
287    HBaseConfiguration.create().writeXml(System.out);
288  }
289}