001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase; 019 020import java.io.DataOutput; 021import java.io.IOException; 022import java.io.OutputStream; 023import java.util.ArrayList; 024import java.util.HashMap; 025import java.util.Iterator; 026import java.util.List; 027import java.util.Map; 028import org.apache.hadoop.conf.Configuration; 029import org.apache.hadoop.hbase.util.Bytes; 030import org.apache.yetus.audience.InterfaceAudience; 031 032import org.apache.hbase.thirdparty.org.apache.commons.collections4.iterators.UnmodifiableIterator; 033 034/** 035 * Do a shallow merge of multiple KV configuration pools. This is a very useful utility class to 036 * easily add per-object configurations in addition to wider scope settings. This is different from 037 * Configuration.addResource() functionality, which performs a deep merge and mutates the common 038 * data structure. 039 * <p> 040 * The iterator on CompoundConfiguration is unmodifiable. Obtaining iterator is an expensive 041 * operation. 042 * <p> 043 * For clarity: the shallow merge allows the user to mutate either of the configuration objects and 044 * have changes reflected everywhere. In contrast to a deep merge, that requires you to explicitly 045 * know all applicable copies to propagate changes. WARNING: The values set in the 046 * CompoundConfiguration are do not handle Property variable substitution. However, if they are set 047 * in the underlying configuration substitutions are done. 048 */ 049@InterfaceAudience.Private 050public class CompoundConfiguration extends Configuration { 051 052 private Configuration mutableConf = null; 053 054 /** 055 * Default Constructor. Initializes empty configuration 056 */ 057 public CompoundConfiguration() { 058 } 059 060 // Devs: these APIs are the same contract as their counterparts in 061 // Configuration.java 062 private interface ImmutableConfigMap extends Iterable<Map.Entry<String, String>> { 063 String get(String key); 064 065 String getRaw(String key); 066 067 Class<?> getClassByName(String name) throws ClassNotFoundException; 068 069 int size(); 070 } 071 072 private final List<ImmutableConfigMap> configs = new ArrayList<>(); 073 074 static class ImmutableConfWrapper implements ImmutableConfigMap { 075 private final Configuration c; 076 077 ImmutableConfWrapper(Configuration conf) { 078 c = conf; 079 } 080 081 @Override 082 public Iterator<Map.Entry<String, String>> iterator() { 083 return c.iterator(); 084 } 085 086 @Override 087 public String get(String key) { 088 return c.get(key); 089 } 090 091 @Override 092 public String getRaw(String key) { 093 return c.getRaw(key); 094 } 095 096 @Override 097 public Class<?> getClassByName(String name) throws ClassNotFoundException { 098 return c.getClassByName(name); 099 } 100 101 @Override 102 public int size() { 103 return c.size(); 104 } 105 106 @Override 107 public String toString() { 108 return c.toString(); 109 } 110 } 111 112 /** 113 * If set has been called, it will create a mutableConf. This converts the mutableConf to an 114 * immutable one and resets it to allow a new mutable conf. This is used when a new map or conf is 115 * added to the compound configuration to preserve proper override semantics. 116 */ 117 void freezeMutableConf() { 118 if (mutableConf == null) { 119 // do nothing if there is no current mutableConf 120 return; 121 } 122 123 this.configs.add(0, new ImmutableConfWrapper(mutableConf)); 124 mutableConf = null; 125 } 126 127 /** 128 * Add Hadoop Configuration object to config list. The added configuration overrides the previous 129 * ones if there are name collisions. 130 * @param conf configuration object 131 * @return this, for builder pattern 132 */ 133 public CompoundConfiguration add(final Configuration conf) { 134 freezeMutableConf(); 135 136 if (conf instanceof CompoundConfiguration) { 137 this.configs.addAll(0, ((CompoundConfiguration) conf).configs); 138 return this; 139 } 140 // put new config at the front of the list (top priority) 141 this.configs.add(0, new ImmutableConfWrapper(conf)); 142 return this; 143 } 144 145 /** 146 * Add Bytes map to config list. This map is generally created by HTableDescriptor or 147 * HColumnDescriptor, but can be abstractly used. The added configuration overrides the previous 148 * ones if there are name collisions. n * Bytes map 149 * @return this, for builder pattern 150 */ 151 public CompoundConfiguration addBytesMap(final Map<Bytes, Bytes> map) { 152 freezeMutableConf(); 153 154 // put new map at the front of the list (top priority) 155 this.configs.add(0, new ImmutableConfigMap() { 156 private final Map<Bytes, Bytes> m = map; 157 158 @Override 159 public Iterator<Map.Entry<String, String>> iterator() { 160 Map<String, String> ret = new HashMap<>(); 161 for (Map.Entry<Bytes, Bytes> entry : map.entrySet()) { 162 String key = Bytes.toString(entry.getKey().get()); 163 String val = entry.getValue() == null ? null : Bytes.toString(entry.getValue().get()); 164 ret.put(key, val); 165 } 166 return ret.entrySet().iterator(); 167 } 168 169 @Override 170 public String get(String key) { 171 Bytes ibw = new Bytes(Bytes.toBytes(key)); 172 if (!m.containsKey(ibw)) return null; 173 Bytes value = m.get(ibw); 174 if (value == null || value.get() == null) return null; 175 return Bytes.toString(value.get()); 176 } 177 178 @Override 179 public String getRaw(String key) { 180 return get(key); 181 } 182 183 @Override 184 public Class<?> getClassByName(String name) throws ClassNotFoundException { 185 return null; 186 } 187 188 @Override 189 public int size() { 190 return m.size(); 191 } 192 193 @Override 194 public String toString() { 195 return m.toString(); 196 } 197 }); 198 return this; 199 } 200 201 /** 202 * Add String map to config list. This map is generally created by HTableDescriptor or 203 * HColumnDescriptor, but can be abstractly used. The added configuration overrides the previous 204 * ones if there are name collisions. 205 * @return this, for builder pattern 206 */ 207 public CompoundConfiguration addStringMap(final Map<String, String> map) { 208 freezeMutableConf(); 209 210 // put new map at the front of the list (top priority) 211 this.configs.add(0, new ImmutableConfigMap() { 212 private final Map<String, String> m = map; 213 214 @Override 215 public Iterator<Map.Entry<String, String>> iterator() { 216 return map.entrySet().iterator(); 217 } 218 219 @Override 220 public String get(String key) { 221 return m.get(key); 222 } 223 224 @Override 225 public String getRaw(String key) { 226 return get(key); 227 } 228 229 @Override 230 public Class<?> getClassByName(String name) throws ClassNotFoundException { 231 return null; 232 } 233 234 @Override 235 public int size() { 236 return m.size(); 237 } 238 239 @Override 240 public String toString() { 241 return m.toString(); 242 } 243 }); 244 return this; 245 } 246 247 @Override 248 public String toString() { 249 StringBuilder sb = new StringBuilder(); 250 sb.append("CompoundConfiguration: " + this.configs.size() + " configs"); 251 for (ImmutableConfigMap m : this.configs) { 252 sb.append(m); 253 } 254 return sb.toString(); 255 } 256 257 @Override 258 public String get(String key) { 259 if (mutableConf != null) { 260 String value = mutableConf.get(key); 261 if (value != null) { 262 return value; 263 } 264 } 265 266 for (ImmutableConfigMap m : this.configs) { 267 String value = m.get(key); 268 if (value != null) { 269 return value; 270 } 271 } 272 return null; 273 } 274 275 @Override 276 public String getRaw(String key) { 277 if (mutableConf != null) { 278 String value = mutableConf.getRaw(key); 279 if (value != null) { 280 return value; 281 } 282 } 283 284 for (ImmutableConfigMap m : this.configs) { 285 String value = m.getRaw(key); 286 if (value != null) { 287 return value; 288 } 289 } 290 return null; 291 } 292 293 @Override 294 public Class<?> getClassByName(String name) throws ClassNotFoundException { 295 if (mutableConf != null) { 296 Class<?> value = mutableConf.getClassByName(name); 297 if (value != null) { 298 return value; 299 } 300 } 301 302 for (ImmutableConfigMap m : this.configs) { 303 Class<?> value = m.getClassByName(name); 304 if (value != null) { 305 return value; 306 } 307 } 308 throw new ClassNotFoundException(); 309 } 310 311 // TODO: This method overestimates the number of configuration settings -- if a value is masked 312 // by an overriding config or map, it will be counted multiple times. 313 @Override 314 public int size() { 315 int ret = 0; 316 317 if (mutableConf != null) { 318 ret += mutableConf.size(); 319 } 320 321 for (ImmutableConfigMap m : this.configs) { 322 ret += m.size(); 323 } 324 return ret; 325 } 326 327 /** 328 * Get the value of the <code>name</code>. If the key is deprecated, it returns the value of the 329 * first key which replaces the deprecated key and is not null. If no such property exists, then 330 * <code>defaultValue</code> is returned. The CompooundConfiguration does not do property 331 * substitution. To do so we need Configuration.getProps to be protected or package visible. 332 * Though in hadoop2 it is protected, in hadoop1 the method is private and not accessible. All of 333 * the get* methods call this overridden get method. 334 * @param name property name. 335 * @param defaultValue default value. 336 * @return property value, or <code>defaultValue</code> if the property doesn't exist. 337 **/ 338 @Override 339 public String get(String name, String defaultValue) { 340 String ret = get(name); 341 return ret == null ? defaultValue : ret; 342 } 343 344 @Override 345 public Iterator<Map.Entry<String, String>> iterator() { 346 Map<String, String> ret = new HashMap<>(); 347 348 // add in reverse order so that oldest get overridden. 349 if (!configs.isEmpty()) { 350 for (int i = configs.size() - 1; i >= 0; i--) { 351 ImmutableConfigMap map = configs.get(i); 352 Iterator<Map.Entry<String, String>> iter = map.iterator(); 353 while (iter.hasNext()) { 354 Map.Entry<String, String> entry = iter.next(); 355 ret.put(entry.getKey(), entry.getValue()); 356 } 357 } 358 } 359 360 // add mutations to this CompoundConfiguration last. 361 if (mutableConf != null) { 362 Iterator<Map.Entry<String, String>> miter = mutableConf.iterator(); 363 while (miter.hasNext()) { 364 Map.Entry<String, String> entry = miter.next(); 365 ret.put(entry.getKey(), entry.getValue()); 366 } 367 } 368 369 return UnmodifiableIterator.unmodifiableIterator(ret.entrySet().iterator()); 370 } 371 372 @Override 373 public void set(String name, String value) { 374 if (mutableConf == null) { 375 // not thread safe 376 mutableConf = new Configuration(false); // an empty configuration 377 } 378 mutableConf.set(name, value); 379 } 380 381 /*********************************************************************************************** 382 * These methods are unsupported, and no code using CompoundConfiguration depend upon them. 383 * Quickly abort upon any attempts to use them. 384 **********************************************************************************************/ 385 386 @Override 387 public void clear() { 388 throw new UnsupportedOperationException("Immutable Configuration"); 389 } 390 391 @Override 392 public void write(DataOutput out) throws IOException { 393 throw new UnsupportedOperationException("Immutable Configuration"); 394 } 395 396 @Override 397 public void writeXml(OutputStream out) throws IOException { 398 throw new UnsupportedOperationException("Immutable Configuration"); 399 } 400}