View Javadoc

1   /**
2    * Copyright The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase;
21  
22  import java.io.DataOutput;
23  import java.io.IOException; 
24  import java.io.OutputStream;
25  import java.util.ArrayList;
26  import java.util.HashMap;
27  import java.util.Iterator;
28  import java.util.List;
29  import java.util.Map;
30  
31  import org.apache.commons.collections.iterators.UnmodifiableIterator;
32  import org.apache.hadoop.classification.InterfaceAudience;
33  import org.apache.hadoop.conf.Configuration;
34  import org.apache.hadoop.hbase.util.Bytes;
35  import org.apache.hadoop.hbase.util.Bytes;
36  
37  /**
38   * Do a shallow merge of multiple KV configuration pools. This is a very useful
39   * utility class to easily add per-object configurations in addition to wider
40   * scope settings. This is different from Configuration.addResource()
41   * functionality, which performs a deep merge and mutates the common data
42   * structure.
43   * <p>
44   * The iterator on CompoundConfiguration is unmodifiable. Obtaining iterator is an expensive
45   * operation.
46   * <p>
47   * For clarity: the shallow merge allows the user to mutate either of the
48   * configuration objects and have changes reflected everywhere. In contrast to a
49   * deep merge, that requires you to explicitly know all applicable copies to
50   * propagate changes.
51   * 
52   * WARNING: The values set in the CompoundConfiguration are do not handle Property variable
53   * substitution.  However, if they are set in the underlying configuration substitutions are
54   * done. 
55   */
56  @InterfaceAudience.Private
57  public class CompoundConfiguration extends Configuration {
58  
59    private Configuration mutableConf = null;
60  
61    /**
62     * Default Constructor. Initializes empty configuration
63     */
64    public CompoundConfiguration() {
65    }
66  
67    // Devs: these APIs are the same contract as their counterparts in
68    // Configuration.java
69    private interface ImmutableConfigMap extends Iterable<Map.Entry<String,String>> {
70      String get(String key);
71      String getRaw(String key);
72      Class<?> getClassByName(String name) throws ClassNotFoundException;
73      int size();
74    }
75  
76    protected List<ImmutableConfigMap> configs
77      = new ArrayList<ImmutableConfigMap>();
78  
79    static class ImmutableConfWrapper implements  ImmutableConfigMap {
80      Configuration c;
81      
82      ImmutableConfWrapper(Configuration conf) {
83        c = conf;
84      }
85  
86      @Override
87      public Iterator<Map.Entry<String,String>> iterator() {
88        return c.iterator();
89      }
90      
91      @Override
92      public String get(String key) {
93        return c.get(key);
94      }
95  
96      @Override
97      public String getRaw(String key) {
98        return c.getRaw(key);
99      }
100 
101     @Override
102     public Class<?> getClassByName(String name)
103         throws ClassNotFoundException {
104       return c.getClassByName(name);
105     }
106 
107     @Override
108     public int size() {
109       return c.size();
110     }
111 
112     @Override
113     public String toString() {
114       return c.toString();
115     }
116   }
117 
118   /**
119    * If set has been called, it will create a mutableConf.  This converts the mutableConf to an
120    * immutable one and resets it to allow a new mutable conf.  This is used when a new map or
121    * conf is added to the compound configuration to preserve proper override semantics.
122    */
123   void freezeMutableConf() {
124     if (mutableConf == null) {
125       // do nothing if there is no current mutableConf
126       return;
127     }
128 
129     this.configs.add(0, new ImmutableConfWrapper(mutableConf));
130     mutableConf = null;
131   }
132 
133   /**
134    * Add Hadoop Configuration object to config list.
135    * The added configuration overrides the previous ones if there are name collisions.
136    * @param conf configuration object
137    * @return this, for builder pattern
138    */
139   public CompoundConfiguration add(final Configuration conf) {
140     freezeMutableConf();
141 
142     if (conf instanceof CompoundConfiguration) {
143       this.configs.addAll(0, ((CompoundConfiguration) conf).configs);
144       return this;
145     }
146     // put new config at the front of the list (top priority)
147     this.configs.add(0, new ImmutableConfWrapper(conf));
148     return this;
149   }
150 
151   /**
152    * Add Bytes map to config list. This map is generally
153    * created by HTableDescriptor or HColumnDescriptor, but can be abstractly
154    * used. The added configuration overrides the previous ones if there are
155    * name collisions.
156    *
157    * @param map
158    *          Bytes map
159    * @return this, for builder pattern
160    */
161   public CompoundConfiguration addBytesMap(
162       final Map<Bytes, Bytes> map) {
163     freezeMutableConf();
164 
165     // put new map at the front of the list (top priority)
166     this.configs.add(0, new ImmutableConfigMap() {
167       Map<Bytes, Bytes> m = map;
168 
169       @Override
170       public Iterator<Map.Entry<String,String>> iterator() {
171         Map<String, String> ret = new HashMap<String, String>();
172         for (Map.Entry<Bytes, Bytes> entry : map.entrySet()) {
173           String key = Bytes.toString(entry.getKey().get());
174           String val = entry.getValue() == null ? null : Bytes.toString(entry.getValue().get());
175           ret.put(key, val);
176         }
177         return ret.entrySet().iterator();
178       }
179       
180       @Override
181       public String get(String key) {
182         Bytes ibw = new Bytes(Bytes
183             .toBytes(key));
184         if (!m.containsKey(ibw))
185           return null;
186         Bytes value = m.get(ibw);
187         if (value == null || value.get() == null)
188           return null;
189         return Bytes.toString(value.get());
190       }
191 
192       @Override
193       public String getRaw(String key) {
194         return get(key);
195       }
196 
197       @Override
198       public Class<?> getClassByName(String name)
199       throws ClassNotFoundException {
200         return null;
201       }
202 
203       @Override
204       public int size() {
205         return m.size();
206       }
207 
208       @Override
209       public String toString() {
210         return m.toString();
211       }
212     });
213     return this;
214   }
215 
216   /**
217    * Add String map to config list. This map is generally created by HTableDescriptor
218    * or HColumnDescriptor, but can be abstractly used. The added configuration
219    * overrides the previous ones if there are name collisions.
220    *
221    * @return this, for builder pattern
222    */
223   public CompoundConfiguration addStringMap(final Map<String, String> map) {
224     freezeMutableConf();
225 
226     // put new map at the front of the list (top priority)
227     this.configs.add(0, new ImmutableConfigMap() {
228       Map<String, String> m = map;
229 
230       @Override
231       public Iterator<Map.Entry<String,String>> iterator() {
232         return map.entrySet().iterator();
233       }
234 
235       @Override
236       public String get(String key) {
237         return m.get(key);
238       }
239 
240       @Override
241       public String getRaw(String key) {
242         return get(key);
243       }
244 
245       @Override
246       public Class<?> getClassByName(String name)
247       throws ClassNotFoundException {
248         return null;
249       }
250 
251       @Override
252       public int size() {
253         return m.size();
254       }
255 
256       @Override
257       public String toString() {
258         return m.toString();
259       }
260     });
261     return this;
262   }
263 
264   @Override
265   public String toString() {
266     StringBuffer sb = new StringBuffer();
267     sb.append("CompoundConfiguration: " + this.configs.size() + " configs");
268     for (ImmutableConfigMap m : this.configs) {
269       sb.append(this.configs);
270     }
271     return sb.toString();
272   }
273 
274   @Override
275   public String get(String key) {
276     if (mutableConf != null) {
277       String value = mutableConf.get(key);
278       if (value != null) {
279         return value;
280       }
281     }
282 
283     for (ImmutableConfigMap m : this.configs) {
284       String value = m.get(key);
285       if (value != null) {
286         return value;
287       }
288     }
289     return null;
290   }
291 
292   @Override
293   public String getRaw(String key) {
294     if (mutableConf != null) {
295       String value = mutableConf.getRaw(key);
296       if (value != null) {
297         return value;
298       }
299     }
300 
301     for (ImmutableConfigMap m : this.configs) {
302       String value = m.getRaw(key);
303       if (value != null) {
304         return value;
305       }
306     }
307     return null;
308   }
309 
310   @Override
311   public Class<?> getClassByName(String name) throws ClassNotFoundException {
312     if (mutableConf != null) {
313       Class<?> value = mutableConf.getClassByName(name);
314       if (value != null) {
315         return value;
316       }
317     }
318 
319     for (ImmutableConfigMap m : this.configs) {
320       Class<?> value = m.getClassByName(name);
321       if (value != null) {
322         return value;
323       }
324     }
325     throw new ClassNotFoundException();
326   }
327 
328   // TODO: This method overestimates the number of configuration settings -- if a value is masked
329   // by an overriding config or map, it will be counted multiple times. 
330   @Override
331   public int size() {
332     int ret = 0;
333 
334     if (mutableConf != null) {
335       ret += mutableConf.size();
336     }
337 
338     for (ImmutableConfigMap m : this.configs) {
339       ret += m.size();
340     }
341     return ret;
342   }
343 
344   /**
345    * Get the value of the <code>name</code>. If the key is deprecated,
346    * it returns the value of the first key which replaces the deprecated key
347    * and is not null.
348    * If no such property exists,
349    * then <code>defaultValue</code> is returned.
350 
351    * The CompooundConfiguration does not do property substitution.  To do so we need
352    * Configuration.getProps to be protected or package visible.  Though in hadoop2 it is
353    * protected, in hadoop1 the method is private and not accessible.
354    * 
355    * All of the get* methods call this overridden get method.
356    * 
357    * @param name property name.
358    * @param defaultValue default value.
359    * @return property value, or <code>defaultValue</code> if the property 
360    *         doesn't exist.                    
361    **/
362   @Override
363   public String get(String name, String defaultValue) {
364     String ret = get(name);
365     return ret == null ? defaultValue : ret;
366   }
367 
368   @Override
369   public Iterator<Map.Entry<String, String>> iterator() {
370     Map<String, String> ret = new HashMap<String, String>();
371 
372     // add in reverse order so that oldest get overridden.
373     if (!configs.isEmpty()) {
374       for (int i = configs.size() - 1; i >= 0; i--) {
375         ImmutableConfigMap map = configs.get(i);
376         Iterator<Map.Entry<String, String>> iter = map.iterator();
377         while (iter.hasNext()) {
378           Map.Entry<String, String> entry = iter.next();
379           ret.put(entry.getKey(), entry.getValue());
380         }
381       }
382     }
383 
384     // add mutations to this CompoundConfiguration last.
385     if (mutableConf != null) {
386       Iterator<Map.Entry<String, String>> miter = mutableConf.iterator();
387       while (miter.hasNext()) {
388         Map.Entry<String, String> entry = miter.next();
389         ret.put(entry.getKey(), entry.getValue());
390       }
391     }
392 
393     return UnmodifiableIterator.decorate(ret.entrySet().iterator());
394   }
395 
396   @Override
397   public void set(String name, String value) {
398     if (mutableConf == null) {
399       // not thread safe
400       mutableConf = new Configuration(false); // an empty configuration
401     }
402     mutableConf.set(name,  value);
403   }
404 
405   /***********************************************************************************************
406    * These methods are unsupported, and no code using CompoundConfiguration depend upon them.
407    * Quickly abort upon any attempts to use them. 
408    **********************************************************************************************/
409 
410   @Override
411   public void clear() {
412     throw new UnsupportedOperationException("Immutable Configuration");
413   }
414 
415   @Override
416   public void write(DataOutput out) throws IOException {
417     throw new UnsupportedOperationException("Immutable Configuration");
418   }
419 
420   @Override
421   public void writeXml(OutputStream out) throws IOException {
422     throw new UnsupportedOperationException("Immutable Configuration");
423   }
424 };