View Javadoc

1   /**
2    * Copyright The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.regionserver;
21  
22  import java.io.DataOutput;
23  import java.io.IOException; 
24  import java.io.OutputStream;
25  import java.util.ArrayList;
26  import java.util.HashMap;
27  import java.util.Iterator;
28  import java.util.List;
29  import java.util.Map;
30  import java.util.Map.Entry;
31  
32  import org.apache.commons.collections.iterators.UnmodifiableIterator;
33  import org.apache.hadoop.classification.InterfaceAudience;
34  import org.apache.hadoop.conf.Configuration;
35  import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
36  import org.apache.hadoop.hbase.util.Bytes;
37  
38  /**
39   * Do a shallow merge of multiple KV configuration pools. This is a very useful
40   * utility class to easily add per-object configurations in addition to wider
41   * scope settings. This is different from Configuration.addResource()
42   * functionality, which performs a deep merge and mutates the common data
43   * structure.
44   * <p>
45   * For clarity: the shallow merge allows the user to mutate either of the
46   * configuration objects and have changes reflected everywhere. In contrast to a
47   * deep merge, that requires you to explicitly know all applicable copies to
48   * propagate changes.
49   * <p>
50   * This class is package private because we expect significant refactoring here
51   * on the HBase side when certain HDFS changes are added & ubiquitous. Will
52   * revisit expanding access at that point.
53   *
54   * WARNING: The values set in the CompoundConfiguration are do not handle Property variable
55   * substitution.  However, if they are set in the underlying configuration substitutions are
56   * done.
57   */
58  @InterfaceAudience.Private
59  class CompoundConfiguration extends Configuration {
60  	private Configuration mutableConf = null;
61  
62    /**
63     * Default Constructor. Initializes empty configuration
64     */
65    public CompoundConfiguration() {
66    }
67  
68    // Devs: these APIs are the same contract as their counterparts in
69    // Configuration.java
70    private static interface ImmutableConfigMap extends Iterable<Map.Entry<String,String>> {
71      String get(String key);
72      String getRaw(String key);
73      Class<?> getClassByName(String name) throws ClassNotFoundException;
74      int size();
75    }
76  
77    protected List<ImmutableConfigMap> configs
78      = new ArrayList<ImmutableConfigMap>();
79  
80    /****************************************************************************
81     * These initial APIs actually required original thought
82     ***************************************************************************/
83  
84    static class ImmutableConfWrapper implements  ImmutableConfigMap {
85      Configuration c;
86      
87      ImmutableConfWrapper(Configuration conf) {
88        c = conf;
89      }
90  
91      @Override
92      public Iterator<Map.Entry<String,String>> iterator() {
93        return c.iterator();
94      }
95      
96      @Override
97      public String get(String key) {
98        return c.get(key);
99      }
100 
101     @Override
102     public String getRaw(String key) {
103       return c.getRaw(key);
104     }
105 
106     @Override
107     public Class<?> getClassByName(String name)
108         throws ClassNotFoundException {
109       return c.getClassByName(name);
110     }
111 
112     @Override
113     public int size() {
114       return c.size();
115     }
116 
117     @Override
118     public String toString() {
119       return c.toString();
120     }
121   }
122 
123   /**
124    * If set has been called, it will create a mutableConf.  This converts the mutableConf to an
125    * immutable one and resets it to allow a new mutable conf.  This is used when a new map or
126    * conf is added to the compound configuration to preserve proper override semantics.
127    */
128   void freezeMutableConf() {
129     if (mutableConf == null) {
130       // do nothing if there is no current mutableConf
131       return;
132     }
133 
134     this.configs.add(0, new ImmutableConfWrapper(mutableConf));
135     mutableConf = null;
136   }
137 
138   /**
139    * Add Hadoop Configuration object to config list
140    * @param conf configuration object
141    * @return this, for builder pattern
142    */
143   public CompoundConfiguration add(final Configuration conf) {
144     freezeMutableConf();
145     if (conf instanceof CompoundConfiguration) {
146       this.configs.addAll(0, ((CompoundConfiguration) conf).configs);
147       return this;
148     }
149     // put new config at the front of the list (top priority)
150     this.configs.add(0, new ImmutableConfWrapper(conf));
151     return this;
152   }
153 
154   /**
155    * Add ImmutableBytesWritable map to config list. This map is generally
156    * created by HTableDescriptor or HColumnDescriptor, but can be abstractly
157    * used.
158    *
159    * @param map
160    *          ImmutableBytesWritable map
161    * @return this, for builder pattern
162    */
163   public CompoundConfiguration add(
164       final Map<ImmutableBytesWritable, ImmutableBytesWritable> map) {
165     freezeMutableConf();
166 
167     // put new map at the front of the list (top priority)
168     this.configs.add(0, new ImmutableConfigMap() {
169       Map<ImmutableBytesWritable, ImmutableBytesWritable> m = map;
170 
171       @Override
172       public String get(String key) {
173         ImmutableBytesWritable ibw = new ImmutableBytesWritable(Bytes
174             .toBytes(key));
175         if (!m.containsKey(ibw))
176           return null;
177         ImmutableBytesWritable value = m.get(ibw);
178         if (value == null || value.get() == null)
179           return null;
180         return Bytes.toString(value.get());
181       }
182 
183       @Override
184       public String getRaw(String key) {
185         return get(key);
186       }
187 
188       @Override
189       public Class<?> getClassByName(String name)
190       throws ClassNotFoundException {
191         return null;
192       }
193 
194       @Override
195       public int size() {
196         // TODO Auto-generated method stub
197         return m.size();
198       }
199 
200       @Override
201       public String toString() {
202         return m.toString();
203       }
204 
205       @Override
206       public Iterator<Entry<String, String>> iterator() {
207         final Iterator<Entry<ImmutableBytesWritable, ImmutableBytesWritable>> entries = m
208             .entrySet().iterator();
209         return new Iterator<Entry<String, String>>() {
210 
211           @Override
212           public boolean hasNext() {
213             return entries.hasNext();
214           }
215 
216           @Override
217           public Entry<String, String> next() {
218             final Entry<ImmutableBytesWritable, ImmutableBytesWritable> e = entries.next();
219             return new Entry<String, String>() {
220 
221               @Override
222               public String setValue(String value) {
223                 throw new UnsupportedOperationException(
224                     "Cannot set value on entry from a CompoundConfiguration!");
225               }
226 
227               @Override
228               public String getValue() {
229                 ImmutableBytesWritable bytes = e.getValue();
230                 // unlike regular configuration, ImmutableBytesWritableMaps can take a null value
231                 if (bytes != null) {
232                   return Bytes.toString(bytes.get(), bytes.getOffset(), bytes.getLength());
233                 }
234                 return null;
235               }
236 
237               @Override
238               public String getKey() {
239                 ImmutableBytesWritable bytes = e.getKey();
240                 return Bytes.toString(bytes.get(), bytes.getOffset(), bytes.getLength());
241               }
242             };
243           }
244 
245           @Override
246           public void remove() {
247             throw new UnsupportedOperationException(
248                 "Cannot remove an entry from a CompoundConfiguration iterator");
249           }
250         };
251 
252       }
253     });
254     return this;
255   }
256 
257   /**
258    * Add String map to config list. This map is generally created by HTableDescriptor
259    * or HColumnDescriptor, but can be abstractly used. The added configuration
260    * overrides the previous ones if there are name collisions.
261    *
262    * @return this, for builder pattern
263    */
264   public CompoundConfiguration addStringMap(final Map<String, String> map) {
265     freezeMutableConf();
266 
267     // put new map at the front of the list (top priority)
268     this.configs.add(0, new ImmutableConfigMap() {
269       Map<String, String> m = map;
270 
271       @Override
272       public Iterator<Map.Entry<String,String>> iterator() {
273         return map.entrySet().iterator();
274       }
275 
276       @Override
277       public String get(String key) {
278         return m.get(key);
279       }
280 
281       @Override
282       public String getRaw(String key) {
283         return get(key);
284       }
285 
286       @Override
287       public Class<?> getClassByName(String name)
288       throws ClassNotFoundException {
289         return null;
290       }
291 
292       @Override
293       public int size() {
294         return m.size();
295       }
296 
297       @Override
298       public String toString() {
299         return m.toString();
300       }
301     });
302     return this;
303   }
304 
305   @Override
306   public String toString() {
307     StringBuffer sb = new StringBuffer();
308     sb.append("CompoundConfiguration: " + this.configs.size() + " configs");
309     for (ImmutableConfigMap m : this.configs) {
310       sb.append(this.configs);
311     }
312     return sb.toString();
313   }
314 
315   @Override
316   public String get(String key) {
317     if (mutableConf != null) {
318       String value = mutableConf.get(key);
319       if (value != null) {
320         return value;
321       }
322     }
323 
324     for (ImmutableConfigMap m : this.configs) {
325       String value = m.get(key);
326       if (value != null) {
327         return value;
328       }
329     }
330     return null;
331   }
332 
333   @Override
334   public String getRaw(String key) {
335     if (mutableConf != null) {
336       String value = mutableConf.getRaw(key);
337       if (value != null) {
338         return value;
339       }
340     }
341 
342     for (ImmutableConfigMap m : this.configs) {
343       String value = m.getRaw(key);
344       if (value != null) {
345         return value;
346       }
347     }
348     return null;
349   }
350 
351   @Override
352   public Class<?> getClassByName(String name) throws ClassNotFoundException {
353     if (mutableConf != null) {
354       Class<?> value = mutableConf.getClassByName(name);
355       if (value != null) {
356         return value;
357       }
358     }
359 
360     for (ImmutableConfigMap m : this.configs) {
361       try {
362         Class<?> value = m.getClassByName(name);
363         if (value != null) {
364           return value;
365         }
366       } catch (ClassNotFoundException e) {
367         // don't propagate an exception until all configs fail
368         continue;
369       }
370     }
371     throw new ClassNotFoundException();
372   }
373 
374   // TODO: This method overestimates the number of configuration settings -- if a value is masked
375   // by an overriding config or map, it will be counted multiple times.
376   @Override
377   public int size() {
378     int ret = 0;
379     if (mutableConf != null) {
380       ret += mutableConf.size();
381     }
382     for (ImmutableConfigMap m : this.configs) {
383       ret += m.size();
384     }
385     return ret;
386   }
387 
388   @Override
389   public Iterator<Map.Entry<String, String>> iterator() {
390     Map<String, String> ret = new HashMap<String, String>();
391 
392     // add in reverse order so that oldest get overridden.
393     if (!configs.isEmpty()) {
394       for (int i = configs.size() - 1; i >= 0; i--) {
395         ImmutableConfigMap map = configs.get(i);
396         Iterator<Map.Entry<String, String>> iter = map.iterator();
397         while (iter.hasNext()) {
398           Map.Entry<String, String> entry = iter.next();
399           ret.put(entry.getKey(), entry.getValue());
400         }
401       }
402     }
403     // add mutations to this CompoundConfiguration last.
404     if (mutableConf != null) {
405       Iterator<Map.Entry<String, String>> miter = mutableConf.iterator();
406       while (miter.hasNext()) {
407         Map.Entry<String, String> entry = miter.next();
408         ret.put(entry.getKey(), entry.getValue());
409       }
410     }
411 
412     return UnmodifiableIterator.decorate(ret.entrySet().iterator());
413   }
414 
415   /**
416    * Get the value of the <code>name</code>. If the key is deprecated,
417    * it returns the value of the first key which replaces the deprecated key
418    * and is not null.
419    * If no such property exists,
420    * then <code>defaultValue</code> is returned.
421 
422    * The CompooundConfiguration does not do property substitution.  To do so we need
423    * Configuration.getProps to be protected or package visible.  Though in hadoop2 it is
424    * protected, in hadoop1 the method is private and not accessible.
425    *
426    * All of the get* methods call this overridden get method.
427    *
428    * @param name property name.
429    * @param defaultValue default value.
430    * @return property value, or <code>defaultValue</code> if the property
431    *         doesn't exist.
432    **/
433   @Override
434   public String get(String name, String defaultValue) {
435     String ret = get(name);
436     return ret == null ? defaultValue : ret;
437   }
438 
439   @Override
440   public void set(String name, String value) {
441     if (mutableConf == null) {
442       // not thread safe
443       mutableConf = new Configuration(false); // an empty configuration
444     }
445     mutableConf.set(name,  value);
446   }
447   
448   /***********************************************************************************************
449    * These methods are unsupported, and no code using CompoundConfiguration depend upon them.
450    * Quickly abort upon any attempts to use them.
451    **********************************************************************************************/
452   @Override
453   public void clear() {
454     throw new UnsupportedOperationException("Immutable Configuration");
455   }
456 
457   @Override
458   public void write(DataOutput out) throws IOException {
459     throw new UnsupportedOperationException("Immutable Configuration");
460   }
461 
462   @Override
463   public void writeXml(OutputStream out) throws IOException {
464     throw new UnsupportedOperationException("Immutable Configuration");
465   }
466 };