View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.metrics2.lib;
20  
21  import java.util.Collection;
22  import java.util.concurrent.ConcurrentMap;
23  
24  import org.apache.commons.logging.Log;
25  import org.apache.commons.logging.LogFactory;
26  import org.apache.hadoop.hbase.classification.InterfaceAudience;
27  import org.apache.hadoop.metrics2.MetricsException;
28  import org.apache.hadoop.metrics2.MetricsInfo;
29  import org.apache.hadoop.metrics2.MetricsRecordBuilder;
30  import org.apache.hadoop.metrics2.MetricsTag;
31  import org.apache.hadoop.metrics2.impl.MsInfo;
32  
33  import com.google.common.base.Objects;
34  import com.google.common.collect.Maps;
35  
36  /**
37   * An optional metrics registry class for creating and maintaining a
38   * collection of MetricsMutables, making writing metrics source easier.
39   * NOTE: this is a copy of org.apache.hadoop.metrics2.lib.MetricsRegistry with added one
40   *       feature: metrics can be removed. When HADOOP-8313 is fixed, usages of this class
41   *       should be substituted with org.apache.hadoop.metrics2.lib.MetricsRegistry.
42   *       This implementation also provides handy methods for creating metrics
43   *       dynamically.
44   *       Another difference is that metricsMap implementation is substituted with
45   *       thread-safe map, as we allow dynamic metrics additions/removals.
46   */
47  @InterfaceAudience.Private
48  public class DynamicMetricsRegistry {
49    private static final Log LOG = LogFactory.getLog(DynamicMetricsRegistry.class);
50  
51    private final ConcurrentMap<String, MutableMetric> metricsMap =
52        Maps.newConcurrentMap();
53    private final ConcurrentMap<String, MetricsTag> tagsMap =
54        Maps.newConcurrentMap();
55    private final MetricsInfo metricsInfo;
56    private final DefaultMetricsSystemHelper helper = new DefaultMetricsSystemHelper();
57    private final static String[] histogramSuffixes = new String[]{
58        "_num_ops",
59        "_min",
60        "_max",
61        "_median",
62        "_75th_percentile",
63        "_95th_percentile",
64        "_99th_percentile"};
65  
66    /**
67     * Construct the registry with a record name
68     * @param name  of the record of the metrics
69     */
70    public DynamicMetricsRegistry(String name) {
71      this(Interns.info(name,name));
72    }
73  
74    /**
75     * Construct the registry with a metadata object
76     * @param info  the info object for the metrics record/group
77     */
78    public DynamicMetricsRegistry(MetricsInfo info) {
79      metricsInfo = info;
80    }
81  
82    /**
83     * @return the info object of the metrics registry
84     */
85    public MetricsInfo info() {
86      return metricsInfo;
87    }
88  
89    /**
90     * Get a metric by name
91     * @param name  of the metric
92     * @return the metric object
93     */
94    public MutableMetric get(String name) {
95      return metricsMap.get(name);
96    }
97  
98    /**
99     * Get a tag by name
100    * @param name  of the tag
101    * @return the tag object
102    */
103   public MetricsTag getTag(String name) {
104     return tagsMap.get(name);
105   }
106 
107   /**
108    * Create a mutable integer counter
109    * @param name  of the metric
110    * @param desc  metric description
111    * @param iVal  initial value
112    * @return a new counter object
113    */
114   public MutableCounterInt newCounter(String name, String desc, int iVal) {
115     return newCounter(new MetricsInfoImpl(name, desc), iVal);
116   }
117 
118   /**
119    * Create a mutable integer counter
120    * @param info  metadata of the metric
121    * @param iVal  initial value
122    * @return a new counter object
123    */
124   public MutableCounterInt newCounter(MetricsInfo info, int iVal) {
125     MutableCounterInt ret = new MutableCounterInt(info, iVal);
126     return addNewMetricIfAbsent(info.name(), ret, MutableCounterInt.class);
127   }
128 
129   /**
130    * Create a mutable long integer counter
131    * @param name  of the metric
132    * @param desc  metric description
133    * @param iVal  initial value
134    * @return a new counter object
135    */
136   public MutableCounterLong newCounter(String name, String desc, long iVal) {
137     return newCounter(new MetricsInfoImpl(name, desc), iVal);
138   }
139 
140   /**
141    * Create a mutable long integer counter
142    * @param info  metadata of the metric
143    * @param iVal  initial value
144    * @return a new counter object
145    */
146   public MutableCounterLong newCounter(MetricsInfo info, long iVal) {
147     MutableCounterLong ret = new MutableCounterLong(info, iVal);
148     return addNewMetricIfAbsent(info.name(), ret, MutableCounterLong.class);
149   }
150 
151   /**
152    * Create a mutable integer gauge
153    * @param name  of the metric
154    * @param desc  metric description
155    * @param iVal  initial value
156    * @return a new gauge object
157    */
158   public MutableGaugeInt newGauge(String name, String desc, int iVal) {
159     return newGauge(new MetricsInfoImpl(name, desc), iVal);
160   }
161   /**
162    * Create a mutable integer gauge
163    * @param info  metadata of the metric
164    * @param iVal  initial value
165    * @return a new gauge object
166    */
167   public MutableGaugeInt newGauge(MetricsInfo info, int iVal) {
168     MutableGaugeInt ret = new MutableGaugeInt(info, iVal);
169     return addNewMetricIfAbsent(info.name(), ret, MutableGaugeInt.class);
170   }
171 
172   /**
173    * Create a mutable long integer gauge
174    * @param name  of the metric
175    * @param desc  metric description
176    * @param iVal  initial value
177    * @return a new gauge object
178    */
179   public MutableGaugeLong newGauge(String name, String desc, long iVal) {
180     return newGauge(new MetricsInfoImpl(name, desc), iVal);
181   }
182 
183   /**
184    * Create a mutable long integer gauge
185    * @param info  metadata of the metric
186    * @param iVal  initial value
187    * @return a new gauge object
188    */
189   public MutableGaugeLong newGauge(MetricsInfo info, long iVal) {
190     MutableGaugeLong ret = new MutableGaugeLong(info, iVal);
191     return addNewMetricIfAbsent(info.name(), ret, MutableGaugeLong.class);
192   }
193 
194   /**
195    * Create a mutable metric with stats
196    * @param name  of the metric
197    * @param desc  metric description
198    * @param sampleName  of the metric (e.g., "Ops")
199    * @param valueName   of the metric (e.g., "Time" or "Latency")
200    * @param extended    produce extended stat (stdev, min/max etc.) if true.
201    * @return a new mutable stat metric object
202    */
203   public MutableStat newStat(String name, String desc,
204       String sampleName, String valueName, boolean extended) {
205     MutableStat ret =
206         new MutableStat(name, desc, sampleName, valueName, extended);
207     return addNewMetricIfAbsent(name, ret, MutableStat.class);
208   }
209 
210   /**
211    * Create a mutable metric with stats
212    * @param name  of the metric
213    * @param desc  metric description
214    * @param sampleName  of the metric (e.g., "Ops")
215    * @param valueName   of the metric (e.g., "Time" or "Latency")
216    * @return a new mutable metric object
217    */
218   public MutableStat newStat(String name, String desc,
219                              String sampleName, String valueName) {
220     return newStat(name, desc, sampleName, valueName, false);
221   }
222 
223   /**
224    * Create a mutable rate metric
225    * @param name  of the metric
226    * @return a new mutable metric object
227    */
228   public MutableRate newRate(String name) {
229     return newRate(name, name, false);
230   }
231 
232   /**
233    * Create a mutable rate metric
234    * @param name  of the metric
235    * @param description of the metric
236    * @return a new mutable rate metric object
237    */
238   public MutableRate newRate(String name, String description) {
239     return newRate(name, description, false);
240   }
241 
242   /**
243    * Create a mutable rate metric (for throughput measurement)
244    * @param name  of the metric
245    * @param desc  description
246    * @param extended  produce extended stat (stdev/min/max etc.) if true
247    * @return a new mutable rate metric object
248    */
249   public MutableRate newRate(String name, String desc, boolean extended) {
250     return newRate(name, desc, extended, true);
251   }
252 
253   @InterfaceAudience.Private
254   public MutableRate newRate(String name, String desc,
255       boolean extended, boolean returnExisting) {
256     if (returnExisting) {
257       MutableMetric rate = metricsMap.get(name);
258       if (rate != null) {
259         if (rate instanceof MutableRate) return (MutableRate) rate;
260         throw new MetricsException("Unexpected metrics type "+ rate.getClass()
261                                    +" for "+ name);
262       }
263     }
264     MutableRate ret = new MutableRate(name, desc, extended);
265     return addNewMetricIfAbsent(name, ret, MutableRate.class);
266   }
267 
268   /**
269    * Create a new histogram.
270    * @param name Name of the histogram.
271    * @return A new MutableHistogram
272    */
273   public MutableHistogram newHistogram(String name) {
274      return newHistogram(name, "");
275   }
276 
277   /**
278    * Create a new histogram.
279    * @param name The name of the histogram
280    * @param desc The description of the data in the histogram.
281    * @return A new MutableHistogram
282    */
283   public MutableHistogram newHistogram(String name, String desc) {
284     MutableHistogram histo = new MutableHistogram(name, desc);
285     return addNewMetricIfAbsent(name, histo, MutableHistogram.class);
286   }
287 
288   /**
289    * Create a new MutableQuantile(A more accurate histogram).
290    * @param name The name of the histogram
291    * @return a new MutableQuantile
292    */
293   public MetricMutableQuantiles newQuantile(String name) {
294     return newQuantile(name, "");
295   }
296 
297   public MetricMutableQuantiles newQuantile(String name, String desc) {
298     MetricMutableQuantiles histo = new MetricMutableQuantiles(name, desc, "Ops", "", 60);
299     return addNewMetricIfAbsent(name, histo, MetricMutableQuantiles.class);
300   }
301 
302   synchronized void add(String name, MutableMetric metric) {
303     addNewMetricIfAbsent(name, metric, MutableMetric.class);
304   }
305 
306   /**
307    * Add sample to a stat metric by name.
308    * @param name  of the metric
309    * @param value of the snapshot to add
310    */
311   public void add(String name, long value) {
312     MutableMetric m = metricsMap.get(name);
313 
314     if (m != null) {
315       if (m instanceof MutableStat) {
316         ((MutableStat) m).add(value);
317       }
318       else {
319         throw new MetricsException("Unsupported add(value) for metric "+ name);
320       }
321     }
322     else {
323       metricsMap.put(name, newRate(name)); // default is a rate metric
324       add(name, value);
325     }
326   }
327 
328   /**
329    * Set the metrics context tag
330    * @param name of the context
331    * @return the registry itself as a convenience
332    */
333   public DynamicMetricsRegistry setContext(String name) {
334     return tag(MsInfo.Context, name, true);
335   }
336 
337   /**
338    * Add a tag to the metrics
339    * @param name  of the tag
340    * @param description of the tag
341    * @param value of the tag
342    * @return the registry (for keep adding tags)
343    */
344   public DynamicMetricsRegistry tag(String name, String description, String value) {
345     return tag(name, description, value, false);
346   }
347 
348   /**
349    * Add a tag to the metrics
350    * @param name  of the tag
351    * @param description of the tag
352    * @param value of the tag
353    * @param override  existing tag if true
354    * @return the registry (for keep adding tags)
355    */
356   public DynamicMetricsRegistry tag(String name, String description, String value,
357                              boolean override) {
358     return tag(new MetricsInfoImpl(name, description), value, override);
359   }
360 
361   /**
362    * Add a tag to the metrics
363    * @param info  metadata of the tag
364    * @param value of the tag
365    * @param override existing tag if true
366    * @return the registry (for keep adding tags etc.)
367    */
368   public DynamicMetricsRegistry tag(MetricsInfo info, String value, boolean override) {
369     MetricsTag tag = Interns.tag(info, value);
370 
371     if (!override) {
372       MetricsTag existing = tagsMap.putIfAbsent(info.name(), tag);
373       if (existing != null) {
374         throw new MetricsException("Tag "+ info.name() +" already exists!");
375       }
376       return this;
377     }
378 
379     tagsMap.put(info.name(), tag);
380 
381     return this;
382   }
383 
384   public DynamicMetricsRegistry tag(MetricsInfo info, String value) {
385     return tag(info, value, false);
386   }
387 
388   Collection<MetricsTag> tags() {
389     return tagsMap.values();
390   }
391 
392   Collection<MutableMetric> metrics() {
393     return metricsMap.values();
394   }
395 
396   /**
397    * Sample all the mutable metrics and put the snapshot in the builder
398    * @param builder to contain the metrics snapshot
399    * @param all get all the metrics even if the values are not changed.
400    */
401   public void snapshot(MetricsRecordBuilder builder, boolean all) {
402     for (MetricsTag tag : tags()) {
403       builder.add(tag);
404     }
405     for (MutableMetric metric : metrics()) {
406       metric.snapshot(builder, all);
407     }
408   }
409 
410   @Override public String toString() {
411     return Objects.toStringHelper(this)
412         .add("info", metricsInfo).add("tags", tags()).add("metrics", metrics())
413         .toString();
414   }
415 
416   /**
417    * Removes metric by name
418    * @param name name of the metric to remove
419    */
420   public void removeMetric(String name) {
421     helper.removeObjectName(name);
422     metricsMap.remove(name);
423   }
424 
425   public void removeHistogramMetrics(String baseName) {
426     for (String suffix:histogramSuffixes) {
427       removeMetric(baseName+suffix);
428     }
429   }
430 
431   /**
432    * Get a MetricMutableGaugeLong from the storage.  If it is not there atomically put it.
433    *
434    * @param gaugeName              name of the gauge to create or get.
435    * @param potentialStartingValue value of the new gauge if we have to create it.
436    */
437   public MutableGaugeLong getLongGauge(String gaugeName, long potentialStartingValue) {
438     //Try and get the guage.
439     MutableMetric metric = metricsMap.get(gaugeName);
440 
441     //If it's not there then try and put a new one in the storage.
442     if (metric == null) {
443 
444       //Create the potential new gauge.
445       MutableGaugeLong newGauge = new MutableGaugeLong(new MetricsInfoImpl(gaugeName, ""),
446               potentialStartingValue);
447 
448       // Try and put the gauge in.  This is atomic.
449       metric = metricsMap.putIfAbsent(gaugeName, newGauge);
450 
451       //If the value we get back is null then the put was successful and we will return that.
452       //otherwise gaugeLong should contain the thing that was in before the put could be completed.
453       if (metric == null) {
454         return newGauge;
455       }
456     }
457 
458     if (!(metric instanceof MutableGaugeLong)) {
459       throw new MetricsException("Metric already exists in registry for metric name: " + gaugeName +
460               " and not of type MetricMutableGaugeLong");
461     }
462 
463     return (MutableGaugeLong) metric;
464   }
465 
466   /**
467    * Get a MetricMutableCounterLong from the storage.  If it is not there atomically put it.
468    *
469    * @param counterName            Name of the counter to get
470    * @param potentialStartingValue starting value if we have to create a new counter
471    */
472   public MutableCounterLong getLongCounter(String counterName, long potentialStartingValue) {
473     //See getLongGauge for description on how this works.
474     MutableMetric counter = metricsMap.get(counterName);
475     if (counter == null) {
476       MutableCounterLong newCounter =
477               new MutableCounterLong(new MetricsInfoImpl(counterName, ""), potentialStartingValue);
478       counter = metricsMap.putIfAbsent(counterName, newCounter);
479       if (counter == null) {
480         return newCounter;
481       }
482     }
483 
484 
485     if (!(counter instanceof MutableCounterLong)) {
486       throw new MetricsException("Metric already exists in registry for metric name: " +
487               counterName + " and not of type MetricMutableCounterLong");
488     }
489 
490     return (MutableCounterLong) counter;
491   }
492 
493   public MutableHistogram getHistogram(String histoName) {
494     //See getLongGauge for description on how this works.
495     MutableMetric histo = metricsMap.get(histoName);
496     if (histo == null) {
497       MutableHistogram newCounter =
498           new MutableHistogram(new MetricsInfoImpl(histoName, ""));
499       histo = metricsMap.putIfAbsent(histoName, newCounter);
500       if (histo == null) {
501         return newCounter;
502       }
503     }
504 
505 
506     if (!(histo instanceof MutableHistogram)) {
507       throw new MetricsException("Metric already exists in registry for metric name: " +
508           histoName + " and not of type MutableHistogram");
509     }
510 
511     return (MutableHistogram) histo;
512   }
513 
514   public MetricMutableQuantiles getQuantile(String histoName) {
515     //See getLongGauge for description on how this works.
516     MutableMetric histo = metricsMap.get(histoName);
517     if (histo == null) {
518       MetricMutableQuantiles newCounter =
519           new MetricMutableQuantiles(histoName, "", "Ops", "", 60);
520       histo = metricsMap.putIfAbsent(histoName, newCounter);
521       if (histo == null) {
522         return newCounter;
523       }
524     }
525 
526 
527     if (!(histo instanceof MetricMutableQuantiles)) {
528       throw new MetricsException("Metric already exists in registry for metric name: " +
529           histoName + " and not of type MutableHistogram");
530     }
531 
532     return (MetricMutableQuantiles) histo;
533   }
534 
535   private<T extends MutableMetric> T
536   addNewMetricIfAbsent(String name,
537                        T ret,
538                        Class<T> metricClass) {
539     //If the value we get back is null then the put was successful and we will
540     // return that. Otherwise metric should contain the thing that was in
541     // before the put could be completed.
542     MutableMetric metric = metricsMap.putIfAbsent(name, ret);
543     if (metric == null) {
544       return ret;
545     }
546 
547     return returnExistingWithCast(metric, metricClass, name);
548   }
549 
550   @SuppressWarnings("unchecked")
551   private<T> T returnExistingWithCast(MutableMetric metric,
552                                       Class<T> metricClass, String name) {
553     if (!metricClass.isAssignableFrom(metric.getClass())) {
554       throw new MetricsException("Metric already exists in registry for metric name: " +
555               name + " and not of type " + metricClass +
556               " but instead of type " + metric.getClass());
557     }
558 
559     return (T) metric;
560   }
561 
562   public void clearMetrics() {
563     for (String name:metricsMap.keySet()) {
564       helper.removeObjectName(name);
565     }
566     metricsMap.clear();
567   }
568 }