View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.metrics2.lib;
20  
21  import java.util.Collection;
22  import java.util.concurrent.ConcurrentMap;
23  
24  import org.apache.commons.logging.Log;
25  import org.apache.commons.logging.LogFactory;
26  import org.apache.hadoop.hbase.classification.InterfaceAudience;
27  import org.apache.hadoop.metrics2.MetricsException;
28  import org.apache.hadoop.metrics2.MetricsInfo;
29  import org.apache.hadoop.metrics2.MetricsRecordBuilder;
30  import org.apache.hadoop.metrics2.MetricsTag;
31  import org.apache.hadoop.metrics2.impl.MsInfo;
32  
33  import com.google.common.base.Objects;
34  import com.google.common.collect.Maps;
35  
36  /**
37   * An optional metrics registry class for creating and maintaining a
38   * collection of MetricsMutables, making writing metrics source easier.
39   * NOTE: this is a copy of org.apache.hadoop.metrics2.lib.MetricsRegistry with added one
40   *       feature: metrics can be removed. When HADOOP-8313 is fixed, usages of this class
41   *       should be substituted with org.apache.hadoop.metrics2.lib.MetricsRegistry.
42   *       This implementation also provides handy methods for creating metrics
43   *       dynamically.
44   *       Another difference is that metricsMap implementation is substituted with
45   *       thread-safe map, as we allow dynamic metrics additions/removals.
46   */
47  @InterfaceAudience.Private
48  public class DynamicMetricsRegistry {
49    private static final Log LOG = LogFactory.getLog(DynamicMetricsRegistry.class);
50  
51    private final ConcurrentMap<String, MutableMetric> metricsMap =
52        Maps.newConcurrentMap();
53    private final ConcurrentMap<String, MetricsTag> tagsMap =
54        Maps.newConcurrentMap();
55    private final MetricsInfo metricsInfo;
56    private final DefaultMetricsSystemHelper helper = new DefaultMetricsSystemHelper();
57    private final static String[] histogramSuffixes = new String[]{
58        "_num_ops",
59        "_min",
60        "_max",
61        "_median",
62        "_75th_percentile",
63        "_90th_percentile",
64        "_95th_percentile",
65        "_99th_percentile"};
66  
67    /**
68     * Construct the registry with a record name
69     * @param name  of the record of the metrics
70     */
71    public DynamicMetricsRegistry(String name) {
72      this(Interns.info(name,name));
73    }
74  
75    /**
76     * Construct the registry with a metadata object
77     * @param info  the info object for the metrics record/group
78     */
79    public DynamicMetricsRegistry(MetricsInfo info) {
80      metricsInfo = info;
81    }
82  
83    /**
84     * @return the info object of the metrics registry
85     */
86    public MetricsInfo info() {
87      return metricsInfo;
88    }
89  
90    /**
91     * Get a metric by name
92     * @param name  of the metric
93     * @return the metric object
94     */
95    public MutableMetric get(String name) {
96      return metricsMap.get(name);
97    }
98  
99    /**
100    * Get a tag by name
101    * @param name  of the tag
102    * @return the tag object
103    */
104   public MetricsTag getTag(String name) {
105     return tagsMap.get(name);
106   }
107 
108   /**
109    * Create a mutable integer counter
110    * @param name  of the metric
111    * @param desc  metric description
112    * @param iVal  initial value
113    * @return a new counter object
114    */
115   public MutableCounterInt newCounter(String name, String desc, int iVal) {
116     return newCounter(new MetricsInfoImpl(name, desc), iVal);
117   }
118 
119   /**
120    * Create a mutable integer counter
121    * @param info  metadata of the metric
122    * @param iVal  initial value
123    * @return a new counter object
124    */
125   public MutableCounterInt newCounter(MetricsInfo info, int iVal) {
126     MutableCounterInt ret = new MutableCounterInt(info, iVal);
127     return addNewMetricIfAbsent(info.name(), ret, MutableCounterInt.class);
128   }
129 
130   /**
131    * Create a mutable long integer counter
132    * @param name  of the metric
133    * @param desc  metric description
134    * @param iVal  initial value
135    * @return a new counter object
136    */
137   public MutableCounterLong newCounter(String name, String desc, long iVal) {
138     return newCounter(new MetricsInfoImpl(name, desc), iVal);
139   }
140 
141   /**
142    * Create a mutable long integer counter
143    * @param info  metadata of the metric
144    * @param iVal  initial value
145    * @return a new counter object
146    */
147   public MutableCounterLong newCounter(MetricsInfo info, long iVal) {
148     MutableCounterLong ret = new MutableCounterLong(info, iVal);
149     return addNewMetricIfAbsent(info.name(), ret, MutableCounterLong.class);
150   }
151 
152   /**
153    * Create a mutable integer gauge
154    * @param name  of the metric
155    * @param desc  metric description
156    * @param iVal  initial value
157    * @return a new gauge object
158    */
159   public MutableGaugeInt newGauge(String name, String desc, int iVal) {
160     return newGauge(new MetricsInfoImpl(name, desc), iVal);
161   }
162   /**
163    * Create a mutable integer gauge
164    * @param info  metadata of the metric
165    * @param iVal  initial value
166    * @return a new gauge object
167    */
168   public MutableGaugeInt newGauge(MetricsInfo info, int iVal) {
169     MutableGaugeInt ret = new MutableGaugeInt(info, iVal);
170     return addNewMetricIfAbsent(info.name(), ret, MutableGaugeInt.class);
171   }
172 
173   /**
174    * Create a mutable long integer gauge
175    * @param name  of the metric
176    * @param desc  metric description
177    * @param iVal  initial value
178    * @return a new gauge object
179    */
180   public MutableGaugeLong newGauge(String name, String desc, long iVal) {
181     return newGauge(new MetricsInfoImpl(name, desc), iVal);
182   }
183 
184   /**
185    * Create a mutable long integer gauge
186    * @param info  metadata of the metric
187    * @param iVal  initial value
188    * @return a new gauge object
189    */
190   public MutableGaugeLong newGauge(MetricsInfo info, long iVal) {
191     MutableGaugeLong ret = new MutableGaugeLong(info, iVal);
192     return addNewMetricIfAbsent(info.name(), ret, MutableGaugeLong.class);
193   }
194 
195   /**
196    * Create a mutable metric with stats
197    * @param name  of the metric
198    * @param desc  metric description
199    * @param sampleName  of the metric (e.g., "Ops")
200    * @param valueName   of the metric (e.g., "Time" or "Latency")
201    * @param extended    produce extended stat (stdev, min/max etc.) if true.
202    * @return a new mutable stat metric object
203    */
204   public MutableStat newStat(String name, String desc,
205       String sampleName, String valueName, boolean extended) {
206     MutableStat ret =
207         new MutableStat(name, desc, sampleName, valueName, extended);
208     return addNewMetricIfAbsent(name, ret, MutableStat.class);
209   }
210 
211   /**
212    * Create a mutable metric with stats
213    * @param name  of the metric
214    * @param desc  metric description
215    * @param sampleName  of the metric (e.g., "Ops")
216    * @param valueName   of the metric (e.g., "Time" or "Latency")
217    * @return a new mutable metric object
218    */
219   public MutableStat newStat(String name, String desc,
220                              String sampleName, String valueName) {
221     return newStat(name, desc, sampleName, valueName, false);
222   }
223 
224   /**
225    * Create a mutable rate metric
226    * @param name  of the metric
227    * @return a new mutable metric object
228    */
229   public MutableRate newRate(String name) {
230     return newRate(name, name, false);
231   }
232 
233   /**
234    * Create a mutable rate metric
235    * @param name  of the metric
236    * @param description of the metric
237    * @return a new mutable rate metric object
238    */
239   public MutableRate newRate(String name, String description) {
240     return newRate(name, description, false);
241   }
242 
243   /**
244    * Create a mutable rate metric (for throughput measurement)
245    * @param name  of the metric
246    * @param desc  description
247    * @param extended  produce extended stat (stdev/min/max etc.) if true
248    * @return a new mutable rate metric object
249    */
250   public MutableRate newRate(String name, String desc, boolean extended) {
251     return newRate(name, desc, extended, true);
252   }
253 
254   @InterfaceAudience.Private
255   public MutableRate newRate(String name, String desc,
256       boolean extended, boolean returnExisting) {
257     if (returnExisting) {
258       MutableMetric rate = metricsMap.get(name);
259       if (rate != null) {
260         if (rate instanceof MutableRate) return (MutableRate) rate;
261         throw new MetricsException("Unexpected metrics type "+ rate.getClass()
262                                    +" for "+ name);
263       }
264     }
265     MutableRate ret = new MutableRate(name, desc, extended);
266     return addNewMetricIfAbsent(name, ret, MutableRate.class);
267   }
268 
269   /**
270    * Create a new histogram.
271    * @param name Name of the histogram.
272    * @return A new MutableHistogram
273    */
274   public MutableHistogram newHistogram(String name) {
275      return newHistogram(name, "");
276   }
277 
278   /**
279    * Create a new histogram.
280    * @param name The name of the histogram
281    * @param desc The description of the data in the histogram.
282    * @return A new MutableHistogram
283    */
284   public MutableHistogram newHistogram(String name, String desc) {
285     MutableHistogram histo = new MutableHistogram(name, desc);
286     return addNewMetricIfAbsent(name, histo, MutableHistogram.class);
287   }
288   
289   /**
290    * Create a new histogram with time range counts.
291    * @param name Name of the histogram.
292    * @return A new MutableTimeHistogram
293    */
294   public MutableTimeHistogram newTimeHistogram(String name) {
295      return newTimeHistogram(name, "");
296   }
297 
298   /**
299    * Create a new histogram with time range counts.
300    * @param name The name of the histogram
301    * @param desc The description of the data in the histogram.
302    * @return A new MutableTimeHistogram
303    */
304   public MutableTimeHistogram newTimeHistogram(String name, String desc) {
305     MutableTimeHistogram histo = new MutableTimeHistogram(name, desc);
306     return addNewMetricIfAbsent(name, histo, MutableTimeHistogram.class);
307   }
308   
309   /**
310    * Create a new histogram with size range counts.
311    * @param name Name of the histogram.
312    * @return A new MutableSizeHistogram
313    */
314   public MutableSizeHistogram newSizeHistogram(String name) {
315      return newSizeHistogram(name, "");
316   }
317 
318   /**
319    * Create a new histogram with size range counts.
320    * @param name The name of the histogram
321    * @param desc The description of the data in the histogram.
322    * @return A new MutableSizeHistogram
323    */
324   public MutableSizeHistogram newSizeHistogram(String name, String desc) {
325     MutableSizeHistogram histo = new MutableSizeHistogram(name, desc);
326     return addNewMetricIfAbsent(name, histo, MutableSizeHistogram.class);
327   }
328 
329   /**
330    * Create a new MutableQuantile(A more accurate histogram).
331    * @param name The name of the histogram
332    * @return a new MutableQuantile
333    */
334   public MetricMutableQuantiles newQuantile(String name) {
335     return newQuantile(name, "");
336   }
337 
338   public MetricMutableQuantiles newQuantile(String name, String desc) {
339     MetricMutableQuantiles histo = new MetricMutableQuantiles(name, desc, "Ops", "", 60);
340     return addNewMetricIfAbsent(name, histo, MetricMutableQuantiles.class);
341   }
342 
343   synchronized void add(String name, MutableMetric metric) {
344     addNewMetricIfAbsent(name, metric, MutableMetric.class);
345   }
346 
347   /**
348    * Add sample to a stat metric by name.
349    * @param name  of the metric
350    * @param value of the snapshot to add
351    */
352   public void add(String name, long value) {
353     MutableMetric m = metricsMap.get(name);
354 
355     if (m != null) {
356       if (m instanceof MutableStat) {
357         ((MutableStat) m).add(value);
358       }
359       else {
360         throw new MetricsException("Unsupported add(value) for metric "+ name);
361       }
362     }
363     else {
364       metricsMap.put(name, newRate(name)); // default is a rate metric
365       add(name, value);
366     }
367   }
368 
369   /**
370    * Set the metrics context tag
371    * @param name of the context
372    * @return the registry itself as a convenience
373    */
374   public DynamicMetricsRegistry setContext(String name) {
375     return tag(MsInfo.Context, name, true);
376   }
377 
378   /**
379    * Add a tag to the metrics
380    * @param name  of the tag
381    * @param description of the tag
382    * @param value of the tag
383    * @return the registry (for keep adding tags)
384    */
385   public DynamicMetricsRegistry tag(String name, String description, String value) {
386     return tag(name, description, value, false);
387   }
388 
389   /**
390    * Add a tag to the metrics
391    * @param name  of the tag
392    * @param description of the tag
393    * @param value of the tag
394    * @param override  existing tag if true
395    * @return the registry (for keep adding tags)
396    */
397   public DynamicMetricsRegistry tag(String name, String description, String value,
398                              boolean override) {
399     return tag(new MetricsInfoImpl(name, description), value, override);
400   }
401 
402   /**
403    * Add a tag to the metrics
404    * @param info  metadata of the tag
405    * @param value of the tag
406    * @param override existing tag if true
407    * @return the registry (for keep adding tags etc.)
408    */
409   public DynamicMetricsRegistry tag(MetricsInfo info, String value, boolean override) {
410     MetricsTag tag = Interns.tag(info, value);
411 
412     if (!override) {
413       MetricsTag existing = tagsMap.putIfAbsent(info.name(), tag);
414       if (existing != null) {
415         throw new MetricsException("Tag "+ info.name() +" already exists!");
416       }
417       return this;
418     }
419 
420     tagsMap.put(info.name(), tag);
421 
422     return this;
423   }
424 
425   public DynamicMetricsRegistry tag(MetricsInfo info, String value) {
426     return tag(info, value, false);
427   }
428 
429   Collection<MetricsTag> tags() {
430     return tagsMap.values();
431   }
432 
433   Collection<MutableMetric> metrics() {
434     return metricsMap.values();
435   }
436 
437   /**
438    * Sample all the mutable metrics and put the snapshot in the builder
439    * @param builder to contain the metrics snapshot
440    * @param all get all the metrics even if the values are not changed.
441    */
442   public void snapshot(MetricsRecordBuilder builder, boolean all) {
443     for (MetricsTag tag : tags()) {
444       builder.add(tag);
445     }
446     for (MutableMetric metric : metrics()) {
447       metric.snapshot(builder, all);
448     }
449   }
450 
451   @Override public String toString() {
452     return Objects.toStringHelper(this)
453         .add("info", metricsInfo).add("tags", tags()).add("metrics", metrics())
454         .toString();
455   }
456 
457   /**
458    * Removes metric by name
459    * @param name name of the metric to remove
460    */
461   public void removeMetric(String name) {
462     helper.removeObjectName(name);
463     metricsMap.remove(name);
464   }
465 
466   public void removeHistogramMetrics(String baseName) {
467     for (String suffix:histogramSuffixes) {
468       removeMetric(baseName+suffix);
469     }
470   }
471 
472   /**
473    * Get a MetricMutableGaugeLong from the storage.  If it is not there atomically put it.
474    *
475    * @param gaugeName              name of the gauge to create or get.
476    * @param potentialStartingValue value of the new gauge if we have to create it.
477    */
478   public MutableGaugeLong getLongGauge(String gaugeName, long potentialStartingValue) {
479     //Try and get the guage.
480     MutableMetric metric = metricsMap.get(gaugeName);
481 
482     //If it's not there then try and put a new one in the storage.
483     if (metric == null) {
484 
485       //Create the potential new gauge.
486       MutableGaugeLong newGauge = new MutableGaugeLong(new MetricsInfoImpl(gaugeName, ""),
487               potentialStartingValue);
488 
489       // Try and put the gauge in.  This is atomic.
490       metric = metricsMap.putIfAbsent(gaugeName, newGauge);
491 
492       //If the value we get back is null then the put was successful and we will return that.
493       //otherwise gaugeLong should contain the thing that was in before the put could be completed.
494       if (metric == null) {
495         return newGauge;
496       }
497     }
498 
499     if (!(metric instanceof MutableGaugeLong)) {
500       throw new MetricsException("Metric already exists in registry for metric name: " + gaugeName +
501               " and not of type MetricMutableGaugeLong");
502     }
503 
504     return (MutableGaugeLong) metric;
505   }
506 
507   /**
508    * Get a MetricMutableCounterLong from the storage.  If it is not there atomically put it.
509    *
510    * @param counterName            Name of the counter to get
511    * @param potentialStartingValue starting value if we have to create a new counter
512    */
513   public MutableCounterLong getLongCounter(String counterName, long potentialStartingValue) {
514     //See getLongGauge for description on how this works.
515     MutableMetric counter = metricsMap.get(counterName);
516     if (counter == null) {
517       MutableCounterLong newCounter =
518               new MutableCounterLong(new MetricsInfoImpl(counterName, ""), potentialStartingValue);
519       counter = metricsMap.putIfAbsent(counterName, newCounter);
520       if (counter == null) {
521         return newCounter;
522       }
523     }
524 
525 
526     if (!(counter instanceof MutableCounterLong)) {
527       throw new MetricsException("Metric already exists in registry for metric name: " +
528               counterName + " and not of type MetricMutableCounterLong");
529     }
530 
531     return (MutableCounterLong) counter;
532   }
533 
534   public MutableHistogram getHistogram(String histoName) {
535     //See getLongGauge for description on how this works.
536     MutableMetric histo = metricsMap.get(histoName);
537     if (histo == null) {
538       MutableHistogram newCounter =
539           new MutableHistogram(new MetricsInfoImpl(histoName, ""));
540       histo = metricsMap.putIfAbsent(histoName, newCounter);
541       if (histo == null) {
542         return newCounter;
543       }
544     }
545 
546 
547     if (!(histo instanceof MutableHistogram)) {
548       throw new MetricsException("Metric already exists in registry for metric name: " +
549           histoName + " and not of type MutableHistogram");
550     }
551 
552     return (MutableHistogram) histo;
553   }
554 
555   public MetricMutableQuantiles getQuantile(String histoName) {
556     //See getLongGauge for description on how this works.
557     MutableMetric histo = metricsMap.get(histoName);
558     if (histo == null) {
559       MetricMutableQuantiles newCounter =
560           new MetricMutableQuantiles(histoName, "", "Ops", "", 60);
561       histo = metricsMap.putIfAbsent(histoName, newCounter);
562       if (histo == null) {
563         return newCounter;
564       }
565     }
566 
567 
568     if (!(histo instanceof MetricMutableQuantiles)) {
569       throw new MetricsException("Metric already exists in registry for metric name: " +
570           histoName + " and not of type MutableHistogram");
571     }
572 
573     return (MetricMutableQuantiles) histo;
574   }
575 
576   private<T extends MutableMetric> T
577   addNewMetricIfAbsent(String name,
578                        T ret,
579                        Class<T> metricClass) {
580     //If the value we get back is null then the put was successful and we will
581     // return that. Otherwise metric should contain the thing that was in
582     // before the put could be completed.
583     MutableMetric metric = metricsMap.putIfAbsent(name, ret);
584     if (metric == null) {
585       return ret;
586     }
587 
588     return returnExistingWithCast(metric, metricClass, name);
589   }
590 
591   @SuppressWarnings("unchecked")
592   private<T> T returnExistingWithCast(MutableMetric metric,
593                                       Class<T> metricClass, String name) {
594     if (!metricClass.isAssignableFrom(metric.getClass())) {
595       throw new MetricsException("Metric already exists in registry for metric name: " +
596               name + " and not of type " + metricClass +
597               " but instead of type " + metric.getClass());
598     }
599 
600     return (T) metric;
601   }
602 
603   public void clearMetrics() {
604     for (String name:metricsMap.keySet()) {
605       helper.removeObjectName(name);
606     }
607     metricsMap.clear();
608   }
609 }