View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.metrics2.lib;
20  
21  import java.util.Collection;
22  import java.util.concurrent.ConcurrentMap;
23  
24  import org.apache.commons.logging.Log;
25  import org.apache.commons.logging.LogFactory;
26  import org.apache.hadoop.hbase.classification.InterfaceAudience;
27  import org.apache.hadoop.hbase.metrics.Interns;
28  import org.apache.hadoop.metrics2.MetricsException;
29  import org.apache.hadoop.metrics2.MetricsInfo;
30  import org.apache.hadoop.metrics2.MetricsRecordBuilder;
31  import org.apache.hadoop.metrics2.MetricsTag;
32  import org.apache.hadoop.metrics2.impl.MsInfo;
33  
34  import com.google.common.base.Objects;
35  import com.google.common.collect.Maps;
36  
37  /**
38   * An optional metrics registry class for creating and maintaining a
39   * collection of MetricsMutables, making writing metrics source easier.
40   * NOTE: this is a copy of org.apache.hadoop.metrics2.lib.MetricsRegistry with added one
41   *       feature: metrics can be removed. When HADOOP-8313 is fixed, usages of this class
42   *       should be substituted with org.apache.hadoop.metrics2.lib.MetricsRegistry.
43   *       This implementation also provides handy methods for creating metrics
44   *       dynamically.
45   *       Another difference is that metricsMap implementation is substituted with
46   *       thread-safe map, as we allow dynamic metrics additions/removals.
47   */
48  @InterfaceAudience.Private
49  public class DynamicMetricsRegistry {
50    private static final Log LOG = LogFactory.getLog(DynamicMetricsRegistry.class);
51  
52    private final ConcurrentMap<String, MutableMetric> metricsMap =
53        Maps.newConcurrentMap();
54    private final ConcurrentMap<String, MetricsTag> tagsMap =
55        Maps.newConcurrentMap();
56    private final MetricsInfo metricsInfo;
57    private final DefaultMetricsSystemHelper helper = new DefaultMetricsSystemHelper();
58    private final static String[] histogramSuffixes = new String[]{
59        "_num_ops",
60        "_min",
61        "_max",
62        "_median",
63        "_75th_percentile",
64        "_90th_percentile",
65        "_95th_percentile",
66        "_99th_percentile"};
67  
68    /**
69     * Construct the registry with a record name
70     * @param name  of the record of the metrics
71     */
72    public DynamicMetricsRegistry(String name) {
73      this(Interns.info(name,name));
74    }
75  
76    /**
77     * Construct the registry with a metadata object
78     * @param info  the info object for the metrics record/group
79     */
80    public DynamicMetricsRegistry(MetricsInfo info) {
81      metricsInfo = info;
82    }
83  
84    /**
85     * @return the info object of the metrics registry
86     */
87    public MetricsInfo info() {
88      return metricsInfo;
89    }
90  
91    /**
92     * Get a metric by name
93     * @param name  of the metric
94     * @return the metric object
95     */
96    public MutableMetric get(String name) {
97      return metricsMap.get(name);
98    }
99  
100   /**
101    * Get a tag by name
102    * @param name  of the tag
103    * @return the tag object
104    */
105   public MetricsTag getTag(String name) {
106     return tagsMap.get(name);
107   }
108 
109   /**
110    * Create a mutable integer counter
111    * @param name  of the metric
112    * @param desc  metric description
113    * @param iVal  initial value
114    * @return a new counter object
115    */
116   public MutableCounterInt newCounter(String name, String desc, int iVal) {
117     return newCounter(new MetricsInfoImpl(name, desc), iVal);
118   }
119 
120   /**
121    * Create a mutable integer counter
122    * @param info  metadata of the metric
123    * @param iVal  initial value
124    * @return a new counter object
125    */
126   public MutableCounterInt newCounter(MetricsInfo info, int iVal) {
127     MutableCounterInt ret = new MutableCounterInt(info, iVal);
128     return addNewMetricIfAbsent(info.name(), ret, MutableCounterInt.class);
129   }
130 
131   /**
132    * Create a mutable long integer counter
133    * @param name  of the metric
134    * @param desc  metric description
135    * @param iVal  initial value
136    * @return a new counter object
137    */
138   public MutableCounterLong newCounter(String name, String desc, long iVal) {
139     return newCounter(new MetricsInfoImpl(name, desc), iVal);
140   }
141 
142   /**
143    * Create a mutable long integer counter
144    * @param info  metadata of the metric
145    * @param iVal  initial value
146    * @return a new counter object
147    */
148   public MutableCounterLong newCounter(MetricsInfo info, long iVal) {
149     MutableCounterLong ret = new MutableCounterLong(info, iVal);
150     return addNewMetricIfAbsent(info.name(), ret, MutableCounterLong.class);
151   }
152 
153   /**
154    * Create a mutable integer gauge
155    * @param name  of the metric
156    * @param desc  metric description
157    * @param iVal  initial value
158    * @return a new gauge object
159    */
160   public MutableGaugeInt newGauge(String name, String desc, int iVal) {
161     return newGauge(new MetricsInfoImpl(name, desc), iVal);
162   }
163   /**
164    * Create a mutable integer gauge
165    * @param info  metadata of the metric
166    * @param iVal  initial value
167    * @return a new gauge object
168    */
169   public MutableGaugeInt newGauge(MetricsInfo info, int iVal) {
170     MutableGaugeInt ret = new MutableGaugeInt(info, iVal);
171     return addNewMetricIfAbsent(info.name(), ret, MutableGaugeInt.class);
172   }
173 
174   /**
175    * Create a mutable long integer gauge
176    * @param name  of the metric
177    * @param desc  metric description
178    * @param iVal  initial value
179    * @return a new gauge object
180    */
181   public MutableGaugeLong newGauge(String name, String desc, long iVal) {
182     return newGauge(new MetricsInfoImpl(name, desc), iVal);
183   }
184 
185   /**
186    * Create a mutable long integer gauge
187    * @param info  metadata of the metric
188    * @param iVal  initial value
189    * @return a new gauge object
190    */
191   public MutableGaugeLong newGauge(MetricsInfo info, long iVal) {
192     MutableGaugeLong ret = new MutableGaugeLong(info, iVal);
193     return addNewMetricIfAbsent(info.name(), ret, MutableGaugeLong.class);
194   }
195 
196   /**
197    * Create a mutable metric with stats
198    * @param name  of the metric
199    * @param desc  metric description
200    * @param sampleName  of the metric (e.g., "Ops")
201    * @param valueName   of the metric (e.g., "Time" or "Latency")
202    * @param extended    produce extended stat (stdev, min/max etc.) if true.
203    * @return a new mutable stat metric object
204    */
205   public MutableStat newStat(String name, String desc,
206       String sampleName, String valueName, boolean extended) {
207     MutableStat ret =
208         new MutableStat(name, desc, sampleName, valueName, extended);
209     return addNewMetricIfAbsent(name, ret, MutableStat.class);
210   }
211 
212   /**
213    * Create a mutable metric with stats
214    * @param name  of the metric
215    * @param desc  metric description
216    * @param sampleName  of the metric (e.g., "Ops")
217    * @param valueName   of the metric (e.g., "Time" or "Latency")
218    * @return a new mutable metric object
219    */
220   public MutableStat newStat(String name, String desc,
221                              String sampleName, String valueName) {
222     return newStat(name, desc, sampleName, valueName, false);
223   }
224 
225   /**
226    * Create a mutable rate metric
227    * @param name  of the metric
228    * @return a new mutable metric object
229    */
230   public MutableRate newRate(String name) {
231     return newRate(name, name, false);
232   }
233 
234   /**
235    * Create a mutable rate metric
236    * @param name  of the metric
237    * @param description of the metric
238    * @return a new mutable rate metric object
239    */
240   public MutableRate newRate(String name, String description) {
241     return newRate(name, description, false);
242   }
243 
244   /**
245    * Create a mutable rate metric (for throughput measurement)
246    * @param name  of the metric
247    * @param desc  description
248    * @param extended  produce extended stat (stdev/min/max etc.) if true
249    * @return a new mutable rate metric object
250    */
251   public MutableRate newRate(String name, String desc, boolean extended) {
252     return newRate(name, desc, extended, true);
253   }
254 
255   @InterfaceAudience.Private
256   public MutableRate newRate(String name, String desc,
257       boolean extended, boolean returnExisting) {
258     if (returnExisting) {
259       MutableMetric rate = metricsMap.get(name);
260       if (rate != null) {
261         if (rate instanceof MutableRate) return (MutableRate) rate;
262         throw new MetricsException("Unexpected metrics type "+ rate.getClass()
263                                    +" for "+ name);
264       }
265     }
266     MutableRate ret = new MutableRate(name, desc, extended);
267     return addNewMetricIfAbsent(name, ret, MutableRate.class);
268   }
269 
270   /**
271    * Create a new histogram.
272    * @param name Name of the histogram.
273    * @return A new MutableHistogram
274    */
275   public MutableHistogram newHistogram(String name) {
276      return newHistogram(name, "");
277   }
278 
279   /**
280    * Create a new histogram.
281    * @param name The name of the histogram
282    * @param desc The description of the data in the histogram.
283    * @return A new MutableHistogram
284    */
285   public MutableHistogram newHistogram(String name, String desc) {
286     MutableHistogram histo = new MutableHistogram(name, desc);
287     return addNewMetricIfAbsent(name, histo, MutableHistogram.class);
288   }
289   
290   /**
291    * Create a new histogram with time range counts.
292    * @param name Name of the histogram.
293    * @return A new MutableTimeHistogram
294    */
295   public MutableTimeHistogram newTimeHistogram(String name) {
296      return newTimeHistogram(name, "");
297   }
298 
299   /**
300    * Create a new histogram with time range counts.
301    * @param name The name of the histogram
302    * @param desc The description of the data in the histogram.
303    * @return A new MutableTimeHistogram
304    */
305   public MutableTimeHistogram newTimeHistogram(String name, String desc) {
306     MutableTimeHistogram histo = new MutableTimeHistogram(name, desc);
307     return addNewMetricIfAbsent(name, histo, MutableTimeHistogram.class);
308   }
309   
310   /**
311    * Create a new histogram with size range counts.
312    * @param name Name of the histogram.
313    * @return A new MutableSizeHistogram
314    */
315   public MutableSizeHistogram newSizeHistogram(String name) {
316      return newSizeHistogram(name, "");
317   }
318 
319   /**
320    * Create a new histogram with size range counts.
321    * @param name The name of the histogram
322    * @param desc The description of the data in the histogram.
323    * @return A new MutableSizeHistogram
324    */
325   public MutableSizeHistogram newSizeHistogram(String name, String desc) {
326     MutableSizeHistogram histo = new MutableSizeHistogram(name, desc);
327     return addNewMetricIfAbsent(name, histo, MutableSizeHistogram.class);
328   }
329 
330   /**
331    * Create a new MutableQuantile(A more accurate histogram).
332    * @param name The name of the histogram
333    * @return a new MutableQuantile
334    */
335   public MetricMutableQuantiles newQuantile(String name) {
336     return newQuantile(name, "");
337   }
338 
339   public MetricMutableQuantiles newQuantile(String name, String desc) {
340     MetricMutableQuantiles histo = new MetricMutableQuantiles(name, desc, "Ops", "", 60);
341     return addNewMetricIfAbsent(name, histo, MetricMutableQuantiles.class);
342   }
343 
344   synchronized void add(String name, MutableMetric metric) {
345     addNewMetricIfAbsent(name, metric, MutableMetric.class);
346   }
347 
348   /**
349    * Add sample to a stat metric by name.
350    * @param name  of the metric
351    * @param value of the snapshot to add
352    */
353   public void add(String name, long value) {
354     MutableMetric m = metricsMap.get(name);
355 
356     if (m != null) {
357       if (m instanceof MutableStat) {
358         ((MutableStat) m).add(value);
359       }
360       else {
361         throw new MetricsException("Unsupported add(value) for metric "+ name);
362       }
363     }
364     else {
365       metricsMap.put(name, newRate(name)); // default is a rate metric
366       add(name, value);
367     }
368   }
369 
370   /**
371    * Set the metrics context tag
372    * @param name of the context
373    * @return the registry itself as a convenience
374    */
375   public DynamicMetricsRegistry setContext(String name) {
376     return tag(MsInfo.Context, name, true);
377   }
378 
379   /**
380    * Add a tag to the metrics
381    * @param name  of the tag
382    * @param description of the tag
383    * @param value of the tag
384    * @return the registry (for keep adding tags)
385    */
386   public DynamicMetricsRegistry tag(String name, String description, String value) {
387     return tag(name, description, value, false);
388   }
389 
390   /**
391    * Add a tag to the metrics
392    * @param name  of the tag
393    * @param description of the tag
394    * @param value of the tag
395    * @param override  existing tag if true
396    * @return the registry (for keep adding tags)
397    */
398   public DynamicMetricsRegistry tag(String name, String description, String value,
399                              boolean override) {
400     return tag(new MetricsInfoImpl(name, description), value, override);
401   }
402 
403   /**
404    * Add a tag to the metrics
405    * @param info  metadata of the tag
406    * @param value of the tag
407    * @param override existing tag if true
408    * @return the registry (for keep adding tags etc.)
409    */
410   public DynamicMetricsRegistry tag(MetricsInfo info, String value, boolean override) {
411     MetricsTag tag = Interns.tag(info, value);
412 
413     if (!override) {
414       MetricsTag existing = tagsMap.putIfAbsent(info.name(), tag);
415       if (existing != null) {
416         throw new MetricsException("Tag "+ info.name() +" already exists!");
417       }
418       return this;
419     }
420 
421     tagsMap.put(info.name(), tag);
422 
423     return this;
424   }
425 
426   public DynamicMetricsRegistry tag(MetricsInfo info, String value) {
427     return tag(info, value, false);
428   }
429 
430   Collection<MetricsTag> tags() {
431     return tagsMap.values();
432   }
433 
434   Collection<MutableMetric> metrics() {
435     return metricsMap.values();
436   }
437 
438   /**
439    * Sample all the mutable metrics and put the snapshot in the builder
440    * @param builder to contain the metrics snapshot
441    * @param all get all the metrics even if the values are not changed.
442    */
443   public void snapshot(MetricsRecordBuilder builder, boolean all) {
444     for (MetricsTag tag : tags()) {
445       builder.add(tag);
446     }
447     for (MutableMetric metric : metrics()) {
448       metric.snapshot(builder, all);
449     }
450   }
451 
452   @Override public String toString() {
453     return Objects.toStringHelper(this)
454         .add("info", metricsInfo).add("tags", tags()).add("metrics", metrics())
455         .toString();
456   }
457 
458   /**
459    * Removes metric by name
460    * @param name name of the metric to remove
461    */
462   public void removeMetric(String name) {
463     helper.removeObjectName(name);
464     metricsMap.remove(name);
465   }
466 
467   public void removeHistogramMetrics(String baseName) {
468     for (String suffix:histogramSuffixes) {
469       removeMetric(baseName+suffix);
470     }
471   }
472 
473   /**
474    * Get a MetricMutableGaugeLong from the storage.  If it is not there atomically put it.
475    *
476    * @param gaugeName              name of the gauge to create or get.
477    * @param potentialStartingValue value of the new gauge if we have to create it.
478    */
479   public MutableGaugeLong getLongGauge(String gaugeName, long potentialStartingValue) {
480     //Try and get the guage.
481     MutableMetric metric = metricsMap.get(gaugeName);
482 
483     //If it's not there then try and put a new one in the storage.
484     if (metric == null) {
485 
486       //Create the potential new gauge.
487       MutableGaugeLong newGauge = new MutableGaugeLong(new MetricsInfoImpl(gaugeName, ""),
488               potentialStartingValue);
489 
490       // Try and put the gauge in.  This is atomic.
491       metric = metricsMap.putIfAbsent(gaugeName, newGauge);
492 
493       //If the value we get back is null then the put was successful and we will return that.
494       //otherwise gaugeLong should contain the thing that was in before the put could be completed.
495       if (metric == null) {
496         return newGauge;
497       }
498     }
499 
500     if (!(metric instanceof MutableGaugeLong)) {
501       throw new MetricsException("Metric already exists in registry for metric name: " + gaugeName +
502               " and not of type MetricMutableGaugeLong");
503     }
504 
505     return (MutableGaugeLong) metric;
506   }
507 
508   /**
509    * Get a MetricMutableCounterLong from the storage.  If it is not there atomically put it.
510    *
511    * @param counterName            Name of the counter to get
512    * @param potentialStartingValue starting value if we have to create a new counter
513    */
514   public MutableCounterLong getLongCounter(String counterName, long potentialStartingValue) {
515     //See getLongGauge for description on how this works.
516     MutableMetric counter = metricsMap.get(counterName);
517     if (counter == null) {
518       MutableCounterLong newCounter =
519               new MutableCounterLong(new MetricsInfoImpl(counterName, ""), potentialStartingValue);
520       counter = metricsMap.putIfAbsent(counterName, newCounter);
521       if (counter == null) {
522         return newCounter;
523       }
524     }
525 
526 
527     if (!(counter instanceof MutableCounterLong)) {
528       throw new MetricsException("Metric already exists in registry for metric name: " +
529               counterName + " and not of type MetricMutableCounterLong");
530     }
531 
532     return (MutableCounterLong) counter;
533   }
534 
535   public MutableHistogram getHistogram(String histoName) {
536     //See getLongGauge for description on how this works.
537     MutableMetric histo = metricsMap.get(histoName);
538     if (histo == null) {
539       MutableHistogram newCounter =
540           new MutableHistogram(new MetricsInfoImpl(histoName, ""));
541       histo = metricsMap.putIfAbsent(histoName, newCounter);
542       if (histo == null) {
543         return newCounter;
544       }
545     }
546 
547 
548     if (!(histo instanceof MutableHistogram)) {
549       throw new MetricsException("Metric already exists in registry for metric name: " +
550           histoName + " and not of type MutableHistogram");
551     }
552 
553     return (MutableHistogram) histo;
554   }
555 
556   public MetricMutableQuantiles getQuantile(String histoName) {
557     //See getLongGauge for description on how this works.
558     MutableMetric histo = metricsMap.get(histoName);
559     if (histo == null) {
560       MetricMutableQuantiles newCounter =
561           new MetricMutableQuantiles(histoName, "", "Ops", "", 60);
562       histo = metricsMap.putIfAbsent(histoName, newCounter);
563       if (histo == null) {
564         return newCounter;
565       }
566     }
567 
568 
569     if (!(histo instanceof MetricMutableQuantiles)) {
570       throw new MetricsException("Metric already exists in registry for metric name: " +
571           histoName + " and not of type MutableHistogram");
572     }
573 
574     return (MetricMutableQuantiles) histo;
575   }
576 
577   private<T extends MutableMetric> T
578   addNewMetricIfAbsent(String name,
579                        T ret,
580                        Class<T> metricClass) {
581     //If the value we get back is null then the put was successful and we will
582     // return that. Otherwise metric should contain the thing that was in
583     // before the put could be completed.
584     MutableMetric metric = metricsMap.putIfAbsent(name, ret);
585     if (metric == null) {
586       return ret;
587     }
588 
589     return returnExistingWithCast(metric, metricClass, name);
590   }
591 
592   @SuppressWarnings("unchecked")
593   private<T> T returnExistingWithCast(MutableMetric metric,
594                                       Class<T> metricClass, String name) {
595     if (!metricClass.isAssignableFrom(metric.getClass())) {
596       throw new MetricsException("Metric already exists in registry for metric name: " +
597               name + " and not of type " + metricClass +
598               " but instead of type " + metric.getClass());
599     }
600 
601     return (T) metric;
602   }
603 
604   public void clearMetrics() {
605     for (String name:metricsMap.keySet()) {
606       helper.removeObjectName(name);
607     }
608     metricsMap.clear();
609   }
610 }