View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.metrics2.lib;
20  
21  import java.util.Collection;
22  import java.util.concurrent.ConcurrentMap;
23  
24  import org.apache.commons.logging.Log;
25  import org.apache.commons.logging.LogFactory;
26  import org.apache.hadoop.hbase.classification.InterfaceAudience;
27  import org.apache.hadoop.metrics2.MetricsException;
28  import org.apache.hadoop.metrics2.MetricsInfo;
29  import org.apache.hadoop.metrics2.MetricsRecordBuilder;
30  import org.apache.hadoop.metrics2.MetricsTag;
31  import org.apache.hadoop.metrics2.impl.MsInfo;
32  
33  import com.google.common.base.Objects;
34  import com.google.common.collect.Maps;
35  
36  /**
37   * An optional metrics registry class for creating and maintaining a
38   * collection of MetricsMutables, making writing metrics source easier.
39   * NOTE: this is a copy of org.apache.hadoop.metrics2.lib.MetricsRegistry with added one
40   *       feature: metrics can be removed. When HADOOP-8313 is fixed, usages of this class
41   *       should be substituted with org.apache.hadoop.metrics2.lib.MetricsRegistry.
42   *       This implementation also provides handy methods for creating metrics
43   *       dynamically.
44   *       Another difference is that metricsMap implementation is substituted with
45   *       thread-safe map, as we allow dynamic metrics additions/removals.
46   */
47  @InterfaceAudience.Private
48  public class DynamicMetricsRegistry {
49    private static final Log LOG = LogFactory.getLog(DynamicMetricsRegistry.class);
50  
51    private final ConcurrentMap<String, MutableMetric> metricsMap =
52        Maps.newConcurrentMap();
53    private final ConcurrentMap<String, MetricsTag> tagsMap =
54        Maps.newConcurrentMap();
55    private final MetricsInfo metricsInfo;
56    private final DefaultMetricsSystemHelper helper = new DefaultMetricsSystemHelper();
57    private final static String[] histogramSuffixes = new String[]{
58        "_num_ops",
59        "_min",
60        "_max",
61        "_median",
62        "_75th_percentile",
63        "_90th_percentile",
64        "_95th_percentile",
65        "_99th_percentile"};
66  
67    /**
68     * Construct the registry with a record name
69     * @param name  of the record of the metrics
70     */
71    public DynamicMetricsRegistry(String name) {
72      this(Interns.info(name,name));
73    }
74  
75    /**
76     * Construct the registry with a metadata object
77     * @param info  the info object for the metrics record/group
78     */
79    public DynamicMetricsRegistry(MetricsInfo info) {
80      metricsInfo = info;
81    }
82  
83    /**
84     * @return the info object of the metrics registry
85     */
86    public MetricsInfo info() {
87      return metricsInfo;
88    }
89  
90    /**
91     * Get a metric by name
92     * @param name  of the metric
93     * @return the metric object
94     */
95    public MutableMetric get(String name) {
96      return metricsMap.get(name);
97    }
98  
99    /**
100    * Get a tag by name
101    * @param name  of the tag
102    * @return the tag object
103    */
104   public MetricsTag getTag(String name) {
105     return tagsMap.get(name);
106   }
107 
108   /**
109    * Create a mutable long integer counter
110    * @param name  of the metric
111    * @param desc  metric description
112    * @param iVal  initial value
113    * @return a new counter object
114    */
115   public MutableFastCounter newCounter(String name, String desc, long iVal) {
116     return newCounter(new MetricsInfoImpl(name, desc), iVal);
117   }
118 
119   /**
120    * Create a mutable long integer counter
121    * @param info  metadata of the metric
122    * @param iVal  initial value
123    * @return a new counter object
124    */
125   public MutableFastCounter newCounter(MetricsInfo info, long iVal) {
126     MutableFastCounter ret = new MutableFastCounter(info, iVal);
127     return addNewMetricIfAbsent(info.name(), ret, MutableFastCounter.class);
128   }
129 
130   /**
131    * Create a mutable long integer gauge
132    * @param name  of the metric
133    * @param desc  metric description
134    * @param iVal  initial value
135    * @return a new gauge object
136    */
137   public MutableGaugeLong newGauge(String name, String desc, long iVal) {
138     return newGauge(new MetricsInfoImpl(name, desc), iVal);
139   }
140 
141   /**
142    * Create a mutable long integer gauge
143    * @param info  metadata of the metric
144    * @param iVal  initial value
145    * @return a new gauge object
146    */
147   public MutableGaugeLong newGauge(MetricsInfo info, long iVal) {
148     MutableGaugeLong ret = new MutableGaugeLong(info, iVal);
149     return addNewMetricIfAbsent(info.name(), ret, MutableGaugeLong.class);
150   }
151 
152   /**
153    * Create a mutable metric with stats
154    * @param name  of the metric
155    * @param desc  metric description
156    * @param sampleName  of the metric (e.g., "Ops")
157    * @param valueName   of the metric (e.g., "Time" or "Latency")
158    * @param extended    produce extended stat (stdev, min/max etc.) if true.
159    * @return a new mutable stat metric object
160    */
161   public MutableStat newStat(String name, String desc,
162       String sampleName, String valueName, boolean extended) {
163     MutableStat ret =
164         new MutableStat(name, desc, sampleName, valueName, extended);
165     return addNewMetricIfAbsent(name, ret, MutableStat.class);
166   }
167 
168   /**
169    * Create a mutable metric with stats
170    * @param name  of the metric
171    * @param desc  metric description
172    * @param sampleName  of the metric (e.g., "Ops")
173    * @param valueName   of the metric (e.g., "Time" or "Latency")
174    * @return a new mutable metric object
175    */
176   public MutableStat newStat(String name, String desc,
177                              String sampleName, String valueName) {
178     return newStat(name, desc, sampleName, valueName, false);
179   }
180 
181   /**
182    * Create a mutable rate metric
183    * @param name  of the metric
184    * @return a new mutable metric object
185    */
186   public MutableRate newRate(String name) {
187     return newRate(name, name, false);
188   }
189 
190   /**
191    * Create a mutable rate metric
192    * @param name  of the metric
193    * @param description of the metric
194    * @return a new mutable rate metric object
195    */
196   public MutableRate newRate(String name, String description) {
197     return newRate(name, description, false);
198   }
199 
200   /**
201    * Create a mutable rate metric (for throughput measurement)
202    * @param name  of the metric
203    * @param desc  description
204    * @param extended  produce extended stat (stdev/min/max etc.) if true
205    * @return a new mutable rate metric object
206    */
207   public MutableRate newRate(String name, String desc, boolean extended) {
208     return newRate(name, desc, extended, true);
209   }
210 
211   @InterfaceAudience.Private
212   public MutableRate newRate(String name, String desc,
213       boolean extended, boolean returnExisting) {
214     if (returnExisting) {
215       MutableMetric rate = metricsMap.get(name);
216       if (rate != null) {
217         if (rate instanceof MutableRate) return (MutableRate) rate;
218         throw new MetricsException("Unexpected metrics type "+ rate.getClass()
219                                    +" for "+ name);
220       }
221     }
222     MutableRate ret = new MutableRate(name, desc, extended);
223     return addNewMetricIfAbsent(name, ret, MutableRate.class);
224   }
225 
226   /**
227    * Create a new histogram.
228    * @param name Name of the histogram.
229    * @return A new MutableHistogram
230    */
231   public MutableHistogram newHistogram(String name) {
232      return newHistogram(name, "");
233   }
234 
235   /**
236    * Create a new histogram.
237    * @param name The name of the histogram
238    * @param desc The description of the data in the histogram.
239    * @return A new MutableHistogram
240    */
241   public MutableHistogram newHistogram(String name, String desc) {
242     MutableHistogram histo = new MutableHistogram(name, desc);
243     return addNewMetricIfAbsent(name, histo, MutableHistogram.class);
244   }
245   
246   /**
247    * Create a new histogram with time range counts.
248    * @param name Name of the histogram.
249    * @return A new MutableTimeHistogram
250    */
251   public MutableTimeHistogram newTimeHistogram(String name) {
252      return newTimeHistogram(name, "");
253   }
254 
255   /**
256    * Create a new histogram with time range counts.
257    * @param name The name of the histogram
258    * @param desc The description of the data in the histogram.
259    * @return A new MutableTimeHistogram
260    */
261   public MutableTimeHistogram newTimeHistogram(String name, String desc) {
262     MutableTimeHistogram histo = new MutableTimeHistogram(name, desc);
263     return addNewMetricIfAbsent(name, histo, MutableTimeHistogram.class);
264   }
265   
266   /**
267    * Create a new histogram with size range counts.
268    * @param name Name of the histogram.
269    * @return A new MutableSizeHistogram
270    */
271   public MutableSizeHistogram newSizeHistogram(String name) {
272      return newSizeHistogram(name, "");
273   }
274 
275   /**
276    * Create a new histogram with size range counts.
277    * @param name The name of the histogram
278    * @param desc The description of the data in the histogram.
279    * @return A new MutableSizeHistogram
280    */
281   public MutableSizeHistogram newSizeHistogram(String name, String desc) {
282     MutableSizeHistogram histo = new MutableSizeHistogram(name, desc);
283     return addNewMetricIfAbsent(name, histo, MutableSizeHistogram.class);
284   }
285 
286 
287   synchronized void add(String name, MutableMetric metric) {
288     addNewMetricIfAbsent(name, metric, MutableMetric.class);
289   }
290 
291   /**
292    * Add sample to a stat metric by name.
293    * @param name  of the metric
294    * @param value of the snapshot to add
295    */
296   public void add(String name, long value) {
297     MutableMetric m = metricsMap.get(name);
298 
299     if (m != null) {
300       if (m instanceof MutableStat) {
301         ((MutableStat) m).add(value);
302       }
303       else {
304         throw new MetricsException("Unsupported add(value) for metric "+ name);
305       }
306     }
307     else {
308       metricsMap.put(name, newRate(name)); // default is a rate metric
309       add(name, value);
310     }
311   }
312 
313   /**
314    * Set the metrics context tag
315    * @param name of the context
316    * @return the registry itself as a convenience
317    */
318   public DynamicMetricsRegistry setContext(String name) {
319     return tag(MsInfo.Context, name, true);
320   }
321 
322   /**
323    * Add a tag to the metrics
324    * @param name  of the tag
325    * @param description of the tag
326    * @param value of the tag
327    * @return the registry (for keep adding tags)
328    */
329   public DynamicMetricsRegistry tag(String name, String description, String value) {
330     return tag(name, description, value, false);
331   }
332 
333   /**
334    * Add a tag to the metrics
335    * @param name  of the tag
336    * @param description of the tag
337    * @param value of the tag
338    * @param override  existing tag if true
339    * @return the registry (for keep adding tags)
340    */
341   public DynamicMetricsRegistry tag(String name, String description, String value,
342                              boolean override) {
343     return tag(new MetricsInfoImpl(name, description), value, override);
344   }
345 
346   /**
347    * Add a tag to the metrics
348    * @param info  metadata of the tag
349    * @param value of the tag
350    * @param override existing tag if true
351    * @return the registry (for keep adding tags etc.)
352    */
353   public DynamicMetricsRegistry tag(MetricsInfo info, String value, boolean override) {
354     MetricsTag tag = Interns.tag(info, value);
355 
356     if (!override) {
357       MetricsTag existing = tagsMap.putIfAbsent(info.name(), tag);
358       if (existing != null) {
359         throw new MetricsException("Tag "+ info.name() +" already exists!");
360       }
361       return this;
362     }
363 
364     tagsMap.put(info.name(), tag);
365 
366     return this;
367   }
368 
369   public DynamicMetricsRegistry tag(MetricsInfo info, String value) {
370     return tag(info, value, false);
371   }
372 
373   Collection<MetricsTag> tags() {
374     return tagsMap.values();
375   }
376 
377   Collection<MutableMetric> metrics() {
378     return metricsMap.values();
379   }
380 
381   /**
382    * Sample all the mutable metrics and put the snapshot in the builder
383    * @param builder to contain the metrics snapshot
384    * @param all get all the metrics even if the values are not changed.
385    */
386   public void snapshot(MetricsRecordBuilder builder, boolean all) {
387     for (MetricsTag tag : tags()) {
388       builder.add(tag);
389     }
390     for (MutableMetric metric : metrics()) {
391       metric.snapshot(builder, all);
392     }
393   }
394 
395   @Override public String toString() {
396     return Objects.toStringHelper(this)
397         .add("info", metricsInfo).add("tags", tags()).add("metrics", metrics())
398         .toString();
399   }
400 
401   /**
402    * Removes metric by name
403    * @param name name of the metric to remove
404    */
405   public void removeMetric(String name) {
406     helper.removeObjectName(name);
407     metricsMap.remove(name);
408   }
409 
410   public void removeHistogramMetrics(String baseName) {
411     for (String suffix:histogramSuffixes) {
412       removeMetric(baseName+suffix);
413     }
414   }
415 
416   /**
417    * Get a MetricMutableGaugeLong from the storage.  If it is not there atomically put it.
418    *
419    * @param gaugeName              name of the gauge to create or get.
420    * @param potentialStartingValue value of the new gauge if we have to create it.
421    */
422   public MutableGaugeLong getGauge(String gaugeName, long potentialStartingValue) {
423     //Try and get the guage.
424     MutableMetric metric = metricsMap.get(gaugeName);
425 
426     //If it's not there then try and put a new one in the storage.
427     if (metric == null) {
428 
429       //Create the potential new gauge.
430       MutableGaugeLong newGauge = new MutableGaugeLong(new MetricsInfoImpl(gaugeName, ""),
431               potentialStartingValue);
432 
433       // Try and put the gauge in.  This is atomic.
434       metric = metricsMap.putIfAbsent(gaugeName, newGauge);
435 
436       //If the value we get back is null then the put was successful and we will return that.
437       //otherwise gaugeLong should contain the thing that was in before the put could be completed.
438       if (metric == null) {
439         return newGauge;
440       }
441     }
442 
443     if (!(metric instanceof MutableGaugeLong)) {
444       throw new MetricsException("Metric already exists in registry for metric name: " + gaugeName +
445               " and not of type MetricMutableGaugeLong");
446     }
447 
448     return (MutableGaugeLong) metric;
449   }
450 
451   /**
452    * Get a MetricMutableCounterLong from the storage.  If it is not there atomically put it.
453    *
454    * @param counterName            Name of the counter to get
455    * @param potentialStartingValue starting value if we have to create a new counter
456    */
457   public MutableFastCounter getCounter(String counterName, long potentialStartingValue) {
458     //See getGauge for description on how this works.
459     MutableMetric counter = metricsMap.get(counterName);
460     if (counter == null) {
461       MutableFastCounter newCounter =
462               new MutableFastCounter(new MetricsInfoImpl(counterName, ""), potentialStartingValue);
463       counter = metricsMap.putIfAbsent(counterName, newCounter);
464       if (counter == null) {
465         return newCounter;
466       }
467     }
468 
469 
470     if (!(counter instanceof MutableCounter)) {
471       throw new MetricsException("Metric already exists in registry for metric name: " +
472               counterName + " and not of type MutableCounter");
473     }
474 
475     return (MutableFastCounter) counter;
476   }
477 
478   public MutableHistogram getHistogram(String histoName) {
479     //See getGauge for description on how this works.
480     MutableMetric histo = metricsMap.get(histoName);
481     if (histo == null) {
482       MutableHistogram newCounter =
483           new MutableHistogram(new MetricsInfoImpl(histoName, ""));
484       histo = metricsMap.putIfAbsent(histoName, newCounter);
485       if (histo == null) {
486         return newCounter;
487       }
488     }
489 
490 
491     if (!(histo instanceof MutableHistogram)) {
492       throw new MetricsException("Metric already exists in registry for metric name: " +
493           histoName + " and not of type MutableHistogram");
494     }
495 
496     return (MutableHistogram) histo;
497   }
498 
499   private<T extends MutableMetric> T
500   addNewMetricIfAbsent(String name,
501                        T ret,
502                        Class<T> metricClass) {
503     //If the value we get back is null then the put was successful and we will
504     // return that. Otherwise metric should contain the thing that was in
505     // before the put could be completed.
506     MutableMetric metric = metricsMap.putIfAbsent(name, ret);
507     if (metric == null) {
508       return ret;
509     }
510 
511     return returnExistingWithCast(metric, metricClass, name);
512   }
513 
514   @SuppressWarnings("unchecked")
515   private<T> T returnExistingWithCast(MutableMetric metric,
516                                       Class<T> metricClass, String name) {
517     if (!metricClass.isAssignableFrom(metric.getClass())) {
518       throw new MetricsException("Metric already exists in registry for metric name: " +
519               name + " and not of type " + metricClass +
520               " but instead of type " + metric.getClass());
521     }
522 
523     return (T) metric;
524   }
525 
526   public void clearMetrics() {
527     for (String name:metricsMap.keySet()) {
528       helper.removeObjectName(name);
529     }
530     metricsMap.clear();
531   }
532 }