View Javadoc

1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.regionserver.metrics;
21  
22  import java.io.IOException;
23  import java.lang.management.ManagementFactory;
24  import java.lang.management.MemoryUsage;
25  import java.util.List;
26  
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.hadoop.hbase.io.hfile.HFile;
30  import org.apache.hadoop.hbase.metrics.HBaseInfo;
31  import org.apache.hadoop.hbase.metrics.MetricsRate;
32  import org.apache.hadoop.hbase.metrics.PersistentMetricsTimeVaryingRate;
33  import org.apache.hadoop.hbase.metrics.histogram.MetricsHistogram;
34  import com.yammer.metrics.stats.Snapshot;
35  import org.apache.hadoop.hbase.regionserver.wal.HLog;
36  import org.apache.hadoop.hbase.util.Pair;
37  import org.apache.hadoop.hbase.util.Strings;
38  import org.apache.hadoop.metrics.ContextFactory;
39  import org.apache.hadoop.metrics.MetricsContext;
40  import org.apache.hadoop.metrics.MetricsRecord;
41  import org.apache.hadoop.metrics.MetricsUtil;
42  import org.apache.hadoop.metrics.Updater;
43  import org.apache.hadoop.metrics.jvm.JvmMetrics;
44  import org.apache.hadoop.metrics.util.MetricsIntValue;
45  import org.apache.hadoop.metrics.util.MetricsLongValue;
46  import org.apache.hadoop.metrics.util.MetricsRegistry;
47  import org.apache.hadoop.metrics.util.MetricsTimeVaryingRate;
48  import org.apache.hadoop.metrics.util.MetricsTimeVaryingLong;
49  import org.apache.hadoop.util.StringUtils;
50  
51  /**
52   * This class is for maintaining the various regionserver statistics
53   * and publishing them through the metrics interfaces.
54   * <p>
55   * This class has a number of metrics variables that are publicly accessible;
56   * these variables (objects) have methods to update their values.
57   */
58  public class RegionServerMetrics implements Updater {
59    @SuppressWarnings({"FieldCanBeLocal"})
60    private final Log LOG = LogFactory.getLog(this.getClass());
61    private final MetricsRecord metricsRecord;
62    private long lastUpdate = System.currentTimeMillis();
63    private long lastExtUpdate = System.currentTimeMillis();
64    private long extendedPeriod = 0;
65    private static final int MB = 1024*1024;
66    private MetricsRegistry registry = new MetricsRegistry();
67    private final RegionServerStatistics statistics;
68  
69    public final MetricsTimeVaryingRate atomicIncrementTime =
70        new MetricsTimeVaryingRate("atomicIncrementTime", registry);
71  
72    /**
73     * Count of regions carried by this regionserver
74     */
75    public final MetricsIntValue regions =
76      new MetricsIntValue("regions", registry);
77  
78    /**
79     * Block cache size.
80     */
81    public final MetricsLongValue blockCacheSize = 
82        new MetricsLongValue("blockCacheSize", registry);
83  
84    /**
85     * Block cache free size.
86     */
87    public final MetricsLongValue blockCacheFree = 
88        new MetricsLongValue("blockCacheFree", registry);
89  
90    /**
91     * Block cache item count.
92     */
93    public final MetricsLongValue blockCacheCount = 
94        new MetricsLongValue("blockCacheCount", registry);
95  
96    /**
97     * Block cache hit count.
98     */
99    public final MetricsLongValue blockCacheHitCount = 
100       new MetricsLongValue("blockCacheHitCount", registry);
101 
102   /**
103    * Block cache miss count.
104    */
105   public final MetricsLongValue blockCacheMissCount = 
106       new MetricsLongValue("blockCacheMissCount", registry);
107 
108   /**
109    * Block cache evict count.
110    */
111   public final MetricsLongValue blockCacheEvictedCount = 
112       new MetricsLongValue("blockCacheEvictedCount", registry);
113 
114   /**
115    * Block hit ratio.
116    */
117   public final MetricsIntValue blockCacheHitRatio = 
118       new MetricsIntValue("blockCacheHitRatio", registry);
119 
120   /**
121    * Block hit caching ratio.  This only includes the requests to the block
122    * cache where caching was turned on.  See HBASE-2253.
123    */
124   public final MetricsIntValue blockCacheHitCachingRatio = 
125       new MetricsIntValue("blockCacheHitCachingRatio", registry);
126 
127   /** Block hit ratio for past N periods. */
128   public final MetricsIntValue blockCacheHitRatioPastNPeriods = new MetricsIntValue("blockCacheHitRatioPastNPeriods", registry);
129 
130   /** Block hit caching ratio for past N periods */
131   public final MetricsIntValue blockCacheHitCachingRatioPastNPeriods = new MetricsIntValue("blockCacheHitCachingRatioPastNPeriods", registry);
132 
133   /*
134    * Count of requests to the regionservers since last call to metrics update
135    */
136   public final MetricsRate requests = new MetricsRate("requests", registry);
137 
138   /**
139    * Count of stores open on the regionserver.
140    */
141   public final MetricsIntValue stores = new MetricsIntValue("stores", registry);
142 
143   /**
144    * Count of storefiles open on the regionserver.
145    */
146   public final MetricsIntValue storefiles = 
147       new MetricsIntValue("storefiles", registry);
148 
149   /**
150    * Count of hlogfiles
151    */
152   public final MetricsIntValue hlogFileCount = 
153       new MetricsIntValue("hlogFileCount", registry);
154   
155   /**
156    * the total size of hlog files in MB
157    */
158   public final MetricsLongValue hlogFileSizeMB = 
159       new MetricsLongValue("hlogFileSizeMB", registry);
160   
161   /**
162    * Count of read requests
163    */
164   public final MetricsLongValue readRequestsCount = 
165       new MetricsLongValue("readRequestsCount", registry);
166 
167   /**
168    * Count of write requests
169    */
170   public final MetricsLongValue writeRequestsCount = 
171       new MetricsLongValue("writeRequestsCount", registry);
172 
173   /**
174    */
175   public final MetricsIntValue storefileIndexSizeMB =
176     new MetricsIntValue("storefileIndexSizeMB", registry);
177 
178   /** The total size of block index root levels in this regionserver in KB. */
179   public final MetricsIntValue rootIndexSizeKB =
180     new MetricsIntValue("rootIndexSizeKB", registry);
181 
182   /** Total size of all block indexes (not necessarily loaded in memory) */
183   public final MetricsIntValue totalStaticIndexSizeKB =
184     new MetricsIntValue("totalStaticIndexSizeKB", registry);
185 
186   /** Total size of all Bloom filters (not necessarily loaded in memory) */
187   public final MetricsIntValue totalStaticBloomSizeKB =
188     new MetricsIntValue("totalStaticBloomSizeKB", registry);
189 
190   /**
191    * HDFS blocks locality index
192    */
193   public final MetricsIntValue hdfsBlocksLocalityIndex =
194     new MetricsIntValue("hdfsBlocksLocalityIndex", registry);
195   
196   /**
197    * Sum of all the memstore sizes in this regionserver in MB
198    */
199   public final MetricsIntValue memstoreSizeMB =
200     new MetricsIntValue("memstoreSizeMB", registry);
201 
202   /**
203    * Number of put with WAL disabled in this regionserver in MB
204    */
205   public final MetricsLongValue numPutsWithoutWAL =
206     new MetricsLongValue("numPutsWithoutWAL", registry);
207 
208   /**
209    * Possible data loss sizes (due to put with WAL disabled) in this regionserver in MB
210    */
211   public final MetricsIntValue mbInMemoryWithoutWAL =
212     new MetricsIntValue("mbInMemoryWithoutWAL", registry);
213 
214   /**
215    * Size of the compaction queue.
216    */
217   public final MetricsIntValue compactionQueueSize =
218     new MetricsIntValue("compactionQueueSize", registry);
219   
220   /**
221    * Size of the flush queue.
222    */
223   public final MetricsIntValue flushQueueSize =
224     new MetricsIntValue("flushQueueSize", registry);
225 
226   /**
227    * filesystem sequential read latency distribution
228    */
229   public final MetricsHistogram fsReadLatencyHistogram = 
230       new MetricsHistogram("fsReadLatencyHistogram", registry);
231 
232   /**
233    * filesystem pread latency distribution
234    */
235   public final MetricsHistogram fsPreadLatencyHistogram = 
236       new MetricsHistogram("fsPreadLatencyHistogram", registry);
237 
238   /**
239    * Metrics on the distribution of filesystem write latencies (improved version of fsWriteLatency)
240    */
241   public final MetricsHistogram fsWriteLatencyHistogram = 
242       new MetricsHistogram("fsWriteLatencyHistogram", registry);
243 
244   
245   /**
246    * filesystem read latency
247    */
248   public final MetricsTimeVaryingRate fsReadLatency =
249     new MetricsTimeVaryingRate("fsReadLatency", registry);
250 
251   /**
252    * filesystem positional read latency
253    */
254   public final MetricsTimeVaryingRate fsPreadLatency =
255     new MetricsTimeVaryingRate("fsPreadLatency", registry);
256 
257   /**
258    * filesystem write latency
259    */
260   public final MetricsTimeVaryingRate fsWriteLatency =
261     new MetricsTimeVaryingRate("fsWriteLatency", registry);
262 
263   /**
264    * size (in bytes) of data in HLog append calls
265    */
266   public final MetricsTimeVaryingRate fsWriteSize =
267     new MetricsTimeVaryingRate("fsWriteSize", registry);
268 
269   /**
270    * filesystem sync latency
271    */
272   public final MetricsTimeVaryingRate fsSyncLatency =
273     new MetricsTimeVaryingRate("fsSyncLatency", registry);
274 
275   
276   /**
277    * time each scheduled compaction takes
278    */
279   protected final PersistentMetricsTimeVaryingRate compactionTime =
280     new PersistentMetricsTimeVaryingRate("compactionTime", registry);
281 
282   protected final PersistentMetricsTimeVaryingRate compactionSize =
283     new PersistentMetricsTimeVaryingRate("compactionSize", registry);
284 
285   /**
286    * time each scheduled flush takes
287    */
288   protected final PersistentMetricsTimeVaryingRate flushTime =
289     new PersistentMetricsTimeVaryingRate("flushTime", registry);
290 
291   protected final PersistentMetricsTimeVaryingRate flushSize =
292     new PersistentMetricsTimeVaryingRate("flushSize", registry);
293   
294   public final MetricsLongValue slowHLogAppendCount =
295       new MetricsLongValue("slowHLogAppendCount", registry);
296   
297   public final MetricsTimeVaryingRate slowHLogAppendTime =
298       new MetricsTimeVaryingRate("slowHLogAppendTime", registry);
299   
300   public final MetricsTimeVaryingLong regionSplitSuccessCount =
301       new MetricsTimeVaryingLong("regionSplitSuccessCount", registry);
302   
303   public final MetricsTimeVaryingLong regionSplitFailureCount =
304       new MetricsTimeVaryingLong("regionSplitFailureCount", registry);
305 
306   /**
307    * Number of times checksum verification failed.
308    */
309   public final MetricsLongValue checksumFailuresCount =
310     new MetricsLongValue("checksumFailuresCount", registry);
311 
312   /**
313    * time blocked on lack of resources
314    */
315   public final MetricsLongValue updatesBlockedSeconds = new MetricsLongValue(
316       "updatesBlockedSeconds", registry);
317 
318   /**
319    * time blocked on memstoreHW
320    */
321   public final MetricsLongValue updatesBlockedSecondsHighWater = new MetricsLongValue(
322       "updatesBlockedSecondsHighWater",registry);
323 
324   public RegionServerMetrics() {
325     MetricsContext context = MetricsUtil.getContext("hbase");
326     metricsRecord = MetricsUtil.createRecord(context, "regionserver");
327     String name = Thread.currentThread().getName();
328     metricsRecord.setTag("RegionServer", name);
329     context.registerUpdater(this);
330     // Add jvmmetrics.
331     JvmMetrics.init("RegionServer", name);
332     // Add Hbase Info metrics
333     HBaseInfo.init();
334 
335     // export for JMX
336     statistics = new RegionServerStatistics(this.registry, name);
337 
338     // get custom attributes
339     try {
340       Object m = ContextFactory.getFactory().getAttribute("hbase.extendedperiod");
341       if (m instanceof String) {
342         this.extendedPeriod = Long.parseLong((String) m)*1000;
343       }
344     } catch (IOException ioe) {
345       LOG.info("Couldn't load ContextFactory for Metrics config info");
346     }
347 
348     LOG.info("Initialized");
349   }
350 
351   public void shutdown() {
352     if (statistics != null)
353       statistics.shutdown();
354   }
355 
356   /**
357    * Since this object is a registered updater, this method will be called
358    * periodically, e.g. every 5 seconds.
359    * @param caller the metrics context that this responsible for calling us
360    */
361   public void doUpdates(MetricsContext caller) {
362     synchronized (this) {
363       this.lastUpdate = System.currentTimeMillis();
364 
365       // has the extended period for long-living stats elapsed?
366       if (this.extendedPeriod > 0 &&
367           this.lastUpdate - this.lastExtUpdate >= this.extendedPeriod) {
368         this.lastExtUpdate = this.lastUpdate;
369         this.compactionTime.resetMinMaxAvg();
370         this.compactionSize.resetMinMaxAvg();
371         this.flushTime.resetMinMaxAvg();
372         this.flushSize.resetMinMaxAvg();
373         this.resetAllMinMax();
374       }
375 
376       this.stores.pushMetric(this.metricsRecord);
377       this.storefiles.pushMetric(this.metricsRecord);
378       this.hlogFileCount.pushMetric(this.metricsRecord);
379       this.hlogFileSizeMB.pushMetric(this.metricsRecord);
380       this.storefileIndexSizeMB.pushMetric(this.metricsRecord);
381       this.rootIndexSizeKB.pushMetric(this.metricsRecord);
382       this.totalStaticIndexSizeKB.pushMetric(this.metricsRecord);
383       this.totalStaticBloomSizeKB.pushMetric(this.metricsRecord);
384       this.memstoreSizeMB.pushMetric(this.metricsRecord);
385       this.mbInMemoryWithoutWAL.pushMetric(this.metricsRecord);
386       this.numPutsWithoutWAL.pushMetric(this.metricsRecord);
387       this.readRequestsCount.pushMetric(this.metricsRecord);
388       this.writeRequestsCount.pushMetric(this.metricsRecord);
389       this.regions.pushMetric(this.metricsRecord);
390       this.requests.pushMetric(this.metricsRecord);
391       this.compactionQueueSize.pushMetric(this.metricsRecord);
392       this.flushQueueSize.pushMetric(this.metricsRecord);
393       this.blockCacheSize.pushMetric(this.metricsRecord);
394       this.blockCacheFree.pushMetric(this.metricsRecord);
395       this.blockCacheCount.pushMetric(this.metricsRecord);
396       this.blockCacheHitCount.pushMetric(this.metricsRecord);
397       this.blockCacheMissCount.pushMetric(this.metricsRecord);
398       this.blockCacheEvictedCount.pushMetric(this.metricsRecord);
399       this.blockCacheHitRatio.pushMetric(this.metricsRecord);
400       this.blockCacheHitCachingRatio.pushMetric(this.metricsRecord);
401       this.hdfsBlocksLocalityIndex.pushMetric(this.metricsRecord);
402       this.blockCacheHitRatioPastNPeriods.pushMetric(this.metricsRecord);
403       this.blockCacheHitCachingRatioPastNPeriods.pushMetric(this.metricsRecord);
404 
405       // Mix in HFile and HLog metrics
406       // Be careful. Here is code for MTVR from up in hadoop:
407       // public synchronized void inc(final int numOps, final long time) {
408       //   currentData.numOperations += numOps;
409       //   currentData.time += time;
410       //   long timePerOps = time/numOps;
411       //    minMax.update(timePerOps);
412       // }
413       // Means you can't pass a numOps of zero or get a ArithmeticException / by zero.
414       // HLog metrics
415       addHLogMetric(HLog.getWriteTime(), this.fsWriteLatency);
416       addHLogMetric(HLog.getWriteSize(), this.fsWriteSize);
417       addHLogMetric(HLog.getSyncTime(), this.fsSyncLatency);
418       addHLogMetric(HLog.getSlowAppendTime(), this.slowHLogAppendTime);
419       this.slowHLogAppendCount.set(HLog.getSlowAppendCount());
420       // HFile metrics, sequential reads
421       int ops = HFile.getReadOps(); 
422       if (ops != 0) this.fsReadLatency.inc(ops, HFile.getReadTimeMs());
423       // HFile metrics, positional reads
424       ops = HFile.getPreadOps(); 
425       if (ops != 0) this.fsPreadLatency.inc(ops, HFile.getPreadTimeMs());
426       this.checksumFailuresCount.set(HFile.getChecksumFailuresCount());
427 
428       /* NOTE: removed HFile write latency.  2 reasons:
429        * 1) Mixing HLog latencies are far higher priority since they're 
430        *      on-demand and HFile is used in background (compact/flush)
431        * 2) HFile metrics are being handled at a higher level 
432        *      by compaction & flush metrics.
433        */
434 
435       for(Long latency : HFile.getReadLatenciesNanos()) {
436         this.fsReadLatencyHistogram.update(latency);
437       }
438       for(Long latency : HFile.getPreadLatenciesNanos()) {
439         this.fsPreadLatencyHistogram.update(latency);
440       }
441       for(Long latency : HFile.getWriteLatenciesNanos()) {
442         this.fsWriteLatencyHistogram.update(latency);
443       }
444             
445 
446       // push the result
447       this.fsPreadLatency.pushMetric(this.metricsRecord);
448       this.fsReadLatency.pushMetric(this.metricsRecord);
449       this.fsWriteLatency.pushMetric(this.metricsRecord);
450       this.fsWriteSize.pushMetric(this.metricsRecord);
451       
452       this.fsReadLatencyHistogram.pushMetric(this.metricsRecord);
453       this.fsWriteLatencyHistogram.pushMetric(this.metricsRecord);
454       this.fsPreadLatencyHistogram.pushMetric(this.metricsRecord);
455 
456       this.fsSyncLatency.pushMetric(this.metricsRecord);
457       this.compactionTime.pushMetric(this.metricsRecord);
458       this.compactionSize.pushMetric(this.metricsRecord);
459       this.flushTime.pushMetric(this.metricsRecord);
460       this.flushSize.pushMetric(this.metricsRecord);
461       this.slowHLogAppendCount.pushMetric(this.metricsRecord);
462       this.regionSplitSuccessCount.pushMetric(this.metricsRecord);
463       this.regionSplitFailureCount.pushMetric(this.metricsRecord);
464       this.checksumFailuresCount.pushMetric(this.metricsRecord);
465       this.updatesBlockedSeconds.pushMetric(this.metricsRecord);
466       this.updatesBlockedSecondsHighWater.pushMetric(this.metricsRecord);
467     }
468     this.metricsRecord.update();
469   }
470 
471   private void addHLogMetric(HLog.Metric logMetric,
472       MetricsTimeVaryingRate hadoopMetric) {
473     if (logMetric.count > 0)
474       hadoopMetric.inc(logMetric.min);
475     if (logMetric.count > 1)
476       hadoopMetric.inc(logMetric.max);
477     if (logMetric.count > 2) {
478       int ops = logMetric.count - 2;
479       hadoopMetric.inc(ops, logMetric.total - logMetric.max - logMetric.min);
480     }
481   }
482 
483   public void resetAllMinMax() {
484     this.atomicIncrementTime.resetMinMax();
485     this.fsReadLatency.resetMinMax();
486     this.fsWriteLatency.resetMinMax();
487     this.fsWriteSize.resetMinMax();
488     this.fsSyncLatency.resetMinMax();
489     this.slowHLogAppendTime.resetMinMax();
490   }
491 
492   /**
493    * @return Count of requests.
494    */
495   public float getRequests() {
496     return this.requests.getPreviousIntervalValue();
497   }
498 
499   /**
500    * @param time time that compaction took
501    * @param size bytesize of storefiles in the compaction
502    */
503   public synchronized void addCompaction(long time, long size) {
504     this.compactionTime.inc(time);
505     this.compactionSize.inc(size);
506   }
507 
508   /**
509    * @param flushes history in <time, size>
510    */
511   public synchronized void addFlush(final List<Pair<Long,Long>> flushes) {
512     for (Pair<Long,Long> f : flushes) {
513       this.flushTime.inc(f.getFirst());
514       this.flushSize.inc(f.getSecond());
515     }
516   }
517 
518   /**
519    * @param inc How much to add to requests.
520    */
521   public void incrementRequests(final int inc) {
522     this.requests.inc(inc);
523   }
524   
525   public void incrementSplitSuccessCount() {
526     this.regionSplitSuccessCount.inc();
527   }
528   
529   public void incrementSplitFailureCount() {
530     this.regionSplitFailureCount.inc();
531   }
532 
533   @Override
534   public String toString() {
535     StringBuilder sb = new StringBuilder();
536     sb = Strings.appendKeyValue(sb, "requestsPerSecond", Integer
537         .valueOf((int) this.requests.getPreviousIntervalValue()));
538     sb = Strings.appendKeyValue(sb, "numberOfOnlineRegions",
539       Integer.valueOf(this.regions.get()));
540     sb = Strings.appendKeyValue(sb, "numberOfStores",
541       Integer.valueOf(this.stores.get()));
542     sb = Strings.appendKeyValue(sb, this.hlogFileCount.getName(),
543       Integer.valueOf(this.hlogFileCount.get()));
544     sb = Strings.appendKeyValue(sb, this.hlogFileSizeMB.getName(),
545       Long.valueOf(this.hlogFileSizeMB.get()));
546     sb = Strings.appendKeyValue(sb, "numberOfStorefiles",
547       Integer.valueOf(this.storefiles.get()));
548     sb = Strings.appendKeyValue(sb, this.storefileIndexSizeMB.getName(),
549       Integer.valueOf(this.storefileIndexSizeMB.get()));
550     sb = Strings.appendKeyValue(sb, "rootIndexSizeKB",
551         Integer.valueOf(this.rootIndexSizeKB.get()));
552     sb = Strings.appendKeyValue(sb, "totalStaticIndexSizeKB",
553         Integer.valueOf(this.totalStaticIndexSizeKB.get()));
554     sb = Strings.appendKeyValue(sb, "totalStaticBloomSizeKB",
555         Integer.valueOf(this.totalStaticBloomSizeKB.get()));
556     sb = Strings.appendKeyValue(sb, this.memstoreSizeMB.getName(),
557       Integer.valueOf(this.memstoreSizeMB.get()));
558     sb = Strings.appendKeyValue(sb, "mbInMemoryWithoutWAL",
559       Integer.valueOf(this.mbInMemoryWithoutWAL.get()));
560     sb = Strings.appendKeyValue(sb, "numberOfPutsWithoutWAL",
561       Long.valueOf(this.numPutsWithoutWAL.get()));
562     sb = Strings.appendKeyValue(sb, "readRequestsCount",
563         Long.valueOf(this.readRequestsCount.get()));
564     sb = Strings.appendKeyValue(sb, "writeRequestsCount",
565         Long.valueOf(this.writeRequestsCount.get()));
566     sb = Strings.appendKeyValue(sb, "compactionQueueSize",
567       Integer.valueOf(this.compactionQueueSize.get()));
568     sb = Strings.appendKeyValue(sb, "flushQueueSize",
569       Integer.valueOf(this.flushQueueSize.get()));
570     // Duplicate from jvmmetrics because metrics are private there so
571     // inaccessible.
572     MemoryUsage memory =
573       ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
574     sb = Strings.appendKeyValue(sb, "usedHeapMB",
575       Long.valueOf(memory.getUsed()/MB));
576     sb = Strings.appendKeyValue(sb, "maxHeapMB",
577       Long.valueOf(memory.getMax()/MB));
578     sb = Strings.appendKeyValue(sb, this.blockCacheSize.getName()+"MB",
579     	StringUtils.limitDecimalTo2((float)this.blockCacheSize.get()/MB));
580     sb = Strings.appendKeyValue(sb, this.blockCacheFree.getName()+"MB",
581     	StringUtils.limitDecimalTo2((float)this.blockCacheFree.get()/MB));
582     sb = Strings.appendKeyValue(sb, this.blockCacheCount.getName(),
583         Long.valueOf(this.blockCacheCount.get()));
584     sb = Strings.appendKeyValue(sb, this.blockCacheHitCount.getName(),
585         Long.valueOf(this.blockCacheHitCount.get()));
586     sb = Strings.appendKeyValue(sb, this.blockCacheMissCount.getName(),
587         Long.valueOf(this.blockCacheMissCount.get()));
588     sb = Strings.appendKeyValue(sb, this.blockCacheEvictedCount.getName(),
589         Long.valueOf(this.blockCacheEvictedCount.get()));
590     sb = Strings.appendKeyValue(sb, this.blockCacheHitRatio.getName(),
591         Long.valueOf(this.blockCacheHitRatio.get())+"%");
592     sb = Strings.appendKeyValue(sb, this.blockCacheHitCachingRatio.getName(),
593         Long.valueOf(this.blockCacheHitCachingRatio.get())+"%");
594     sb = Strings.appendKeyValue(sb, this.hdfsBlocksLocalityIndex.getName(),
595         Long.valueOf(this.hdfsBlocksLocalityIndex.get()));
596     sb = Strings.appendKeyValue(sb, "slowHLogAppendCount",
597         Long.valueOf(this.slowHLogAppendCount.get()));
598     sb = appendHistogram(sb, this.fsReadLatencyHistogram);
599     sb = appendHistogram(sb, this.fsPreadLatencyHistogram);
600     sb = appendHistogram(sb, this.fsWriteLatencyHistogram);
601 
602     return sb.toString();
603   }
604   
605   private StringBuilder appendHistogram(StringBuilder sb, 
606       MetricsHistogram histogram) {
607     sb = Strings.appendKeyValue(sb, 
608         histogram.getName() + "Mean", 
609         StringUtils.limitDecimalTo2(histogram.getMean()));
610     sb = Strings.appendKeyValue(sb, 
611         histogram.getName() + "Count", 
612         StringUtils.limitDecimalTo2(histogram.getCount()));
613     final Snapshot s = histogram.getSnapshot();
614     sb = Strings.appendKeyValue(sb, 
615         histogram.getName() + "Median", 
616         StringUtils.limitDecimalTo2(s.getMedian()));
617     sb = Strings.appendKeyValue(sb, 
618         histogram.getName() + "75th", 
619         StringUtils.limitDecimalTo2(s.get75thPercentile()));
620     sb = Strings.appendKeyValue(sb, 
621         histogram.getName() + "95th", 
622         StringUtils.limitDecimalTo2(s.get95thPercentile()));
623     sb = Strings.appendKeyValue(sb, 
624         histogram.getName() + "99th", 
625         StringUtils.limitDecimalTo2(s.get99thPercentile()));
626     sb = Strings.appendKeyValue(sb, 
627         histogram.getName() + "999th", 
628         StringUtils.limitDecimalTo2(s.get999thPercentile()));
629     return sb;
630   }
631 }