View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.io.hfile;
20  
21  import java.util.concurrent.atomic.AtomicLong;
22  
23  import org.apache.hadoop.hbase.classification.InterfaceAudience;
24  
25  import org.apache.hadoop.hbase.util.Counter;
26  import org.apache.hadoop.hbase.util.FastLongHistogram;
27  
28  
29  /**
30   * Class that implements cache metrics.
31   */
32  @InterfaceAudience.Private
33  public class CacheStats {
34  
35    /** Sliding window statistics. The number of metric periods to include in
36     * sliding window hit ratio calculations.
37     */
38    static final int DEFAULT_WINDOW_PERIODS = 5;
39  
40    /** The number of getBlock requests that were cache hits */
41    private final Counter hitCount = new Counter();
42  
43    /** The number of getBlock requests that were cache hits from primary replica */
44    private final Counter primaryHitCount = new Counter();
45    
46    /**
47     * The number of getBlock requests that were cache hits, but only from
48     * requests that were set to use the block cache.  This is because all reads
49     * attempt to read from the block cache even if they will not put new blocks
50     * into the block cache.  See HBASE-2253 for more information.
51     */
52    private final Counter hitCachingCount = new Counter();
53  
54    /** The number of getBlock requests that were cache misses */
55    private final Counter missCount = new Counter();
56  
57    /** The number of getBlock requests for primary replica that were cache misses */
58    private final Counter primaryMissCount = new Counter();
59    /**
60     * The number of getBlock requests that were cache misses, but only from
61     * requests that were set to use the block cache.
62     */
63    private final Counter missCachingCount = new Counter();
64  
65    /** The number of times an eviction has occurred */
66    private final Counter evictionCount = new Counter();
67  
68    /** The total number of blocks that have been evicted */
69    private final Counter evictedBlockCount = new Counter();
70  
71    /** The total number of blocks for primary replica that have been evicted */
72    private final Counter primaryEvictedBlockCount = new Counter();
73  
74    /** The total number of blocks that were not inserted. */
75    private final AtomicLong failedInserts = new AtomicLong(0);
76  
77    /** Per Block Type Counts */
78    private final Counter dataMissCount = new Counter(0);
79    private final Counter leafIndexMissCount = new Counter(0);
80    private final Counter bloomChunkMissCount = new Counter(0);
81    private final Counter metaMissCount = new Counter(0);
82    private final Counter rootIndexMissCount = new Counter(0);
83    private final Counter intermediateIndexMissCount = new Counter(0);
84    private final Counter fileInfoMissCount = new Counter(0);
85    private final Counter generalBloomMetaMissCount = new Counter(0);
86    private final Counter deleteFamilyBloomMissCount = new Counter(0);
87    private final Counter trailerMissCount = new Counter(0);
88  
89    private final Counter dataHitCount = new Counter(0);
90    private final Counter leafIndexHitCount = new Counter(0);
91    private final Counter bloomChunkHitCount = new Counter(0);
92    private final Counter metaHitCount = new Counter(0);
93    private final Counter rootIndexHitCount = new Counter(0);
94    private final Counter intermediateIndexHitCount = new Counter(0);
95    private final Counter fileInfoHitCount = new Counter(0);
96    private final Counter generalBloomMetaHitCount = new Counter(0);
97    private final Counter deleteFamilyBloomHitCount = new Counter(0);
98    private final Counter trailerHitCount = new Counter(0);
99  
100   /** The number of metrics periods to include in window */
101   private final int numPeriodsInWindow;
102   /** Hit counts for each period in window */
103   private final long [] hitCounts;
104   /** Caching hit counts for each period in window */
105   private final long [] hitCachingCounts;
106   /** Access counts for each period in window */
107   private final long [] requestCounts;
108   /** Caching access counts for each period in window */
109   private final long [] requestCachingCounts;
110   /** Last hit count read */
111   private long lastHitCount = 0;
112   /** Last hit caching count read */
113   private long lastHitCachingCount = 0;
114   /** Last request count read */
115   private long lastRequestCount = 0;
116   /** Last request caching count read */
117   private long lastRequestCachingCount = 0;
118   /** Current window index (next to be updated) */
119   private int windowIndex = 0;
120   /**
121    * Keep running age at eviction time
122    */
123   private FastLongHistogram ageAtEviction;
124   private long startTime = System.nanoTime();
125 
126   public CacheStats(final String name) {
127     this(name, DEFAULT_WINDOW_PERIODS);
128   }
129 
130   public CacheStats(final String name, int numPeriodsInWindow) {
131     this.numPeriodsInWindow = numPeriodsInWindow;
132     this.hitCounts = initializeZeros(numPeriodsInWindow);
133     this.hitCachingCounts = initializeZeros(numPeriodsInWindow);
134     this.requestCounts = initializeZeros(numPeriodsInWindow);
135     this.requestCachingCounts = initializeZeros(numPeriodsInWindow);
136     this.ageAtEviction = new FastLongHistogram();
137   }
138 
139   @Override
140   public String toString() {
141     AgeSnapshot snapshot = getAgeAtEvictionSnapshot();
142     return "hitCount=" + getHitCount() + ", hitCachingCount=" + getHitCachingCount() +
143       ", missCount=" + getMissCount() + ", missCachingCount=" + getMissCachingCount() +
144       ", evictionCount=" + getEvictionCount() +
145       ", evictedBlockCount=" + getEvictedCount() +
146       ", primaryMissCount=" + getPrimaryMissCount() +
147       ", primaryHitCount=" + getPrimaryHitCount() +
148       ", evictedAgeMean=" + snapshot.getMean();
149   }
150 
151 
152   public void miss(boolean caching, boolean primary, BlockType type) {
153     missCount.increment();
154     if (primary) primaryMissCount.increment();
155     if (caching) missCachingCount.increment();
156     if (type == null) {
157       return;
158     }
159     switch (type) {
160       case DATA:
161       case ENCODED_DATA:
162         dataMissCount.increment();
163         break;
164       case LEAF_INDEX:
165         leafIndexMissCount.increment();
166         break;
167       case BLOOM_CHUNK:
168         bloomChunkMissCount.increment();
169         break;
170       case META:
171         metaMissCount.increment();
172         break;
173       case INTERMEDIATE_INDEX:
174         intermediateIndexMissCount.increment();
175         break;
176       case ROOT_INDEX:
177         rootIndexMissCount.increment();
178         break;
179       case FILE_INFO:
180         fileInfoMissCount.increment();
181         break;
182       case GENERAL_BLOOM_META:
183         generalBloomMetaMissCount.increment();
184         break;
185       case DELETE_FAMILY_BLOOM_META:
186         deleteFamilyBloomMissCount.increment();
187         break;
188       case TRAILER:
189         trailerMissCount.increment();
190         break;
191       default:
192         // If there's a new type that's fine
193         // Ignore it for now. This is metrics don't exception.
194         break;
195     }
196   }
197 
198   public void hit(boolean caching, boolean primary, BlockType type) {
199     hitCount.increment();
200     if (primary) primaryHitCount.increment();
201     if (caching) hitCachingCount.increment();
202 
203 
204     if (type == null) {
205       return;
206     }
207     switch (type) {
208       case DATA:
209       case ENCODED_DATA:
210         dataHitCount.increment();
211         break;
212       case LEAF_INDEX:
213         leafIndexHitCount.increment();
214         break;
215       case BLOOM_CHUNK:
216         bloomChunkHitCount.increment();
217         break;
218       case META:
219         metaHitCount.increment();
220         break;
221       case INTERMEDIATE_INDEX:
222         intermediateIndexHitCount.increment();
223         break;
224       case ROOT_INDEX:
225         rootIndexHitCount.increment();
226         break;
227       case FILE_INFO:
228         fileInfoHitCount.increment();
229         break;
230       case GENERAL_BLOOM_META:
231         generalBloomMetaHitCount.increment();
232         break;
233       case DELETE_FAMILY_BLOOM_META:
234         deleteFamilyBloomHitCount.increment();
235         break;
236       case TRAILER:
237         trailerHitCount.increment();
238         break;
239       default:
240         // If there's a new type that's fine
241         // Ignore it for now. This is metrics don't exception.
242         break;
243     }
244   }
245 
246   public void evict() {
247     evictionCount.increment();
248   }
249 
250   public void evicted(final long t, boolean primary) {
251     if (t > this.startTime) {
252       this.ageAtEviction.add((t - this.startTime) / BlockCacheUtil.NANOS_PER_SECOND, 1);
253     }
254     this.evictedBlockCount.increment();
255     if (primary) {
256       primaryEvictedBlockCount.increment();
257     }
258   }
259
260   public long failInsert() {
261     return failedInserts.incrementAndGet();
262   }
263
264
265   // All of the counts of misses and hits.
266   public long getDataMissCount() {
267     return dataMissCount.get();
268   }
269
270   public long getLeafIndexMissCount() {
271     return leafIndexMissCount.get();
272   }
273
274   public long getBloomChunkMissCount() {
275     return bloomChunkMissCount.get();
276   }
277
278   public long getMetaMissCount() {
279     return metaMissCount.get();
280   }
281
282   public long getRootIndexMissCount() {
283     return rootIndexMissCount.get();
284   }
285
286   public long getIntermediateIndexMissCount() {
287     return intermediateIndexMissCount.get();
288   }
289
290   public long getFileInfoMissCount() {
291     return fileInfoMissCount.get();
292   }
293
294   public long getGeneralBloomMetaMissCount() {
295     return generalBloomMetaMissCount.get();
296   }
297
298   public long getDeleteFamilyBloomMissCount() {
299     return deleteFamilyBloomMissCount.get();
300   }
301
302   public long getTrailerMissCount() {
303     return trailerMissCount.get();
304   }
305
306   public long getDataHitCount() {
307     return dataHitCount.get();
308   }
309
310   public long getLeafIndexHitCount() {
311     return leafIndexHitCount.get();
312   }
313
314   public long getBloomChunkHitCount() {
315     return bloomChunkHitCount.get();
316   }
317
318   public long getMetaHitCount() {
319     return metaHitCount.get();
320   }
321
322   public long getRootIndexHitCount() {
323     return rootIndexHitCount.get();
324   }
325
326   public long getIntermediateIndexHitCount() {
327     return intermediateIndexHitCount.get();
328   }
329
330   public long getFileInfoHitCount() {
331     return fileInfoHitCount.get();
332   }
333
334   public long getGeneralBloomMetaHitCount() {
335     return generalBloomMetaHitCount.get();
336   }
337
338   public long getDeleteFamilyBloomHitCount() {
339     return deleteFamilyBloomHitCount.get();
340   }
341
342   public long getTrailerHitCount() {
343     return trailerHitCount.get();
344   }
345
346   public long getRequestCount() {
347     return getHitCount() + getMissCount();
348   }
349
350   public long getRequestCachingCount() {
351     return getHitCachingCount() + getMissCachingCount();
352   }
353
354   public long getMissCount() {
355     return missCount.get();
356   }
357
358   public long getPrimaryMissCount() {
359     return primaryMissCount.get();
360   }
361
362   public long getMissCachingCount() {
363     return missCachingCount.get();
364   }
365
366   public long getHitCount() {
367     return hitCount.get();
368   }
369
370   public long getPrimaryHitCount() {
371     return primaryHitCount.get();
372   }
373
374   public long getHitCachingCount() {
375     return hitCachingCount.get();
376   }
377
378   public long getEvictionCount() {
379     return evictionCount.get();
380   }
381
382   public long getEvictedCount() {
383     return this.evictedBlockCount.get();
384   }
385
386   public long getPrimaryEvictedCount() {
387     return primaryEvictedBlockCount.get();
388   }
389
390   public double getHitRatio() {
391     return ((float)getHitCount()/(float)getRequestCount());
392   }
393
394   public double getHitCachingRatio() {
395     return ((float)getHitCachingCount()/(float)getRequestCachingCount());
396   }
397
398   public double getMissRatio() {
399     return ((float)getMissCount()/(float)getRequestCount());
400   }
401
402   public double getMissCachingRatio() {
403     return ((float)getMissCachingCount()/(float)getRequestCachingCount());
404   }
405
406   public double evictedPerEviction() {
407     return ((float)getEvictedCount()/(float)getEvictionCount());
408   }
409
410   public long getFailedInserts() {
411     return failedInserts.get();
412   }
413
414   public void rollMetricsPeriod() {
415     hitCounts[windowIndex] = getHitCount() - lastHitCount;
416     lastHitCount = getHitCount();
417     hitCachingCounts[windowIndex] =
418       getHitCachingCount() - lastHitCachingCount;
419     lastHitCachingCount = getHitCachingCount();
420     requestCounts[windowIndex] = getRequestCount() - lastRequestCount;
421     lastRequestCount = getRequestCount();
422     requestCachingCounts[windowIndex] =
423       getRequestCachingCount() - lastRequestCachingCount;
424     lastRequestCachingCount = getRequestCachingCount();
425     windowIndex = (windowIndex + 1) % numPeriodsInWindow;
426   }
427
428   public long getSumHitCountsPastNPeriods() {
429     return sum(hitCounts);
430   }
431
432   public long getSumRequestCountsPastNPeriods() {
433     return sum(requestCounts);
434   }
435
436   public long getSumHitCachingCountsPastNPeriods() {
437     return sum(hitCachingCounts);
438   }
439
440   public long getSumRequestCachingCountsPastNPeriods() {
441     return sum(requestCachingCounts);
442   }
443
444   public double getHitRatioPastNPeriods() {
445     double ratio = ((double)getSumHitCountsPastNPeriods() /
446         (double)getSumRequestCountsPastNPeriods());
447     return Double.isNaN(ratio) ? 0 : ratio;
448   }
449
450   public double getHitCachingRatioPastNPeriods() {
451     double ratio = ((double)getSumHitCachingCountsPastNPeriods() /
452         (double)getSumRequestCachingCountsPastNPeriods());
453     return Double.isNaN(ratio) ? 0 : ratio;
454   }
455
456   public AgeSnapshot getAgeAtEvictionSnapshot() {
457     return new AgeSnapshot(this.ageAtEviction);
458   }
459
460   private static long sum(long [] counts) {
461     long sum = 0;
462     for (long count : counts) sum += count;
463     return sum;
464   }
465
466   private static long [] initializeZeros(int n) {
467     long [] zeros = new long [n];
468     for (int i=0; i<n; i++) {
469       zeros[i] = 0L;
470     }
471     return zeros;
472   }
473 }