View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.io.hfile;
20  
21  import java.util.concurrent.atomic.AtomicLong;
22  
23  import org.apache.hadoop.hbase.classification.InterfaceAudience;
24  
25  import org.apache.hadoop.hbase.util.Counter;
26  import org.apache.hadoop.hbase.util.FastLongHistogram;
27  
28  
29  /**
30   * Class that implements cache metrics.
31   */
32  @InterfaceAudience.Private
33  public class CacheStats {
34  
35    /** Sliding window statistics. The number of metric periods to include in
36     * sliding window hit ratio calculations.
37     */
38    static final int DEFAULT_WINDOW_PERIODS = 5;
39  
40    /** The number of getBlock requests that were cache hits */
41    private final Counter hitCount = new Counter();
42  
43    /** The number of getBlock requests that were cache hits from primary replica */
44    private final Counter primaryHitCount = new Counter();
45    
46    /**
47     * The number of getBlock requests that were cache hits, but only from
48     * requests that were set to use the block cache.  This is because all reads
49     * attempt to read from the block cache even if they will not put new blocks
50     * into the block cache.  See HBASE-2253 for more information.
51     */
52    private final Counter hitCachingCount = new Counter();
53  
54    /** The number of getBlock requests that were cache misses */
55    private final Counter missCount = new Counter();
56  
57    /** The number of getBlock requests for primary replica that were cache misses */
58    private final Counter primaryMissCount = new Counter();
59    /**
60     * The number of getBlock requests that were cache misses, but only from
61     * requests that were set to use the block cache.
62     */
63    private final Counter missCachingCount = new Counter();
64  
65    /** The number of times an eviction has occurred */
66    private final Counter evictionCount = new Counter();
67  
68    /** The total number of blocks that have been evicted */
69    private final Counter evictedBlockCount = new Counter();
70  
71    /** The total number of blocks for primary replica that have been evicted */
72    private final Counter primaryEvictedBlockCount = new Counter();
73  
74    /** The total number of blocks that were not inserted. */
75    private final AtomicLong failedInserts = new AtomicLong(0);
76  
77    /** Per Block Type Counts */
78    private final Counter dataMissCount = new Counter(0);
79    private final Counter leafIndexMissCount = new Counter(0);
80    private final Counter bloomChunkMissCount = new Counter(0);
81    private final Counter metaMissCount = new Counter(0);
82    private final Counter rootIndexMissCount = new Counter(0);
83    private final Counter intermediateIndexMissCount = new Counter(0);
84    private final Counter fileInfoMissCount = new Counter(0);
85    private final Counter generalBloomMetaMissCount = new Counter(0);
86    private final Counter deleteFamilyBloomMissCount = new Counter(0);
87    private final Counter trailerMissCount = new Counter(0);
88  
89    private final Counter dataHitCount = new Counter(0);
90    private final Counter leafIndexHitCount = new Counter(0);
91    private final Counter bloomChunkHitCount = new Counter(0);
92    private final Counter metaHitCount = new Counter(0);
93    private final Counter rootIndexHitCount = new Counter(0);
94    private final Counter intermediateIndexHitCount = new Counter(0);
95    private final Counter fileInfoHitCount = new Counter(0);
96    private final Counter generalBloomMetaHitCount = new Counter(0);
97    private final Counter deleteFamilyBloomHitCount = new Counter(0);
98    private final Counter trailerHitCount = new Counter(0);
99  
100   /** The number of metrics periods to include in window */
101   private final int numPeriodsInWindow;
102   /** Hit counts for each period in window */
103   private final long [] hitCounts;
104   /** Caching hit counts for each period in window */
105   private final long [] hitCachingCounts;
106   /** Access counts for each period in window */
107   private final long [] requestCounts;
108   /** Caching access counts for each period in window */
109   private final long [] requestCachingCounts;
110   /** Last hit count read */
111   private long lastHitCount = 0;
112   /** Last hit caching count read */
113   private long lastHitCachingCount = 0;
114   /** Last request count read */
115   private long lastRequestCount = 0;
116   /** Last request caching count read */
117   private long lastRequestCachingCount = 0;
118   /** Current window index (next to be updated) */
119   private int windowIndex = 0;
120   /**
121    * Keep running age at eviction time
122    */
123   private FastLongHistogram ageAtEviction;
124   private long startTime = System.nanoTime();
125 
126   public CacheStats(final String name) {
127     this(name, DEFAULT_WINDOW_PERIODS);
128   }
129 
130   public CacheStats(final String name, int numPeriodsInWindow) {
131     this.numPeriodsInWindow = numPeriodsInWindow;
132     this.hitCounts = initializeZeros(numPeriodsInWindow);
133     this.hitCachingCounts = initializeZeros(numPeriodsInWindow);
134     this.requestCounts = initializeZeros(numPeriodsInWindow);
135     this.requestCachingCounts = initializeZeros(numPeriodsInWindow);
136     this.ageAtEviction = new FastLongHistogram();
137   }
138 
139   @Override
140   public String toString() {
141     AgeSnapshot snapshot = getAgeAtEvictionSnapshot();
142     return "hitCount=" + getHitCount() + ", hitCachingCount=" + getHitCachingCount() +
143       ", missCount=" + getMissCount() + ", missCachingCount=" + getMissCachingCount() +
144       ", evictionCount=" + getEvictionCount() +
145       ", evictedBlockCount=" + getEvictedCount() +
146       ", primaryMissCount=" + getPrimaryMissCount() +
147       ", primaryHitCount=" + getPrimaryHitCount() +
148       ", evictedAgeMean=" + snapshot.getMean();
149   }
150 
151 
152   public void miss(boolean caching, boolean primary, BlockType type) {
153     missCount.increment();
154     if (primary) primaryMissCount.increment();
155     if (caching) missCachingCount.increment();
156     if (type == null) {
157       return;
158     }
159     switch (type) {
160       case DATA:
161       case ENCODED_DATA:
162         dataMissCount.increment();
163         break;
164       case LEAF_INDEX:
165         leafIndexMissCount.increment();
166         break;
167       case BLOOM_CHUNK:
168         bloomChunkMissCount.increment();
169         break;
170       case META:
171         metaMissCount.increment();
172         break;
173       case INTERMEDIATE_INDEX:
174         intermediateIndexMissCount.increment();
175         break;
176       case ROOT_INDEX:
177         rootIndexMissCount.increment();
178         break;
179       case FILE_INFO:
180         fileInfoMissCount.increment();
181         break;
182       case GENERAL_BLOOM_META:
183         generalBloomMetaMissCount.increment();
184         break;
185       case DELETE_FAMILY_BLOOM_META:
186         deleteFamilyBloomMissCount.increment();
187         break;
188       case TRAILER:
189         trailerMissCount.increment();
190         break;
191       default:
192         // If there's a new type that's fine
193         // Ignore it for now. This is metrics don't exception.
194         break;
195     }
196   }
197 
198   public void hit(boolean caching, boolean primary, BlockType type) {
199     hitCount.increment();
200     if (primary) primaryHitCount.increment();
201     if (caching) hitCachingCount.increment();
202 
203 
204     if (type == null) {
205       return;
206     }
207     switch (type) {
208       case DATA:
209       case ENCODED_DATA:
210         dataHitCount.increment();
211         break;
212       case LEAF_INDEX:
213         leafIndexHitCount.increment();
214         break;
215       case BLOOM_CHUNK:
216         bloomChunkHitCount.increment();
217         break;
218       case META:
219         metaHitCount.increment();
220         break;
221       case INTERMEDIATE_INDEX:
222         intermediateIndexHitCount.increment();
223         break;
224       case ROOT_INDEX:
225         rootIndexHitCount.increment();
226         break;
227       case FILE_INFO:
228         fileInfoHitCount.increment();
229         break;
230       case GENERAL_BLOOM_META:
231         generalBloomMetaHitCount.increment();
232         break;
233       case DELETE_FAMILY_BLOOM_META:
234         deleteFamilyBloomHitCount.increment();
235         break;
236       case TRAILER:
237         trailerHitCount.increment();
238         break;
239       default:
240         // If there's a new type that's fine
241         // Ignore it for now. This is metrics don't exception.
242         break;
243     }
244   }
245 
246   public void evict() {
247     evictionCount.increment();
248   }
249 
250   public void evicted(final long t, boolean primary) {
251     if (t > this.startTime) this.ageAtEviction.add(t - this.startTime,1);
252     this.evictedBlockCount.increment();
253     if (primary) {
254       primaryEvictedBlockCount.increment();
255     }
256   }
257 
258   public long failInsert() {
259     return failedInserts.incrementAndGet();
260   }
261 
262 
263   // All of the counts of misses and hits.
264   public long getDataMissCount() {
265     return dataMissCount.get();
266   }
267 
268   public long getLeafIndexMissCount() {
269     return leafIndexMissCount.get();
270   }
271 
272   public long getBloomChunkMissCount() {
273     return bloomChunkMissCount.get();
274   }
275 
276   public long getMetaMissCount() {
277     return metaMissCount.get();
278   }
279 
280   public long getRootIndexMissCount() {
281     return rootIndexMissCount.get();
282   }
283 
284   public long getIntermediateIndexMissCount() {
285     return intermediateIndexMissCount.get();
286   }
287 
288   public long getFileInfoMissCount() {
289     return fileInfoMissCount.get();
290   }
291 
292   public long getGeneralBloomMetaMissCount() {
293     return generalBloomMetaMissCount.get();
294   }
295 
296   public long getDeleteFamilyBloomMissCount() {
297     return deleteFamilyBloomMissCount.get();
298   }
299 
300   public long getTrailerMissCount() {
301     return trailerMissCount.get();
302   }
303 
304   public long getDataHitCount() {
305     return dataHitCount.get();
306   }
307 
308   public long getLeafIndexHitCount() {
309     return leafIndexHitCount.get();
310   }
311 
312   public long getBloomChunkHitCount() {
313     return bloomChunkHitCount.get();
314   }
315 
316   public long getMetaHitCount() {
317     return metaHitCount.get();
318   }
319 
320   public long getRootIndexHitCount() {
321     return rootIndexHitCount.get();
322   }
323 
324   public long getIntermediateIndexHitCount() {
325     return intermediateIndexHitCount.get();
326   }
327 
328   public long getFileInfoHitCount() {
329     return fileInfoHitCount.get();
330   }
331 
332   public long getGeneralBloomMetaHitCount() {
333     return generalBloomMetaHitCount.get();
334   }
335 
336   public long getDeleteFamilyBloomHitCount() {
337     return deleteFamilyBloomHitCount.get();
338   }
339 
340   public long getTrailerHitCount() {
341     return trailerHitCount.get();
342   }
343 
344   public long getRequestCount() {
345     return getHitCount() + getMissCount();
346   }
347 
348   public long getRequestCachingCount() {
349     return getHitCachingCount() + getMissCachingCount();
350   }
351 
352   public long getMissCount() {
353     return missCount.get();
354   }
355 
356   public long getPrimaryMissCount() {
357     return primaryMissCount.get();
358   }
359 
360   public long getMissCachingCount() {
361     return missCachingCount.get();
362   }
363 
364   public long getHitCount() {
365     return hitCount.get();
366   }
367 
368   public long getPrimaryHitCount() {
369     return primaryHitCount.get();
370   }
371 
372   public long getHitCachingCount() {
373     return hitCachingCount.get();
374   }
375 
376   public long getEvictionCount() {
377     return evictionCount.get();
378   }
379 
380   public long getEvictedCount() {
381     return this.evictedBlockCount.get();
382   }
383 
384   public long getPrimaryEvictedCount() {
385     return primaryEvictedBlockCount.get();
386   }
387 
388   public double getHitRatio() {
389     return ((float)getHitCount()/(float)getRequestCount());
390   }
391 
392   public double getHitCachingRatio() {
393     return ((float)getHitCachingCount()/(float)getRequestCachingCount());
394   }
395 
396   public double getMissRatio() {
397     return ((float)getMissCount()/(float)getRequestCount());
398   }
399 
400   public double getMissCachingRatio() {
401     return ((float)getMissCachingCount()/(float)getRequestCachingCount());
402   }
403 
404   public double evictedPerEviction() {
405     return ((float)getEvictedCount()/(float)getEvictionCount());
406   }
407 
408   public long getFailedInserts() {
409     return failedInserts.get();
410   }
411 
412   public void rollMetricsPeriod() {
413     hitCounts[windowIndex] = getHitCount() - lastHitCount;
414     lastHitCount = getHitCount();
415     hitCachingCounts[windowIndex] =
416       getHitCachingCount() - lastHitCachingCount;
417     lastHitCachingCount = getHitCachingCount();
418     requestCounts[windowIndex] = getRequestCount() - lastRequestCount;
419     lastRequestCount = getRequestCount();
420     requestCachingCounts[windowIndex] =
421       getRequestCachingCount() - lastRequestCachingCount;
422     lastRequestCachingCount = getRequestCachingCount();
423     windowIndex = (windowIndex + 1) % numPeriodsInWindow;
424   }
425 
426   public long getSumHitCountsPastNPeriods() {
427     return sum(hitCounts);
428   }
429 
430   public long getSumRequestCountsPastNPeriods() {
431     return sum(requestCounts);
432   }
433 
434   public long getSumHitCachingCountsPastNPeriods() {
435     return sum(hitCachingCounts);
436   }
437 
438   public long getSumRequestCachingCountsPastNPeriods() {
439     return sum(requestCachingCounts);
440   }
441 
442   public double getHitRatioPastNPeriods() {
443     double ratio = ((double)getSumHitCountsPastNPeriods() /
444         (double)getSumRequestCountsPastNPeriods());
445     return Double.isNaN(ratio) ? 0 : ratio;
446   }
447 
448   public double getHitCachingRatioPastNPeriods() {
449     double ratio = ((double)getSumHitCachingCountsPastNPeriods() /
450         (double)getSumRequestCachingCountsPastNPeriods());
451     return Double.isNaN(ratio) ? 0 : ratio;
452   }
453 
454   public AgeSnapshot getAgeAtEvictionSnapshot() {
455     return new AgeSnapshot(this.ageAtEviction);
456   }
457 
458   private static long sum(long [] counts) {
459     long sum = 0;
460     for (long count : counts) sum += count;
461     return sum;
462   }
463 
464   private static long [] initializeZeros(int n) {
465     long [] zeros = new long [n];
466     for (int i=0; i<n; i++) {
467       zeros[i] = 0L;
468     }
469     return zeros;
470   }
471 }