1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.io.hfile;
20
21 import java.lang.ref.WeakReference;
22 import java.nio.ByteBuffer;
23 import java.util.EnumMap;
24 import java.util.Iterator;
25 import java.util.List;
26 import java.util.Map;
27 import java.util.PriorityQueue;
28 import java.util.SortedSet;
29 import java.util.TreeSet;
30 import java.util.concurrent.ConcurrentHashMap;
31 import java.util.concurrent.Executors;
32 import java.util.concurrent.ScheduledExecutorService;
33 import java.util.concurrent.TimeUnit;
34 import java.util.concurrent.atomic.AtomicLong;
35 import java.util.concurrent.locks.ReentrantLock;
36
37 import com.google.common.base.Objects;
38 import org.apache.commons.logging.Log;
39 import org.apache.commons.logging.LogFactory;
40 import org.apache.hadoop.hbase.classification.InterfaceAudience;
41 import org.apache.hadoop.conf.Configuration;
42 import org.apache.hadoop.hbase.io.HeapSize;
43 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
44 import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache;
45 import org.apache.hadoop.hbase.util.Bytes;
46 import org.apache.hadoop.hbase.util.ClassSize;
47 import org.apache.hadoop.hbase.util.HasThread;
48 import org.apache.hadoop.util.StringUtils;
49 import org.codehaus.jackson.annotate.JsonIgnoreProperties;
50
51 import com.google.common.annotations.VisibleForTesting;
52 import com.google.common.util.concurrent.ThreadFactoryBuilder;
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98 @InterfaceAudience.Private
99 @JsonIgnoreProperties({"encodingCountsForTest"})
100 public class LruBlockCache implements ResizableBlockCache, HeapSize {
101
102 static final Log LOG = LogFactory.getLog(LruBlockCache.class);
103
104
105
106
107
108 static final String LRU_MIN_FACTOR_CONFIG_NAME = "hbase.lru.blockcache.min.factor";
109
110
111
112
113 static final String LRU_ACCEPTABLE_FACTOR_CONFIG_NAME = "hbase.lru.blockcache.acceptable.factor";
114
115 static final String LRU_SINGLE_PERCENTAGE_CONFIG_NAME = "hbase.lru.blockcache.single.percentage";
116 static final String LRU_MULTI_PERCENTAGE_CONFIG_NAME = "hbase.lru.blockcache.multi.percentage";
117 static final String LRU_MEMORY_PERCENTAGE_CONFIG_NAME = "hbase.lru.blockcache.memory.percentage";
118
119
120
121
122
123
124 static final String LRU_IN_MEMORY_FORCE_MODE_CONFIG_NAME = "hbase.lru.rs.inmemoryforcemode";
125
126
127
128
129 static final float DEFAULT_LOAD_FACTOR = 0.75f;
130 static final int DEFAULT_CONCURRENCY_LEVEL = 16;
131
132
133 static final float DEFAULT_MIN_FACTOR = 0.95f;
134 static final float DEFAULT_ACCEPTABLE_FACTOR = 0.99f;
135
136
137 static final float DEFAULT_SINGLE_FACTOR = 0.25f;
138 static final float DEFAULT_MULTI_FACTOR = 0.50f;
139 static final float DEFAULT_MEMORY_FACTOR = 0.25f;
140
141 static final boolean DEFAULT_IN_MEMORY_FORCE_MODE = false;
142
143
144 static final int statThreadPeriod = 60 * 5;
145
146
147 private final Map<BlockCacheKey,LruCachedBlock> map;
148
149
150 private final ReentrantLock evictionLock = new ReentrantLock(true);
151
152
153 private volatile boolean evictionInProgress = false;
154
155
156 private final EvictionThread evictionThread;
157
158
159 private final ScheduledExecutorService scheduleThreadPool = Executors.newScheduledThreadPool(1,
160 new ThreadFactoryBuilder().setNameFormat("LruBlockCacheStatsExecutor").setDaemon(true).build());
161
162
163 private final AtomicLong size;
164
165
166 private final AtomicLong elements;
167
168
169 private final AtomicLong count;
170
171
172 private final CacheStats stats;
173
174
175 private long maxSize;
176
177
178 private long blockSize;
179
180
181 private float acceptableFactor;
182
183
184 private float minFactor;
185
186
187 private float singleFactor;
188
189
190 private float multiFactor;
191
192
193 private float memoryFactor;
194
195
196 private long overhead;
197
198
199 private boolean forceInMemory;
200
201
202 private BlockCache victimHandler = null;
203
204
205
206
207
208
209
210
211
212
213 public LruBlockCache(long maxSize, long blockSize) {
214 this(maxSize, blockSize, true);
215 }
216
217
218
219
220 public LruBlockCache(long maxSize, long blockSize, boolean evictionThread) {
221 this(maxSize, blockSize, evictionThread,
222 (int)Math.ceil(1.2*maxSize/blockSize),
223 DEFAULT_LOAD_FACTOR, DEFAULT_CONCURRENCY_LEVEL,
224 DEFAULT_MIN_FACTOR, DEFAULT_ACCEPTABLE_FACTOR,
225 DEFAULT_SINGLE_FACTOR,
226 DEFAULT_MULTI_FACTOR,
227 DEFAULT_MEMORY_FACTOR,
228 false
229 );
230 }
231
232 public LruBlockCache(long maxSize, long blockSize, boolean evictionThread, Configuration conf) {
233 this(maxSize, blockSize, evictionThread,
234 (int)Math.ceil(1.2*maxSize/blockSize),
235 DEFAULT_LOAD_FACTOR,
236 DEFAULT_CONCURRENCY_LEVEL,
237 conf.getFloat(LRU_MIN_FACTOR_CONFIG_NAME, DEFAULT_MIN_FACTOR),
238 conf.getFloat(LRU_ACCEPTABLE_FACTOR_CONFIG_NAME, DEFAULT_ACCEPTABLE_FACTOR),
239 conf.getFloat(LRU_SINGLE_PERCENTAGE_CONFIG_NAME, DEFAULT_SINGLE_FACTOR),
240 conf.getFloat(LRU_MULTI_PERCENTAGE_CONFIG_NAME, DEFAULT_MULTI_FACTOR),
241 conf.getFloat(LRU_MEMORY_PERCENTAGE_CONFIG_NAME, DEFAULT_MEMORY_FACTOR),
242 conf.getBoolean(LRU_IN_MEMORY_FORCE_MODE_CONFIG_NAME, DEFAULT_IN_MEMORY_FORCE_MODE)
243 );
244 }
245
246 public LruBlockCache(long maxSize, long blockSize, Configuration conf) {
247 this(maxSize, blockSize, true, conf);
248 }
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264 public LruBlockCache(long maxSize, long blockSize, boolean evictionThread,
265 int mapInitialSize, float mapLoadFactor, int mapConcurrencyLevel,
266 float minFactor, float acceptableFactor, float singleFactor,
267 float multiFactor, float memoryFactor, boolean forceInMemory) {
268 if(singleFactor + multiFactor + memoryFactor != 1 ||
269 singleFactor < 0 || multiFactor < 0 || memoryFactor < 0) {
270 throw new IllegalArgumentException("Single, multi, and memory factors " +
271 " should be non-negative and total 1.0");
272 }
273 if(minFactor >= acceptableFactor) {
274 throw new IllegalArgumentException("minFactor must be smaller than acceptableFactor");
275 }
276 if(minFactor >= 1.0f || acceptableFactor >= 1.0f) {
277 throw new IllegalArgumentException("all factors must be < 1");
278 }
279 this.maxSize = maxSize;
280 this.blockSize = blockSize;
281 this.forceInMemory = forceInMemory;
282 map = new ConcurrentHashMap<BlockCacheKey,LruCachedBlock>(mapInitialSize,
283 mapLoadFactor, mapConcurrencyLevel);
284 this.minFactor = minFactor;
285 this.acceptableFactor = acceptableFactor;
286 this.singleFactor = singleFactor;
287 this.multiFactor = multiFactor;
288 this.memoryFactor = memoryFactor;
289 this.stats = new CacheStats(this.getClass().getSimpleName());
290 this.count = new AtomicLong(0);
291 this.elements = new AtomicLong(0);
292 this.overhead = calculateOverhead(maxSize, blockSize, mapConcurrencyLevel);
293 this.size = new AtomicLong(this.overhead);
294 if(evictionThread) {
295 this.evictionThread = new EvictionThread(this);
296 this.evictionThread.start();
297 } else {
298 this.evictionThread = null;
299 }
300
301
302 this.scheduleThreadPool.scheduleAtFixedRate(new StatisticsThread(this),
303 statThreadPeriod, statThreadPeriod, TimeUnit.SECONDS);
304 }
305
306 @Override
307 public void setMaxSize(long maxSize) {
308 this.maxSize = maxSize;
309 if(this.size.get() > acceptableSize() && !evictionInProgress) {
310 runEviction();
311 }
312 }
313
314
315
316
317
318
319
320
321
322
323
324
325
326 @Override
327 public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory,
328 final boolean cacheDataInL1) {
329 LruCachedBlock cb = map.get(cacheKey);
330 if (cb != null) {
331
332 if (compare(buf, cb.getBuffer()) != 0) {
333 throw new RuntimeException("Cached block contents differ, which should not have happened."
334 + "cacheKey:" + cacheKey);
335 }
336 String msg = "Cached an already cached block: " + cacheKey + " cb:" + cb.getCacheKey();
337 msg += ". This is harmless and can happen in rare cases (see HBASE-8547)";
338 LOG.warn(msg);
339 return;
340 }
341 cb = new LruCachedBlock(cacheKey, buf, count.incrementAndGet(), inMemory);
342 long newSize = updateSizeMetrics(cb, false);
343 map.put(cacheKey, cb);
344 long val = elements.incrementAndGet();
345 if (LOG.isTraceEnabled()) {
346 long size = map.size();
347 assertCounterSanity(size, val);
348 }
349 if (newSize > acceptableSize() && !evictionInProgress) {
350 runEviction();
351 }
352 }
353
354
355
356
357
358 private static void assertCounterSanity(long mapSize, long counterVal) {
359 if (counterVal < 0) {
360 LOG.trace("counterVal overflow. Assertions unreliable. counterVal=" + counterVal +
361 ", mapSize=" + mapSize);
362 return;
363 }
364 if (mapSize < Integer.MAX_VALUE) {
365 double pct_diff = Math.abs((((double) counterVal) / ((double) mapSize)) - 1.);
366 if (pct_diff > 0.05) {
367 LOG.trace("delta between reported and actual size > 5%. counterVal=" + counterVal +
368 ", mapSize=" + mapSize);
369 }
370 }
371 }
372
373 private int compare(Cacheable left, Cacheable right) {
374 ByteBuffer l = ByteBuffer.allocate(left.getSerializedLength());
375 left.serialize(l);
376 ByteBuffer r = ByteBuffer.allocate(right.getSerializedLength());
377 right.serialize(r);
378 return Bytes.compareTo(l.array(), l.arrayOffset(), l.limit(),
379 r.array(), r.arrayOffset(), r.limit());
380 }
381
382
383
384
385
386
387
388 public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf) {
389 cacheBlock(cacheKey, buf, false, false);
390 }
391
392
393
394
395
396
397
398
399
400 protected long updateSizeMetrics(LruCachedBlock cb, boolean evict) {
401 long heapsize = cb.heapSize();
402 if (evict) {
403 heapsize *= -1;
404 }
405 return size.addAndGet(heapsize);
406 }
407
408
409
410
411
412
413
414
415
416
417 @Override
418 public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching, boolean repeat,
419 boolean updateCacheMetrics) {
420 LruCachedBlock cb = map.get(cacheKey);
421 if (cb == null) {
422 if (!repeat && updateCacheMetrics) stats.miss(caching);
423
424
425
426 if (victimHandler != null && !repeat) {
427 Cacheable result = victimHandler.getBlock(cacheKey, caching, repeat, updateCacheMetrics);
428
429
430 if (result != null && caching) {
431 cacheBlock(cacheKey, result,
432 }
433 return result;
434 }
435 return null;
436 }
437 if (updateCacheMetrics) stats.hit(caching);
438 cb.access(count.incrementAndGet());
439 return cb.getBuffer();
440 }
441
442
443
444
445
446
447 public boolean containsBlock(BlockCacheKey cacheKey) {
448 return map.containsKey(cacheKey);
449 }
450
451 @Override
452 public boolean evictBlock(BlockCacheKey cacheKey) {
453 LruCachedBlock cb = map.get(cacheKey);
454 if (cb == null) return false;
455 evictBlock(cb, false);
456 return true;
457 }
458
459
460
461
462
463
464
465
466
467
468
469 @Override
470 public int evictBlocksByHfileName(String hfileName) {
471 int numEvicted = 0;
472 for (BlockCacheKey key : map.keySet()) {
473 if (key.getHfileName().equals(hfileName)) {
474 if (evictBlock(key))
475 ++numEvicted;
476 }
477 }
478 if (victimHandler != null) {
479 numEvicted += victimHandler.evictBlocksByHfileName(hfileName);
480 }
481 return numEvicted;
482 }
483
484
485
486
487
488
489
490
491
492 protected long evictBlock(LruCachedBlock block, boolean evictedByEvictionProcess) {
493 map.remove(block.getCacheKey());
494 updateSizeMetrics(block, true);
495 long val = elements.decrementAndGet();
496 if (LOG.isTraceEnabled()) {
497 long size = map.size();
498 assertCounterSanity(size, val);
499 }
500 stats.evicted(block.getCachedTime());
501 if (evictedByEvictionProcess && victimHandler != null) {
502 if (victimHandler instanceof BucketCache) {
503 boolean wait = getCurrentSize() < acceptableSize();
504 boolean inMemory = block.getPriority() == BlockPriority.MEMORY;
505 ((BucketCache)victimHandler).cacheBlockWithWait(block.getCacheKey(), block.getBuffer(),
506 inMemory, wait);
507 } else {
508 victimHandler.cacheBlock(block.getCacheKey(), block.getBuffer());
509 }
510 }
511 return block.heapSize();
512 }
513
514
515
516
517 private void runEviction() {
518 if(evictionThread == null) {
519 evict();
520 } else {
521 evictionThread.evict();
522 }
523 }
524
525
526
527
528 void evict() {
529
530
531 if(!evictionLock.tryLock()) return;
532
533 try {
534 evictionInProgress = true;
535 long currentSize = this.size.get();
536 long bytesToFree = currentSize - minSize();
537
538 if (LOG.isTraceEnabled()) {
539 LOG.trace("Block cache LRU eviction started; Attempting to free " +
540 StringUtils.byteDesc(bytesToFree) + " of total=" +
541 StringUtils.byteDesc(currentSize));
542 }
543
544 if(bytesToFree <= 0) return;
545
546
547 BlockBucket bucketSingle = new BlockBucket("single", bytesToFree, blockSize,
548 singleSize());
549 BlockBucket bucketMulti = new BlockBucket("multi", bytesToFree, blockSize,
550 multiSize());
551 BlockBucket bucketMemory = new BlockBucket("memory", bytesToFree, blockSize,
552 memorySize());
553
554
555 for(LruCachedBlock cachedBlock : map.values()) {
556 switch(cachedBlock.getPriority()) {
557 case SINGLE: {
558 bucketSingle.add(cachedBlock);
559 break;
560 }
561 case MULTI: {
562 bucketMulti.add(cachedBlock);
563 break;
564 }
565 case MEMORY: {
566 bucketMemory.add(cachedBlock);
567 break;
568 }
569 }
570 }
571
572 long bytesFreed = 0;
573 if (forceInMemory || memoryFactor > 0.999f) {
574 long s = bucketSingle.totalSize();
575 long m = bucketMulti.totalSize();
576 if (bytesToFree > (s + m)) {
577
578
579 bytesFreed = bucketSingle.free(s);
580 bytesFreed += bucketMulti.free(m);
581 if (LOG.isTraceEnabled()) {
582 LOG.trace("freed " + StringUtils.byteDesc(bytesFreed) +
583 " from single and multi buckets");
584 }
585 bytesFreed += bucketMemory.free(bytesToFree - bytesFreed);
586 if (LOG.isTraceEnabled()) {
587 LOG.trace("freed " + StringUtils.byteDesc(bytesFreed) +
588 " total from all three buckets ");
589 }
590 } else {
591
592
593
594 long bytesRemain = s + m - bytesToFree;
595 if (3 * s <= bytesRemain) {
596
597
598 bytesFreed = bucketMulti.free(bytesToFree);
599 } else if (3 * m <= 2 * bytesRemain) {
600
601
602 bytesFreed = bucketSingle.free(bytesToFree);
603 } else {
604
605 bytesFreed = bucketSingle.free(s - bytesRemain / 3);
606 if (bytesFreed < bytesToFree) {
607 bytesFreed += bucketMulti.free(bytesToFree - bytesFreed);
608 }
609 }
610 }
611 } else {
612 PriorityQueue<BlockBucket> bucketQueue =
613 new PriorityQueue<BlockBucket>(3);
614
615 bucketQueue.add(bucketSingle);
616 bucketQueue.add(bucketMulti);
617 bucketQueue.add(bucketMemory);
618
619 int remainingBuckets = 3;
620
621 BlockBucket bucket;
622 while((bucket = bucketQueue.poll()) != null) {
623 long overflow = bucket.overflow();
624 if(overflow > 0) {
625 long bucketBytesToFree = Math.min(overflow,
626 (bytesToFree - bytesFreed) / remainingBuckets);
627 bytesFreed += bucket.free(bucketBytesToFree);
628 }
629 remainingBuckets--;
630 }
631 }
632
633 if (LOG.isTraceEnabled()) {
634 long single = bucketSingle.totalSize();
635 long multi = bucketMulti.totalSize();
636 long memory = bucketMemory.totalSize();
637 LOG.trace("Block cache LRU eviction completed; " +
638 "freed=" + StringUtils.byteDesc(bytesFreed) + ", " +
639 "total=" + StringUtils.byteDesc(this.size.get()) + ", " +
640 "single=" + StringUtils.byteDesc(single) + ", " +
641 "multi=" + StringUtils.byteDesc(multi) + ", " +
642 "memory=" + StringUtils.byteDesc(memory));
643 }
644 } finally {
645 stats.evict();
646 evictionInProgress = false;
647 evictionLock.unlock();
648 }
649 }
650
651 @Override
652 public String toString() {
653 return Objects.toStringHelper(this)
654 .add("blockCount", getBlockCount())
655 .add("currentSize", getCurrentSize())
656 .add("freeSize", getFreeSize())
657 .add("maxSize", getMaxSize())
658 .add("heapSize", heapSize())
659 .add("minSize", minSize())
660 .add("minFactor", minFactor)
661 .add("multiSize", multiSize())
662 .add("multiFactor", multiFactor)
663 .add("singleSize", singleSize())
664 .add("singleFactor", singleFactor)
665 .toString();
666 }
667
668
669
670
671
672
673
674 private class BlockBucket implements Comparable<BlockBucket> {
675
676 private final String name;
677 private LruCachedBlockQueue queue;
678 private long totalSize = 0;
679 private long bucketSize;
680
681 public BlockBucket(String name, long bytesToFree, long blockSize, long bucketSize) {
682 this.name = name;
683 this.bucketSize = bucketSize;
684 queue = new LruCachedBlockQueue(bytesToFree, blockSize);
685 totalSize = 0;
686 }
687
688 public void add(LruCachedBlock block) {
689 totalSize += block.heapSize();
690 queue.add(block);
691 }
692
693 public long free(long toFree) {
694 if (LOG.isTraceEnabled()) {
695 LOG.trace("freeing " + StringUtils.byteDesc(toFree) + " from " + this);
696 }
697 LruCachedBlock cb;
698 long freedBytes = 0;
699 while ((cb = queue.pollLast()) != null) {
700 freedBytes += evictBlock(cb, true);
701 if (freedBytes >= toFree) {
702 return freedBytes;
703 }
704 }
705 if (LOG.isTraceEnabled()) {
706 LOG.trace("freed " + StringUtils.byteDesc(freedBytes) + " from " + this);
707 }
708 return freedBytes;
709 }
710
711 public long overflow() {
712 return totalSize - bucketSize;
713 }
714
715 public long totalSize() {
716 return totalSize;
717 }
718
719 public int compareTo(BlockBucket that) {
720 return Long.compare(this.overflow(), that.overflow());
721 }
722
723 @Override
724 public boolean equals(Object that) {
725 if (that == null || !(that instanceof BlockBucket)){
726 return false;
727 }
728 return compareTo((BlockBucket)that) == 0;
729 }
730
731 @Override
732 public int hashCode() {
733 return Objects.hashCode(name, bucketSize, queue, totalSize);
734 }
735
736 @Override
737 public String toString() {
738 return Objects.toStringHelper(this)
739 .add("name", name)
740 .add("totalSize", StringUtils.byteDesc(totalSize))
741 .add("bucketSize", StringUtils.byteDesc(bucketSize))
742 .toString();
743 }
744 }
745
746
747
748
749
750 public long getMaxSize() {
751 return this.maxSize;
752 }
753
754 @Override
755 public long getCurrentSize() {
756 return this.size.get();
757 }
758
759 @Override
760 public long getFreeSize() {
761 return getMaxSize() - getCurrentSize();
762 }
763
764 @Override
765 public long size() {
766 return getMaxSize();
767 }
768
769 @Override
770 public long getBlockCount() {
771 return this.elements.get();
772 }
773
774 EvictionThread getEvictionThread() {
775 return this.evictionThread;
776 }
777
778
779
780
781
782
783
784 static class EvictionThread extends HasThread {
785 private WeakReference<LruBlockCache> cache;
786 private volatile boolean go = true;
787
788 private boolean enteringRun = false;
789
790 public EvictionThread(LruBlockCache cache) {
791 super(Thread.currentThread().getName() + ".LruBlockCache.EvictionThread");
792 setDaemon(true);
793 this.cache = new WeakReference<LruBlockCache>(cache);
794 }
795
796 @Override
797 public void run() {
798 enteringRun = true;
799 while (this.go) {
800 synchronized(this) {
801 try {
802 this.wait(1000 * 10
803 } catch(InterruptedException e) {
804 LOG.warn("Interrupted eviction thread ", e);
805 Thread.currentThread().interrupt();
806 }
807 }
808 LruBlockCache cache = this.cache.get();
809 if (cache == null) break;
810 cache.evict();
811 }
812 }
813
814 @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="NN_NAKED_NOTIFY",
815 justification="This is what we want")
816 public void evict() {
817 synchronized(this) {
818 this.notifyAll();
819 }
820 }
821
822 synchronized void shutdown() {
823 this.go = false;
824 this.notifyAll();
825 }
826
827
828
829
830 boolean isEnteringRun() {
831 return this.enteringRun;
832 }
833 }
834
835
836
837
838 static class StatisticsThread extends Thread {
839 private final LruBlockCache lru;
840
841 public StatisticsThread(LruBlockCache lru) {
842 super("LruBlockCacheStats");
843 setDaemon(true);
844 this.lru = lru;
845 }
846
847 @Override
848 public void run() {
849 lru.logStats();
850 }
851 }
852
853 public void logStats() {
854
855 long totalSize = heapSize();
856 long freeSize = maxSize - totalSize;
857 LruBlockCache.LOG.info("totalSize=" + StringUtils.byteDesc(totalSize) + ", " +
858 "freeSize=" + StringUtils.byteDesc(freeSize) + ", " +
859 "max=" + StringUtils.byteDesc(this.maxSize) + ", " +
860 "blockCount=" + getBlockCount() + ", " +
861 "accesses=" + stats.getRequestCount() + ", " +
862 "hits=" + stats.getHitCount() + ", " +
863 "hitRatio=" + (stats.getHitCount() == 0 ?
864 "0" : (StringUtils.formatPercent(stats.getHitRatio(), 2)+ ", ")) + ", " +
865 "cachingAccesses=" + stats.getRequestCachingCount() + ", " +
866 "cachingHits=" + stats.getHitCachingCount() + ", " +
867 "cachingHitsRatio=" + (stats.getHitCachingCount() == 0 ?
868 "0,": (StringUtils.formatPercent(stats.getHitCachingRatio(), 2) + ", ")) +
869 "evictions=" + stats.getEvictionCount() + ", " +
870 "evicted=" + stats.getEvictedCount() + ", " +
871 "evictedPerRun=" + stats.evictedPerEviction());
872 }
873
874
875
876
877
878
879
880 public CacheStats getStats() {
881 return this.stats;
882 }
883
884 public final static long CACHE_FIXED_OVERHEAD = ClassSize.align(
885 (3 * Bytes.SIZEOF_LONG) + (9 * ClassSize.REFERENCE) +
886 (5 * Bytes.SIZEOF_FLOAT) + Bytes.SIZEOF_BOOLEAN
887 + ClassSize.OBJECT);
888
889 @Override
890 public long heapSize() {
891 return getCurrentSize();
892 }
893
894 public static long calculateOverhead(long maxSize, long blockSize, int concurrency){
895
896 return CACHE_FIXED_OVERHEAD + ClassSize.CONCURRENT_HASHMAP +
897 ((long)Math.ceil(maxSize*1.2/blockSize)
898 * ClassSize.CONCURRENT_HASHMAP_ENTRY) +
899 ((long)concurrency * ClassSize.CONCURRENT_HASHMAP_SEGMENT);
900 }
901
902 @Override
903 public Iterator<CachedBlock> iterator() {
904 final Iterator<LruCachedBlock> iterator = map.values().iterator();
905
906 return new Iterator<CachedBlock>() {
907 private final long now = System.nanoTime();
908
909 @Override
910 public boolean hasNext() {
911 return iterator.hasNext();
912 }
913
914 @Override
915 public CachedBlock next() {
916 final LruCachedBlock b = iterator.next();
917 return new CachedBlock() {
918 @Override
919 public String toString() {
920 return BlockCacheUtil.toString(this, now);
921 }
922
923 @Override
924 public BlockPriority getBlockPriority() {
925 return b.getPriority();
926 }
927
928 @Override
929 public BlockType getBlockType() {
930 return b.getBuffer().getBlockType();
931 }
932
933 @Override
934 public long getOffset() {
935 return b.getCacheKey().getOffset();
936 }
937
938 @Override
939 public long getSize() {
940 return b.getBuffer().heapSize();
941 }
942
943 @Override
944 public long getCachedTime() {
945 return b.getCachedTime();
946 }
947
948 @Override
949 public String getFilename() {
950 return b.getCacheKey().getHfileName();
951 }
952
953 @Override
954 public int compareTo(CachedBlock other) {
955 int diff = this.getFilename().compareTo(other.getFilename());
956 if (diff != 0) return diff;
957 diff = Long.compare(this.getOffset(), other.getOffset());
958 if (diff != 0) return diff;
959 if (other.getCachedTime() < 0 || this.getCachedTime() < 0) {
960 throw new IllegalStateException("" + this.getCachedTime() + ", " +
961 other.getCachedTime());
962 }
963 return Long.compare(other.getCachedTime(), this.getCachedTime());
964 }
965
966 @Override
967 public int hashCode() {
968 return b.hashCode();
969 }
970
971 @Override
972 public boolean equals(Object obj) {
973 if (obj instanceof CachedBlock) {
974 CachedBlock cb = (CachedBlock)obj;
975 return compareTo(cb) == 0;
976 } else {
977 return false;
978 }
979 }
980 };
981 }
982
983 @Override
984 public void remove() {
985 throw new UnsupportedOperationException();
986 }
987 };
988 }
989
990
991
992 long acceptableSize() {
993 return (long)Math.floor(this.maxSize * this.acceptableFactor);
994 }
995 private long minSize() {
996 return (long)Math.floor(this.maxSize * this.minFactor);
997 }
998 private long singleSize() {
999 return (long)Math.floor(this.maxSize * this.singleFactor * this.minFactor);
1000 }
1001 private long multiSize() {
1002 return (long)Math.floor(this.maxSize * this.multiFactor * this.minFactor);
1003 }
1004 private long memorySize() {
1005 return (long)Math.floor(this.maxSize * this.memoryFactor * this.minFactor);
1006 }
1007
1008 public void shutdown() {
1009 if (victimHandler != null)
1010 victimHandler.shutdown();
1011 this.scheduleThreadPool.shutdown();
1012 for (int i = 0; i < 10; i++) {
1013 if (!this.scheduleThreadPool.isShutdown()) {
1014 try {
1015 Thread.sleep(10);
1016 } catch (InterruptedException e) {
1017 LOG.warn("Interrupted while sleeping");
1018 Thread.currentThread().interrupt();
1019 break;
1020 }
1021 }
1022 }
1023
1024 if (!this.scheduleThreadPool.isShutdown()) {
1025 List<Runnable> runnables = this.scheduleThreadPool.shutdownNow();
1026 LOG.debug("Still running " + runnables);
1027 }
1028 this.evictionThread.shutdown();
1029 }
1030
1031
1032 @VisibleForTesting
1033 public void clearCache() {
1034 this.map.clear();
1035 this.elements.set(0);
1036 }
1037
1038
1039
1040
1041
1042 @VisibleForTesting
1043 SortedSet<String> getCachedFileNamesForTest() {
1044 SortedSet<String> fileNames = new TreeSet<String>();
1045 for (BlockCacheKey cacheKey : map.keySet()) {
1046 fileNames.add(cacheKey.getHfileName());
1047 }
1048 return fileNames;
1049 }
1050
1051 @VisibleForTesting
1052 Map<BlockType, Integer> getBlockTypeCountsForTest() {
1053 Map<BlockType, Integer> counts =
1054 new EnumMap<BlockType, Integer>(BlockType.class);
1055 for (LruCachedBlock cb : map.values()) {
1056 BlockType blockType = ((Cacheable)cb.getBuffer()).getBlockType();
1057 Integer count = counts.get(blockType);
1058 counts.put(blockType, (count == null ? 0 : count) + 1);
1059 }
1060 return counts;
1061 }
1062
1063 @VisibleForTesting
1064 public Map<DataBlockEncoding, Integer> getEncodingCountsForTest() {
1065 Map<DataBlockEncoding, Integer> counts =
1066 new EnumMap<DataBlockEncoding, Integer>(DataBlockEncoding.class);
1067 for (LruCachedBlock block : map.values()) {
1068 DataBlockEncoding encoding =
1069 ((HFileBlock) block.getBuffer()).getDataBlockEncoding();
1070 Integer count = counts.get(encoding);
1071 counts.put(encoding, (count == null ? 0 : count) + 1);
1072 }
1073 return counts;
1074 }
1075
1076 public void setVictimCache(BlockCache handler) {
1077 assert victimHandler == null;
1078 victimHandler = handler;
1079 }
1080
1081 @VisibleForTesting
1082 Map<BlockCacheKey, LruCachedBlock> getMapForTests() {
1083 return map;
1084 }
1085
1086 BlockCache getVictimHandler() {
1087 return this.victimHandler;
1088 }
1089
1090 @Override
1091 public BlockCache[] getBlockCaches() {
1092 return null;
1093 }
1094 }