1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.io.hfile;
19
20 import static org.apache.hadoop.hbase.HConstants.BUCKET_CACHE_IOENGINE_KEY;
21 import static org.apache.hadoop.hbase.HConstants.BUCKET_CACHE_SIZE_KEY;
22
23 import java.io.IOException;
24 import java.lang.management.ManagementFactory;
25 import java.lang.management.MemoryUsage;
26
27 import org.apache.commons.logging.Log;
28 import org.apache.commons.logging.LogFactory;
29 import org.apache.hadoop.hbase.classification.InterfaceAudience;
30 import org.apache.hadoop.conf.Configuration;
31 import org.apache.hadoop.hbase.HColumnDescriptor;
32 import org.apache.hadoop.hbase.HConstants;
33 import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory;
34 import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache;
35 import org.apache.hadoop.hbase.util.ReflectionUtils;
36 import org.apache.hadoop.util.StringUtils;
37
38 import com.google.common.annotations.VisibleForTesting;
39
40
41
42
43 @InterfaceAudience.Private
44 public class CacheConfig {
45 private static final Log LOG = LogFactory.getLog(CacheConfig.class.getName());
46
47
48
49
50
51 public static final String CACHE_BLOCKS_ON_WRITE_KEY =
52 "hbase.rs.cacheblocksonwrite";
53
54
55
56
57
58 public static final String CACHE_INDEX_BLOCKS_ON_WRITE_KEY =
59 "hfile.block.index.cacheonwrite";
60
61
62
63
64 public static final String CACHE_BLOOM_BLOCKS_ON_WRITE_KEY =
65 "hfile.block.bloom.cacheonwrite";
66
67
68
69
70 public static final String CACHE_DATA_BLOCKS_COMPRESSED_KEY =
71 "hbase.block.data.cachecompressed";
72
73
74
75
76
77 public static final String EVICT_BLOCKS_ON_CLOSE_KEY =
78 "hbase.rs.evictblocksonclose";
79
80
81
82
83
84
85
86
87
88 public static final String BUCKET_CACHE_PERSISTENT_PATH_KEY =
89 "hbase.bucketcache.persistent.path";
90
91
92
93
94
95
96 public static final String BUCKET_CACHE_COMBINED_KEY =
97 "hbase.bucketcache.combinedcache.enabled";
98
99 public static final String BUCKET_CACHE_WRITER_THREADS_KEY = "hbase.bucketcache.writer.threads";
100 public static final String BUCKET_CACHE_WRITER_QUEUE_KEY =
101 "hbase.bucketcache.writer.queuelength";
102
103
104
105
106 public static final String BUCKET_CACHE_BUCKETS_KEY = "hbase.bucketcache.bucket.sizes";
107
108
109
110
111 public static final boolean DEFAULT_BUCKET_CACHE_COMBINED = true;
112 public static final int DEFAULT_BUCKET_CACHE_WRITER_THREADS = 3;
113 public static final int DEFAULT_BUCKET_CACHE_WRITER_QUEUE = 64;
114
115
116
117
118
119 public static final String PREFETCH_BLOCKS_ON_OPEN_KEY =
120 "hbase.rs.prefetchblocksonopen";
121
122
123
124
125
126
127
128 public static final String BLOCKCACHE_BLOCKSIZE_KEY = "hbase.offheapcache.minblocksize";
129
130 private static final String EXTERNAL_BLOCKCACHE_KEY = "hbase.blockcache.use.external";
131 private static final boolean EXTERNAL_BLOCKCACHE_DEFAULT = false;
132
133 private static final String EXTERNAL_BLOCKCACHE_CLASS_KEY="hbase.blockcache.external.class";
134
135
136
137
138
139 private static enum ExternalBlockCaches {
140 memcached(MemcachedBlockCache.class);
141
142 Class<? extends BlockCache> clazz;
143 ExternalBlockCaches(Class<? extends BlockCache> clazz) {
144 this.clazz = clazz;
145 }
146 }
147
148
149 public static final boolean DEFAULT_CACHE_DATA_ON_READ = true;
150 public static final boolean DEFAULT_CACHE_DATA_ON_WRITE = false;
151 public static final boolean DEFAULT_IN_MEMORY = false;
152 public static final boolean DEFAULT_CACHE_INDEXES_ON_WRITE = false;
153 public static final boolean DEFAULT_CACHE_BLOOMS_ON_WRITE = false;
154 public static final boolean DEFAULT_EVICT_ON_CLOSE = false;
155 public static final boolean DEFAULT_CACHE_DATA_COMPRESSED = false;
156 public static final boolean DEFAULT_PREFETCH_ON_OPEN = false;
157
158
159 private final BlockCache blockCache;
160
161
162
163
164
165
166
167 private boolean cacheDataOnRead;
168
169
170 private final boolean inMemory;
171
172
173 private boolean cacheDataOnWrite;
174
175
176 private final boolean cacheIndexesOnWrite;
177
178
179 private final boolean cacheBloomsOnWrite;
180
181
182 private boolean evictOnClose;
183
184
185 private final boolean cacheDataCompressed;
186
187
188 private final boolean prefetchOnOpen;
189
190
191
192
193
194
195 private boolean cacheDataInL1;
196
197
198
199
200
201
202
203 public CacheConfig(Configuration conf, HColumnDescriptor family) {
204 this(CacheConfig.instantiateBlockCache(conf),
205 family.isBlockCacheEnabled(),
206 family.isInMemory(),
207
208
209 conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY,
210 DEFAULT_CACHE_DATA_ON_WRITE) || family.isCacheDataOnWrite(),
211 conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY,
212 DEFAULT_CACHE_INDEXES_ON_WRITE) || family.isCacheIndexesOnWrite(),
213 conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY,
214 DEFAULT_CACHE_BLOOMS_ON_WRITE) || family.isCacheBloomsOnWrite(),
215 conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY,
216 DEFAULT_EVICT_ON_CLOSE) || family.isEvictBlocksOnClose(),
217 conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, DEFAULT_CACHE_DATA_COMPRESSED),
218 conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY,
219 DEFAULT_PREFETCH_ON_OPEN) || family.isPrefetchBlocksOnOpen(),
220 conf.getBoolean(HColumnDescriptor.CACHE_DATA_IN_L1,
221 HColumnDescriptor.DEFAULT_CACHE_DATA_IN_L1) || family.isCacheDataInL1()
222 );
223 }
224
225
226
227
228
229
230 public CacheConfig(Configuration conf) {
231 this(CacheConfig.instantiateBlockCache(conf),
232 DEFAULT_CACHE_DATA_ON_READ,
233 DEFAULT_IN_MEMORY,
234
235 conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_DATA_ON_WRITE),
236 conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_INDEXES_ON_WRITE),
237 conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_BLOOMS_ON_WRITE),
238 conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, DEFAULT_EVICT_ON_CLOSE),
239 conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, DEFAULT_CACHE_DATA_COMPRESSED),
240 conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY, DEFAULT_PREFETCH_ON_OPEN),
241 conf.getBoolean(HColumnDescriptor.CACHE_DATA_IN_L1,
242 HColumnDescriptor.DEFAULT_CACHE_DATA_IN_L1)
243 );
244 }
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262 CacheConfig(final BlockCache blockCache,
263 final boolean cacheDataOnRead, final boolean inMemory,
264 final boolean cacheDataOnWrite, final boolean cacheIndexesOnWrite,
265 final boolean cacheBloomsOnWrite, final boolean evictOnClose,
266 final boolean cacheDataCompressed, final boolean prefetchOnOpen,
267 final boolean cacheDataInL1) {
268 this.blockCache = blockCache;
269 this.cacheDataOnRead = cacheDataOnRead;
270 this.inMemory = inMemory;
271 this.cacheDataOnWrite = cacheDataOnWrite;
272 this.cacheIndexesOnWrite = cacheIndexesOnWrite;
273 this.cacheBloomsOnWrite = cacheBloomsOnWrite;
274 this.evictOnClose = evictOnClose;
275 this.cacheDataCompressed = cacheDataCompressed;
276 this.prefetchOnOpen = prefetchOnOpen;
277 this.cacheDataInL1 = cacheDataInL1;
278 LOG.info(this);
279 }
280
281
282
283
284
285 public CacheConfig(CacheConfig cacheConf) {
286 this(cacheConf.blockCache, cacheConf.cacheDataOnRead, cacheConf.inMemory,
287 cacheConf.cacheDataOnWrite, cacheConf.cacheIndexesOnWrite,
288 cacheConf.cacheBloomsOnWrite, cacheConf.evictOnClose,
289 cacheConf.cacheDataCompressed, cacheConf.prefetchOnOpen,
290 cacheConf.cacheDataInL1);
291 }
292
293
294
295
296 public boolean isBlockCacheEnabled() {
297 return this.blockCache != null;
298 }
299
300
301
302
303
304 public BlockCache getBlockCache() {
305 return this.blockCache;
306 }
307
308
309
310
311
312
313 public boolean shouldCacheDataOnRead() {
314 return isBlockCacheEnabled() && cacheDataOnRead;
315 }
316
317
318
319
320
321
322 public boolean shouldCacheBlockOnRead(BlockCategory category) {
323 return isBlockCacheEnabled()
324 && (cacheDataOnRead ||
325 category == BlockCategory.INDEX ||
326 category == BlockCategory.BLOOM ||
327 (prefetchOnOpen &&
328 (category != BlockCategory.META &&
329 category != BlockCategory.UNKNOWN)));
330 }
331
332
333
334
335 public boolean isInMemory() {
336 return isBlockCacheEnabled() && this.inMemory;
337 }
338
339
340
341
342 public boolean isCacheDataInL1() {
343 return isBlockCacheEnabled() && this.cacheDataInL1;
344 }
345
346
347
348
349
350 public boolean shouldCacheDataOnWrite() {
351 return isBlockCacheEnabled() && this.cacheDataOnWrite;
352 }
353
354
355
356
357
358
359 @VisibleForTesting
360 public void setCacheDataOnWrite(boolean cacheDataOnWrite) {
361 this.cacheDataOnWrite = cacheDataOnWrite;
362 }
363
364
365
366
367
368
369 @VisibleForTesting
370 public void setCacheDataInL1(boolean cacheDataInL1) {
371 this.cacheDataInL1 = cacheDataInL1;
372 }
373
374
375
376
377
378 public boolean shouldCacheIndexesOnWrite() {
379 return isBlockCacheEnabled() && this.cacheIndexesOnWrite;
380 }
381
382
383
384
385
386 public boolean shouldCacheBloomsOnWrite() {
387 return isBlockCacheEnabled() && this.cacheBloomsOnWrite;
388 }
389
390
391
392
393
394 public boolean shouldEvictOnClose() {
395 return isBlockCacheEnabled() && this.evictOnClose;
396 }
397
398
399
400
401
402
403 public void setEvictOnClose(boolean evictOnClose) {
404 this.evictOnClose = evictOnClose;
405 }
406
407
408
409
410 public boolean shouldCacheDataCompressed() {
411 return isBlockCacheEnabled() && this.cacheDataCompressed;
412 }
413
414
415
416
417 public boolean shouldCacheCompressed(BlockCategory category) {
418 if (!isBlockCacheEnabled()) return false;
419 switch (category) {
420 case DATA:
421 return this.cacheDataCompressed;
422 default:
423 return false;
424 }
425 }
426
427
428
429
430 public boolean shouldPrefetchOnOpen() {
431 return isBlockCacheEnabled() && this.prefetchOnOpen;
432 }
433
434
435
436
437
438
439
440
441
442 public boolean shouldReadBlockFromCache(BlockType blockType) {
443 if (!isBlockCacheEnabled()) {
444 return false;
445 }
446 if (cacheDataOnRead) {
447 return true;
448 }
449 if (prefetchOnOpen) {
450 return true;
451 }
452 if (cacheDataOnWrite) {
453 return true;
454 }
455 if (blockType == null) {
456 return true;
457 }
458 if (blockType.getCategory() == BlockCategory.BLOOM ||
459 blockType.getCategory() == BlockCategory.INDEX) {
460 return true;
461 }
462 return false;
463 }
464
465
466
467
468
469 public boolean shouldLockOnCacheMiss(BlockType blockType) {
470 if (blockType == null) {
471 return true;
472 }
473 return shouldCacheBlockOnRead(blockType.getCategory());
474 }
475
476 @Override
477 public String toString() {
478 if (!isBlockCacheEnabled()) {
479 return "CacheConfig:disabled";
480 }
481 return "blockCache=" + getBlockCache() +
482 ", cacheDataOnRead=" + shouldCacheDataOnRead() +
483 ", cacheDataOnWrite=" + shouldCacheDataOnWrite() +
484 ", cacheIndexesOnWrite=" + shouldCacheIndexesOnWrite() +
485 ", cacheBloomsOnWrite=" + shouldCacheBloomsOnWrite() +
486 ", cacheEvictOnClose=" + shouldEvictOnClose() +
487 ", cacheDataCompressed=" + shouldCacheDataCompressed() +
488 ", prefetchOnOpen=" + shouldPrefetchOnOpen();
489 }
490
491
492
493
494
495
496
497
498 @VisibleForTesting
499 static BlockCache GLOBAL_BLOCK_CACHE_INSTANCE;
500
501
502 @VisibleForTesting
503 static boolean blockCacheDisabled = false;
504
505 static long getLruCacheSize(final Configuration conf, final MemoryUsage mu) {
506 float cachePercentage = conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY,
507 HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
508 if (cachePercentage <= 0.0001f) {
509 blockCacheDisabled = true;
510 return -1;
511 }
512 if (cachePercentage > 1.0) {
513 throw new IllegalArgumentException(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY +
514 " must be between 0.0 and 1.0, and not > 1.0");
515 }
516
517
518 return (long) (mu.getMax() * cachePercentage);
519 }
520
521
522
523
524
525
526 private static LruBlockCache getL1(final Configuration c, final MemoryUsage mu) {
527 long lruCacheSize = getLruCacheSize(c, mu);
528 if (lruCacheSize < 0) return null;
529 int blockSize = c.getInt(BLOCKCACHE_BLOCKSIZE_KEY, HConstants.DEFAULT_BLOCKSIZE);
530 LOG.info("Allocating LruBlockCache size=" +
531 StringUtils.byteDesc(lruCacheSize) + ", blockSize=" + StringUtils.byteDesc(blockSize));
532 return new LruBlockCache(lruCacheSize, blockSize, true, c);
533 }
534
535
536
537
538
539
540
541 private static BlockCache getL2(final Configuration c, final MemoryUsage mu) {
542 final boolean useExternal = c.getBoolean(EXTERNAL_BLOCKCACHE_KEY, EXTERNAL_BLOCKCACHE_DEFAULT);
543 if (LOG.isDebugEnabled()) {
544 LOG.debug("Trying to use " + (useExternal?" External":" Internal") + " l2 cache");
545 }
546
547
548 if (useExternal) {
549 return getExternalBlockcache(c);
550 }
551
552
553 return getBucketCache(c, mu);
554
555 }
556
557 private static BlockCache getExternalBlockcache(Configuration c) {
558 Class klass = null;
559
560
561 try {
562 klass = ExternalBlockCaches.valueOf(c.get(EXTERNAL_BLOCKCACHE_CLASS_KEY, "memcache")).clazz;
563 } catch (IllegalArgumentException exception) {
564 klass = c.getClass(EXTERNAL_BLOCKCACHE_CLASS_KEY, MemcachedBlockCache.class);
565 }
566
567
568 try {
569 LOG.info("Creating external block cache of type: " + klass);
570 return (BlockCache) ReflectionUtils.newInstance(klass, c);
571 } catch (Exception e) {
572 LOG.warn("Error creating external block cache", e);
573 }
574 return null;
575
576 }
577
578 private static BlockCache getBucketCache(Configuration c, MemoryUsage mu) {
579
580 String bucketCacheIOEngineName = c.get(BUCKET_CACHE_IOENGINE_KEY, null);
581 if (bucketCacheIOEngineName == null || bucketCacheIOEngineName.length() <= 0) return null;
582
583 int blockSize = c.getInt(BLOCKCACHE_BLOCKSIZE_KEY, HConstants.DEFAULT_BLOCKSIZE);
584 float bucketCachePercentage = c.getFloat(BUCKET_CACHE_SIZE_KEY, 0F);
585 long bucketCacheSize = (long) (bucketCachePercentage < 1? mu.getMax() * bucketCachePercentage:
586 bucketCachePercentage * 1024 * 1024);
587 if (bucketCacheSize <= 0) {
588 throw new IllegalStateException("bucketCacheSize <= 0; Check " +
589 BUCKET_CACHE_SIZE_KEY + " setting and/or server java heap size");
590 }
591 if (c.get("hbase.bucketcache.percentage.in.combinedcache") != null) {
592 LOG.warn("Configuration 'hbase.bucketcache.percentage.in.combinedcache' is no longer "
593 + "respected. See comments in http://hbase.apache.org/book.html#_changes_of_note");
594 }
595 int writerThreads = c.getInt(BUCKET_CACHE_WRITER_THREADS_KEY,
596 DEFAULT_BUCKET_CACHE_WRITER_THREADS);
597 int writerQueueLen = c.getInt(BUCKET_CACHE_WRITER_QUEUE_KEY,
598 DEFAULT_BUCKET_CACHE_WRITER_QUEUE);
599 String persistentPath = c.get(BUCKET_CACHE_PERSISTENT_PATH_KEY);
600 String[] configuredBucketSizes = c.getStrings(BUCKET_CACHE_BUCKETS_KEY);
601 int [] bucketSizes = null;
602 if (configuredBucketSizes != null) {
603 bucketSizes = new int[configuredBucketSizes.length];
604 for (int i = 0; i < configuredBucketSizes.length; i++) {
605 bucketSizes[i] = Integer.parseInt(configuredBucketSizes[i].trim());
606 }
607 }
608 BucketCache bucketCache = null;
609 try {
610 int ioErrorsTolerationDuration = c.getInt(
611 "hbase.bucketcache.ioengine.errors.tolerated.duration",
612 BucketCache.DEFAULT_ERROR_TOLERATION_DURATION);
613
614 bucketCache = new BucketCache(bucketCacheIOEngineName,
615 bucketCacheSize, blockSize, bucketSizes, writerThreads, writerQueueLen, persistentPath,
616 ioErrorsTolerationDuration);
617 } catch (IOException ioex) {
618 LOG.error("Can't instantiate bucket cache", ioex); throw new RuntimeException(ioex);
619 }
620 return bucketCache;
621 }
622
623
624
625
626
627
628
629
630 public static synchronized BlockCache instantiateBlockCache(Configuration conf) {
631 if (GLOBAL_BLOCK_CACHE_INSTANCE != null) return GLOBAL_BLOCK_CACHE_INSTANCE;
632 if (blockCacheDisabled) return null;
633 MemoryUsage mu = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
634 LruBlockCache l1 = getL1(conf, mu);
635
636 if (blockCacheDisabled) return null;
637 BlockCache l2 = getL2(conf, mu);
638 if (l2 == null) {
639 GLOBAL_BLOCK_CACHE_INSTANCE = l1;
640 } else {
641 boolean useExternal = conf.getBoolean(EXTERNAL_BLOCKCACHE_KEY, EXTERNAL_BLOCKCACHE_DEFAULT);
642 boolean combinedWithLru = conf.getBoolean(BUCKET_CACHE_COMBINED_KEY,
643 DEFAULT_BUCKET_CACHE_COMBINED);
644 if (useExternal) {
645 GLOBAL_BLOCK_CACHE_INSTANCE = new InclusiveCombinedBlockCache(l1, l2);
646 } else {
647 if (combinedWithLru) {
648 GLOBAL_BLOCK_CACHE_INSTANCE = new CombinedBlockCache(l1, l2);
649 } else {
650
651
652
653
654 GLOBAL_BLOCK_CACHE_INSTANCE = l1;
655 }
656 }
657 l1.setVictimCache(l2);
658 }
659 return GLOBAL_BLOCK_CACHE_INSTANCE;
660 }
661 }