View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.io.hfile;
19  
20  import static org.apache.hadoop.hbase.HConstants.BUCKET_CACHE_IOENGINE_KEY;
21  import static org.apache.hadoop.hbase.HConstants.BUCKET_CACHE_SIZE_KEY;
22  
23  import java.io.IOException;
24  import java.lang.management.ManagementFactory;
25  import java.lang.management.MemoryUsage;
26  
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.hadoop.hbase.classification.InterfaceAudience;
30  import org.apache.hadoop.conf.Configuration;
31  import org.apache.hadoop.hbase.HColumnDescriptor;
32  import org.apache.hadoop.hbase.HConstants;
33  import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory;
34  import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache;
35  import org.apache.hadoop.hbase.util.ReflectionUtils;
36  import org.apache.hadoop.util.StringUtils;
37  
38  import com.google.common.annotations.VisibleForTesting;
39  
40  /**
41   * Stores all of the cache objects and configuration for a single HFile.
42   */
43  @InterfaceAudience.Private
44  public class CacheConfig {
45    private static final Log LOG = LogFactory.getLog(CacheConfig.class.getName());
46  
47    /**
48     * Configuration key to cache data blocks on write. There are separate
49     * switches for bloom blocks and non-root index blocks.
50     */
51    public static final String CACHE_BLOCKS_ON_WRITE_KEY =
52        "hbase.rs.cacheblocksonwrite";
53  
54    /**
55     * Configuration key to cache leaf and intermediate-level index blocks on
56     * write.
57     */
58    public static final String CACHE_INDEX_BLOCKS_ON_WRITE_KEY =
59        "hfile.block.index.cacheonwrite";
60  
61    /**
62     * Configuration key to cache compound bloom filter blocks on write.
63     */
64    public static final String CACHE_BLOOM_BLOCKS_ON_WRITE_KEY =
65        "hfile.block.bloom.cacheonwrite";
66  
67    /**
68     * Configuration key to cache data blocks in compressed and/or encrypted format.
69     */
70    public static final String CACHE_DATA_BLOCKS_COMPRESSED_KEY =
71        "hbase.block.data.cachecompressed";
72  
73    /**
74     * Configuration key to evict all blocks of a given file from the block cache
75     * when the file is closed.
76     */
77    public static final String EVICT_BLOCKS_ON_CLOSE_KEY =
78        "hbase.rs.evictblocksonclose";
79  
80    /**
81     * Configuration keys for Bucket cache
82     */
83  
84    /**
85     * If the chosen ioengine can persist its state across restarts, the path to the file to
86     * persist to.
87     */
88    public static final String BUCKET_CACHE_PERSISTENT_PATH_KEY = 
89        "hbase.bucketcache.persistent.path";
90  
91    /**
92     * If the bucket cache is used in league with the lru on-heap block cache (meta blocks such
93     * as indices and blooms are kept in the lru blockcache and the data blocks in the
94     * bucket cache).
95     */
96    public static final String BUCKET_CACHE_COMBINED_KEY = 
97        "hbase.bucketcache.combinedcache.enabled";
98  
99    public static final String BUCKET_CACHE_WRITER_THREADS_KEY = "hbase.bucketcache.writer.threads";
100   public static final String BUCKET_CACHE_WRITER_QUEUE_KEY = 
101       "hbase.bucketcache.writer.queuelength";
102 
103   /**
104    * A comma-delimited array of values for use as bucket sizes.
105    */
106   public static final String BUCKET_CACHE_BUCKETS_KEY = "hbase.bucketcache.bucket.sizes";
107 
108   /**
109    * Defaults for Bucket cache
110    */
111   public static final boolean DEFAULT_BUCKET_CACHE_COMBINED = true;
112   public static final int DEFAULT_BUCKET_CACHE_WRITER_THREADS = 3;
113   public static final int DEFAULT_BUCKET_CACHE_WRITER_QUEUE = 64;
114 
115  /**
116    * Configuration key to prefetch all blocks of a given file into the block cache
117    * when the file is opened.
118    */
119   public static final String PREFETCH_BLOCKS_ON_OPEN_KEY =
120       "hbase.rs.prefetchblocksonopen";
121 
122   /**
123    * The target block size used by blockcache instances. Defaults to
124    * {@link HConstants#DEFAULT_BLOCKSIZE}.
125    * TODO: this config point is completely wrong, as it's used to determine the
126    * target block size of BlockCache instances. Rename.
127    */
128   public static final String BLOCKCACHE_BLOCKSIZE_KEY = "hbase.offheapcache.minblocksize";
129 
130   private static final String EXTERNAL_BLOCKCACHE_KEY = "hbase.blockcache.use.external";
131   private static final boolean EXTERNAL_BLOCKCACHE_DEFAULT = false;
132 
133   private static final String EXTERNAL_BLOCKCACHE_CLASS_KEY="hbase.blockcache.external.class";
134 
135   /**
136    * Enum of all built in external block caches.
137    * This is used for config.
138    */
139   private static enum ExternalBlockCaches {
140     memcached(MemcachedBlockCache.class);
141     // TODO(eclark): Consider more. Redis, etc.
142     Class<? extends BlockCache> clazz;
143     ExternalBlockCaches(Class<? extends BlockCache> clazz) {
144       this.clazz = clazz;
145     }
146   }
147 
148   // Defaults
149   public static final boolean DEFAULT_CACHE_DATA_ON_READ = true;
150   public static final boolean DEFAULT_CACHE_DATA_ON_WRITE = false;
151   public static final boolean DEFAULT_IN_MEMORY = false;
152   public static final boolean DEFAULT_CACHE_INDEXES_ON_WRITE = false;
153   public static final boolean DEFAULT_CACHE_BLOOMS_ON_WRITE = false;
154   public static final boolean DEFAULT_EVICT_ON_CLOSE = false;
155   public static final boolean DEFAULT_CACHE_DATA_COMPRESSED = false;
156   public static final boolean DEFAULT_PREFETCH_ON_OPEN = false;
157 
158   /** Local reference to the block cache, null if completely disabled */
159   private final BlockCache blockCache;
160 
161   /**
162    * Whether blocks should be cached on read (default is on if there is a
163    * cache but this can be turned off on a per-family or per-request basis).
164    * If off we will STILL cache meta blocks; i.e. INDEX and BLOOM types.
165    * This cannot be disabled.
166    */
167   private boolean cacheDataOnRead;
168 
169   /** Whether blocks should be flagged as in-memory when being cached */
170   private final boolean inMemory;
171 
172   /** Whether data blocks should be cached when new files are written */
173   private boolean cacheDataOnWrite;
174 
175   /** Whether index blocks should be cached when new files are written */
176   private final boolean cacheIndexesOnWrite;
177 
178   /** Whether compound bloom filter blocks should be cached on write */
179   private final boolean cacheBloomsOnWrite;
180 
181   /** Whether blocks of a file should be evicted when the file is closed */
182   private boolean evictOnClose;
183 
184   /** Whether data blocks should be stored in compressed and/or encrypted form in the cache */
185   private final boolean cacheDataCompressed;
186 
187   /** Whether data blocks should be prefetched into the cache */
188   private final boolean prefetchOnOpen;
189 
190   /**
191    * If true and if more than one tier in this cache deploy -- e.g. CombinedBlockCache has an L1
192    * and an L2 tier -- then cache data blocks up in the L1 tier (The meta blocks are likely being
193    * cached up in L1 already.  At least this is the case if CombinedBlockCache).
194    */
195   private boolean cacheDataInL1;
196 
197   /**
198    * Create a cache configuration using the specified configuration object and
199    * family descriptor.
200    * @param conf hbase configuration
201    * @param family column family configuration
202    */
203   public CacheConfig(Configuration conf, HColumnDescriptor family) {
204     this(CacheConfig.instantiateBlockCache(conf),
205         family.isBlockCacheEnabled(),
206         family.isInMemory(),
207         // For the following flags we enable them regardless of per-schema settings
208         // if they are enabled in the global configuration.
209         conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY,
210             DEFAULT_CACHE_DATA_ON_WRITE) || family.isCacheDataOnWrite(),
211         conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY,
212             DEFAULT_CACHE_INDEXES_ON_WRITE) || family.isCacheIndexesOnWrite(),
213         conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY,
214             DEFAULT_CACHE_BLOOMS_ON_WRITE) || family.isCacheBloomsOnWrite(),
215         conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY,
216             DEFAULT_EVICT_ON_CLOSE) || family.isEvictBlocksOnClose(),
217         conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, DEFAULT_CACHE_DATA_COMPRESSED),
218         conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY,
219             DEFAULT_PREFETCH_ON_OPEN) || family.isPrefetchBlocksOnOpen(),
220         conf.getBoolean(HColumnDescriptor.CACHE_DATA_IN_L1,
221             HColumnDescriptor.DEFAULT_CACHE_DATA_IN_L1) || family.isCacheDataInL1()
222      );
223   }
224 
225   /**
226    * Create a cache configuration using the specified configuration object and
227    * defaults for family level settings.
228    * @param conf hbase configuration
229    */
230   public CacheConfig(Configuration conf) {
231     this(CacheConfig.instantiateBlockCache(conf),
232         DEFAULT_CACHE_DATA_ON_READ,
233         DEFAULT_IN_MEMORY, // This is a family-level setting so can't be set
234                            // strictly from conf
235         conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_DATA_ON_WRITE),
236         conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_INDEXES_ON_WRITE),
237         conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_BLOOMS_ON_WRITE),
238         conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, DEFAULT_EVICT_ON_CLOSE),
239         conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, DEFAULT_CACHE_DATA_COMPRESSED),
240         conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY, DEFAULT_PREFETCH_ON_OPEN),
241         conf.getBoolean(HColumnDescriptor.CACHE_DATA_IN_L1,
242           HColumnDescriptor.DEFAULT_CACHE_DATA_IN_L1)
243      );
244   }
245 
246   /**
247    * Create a block cache configuration with the specified cache and
248    * configuration parameters.
249    * @param blockCache reference to block cache, null if completely disabled
250    * @param cacheDataOnRead whether DATA blocks should be cached on read (we always cache INDEX
251    * blocks and BLOOM blocks; this cannot be disabled).
252    * @param inMemory whether blocks should be flagged as in-memory
253    * @param cacheDataOnWrite whether data blocks should be cached on write
254    * @param cacheIndexesOnWrite whether index blocks should be cached on write
255    * @param cacheBloomsOnWrite whether blooms should be cached on write
256    * @param evictOnClose whether blocks should be evicted when HFile is closed
257    * @param cacheDataCompressed whether to store blocks as compressed in the cache
258    * @param prefetchOnOpen whether to prefetch blocks upon open
259    * @param cacheDataInL1 If more than one cache tier deployed, if true, cache this column families
260    * data blocks up in the L1 tier.
261    */
262   CacheConfig(final BlockCache blockCache,
263       final boolean cacheDataOnRead, final boolean inMemory,
264       final boolean cacheDataOnWrite, final boolean cacheIndexesOnWrite,
265       final boolean cacheBloomsOnWrite, final boolean evictOnClose,
266       final boolean cacheDataCompressed, final boolean prefetchOnOpen,
267       final boolean cacheDataInL1) {
268     this.blockCache = blockCache;
269     this.cacheDataOnRead = cacheDataOnRead;
270     this.inMemory = inMemory;
271     this.cacheDataOnWrite = cacheDataOnWrite;
272     this.cacheIndexesOnWrite = cacheIndexesOnWrite;
273     this.cacheBloomsOnWrite = cacheBloomsOnWrite;
274     this.evictOnClose = evictOnClose;
275     this.cacheDataCompressed = cacheDataCompressed;
276     this.prefetchOnOpen = prefetchOnOpen;
277     this.cacheDataInL1 = cacheDataInL1;
278     LOG.info(this);
279   }
280 
281   /**
282    * Constructs a cache configuration copied from the specified configuration.
283    * @param cacheConf
284    */
285   public CacheConfig(CacheConfig cacheConf) {
286     this(cacheConf.blockCache, cacheConf.cacheDataOnRead, cacheConf.inMemory,
287         cacheConf.cacheDataOnWrite, cacheConf.cacheIndexesOnWrite,
288         cacheConf.cacheBloomsOnWrite, cacheConf.evictOnClose,
289         cacheConf.cacheDataCompressed, cacheConf.prefetchOnOpen,
290         cacheConf.cacheDataInL1);
291   }
292 
293   /**
294    * Checks whether the block cache is enabled.
295    */
296   public boolean isBlockCacheEnabled() {
297     return this.blockCache != null;
298   }
299 
300   /**
301    * Returns the block cache.
302    * @return the block cache, or null if caching is completely disabled
303    */
304   public BlockCache getBlockCache() {
305     return this.blockCache;
306   }
307 
308   /**
309    * Returns whether the DATA blocks of this HFile should be cached on read or not (we always
310    * cache the meta blocks, the INDEX and BLOOM blocks).
311    * @return true if blocks should be cached on read, false if not
312    */
313   public boolean shouldCacheDataOnRead() {
314     return isBlockCacheEnabled() && cacheDataOnRead;
315   }
316 
317   /**
318    * Should we cache a block of a particular category? We always cache
319    * important blocks such as index blocks, as long as the block cache is
320    * available.
321    */
322   public boolean shouldCacheBlockOnRead(BlockCategory category) {
323     return isBlockCacheEnabled()
324         && (cacheDataOnRead ||
325             category == BlockCategory.INDEX ||
326             category == BlockCategory.BLOOM ||
327             (prefetchOnOpen &&
328                 (category != BlockCategory.META &&
329                  category != BlockCategory.UNKNOWN)));
330   }
331 
332   /**
333    * @return true if blocks in this file should be flagged as in-memory
334    */
335   public boolean isInMemory() {
336     return isBlockCacheEnabled() && this.inMemory;
337   }
338 
339   /**
340    * @return True if cache data blocks in L1 tier (if more than one tier in block cache deploy).
341    */
342   public boolean isCacheDataInL1() {
343     return isBlockCacheEnabled() && this.cacheDataInL1;
344   }
345 
346   /**
347    * @return true if data blocks should be written to the cache when an HFile is
348    *         written, false if not
349    */
350   public boolean shouldCacheDataOnWrite() {
351     return isBlockCacheEnabled() && this.cacheDataOnWrite;
352   }
353 
354   /**
355    * Only used for testing.
356    * @param cacheDataOnWrite whether data blocks should be written to the cache
357    *                         when an HFile is written
358    */
359   @VisibleForTesting
360   public void setCacheDataOnWrite(boolean cacheDataOnWrite) {
361     this.cacheDataOnWrite = cacheDataOnWrite;
362   }
363 
364   /**
365    * Only used for testing.
366    * @param cacheDataInL1 Whether to cache data blocks up in l1 (if a multi-tier cache
367    * implementation).
368    */
369   @VisibleForTesting
370   public void setCacheDataInL1(boolean cacheDataInL1) {
371     this.cacheDataInL1 = cacheDataInL1;
372   }
373 
374   /**
375    * @return true if index blocks should be written to the cache when an HFile
376    *         is written, false if not
377    */
378   public boolean shouldCacheIndexesOnWrite() {
379     return isBlockCacheEnabled() && this.cacheIndexesOnWrite;
380   }
381 
382   /**
383    * @return true if bloom blocks should be written to the cache when an HFile
384    *         is written, false if not
385    */
386   public boolean shouldCacheBloomsOnWrite() {
387     return isBlockCacheEnabled() && this.cacheBloomsOnWrite;
388   }
389 
390   /**
391    * @return true if blocks should be evicted from the cache when an HFile
392    *         reader is closed, false if not
393    */
394   public boolean shouldEvictOnClose() {
395     return isBlockCacheEnabled() && this.evictOnClose;
396   }
397 
398   /**
399    * Only used for testing.
400    * @param evictOnClose whether blocks should be evicted from the cache when an
401    *                     HFile reader is closed
402    */
403   public void setEvictOnClose(boolean evictOnClose) {
404     this.evictOnClose = evictOnClose;
405   }
406 
407   /**
408    * @return true if data blocks should be compressed in the cache, false if not
409    */
410   public boolean shouldCacheDataCompressed() {
411     return isBlockCacheEnabled() && this.cacheDataCompressed;
412   }
413 
414   /**
415    * @return true if this {@link BlockCategory} should be compressed in blockcache, false otherwise
416    */
417   public boolean shouldCacheCompressed(BlockCategory category) {
418     if (!isBlockCacheEnabled()) return false;
419     switch (category) {
420       case DATA:
421         return this.cacheDataCompressed;
422       default:
423         return false;
424     }
425   }
426 
427   /**
428    * @return true if blocks should be prefetched into the cache on open, false if not
429    */
430   public boolean shouldPrefetchOnOpen() {
431     return isBlockCacheEnabled() && this.prefetchOnOpen;
432   }
433 
434   @Override
435   public String toString() {
436     if (!isBlockCacheEnabled()) {
437       return "CacheConfig:disabled";
438     }
439     return "blockCache=" + getBlockCache() +
440       ", cacheDataOnRead=" + shouldCacheDataOnRead() +
441       ", cacheDataOnWrite=" + shouldCacheDataOnWrite() +
442       ", cacheIndexesOnWrite=" + shouldCacheIndexesOnWrite() +
443       ", cacheBloomsOnWrite=" + shouldCacheBloomsOnWrite() +
444       ", cacheEvictOnClose=" + shouldEvictOnClose() +
445       ", cacheDataCompressed=" + shouldCacheDataCompressed() +
446       ", prefetchOnOpen=" + shouldPrefetchOnOpen();
447   }
448 
449   // Static block cache reference and methods
450 
451   /**
452    * Static reference to the block cache, or null if no caching should be used
453    * at all.
454    */
455   // Clear this if in tests you'd make more than one block cache instance.
456   @VisibleForTesting
457   static BlockCache GLOBAL_BLOCK_CACHE_INSTANCE;
458 
459   /** Boolean whether we have disabled the block cache entirely. */
460   @VisibleForTesting
461   static boolean blockCacheDisabled = false;
462 
463   static long getLruCacheSize(final Configuration conf, final MemoryUsage mu) {
464     float cachePercentage = conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY,
465       HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
466     if (cachePercentage <= 0.0001f) {
467       blockCacheDisabled = true;
468       return -1;
469     }
470     if (cachePercentage > 1.0) {
471       throw new IllegalArgumentException(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY +
472         " must be between 0.0 and 1.0, and not > 1.0");
473     }
474 
475     // Calculate the amount of heap to give the heap.
476     return (long) (mu.getMax() * cachePercentage);
477   }
478 
479   /**
480    * @param c Configuration to use.
481    * @param mu JMX Memory Bean
482    * @return An L1 instance.  Currently an instance of LruBlockCache.
483    */
484   private static LruBlockCache getL1(final Configuration c, final MemoryUsage mu) {
485     long lruCacheSize = getLruCacheSize(c, mu);
486     if (lruCacheSize < 0) return null;
487     int blockSize = c.getInt(BLOCKCACHE_BLOCKSIZE_KEY, HConstants.DEFAULT_BLOCKSIZE);
488     LOG.info("Allocating LruBlockCache size=" +
489       StringUtils.byteDesc(lruCacheSize) + ", blockSize=" + StringUtils.byteDesc(blockSize));
490     return new LruBlockCache(lruCacheSize, blockSize, true, c);
491   }
492 
493   /**
494    * @param c Configuration to use.
495    * @param mu JMX Memory Bean
496    * @return Returns L2 block cache instance (for now it is BucketCache BlockCache all the time)
497    * or null if not supposed to be a L2.
498    */
499   private static BlockCache getL2(final Configuration c, final MemoryUsage mu) {
500     final boolean useExternal = c.getBoolean(EXTERNAL_BLOCKCACHE_KEY, EXTERNAL_BLOCKCACHE_DEFAULT);
501     if (LOG.isDebugEnabled()) {
502       LOG.debug("Trying to use " + (useExternal?" External":" Internal") + " l2 cache");
503     }
504 
505     // If we want to use an external block cache then create that.
506     if (useExternal) {
507       return getExternalBlockcache(c);
508     }
509 
510     // otherwise use the bucket cache.
511     return getBucketCache(c, mu);
512 
513   }
514 
515   private static BlockCache getExternalBlockcache(Configuration c) {
516     Class klass = null;
517 
518     // Get the class, from the config. s
519     try {
520       klass = ExternalBlockCaches.valueOf(c.get(EXTERNAL_BLOCKCACHE_CLASS_KEY, "memcache")).clazz;
521     } catch (IllegalArgumentException exception) {
522       klass = c.getClass(EXTERNAL_BLOCKCACHE_CLASS_KEY, MemcachedBlockCache.class);
523     }
524 
525     // Now try and create an instance of the block cache.
526     try {
527       LOG.info("Creating external block cache of type: " + klass);
528       return (BlockCache) ReflectionUtils.newInstance(klass, c);
529     } catch (Exception e) {
530       LOG.warn("Error creating external block cache", e);
531     }
532     return null;
533 
534   }
535 
536   private static BlockCache getBucketCache(Configuration c, MemoryUsage mu) {
537     // Check for L2.  ioengine name must be non-null.
538     String bucketCacheIOEngineName = c.get(BUCKET_CACHE_IOENGINE_KEY, null);
539     if (bucketCacheIOEngineName == null || bucketCacheIOEngineName.length() <= 0) return null;
540 
541     int blockSize = c.getInt(BLOCKCACHE_BLOCKSIZE_KEY, HConstants.DEFAULT_BLOCKSIZE);
542     float bucketCachePercentage = c.getFloat(BUCKET_CACHE_SIZE_KEY, 0F);
543     long bucketCacheSize = (long) (bucketCachePercentage < 1? mu.getMax() * bucketCachePercentage:
544       bucketCachePercentage * 1024 * 1024);
545     if (bucketCacheSize <= 0) {
546       throw new IllegalStateException("bucketCacheSize <= 0; Check " +
547         BUCKET_CACHE_SIZE_KEY + " setting and/or server java heap size");
548     }
549     if (c.get("hbase.bucketcache.percentage.in.combinedcache") != null) {
550       LOG.warn("Configuration 'hbase.bucketcache.percentage.in.combinedcache' is no longer "
551           + "respected. See comments in http://hbase.apache.org/book.html#_changes_of_note");
552     }
553     int writerThreads = c.getInt(BUCKET_CACHE_WRITER_THREADS_KEY,
554       DEFAULT_BUCKET_CACHE_WRITER_THREADS);
555     int writerQueueLen = c.getInt(BUCKET_CACHE_WRITER_QUEUE_KEY,
556       DEFAULT_BUCKET_CACHE_WRITER_QUEUE);
557     String persistentPath = c.get(BUCKET_CACHE_PERSISTENT_PATH_KEY);
558     String[] configuredBucketSizes = c.getStrings(BUCKET_CACHE_BUCKETS_KEY);
559     int [] bucketSizes = null;
560     if (configuredBucketSizes != null) {
561       bucketSizes = new int[configuredBucketSizes.length];
562       for (int i = 0; i < configuredBucketSizes.length; i++) {
563         bucketSizes[i] = Integer.parseInt(configuredBucketSizes[i].trim());
564       }
565     }
566     BucketCache bucketCache = null;
567     try {
568       int ioErrorsTolerationDuration = c.getInt(
569         "hbase.bucketcache.ioengine.errors.tolerated.duration",
570         BucketCache.DEFAULT_ERROR_TOLERATION_DURATION);
571       // Bucket cache logs its stats on creation internal to the constructor.
572       bucketCache = new BucketCache(bucketCacheIOEngineName,
573         bucketCacheSize, blockSize, bucketSizes, writerThreads, writerQueueLen, persistentPath,
574         ioErrorsTolerationDuration);
575     } catch (IOException ioex) {
576       LOG.error("Can't instantiate bucket cache", ioex); throw new RuntimeException(ioex);
577     }
578     return bucketCache;
579   }
580 
581   /**
582    * Returns the block cache or <code>null</code> in case none should be used.
583    * Sets GLOBAL_BLOCK_CACHE_INSTANCE
584    *
585    * @param conf  The current configuration.
586    * @return The block cache or <code>null</code>.
587    */
588   public static synchronized BlockCache instantiateBlockCache(Configuration conf) {
589     if (GLOBAL_BLOCK_CACHE_INSTANCE != null) return GLOBAL_BLOCK_CACHE_INSTANCE;
590     if (blockCacheDisabled) return null;
591     MemoryUsage mu = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
592     LruBlockCache l1 = getL1(conf, mu);
593     // blockCacheDisabled is set as a side-effect of getL1(), so check it again after the call.
594     if (blockCacheDisabled) return null;
595     BlockCache l2 = getL2(conf, mu);
596     if (l2 == null) {
597       GLOBAL_BLOCK_CACHE_INSTANCE = l1;
598     } else {
599       boolean useExternal = conf.getBoolean(EXTERNAL_BLOCKCACHE_KEY, EXTERNAL_BLOCKCACHE_DEFAULT);
600       boolean combinedWithLru = conf.getBoolean(BUCKET_CACHE_COMBINED_KEY,
601         DEFAULT_BUCKET_CACHE_COMBINED);
602       if (useExternal) {
603         GLOBAL_BLOCK_CACHE_INSTANCE = new InclusiveCombinedBlockCache(l1, l2);
604       } else {
605         if (combinedWithLru) {
606           GLOBAL_BLOCK_CACHE_INSTANCE = new CombinedBlockCache(l1, l2);
607         } else {
608           // L1 and L2 are not 'combined'.  They are connected via the LruBlockCache victimhandler
609           // mechanism.  It is a little ugly but works according to the following: when the
610           // background eviction thread runs, blocks evicted from L1 will go to L2 AND when we get
611           // a block from the L1 cache, if not in L1, we will search L2.
612           GLOBAL_BLOCK_CACHE_INSTANCE = l1;
613         }
614       }
615       l1.setVictimCache(l2);
616     }
617     return GLOBAL_BLOCK_CACHE_INSTANCE;
618   }
619 }