View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.io.hfile;
19  
20  import static org.apache.hadoop.hbase.HConstants.BUCKET_CACHE_IOENGINE_KEY;
21  import static org.apache.hadoop.hbase.HConstants.BUCKET_CACHE_SIZE_KEY;
22  
23  import java.io.IOException;
24  import java.lang.management.ManagementFactory;
25  import java.lang.management.MemoryUsage;
26  
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.hadoop.classification.InterfaceAudience;
30  import org.apache.hadoop.conf.Configuration;
31  import org.apache.hadoop.hbase.HColumnDescriptor;
32  import org.apache.hadoop.hbase.HConstants;
33  import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory;
34  import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache;
35  import org.apache.hadoop.util.StringUtils;
36  
37  import com.google.common.annotations.VisibleForTesting;
38  
39  /**
40   * Stores all of the cache objects and configuration for a single HFile.
41   */
42  @InterfaceAudience.Private
43  public class CacheConfig {
44    private static final Log LOG = LogFactory.getLog(CacheConfig.class.getName());
45  
46    /**
47     * Configuration key to cache data blocks on write. There are separate
48     * switches for bloom blocks and non-root index blocks.
49     */
50    public static final String CACHE_BLOCKS_ON_WRITE_KEY =
51        "hbase.rs.cacheblocksonwrite";
52  
53    /**
54     * Configuration key to cache leaf and intermediate-level index blocks on
55     * write.
56     */
57    public static final String CACHE_INDEX_BLOCKS_ON_WRITE_KEY =
58        "hfile.block.index.cacheonwrite";
59  
60    /**
61     * Configuration key to cache compound bloom filter blocks on write.
62     */
63    public static final String CACHE_BLOOM_BLOCKS_ON_WRITE_KEY =
64        "hfile.block.bloom.cacheonwrite";
65  
66    /**
67     * TODO: Implement this (jgray)
68     * Configuration key to cache data blocks in compressed format.
69     */
70    public static final String CACHE_DATA_BLOCKS_COMPRESSED_KEY =
71        "hbase.rs.blockcache.cachedatacompressed";
72  
73    /**
74     * Configuration key to evict all blocks of a given file from the block cache
75     * when the file is closed.
76     */
77    public static final String EVICT_BLOCKS_ON_CLOSE_KEY =
78        "hbase.rs.evictblocksonclose";
79  
80    /**
81     * Configuration keys for Bucket cache
82     */
83  
84    /**
85     * If the chosen ioengine can persist its state across restarts, the path to the file to
86     * persist to.
87     */
88    public static final String BUCKET_CACHE_PERSISTENT_PATH_KEY = 
89        "hbase.bucketcache.persistent.path";
90  
91    /**
92     * If the bucket cache is used in league with the lru on-heap block cache (meta blocks such
93     * as indices and blooms are kept in the lru blockcache and the data blocks in the
94     * bucket cache).
95     */
96    public static final String BUCKET_CACHE_COMBINED_KEY = 
97        "hbase.bucketcache.combinedcache.enabled";
98  
99    public static final String BUCKET_CACHE_WRITER_THREADS_KEY = "hbase.bucketcache.writer.threads";
100   public static final String BUCKET_CACHE_WRITER_QUEUE_KEY = 
101       "hbase.bucketcache.writer.queuelength";
102 
103   /**
104    * A comma-delimited array of values for use as bucket sizes.
105    */
106   public static final String BUCKET_CACHE_BUCKETS_KEY = "hbase.bucketcache.bucket.sizes";
107 
108   /**
109    * Defaults for Bucket cache
110    */
111   public static final boolean DEFAULT_BUCKET_CACHE_COMBINED = true;
112   public static final int DEFAULT_BUCKET_CACHE_WRITER_THREADS = 3;
113   public static final int DEFAULT_BUCKET_CACHE_WRITER_QUEUE = 64;
114   public static final float DEFAULT_BUCKET_CACHE_COMBINED_PERCENTAGE = 0.9f;
115 
116  /**
117    * Configuration key to prefetch all blocks of a given file into the block cache
118    * when the file is opened.
119    */
120   public static final String PREFETCH_BLOCKS_ON_OPEN_KEY =
121       "hbase.rs.prefetchblocksonopen";
122 
123   // Defaults
124 
125   public static final boolean DEFAULT_CACHE_DATA_ON_READ = true;
126   public static final boolean DEFAULT_CACHE_DATA_ON_WRITE = false;
127   public static final boolean DEFAULT_IN_MEMORY = false;
128   public static final boolean DEFAULT_CACHE_INDEXES_ON_WRITE = false;
129   public static final boolean DEFAULT_CACHE_BLOOMS_ON_WRITE = false;
130   public static final boolean DEFAULT_EVICT_ON_CLOSE = false;
131   public static final boolean DEFAULT_COMPRESSED_CACHE = false;
132   public static final boolean DEFAULT_PREFETCH_ON_OPEN = false;
133 
134   /** Local reference to the block cache, null if completely disabled */
135   private final BlockCache blockCache;
136 
137   /**
138    * Whether blocks should be cached on read (default is on if there is a
139    * cache but this can be turned off on a per-family or per-request basis).
140    * If off we will STILL cache meta blocks; i.e. INDEX and BLOOM types.
141    * This cannot be disabled.
142    */
143   private boolean cacheDataOnRead;
144 
145   /** Whether blocks should be flagged as in-memory when being cached */
146   private final boolean inMemory;
147 
148   /** Whether data blocks should be cached when new files are written */
149   private boolean cacheDataOnWrite;
150 
151   /** Whether index blocks should be cached when new files are written */
152   private final boolean cacheIndexesOnWrite;
153 
154   /** Whether compound bloom filter blocks should be cached on write */
155   private final boolean cacheBloomsOnWrite;
156 
157   /** Whether blocks of a file should be evicted when the file is closed */
158   private boolean evictOnClose;
159 
160   /** Whether data blocks should be stored in compressed form in the cache */
161   private final boolean cacheCompressed;
162 
163   /** Whether data blocks should be prefetched into the cache */
164   private final boolean prefetchOnOpen;
165 
166   /**
167    * If true and if more than one tier in this cache deploy -- e.g. CombinedBlockCache has an L1
168    * and an L2 tier -- then cache data blocks up in the L1 tier (The meta blocks are likely being
169    * cached up in L1 already.  At least this is the case if CombinedBlockCache).
170    */
171   private boolean cacheDataInL1;
172 
173   /**
174    * Create a cache configuration using the specified configuration object and
175    * family descriptor.
176    * @param conf hbase configuration
177    * @param family column family configuration
178    */
179   public CacheConfig(Configuration conf, HColumnDescriptor family) {
180     this(CacheConfig.instantiateBlockCache(conf),
181         family.isBlockCacheEnabled(),
182         family.isInMemory(),
183         // For the following flags we enable them regardless of per-schema settings
184         // if they are enabled in the global configuration.
185         conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY,
186             DEFAULT_CACHE_DATA_ON_WRITE) || family.shouldCacheDataOnWrite(),
187         conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY,
188             DEFAULT_CACHE_INDEXES_ON_WRITE) || family.shouldCacheIndexesOnWrite(),
189         conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY,
190             DEFAULT_CACHE_BLOOMS_ON_WRITE) || family.shouldCacheBloomsOnWrite(),
191         conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY,
192             DEFAULT_EVICT_ON_CLOSE) || family.shouldEvictBlocksOnClose(),
193         conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, DEFAULT_COMPRESSED_CACHE),
194         conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY,
195             DEFAULT_PREFETCH_ON_OPEN) || family.shouldPrefetchBlocksOnOpen(),
196         conf.getBoolean(HColumnDescriptor.CACHE_DATA_IN_L1,
197             HColumnDescriptor.DEFAULT_CACHE_DATA_IN_L1) || family.shouldCacheDataInL1()
198      );
199   }
200 
201   /**
202    * Create a cache configuration using the specified configuration object and
203    * defaults for family level settings.
204    * @param conf hbase configuration
205    */
206   public CacheConfig(Configuration conf) {
207     this(CacheConfig.instantiateBlockCache(conf),
208         DEFAULT_CACHE_DATA_ON_READ,
209         DEFAULT_IN_MEMORY, // This is a family-level setting so can't be set
210                            // strictly from conf
211         conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_DATA_ON_WRITE),
212         conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY,
213             DEFAULT_CACHE_INDEXES_ON_WRITE),
214             conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY,
215                 DEFAULT_CACHE_BLOOMS_ON_WRITE),
216         conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, DEFAULT_EVICT_ON_CLOSE),
217         conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY,
218             DEFAULT_COMPRESSED_CACHE),
219         conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY, DEFAULT_PREFETCH_ON_OPEN),
220         conf.getBoolean(HColumnDescriptor.CACHE_DATA_IN_L1,
221           HColumnDescriptor.DEFAULT_CACHE_DATA_IN_L1)
222      );
223   }
224 
225   /**
226    * Create a block cache configuration with the specified cache and
227    * configuration parameters.
228    * @param blockCache reference to block cache, null if completely disabled
229    * @param cacheDataOnRead whether DATA blocks should be cached on read (we always cache INDEX
230    * blocks and BLOOM blocks; this cannot be disabled).
231    * @param inMemory whether blocks should be flagged as in-memory
232    * @param cacheDataOnWrite whether data blocks should be cached on write
233    * @param cacheIndexesOnWrite whether index blocks should be cached on write
234    * @param cacheBloomsOnWrite whether blooms should be cached on write
235    * @param evictOnClose whether blocks should be evicted when HFile is closed
236    * @param cacheCompressed whether to store blocks as compressed in the cache
237    * @param prefetchOnOpen whether to prefetch blocks upon open
238    * @param cacheDataInL1 If more than one cache tier deployed, if true, cache this column families
239    * data blocks up in the L1 tier.
240    */
241   CacheConfig(final BlockCache blockCache,
242       final boolean cacheDataOnRead, final boolean inMemory,
243       final boolean cacheDataOnWrite, final boolean cacheIndexesOnWrite,
244       final boolean cacheBloomsOnWrite, final boolean evictOnClose,
245       final boolean cacheCompressed, final boolean prefetchOnOpen,
246       final boolean cacheDataInL1) {
247     this.blockCache = blockCache;
248     this.cacheDataOnRead = cacheDataOnRead;
249     this.inMemory = inMemory;
250     this.cacheDataOnWrite = cacheDataOnWrite;
251     this.cacheIndexesOnWrite = cacheIndexesOnWrite;
252     this.cacheBloomsOnWrite = cacheBloomsOnWrite;
253     this.evictOnClose = evictOnClose;
254     this.cacheCompressed = cacheCompressed;
255     this.prefetchOnOpen = prefetchOnOpen;
256     this.cacheDataInL1 = cacheDataInL1;
257     LOG.info(this);
258   }
259 
260   /**
261    * Constructs a cache configuration copied from the specified configuration.
262    * @param cacheConf
263    */
264   public CacheConfig(CacheConfig cacheConf) {
265     this(cacheConf.blockCache, cacheConf.cacheDataOnRead, cacheConf.inMemory,
266         cacheConf.cacheDataOnWrite, cacheConf.cacheIndexesOnWrite,
267         cacheConf.cacheBloomsOnWrite, cacheConf.evictOnClose,
268         cacheConf.cacheCompressed, cacheConf.prefetchOnOpen,
269         cacheConf.cacheDataInL1);
270   }
271 
272   /**
273    * Checks whether the block cache is enabled.
274    */
275   public boolean isBlockCacheEnabled() {
276     return this.blockCache != null;
277   }
278 
279   /**
280    * Returns the block cache.
281    * @return the block cache, or null if caching is completely disabled
282    */
283   public BlockCache getBlockCache() {
284     return this.blockCache;
285   }
286 
287   /**
288    * Returns whether the DATA blocks of this HFile should be cached on read or not (we always
289    * cache the meta blocks, the INDEX and BLOOM blocks).
290    * @return true if blocks should be cached on read, false if not
291    */
292   public boolean shouldCacheDataOnRead() {
293     return isBlockCacheEnabled() && cacheDataOnRead;
294   }
295 
296   /**
297    * Should we cache a block of a particular category? We always cache
298    * important blocks such as index blocks, as long as the block cache is
299    * available.
300    */
301   public boolean shouldCacheBlockOnRead(BlockCategory category) {
302     boolean shouldCache = isBlockCacheEnabled()
303         && (cacheDataOnRead ||
304             category == BlockCategory.INDEX ||
305             category == BlockCategory.BLOOM ||
306             (prefetchOnOpen &&
307                 (category != BlockCategory.META &&
308                  category != BlockCategory.UNKNOWN)));
309     return shouldCache;
310   }
311 
312   /**
313    * @return true if blocks in this file should be flagged as in-memory
314    */
315   public boolean isInMemory() {
316     return isBlockCacheEnabled() && this.inMemory;
317   }
318 
319   /**
320    * @return True if cache data blocks in L1 tier (if more than one tier in block cache deploy).
321    */
322   public boolean isCacheDataInL1() {
323     return isBlockCacheEnabled() && this.cacheDataInL1;
324   }
325 
326   /**
327    * @return true if data blocks should be written to the cache when an HFile is
328    *         written, false if not
329    */
330   public boolean shouldCacheDataOnWrite() {
331     return isBlockCacheEnabled() && this.cacheDataOnWrite;
332   }
333 
334   /**
335    * Only used for testing.
336    * @param cacheDataOnWrite whether data blocks should be written to the cache
337    *                         when an HFile is written
338    */
339   @VisibleForTesting
340   public void setCacheDataOnWrite(boolean cacheDataOnWrite) {
341     this.cacheDataOnWrite = cacheDataOnWrite;
342   }
343 
344   /**
345    * Only used for testing.
346    * @param cacheDataInL1 Whether to cache data blocks up in l1 (if a multi-tier cache
347    * implementation).
348    */
349   @VisibleForTesting
350   public void setCacheDataInL1(boolean cacheDataInL1) {
351     this.cacheDataInL1 = cacheDataInL1;
352   }
353 
354   /**
355    * @return true if index blocks should be written to the cache when an HFile
356    *         is written, false if not
357    */
358   public boolean shouldCacheIndexesOnWrite() {
359     return isBlockCacheEnabled() && this.cacheIndexesOnWrite;
360   }
361 
362   /**
363    * @return true if bloom blocks should be written to the cache when an HFile
364    *         is written, false if not
365    */
366   public boolean shouldCacheBloomsOnWrite() {
367     return isBlockCacheEnabled() && this.cacheBloomsOnWrite;
368   }
369 
370   /**
371    * @return true if blocks should be evicted from the cache when an HFile
372    *         reader is closed, false if not
373    */
374   public boolean shouldEvictOnClose() {
375     return isBlockCacheEnabled() && this.evictOnClose;
376   }
377 
378   /**
379    * Only used for testing.
380    * @param evictOnClose whether blocks should be evicted from the cache when an
381    *                     HFile reader is closed
382    */
383   public void setEvictOnClose(boolean evictOnClose) {
384     this.evictOnClose = evictOnClose;
385   }
386 
387   /**
388    * @return true if blocks should be compressed in the cache, false if not
389    */
390   public boolean shouldCacheCompressed() {
391     return isBlockCacheEnabled() && this.cacheCompressed;
392   }
393 
394   /**
395    * @return true if blocks should be prefetched into the cache on open, false if not
396    */
397   public boolean shouldPrefetchOnOpen() {
398     return isBlockCacheEnabled() && this.prefetchOnOpen;
399   }
400 
401   @Override
402   public String toString() {
403     if (!isBlockCacheEnabled()) {
404       return "CacheConfig:disabled";
405     }
406     return "blockCache=" + getBlockCache() +
407       ", cacheDataOnRead=" + shouldCacheDataOnRead() +
408       ", cacheDataOnWrite=" + shouldCacheDataOnWrite() +
409       ", cacheIndexesOnWrite=" + shouldCacheIndexesOnWrite() +
410       ", cacheBloomsOnWrite=" + shouldCacheBloomsOnWrite() +
411       ", cacheEvictOnClose=" + shouldEvictOnClose() +
412       ", cacheCompressed=" + shouldCacheCompressed() +
413       ", prefetchOnOpen=" + shouldPrefetchOnOpen();
414   }
415 
416   // Static block cache reference and methods
417 
418   /**
419    * Static reference to the block cache, or null if no caching should be used
420    * at all.
421    */
422   // Clear this if in tests you'd make more than one block cache instance.
423   @VisibleForTesting
424   static BlockCache GLOBAL_BLOCK_CACHE_INSTANCE;
425 
426   /** Boolean whether we have disabled the block cache entirely. */
427   @VisibleForTesting
428   static boolean blockCacheDisabled = false;
429 
430   static long getLruCacheSize(final Configuration conf, final MemoryUsage mu) {
431     float cachePercentage = conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY,
432       HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
433     if (cachePercentage <= 0.0001f) {
434       blockCacheDisabled = true;
435       return -1;
436     }
437     if (cachePercentage > 1.0) {
438       throw new IllegalArgumentException(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY +
439         " must be between 0.0 and 1.0, and not > 1.0");
440     }
441 
442     // Calculate the amount of heap to give the heap.
443     return (long) (mu.getMax() * cachePercentage);
444   }
445 
446   /**
447    * @param c Configuration to use.
448    * @param mu JMX Memory Bean
449    * @return An L1 instance.  Currently an instance of LruBlockCache.
450    */
451   private static LruBlockCache getL1(final Configuration c, final MemoryUsage mu) {
452     long lruCacheSize = getLruCacheSize(c, mu);
453     int blockSize = c.getInt("hbase.offheapcache.minblocksize", HConstants.DEFAULT_BLOCKSIZE);
454     LOG.info("Allocating LruBlockCache size=" +
455       StringUtils.byteDesc(lruCacheSize) + ", blockSize=" + StringUtils.byteDesc(blockSize));
456     return new LruBlockCache(lruCacheSize, blockSize, true, c);
457   }
458 
459   /**
460    * @param c Configuration to use.
461    * @param mu JMX Memory Bean
462    * @return Returns L2 block cache instance (for now it is BucketCache BlockCache all the time)
463    * or null if not supposed to be a L2.
464    */
465   private static BucketCache getL2(final Configuration c, final MemoryUsage mu) {
466     // Check for L2.  ioengine name must be non-null.
467     String bucketCacheIOEngineName = c.get(BUCKET_CACHE_IOENGINE_KEY, null);
468     if (bucketCacheIOEngineName == null || bucketCacheIOEngineName.length() <= 0) return null;
469 
470     int blockSize = c.getInt("hbase.offheapcache.minblocksize", HConstants.DEFAULT_BLOCKSIZE);
471     float bucketCachePercentage = c.getFloat(BUCKET_CACHE_SIZE_KEY, 0F);
472     long bucketCacheSize = (long) (bucketCachePercentage < 1? mu.getMax() * bucketCachePercentage:
473       bucketCachePercentage * 1024 * 1024);
474     if (bucketCacheSize <= 0) {
475       throw new IllegalStateException("bucketCacheSize <= 0; Check " +
476         BUCKET_CACHE_SIZE_KEY + " setting and/or server java heap size");
477     }
478     int writerThreads = c.getInt(BUCKET_CACHE_WRITER_THREADS_KEY,
479       DEFAULT_BUCKET_CACHE_WRITER_THREADS);
480     int writerQueueLen = c.getInt(BUCKET_CACHE_WRITER_QUEUE_KEY,
481       DEFAULT_BUCKET_CACHE_WRITER_QUEUE);
482     String persistentPath = c.get(BUCKET_CACHE_PERSISTENT_PATH_KEY);
483     String[] configuredBucketSizes = c.getStrings(BUCKET_CACHE_BUCKETS_KEY);
484     int [] bucketSizes = null;
485     if (configuredBucketSizes != null) {
486       bucketSizes = new int[configuredBucketSizes.length];
487       for (int i = 0; i < configuredBucketSizes.length; i++) {
488         bucketSizes[i] = Integer.parseInt(configuredBucketSizes[i]);
489       }
490     }
491     BucketCache bucketCache = null;
492     try {
493       int ioErrorsTolerationDuration = c.getInt(
494         "hbase.bucketcache.ioengine.errors.tolerated.duration",
495         BucketCache.DEFAULT_ERROR_TOLERATION_DURATION);
496       // Bucket cache logs its stats on creation internal to the constructor.
497       bucketCache = new BucketCache(bucketCacheIOEngineName,
498         bucketCacheSize, blockSize, bucketSizes, writerThreads, writerQueueLen, persistentPath,
499         ioErrorsTolerationDuration);
500     } catch (IOException ioex) {
501       LOG.error("Can't instantiate bucket cache", ioex); throw new RuntimeException(ioex);
502     }
503     return bucketCache;
504   }
505 
506   /**
507    * Returns the block cache or <code>null</code> in case none should be used.
508    * Sets GLOBAL_BLOCK_CACHE_INSTANCE
509    *
510    * @param conf  The current configuration.
511    * @return The block cache or <code>null</code>.
512    */
513   public static synchronized BlockCache instantiateBlockCache(Configuration conf) {
514     if (GLOBAL_BLOCK_CACHE_INSTANCE != null) return GLOBAL_BLOCK_CACHE_INSTANCE;
515     if (blockCacheDisabled) return null;
516     MemoryUsage mu = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
517     LruBlockCache l1 = getL1(conf, mu);
518     BucketCache l2 = getL2(conf, mu);
519     if (l2 == null) {
520       GLOBAL_BLOCK_CACHE_INSTANCE = l1;
521     } else {
522       boolean combinedWithLru = conf.getBoolean(BUCKET_CACHE_COMBINED_KEY,
523         DEFAULT_BUCKET_CACHE_COMBINED);
524       if (combinedWithLru) {
525         GLOBAL_BLOCK_CACHE_INSTANCE = new CombinedBlockCache(l1, l2);
526       } else {
527         // L1 and L2 are not 'combined'.  They are connected via the LruBlockCache victimhandler
528         // mechanism.  It is a little ugly but works according to the following: when the
529         // background eviction thread runs, blocks evicted from L1 will go to L2 AND when we get
530         // a block from the L1 cache, if not in L1, we will search L2.
531         l1.setVictimCache(l2);
532         GLOBAL_BLOCK_CACHE_INSTANCE = l1;
533       }
534     }
535     return GLOBAL_BLOCK_CACHE_INSTANCE;
536   }
537 }