View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.io.hfile;
19  
20  import static org.apache.hadoop.hbase.HConstants.BUCKET_CACHE_IOENGINE_KEY;
21  import static org.apache.hadoop.hbase.HConstants.BUCKET_CACHE_SIZE_KEY;
22  
23  import java.io.IOException;
24  import java.lang.management.ManagementFactory;
25  import java.lang.management.MemoryUsage;
26  
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.hadoop.classification.InterfaceAudience;
30  import org.apache.hadoop.conf.Configuration;
31  import org.apache.hadoop.hbase.HColumnDescriptor;
32  import org.apache.hadoop.hbase.HConstants;
33  import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory;
34  import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache;
35  import org.apache.hadoop.util.StringUtils;
36  
37  import com.google.common.annotations.VisibleForTesting;
38  
39  /**
40   * Stores all of the cache objects and configuration for a single HFile.
41   */
42  @InterfaceAudience.Private
43  public class CacheConfig {
44    private static final Log LOG = LogFactory.getLog(CacheConfig.class.getName());
45  
46    /**
47     * Configuration key to cache data blocks on write. There are separate
48     * switches for bloom blocks and non-root index blocks.
49     */
50    public static final String CACHE_BLOCKS_ON_WRITE_KEY =
51        "hbase.rs.cacheblocksonwrite";
52  
53    /**
54     * Configuration key to cache leaf and intermediate-level index blocks on
55     * write.
56     */
57    public static final String CACHE_INDEX_BLOCKS_ON_WRITE_KEY =
58        "hfile.block.index.cacheonwrite";
59  
60    /**
61     * Configuration key to cache compound bloom filter blocks on write.
62     */
63    public static final String CACHE_BLOOM_BLOCKS_ON_WRITE_KEY =
64        "hfile.block.bloom.cacheonwrite";
65  
66    /**
67     * TODO: Implement this (jgray)
68     * Configuration key to cache data blocks in compressed format.
69     */
70    public static final String CACHE_DATA_BLOCKS_COMPRESSED_KEY =
71        "hbase.rs.blockcache.cachedatacompressed";
72  
73    /**
74     * Configuration key to evict all blocks of a given file from the block cache
75     * when the file is closed.
76     */
77    public static final String EVICT_BLOCKS_ON_CLOSE_KEY =
78        "hbase.rs.evictblocksonclose";
79  
80    /**
81     * Configuration keys for Bucket cache
82     */
83  
84    /**
85     * If the chosen ioengine can persist its state across restarts, the path to the file to
86     * persist to.
87     */
88    public static final String BUCKET_CACHE_PERSISTENT_PATH_KEY = 
89        "hbase.bucketcache.persistent.path";
90  
91    /**
92     * If the bucket cache is used in league with the lru on-heap block cache (meta blocks such
93     * as indices and blooms are kept in the lru blockcache and the data blocks in the
94     * bucket cache).
95     */
96    public static final String BUCKET_CACHE_COMBINED_KEY = 
97        "hbase.bucketcache.combinedcache.enabled";
98  
99    public static final String BUCKET_CACHE_WRITER_THREADS_KEY = "hbase.bucketcache.writer.threads";
100   public static final String BUCKET_CACHE_WRITER_QUEUE_KEY = 
101       "hbase.bucketcache.writer.queuelength";
102 
103   /**
104    * A comma-delimited array of values for use as bucket sizes.
105    */
106   public static final String BUCKET_CACHE_BUCKETS_KEY = "hbase.bucketcache.bucket.sizes";
107 
108   /**
109    * Defaults for Bucket cache
110    */
111   public static final boolean DEFAULT_BUCKET_CACHE_COMBINED = true;
112   public static final int DEFAULT_BUCKET_CACHE_WRITER_THREADS = 3;
113   public static final int DEFAULT_BUCKET_CACHE_WRITER_QUEUE = 64;
114 
115  /**
116    * Configuration key to prefetch all blocks of a given file into the block cache
117    * when the file is opened.
118    */
119   public static final String PREFETCH_BLOCKS_ON_OPEN_KEY =
120       "hbase.rs.prefetchblocksonopen";
121 
122   // Defaults
123 
124   public static final boolean DEFAULT_CACHE_DATA_ON_READ = true;
125   public static final boolean DEFAULT_CACHE_DATA_ON_WRITE = false;
126   public static final boolean DEFAULT_IN_MEMORY = false;
127   public static final boolean DEFAULT_CACHE_INDEXES_ON_WRITE = false;
128   public static final boolean DEFAULT_CACHE_BLOOMS_ON_WRITE = false;
129   public static final boolean DEFAULT_EVICT_ON_CLOSE = false;
130   public static final boolean DEFAULT_COMPRESSED_CACHE = false;
131   public static final boolean DEFAULT_PREFETCH_ON_OPEN = false;
132 
133   /** Local reference to the block cache, null if completely disabled */
134   private final BlockCache blockCache;
135 
136   /**
137    * Whether blocks should be cached on read (default is on if there is a
138    * cache but this can be turned off on a per-family or per-request basis).
139    * If off we will STILL cache meta blocks; i.e. INDEX and BLOOM types.
140    * This cannot be disabled.
141    */
142   private boolean cacheDataOnRead;
143 
144   /** Whether blocks should be flagged as in-memory when being cached */
145   private final boolean inMemory;
146 
147   /** Whether data blocks should be cached when new files are written */
148   private boolean cacheDataOnWrite;
149 
150   /** Whether index blocks should be cached when new files are written */
151   private final boolean cacheIndexesOnWrite;
152 
153   /** Whether compound bloom filter blocks should be cached on write */
154   private final boolean cacheBloomsOnWrite;
155 
156   /** Whether blocks of a file should be evicted when the file is closed */
157   private boolean evictOnClose;
158 
159   /** Whether data blocks should be stored in compressed form in the cache */
160   private final boolean cacheCompressed;
161 
162   /** Whether data blocks should be prefetched into the cache */
163   private final boolean prefetchOnOpen;
164 
165   /**
166    * If true and if more than one tier in this cache deploy -- e.g. CombinedBlockCache has an L1
167    * and an L2 tier -- then cache data blocks up in the L1 tier (The meta blocks are likely being
168    * cached up in L1 already.  At least this is the case if CombinedBlockCache).
169    */
170   private boolean cacheDataInL1;
171 
172   /**
173    * Create a cache configuration using the specified configuration object and
174    * family descriptor.
175    * @param conf hbase configuration
176    * @param family column family configuration
177    */
178   public CacheConfig(Configuration conf, HColumnDescriptor family) {
179     this(CacheConfig.instantiateBlockCache(conf),
180         family.isBlockCacheEnabled(),
181         family.isInMemory(),
182         // For the following flags we enable them regardless of per-schema settings
183         // if they are enabled in the global configuration.
184         conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY,
185             DEFAULT_CACHE_DATA_ON_WRITE) || family.shouldCacheDataOnWrite(),
186         conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY,
187             DEFAULT_CACHE_INDEXES_ON_WRITE) || family.shouldCacheIndexesOnWrite(),
188         conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY,
189             DEFAULT_CACHE_BLOOMS_ON_WRITE) || family.shouldCacheBloomsOnWrite(),
190         conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY,
191             DEFAULT_EVICT_ON_CLOSE) || family.shouldEvictBlocksOnClose(),
192         conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, DEFAULT_COMPRESSED_CACHE),
193         conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY,
194             DEFAULT_PREFETCH_ON_OPEN) || family.shouldPrefetchBlocksOnOpen(),
195         conf.getBoolean(HColumnDescriptor.CACHE_DATA_IN_L1,
196             HColumnDescriptor.DEFAULT_CACHE_DATA_IN_L1) || family.shouldCacheDataInL1()
197      );
198   }
199 
200   /**
201    * Create a cache configuration using the specified configuration object and
202    * defaults for family level settings.
203    * @param conf hbase configuration
204    */
205   public CacheConfig(Configuration conf) {
206     this(CacheConfig.instantiateBlockCache(conf),
207         DEFAULT_CACHE_DATA_ON_READ,
208         DEFAULT_IN_MEMORY, // This is a family-level setting so can't be set
209                            // strictly from conf
210         conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_DATA_ON_WRITE),
211         conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY,
212             DEFAULT_CACHE_INDEXES_ON_WRITE),
213             conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY,
214                 DEFAULT_CACHE_BLOOMS_ON_WRITE),
215         conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, DEFAULT_EVICT_ON_CLOSE),
216         conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY,
217             DEFAULT_COMPRESSED_CACHE),
218         conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY, DEFAULT_PREFETCH_ON_OPEN),
219         conf.getBoolean(HColumnDescriptor.CACHE_DATA_IN_L1,
220           HColumnDescriptor.DEFAULT_CACHE_DATA_IN_L1)
221      );
222   }
223 
224   /**
225    * Create a block cache configuration with the specified cache and
226    * configuration parameters.
227    * @param blockCache reference to block cache, null if completely disabled
228    * @param cacheDataOnRead whether DATA blocks should be cached on read (we always cache INDEX
229    * blocks and BLOOM blocks; this cannot be disabled).
230    * @param inMemory whether blocks should be flagged as in-memory
231    * @param cacheDataOnWrite whether data blocks should be cached on write
232    * @param cacheIndexesOnWrite whether index blocks should be cached on write
233    * @param cacheBloomsOnWrite whether blooms should be cached on write
234    * @param evictOnClose whether blocks should be evicted when HFile is closed
235    * @param cacheCompressed whether to store blocks as compressed in the cache
236    * @param prefetchOnOpen whether to prefetch blocks upon open
237    * @param cacheDataInL1 If more than one cache tier deployed, if true, cache this column families
238    * data blocks up in the L1 tier.
239    */
240   CacheConfig(final BlockCache blockCache,
241       final boolean cacheDataOnRead, final boolean inMemory,
242       final boolean cacheDataOnWrite, final boolean cacheIndexesOnWrite,
243       final boolean cacheBloomsOnWrite, final boolean evictOnClose,
244       final boolean cacheCompressed, final boolean prefetchOnOpen,
245       final boolean cacheDataInL1) {
246     this.blockCache = blockCache;
247     this.cacheDataOnRead = cacheDataOnRead;
248     this.inMemory = inMemory;
249     this.cacheDataOnWrite = cacheDataOnWrite;
250     this.cacheIndexesOnWrite = cacheIndexesOnWrite;
251     this.cacheBloomsOnWrite = cacheBloomsOnWrite;
252     this.evictOnClose = evictOnClose;
253     this.cacheCompressed = cacheCompressed;
254     this.prefetchOnOpen = prefetchOnOpen;
255     this.cacheDataInL1 = cacheDataInL1;
256     LOG.info(this);
257   }
258 
259   /**
260    * Constructs a cache configuration copied from the specified configuration.
261    * @param cacheConf
262    */
263   public CacheConfig(CacheConfig cacheConf) {
264     this(cacheConf.blockCache, cacheConf.cacheDataOnRead, cacheConf.inMemory,
265         cacheConf.cacheDataOnWrite, cacheConf.cacheIndexesOnWrite,
266         cacheConf.cacheBloomsOnWrite, cacheConf.evictOnClose,
267         cacheConf.cacheCompressed, cacheConf.prefetchOnOpen,
268         cacheConf.cacheDataInL1);
269   }
270 
271   /**
272    * Checks whether the block cache is enabled.
273    */
274   public boolean isBlockCacheEnabled() {
275     return this.blockCache != null;
276   }
277 
278   /**
279    * Returns the block cache.
280    * @return the block cache, or null if caching is completely disabled
281    */
282   public BlockCache getBlockCache() {
283     return this.blockCache;
284   }
285 
286   /**
287    * Returns whether the DATA blocks of this HFile should be cached on read or not (we always
288    * cache the meta blocks, the INDEX and BLOOM blocks).
289    * @return true if blocks should be cached on read, false if not
290    */
291   public boolean shouldCacheDataOnRead() {
292     return isBlockCacheEnabled() && cacheDataOnRead;
293   }
294 
295   /**
296    * Should we cache a block of a particular category? We always cache
297    * important blocks such as index blocks, as long as the block cache is
298    * available.
299    */
300   public boolean shouldCacheBlockOnRead(BlockCategory category) {
301     boolean shouldCache = isBlockCacheEnabled()
302         && (cacheDataOnRead ||
303             category == BlockCategory.INDEX ||
304             category == BlockCategory.BLOOM ||
305             (prefetchOnOpen &&
306                 (category != BlockCategory.META &&
307                  category != BlockCategory.UNKNOWN)));
308     return shouldCache;
309   }
310 
311   /**
312    * @return true if blocks in this file should be flagged as in-memory
313    */
314   public boolean isInMemory() {
315     return isBlockCacheEnabled() && this.inMemory;
316   }
317 
318   /**
319    * @return True if cache data blocks in L1 tier (if more than one tier in block cache deploy).
320    */
321   public boolean isCacheDataInL1() {
322     return isBlockCacheEnabled() && this.cacheDataInL1;
323   }
324 
325   /**
326    * @return true if data blocks should be written to the cache when an HFile is
327    *         written, false if not
328    */
329   public boolean shouldCacheDataOnWrite() {
330     return isBlockCacheEnabled() && this.cacheDataOnWrite;
331   }
332 
333   /**
334    * Only used for testing.
335    * @param cacheDataOnWrite whether data blocks should be written to the cache
336    *                         when an HFile is written
337    */
338   @VisibleForTesting
339   public void setCacheDataOnWrite(boolean cacheDataOnWrite) {
340     this.cacheDataOnWrite = cacheDataOnWrite;
341   }
342 
343   /**
344    * Only used for testing.
345    * @param cacheDataInL1 Whether to cache data blocks up in l1 (if a multi-tier cache
346    * implementation).
347    */
348   @VisibleForTesting
349   public void setCacheDataInL1(boolean cacheDataInL1) {
350     this.cacheDataInL1 = cacheDataInL1;
351   }
352 
353   /**
354    * @return true if index blocks should be written to the cache when an HFile
355    *         is written, false if not
356    */
357   public boolean shouldCacheIndexesOnWrite() {
358     return isBlockCacheEnabled() && this.cacheIndexesOnWrite;
359   }
360 
361   /**
362    * @return true if bloom blocks should be written to the cache when an HFile
363    *         is written, false if not
364    */
365   public boolean shouldCacheBloomsOnWrite() {
366     return isBlockCacheEnabled() && this.cacheBloomsOnWrite;
367   }
368 
369   /**
370    * @return true if blocks should be evicted from the cache when an HFile
371    *         reader is closed, false if not
372    */
373   public boolean shouldEvictOnClose() {
374     return isBlockCacheEnabled() && this.evictOnClose;
375   }
376 
377   /**
378    * Only used for testing.
379    * @param evictOnClose whether blocks should be evicted from the cache when an
380    *                     HFile reader is closed
381    */
382   public void setEvictOnClose(boolean evictOnClose) {
383     this.evictOnClose = evictOnClose;
384   }
385 
386   /**
387    * @return true if blocks should be compressed in the cache, false if not
388    */
389   public boolean shouldCacheCompressed() {
390     return isBlockCacheEnabled() && this.cacheCompressed;
391   }
392 
393   /**
394    * @return true if blocks should be prefetched into the cache on open, false if not
395    */
396   public boolean shouldPrefetchOnOpen() {
397     return isBlockCacheEnabled() && this.prefetchOnOpen;
398   }
399 
400   @Override
401   public String toString() {
402     if (!isBlockCacheEnabled()) {
403       return "CacheConfig:disabled";
404     }
405     return "blockCache=" + getBlockCache() +
406       ", cacheDataOnRead=" + shouldCacheDataOnRead() +
407       ", cacheDataOnWrite=" + shouldCacheDataOnWrite() +
408       ", cacheIndexesOnWrite=" + shouldCacheIndexesOnWrite() +
409       ", cacheBloomsOnWrite=" + shouldCacheBloomsOnWrite() +
410       ", cacheEvictOnClose=" + shouldEvictOnClose() +
411       ", cacheCompressed=" + shouldCacheCompressed() +
412       ", prefetchOnOpen=" + shouldPrefetchOnOpen();
413   }
414 
415   // Static block cache reference and methods
416 
417   /**
418    * Static reference to the block cache, or null if no caching should be used
419    * at all.
420    */
421   // Clear this if in tests you'd make more than one block cache instance.
422   @VisibleForTesting
423   static BlockCache GLOBAL_BLOCK_CACHE_INSTANCE;
424 
425   /** Boolean whether we have disabled the block cache entirely. */
426   @VisibleForTesting
427   static boolean blockCacheDisabled = false;
428 
429   static long getLruCacheSize(final Configuration conf, final MemoryUsage mu) {
430     float cachePercentage = conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY,
431       HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
432     if (cachePercentage <= 0.0001f) {
433       blockCacheDisabled = true;
434       return -1;
435     }
436     if (cachePercentage > 1.0) {
437       throw new IllegalArgumentException(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY +
438         " must be between 0.0 and 1.0, and not > 1.0");
439     }
440 
441     // Calculate the amount of heap to give the heap.
442     return (long) (mu.getMax() * cachePercentage);
443   }
444 
445   /**
446    * @param c Configuration to use.
447    * @param mu JMX Memory Bean
448    * @return An L1 instance.  Currently an instance of LruBlockCache.
449    */
450   private static LruBlockCache getL1(final Configuration c, final MemoryUsage mu) {
451     long lruCacheSize = getLruCacheSize(c, mu);
452     int blockSize = c.getInt("hbase.offheapcache.minblocksize", HConstants.DEFAULT_BLOCKSIZE);
453     LOG.info("Allocating LruBlockCache size=" +
454       StringUtils.byteDesc(lruCacheSize) + ", blockSize=" + StringUtils.byteDesc(blockSize));
455     return new LruBlockCache(lruCacheSize, blockSize, true, c);
456   }
457 
458   /**
459    * @param c Configuration to use.
460    * @param mu JMX Memory Bean
461    * @return Returns L2 block cache instance (for now it is BucketCache BlockCache all the time)
462    * or null if not supposed to be a L2.
463    */
464   private static BucketCache getL2(final Configuration c, final MemoryUsage mu) {
465     // Check for L2.  ioengine name must be non-null.
466     String bucketCacheIOEngineName = c.get(BUCKET_CACHE_IOENGINE_KEY, null);
467     if (bucketCacheIOEngineName == null || bucketCacheIOEngineName.length() <= 0) return null;
468 
469     int blockSize = c.getInt("hbase.offheapcache.minblocksize", HConstants.DEFAULT_BLOCKSIZE);
470     float bucketCachePercentage = c.getFloat(BUCKET_CACHE_SIZE_KEY, 0F);
471     long bucketCacheSize = (long) (bucketCachePercentage < 1? mu.getMax() * bucketCachePercentage:
472       bucketCachePercentage * 1024 * 1024);
473     if (bucketCacheSize <= 0) {
474       throw new IllegalStateException("bucketCacheSize <= 0; Check " +
475         BUCKET_CACHE_SIZE_KEY + " setting and/or server java heap size");
476     }
477     int writerThreads = c.getInt(BUCKET_CACHE_WRITER_THREADS_KEY,
478       DEFAULT_BUCKET_CACHE_WRITER_THREADS);
479     int writerQueueLen = c.getInt(BUCKET_CACHE_WRITER_QUEUE_KEY,
480       DEFAULT_BUCKET_CACHE_WRITER_QUEUE);
481     String persistentPath = c.get(BUCKET_CACHE_PERSISTENT_PATH_KEY);
482     String[] configuredBucketSizes = c.getStrings(BUCKET_CACHE_BUCKETS_KEY);
483     int [] bucketSizes = null;
484     if (configuredBucketSizes != null) {
485       bucketSizes = new int[configuredBucketSizes.length];
486       for (int i = 0; i < configuredBucketSizes.length; i++) {
487         bucketSizes[i] = Integer.parseInt(configuredBucketSizes[i].trim());
488       }
489     }
490     BucketCache bucketCache = null;
491     try {
492       int ioErrorsTolerationDuration = c.getInt(
493         "hbase.bucketcache.ioengine.errors.tolerated.duration",
494         BucketCache.DEFAULT_ERROR_TOLERATION_DURATION);
495       // Bucket cache logs its stats on creation internal to the constructor.
496       bucketCache = new BucketCache(bucketCacheIOEngineName,
497         bucketCacheSize, blockSize, bucketSizes, writerThreads, writerQueueLen, persistentPath,
498         ioErrorsTolerationDuration);
499     } catch (IOException ioex) {
500       LOG.error("Can't instantiate bucket cache", ioex); throw new RuntimeException(ioex);
501     }
502     return bucketCache;
503   }
504 
505   /**
506    * Returns the block cache or <code>null</code> in case none should be used.
507    * Sets GLOBAL_BLOCK_CACHE_INSTANCE
508    *
509    * @param conf  The current configuration.
510    * @return The block cache or <code>null</code>.
511    */
512   public static synchronized BlockCache instantiateBlockCache(Configuration conf) {
513     if (GLOBAL_BLOCK_CACHE_INSTANCE != null) return GLOBAL_BLOCK_CACHE_INSTANCE;
514     if (blockCacheDisabled) return null;
515     MemoryUsage mu = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
516     LruBlockCache l1 = getL1(conf, mu);
517     BucketCache l2 = getL2(conf, mu);
518     if (l2 == null) {
519       GLOBAL_BLOCK_CACHE_INSTANCE = l1;
520     } else {
521       boolean combinedWithLru = conf.getBoolean(BUCKET_CACHE_COMBINED_KEY,
522         DEFAULT_BUCKET_CACHE_COMBINED);
523       if (combinedWithLru) {
524         GLOBAL_BLOCK_CACHE_INSTANCE = new CombinedBlockCache(l1, l2);
525       } else {
526         // L1 and L2 are not 'combined'.  They are connected via the LruBlockCache victimhandler
527         // mechanism.  It is a little ugly but works according to the following: when the
528         // background eviction thread runs, blocks evicted from L1 will go to L2 AND when we get
529         // a block from the L1 cache, if not in L1, we will search L2.
530         l1.setVictimCache(l2);
531         GLOBAL_BLOCK_CACHE_INSTANCE = l1;
532       }
533     }
534     return GLOBAL_BLOCK_CACHE_INSTANCE;
535   }
536 }