001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.io.hfile; 019 020import java.util.Optional; 021 022import org.apache.hadoop.conf.Configuration; 023import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; 024import org.apache.hadoop.hbase.io.ByteBuffAllocator; 025import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory; 026import org.apache.yetus.audience.InterfaceAudience; 027import org.slf4j.Logger; 028import org.slf4j.LoggerFactory; 029 030import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting; 031 032/** 033 * Stores all of the cache objects and configuration for a single HFile. 034 */ 035@InterfaceAudience.Private 036public class CacheConfig { 037 private static final Logger LOG = LoggerFactory.getLogger(CacheConfig.class.getName()); 038 039 /** 040 * Disabled cache configuration 041 */ 042 public static final CacheConfig DISABLED = new CacheConfig(); 043 044 /** 045 * Configuration key to cache data blocks on read. Bloom blocks and index blocks are always be 046 * cached if the block cache is enabled. 047 */ 048 public static final String CACHE_DATA_ON_READ_KEY = "hbase.block.data.cacheonread"; 049 050 /** 051 * Configuration key to cache data blocks on write. There are separate 052 * switches for bloom blocks and non-root index blocks. 053 */ 054 public static final String CACHE_BLOCKS_ON_WRITE_KEY = "hbase.rs.cacheblocksonwrite"; 055 056 /** 057 * Configuration key to cache leaf and intermediate-level index blocks on 058 * write. 059 */ 060 public static final String CACHE_INDEX_BLOCKS_ON_WRITE_KEY = "hfile.block.index.cacheonwrite"; 061 062 /** 063 * Configuration key to cache compound bloom filter blocks on write. 064 */ 065 public static final String CACHE_BLOOM_BLOCKS_ON_WRITE_KEY = "hfile.block.bloom.cacheonwrite"; 066 067 /** 068 * Configuration key to cache data blocks in compressed and/or encrypted format. 069 */ 070 public static final String CACHE_DATA_BLOCKS_COMPRESSED_KEY = "hbase.block.data.cachecompressed"; 071 072 /** 073 * Configuration key to evict all blocks of a given file from the block cache 074 * when the file is closed. 075 */ 076 public static final String EVICT_BLOCKS_ON_CLOSE_KEY = "hbase.rs.evictblocksonclose"; 077 078 /** 079 * Configuration key to prefetch all blocks of a given file into the block cache 080 * when the file is opened. 081 */ 082 public static final String PREFETCH_BLOCKS_ON_OPEN_KEY = "hbase.rs.prefetchblocksonopen"; 083 084 /** 085 * Configuration key to cache blocks when a compacted file is written 086 */ 087 public static final String CACHE_COMPACTED_BLOCKS_ON_WRITE_KEY = 088 "hbase.rs.cachecompactedblocksonwrite"; 089 090 /** 091 * Configuration key to determine total size in bytes of compacted files beyond which we do not 092 * cache blocks on compaction 093 */ 094 public static final String CACHE_COMPACTED_BLOCKS_ON_WRITE_THRESHOLD_KEY = 095 "hbase.rs.cachecompactedblocksonwrite.threshold"; 096 097 public static final String DROP_BEHIND_CACHE_COMPACTION_KEY = 098 "hbase.hfile.drop.behind.compaction"; 099 100 // Defaults 101 public static final boolean DEFAULT_CACHE_DATA_ON_READ = true; 102 public static final boolean DEFAULT_CACHE_DATA_ON_WRITE = false; 103 public static final boolean DEFAULT_IN_MEMORY = false; 104 public static final boolean DEFAULT_CACHE_INDEXES_ON_WRITE = false; 105 public static final boolean DEFAULT_CACHE_BLOOMS_ON_WRITE = false; 106 public static final boolean DEFAULT_EVICT_ON_CLOSE = false; 107 public static final boolean DEFAULT_CACHE_DATA_COMPRESSED = false; 108 public static final boolean DEFAULT_PREFETCH_ON_OPEN = false; 109 public static final boolean DEFAULT_CACHE_COMPACTED_BLOCKS_ON_WRITE = false; 110 public static final boolean DROP_BEHIND_CACHE_COMPACTION_DEFAULT = true; 111 public static final long DEFAULT_CACHE_COMPACTED_BLOCKS_ON_WRITE_THRESHOLD = Long.MAX_VALUE; 112 113 /** 114 * Whether blocks should be cached on read (default is on if there is a 115 * cache but this can be turned off on a per-family or per-request basis). 116 * If off we will STILL cache meta blocks; i.e. INDEX and BLOOM types. 117 * This cannot be disabled. 118 */ 119 private final boolean cacheDataOnRead; 120 121 /** Whether blocks should be flagged as in-memory when being cached */ 122 private final boolean inMemory; 123 124 /** Whether data blocks should be cached when new files are written */ 125 private boolean cacheDataOnWrite; 126 127 /** Whether index blocks should be cached when new files are written */ 128 private boolean cacheIndexesOnWrite; 129 130 /** Whether compound bloom filter blocks should be cached on write */ 131 private boolean cacheBloomsOnWrite; 132 133 /** Whether blocks of a file should be evicted when the file is closed */ 134 private volatile boolean evictOnClose; 135 136 /** Whether data blocks should be stored in compressed and/or encrypted form in the cache */ 137 private final boolean cacheDataCompressed; 138 139 /** Whether data blocks should be prefetched into the cache */ 140 private final boolean prefetchOnOpen; 141 142 /** 143 * Whether data blocks should be cached when compacted file is written 144 */ 145 private final boolean cacheCompactedDataOnWrite; 146 147 /** 148 * Determine threshold beyond which we do not cache blocks on compaction 149 */ 150 private long cacheCompactedDataOnWriteThreshold; 151 152 private final boolean dropBehindCompaction; 153 154 // Local reference to the block cache 155 private final BlockCache blockCache; 156 157 private final ByteBuffAllocator byteBuffAllocator; 158 159 /** 160 * Create a cache configuration using the specified configuration object and 161 * defaults for family level settings. Only use if no column family context. 162 * @param conf hbase configuration 163 */ 164 public CacheConfig(Configuration conf) { 165 this(conf, null); 166 } 167 168 public CacheConfig(Configuration conf, BlockCache blockCache) { 169 this(conf, null, blockCache, ByteBuffAllocator.HEAP); 170 } 171 172 /** 173 * Create a cache configuration using the specified configuration object and 174 * family descriptor. 175 * @param conf hbase configuration 176 * @param family column family configuration 177 */ 178 public CacheConfig(Configuration conf, ColumnFamilyDescriptor family, BlockCache blockCache, 179 ByteBuffAllocator byteBuffAllocator) { 180 this.cacheDataOnRead = conf.getBoolean(CACHE_DATA_ON_READ_KEY, DEFAULT_CACHE_DATA_ON_READ) && 181 (family == null ? true : family.isBlockCacheEnabled()); 182 this.inMemory = family == null ? DEFAULT_IN_MEMORY : family.isInMemory(); 183 this.cacheDataCompressed = 184 conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, DEFAULT_CACHE_DATA_COMPRESSED); 185 this.dropBehindCompaction = 186 conf.getBoolean(DROP_BEHIND_CACHE_COMPACTION_KEY, DROP_BEHIND_CACHE_COMPACTION_DEFAULT); 187 // For the following flags we enable them regardless of per-schema settings 188 // if they are enabled in the global configuration. 189 this.cacheDataOnWrite = 190 conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_DATA_ON_WRITE) || 191 (family == null ? false : family.isCacheDataOnWrite()); 192 this.cacheIndexesOnWrite = 193 conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_INDEXES_ON_WRITE) || 194 (family == null ? false : family.isCacheIndexesOnWrite()); 195 this.cacheBloomsOnWrite = 196 conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_BLOOMS_ON_WRITE) || 197 (family == null ? false : family.isCacheBloomsOnWrite()); 198 this.evictOnClose = conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, DEFAULT_EVICT_ON_CLOSE) || 199 (family == null ? false : family.isEvictBlocksOnClose()); 200 this.prefetchOnOpen = conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY, DEFAULT_PREFETCH_ON_OPEN) || 201 (family == null ? false : family.isPrefetchBlocksOnOpen()); 202 this.cacheCompactedDataOnWrite = conf.getBoolean(CACHE_COMPACTED_BLOCKS_ON_WRITE_KEY, 203 DEFAULT_CACHE_COMPACTED_BLOCKS_ON_WRITE); 204 this.cacheCompactedDataOnWriteThreshold = getCacheCompactedBlocksOnWriteThreshold(conf); 205 this.blockCache = blockCache; 206 this.byteBuffAllocator = byteBuffAllocator; 207 } 208 209 /** 210 * Constructs a cache configuration copied from the specified configuration. 211 */ 212 public CacheConfig(CacheConfig cacheConf) { 213 this.cacheDataOnRead = cacheConf.cacheDataOnRead; 214 this.inMemory = cacheConf.inMemory; 215 this.cacheDataOnWrite = cacheConf.cacheDataOnWrite; 216 this.cacheIndexesOnWrite = cacheConf.cacheIndexesOnWrite; 217 this.cacheBloomsOnWrite = cacheConf.cacheBloomsOnWrite; 218 this.evictOnClose = cacheConf.evictOnClose; 219 this.cacheDataCompressed = cacheConf.cacheDataCompressed; 220 this.prefetchOnOpen = cacheConf.prefetchOnOpen; 221 this.cacheCompactedDataOnWrite = cacheConf.cacheCompactedDataOnWrite; 222 this.cacheCompactedDataOnWriteThreshold = cacheConf.cacheCompactedDataOnWriteThreshold; 223 this.dropBehindCompaction = cacheConf.dropBehindCompaction; 224 this.blockCache = cacheConf.blockCache; 225 this.byteBuffAllocator = cacheConf.byteBuffAllocator; 226 } 227 228 private CacheConfig() { 229 this.cacheDataOnRead = false; 230 this.inMemory = false; 231 this.cacheDataOnWrite = false; 232 this.cacheIndexesOnWrite = false; 233 this.cacheBloomsOnWrite = false; 234 this.evictOnClose = false; 235 this.cacheDataCompressed = false; 236 this.prefetchOnOpen = false; 237 this.cacheCompactedDataOnWrite = false; 238 this.dropBehindCompaction = false; 239 this.blockCache = null; 240 this.byteBuffAllocator = ByteBuffAllocator.HEAP; 241 } 242 243 /** 244 * Returns whether the DATA blocks of this HFile should be cached on read or not (we always 245 * cache the meta blocks, the INDEX and BLOOM blocks). 246 * @return true if blocks should be cached on read, false if not 247 */ 248 public boolean shouldCacheDataOnRead() { 249 return cacheDataOnRead; 250 } 251 252 public boolean shouldDropBehindCompaction() { 253 return dropBehindCompaction; 254 } 255 256 /** 257 * Should we cache a block of a particular category? We always cache 258 * important blocks such as index blocks, as long as the block cache is 259 * available. 260 */ 261 public boolean shouldCacheBlockOnRead(BlockCategory category) { 262 return cacheDataOnRead || category == BlockCategory.INDEX || category == BlockCategory.BLOOM || 263 (prefetchOnOpen && (category != BlockCategory.META && category != BlockCategory.UNKNOWN)); 264 } 265 266 /** 267 * @return true if blocks in this file should be flagged as in-memory 268 */ 269 public boolean isInMemory() { 270 return this.inMemory; 271 } 272 273 /** 274 * @return true if data blocks should be written to the cache when an HFile is 275 * written, false if not 276 */ 277 public boolean shouldCacheDataOnWrite() { 278 return this.cacheDataOnWrite; 279 } 280 281 /** 282 * @param cacheDataOnWrite whether data blocks should be written to the cache 283 * when an HFile is written 284 */ 285 @VisibleForTesting 286 public void setCacheDataOnWrite(boolean cacheDataOnWrite) { 287 this.cacheDataOnWrite = cacheDataOnWrite; 288 } 289 290 /** 291 * Enable cache on write including: 292 * cacheDataOnWrite 293 * cacheIndexesOnWrite 294 * cacheBloomsOnWrite 295 */ 296 public void enableCacheOnWrite() { 297 this.cacheDataOnWrite = true; 298 this.cacheIndexesOnWrite = true; 299 this.cacheBloomsOnWrite = true; 300 } 301 302 /** 303 * @return true if index blocks should be written to the cache when an HFile 304 * is written, false if not 305 */ 306 public boolean shouldCacheIndexesOnWrite() { 307 return this.cacheIndexesOnWrite; 308 } 309 310 /** 311 * @return true if bloom blocks should be written to the cache when an HFile 312 * is written, false if not 313 */ 314 public boolean shouldCacheBloomsOnWrite() { 315 return this.cacheBloomsOnWrite; 316 } 317 318 /** 319 * @return true if blocks should be evicted from the cache when an HFile 320 * reader is closed, false if not 321 */ 322 public boolean shouldEvictOnClose() { 323 return this.evictOnClose; 324 } 325 326 /** 327 * Only used for testing. 328 * @param evictOnClose whether blocks should be evicted from the cache when an 329 * HFile reader is closed 330 */ 331 @VisibleForTesting 332 public void setEvictOnClose(boolean evictOnClose) { 333 this.evictOnClose = evictOnClose; 334 } 335 336 /** 337 * @return true if data blocks should be compressed in the cache, false if not 338 */ 339 public boolean shouldCacheDataCompressed() { 340 return this.cacheDataOnRead && this.cacheDataCompressed; 341 } 342 343 /** 344 * @return true if this {@link BlockCategory} should be compressed in blockcache, false otherwise 345 */ 346 public boolean shouldCacheCompressed(BlockCategory category) { 347 switch (category) { 348 case DATA: 349 return this.cacheDataOnRead && this.cacheDataCompressed; 350 default: 351 return false; 352 } 353 } 354 355 /** 356 * @return true if blocks should be prefetched into the cache on open, false if not 357 */ 358 public boolean shouldPrefetchOnOpen() { 359 return this.prefetchOnOpen; 360 } 361 362 /** 363 * @return true if blocks should be cached while writing during compaction, false if not 364 */ 365 public boolean shouldCacheCompactedBlocksOnWrite() { 366 return this.cacheCompactedDataOnWrite; 367 } 368 369 /** 370 * @return total file size in bytes threshold for caching while writing during compaction 371 */ 372 public long getCacheCompactedBlocksOnWriteThreshold() { 373 return this.cacheCompactedDataOnWriteThreshold; 374 } 375 /** 376 * Return true if we may find this type of block in block cache. 377 * <p> 378 * TODO: today {@code family.isBlockCacheEnabled()} only means {@code cacheDataOnRead}, so here we 379 * consider lots of other configurations such as {@code cacheDataOnWrite}. We should fix this in 380 * the future, {@code cacheDataOnWrite} should honor the CF level {@code isBlockCacheEnabled} 381 * configuration. 382 */ 383 public boolean shouldReadBlockFromCache(BlockType blockType) { 384 if (cacheDataOnRead) { 385 return true; 386 } 387 if (prefetchOnOpen) { 388 return true; 389 } 390 if (cacheDataOnWrite) { 391 return true; 392 } 393 if (blockType == null) { 394 return true; 395 } 396 if (blockType.getCategory() == BlockCategory.BLOOM || 397 blockType.getCategory() == BlockCategory.INDEX) { 398 return true; 399 } 400 return false; 401 } 402 403 /** 404 * If we make sure the block could not be cached, we will not acquire the lock 405 * otherwise we will acquire lock 406 */ 407 public boolean shouldLockOnCacheMiss(BlockType blockType) { 408 if (blockType == null) { 409 return true; 410 } 411 return shouldCacheBlockOnRead(blockType.getCategory()); 412 } 413 414 /** 415 * Returns the block cache. 416 * 417 * @return the block cache, or null if caching is completely disabled 418 */ 419 public Optional<BlockCache> getBlockCache() { 420 return Optional.ofNullable(this.blockCache); 421 } 422 423 public boolean isCombinedBlockCache() { 424 return blockCache instanceof CombinedBlockCache; 425 } 426 427 public ByteBuffAllocator getByteBuffAllocator() { 428 return this.byteBuffAllocator; 429 } 430 431 private long getCacheCompactedBlocksOnWriteThreshold(Configuration conf) { 432 long cacheCompactedBlocksOnWriteThreshold = conf 433 .getLong(CACHE_COMPACTED_BLOCKS_ON_WRITE_THRESHOLD_KEY, 434 DEFAULT_CACHE_COMPACTED_BLOCKS_ON_WRITE_THRESHOLD); 435 436 if (cacheCompactedBlocksOnWriteThreshold < 0) { 437 LOG.warn( 438 "cacheCompactedBlocksOnWriteThreshold value : {} is less than 0, resetting it to: {}", 439 cacheCompactedBlocksOnWriteThreshold, DEFAULT_CACHE_COMPACTED_BLOCKS_ON_WRITE_THRESHOLD); 440 cacheCompactedBlocksOnWriteThreshold = DEFAULT_CACHE_COMPACTED_BLOCKS_ON_WRITE_THRESHOLD; 441 } 442 443 return cacheCompactedBlocksOnWriteThreshold; 444 } 445 446 @Override 447 public String toString() { 448 return "cacheDataOnRead=" + shouldCacheDataOnRead() + ", cacheDataOnWrite=" 449 + shouldCacheDataOnWrite() + ", cacheIndexesOnWrite=" + shouldCacheIndexesOnWrite() 450 + ", cacheBloomsOnWrite=" + shouldCacheBloomsOnWrite() + ", cacheEvictOnClose=" 451 + shouldEvictOnClose() + ", cacheDataCompressed=" + shouldCacheDataCompressed() 452 + ", prefetchOnOpen=" + shouldPrefetchOnOpen(); 453 } 454}