001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.io.hfile; 019 020import java.util.Optional; 021import org.apache.hadoop.conf.Configuration; 022import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; 023import org.apache.hadoop.hbase.io.ByteBuffAllocator; 024import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory; 025import org.apache.yetus.audience.InterfaceAudience; 026import org.slf4j.Logger; 027import org.slf4j.LoggerFactory; 028 029/** 030 * Stores all of the cache objects and configuration for a single HFile. 031 */ 032@InterfaceAudience.Private 033public class CacheConfig { 034 private static final Logger LOG = LoggerFactory.getLogger(CacheConfig.class.getName()); 035 036 /** 037 * Disabled cache configuration 038 */ 039 public static final CacheConfig DISABLED = new CacheConfig(); 040 041 /** 042 * Configuration key to cache data blocks on read. Bloom blocks and index blocks are always be 043 * cached if the block cache is enabled. 044 */ 045 public static final String CACHE_DATA_ON_READ_KEY = "hbase.block.data.cacheonread"; 046 047 /** 048 * Configuration key to cache data blocks on write. There are separate switches for bloom blocks 049 * and non-root index blocks. 050 */ 051 public static final String CACHE_BLOCKS_ON_WRITE_KEY = "hbase.rs.cacheblocksonwrite"; 052 053 /** 054 * Configuration key to cache leaf and intermediate-level index blocks on write. 055 */ 056 public static final String CACHE_INDEX_BLOCKS_ON_WRITE_KEY = "hfile.block.index.cacheonwrite"; 057 058 /** 059 * Configuration key to cache compound bloom filter blocks on write. 060 */ 061 public static final String CACHE_BLOOM_BLOCKS_ON_WRITE_KEY = "hfile.block.bloom.cacheonwrite"; 062 063 /** 064 * Configuration key to cache data blocks in compressed and/or encrypted format. 065 */ 066 public static final String CACHE_DATA_BLOCKS_COMPRESSED_KEY = "hbase.block.data.cachecompressed"; 067 068 /** 069 * Configuration key to evict all blocks of a given file from the block cache when the file is 070 * closed. 071 */ 072 public static final String EVICT_BLOCKS_ON_CLOSE_KEY = "hbase.rs.evictblocksonclose"; 073 074 /** 075 * Configuration key to prefetch all blocks of a given file into the block cache when the file is 076 * opened. 077 */ 078 public static final String PREFETCH_BLOCKS_ON_OPEN_KEY = "hbase.rs.prefetchblocksonopen"; 079 080 /** 081 * Configuration key to cache blocks when a compacted file is written 082 */ 083 public static final String CACHE_COMPACTED_BLOCKS_ON_WRITE_KEY = 084 "hbase.rs.cachecompactedblocksonwrite"; 085 086 /** 087 * Configuration key to determine total size in bytes of compacted files beyond which we do not 088 * cache blocks on compaction 089 */ 090 public static final String CACHE_COMPACTED_BLOCKS_ON_WRITE_THRESHOLD_KEY = 091 "hbase.rs.cachecompactedblocksonwrite.threshold"; 092 093 public static final String DROP_BEHIND_CACHE_COMPACTION_KEY = 094 "hbase.hfile.drop.behind.compaction"; 095 096 // Defaults 097 public static final boolean DEFAULT_CACHE_DATA_ON_READ = true; 098 public static final boolean DEFAULT_CACHE_DATA_ON_WRITE = false; 099 public static final boolean DEFAULT_IN_MEMORY = false; 100 public static final boolean DEFAULT_CACHE_INDEXES_ON_WRITE = false; 101 public static final boolean DEFAULT_CACHE_BLOOMS_ON_WRITE = false; 102 public static final boolean DEFAULT_EVICT_ON_CLOSE = false; 103 public static final boolean DEFAULT_CACHE_DATA_COMPRESSED = false; 104 public static final boolean DEFAULT_PREFETCH_ON_OPEN = false; 105 public static final boolean DEFAULT_CACHE_COMPACTED_BLOCKS_ON_WRITE = false; 106 public static final boolean DROP_BEHIND_CACHE_COMPACTION_DEFAULT = true; 107 public static final long DEFAULT_CACHE_COMPACTED_BLOCKS_ON_WRITE_THRESHOLD = Long.MAX_VALUE; 108 109 /** 110 * Whether blocks should be cached on read (default is on if there is a cache but this can be 111 * turned off on a per-family or per-request basis). If off we will STILL cache meta blocks; i.e. 112 * INDEX and BLOOM types. This cannot be disabled. 113 */ 114 private final boolean cacheDataOnRead; 115 116 /** Whether blocks should be flagged as in-memory when being cached */ 117 private final boolean inMemory; 118 119 /** Whether data blocks should be cached when new files are written */ 120 private boolean cacheDataOnWrite; 121 122 /** Whether index blocks should be cached when new files are written */ 123 private boolean cacheIndexesOnWrite; 124 125 /** Whether compound bloom filter blocks should be cached on write */ 126 private boolean cacheBloomsOnWrite; 127 128 /** Whether blocks of a file should be evicted when the file is closed */ 129 private volatile boolean evictOnClose; 130 131 /** Whether data blocks should be stored in compressed and/or encrypted form in the cache */ 132 private final boolean cacheDataCompressed; 133 134 /** Whether data blocks should be prefetched into the cache */ 135 private final boolean prefetchOnOpen; 136 137 /** 138 * Whether data blocks should be cached when compacted file is written 139 */ 140 private final boolean cacheCompactedDataOnWrite; 141 142 /** 143 * Determine threshold beyond which we do not cache blocks on compaction 144 */ 145 private long cacheCompactedDataOnWriteThreshold; 146 147 private final boolean dropBehindCompaction; 148 149 // Local reference to the block cache 150 private final BlockCache blockCache; 151 152 private final ByteBuffAllocator byteBuffAllocator; 153 154 /** 155 * Create a cache configuration using the specified configuration object and defaults for family 156 * level settings. Only use if no column family context. 157 * @param conf hbase configuration 158 */ 159 public CacheConfig(Configuration conf) { 160 this(conf, null); 161 } 162 163 public CacheConfig(Configuration conf, BlockCache blockCache) { 164 this(conf, null, blockCache, ByteBuffAllocator.HEAP); 165 } 166 167 /** 168 * Create a cache configuration using the specified configuration object and family descriptor. 169 * @param conf hbase configuration 170 * @param family column family configuration 171 */ 172 public CacheConfig(Configuration conf, ColumnFamilyDescriptor family, BlockCache blockCache, 173 ByteBuffAllocator byteBuffAllocator) { 174 this.cacheDataOnRead = conf.getBoolean(CACHE_DATA_ON_READ_KEY, DEFAULT_CACHE_DATA_ON_READ) 175 && (family == null ? true : family.isBlockCacheEnabled()); 176 this.inMemory = family == null ? DEFAULT_IN_MEMORY : family.isInMemory(); 177 this.cacheDataCompressed = 178 conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, DEFAULT_CACHE_DATA_COMPRESSED); 179 this.dropBehindCompaction = 180 conf.getBoolean(DROP_BEHIND_CACHE_COMPACTION_KEY, DROP_BEHIND_CACHE_COMPACTION_DEFAULT); 181 // For the following flags we enable them regardless of per-schema settings 182 // if they are enabled in the global configuration. 183 this.cacheDataOnWrite = conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_DATA_ON_WRITE) 184 || (family == null ? false : family.isCacheDataOnWrite()); 185 this.cacheIndexesOnWrite = 186 conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_INDEXES_ON_WRITE) 187 || (family == null ? false : family.isCacheIndexesOnWrite()); 188 this.cacheBloomsOnWrite = 189 conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_BLOOMS_ON_WRITE) 190 || (family == null ? false : family.isCacheBloomsOnWrite()); 191 this.evictOnClose = conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, DEFAULT_EVICT_ON_CLOSE) 192 || (family == null ? false : family.isEvictBlocksOnClose()); 193 this.prefetchOnOpen = conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY, DEFAULT_PREFETCH_ON_OPEN) 194 || (family == null ? false : family.isPrefetchBlocksOnOpen()); 195 this.cacheCompactedDataOnWrite = 196 conf.getBoolean(CACHE_COMPACTED_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_COMPACTED_BLOCKS_ON_WRITE); 197 this.cacheCompactedDataOnWriteThreshold = getCacheCompactedBlocksOnWriteThreshold(conf); 198 this.blockCache = blockCache; 199 this.byteBuffAllocator = byteBuffAllocator; 200 } 201 202 /** 203 * Constructs a cache configuration copied from the specified configuration. 204 */ 205 public CacheConfig(CacheConfig cacheConf) { 206 this.cacheDataOnRead = cacheConf.cacheDataOnRead; 207 this.inMemory = cacheConf.inMemory; 208 this.cacheDataOnWrite = cacheConf.cacheDataOnWrite; 209 this.cacheIndexesOnWrite = cacheConf.cacheIndexesOnWrite; 210 this.cacheBloomsOnWrite = cacheConf.cacheBloomsOnWrite; 211 this.evictOnClose = cacheConf.evictOnClose; 212 this.cacheDataCompressed = cacheConf.cacheDataCompressed; 213 this.prefetchOnOpen = cacheConf.prefetchOnOpen; 214 this.cacheCompactedDataOnWrite = cacheConf.cacheCompactedDataOnWrite; 215 this.cacheCompactedDataOnWriteThreshold = cacheConf.cacheCompactedDataOnWriteThreshold; 216 this.dropBehindCompaction = cacheConf.dropBehindCompaction; 217 this.blockCache = cacheConf.blockCache; 218 this.byteBuffAllocator = cacheConf.byteBuffAllocator; 219 } 220 221 private CacheConfig() { 222 this.cacheDataOnRead = false; 223 this.inMemory = false; 224 this.cacheDataOnWrite = false; 225 this.cacheIndexesOnWrite = false; 226 this.cacheBloomsOnWrite = false; 227 this.evictOnClose = false; 228 this.cacheDataCompressed = false; 229 this.prefetchOnOpen = false; 230 this.cacheCompactedDataOnWrite = false; 231 this.dropBehindCompaction = false; 232 this.blockCache = null; 233 this.byteBuffAllocator = ByteBuffAllocator.HEAP; 234 } 235 236 /** 237 * Returns whether the DATA blocks of this HFile should be cached on read or not (we always cache 238 * the meta blocks, the INDEX and BLOOM blocks). 239 * @return true if blocks should be cached on read, false if not 240 */ 241 public boolean shouldCacheDataOnRead() { 242 return cacheDataOnRead; 243 } 244 245 public boolean shouldDropBehindCompaction() { 246 return dropBehindCompaction; 247 } 248 249 /** 250 * Should we cache a block of a particular category? We always cache important blocks such as 251 * index blocks, as long as the block cache is available. 252 */ 253 public boolean shouldCacheBlockOnRead(BlockCategory category) { 254 return cacheDataOnRead || category == BlockCategory.INDEX || category == BlockCategory.BLOOM 255 || (prefetchOnOpen && (category != BlockCategory.META && category != BlockCategory.UNKNOWN)); 256 } 257 258 /** Returns true if blocks in this file should be flagged as in-memory */ 259 public boolean isInMemory() { 260 return this.inMemory; 261 } 262 263 /** 264 * @return true if data blocks should be written to the cache when an HFile is written, false if 265 * not 266 */ 267 public boolean shouldCacheDataOnWrite() { 268 return this.cacheDataOnWrite; 269 } 270 271 /** 272 * @param cacheDataOnWrite whether data blocks should be written to the cache when an HFile is 273 * written 274 */ 275 public void setCacheDataOnWrite(boolean cacheDataOnWrite) { 276 this.cacheDataOnWrite = cacheDataOnWrite; 277 } 278 279 /** 280 * Enable cache on write including: cacheDataOnWrite cacheIndexesOnWrite cacheBloomsOnWrite 281 */ 282 public void enableCacheOnWrite() { 283 this.cacheDataOnWrite = true; 284 this.cacheIndexesOnWrite = true; 285 this.cacheBloomsOnWrite = true; 286 } 287 288 /** 289 * @return true if index blocks should be written to the cache when an HFile is written, false if 290 * not 291 */ 292 public boolean shouldCacheIndexesOnWrite() { 293 return this.cacheIndexesOnWrite; 294 } 295 296 /** 297 * @return true if bloom blocks should be written to the cache when an HFile is written, false if 298 * not 299 */ 300 public boolean shouldCacheBloomsOnWrite() { 301 return this.cacheBloomsOnWrite; 302 } 303 304 /** 305 * @return true if blocks should be evicted from the cache when an HFile reader is closed, false 306 * if not 307 */ 308 public boolean shouldEvictOnClose() { 309 return this.evictOnClose; 310 } 311 312 /** 313 * Only used for testing. 314 * @param evictOnClose whether blocks should be evicted from the cache when an HFile reader is 315 * closed 316 */ 317 public void setEvictOnClose(boolean evictOnClose) { 318 this.evictOnClose = evictOnClose; 319 } 320 321 /** Returns true if data blocks should be compressed in the cache, false if not */ 322 public boolean shouldCacheDataCompressed() { 323 return this.cacheDataOnRead && this.cacheDataCompressed; 324 } 325 326 /** 327 * Returns true if this {@link BlockCategory} should be compressed in blockcache, false otherwise 328 */ 329 public boolean shouldCacheCompressed(BlockCategory category) { 330 switch (category) { 331 case DATA: 332 return this.cacheDataOnRead && this.cacheDataCompressed; 333 default: 334 return false; 335 } 336 } 337 338 /** Returns true if blocks should be prefetched into the cache on open, false if not */ 339 public boolean shouldPrefetchOnOpen() { 340 return this.prefetchOnOpen; 341 } 342 343 /** Returns true if blocks should be cached while writing during compaction, false if not */ 344 public boolean shouldCacheCompactedBlocksOnWrite() { 345 return this.cacheCompactedDataOnWrite; 346 } 347 348 /** Returns total file size in bytes threshold for caching while writing during compaction */ 349 public long getCacheCompactedBlocksOnWriteThreshold() { 350 return this.cacheCompactedDataOnWriteThreshold; 351 } 352 353 /** 354 * Return true if we may find this type of block in block cache. 355 * <p> 356 * TODO: today {@code family.isBlockCacheEnabled()} only means {@code cacheDataOnRead}, so here we 357 * consider lots of other configurations such as {@code cacheDataOnWrite}. We should fix this in 358 * the future, {@code cacheDataOnWrite} should honor the CF level {@code isBlockCacheEnabled} 359 * configuration. 360 */ 361 public boolean shouldReadBlockFromCache(BlockType blockType) { 362 if (cacheDataOnRead) { 363 return true; 364 } 365 if (prefetchOnOpen) { 366 return true; 367 } 368 if (cacheDataOnWrite) { 369 return true; 370 } 371 if (blockType == null) { 372 return true; 373 } 374 if ( 375 blockType.getCategory() == BlockCategory.BLOOM 376 || blockType.getCategory() == BlockCategory.INDEX 377 ) { 378 return true; 379 } 380 return false; 381 } 382 383 /** 384 * If we make sure the block could not be cached, we will not acquire the lock otherwise we will 385 * acquire lock 386 */ 387 public boolean shouldLockOnCacheMiss(BlockType blockType) { 388 if (blockType == null) { 389 return true; 390 } 391 return shouldCacheBlockOnRead(blockType.getCategory()); 392 } 393 394 /** 395 * Returns the block cache. 396 * @return the block cache, or null if caching is completely disabled 397 */ 398 public Optional<BlockCache> getBlockCache() { 399 return Optional.ofNullable(this.blockCache); 400 } 401 402 public boolean isCombinedBlockCache() { 403 return blockCache instanceof CombinedBlockCache; 404 } 405 406 public ByteBuffAllocator getByteBuffAllocator() { 407 return this.byteBuffAllocator; 408 } 409 410 private long getCacheCompactedBlocksOnWriteThreshold(Configuration conf) { 411 long cacheCompactedBlocksOnWriteThreshold = 412 conf.getLong(CACHE_COMPACTED_BLOCKS_ON_WRITE_THRESHOLD_KEY, 413 DEFAULT_CACHE_COMPACTED_BLOCKS_ON_WRITE_THRESHOLD); 414 415 if (cacheCompactedBlocksOnWriteThreshold < 0) { 416 LOG.warn( 417 "cacheCompactedBlocksOnWriteThreshold value : {} is less than 0, resetting it to: {}", 418 cacheCompactedBlocksOnWriteThreshold, DEFAULT_CACHE_COMPACTED_BLOCKS_ON_WRITE_THRESHOLD); 419 cacheCompactedBlocksOnWriteThreshold = DEFAULT_CACHE_COMPACTED_BLOCKS_ON_WRITE_THRESHOLD; 420 } 421 422 return cacheCompactedBlocksOnWriteThreshold; 423 } 424 425 @Override 426 public String toString() { 427 return "cacheDataOnRead=" + shouldCacheDataOnRead() + ", cacheDataOnWrite=" 428 + shouldCacheDataOnWrite() + ", cacheIndexesOnWrite=" + shouldCacheIndexesOnWrite() 429 + ", cacheBloomsOnWrite=" + shouldCacheBloomsOnWrite() + ", cacheEvictOnClose=" 430 + shouldEvictOnClose() + ", cacheDataCompressed=" + shouldCacheDataCompressed() 431 + ", prefetchOnOpen=" + shouldPrefetchOnOpen(); 432 } 433}