001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.io.hfile; 019 020import static org.junit.jupiter.api.Assertions.assertArrayEquals; 021import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; 022import static org.junit.jupiter.api.Assertions.assertEquals; 023import static org.junit.jupiter.api.Assertions.assertFalse; 024import static org.junit.jupiter.api.Assertions.assertNotNull; 025import static org.junit.jupiter.api.Assertions.assertNull; 026import static org.junit.jupiter.api.Assertions.assertTrue; 027import static org.junit.jupiter.api.Assertions.fail; 028 029import java.io.IOException; 030import java.lang.reflect.Field; 031import java.lang.reflect.Modifier; 032import java.nio.ByteBuffer; 033import java.util.Arrays; 034import java.util.HashSet; 035import java.util.Random; 036import java.util.concurrent.ConcurrentLinkedQueue; 037import java.util.concurrent.ThreadLocalRandom; 038import java.util.concurrent.atomic.AtomicInteger; 039import org.apache.hadoop.conf.Configuration; 040import org.apache.hadoop.fs.Path; 041import org.apache.hadoop.hbase.HConstants; 042import org.apache.hadoop.hbase.MultithreadedTestUtil; 043import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread; 044import org.apache.hadoop.hbase.io.ByteBuffAllocator; 045import org.apache.hadoop.hbase.io.HeapSize; 046import org.apache.hadoop.hbase.io.compress.Compression; 047import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; 048import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache; 049import org.apache.hadoop.hbase.nio.ByteBuff; 050import org.apache.hadoop.hbase.util.Bytes; 051import org.apache.hadoop.hbase.util.ChecksumType; 052 053import org.apache.hbase.thirdparty.com.google.gson.JsonObject; 054import org.apache.hbase.thirdparty.com.google.gson.JsonParser; 055 056public class CacheTestUtils { 057 058 private static final boolean includesMemstoreTS = true; 059 060 /** 061 * Just checks if heapsize grows when something is cached, and gets smaller when the same object 062 * is evicted 063 */ 064 065 public static void testHeapSizeChanges(final BlockCache toBeTested, final int blockSize) { 066 HFileBlockPair[] blocks = generateHFileBlocks(blockSize, 1); 067 long heapSize = ((HeapSize) toBeTested).heapSize(); 068 toBeTested.cacheBlock(blocks[0].blockName, blocks[0].block); 069 070 /* When we cache something HeapSize should always increase */ 071 assertTrue(heapSize < ((HeapSize) toBeTested).heapSize()); 072 073 toBeTested.evictBlock(blocks[0].blockName); 074 075 /* Post eviction, heapsize should be the same */ 076 assertEquals(heapSize, ((HeapSize) toBeTested).heapSize()); 077 } 078 079 public static void testCacheMultiThreaded(final BlockCache toBeTested, final int blockSize, 080 final int numThreads, final int numQueries, final double passingScore) throws Exception { 081 082 Configuration conf = new Configuration(); 083 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(conf); 084 085 final AtomicInteger totalQueries = new AtomicInteger(); 086 final ConcurrentLinkedQueue<HFileBlockPair> blocksToTest = new ConcurrentLinkedQueue<>(); 087 final AtomicInteger hits = new AtomicInteger(); 088 final AtomicInteger miss = new AtomicInteger(); 089 090 HFileBlockPair[] blocks = generateHFileBlocks(numQueries, blockSize); 091 blocksToTest.addAll(Arrays.asList(blocks)); 092 093 for (int i = 0; i < numThreads; i++) { 094 TestThread t = new MultithreadedTestUtil.RepeatingTestThread(ctx) { 095 @Override 096 public void doAnAction() throws Exception { 097 if (!blocksToTest.isEmpty()) { 098 HFileBlockPair ourBlock = blocksToTest.poll(); 099 // if we run out of blocks to test, then we should stop the tests. 100 if (ourBlock == null) { 101 ctx.setStopFlag(true); 102 return; 103 } 104 toBeTested.cacheBlock(ourBlock.blockName, ourBlock.block); 105 Cacheable retrievedBlock = toBeTested.getBlock(ourBlock.blockName, false, false, true); 106 if (retrievedBlock != null) { 107 assertEquals(ourBlock.block, retrievedBlock); 108 toBeTested.evictBlock(ourBlock.blockName); 109 hits.incrementAndGet(); 110 assertNull(toBeTested.getBlock(ourBlock.blockName, false, false, true)); 111 } else { 112 miss.incrementAndGet(); 113 } 114 totalQueries.incrementAndGet(); 115 } 116 } 117 }; 118 t.setDaemon(true); 119 ctx.addThread(t); 120 } 121 ctx.startThreads(); 122 while (!blocksToTest.isEmpty() && ctx.shouldRun()) { 123 Thread.sleep(10); 124 } 125 ctx.stop(); 126 if (hits.get() / ((double) hits.get() + (double) miss.get()) < passingScore) { 127 fail("Too many nulls returned. Hits: " + hits.get() + " Misses: " + miss.get()); 128 } 129 } 130 131 public static void testCacheSimple(BlockCache toBeTested, int blockSize, int numBlocks) 132 throws Exception { 133 134 HFileBlockPair[] blocks = generateHFileBlocks(blockSize, numBlocks); 135 // Confirm empty 136 for (HFileBlockPair block : blocks) { 137 assertNull(toBeTested.getBlock(block.blockName, true, false, true)); 138 } 139 140 // Add blocks 141 for (HFileBlockPair block : blocks) { 142 toBeTested.cacheBlock(block.blockName, block.block); 143 } 144 145 // Check if all blocks are properly cached and contain the right 146 // information, or the blocks are null. 147 // MapMaker makes no guarantees when it will evict, so neither can we. 148 149 for (HFileBlockPair block : blocks) { 150 HFileBlock buf = (HFileBlock) toBeTested.getBlock(block.blockName, true, false, true); 151 if (buf != null) { 152 assertEquals(block.block, buf); 153 } 154 } 155 156 // Re-add some duplicate blocks. Hope nothing breaks. 157 158 for (HFileBlockPair block : blocks) { 159 try { 160 if (toBeTested.getBlock(block.blockName, true, false, true) != null) { 161 toBeTested.cacheBlock(block.blockName, block.block); 162 if (!(toBeTested instanceof BucketCache)) { 163 // BucketCache won't throw exception when caching already cached 164 // block 165 fail("Cache should not allow re-caching a block"); 166 } 167 } 168 } catch (RuntimeException re) { 169 // expected 170 } 171 } 172 173 testConvertToJSON(toBeTested); 174 175 } 176 177 public static void hammerSingleKey(final BlockCache toBeTested, int numThreads, int numQueries) 178 throws Exception { 179 final BlockCacheKey key = new BlockCacheKey("key", 0); 180 final byte[] buf = new byte[5 * 1024]; 181 Arrays.fill(buf, (byte) 5); 182 183 final ByteArrayCacheable bac = new ByteArrayCacheable(buf); 184 Configuration conf = new Configuration(); 185 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(conf); 186 187 final AtomicInteger totalQueries = new AtomicInteger(); 188 toBeTested.cacheBlock(key, bac); 189 190 for (int i = 0; i < numThreads; i++) { 191 TestThread t = new MultithreadedTestUtil.RepeatingTestThread(ctx) { 192 @Override 193 public void doAnAction() throws Exception { 194 ByteArrayCacheable returned = 195 (ByteArrayCacheable) toBeTested.getBlock(key, false, false, true); 196 if (returned != null) { 197 assertArrayEquals(buf, returned.buf); 198 } else { 199 Thread.sleep(10); 200 } 201 totalQueries.incrementAndGet(); 202 } 203 }; 204 205 t.setDaemon(true); 206 ctx.addThread(t); 207 } 208 209 // add a thread to periodically evict and re-cache the block 210 final long blockEvictPeriod = 50; 211 TestThread t = new MultithreadedTestUtil.RepeatingTestThread(ctx) { 212 @Override 213 public void doAnAction() throws Exception { 214 toBeTested.evictBlock(key); 215 toBeTested.cacheBlock(key, bac); 216 Thread.sleep(blockEvictPeriod); 217 } 218 }; 219 t.setDaemon(true); 220 ctx.addThread(t); 221 222 ctx.startThreads(); 223 while (totalQueries.get() < numQueries && ctx.shouldRun()) { 224 Thread.sleep(10); 225 } 226 ctx.stop(); 227 } 228 229 public static class ByteArrayCacheable implements Cacheable { 230 231 static final CacheableDeserializer<Cacheable> blockDeserializer = 232 new CacheableDeserializer<Cacheable>() { 233 @Override 234 public int getDeserializerIdentifier() { 235 return deserializerIdentifier; 236 } 237 238 @Override 239 public Cacheable deserialize(ByteBuff b, ByteBuffAllocator alloc) throws IOException { 240 int len = b.getInt(); 241 Thread.yield(); 242 byte buf[] = new byte[len]; 243 b.get(buf); 244 return new ByteArrayCacheable(buf); 245 } 246 }; 247 248 final byte[] buf; 249 250 public ByteArrayCacheable(byte[] buf) { 251 this.buf = buf; 252 } 253 254 @Override 255 public long heapSize() { 256 return 4L + buf.length; 257 } 258 259 @Override 260 public int getSerializedLength() { 261 return 4 + buf.length; 262 } 263 264 @Override 265 public void serialize(ByteBuffer destination, boolean includeNextBlockMetadata) { 266 destination.putInt(buf.length); 267 Thread.yield(); 268 destination.put(buf); 269 destination.rewind(); 270 } 271 272 @Override 273 public CacheableDeserializer<Cacheable> getDeserializer() { 274 return blockDeserializer; 275 } 276 277 private static final int deserializerIdentifier; 278 static { 279 deserializerIdentifier = 280 CacheableDeserializerIdManager.registerDeserializer(blockDeserializer); 281 } 282 283 @Override 284 public BlockType getBlockType() { 285 return BlockType.DATA; 286 } 287 } 288 289 public static HFileBlockPair[] generateHFileBlocks(int blockSize, int numBlocks) { 290 return generateBlocksForPath(blockSize, numBlocks, null, false); 291 } 292 293 public static String[] getHFileNames(HFileBlockPair[] blocks) { 294 String[] names = new String[blocks.length]; 295 for (int i = 0; i < blocks.length; i++) { 296 names[i] = blocks[i].blockName.getHfileName(); 297 } 298 return names; 299 } 300 301 public static BlockCacheKey[] regenerateKeys(HFileBlockPair[] blocks, String[] names) { 302 BlockCacheKey[] keys = new BlockCacheKey[blocks.length]; 303 for (int i = 0; i < blocks.length; i++) { 304 keys[i] = new BlockCacheKey(names[i], blocks[i].blockName.getOffset(), true, 305 blocks[i].blockName.getBlockType()); 306 } 307 return keys; 308 } 309 310 public static HFileBlockPair[] generateBlocksForPath(int blockSize, int numBlocks, Path path, 311 boolean encoded) { 312 HFileBlockPair[] returnedBlocks = new HFileBlockPair[numBlocks]; 313 Random rand = ThreadLocalRandom.current(); 314 HashSet<String> usedStrings = new HashSet<>(); 315 for (int i = 0; i < numBlocks; i++) { 316 ByteBuffer cachedBuffer = ByteBuffer.allocate(blockSize); 317 Bytes.random(cachedBuffer.array()); 318 cachedBuffer.rewind(); 319 int onDiskSizeWithoutHeader = blockSize; 320 int uncompressedSizeWithoutHeader = blockSize; 321 long prevBlockOffset = rand.nextLong(); 322 BlockType.DATA.write(cachedBuffer); 323 cachedBuffer.putInt(onDiskSizeWithoutHeader); 324 cachedBuffer.putInt(uncompressedSizeWithoutHeader); 325 cachedBuffer.putLong(prevBlockOffset); 326 cachedBuffer.rewind(); 327 HFileContext meta = 328 new HFileContextBuilder().withHBaseCheckSum(false).withIncludesMvcc(includesMemstoreTS) 329 .withIncludesTags(false).withCompression(Compression.Algorithm.NONE) 330 .withDataBlockEncoding(DataBlockEncoding.FAST_DIFF).withBytesPerCheckSum(0) 331 .withChecksumType(ChecksumType.NULL).build(); 332 HFileBlock generated = new HFileBlock(encoded ? BlockType.ENCODED_DATA : BlockType.DATA, 333 onDiskSizeWithoutHeader, uncompressedSizeWithoutHeader, prevBlockOffset, 334 ByteBuff.wrap(cachedBuffer), HFileBlock.DONT_FILL_HEADER, blockSize, 335 onDiskSizeWithoutHeader + HConstants.HFILEBLOCK_HEADER_SIZE, -1, meta, 336 ByteBuffAllocator.HEAP); 337 String key = null; 338 long offset = 0; 339 returnedBlocks[i] = new HFileBlockPair(); 340 if (path != null) { 341 offset = i * blockSize; 342 returnedBlocks[i].blockName = 343 new BlockCacheKey(path, offset, true, encoded ? BlockType.ENCODED_DATA : BlockType.DATA); 344 } else { 345 /* No conflicting keys */ 346 key = Long.toString(rand.nextLong()); 347 while (!usedStrings.add(key)) { 348 key = Long.toString(rand.nextLong()); 349 } 350 returnedBlocks[i].blockName = 351 new BlockCacheKey(key, offset, true, encoded ? BlockType.ENCODED_DATA : BlockType.DATA); 352 } 353 returnedBlocks[i].block = generated; 354 } 355 return returnedBlocks; 356 } 357 358 public static class HFileBlockPair { 359 BlockCacheKey blockName; 360 HFileBlock block; 361 362 public BlockCacheKey getBlockName() { 363 return this.blockName; 364 } 365 366 public HFileBlock getBlock() { 367 return this.block; 368 } 369 } 370 371 public static void getBlockAndAssertEquals(BlockCache cache, BlockCacheKey key, 372 Cacheable blockToCache, ByteBuffer destBuffer, ByteBuffer expectedBuffer) { 373 destBuffer.clear(); 374 cache.cacheBlock(key, blockToCache); 375 Cacheable actualBlock = cache.getBlock(key, false, false, false); 376 try { 377 actualBlock.serialize(destBuffer, true); 378 assertEquals(expectedBuffer, destBuffer); 379 } finally { 380 // Release the reference count increased by getBlock. 381 if (actualBlock != null) { 382 actualBlock.release(); 383 } 384 } 385 } 386 387 public static void testConvertToJSON(BlockCache toBeTested) { 388 String json = assertDoesNotThrow(() -> BlockCacheUtil.toJSON(toBeTested)); 389 assertNotNull(json); 390 // static and transient fields, e.g. the cache internals, should not be serialized 391 JsonObject jsonObject = JsonParser.parseString(json).getAsJsonObject(); 392 for (Class<?> clazz = toBeTested.getClass(); clazz != null; clazz = clazz.getSuperclass()) { 393 for (Field field : clazz.getDeclaredFields()) { 394 int modifiers = field.getModifiers(); 395 if (Modifier.isStatic(modifiers) || Modifier.isTransient(modifiers)) { 396 assertFalse(jsonObject.has(field.getName()), 397 "static or transient field " + field.getName() + " should not be serialized"); 398 } 399 } 400 } 401 } 402}