001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.io.hfile;
019
020import static org.junit.jupiter.api.Assertions.assertArrayEquals;
021import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
022import static org.junit.jupiter.api.Assertions.assertEquals;
023import static org.junit.jupiter.api.Assertions.assertFalse;
024import static org.junit.jupiter.api.Assertions.assertNotNull;
025import static org.junit.jupiter.api.Assertions.assertNull;
026import static org.junit.jupiter.api.Assertions.assertTrue;
027import static org.junit.jupiter.api.Assertions.fail;
028
029import java.io.IOException;
030import java.lang.reflect.Field;
031import java.lang.reflect.Modifier;
032import java.nio.ByteBuffer;
033import java.util.Arrays;
034import java.util.HashSet;
035import java.util.Random;
036import java.util.concurrent.ConcurrentLinkedQueue;
037import java.util.concurrent.ThreadLocalRandom;
038import java.util.concurrent.atomic.AtomicInteger;
039import org.apache.hadoop.conf.Configuration;
040import org.apache.hadoop.fs.Path;
041import org.apache.hadoop.hbase.HConstants;
042import org.apache.hadoop.hbase.MultithreadedTestUtil;
043import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
044import org.apache.hadoop.hbase.io.ByteBuffAllocator;
045import org.apache.hadoop.hbase.io.HeapSize;
046import org.apache.hadoop.hbase.io.compress.Compression;
047import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
048import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache;
049import org.apache.hadoop.hbase.nio.ByteBuff;
050import org.apache.hadoop.hbase.util.Bytes;
051import org.apache.hadoop.hbase.util.ChecksumType;
052
053import org.apache.hbase.thirdparty.com.google.gson.JsonObject;
054import org.apache.hbase.thirdparty.com.google.gson.JsonParser;
055
056public class CacheTestUtils {
057
058  private static final boolean includesMemstoreTS = true;
059
060  /**
061   * Just checks if heapsize grows when something is cached, and gets smaller when the same object
062   * is evicted
063   */
064
065  public static void testHeapSizeChanges(final BlockCache toBeTested, final int blockSize) {
066    HFileBlockPair[] blocks = generateHFileBlocks(blockSize, 1);
067    long heapSize = ((HeapSize) toBeTested).heapSize();
068    toBeTested.cacheBlock(blocks[0].blockName, blocks[0].block);
069
070    /* When we cache something HeapSize should always increase */
071    assertTrue(heapSize < ((HeapSize) toBeTested).heapSize());
072
073    toBeTested.evictBlock(blocks[0].blockName);
074
075    /* Post eviction, heapsize should be the same */
076    assertEquals(heapSize, ((HeapSize) toBeTested).heapSize());
077  }
078
079  public static void testCacheMultiThreaded(final BlockCache toBeTested, final int blockSize,
080    final int numThreads, final int numQueries, final double passingScore) throws Exception {
081
082    Configuration conf = new Configuration();
083    MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(conf);
084
085    final AtomicInteger totalQueries = new AtomicInteger();
086    final ConcurrentLinkedQueue<HFileBlockPair> blocksToTest = new ConcurrentLinkedQueue<>();
087    final AtomicInteger hits = new AtomicInteger();
088    final AtomicInteger miss = new AtomicInteger();
089
090    HFileBlockPair[] blocks = generateHFileBlocks(numQueries, blockSize);
091    blocksToTest.addAll(Arrays.asList(blocks));
092
093    for (int i = 0; i < numThreads; i++) {
094      TestThread t = new MultithreadedTestUtil.RepeatingTestThread(ctx) {
095        @Override
096        public void doAnAction() throws Exception {
097          if (!blocksToTest.isEmpty()) {
098            HFileBlockPair ourBlock = blocksToTest.poll();
099            // if we run out of blocks to test, then we should stop the tests.
100            if (ourBlock == null) {
101              ctx.setStopFlag(true);
102              return;
103            }
104            toBeTested.cacheBlock(ourBlock.blockName, ourBlock.block);
105            Cacheable retrievedBlock = toBeTested.getBlock(ourBlock.blockName, false, false, true);
106            if (retrievedBlock != null) {
107              assertEquals(ourBlock.block, retrievedBlock);
108              toBeTested.evictBlock(ourBlock.blockName);
109              hits.incrementAndGet();
110              assertNull(toBeTested.getBlock(ourBlock.blockName, false, false, true));
111            } else {
112              miss.incrementAndGet();
113            }
114            totalQueries.incrementAndGet();
115          }
116        }
117      };
118      t.setDaemon(true);
119      ctx.addThread(t);
120    }
121    ctx.startThreads();
122    while (!blocksToTest.isEmpty() && ctx.shouldRun()) {
123      Thread.sleep(10);
124    }
125    ctx.stop();
126    if (hits.get() / ((double) hits.get() + (double) miss.get()) < passingScore) {
127      fail("Too many nulls returned. Hits: " + hits.get() + " Misses: " + miss.get());
128    }
129  }
130
131  public static void testCacheSimple(BlockCache toBeTested, int blockSize, int numBlocks)
132    throws Exception {
133
134    HFileBlockPair[] blocks = generateHFileBlocks(blockSize, numBlocks);
135    // Confirm empty
136    for (HFileBlockPair block : blocks) {
137      assertNull(toBeTested.getBlock(block.blockName, true, false, true));
138    }
139
140    // Add blocks
141    for (HFileBlockPair block : blocks) {
142      toBeTested.cacheBlock(block.blockName, block.block);
143    }
144
145    // Check if all blocks are properly cached and contain the right
146    // information, or the blocks are null.
147    // MapMaker makes no guarantees when it will evict, so neither can we.
148
149    for (HFileBlockPair block : blocks) {
150      HFileBlock buf = (HFileBlock) toBeTested.getBlock(block.blockName, true, false, true);
151      if (buf != null) {
152        assertEquals(block.block, buf);
153      }
154    }
155
156    // Re-add some duplicate blocks. Hope nothing breaks.
157
158    for (HFileBlockPair block : blocks) {
159      try {
160        if (toBeTested.getBlock(block.blockName, true, false, true) != null) {
161          toBeTested.cacheBlock(block.blockName, block.block);
162          if (!(toBeTested instanceof BucketCache)) {
163            // BucketCache won't throw exception when caching already cached
164            // block
165            fail("Cache should not allow re-caching a block");
166          }
167        }
168      } catch (RuntimeException re) {
169        // expected
170      }
171    }
172
173    testConvertToJSON(toBeTested);
174
175  }
176
177  public static void hammerSingleKey(final BlockCache toBeTested, int numThreads, int numQueries)
178    throws Exception {
179    final BlockCacheKey key = new BlockCacheKey("key", 0);
180    final byte[] buf = new byte[5 * 1024];
181    Arrays.fill(buf, (byte) 5);
182
183    final ByteArrayCacheable bac = new ByteArrayCacheable(buf);
184    Configuration conf = new Configuration();
185    MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(conf);
186
187    final AtomicInteger totalQueries = new AtomicInteger();
188    toBeTested.cacheBlock(key, bac);
189
190    for (int i = 0; i < numThreads; i++) {
191      TestThread t = new MultithreadedTestUtil.RepeatingTestThread(ctx) {
192        @Override
193        public void doAnAction() throws Exception {
194          ByteArrayCacheable returned =
195            (ByteArrayCacheable) toBeTested.getBlock(key, false, false, true);
196          if (returned != null) {
197            assertArrayEquals(buf, returned.buf);
198          } else {
199            Thread.sleep(10);
200          }
201          totalQueries.incrementAndGet();
202        }
203      };
204
205      t.setDaemon(true);
206      ctx.addThread(t);
207    }
208
209    // add a thread to periodically evict and re-cache the block
210    final long blockEvictPeriod = 50;
211    TestThread t = new MultithreadedTestUtil.RepeatingTestThread(ctx) {
212      @Override
213      public void doAnAction() throws Exception {
214        toBeTested.evictBlock(key);
215        toBeTested.cacheBlock(key, bac);
216        Thread.sleep(blockEvictPeriod);
217      }
218    };
219    t.setDaemon(true);
220    ctx.addThread(t);
221
222    ctx.startThreads();
223    while (totalQueries.get() < numQueries && ctx.shouldRun()) {
224      Thread.sleep(10);
225    }
226    ctx.stop();
227  }
228
229  public static class ByteArrayCacheable implements Cacheable {
230
231    static final CacheableDeserializer<Cacheable> blockDeserializer =
232      new CacheableDeserializer<Cacheable>() {
233        @Override
234        public int getDeserializerIdentifier() {
235          return deserializerIdentifier;
236        }
237
238        @Override
239        public Cacheable deserialize(ByteBuff b, ByteBuffAllocator alloc) throws IOException {
240          int len = b.getInt();
241          Thread.yield();
242          byte buf[] = new byte[len];
243          b.get(buf);
244          return new ByteArrayCacheable(buf);
245        }
246      };
247
248    final byte[] buf;
249
250    public ByteArrayCacheable(byte[] buf) {
251      this.buf = buf;
252    }
253
254    @Override
255    public long heapSize() {
256      return 4L + buf.length;
257    }
258
259    @Override
260    public int getSerializedLength() {
261      return 4 + buf.length;
262    }
263
264    @Override
265    public void serialize(ByteBuffer destination, boolean includeNextBlockMetadata) {
266      destination.putInt(buf.length);
267      Thread.yield();
268      destination.put(buf);
269      destination.rewind();
270    }
271
272    @Override
273    public CacheableDeserializer<Cacheable> getDeserializer() {
274      return blockDeserializer;
275    }
276
277    private static final int deserializerIdentifier;
278    static {
279      deserializerIdentifier =
280        CacheableDeserializerIdManager.registerDeserializer(blockDeserializer);
281    }
282
283    @Override
284    public BlockType getBlockType() {
285      return BlockType.DATA;
286    }
287  }
288
289  public static HFileBlockPair[] generateHFileBlocks(int blockSize, int numBlocks) {
290    return generateBlocksForPath(blockSize, numBlocks, null, false);
291  }
292
293  public static String[] getHFileNames(HFileBlockPair[] blocks) {
294    String[] names = new String[blocks.length];
295    for (int i = 0; i < blocks.length; i++) {
296      names[i] = blocks[i].blockName.getHfileName();
297    }
298    return names;
299  }
300
301  public static BlockCacheKey[] regenerateKeys(HFileBlockPair[] blocks, String[] names) {
302    BlockCacheKey[] keys = new BlockCacheKey[blocks.length];
303    for (int i = 0; i < blocks.length; i++) {
304      keys[i] = new BlockCacheKey(names[i], blocks[i].blockName.getOffset(), true,
305        blocks[i].blockName.getBlockType());
306    }
307    return keys;
308  }
309
310  public static HFileBlockPair[] generateBlocksForPath(int blockSize, int numBlocks, Path path,
311    boolean encoded) {
312    HFileBlockPair[] returnedBlocks = new HFileBlockPair[numBlocks];
313    Random rand = ThreadLocalRandom.current();
314    HashSet<String> usedStrings = new HashSet<>();
315    for (int i = 0; i < numBlocks; i++) {
316      ByteBuffer cachedBuffer = ByteBuffer.allocate(blockSize);
317      Bytes.random(cachedBuffer.array());
318      cachedBuffer.rewind();
319      int onDiskSizeWithoutHeader = blockSize;
320      int uncompressedSizeWithoutHeader = blockSize;
321      long prevBlockOffset = rand.nextLong();
322      BlockType.DATA.write(cachedBuffer);
323      cachedBuffer.putInt(onDiskSizeWithoutHeader);
324      cachedBuffer.putInt(uncompressedSizeWithoutHeader);
325      cachedBuffer.putLong(prevBlockOffset);
326      cachedBuffer.rewind();
327      HFileContext meta =
328        new HFileContextBuilder().withHBaseCheckSum(false).withIncludesMvcc(includesMemstoreTS)
329          .withIncludesTags(false).withCompression(Compression.Algorithm.NONE)
330          .withDataBlockEncoding(DataBlockEncoding.FAST_DIFF).withBytesPerCheckSum(0)
331          .withChecksumType(ChecksumType.NULL).build();
332      HFileBlock generated = new HFileBlock(encoded ? BlockType.ENCODED_DATA : BlockType.DATA,
333        onDiskSizeWithoutHeader, uncompressedSizeWithoutHeader, prevBlockOffset,
334        ByteBuff.wrap(cachedBuffer), HFileBlock.DONT_FILL_HEADER, blockSize,
335        onDiskSizeWithoutHeader + HConstants.HFILEBLOCK_HEADER_SIZE, -1, meta,
336        ByteBuffAllocator.HEAP);
337      String key = null;
338      long offset = 0;
339      returnedBlocks[i] = new HFileBlockPair();
340      if (path != null) {
341        offset = i * blockSize;
342        returnedBlocks[i].blockName =
343          new BlockCacheKey(path, offset, true, encoded ? BlockType.ENCODED_DATA : BlockType.DATA);
344      } else {
345        /* No conflicting keys */
346        key = Long.toString(rand.nextLong());
347        while (!usedStrings.add(key)) {
348          key = Long.toString(rand.nextLong());
349        }
350        returnedBlocks[i].blockName =
351          new BlockCacheKey(key, offset, true, encoded ? BlockType.ENCODED_DATA : BlockType.DATA);
352      }
353      returnedBlocks[i].block = generated;
354    }
355    return returnedBlocks;
356  }
357
358  public static class HFileBlockPair {
359    BlockCacheKey blockName;
360    HFileBlock block;
361
362    public BlockCacheKey getBlockName() {
363      return this.blockName;
364    }
365
366    public HFileBlock getBlock() {
367      return this.block;
368    }
369  }
370
371  public static void getBlockAndAssertEquals(BlockCache cache, BlockCacheKey key,
372    Cacheable blockToCache, ByteBuffer destBuffer, ByteBuffer expectedBuffer) {
373    destBuffer.clear();
374    cache.cacheBlock(key, blockToCache);
375    Cacheable actualBlock = cache.getBlock(key, false, false, false);
376    try {
377      actualBlock.serialize(destBuffer, true);
378      assertEquals(expectedBuffer, destBuffer);
379    } finally {
380      // Release the reference count increased by getBlock.
381      if (actualBlock != null) {
382        actualBlock.release();
383      }
384    }
385  }
386
387  public static void testConvertToJSON(BlockCache toBeTested) {
388    String json = assertDoesNotThrow(() -> BlockCacheUtil.toJSON(toBeTested));
389    assertNotNull(json);
390    // static and transient fields, e.g. the cache internals, should not be serialized
391    JsonObject jsonObject = JsonParser.parseString(json).getAsJsonObject();
392    for (Class<?> clazz = toBeTested.getClass(); clazz != null; clazz = clazz.getSuperclass()) {
393      for (Field field : clazz.getDeclaredFields()) {
394        int modifiers = field.getModifiers();
395        if (Modifier.isStatic(modifiers) || Modifier.isTransient(modifiers)) {
396          assertFalse(jsonObject.has(field.getName()),
397            "static or transient field " + field.getName() + " should not be serialized");
398        }
399      }
400    }
401  }
402}