001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.io.hfile; 019 020import static org.junit.jupiter.api.Assertions.assertEquals; 021import static org.junit.jupiter.api.Assertions.assertFalse; 022import static org.junit.jupiter.api.Assertions.assertTrue; 023import static org.mockito.ArgumentMatchers.any; 024import static org.mockito.ArgumentMatchers.anyInt; 025import static org.mockito.ArgumentMatchers.anyString; 026import static org.mockito.Mockito.mock; 027import static org.mockito.Mockito.when; 028 029import java.io.IOException; 030import java.net.InetSocketAddress; 031import java.util.List; 032import java.util.concurrent.ConcurrentHashMap; 033import java.util.concurrent.ConcurrentMap; 034import java.util.concurrent.CountDownLatch; 035import java.util.concurrent.ForkJoinPool; 036import java.util.concurrent.ThreadLocalRandom; 037import net.spy.memcached.CachedData; 038import net.spy.memcached.ConnectionFactory; 039import net.spy.memcached.FailureMode; 040import net.spy.memcached.MemcachedClient; 041import net.spy.memcached.internal.OperationFuture; 042import net.spy.memcached.ops.Operation; 043import net.spy.memcached.ops.OperationState; 044import net.spy.memcached.ops.OperationStatus; 045import net.spy.memcached.transcoders.Transcoder; 046import org.apache.hadoop.conf.Configuration; 047import org.apache.hadoop.hbase.HConstants; 048import org.apache.hadoop.hbase.Waiter; 049import org.apache.hadoop.hbase.io.hfile.CacheTestUtils.HFileBlockPair; 050import org.apache.hadoop.hbase.testclassification.IOTests; 051import org.apache.hadoop.hbase.testclassification.SmallTests; 052import org.junit.jupiter.api.BeforeEach; 053import org.junit.jupiter.api.Tag; 054import org.junit.jupiter.api.Test; 055 056@Tag(IOTests.TAG) 057@Tag(SmallTests.TAG) 058public class TestMemcachedBlockCache { 059 060 private int port; 061 062 private MemcachedBlockCache cache; 063 064 private ConcurrentMap<String, CachedData> backingMap; 065 066 @BeforeEach 067 public void setup() throws Exception { 068 port = ThreadLocalRandom.current().nextInt(1024, 65536); 069 Configuration conf = new Configuration(); 070 conf.set("hbase.cache.memcached.servers", "localhost:" + port); 071 backingMap = new ConcurrentHashMap<>(); 072 cache = new MockMemcachedBlockCache(conf); 073 } 074 075 /** 076 * Must not be an anonymous or local class, as Gson serializes those as null, which would break 077 * {@link CacheTestUtils#testConvertToJSON(BlockCache)}. 078 */ 079 private class MockMemcachedBlockCache extends MemcachedBlockCache { 080 081 MockMemcachedBlockCache(Configuration conf) throws IOException { 082 super(conf); 083 } 084 085 private <T> OperationFuture<T> createFuture(String key, long opTimeout, T result) { 086 OperationFuture<T> future = 087 new OperationFuture<>(key, new CountDownLatch(0), opTimeout, ForkJoinPool.commonPool()); 088 Operation op = mock(Operation.class); 089 when(op.getState()).thenReturn(OperationState.COMPLETE); 090 future.setOperation(op); 091 future.set(result, new OperationStatus(true, "")); 092 093 return future; 094 } 095 096 @Override 097 protected MemcachedClient createMemcachedClient(ConnectionFactory factory, 098 List<InetSocketAddress> serverAddresses) throws IOException { 099 assertEquals(FailureMode.Redistribute, factory.getFailureMode()); 100 assertTrue(factory.isDaemon()); 101 assertFalse(factory.useNagleAlgorithm()); 102 assertEquals(MAX_SIZE, factory.getReadBufSize()); 103 assertEquals(1, serverAddresses.size()); 104 assertEquals("localhost", serverAddresses.get(0).getHostName()); 105 assertEquals(port, serverAddresses.get(0).getPort()); 106 MemcachedClient client = mock(MemcachedClient.class); 107 when(client.set(anyString(), anyInt(), any(), any())).then(inv -> { 108 String key = inv.getArgument(0); 109 HFileBlock block = inv.getArgument(2); 110 Transcoder<HFileBlock> tc = inv.getArgument(3); 111 CachedData cd = tc.encode(block); 112 backingMap.put(key, cd); 113 return createFuture(key, factory.getOperationTimeout(), true); 114 }); 115 when(client.delete(anyString())).then(inv -> { 116 String key = inv.getArgument(0); 117 backingMap.remove(key); 118 return createFuture(key, factory.getOperationTimeout(), true); 119 }); 120 when(client.get(anyString(), any())).then(inv -> { 121 String key = inv.getArgument(0); 122 Transcoder<HFileBlock> tc = inv.getArgument(1); 123 CachedData cd = backingMap.get(key); 124 return tc.decode(cd); 125 }); 126 return client; 127 } 128 } 129 130 @Test 131 public void testCache() throws Exception { 132 final int numBlocks = 10; 133 HFileBlockPair[] blocks = 134 CacheTestUtils.generateHFileBlocks(HConstants.DEFAULT_BLOCKSIZE, numBlocks); 135 for (int i = 0; i < numBlocks; i++) { 136 cache.cacheBlock(blocks[i].getBlockName(), blocks[i].getBlock()); 137 } 138 Waiter.waitFor(new Configuration(), 10000, () -> backingMap.size() == numBlocks); 139 for (int i = 0; i < numBlocks; i++) { 140 HFileBlock actual = (HFileBlock) cache.getBlock(blocks[i].getBlockName(), false, false, true); 141 HFileBlock expected = blocks[i].getBlock(); 142 assertEquals(expected.getBlockType(), actual.getBlockType()); 143 assertEquals(expected.getSerializedLength(), actual.getSerializedLength()); 144 } 145 146 CacheTestUtils.testConvertToJSON(cache); 147 } 148 149 @Test 150 public void testEviction() throws Exception { 151 final int numBlocks = 10; 152 HFileBlockPair[] blocks = 153 CacheTestUtils.generateHFileBlocks(HConstants.DEFAULT_BLOCKSIZE, numBlocks); 154 for (int i = 0; i < numBlocks; i++) { 155 cache.cacheBlock(blocks[i].getBlockName(), blocks[i].getBlock()); 156 } 157 Waiter.waitFor(new Configuration(), 10000, () -> backingMap.size() == numBlocks); 158 for (int i = 0; i < numBlocks; i++) { 159 cache.evictBlock(blocks[i].getBlockName()); 160 } 161 Waiter.waitFor(new Configuration(), 10000, () -> backingMap.size() == 0); 162 } 163}