001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.io.hfile;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertFalse;
022import static org.junit.jupiter.api.Assertions.assertTrue;
023import static org.mockito.ArgumentMatchers.any;
024import static org.mockito.ArgumentMatchers.anyInt;
025import static org.mockito.ArgumentMatchers.anyString;
026import static org.mockito.Mockito.mock;
027import static org.mockito.Mockito.when;
028
029import java.io.IOException;
030import java.net.InetSocketAddress;
031import java.util.List;
032import java.util.concurrent.ConcurrentHashMap;
033import java.util.concurrent.ConcurrentMap;
034import java.util.concurrent.CountDownLatch;
035import java.util.concurrent.ForkJoinPool;
036import java.util.concurrent.ThreadLocalRandom;
037import net.spy.memcached.CachedData;
038import net.spy.memcached.ConnectionFactory;
039import net.spy.memcached.FailureMode;
040import net.spy.memcached.MemcachedClient;
041import net.spy.memcached.internal.OperationFuture;
042import net.spy.memcached.ops.Operation;
043import net.spy.memcached.ops.OperationState;
044import net.spy.memcached.ops.OperationStatus;
045import net.spy.memcached.transcoders.Transcoder;
046import org.apache.hadoop.conf.Configuration;
047import org.apache.hadoop.hbase.HConstants;
048import org.apache.hadoop.hbase.Waiter;
049import org.apache.hadoop.hbase.io.hfile.CacheTestUtils.HFileBlockPair;
050import org.apache.hadoop.hbase.testclassification.IOTests;
051import org.apache.hadoop.hbase.testclassification.SmallTests;
052import org.junit.jupiter.api.BeforeEach;
053import org.junit.jupiter.api.Tag;
054import org.junit.jupiter.api.Test;
055
056@Tag(IOTests.TAG)
057@Tag(SmallTests.TAG)
058public class TestMemcachedBlockCache {
059
060  private int port;
061
062  private MemcachedBlockCache cache;
063
064  private ConcurrentMap<String, CachedData> backingMap;
065
066  @BeforeEach
067  public void setup() throws Exception {
068    port = ThreadLocalRandom.current().nextInt(1024, 65536);
069    Configuration conf = new Configuration();
070    conf.set("hbase.cache.memcached.servers", "localhost:" + port);
071    backingMap = new ConcurrentHashMap<>();
072    cache = new MockMemcachedBlockCache(conf);
073  }
074
075  /**
076   * Must not be an anonymous or local class, as Gson serializes those as null, which would break
077   * {@link CacheTestUtils#testConvertToJSON(BlockCache)}.
078   */
079  private class MockMemcachedBlockCache extends MemcachedBlockCache {
080
081    MockMemcachedBlockCache(Configuration conf) throws IOException {
082      super(conf);
083    }
084
085    private <T> OperationFuture<T> createFuture(String key, long opTimeout, T result) {
086      OperationFuture<T> future =
087        new OperationFuture<>(key, new CountDownLatch(0), opTimeout, ForkJoinPool.commonPool());
088      Operation op = mock(Operation.class);
089      when(op.getState()).thenReturn(OperationState.COMPLETE);
090      future.setOperation(op);
091      future.set(result, new OperationStatus(true, ""));
092
093      return future;
094    }
095
096    @Override
097    protected MemcachedClient createMemcachedClient(ConnectionFactory factory,
098      List<InetSocketAddress> serverAddresses) throws IOException {
099      assertEquals(FailureMode.Redistribute, factory.getFailureMode());
100      assertTrue(factory.isDaemon());
101      assertFalse(factory.useNagleAlgorithm());
102      assertEquals(MAX_SIZE, factory.getReadBufSize());
103      assertEquals(1, serverAddresses.size());
104      assertEquals("localhost", serverAddresses.get(0).getHostName());
105      assertEquals(port, serverAddresses.get(0).getPort());
106      MemcachedClient client = mock(MemcachedClient.class);
107      when(client.set(anyString(), anyInt(), any(), any())).then(inv -> {
108        String key = inv.getArgument(0);
109        HFileBlock block = inv.getArgument(2);
110        Transcoder<HFileBlock> tc = inv.getArgument(3);
111        CachedData cd = tc.encode(block);
112        backingMap.put(key, cd);
113        return createFuture(key, factory.getOperationTimeout(), true);
114      });
115      when(client.delete(anyString())).then(inv -> {
116        String key = inv.getArgument(0);
117        backingMap.remove(key);
118        return createFuture(key, factory.getOperationTimeout(), true);
119      });
120      when(client.get(anyString(), any())).then(inv -> {
121        String key = inv.getArgument(0);
122        Transcoder<HFileBlock> tc = inv.getArgument(1);
123        CachedData cd = backingMap.get(key);
124        return tc.decode(cd);
125      });
126      return client;
127    }
128  }
129
130  @Test
131  public void testCache() throws Exception {
132    final int numBlocks = 10;
133    HFileBlockPair[] blocks =
134      CacheTestUtils.generateHFileBlocks(HConstants.DEFAULT_BLOCKSIZE, numBlocks);
135    for (int i = 0; i < numBlocks; i++) {
136      cache.cacheBlock(blocks[i].getBlockName(), blocks[i].getBlock());
137    }
138    Waiter.waitFor(new Configuration(), 10000, () -> backingMap.size() == numBlocks);
139    for (int i = 0; i < numBlocks; i++) {
140      HFileBlock actual = (HFileBlock) cache.getBlock(blocks[i].getBlockName(), false, false, true);
141      HFileBlock expected = blocks[i].getBlock();
142      assertEquals(expected.getBlockType(), actual.getBlockType());
143      assertEquals(expected.getSerializedLength(), actual.getSerializedLength());
144    }
145
146    CacheTestUtils.testConvertToJSON(cache);
147  }
148
149  @Test
150  public void testEviction() throws Exception {
151    final int numBlocks = 10;
152    HFileBlockPair[] blocks =
153      CacheTestUtils.generateHFileBlocks(HConstants.DEFAULT_BLOCKSIZE, numBlocks);
154    for (int i = 0; i < numBlocks; i++) {
155      cache.cacheBlock(blocks[i].getBlockName(), blocks[i].getBlock());
156    }
157    Waiter.waitFor(new Configuration(), 10000, () -> backingMap.size() == numBlocks);
158    for (int i = 0; i < numBlocks; i++) {
159      cache.evictBlock(blocks[i].getBlockName());
160    }
161    Waiter.waitFor(new Configuration(), 10000, () -> backingMap.size() == 0);
162  }
163}