001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.io.hfile;
019
020import static org.junit.Assert.*;
021
022import java.io.IOException;
023import java.util.Map;
024import java.util.NavigableSet;
025import java.util.Objects;
026import org.apache.hadoop.conf.Configuration;
027import org.apache.hadoop.hbase.HBaseClassTestRule;
028import org.apache.hadoop.hbase.HBaseConfiguration;
029import org.apache.hadoop.hbase.HConstants;
030import org.apache.hadoop.hbase.io.hfile.TestCacheConfig.DataCacheEntry;
031import org.apache.hadoop.hbase.io.hfile.TestCacheConfig.IndexCacheEntry;
032import org.apache.hadoop.hbase.testclassification.IOTests;
033import org.apache.hadoop.hbase.testclassification.SmallTests;
034import org.junit.After;
035import org.junit.Before;
036import org.junit.ClassRule;
037import org.junit.Test;
038import org.junit.experimental.categories.Category;
039import org.slf4j.Logger;
040import org.slf4j.LoggerFactory;
041
042@Category({IOTests.class, SmallTests.class})
043public class TestBlockCacheReporting {
044
045  @ClassRule
046  public static final HBaseClassTestRule CLASS_RULE =
047      HBaseClassTestRule.forClass(TestBlockCacheReporting.class);
048
049  private static final Logger LOG = LoggerFactory.getLogger(TestBlockCacheReporting.class);
050  private Configuration conf;
051
052  @Before
053  public void setUp() throws Exception {
054    CacheConfig.clearGlobalInstances();
055    this.conf = HBaseConfiguration.create();
056  }
057
058  @After
059  public void tearDown() throws Exception {
060    // Let go of current block cache.
061    CacheConfig.clearGlobalInstances();
062  }
063
064  private void addDataAndHits(final BlockCache bc, final int count) {
065    Cacheable dce = new DataCacheEntry();
066    Cacheable ice = new IndexCacheEntry();
067    for (int i = 0; i < count; i++) {
068      BlockCacheKey bckd = new BlockCacheKey("f", i);
069      BlockCacheKey bcki = new BlockCacheKey("f", i + count);
070      bc.getBlock(bckd, true, false, true);
071      bc.cacheBlock(bckd, dce);
072      bc.cacheBlock(bcki, ice);
073      bc.getBlock(bckd, true, false, true);
074      bc.getBlock(bcki, true, false, true);
075    }
076    assertEquals(2 * count /*Data and Index blocks*/, bc.getStats().getHitCount());
077    BlockCacheKey bckd = new BlockCacheKey("f", 0);
078    BlockCacheKey bcki = new BlockCacheKey("f", 0 + count);
079    bc.evictBlock(bckd);
080    bc.evictBlock(bcki);
081    bc.getStats().getEvictedCount();
082  }
083
084  @Test
085  public void testBucketCache() throws IOException {
086    this.conf.set(HConstants.BUCKET_CACHE_IOENGINE_KEY, "offheap");
087    this.conf.setInt(HConstants.BUCKET_CACHE_SIZE_KEY, 100);
088    CacheConfig.instantiateBlockCache(this.conf);
089    CacheConfig cc = new CacheConfig(this.conf);
090    assertTrue(cc.getBlockCache() instanceof CombinedBlockCache);
091    logPerBlock(cc.getBlockCache());
092    final int count = 3;
093    addDataAndHits(cc.getBlockCache(), count);
094    // The below has no asserts.  It is just exercising toString and toJSON code.
095    LOG.info(Objects.toString(cc.getBlockCache().getStats()));
096    BlockCacheUtil.CachedBlocksByFile cbsbf = logPerBlock(cc.getBlockCache());
097    LOG.info(Objects.toString(cbsbf));
098    logPerFile(cbsbf);
099    bucketCacheReport(cc.getBlockCache());
100    LOG.info(BlockCacheUtil.toJSON(cbsbf));
101  }
102
103  @Test
104  public void testLruBlockCache() throws IOException {
105    CacheConfig.instantiateBlockCache(this.conf);
106    CacheConfig cc = new CacheConfig(this.conf);
107    assertTrue(cc.isBlockCacheEnabled());
108    assertTrue(CacheConfig.DEFAULT_IN_MEMORY == cc.isInMemory());
109    assertTrue(cc.getBlockCache() instanceof LruBlockCache);
110    logPerBlock(cc.getBlockCache());
111    addDataAndHits(cc.getBlockCache(), 3);
112    // The below has no asserts.  It is just exercising toString and toJSON code.
113    BlockCache bc = cc.getBlockCache();
114    LOG.info("count=" + bc.getBlockCount() + ", currentSize=" + bc.getCurrentSize() +
115        ", freeSize=" + bc.getFreeSize() );
116    LOG.info(Objects.toString(cc.getBlockCache().getStats()));
117    BlockCacheUtil.CachedBlocksByFile cbsbf = logPerBlock(cc.getBlockCache());
118    LOG.info(Objects.toString(cbsbf));
119    logPerFile(cbsbf);
120    bucketCacheReport(cc.getBlockCache());
121    LOG.info(BlockCacheUtil.toJSON(cbsbf));
122  }
123
124  private void bucketCacheReport(final BlockCache bc) {
125    LOG.info(bc.getClass().getSimpleName() + ": " + bc.getStats());
126    BlockCache [] bcs = bc.getBlockCaches();
127    if (bcs != null) {
128      for (BlockCache sbc: bc.getBlockCaches()) {
129        LOG.info(bc.getClass().getSimpleName() + ": " + sbc.getStats());
130      }
131    }
132  }
133
134  private void logPerFile(final BlockCacheUtil.CachedBlocksByFile cbsbf) throws IOException {
135    for (Map.Entry<String, NavigableSet<CachedBlock>> e:
136        cbsbf.getCachedBlockStatsByFile().entrySet()) {
137      int count = 0;
138      long size = 0;
139      int countData = 0;
140      long sizeData = 0;
141      for (CachedBlock cb: e.getValue()) {
142        count++;
143        size += cb.getSize();
144        BlockType bt = cb.getBlockType();
145        if (bt != null && bt.isData()) {
146          countData++;
147          sizeData += cb.getSize();
148        }
149      }
150      LOG.info("filename=" + e.getKey() + ", count=" + count + ", countData=" + countData +
151          ", size=" + size + ", sizeData=" + sizeData);
152      LOG.info(BlockCacheUtil.toJSON(e.getKey(), e.getValue()));
153    }
154  }
155
156  private BlockCacheUtil.CachedBlocksByFile logPerBlock(final BlockCache bc) throws IOException {
157    BlockCacheUtil.CachedBlocksByFile cbsbf = new BlockCacheUtil.CachedBlocksByFile();
158    for (CachedBlock cb : bc) {
159      LOG.info(cb.toString());
160      LOG.info(BlockCacheUtil.toJSON(bc));
161      cbsbf.update(cb);
162    }
163    return cbsbf;
164  }
165}