001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.regionserver; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertTrue; 022 023import java.lang.management.ManagementFactory; 024import java.nio.ByteBuffer; 025import java.util.ArrayList; 026import java.util.List; 027import java.util.Random; 028import org.apache.hadoop.conf.Configuration; 029import org.apache.hadoop.hbase.ByteBufferKeyValue; 030import org.apache.hadoop.hbase.Cell; 031import org.apache.hadoop.hbase.HBaseClassTestRule; 032import org.apache.hadoop.hbase.HBaseConfiguration; 033import org.apache.hadoop.hbase.KeyValue; 034import org.apache.hadoop.hbase.testclassification.RegionServerTests; 035import org.apache.hadoop.hbase.testclassification.SmallTests; 036import org.apache.hadoop.hbase.util.Bytes; 037import org.junit.BeforeClass; 038import org.junit.ClassRule; 039import org.junit.Ignore; 040import org.junit.Test; 041import org.junit.experimental.categories.Category; 042 043@Ignore // See HBASE-19742 for issue on reenabling. 044@Category({RegionServerTests.class, SmallTests.class}) 045public class TestMemstoreLABWithoutPool { 046 047 @ClassRule 048 public static final HBaseClassTestRule CLASS_RULE = 049 HBaseClassTestRule.forClass(TestMemstoreLABWithoutPool.class); 050 051 private final static Configuration conf = new Configuration(); 052 053 private static final byte[] rk = Bytes.toBytes("r1"); 054 private static final byte[] cf = Bytes.toBytes("f"); 055 private static final byte[] q = Bytes.toBytes("q"); 056 057 @BeforeClass 058 public static void setUpBeforeClass() throws Exception { 059 long globalMemStoreLimit = (long) (ManagementFactory.getMemoryMXBean().getHeapMemoryUsage() 060 .getMax() * 0.8); 061 // disable pool 062 ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT + Bytes.SIZEOF_LONG, false, globalMemStoreLimit, 063 0.0f, MemStoreLAB.POOL_INITIAL_SIZE_DEFAULT, null); 064 } 065 066 /** 067 * Test a bunch of random allocations 068 */ 069 @Test 070 public void testLABRandomAllocation() { 071 Random rand = new Random(); 072 MemStoreLAB mslab = new MemStoreLABImpl(); 073 int expectedOff = 0; 074 ByteBuffer lastBuffer = null; 075 int lastChunkId = -1; 076 // 100K iterations by 0-1K alloc -> 50MB expected 077 // should be reasonable for unit test and also cover wraparound 078 // behavior 079 for (int i = 0; i < 100000; i++) { 080 int valSize = rand.nextInt(1000); 081 KeyValue kv = new KeyValue(rk, cf, q, new byte[valSize]); 082 int size = kv.getSerializedSize(); 083 ByteBufferKeyValue newKv = (ByteBufferKeyValue) mslab.copyCellInto(kv); 084 if (newKv.getBuffer() != lastBuffer) { 085 // since we add the chunkID at the 0th offset of the chunk and the 086 // chunkid is an int we need to account for those 4 bytes 087 expectedOff = Bytes.SIZEOF_INT; 088 lastBuffer = newKv.getBuffer(); 089 int chunkId = newKv.getBuffer().getInt(0); 090 assertTrue("chunkid should be different", chunkId != lastChunkId); 091 lastChunkId = chunkId; 092 } 093 assertEquals(expectedOff, newKv.getOffset()); 094 assertTrue("Allocation overruns buffer", 095 newKv.getOffset() + size <= newKv.getBuffer().capacity()); 096 expectedOff += size; 097 } 098 } 099 100 /** 101 * Test frequent chunk retirement with chunk pool triggered by lots of threads, making sure 102 * there's no memory leak (HBASE-16195) 103 * @throws Exception if any error occurred 104 */ 105 @Test 106 public void testLABChunkQueueWithMultipleMSLABs() throws Exception { 107 Configuration conf = HBaseConfiguration.create(); 108 MemStoreLABImpl[] mslab = new MemStoreLABImpl[10]; 109 for (int i = 0; i < 10; i++) { 110 mslab[i] = new MemStoreLABImpl(conf); 111 } 112 // launch multiple threads to trigger frequent chunk retirement 113 List<Thread> threads = new ArrayList<>(); 114 // create smaller sized kvs 115 final KeyValue kv = new KeyValue(Bytes.toBytes("r"), Bytes.toBytes("f"), Bytes.toBytes("q"), 116 new byte[0]); 117 for (int i = 0; i < 10; i++) { 118 for (int j = 0; j < 10; j++) { 119 threads.add(getChunkQueueTestThread(mslab[i], "testLABChunkQueue-" + j, kv)); 120 } 121 } 122 for (Thread thread : threads) { 123 thread.start(); 124 } 125 // let it run for some time 126 Thread.sleep(3000); 127 for (Thread thread : threads) { 128 thread.interrupt(); 129 } 130 boolean threadsRunning = true; 131 boolean alive = false; 132 while (threadsRunning) { 133 alive = false; 134 for (Thread thread : threads) { 135 if (thread.isAlive()) { 136 alive = true; 137 break; 138 } 139 } 140 if (!alive) { 141 threadsRunning = false; 142 } 143 } 144 // close the mslab 145 for (int i = 0; i < 10; i++) { 146 mslab[i].close(); 147 } 148 // all of the chunkIds would have been returned back 149 assertTrue("All the chunks must have been cleared", 150 ChunkCreator.instance.numberOfMappedChunks() == 0); 151 } 152 153 private Thread getChunkQueueTestThread(final MemStoreLABImpl mslab, String threadName, 154 Cell cellToCopyInto) { 155 Thread thread = new Thread() { 156 volatile boolean stopped = false; 157 158 @Override 159 public void run() { 160 while (!stopped) { 161 // keep triggering chunk retirement 162 mslab.copyCellInto(cellToCopyInto); 163 } 164 } 165 166 @Override 167 public void interrupt() { 168 this.stopped = true; 169 } 170 }; 171 thread.setName(threadName); 172 thread.setDaemon(true); 173 return thread; 174 } 175}