001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.regionserver; 019 020import static org.apache.hadoop.hbase.regionserver.DefaultStoreEngine.DEFAULT_COMPACTOR_CLASS_KEY; 021 022import java.io.IOException; 023import java.util.List; 024import java.util.concurrent.atomic.AtomicBoolean; 025 026import org.apache.hadoop.conf.Configuration; 027import org.apache.hadoop.fs.Path; 028import org.apache.hadoop.hbase.Cell; 029import org.apache.hadoop.hbase.HBaseClassTestRule; 030import org.apache.hadoop.hbase.HBaseTestingUtility; 031import org.apache.hadoop.hbase.HConstants; 032import org.apache.hadoop.hbase.KeyValue.KeyOnlyKeyValue; 033import org.apache.hadoop.hbase.TableName; 034import org.apache.hadoop.hbase.client.Put; 035import org.apache.hadoop.hbase.client.Table; 036import org.apache.hadoop.hbase.io.hfile.HFileWriterImpl; 037import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequestImpl; 038import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor; 039import org.apache.hadoop.hbase.testclassification.MediumTests; 040import org.apache.hadoop.hbase.testclassification.RegionServerTests; 041import org.apache.hadoop.hbase.util.Bytes; 042import org.junit.AfterClass; 043import org.junit.Assert; 044import org.junit.BeforeClass; 045import org.junit.ClassRule; 046import org.junit.Rule; 047import org.junit.Test; 048import org.junit.experimental.categories.Category; 049import org.junit.rules.TestName; 050 051@Category({ RegionServerTests.class, MediumTests.class }) 052public class TestCompactorMemLeak { 053 054 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); 055 private static final Configuration CONF = UTIL.getConfiguration(); 056 private static final AtomicBoolean IS_LAST_CELL_ON_HEAP = new AtomicBoolean(false); 057 private static final byte[] FAMILY = Bytes.toBytes("f"); 058 private static final byte[] QUALIFIER = Bytes.toBytes("q"); 059 private static final byte[] VALUE = Bytes.toBytes("value"); 060 061 @Rule 062 public TestName name = new TestName(); 063 064 @ClassRule 065 public static final HBaseClassTestRule CLASS_RULE = 066 HBaseClassTestRule.forClass(TestCompactorMemLeak.class); 067 068 @BeforeClass 069 public static void setUp() throws Exception { 070 IS_LAST_CELL_ON_HEAP.set(false); 071 // Must use OFF-HEAP BucketCache here. 072 CONF.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.1f); 073 CONF.set(HConstants.BUCKET_CACHE_IOENGINE_KEY, "offheap"); 074 // 32MB for BucketCache. 075 CONF.setFloat(HConstants.BUCKET_CACHE_SIZE_KEY, 32); 076 // Use the MyCompactor 077 CONF.set(DEFAULT_COMPACTOR_CLASS_KEY, MyCompactor.class.getName()); 078 UTIL.startMiniCluster(); 079 } 080 081 @AfterClass 082 public static void tearDown() throws Exception { 083 IS_LAST_CELL_ON_HEAP.set(false); 084 UTIL.shutdownMiniCluster(); 085 } 086 087 private void assertMajorCompactionOK(TableName tableName) { 088 List<HRegion> regions = UTIL.getHBaseCluster().getRegionServerThreads().get(0).getRegionServer() 089 .getRegions(tableName); 090 Assert.assertEquals(regions.size(), 1); 091 HRegion region = regions.get(0); 092 Assert.assertEquals(region.getStores().size(), 1); 093 HStore store = region.getStore(FAMILY); 094 Assert.assertEquals(store.getStorefilesCount(), 1); 095 } 096 097 @Test 098 public void testMemLeak() throws IOException, InterruptedException { 099 TableName tableName = TableName.valueOf(name.getMethodName()); 100 Table table = UTIL.createTable(tableName, FAMILY); 101 102 // Put and Flush #1 103 Put put = new Put(Bytes.toBytes("row1")).addColumn(FAMILY, QUALIFIER, VALUE); 104 table.put(put); 105 UTIL.getAdmin().flush(tableName); 106 107 // Put and Flush #2 108 put = new Put(Bytes.toBytes("row2")).addColumn(FAMILY, QUALIFIER, VALUE); 109 table.put(put); 110 UTIL.getAdmin().flush(tableName); 111 112 // Major compact 113 UTIL.getAdmin().majorCompact(tableName); 114 Thread.sleep(6000); 115 assertMajorCompactionOK(tableName); 116 117 // The last cell before Compactor#commitWriter must be an heap one. 118 Assert.assertTrue(IS_LAST_CELL_ON_HEAP.get()); 119 } 120 121 public static class MyCompactor extends DefaultCompactor { 122 123 public MyCompactor(Configuration conf, HStore store) { 124 super(conf, store); 125 } 126 127 @Override 128 protected List<Path> commitWriter(StoreFileWriter writer, FileDetails fd, 129 CompactionRequestImpl request) throws IOException { 130 HFileWriterImpl writerImpl = (HFileWriterImpl) writer.writer; 131 Cell cell = writerImpl.getLastCell(); 132 // The cell should be backend with an KeyOnlyKeyValue. 133 IS_LAST_CELL_ON_HEAP.set(cell instanceof KeyOnlyKeyValue); 134 return super.commitWriter(writer, fd, request); 135 } 136 } 137}