001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import java.io.IOException;
021import java.util.ArrayList;
022import java.util.List;
023import org.apache.hadoop.conf.Configuration;
024import org.apache.hadoop.hbase.HBaseClassTestRule;
025import org.apache.hadoop.hbase.HBaseTestingUtil;
026import org.apache.hadoop.hbase.HConstants;
027import org.apache.hadoop.hbase.TableName;
028import org.apache.hadoop.hbase.client.Admin;
029import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
030import org.apache.hadoop.hbase.client.Put;
031import org.apache.hadoop.hbase.client.Table;
032import org.apache.hadoop.hbase.client.TableDescriptor;
033import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
034import org.apache.hadoop.hbase.io.ByteBuffAllocator;
035import org.apache.hadoop.hbase.testclassification.LargeTests;
036import org.apache.hadoop.hbase.util.Bytes;
037import org.apache.hadoop.hbase.util.JVMClusterUtil;
038import org.junit.AfterClass;
039import org.junit.BeforeClass;
040import org.junit.ClassRule;
041import org.junit.Rule;
042import org.junit.Test;
043import org.junit.experimental.categories.Category;
044import org.junit.rules.TestName;
045
046@Category(LargeTests.class)
047public class TestCompactionWithByteBuff {
048  @ClassRule
049  public static final HBaseClassTestRule CLASS_RULE =
050    HBaseClassTestRule.forClass(TestCompactionWithByteBuff.class);
051  @Rule
052  public TestName name = new TestName();
053
054  private static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
055  private static Configuration conf = TEST_UTIL.getConfiguration();
056  private static Admin admin = null;
057
058  private static final byte[] COLUMN = Bytes.toBytes("A");
059  private static final int REGION_COUNT = 5;
060  private static final long ROW_COUNT = 200;
061  private static final int ROW_LENGTH = 20;
062  private static final int VALUE_LENGTH = 5000;
063
064  @BeforeClass
065  public static void setupBeforeClass() throws Exception {
066    conf.setBoolean(ByteBuffAllocator.ALLOCATOR_POOL_ENABLED_KEY, true);
067    conf.setInt(ByteBuffAllocator.BUFFER_SIZE_KEY, 1024 * 5);
068    conf.setInt(CompactSplit.SMALL_COMPACTION_THREADS, REGION_COUNT * 2);
069    conf.setInt(CompactSplit.LARGE_COMPACTION_THREADS, REGION_COUNT * 2);
070    conf.set(HConstants.BUCKET_CACHE_IOENGINE_KEY, "offheap");
071    conf.setInt(HConstants.BUCKET_CACHE_SIZE_KEY, 512);
072    TEST_UTIL.startMiniCluster();
073    admin = TEST_UTIL.getAdmin();
074  }
075
076  @AfterClass
077  public static void tearDownAfterClass() throws Exception {
078    TEST_UTIL.shutdownMiniCluster();
079  }
080
081  @Test
082  public void testCompaction() throws Exception {
083    TableName table = TableName.valueOf("t1");
084    admin.compactionSwitch(false, new ArrayList<>(0));
085    try (Table t = createTable(TEST_UTIL, table)) {
086      for (int i = 0; i < 2; i++) {
087        put(t);
088        admin.flush(table);
089      }
090      admin.compactionSwitch(true, new ArrayList<>(0));
091      admin.majorCompact(table);
092      List<JVMClusterUtil.RegionServerThread> regionServerThreads =
093        TEST_UTIL.getHBaseCluster().getRegionServerThreads();
094      TEST_UTIL.waitFor(2 * 60 * 1000L, () -> {
095        boolean result = true;
096        for (JVMClusterUtil.RegionServerThread regionServerThread : regionServerThreads) {
097          HRegionServer regionServer = regionServerThread.getRegionServer();
098          List<HRegion> regions = regionServer.getRegions(table);
099          for (HRegion region : regions) {
100            List<String> storeFileList = region.getStoreFileList(new byte[][] { COLUMN });
101            if (storeFileList.size() > 1) {
102              result = false;
103            }
104          }
105        }
106        return result;
107      });
108    }
109  }
110
111  private Table createTable(HBaseTestingUtil util, TableName tableName) throws IOException {
112    TableDescriptor td = TableDescriptorBuilder.newBuilder(tableName).setColumnFamily(
113      ColumnFamilyDescriptorBuilder.newBuilder(COLUMN).setBlocksize(1024 * 4).build()).build();
114    byte[][] splits = new byte[REGION_COUNT - 1][];
115    for (int i = 1; i < REGION_COUNT; i++) {
116      splits[i - 1] = Bytes.toBytes(buildRow((int) (ROW_COUNT / REGION_COUNT * i)));
117    }
118    return util.createTable(td, splits);
119  }
120
121  private void put(Table table) throws IOException {
122    for (int i = 0; i < ROW_COUNT; i++) {
123      Put put = new Put(Bytes.toBytes(buildRow(i)));
124      put.addColumn(COLUMN, Bytes.toBytes("filed01"), buildValue(i, 1));
125      put.addColumn(COLUMN, Bytes.toBytes("filed02"), buildValue(i, 2));
126      put.addColumn(COLUMN, Bytes.toBytes("filed03"), buildValue(i, 3));
127      put.addColumn(COLUMN, Bytes.toBytes("filed04"), buildValue(i, 4));
128      put.addColumn(COLUMN, Bytes.toBytes("filed05"), buildValue(i, 5));
129      table.put(put);
130    }
131  }
132
133  private String buildRow(int index) {
134    String value = Long.toString(index);
135    String prefix = "user";
136    for (int i = 0; i < ROW_LENGTH - value.length(); i++) {
137      prefix += '0';
138    }
139    return prefix + value;
140  }
141
142  private byte[] buildValue(int index, int qualifierId) {
143    String row = buildRow(index) + "/f" + qualifierId + "-";
144    StringBuffer result = new StringBuffer();
145    while (result.length() < VALUE_LENGTH) {
146      result.append(row);
147    }
148    return Bytes.toBytes(result.toString().substring(0, VALUE_LENGTH));
149  }
150}