001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import java.io.IOException;
021import java.util.ArrayList;
022import java.util.List;
023import org.apache.hadoop.conf.Configuration;
024import org.apache.hadoop.hbase.HBaseTestingUtil;
025import org.apache.hadoop.hbase.HConstants;
026import org.apache.hadoop.hbase.TableName;
027import org.apache.hadoop.hbase.client.Admin;
028import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
029import org.apache.hadoop.hbase.client.Put;
030import org.apache.hadoop.hbase.client.Table;
031import org.apache.hadoop.hbase.client.TableDescriptor;
032import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
033import org.apache.hadoop.hbase.io.ByteBuffAllocator;
034import org.apache.hadoop.hbase.testclassification.LargeTests;
035import org.apache.hadoop.hbase.util.Bytes;
036import org.apache.hadoop.hbase.util.JVMClusterUtil;
037import org.junit.jupiter.api.AfterAll;
038import org.junit.jupiter.api.BeforeAll;
039import org.junit.jupiter.api.Tag;
040import org.junit.jupiter.api.Test;
041
042@Tag(LargeTests.TAG)
043public class TestCompactionWithByteBuff {
044
045  private static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
046  private static Configuration conf = TEST_UTIL.getConfiguration();
047  private static Admin admin = null;
048
049  private static final byte[] COLUMN = Bytes.toBytes("A");
050  private static final int REGION_COUNT = 5;
051  private static final long ROW_COUNT = 200;
052  private static final int ROW_LENGTH = 20;
053  private static final int VALUE_LENGTH = 5000;
054
055  @BeforeAll
056  public static void setupBeforeClass() throws Exception {
057    conf.setBoolean(ByteBuffAllocator.ALLOCATOR_POOL_ENABLED_KEY, true);
058    conf.setInt(ByteBuffAllocator.BUFFER_SIZE_KEY, 1024 * 5);
059    conf.setInt(CompactSplit.SMALL_COMPACTION_THREADS, REGION_COUNT * 2);
060    conf.setInt(CompactSplit.LARGE_COMPACTION_THREADS, REGION_COUNT * 2);
061    conf.set(HConstants.BUCKET_CACHE_IOENGINE_KEY, "offheap");
062    conf.setInt(HConstants.BUCKET_CACHE_SIZE_KEY, 512);
063    TEST_UTIL.startMiniCluster();
064    admin = TEST_UTIL.getAdmin();
065  }
066
067  @AfterAll
068  public static void tearDownAfterClass() throws Exception {
069    TEST_UTIL.shutdownMiniCluster();
070  }
071
072  @Test
073  public void testCompaction() throws Exception {
074    TableName table = TableName.valueOf("t1");
075    admin.compactionSwitch(false, new ArrayList<>(0));
076    try (Table t = createTable(TEST_UTIL, table)) {
077      for (int i = 0; i < 2; i++) {
078        put(t);
079        admin.flush(table);
080      }
081      admin.compactionSwitch(true, new ArrayList<>(0));
082      admin.majorCompact(table);
083      List<JVMClusterUtil.RegionServerThread> regionServerThreads =
084        TEST_UTIL.getHBaseCluster().getRegionServerThreads();
085      TEST_UTIL.waitFor(2 * 60 * 1000L, () -> {
086        boolean result = true;
087        for (JVMClusterUtil.RegionServerThread regionServerThread : regionServerThreads) {
088          HRegionServer regionServer = regionServerThread.getRegionServer();
089          List<HRegion> regions = regionServer.getRegions(table);
090          for (HRegion region : regions) {
091            List<String> storeFileList = region.getStoreFileList(new byte[][] { COLUMN });
092            if (storeFileList.size() > 1) {
093              result = false;
094            }
095          }
096        }
097        return result;
098      });
099    }
100  }
101
102  private Table createTable(HBaseTestingUtil util, TableName tableName) throws IOException {
103    TableDescriptor td = TableDescriptorBuilder.newBuilder(tableName).setColumnFamily(
104      ColumnFamilyDescriptorBuilder.newBuilder(COLUMN).setBlocksize(1024 * 4).build()).build();
105    byte[][] splits = new byte[REGION_COUNT - 1][];
106    for (int i = 1; i < REGION_COUNT; i++) {
107      splits[i - 1] = Bytes.toBytes(buildRow((int) (ROW_COUNT / REGION_COUNT * i)));
108    }
109    return util.createTable(td, splits);
110  }
111
112  private void put(Table table) throws IOException {
113    for (int i = 0; i < ROW_COUNT; i++) {
114      Put put = new Put(Bytes.toBytes(buildRow(i)));
115      put.addColumn(COLUMN, Bytes.toBytes("filed01"), buildValue(i, 1));
116      put.addColumn(COLUMN, Bytes.toBytes("filed02"), buildValue(i, 2));
117      put.addColumn(COLUMN, Bytes.toBytes("filed03"), buildValue(i, 3));
118      put.addColumn(COLUMN, Bytes.toBytes("filed04"), buildValue(i, 4));
119      put.addColumn(COLUMN, Bytes.toBytes("filed05"), buildValue(i, 5));
120      table.put(put);
121    }
122  }
123
124  private String buildRow(int index) {
125    String value = Long.toString(index);
126    String prefix = "user";
127    for (int i = 0; i < ROW_LENGTH - value.length(); i++) {
128      prefix += '0';
129    }
130    return prefix + value;
131  }
132
133  private byte[] buildValue(int index, int qualifierId) {
134    String row = buildRow(index) + "/f" + qualifierId + "-";
135    StringBuffer result = new StringBuffer();
136    while (result.length() < VALUE_LENGTH) {
137      result.append(row);
138    }
139    return Bytes.toBytes(result.toString().substring(0, VALUE_LENGTH));
140  }
141}