001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertTrue;
022
023import java.io.IOException;
024import java.util.ArrayList;
025import java.util.Arrays;
026import java.util.List;
027import org.apache.hadoop.conf.Configuration;
028import org.apache.hadoop.fs.FileSystem;
029import org.apache.hadoop.fs.Path;
030import org.apache.hadoop.hbase.HBaseTestingUtil;
031import org.apache.hadoop.hbase.HConstants;
032import org.apache.hadoop.hbase.TableName;
033import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
034import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
035import org.apache.hadoop.hbase.client.RegionInfo;
036import org.apache.hadoop.hbase.client.RegionInfoBuilder;
037import org.apache.hadoop.hbase.client.TableDescriptor;
038import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
039import org.apache.hadoop.hbase.regionserver.compactions.CompactionConfiguration;
040import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequestImpl;
041import org.apache.hadoop.hbase.regionserver.compactions.RatioBasedCompactionPolicy;
042import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerForTest;
043import org.apache.hadoop.hbase.regionserver.wal.FSHLog;
044import org.apache.hadoop.hbase.util.Bytes;
045import org.apache.hadoop.hbase.util.CommonFSUtils;
046import org.junit.jupiter.api.AfterEach;
047import org.junit.jupiter.api.BeforeEach;
048import org.slf4j.Logger;
049import org.slf4j.LoggerFactory;
050
051import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
052
053public class TestCompactionPolicy {
054
055  private final static Logger LOG = LoggerFactory.getLogger(TestCompactionPolicy.class);
056  protected final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
057
058  protected Configuration conf;
059  protected HStore store;
060  private static final String DIR =
061    TEST_UTIL.getDataTestDir(TestCompactionPolicy.class.getSimpleName()).toString();
062  protected static Path TEST_FILE;
063  protected static final int minFiles = 3;
064  protected static final int maxFiles = 5;
065
066  protected static final long minSize = 10;
067  protected static final long maxSize = 2100;
068
069  private FSHLog hlog;
070  private HRegion region;
071
072  @BeforeEach
073  public void setUp() throws Exception {
074    config();
075    initialize();
076  }
077
078  /**
079   * setup config values necessary for store
080   */
081  protected void config() {
082    this.conf = TEST_UTIL.getConfiguration();
083    this.conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 0);
084    this.conf.setInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MIN_KEY, minFiles);
085    this.conf.setInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MAX_KEY, maxFiles);
086    this.conf.setLong(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MIN_SIZE_KEY, minSize);
087    this.conf.setLong(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MAX_SIZE_KEY, maxSize);
088    this.conf.setFloat(CompactionConfiguration.HBASE_HSTORE_COMPACTION_RATIO_KEY, 1.0F);
089  }
090
091  /**
092   * Setting up a Store
093   * @throws IOException with error
094   */
095  protected void initialize() throws IOException {
096    Path basedir = new Path(DIR);
097    String logName = "logs";
098    Path logdir = new Path(DIR, logName);
099    ColumnFamilyDescriptor familyDescriptor =
100      ColumnFamilyDescriptorBuilder.of(Bytes.toBytes("family"));
101    FileSystem fs = FileSystem.get(conf);
102
103    fs.delete(logdir, true);
104
105    TableDescriptor tableDescriptor =
106      TableDescriptorBuilder.newBuilder(TableName.valueOf(Bytes.toBytes("table")))
107        .setColumnFamily(familyDescriptor).build();
108    RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
109
110    fs.mkdirs(new Path(basedir, logName));
111    hlog = new FSHLog(fs, basedir, logName, conf);
112    hlog.init();
113    ChunkCreator.initialize(MemStoreLAB.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null,
114      MemStoreLAB.INDEX_CHUNK_SIZE_PERCENTAGE_DEFAULT);
115    region = HRegion.createHRegion(info, basedir, conf, tableDescriptor, hlog);
116    region.close();
117    Path tableDir = CommonFSUtils.getTableDir(basedir, tableDescriptor.getTableName());
118    region = new HRegion(tableDir, hlog, fs, conf, info, tableDescriptor, null);
119
120    store = new HStore(region, familyDescriptor, conf, false);
121
122    TEST_FILE = region.getRegionFileSystem().createTempName();
123    fs.createNewFile(TEST_FILE);
124  }
125
126  @AfterEach
127  public void tearDown() throws IOException {
128    IOException ex = null;
129    try {
130      region.close();
131    } catch (IOException e) {
132      LOG.warn("Caught Exception", e);
133      ex = e;
134    }
135    try {
136      hlog.close();
137    } catch (IOException e) {
138      LOG.warn("Caught Exception", e);
139      ex = e;
140    }
141    if (ex != null) {
142      throw ex;
143    }
144  }
145
146  ArrayList<Long> toArrayList(long... numbers) {
147    ArrayList<Long> result = new ArrayList<>();
148    for (long i : numbers) {
149      result.add(i);
150    }
151    return result;
152  }
153
154  List<HStoreFile> sfCreate(long... sizes) throws IOException {
155    ArrayList<Long> ageInDisk = new ArrayList<>();
156    for (int i = 0; i < sizes.length; i++) {
157      ageInDisk.add(0L);
158    }
159    return sfCreate(toArrayList(sizes), ageInDisk);
160  }
161
162  List<HStoreFile> sfCreate(ArrayList<Long> sizes, ArrayList<Long> ageInDisk) throws IOException {
163    return sfCreate(false, sizes, ageInDisk);
164  }
165
166  List<HStoreFile> sfCreate(boolean isReference, long... sizes) throws IOException {
167    ArrayList<Long> ageInDisk = new ArrayList<>(sizes.length);
168    for (int i = 0; i < sizes.length; i++) {
169      ageInDisk.add(0L);
170    }
171    return sfCreate(isReference, toArrayList(sizes), ageInDisk);
172  }
173
174  List<HStoreFile> sfCreate(boolean isReference, ArrayList<Long> sizes, ArrayList<Long> ageInDisk)
175    throws IOException {
176    List<HStoreFile> ret = Lists.newArrayList();
177    StoreFileTrackerForTest storeFileTrackerForTest =
178      new StoreFileTrackerForTest(conf, true, store.getStoreContext());
179    for (int i = 0; i < sizes.size(); i++) {
180      ret.add(new MockHStoreFile(TEST_UTIL, TEST_FILE, sizes.get(i), ageInDisk.get(i), isReference,
181        i, storeFileTrackerForTest));
182    }
183    return ret;
184  }
185
186  long[] getSizes(List<HStoreFile> sfList) {
187    long[] aNums = new long[sfList.size()];
188    for (int i = 0; i < sfList.size(); ++i) {
189      aNums[i] = sfList.get(i).getReader().length();
190    }
191    return aNums;
192  }
193
194  void compactEquals(List<HStoreFile> candidates, long... expected) throws IOException {
195    compactEquals(candidates, false, false, expected);
196  }
197
198  void compactEquals(List<HStoreFile> candidates, boolean forcemajor, long... expected)
199    throws IOException {
200    compactEquals(candidates, forcemajor, false, expected);
201  }
202
203  void compactEquals(List<HStoreFile> candidates, boolean forcemajor, boolean isOffPeak,
204    long... expected) throws IOException {
205    store.forceMajor = forcemajor;
206    // Test Default compactions
207    CompactionRequestImpl result =
208      ((RatioBasedCompactionPolicy) store.storeEngine.getCompactionPolicy())
209        .selectCompaction(candidates, new ArrayList<>(), false, isOffPeak, forcemajor);
210    List<HStoreFile> actual = new ArrayList<>(result.getFiles());
211    if (isOffPeak && !forcemajor) {
212      assertTrue(result.isOffPeak());
213    }
214    assertEquals(Arrays.toString(expected), Arrays.toString(getSizes(actual)));
215    store.forceMajor = false;
216  }
217}