001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.apache.hadoop.hbase.HBaseTestingUtil.START_KEY_BYTES;
021import static org.apache.hadoop.hbase.HBaseTestingUtil.fam1;
022import static org.apache.hadoop.hbase.HBaseTestingUtil.fam2;
023import static org.junit.jupiter.api.Assertions.assertEquals;
024import static org.junit.jupiter.api.Assertions.assertTrue;
025
026import java.io.IOException;
027import java.util.Collection;
028import java.util.List;
029import java.util.Optional;
030import java.util.stream.Collectors;
031import org.apache.hadoop.conf.Configuration;
032import org.apache.hadoop.hbase.HBaseTestingUtil;
033import org.apache.hadoop.hbase.HConstants;
034import org.apache.hadoop.hbase.HTestConst;
035import org.apache.hadoop.hbase.TableName;
036import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
037import org.apache.hadoop.hbase.client.Delete;
038import org.apache.hadoop.hbase.client.Get;
039import org.apache.hadoop.hbase.client.Result;
040import org.apache.hadoop.hbase.client.Table;
041import org.apache.hadoop.hbase.client.TableDescriptor;
042import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
043import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequestImpl;
044import org.apache.hadoop.hbase.regionserver.compactions.RatioBasedCompactionPolicy;
045import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController;
046import org.apache.hadoop.hbase.testclassification.RegionServerTests;
047import org.apache.hadoop.hbase.testclassification.SmallTests;
048import org.apache.hadoop.hbase.util.Bytes;
049import org.apache.hadoop.hbase.wal.WAL;
050import org.junit.jupiter.api.AfterEach;
051import org.junit.jupiter.api.BeforeAll;
052import org.junit.jupiter.api.BeforeEach;
053import org.junit.jupiter.api.Tag;
054import org.junit.jupiter.api.Test;
055import org.junit.jupiter.api.TestInfo;
056
057/**
058 * Test minor compactions
059 */
060@Tag(RegionServerTests.TAG)
061@Tag(SmallTests.TAG)
062public class TestMinorCompaction {
063
064  private String name;
065  private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
066  private static Configuration CONF = UTIL.getConfiguration();
067
068  private HRegion r = null;
069  private TableDescriptor htd = null;
070  private static int COMPACTION_THRESHOLD;
071  private static byte[] FIRST_ROW_BYTES, SECOND_ROW_BYTES, THIRD_ROW_BYTES;
072  private static byte[] COL1, COL2;
073
074  public static final class MyCompactionPolicy extends RatioBasedCompactionPolicy {
075
076    public MyCompactionPolicy(Configuration conf, StoreConfigInformation storeConfigInfo) {
077      super(conf, storeConfigInfo);
078    }
079
080    @Override
081    public CompactionRequestImpl selectCompaction(Collection<HStoreFile> candidateFiles,
082      List<HStoreFile> filesCompacting, boolean isUserCompaction, boolean mayUseOffPeak,
083      boolean forceMajor) throws IOException {
084      return new CompactionRequestImpl(
085        candidateFiles.stream().filter(f -> !filesCompacting.contains(f))
086          .limit(COMPACTION_THRESHOLD).collect(Collectors.toList()));
087    }
088  }
089
090  @BeforeAll
091  public static void setUpBeforeClass() {
092    // Set cache flush size to 1MB
093    CONF.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024 * 1024);
094    CONF.setInt(HConstants.HREGION_MEMSTORE_BLOCK_MULTIPLIER, 100);
095    COMPACTION_THRESHOLD = CONF.getInt("hbase.hstore.compactionThreshold", 3);
096    CONF.setClass(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY, MyCompactionPolicy.class,
097      RatioBasedCompactionPolicy.class);
098
099    FIRST_ROW_BYTES = START_KEY_BYTES;
100    SECOND_ROW_BYTES = START_KEY_BYTES.clone();
101    // Increment the least significant character so we get to next row.
102    SECOND_ROW_BYTES[START_KEY_BYTES.length - 1]++;
103    THIRD_ROW_BYTES = START_KEY_BYTES.clone();
104    THIRD_ROW_BYTES[START_KEY_BYTES.length - 1] =
105      (byte) (THIRD_ROW_BYTES[START_KEY_BYTES.length - 1] + 2);
106    COL1 = Bytes.toBytes("column1");
107    COL2 = Bytes.toBytes("column2");
108  }
109
110  @BeforeEach
111  public void setUp(TestInfo testInfo) throws Exception {
112    this.name = testInfo.getTestMethod().get().getName();
113    this.htd = UTIL.createTableDescriptor(TableName.valueOf(name),
114      ColumnFamilyDescriptorBuilder.DEFAULT_MIN_VERSIONS, 3, HConstants.FOREVER,
115      ColumnFamilyDescriptorBuilder.DEFAULT_KEEP_DELETED);
116    this.r = UTIL.createLocalHRegion(htd, null, null);
117  }
118
119  @AfterEach
120  public void tearDown() throws Exception {
121    WAL wal = ((HRegion) r).getWAL();
122    ((HRegion) r).close();
123    wal.close();
124  }
125
126  @Test
127  public void testMinorCompactionWithDeleteRow() throws Exception {
128    Delete deleteRow = new Delete(SECOND_ROW_BYTES);
129    testMinorCompactionWithDelete(deleteRow);
130  }
131
132  @Test
133  public void testMinorCompactionWithDeleteColumn1() throws Exception {
134    Delete dc = new Delete(SECOND_ROW_BYTES);
135    /* delete all timestamps in the column */
136    dc.addColumns(fam2, COL2);
137    testMinorCompactionWithDelete(dc);
138  }
139
140  @Test
141  public void testMinorCompactionWithDeleteColumn2() throws Exception {
142    Delete dc = new Delete(SECOND_ROW_BYTES);
143    dc.addColumn(fam2, COL2);
144    /*
145     * compactionThreshold is 3. The table has 4 versions: 0, 1, 2, and 3. we only delete the latest
146     * version. One might expect to see only versions 1 and 2. HBase differs, and gives us 0, 1 and
147     * 2. This is okay as well. Since there was no compaction done before the delete, version 0
148     * seems to stay on.
149     */
150    testMinorCompactionWithDelete(dc, 3);
151  }
152
153  @Test
154  public void testMinorCompactionWithDeleteColumnFamily() throws Exception {
155    Delete deleteCF = new Delete(SECOND_ROW_BYTES);
156    deleteCF.addFamily(fam2);
157    testMinorCompactionWithDelete(deleteCF);
158  }
159
160  @Test
161  public void testMinorCompactionWithDeleteVersion1() throws Exception {
162    Delete deleteVersion = new Delete(SECOND_ROW_BYTES);
163    deleteVersion.addColumns(fam2, COL2, 2);
164    /*
165     * compactionThreshold is 3. The table has 4 versions: 0, 1, 2, and 3. We delete versions 0 ...
166     * 2. So, we still have one remaining.
167     */
168    testMinorCompactionWithDelete(deleteVersion, 1);
169  }
170
171  @Test
172  public void testMinorCompactionWithDeleteVersion2() throws Exception {
173    Delete deleteVersion = new Delete(SECOND_ROW_BYTES);
174    deleteVersion.addColumn(fam2, COL2, 1);
175    /*
176     * the table has 4 versions: 0, 1, 2, and 3. We delete 1. Should have 3 remaining.
177     */
178    testMinorCompactionWithDelete(deleteVersion, 3);
179  }
180
181  /*
182   * A helper function to test the minor compaction algorithm. We check that the delete markers are
183   * left behind. Takes delete as an argument, which can be any delete (row, column, columnfamliy
184   * etc), that essentially deletes row2 and column2. row1 and column1 should be undeleted
185   */
186  private void testMinorCompactionWithDelete(Delete delete) throws Exception {
187    testMinorCompactionWithDelete(delete, 0);
188  }
189
190  private void testMinorCompactionWithDelete(Delete delete, int expectedResultsAfterDelete)
191    throws Exception {
192    Table loader = new RegionAsTable(r);
193    for (int i = 0; i < COMPACTION_THRESHOLD + 1; i++) {
194      HTestConst.addContent(loader, Bytes.toString(fam1), Bytes.toString(COL1), FIRST_ROW_BYTES,
195        THIRD_ROW_BYTES, i);
196      HTestConst.addContent(loader, Bytes.toString(fam1), Bytes.toString(COL2), FIRST_ROW_BYTES,
197        THIRD_ROW_BYTES, i);
198      HTestConst.addContent(loader, Bytes.toString(fam2), Bytes.toString(COL1), FIRST_ROW_BYTES,
199        THIRD_ROW_BYTES, i);
200      HTestConst.addContent(loader, Bytes.toString(fam2), Bytes.toString(COL2), FIRST_ROW_BYTES,
201        THIRD_ROW_BYTES, i);
202      r.flush(true);
203    }
204
205    Result result = r.get(new Get(FIRST_ROW_BYTES).addColumn(fam1, COL1).readVersions(100));
206    assertEquals(COMPACTION_THRESHOLD, result.size());
207    result = r.get(new Get(SECOND_ROW_BYTES).addColumn(fam2, COL2).readVersions(100));
208    assertEquals(COMPACTION_THRESHOLD, result.size());
209
210    // Now add deletes to memstore and then flush it. That will put us over
211    // the compaction threshold of 3 store files. Compacting these store files
212    // should result in a compacted store file that has no references to the
213    // deleted row.
214    r.delete(delete);
215
216    // Make sure that we have only deleted family2 from secondRowBytes
217    result = r.get(new Get(SECOND_ROW_BYTES).addColumn(fam2, COL2).readVersions(100));
218    assertEquals(expectedResultsAfterDelete, result.size());
219    // but we still have firstrow
220    result = r.get(new Get(FIRST_ROW_BYTES).addColumn(fam1, COL1).readVersions(100));
221    assertEquals(COMPACTION_THRESHOLD, result.size());
222
223    r.flush(true);
224    // should not change anything.
225    // Let us check again
226
227    // Make sure that we have only deleted family2 from secondRowBytes
228    result = r.get(new Get(SECOND_ROW_BYTES).addColumn(fam2, COL2).readVersions(100));
229    assertEquals(expectedResultsAfterDelete, result.size());
230    // but we still have firstrow
231    result = r.get(new Get(FIRST_ROW_BYTES).addColumn(fam1, COL1).readVersions(100));
232    assertEquals(COMPACTION_THRESHOLD, result.size());
233
234    // do a compaction
235    HStore store2 = r.getStore(fam2);
236    int numFiles1 = store2.getStorefiles().size();
237    assertTrue(numFiles1 > COMPACTION_THRESHOLD, "Was expecting to see 4 store files"); // > 3
238    Optional<CompactionContext> compaction = store2.requestCompaction();
239    assertTrue(compaction.isPresent());
240    store2.compact(compaction.get(), NoLimitThroughputController.INSTANCE, null); // = 3
241    int numFiles2 = store2.getStorefiles().size();
242    // Check that we did compact
243    assertTrue(numFiles1 > numFiles2, "Number of store files should go down");
244    // Check that it was a minor compaction.
245    assertTrue(numFiles2 > 1, "Was not supposed to be a major compaction");
246
247    // Make sure that we have only deleted family2 from secondRowBytes
248    result = r.get(new Get(SECOND_ROW_BYTES).addColumn(fam2, COL2).readVersions(100));
249    assertEquals(expectedResultsAfterDelete, result.size());
250    // but we still have firstrow
251    result = r.get(new Get(FIRST_ROW_BYTES).addColumn(fam1, COL1).readVersions(100));
252    assertEquals(COMPACTION_THRESHOLD, result.size());
253  }
254}