001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.apache.hadoop.hbase.HBaseTestingUtil.START_KEY_BYTES;
021import static org.apache.hadoop.hbase.HBaseTestingUtil.fam1;
022import static org.apache.hadoop.hbase.HBaseTestingUtil.fam2;
023import static org.junit.Assert.assertEquals;
024import static org.junit.Assert.assertTrue;
025
026import java.io.IOException;
027import java.util.Collection;
028import java.util.List;
029import java.util.Optional;
030import java.util.stream.Collectors;
031import org.apache.hadoop.conf.Configuration;
032import org.apache.hadoop.hbase.HBaseClassTestRule;
033import org.apache.hadoop.hbase.HBaseTestingUtil;
034import org.apache.hadoop.hbase.HConstants;
035import org.apache.hadoop.hbase.HTestConst;
036import org.apache.hadoop.hbase.TableName;
037import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
038import org.apache.hadoop.hbase.client.Delete;
039import org.apache.hadoop.hbase.client.Get;
040import org.apache.hadoop.hbase.client.Result;
041import org.apache.hadoop.hbase.client.Table;
042import org.apache.hadoop.hbase.client.TableDescriptor;
043import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
044import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequestImpl;
045import org.apache.hadoop.hbase.regionserver.compactions.RatioBasedCompactionPolicy;
046import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController;
047import org.apache.hadoop.hbase.testclassification.RegionServerTests;
048import org.apache.hadoop.hbase.testclassification.SmallTests;
049import org.apache.hadoop.hbase.util.Bytes;
050import org.apache.hadoop.hbase.wal.WAL;
051import org.junit.After;
052import org.junit.Before;
053import org.junit.BeforeClass;
054import org.junit.ClassRule;
055import org.junit.Rule;
056import org.junit.Test;
057import org.junit.experimental.categories.Category;
058import org.junit.rules.TestName;
059
060/**
061 * Test minor compactions
062 */
063@Category({RegionServerTests.class, SmallTests.class})
064public class TestMinorCompaction {
065
066  @ClassRule
067  public static final HBaseClassTestRule CLASS_RULE =
068      HBaseClassTestRule.forClass(TestMinorCompaction.class);
069
070  @Rule
071  public TestName name = new TestName();
072  private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
073  private static Configuration CONF = UTIL.getConfiguration();
074
075  private HRegion r = null;
076  private TableDescriptor htd = null;
077  private static int COMPACTION_THRESHOLD;
078  private static byte[] FIRST_ROW_BYTES, SECOND_ROW_BYTES, THIRD_ROW_BYTES;
079  private static byte[] COL1, COL2;
080
081  public static final class MyCompactionPolicy extends RatioBasedCompactionPolicy {
082
083    public MyCompactionPolicy(Configuration conf, StoreConfigInformation storeConfigInfo) {
084      super(conf, storeConfigInfo);
085    }
086
087    @Override
088    public CompactionRequestImpl selectCompaction(Collection<HStoreFile> candidateFiles,
089      List<HStoreFile> filesCompacting, boolean isUserCompaction, boolean mayUseOffPeak,
090      boolean forceMajor) throws IOException {
091      return new CompactionRequestImpl(
092        candidateFiles.stream().filter(f -> !filesCompacting.contains(f))
093          .limit(COMPACTION_THRESHOLD).collect(Collectors.toList()));
094    }
095  }
096
097  @BeforeClass
098  public static void setUpBeforeClass() {
099    // Set cache flush size to 1MB
100    CONF.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024 * 1024);
101    CONF.setInt(HConstants.HREGION_MEMSTORE_BLOCK_MULTIPLIER, 100);
102    COMPACTION_THRESHOLD = CONF.getInt("hbase.hstore.compactionThreshold", 3);
103    CONF.setClass(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY, MyCompactionPolicy.class,
104      RatioBasedCompactionPolicy.class);
105
106    FIRST_ROW_BYTES = START_KEY_BYTES;
107    SECOND_ROW_BYTES = START_KEY_BYTES.clone();
108    // Increment the least significant character so we get to next row.
109    SECOND_ROW_BYTES[START_KEY_BYTES.length - 1]++;
110    THIRD_ROW_BYTES = START_KEY_BYTES.clone();
111    THIRD_ROW_BYTES[START_KEY_BYTES.length - 1] =
112      (byte) (THIRD_ROW_BYTES[START_KEY_BYTES.length - 1] + 2);
113    COL1 = Bytes.toBytes("column1");
114    COL2 = Bytes.toBytes("column2");
115  }
116
117  @Before
118  public void setUp() throws Exception {
119    this.htd = UTIL.createTableDescriptor(TableName.valueOf(name.getMethodName()),
120      ColumnFamilyDescriptorBuilder.DEFAULT_MIN_VERSIONS, 3, HConstants.FOREVER,
121      ColumnFamilyDescriptorBuilder.DEFAULT_KEEP_DELETED);
122    this.r = UTIL.createLocalHRegion(htd, null, null);
123  }
124
125  @After
126  public void tearDown() throws Exception {
127    WAL wal = ((HRegion) r).getWAL();
128    ((HRegion) r).close();
129    wal.close();
130  }
131
132  @Test
133  public void testMinorCompactionWithDeleteRow() throws Exception {
134    Delete deleteRow = new Delete(SECOND_ROW_BYTES);
135    testMinorCompactionWithDelete(deleteRow);
136  }
137
138  @Test
139  public void testMinorCompactionWithDeleteColumn1() throws Exception {
140    Delete dc = new Delete(SECOND_ROW_BYTES);
141    /* delete all timestamps in the column */
142    dc.addColumns(fam2, COL2);
143    testMinorCompactionWithDelete(dc);
144  }
145
146  @Test
147  public void testMinorCompactionWithDeleteColumn2() throws Exception {
148    Delete dc = new Delete(SECOND_ROW_BYTES);
149    dc.addColumn(fam2, COL2);
150    /* compactionThreshold is 3. The table has 4 versions: 0, 1, 2, and 3.
151     * we only delete the latest version. One might expect to see only
152     * versions 1 and 2. HBase differs, and gives us 0, 1 and 2.
153     * This is okay as well. Since there was no compaction done before the
154     * delete, version 0 seems to stay on.
155     */
156    testMinorCompactionWithDelete(dc, 3);
157  }
158
159  @Test
160  public void testMinorCompactionWithDeleteColumnFamily() throws Exception {
161    Delete deleteCF = new Delete(SECOND_ROW_BYTES);
162    deleteCF.addFamily(fam2);
163    testMinorCompactionWithDelete(deleteCF);
164  }
165
166  @Test
167  public void testMinorCompactionWithDeleteVersion1() throws Exception {
168    Delete deleteVersion = new Delete(SECOND_ROW_BYTES);
169    deleteVersion.addColumns(fam2, COL2, 2);
170    /* compactionThreshold is 3. The table has 4 versions: 0, 1, 2, and 3.
171     * We delete versions 0 ... 2. So, we still have one remaining.
172     */
173    testMinorCompactionWithDelete(deleteVersion, 1);
174  }
175
176  @Test
177  public void testMinorCompactionWithDeleteVersion2() throws Exception {
178    Delete deleteVersion = new Delete(SECOND_ROW_BYTES);
179    deleteVersion.addColumn(fam2, COL2, 1);
180    /*
181     * the table has 4 versions: 0, 1, 2, and 3.
182     * We delete 1.
183     * Should have 3 remaining.
184     */
185    testMinorCompactionWithDelete(deleteVersion, 3);
186  }
187
188  /*
189   * A helper function to test the minor compaction algorithm. We check that
190   * the delete markers are left behind. Takes delete as an argument, which
191   * can be any delete (row, column, columnfamliy etc), that essentially
192   * deletes row2 and column2. row1 and column1 should be undeleted
193   */
194  private void testMinorCompactionWithDelete(Delete delete) throws Exception {
195    testMinorCompactionWithDelete(delete, 0);
196  }
197
198  private void testMinorCompactionWithDelete(Delete delete, int expectedResultsAfterDelete)
199      throws Exception {
200    Table loader = new RegionAsTable(r);
201    for (int i = 0; i < COMPACTION_THRESHOLD + 1; i++) {
202      HTestConst.addContent(loader, Bytes.toString(fam1), Bytes.toString(COL1), FIRST_ROW_BYTES,
203        THIRD_ROW_BYTES, i);
204      HTestConst.addContent(loader, Bytes.toString(fam1), Bytes.toString(COL2), FIRST_ROW_BYTES,
205        THIRD_ROW_BYTES, i);
206      HTestConst.addContent(loader, Bytes.toString(fam2), Bytes.toString(COL1), FIRST_ROW_BYTES,
207        THIRD_ROW_BYTES, i);
208      HTestConst.addContent(loader, Bytes.toString(fam2), Bytes.toString(COL2), FIRST_ROW_BYTES,
209        THIRD_ROW_BYTES, i);
210      r.flush(true);
211    }
212
213    Result result = r.get(new Get(FIRST_ROW_BYTES).addColumn(fam1, COL1).readVersions(100));
214    assertEquals(COMPACTION_THRESHOLD, result.size());
215    result = r.get(new Get(SECOND_ROW_BYTES).addColumn(fam2, COL2).readVersions(100));
216    assertEquals(COMPACTION_THRESHOLD, result.size());
217
218    // Now add deletes to memstore and then flush it.  That will put us over
219    // the compaction threshold of 3 store files.  Compacting these store files
220    // should result in a compacted store file that has no references to the
221    // deleted row.
222    r.delete(delete);
223
224    // Make sure that we have only deleted family2 from secondRowBytes
225    result = r.get(new Get(SECOND_ROW_BYTES).addColumn(fam2, COL2).readVersions(100));
226    assertEquals(expectedResultsAfterDelete, result.size());
227    // but we still have firstrow
228    result = r.get(new Get(FIRST_ROW_BYTES).addColumn(fam1, COL1).readVersions(100));
229    assertEquals(COMPACTION_THRESHOLD, result.size());
230
231    r.flush(true);
232    // should not change anything.
233    // Let us check again
234
235    // Make sure that we have only deleted family2 from secondRowBytes
236    result = r.get(new Get(SECOND_ROW_BYTES).addColumn(fam2, COL2).readVersions(100));
237    assertEquals(expectedResultsAfterDelete, result.size());
238    // but we still have firstrow
239    result = r.get(new Get(FIRST_ROW_BYTES).addColumn(fam1, COL1).readVersions(100));
240    assertEquals(COMPACTION_THRESHOLD, result.size());
241
242    // do a compaction
243    HStore store2 = r.getStore(fam2);
244    int numFiles1 = store2.getStorefiles().size();
245    assertTrue("Was expecting to see 4 store files", numFiles1 > COMPACTION_THRESHOLD); // > 3
246    Optional<CompactionContext> compaction = store2.requestCompaction();
247    assertTrue(compaction.isPresent());
248    store2.compact(compaction.get(), NoLimitThroughputController.INSTANCE, null); // = 3
249    int numFiles2 = store2.getStorefiles().size();
250    // Check that we did compact
251    assertTrue("Number of store files should go down", numFiles1 > numFiles2);
252    // Check that it was a minor compaction.
253    assertTrue("Was not supposed to be a major compaction", numFiles2 > 1);
254
255    // Make sure that we have only deleted family2 from secondRowBytes
256    result = r.get(new Get(SECOND_ROW_BYTES).addColumn(fam2, COL2).readVersions(100));
257    assertEquals(expectedResultsAfterDelete, result.size());
258    // but we still have firstrow
259    result = r.get(new Get(FIRST_ROW_BYTES).addColumn(fam1, COL1).readVersions(100));
260    assertEquals(COMPACTION_THRESHOLD, result.size());
261  }
262}