001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.apache.hadoop.hbase.HBaseTestingUtility.START_KEY_BYTES;
021import static org.apache.hadoop.hbase.HBaseTestingUtility.fam1;
022import static org.apache.hadoop.hbase.HBaseTestingUtility.fam2;
023import static org.junit.Assert.assertEquals;
024import static org.junit.Assert.assertTrue;
025
026import java.io.IOException;
027import java.util.Collection;
028import java.util.List;
029import java.util.Optional;
030import java.util.stream.Collectors;
031import org.apache.hadoop.conf.Configuration;
032import org.apache.hadoop.hbase.HBaseClassTestRule;
033import org.apache.hadoop.hbase.HBaseTestingUtility;
034import org.apache.hadoop.hbase.HConstants;
035import org.apache.hadoop.hbase.HTestConst;
036import org.apache.hadoop.hbase.client.Delete;
037import org.apache.hadoop.hbase.client.Get;
038import org.apache.hadoop.hbase.client.Result;
039import org.apache.hadoop.hbase.client.Table;
040import org.apache.hadoop.hbase.client.TableDescriptor;
041import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
042import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequestImpl;
043import org.apache.hadoop.hbase.regionserver.compactions.RatioBasedCompactionPolicy;
044import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController;
045import org.apache.hadoop.hbase.testclassification.RegionServerTests;
046import org.apache.hadoop.hbase.testclassification.SmallTests;
047import org.apache.hadoop.hbase.util.Bytes;
048import org.apache.hadoop.hbase.wal.WAL;
049import org.junit.After;
050import org.junit.Before;
051import org.junit.BeforeClass;
052import org.junit.ClassRule;
053import org.junit.Rule;
054import org.junit.Test;
055import org.junit.experimental.categories.Category;
056import org.junit.rules.TestName;
057
058/**
059 * Test minor compactions
060 */
061@Category({ RegionServerTests.class, SmallTests.class })
062public class TestMinorCompaction {
063
064  @ClassRule
065  public static final HBaseClassTestRule CLASS_RULE =
066    HBaseClassTestRule.forClass(TestMinorCompaction.class);
067
068  @Rule
069  public TestName name = new TestName();
070  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
071  private static Configuration CONF = UTIL.getConfiguration();
072
073  private HRegion r = null;
074  private TableDescriptor htd = null;
075  private static int COMPACTION_THRESHOLD;
076  private static byte[] FIRST_ROW_BYTES, SECOND_ROW_BYTES, THIRD_ROW_BYTES;
077  private static byte[] COL1, COL2;
078
079  public static final class MyCompactionPolicy extends RatioBasedCompactionPolicy {
080
081    public MyCompactionPolicy(Configuration conf, StoreConfigInformation storeConfigInfo) {
082      super(conf, storeConfigInfo);
083    }
084
085    @Override
086    public CompactionRequestImpl selectCompaction(Collection<HStoreFile> candidateFiles,
087      List<HStoreFile> filesCompacting, boolean isUserCompaction, boolean mayUseOffPeak,
088      boolean forceMajor) throws IOException {
089      return new CompactionRequestImpl(
090        candidateFiles.stream().filter(f -> !filesCompacting.contains(f))
091          .limit(COMPACTION_THRESHOLD).collect(Collectors.toList()));
092    }
093  }
094
095  @BeforeClass
096  public static void setUpBeforeClass() {
097    // Set cache flush size to 1MB
098    CONF.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024 * 1024);
099    CONF.setInt(HConstants.HREGION_MEMSTORE_BLOCK_MULTIPLIER, 100);
100    COMPACTION_THRESHOLD = CONF.getInt("hbase.hstore.compactionThreshold", 3);
101    CONF.setClass(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY, MyCompactionPolicy.class,
102      RatioBasedCompactionPolicy.class);
103
104    FIRST_ROW_BYTES = START_KEY_BYTES;
105    SECOND_ROW_BYTES = START_KEY_BYTES.clone();
106    // Increment the least significant character so we get to next row.
107    SECOND_ROW_BYTES[START_KEY_BYTES.length - 1]++;
108    THIRD_ROW_BYTES = START_KEY_BYTES.clone();
109    THIRD_ROW_BYTES[START_KEY_BYTES.length - 1] =
110      (byte) (THIRD_ROW_BYTES[START_KEY_BYTES.length - 1] + 2);
111    COL1 = Bytes.toBytes("column1");
112    COL2 = Bytes.toBytes("column2");
113  }
114
115  @Before
116  public void setUp() throws Exception {
117    this.htd = UTIL.createTableDescriptor(name.getMethodName());
118    this.r = UTIL.createLocalHRegion(htd, null, null);
119  }
120
121  @After
122  public void tearDown() throws Exception {
123    WAL wal = ((HRegion) r).getWAL();
124    ((HRegion) r).close();
125    wal.close();
126  }
127
128  @Test
129  public void testMinorCompactionWithDeleteRow() throws Exception {
130    Delete deleteRow = new Delete(SECOND_ROW_BYTES);
131    testMinorCompactionWithDelete(deleteRow);
132  }
133
134  @Test
135  public void testMinorCompactionWithDeleteColumn1() throws Exception {
136    Delete dc = new Delete(SECOND_ROW_BYTES);
137    /* delete all timestamps in the column */
138    dc.addColumns(fam2, COL2);
139    testMinorCompactionWithDelete(dc);
140  }
141
142  @Test
143  public void testMinorCompactionWithDeleteColumn2() throws Exception {
144    Delete dc = new Delete(SECOND_ROW_BYTES);
145    dc.addColumn(fam2, COL2);
146    /*
147     * compactionThreshold is 3. The table has 4 versions: 0, 1, 2, and 3. we only delete the latest
148     * version. One might expect to see only versions 1 and 2. HBase differs, and gives us 0, 1 and
149     * 2. This is okay as well. Since there was no compaction done before the delete, version 0
150     * seems to stay on.
151     */
152    testMinorCompactionWithDelete(dc, 3);
153  }
154
155  @Test
156  public void testMinorCompactionWithDeleteColumnFamily() throws Exception {
157    Delete deleteCF = new Delete(SECOND_ROW_BYTES);
158    deleteCF.addFamily(fam2);
159    testMinorCompactionWithDelete(deleteCF);
160  }
161
162  @Test
163  public void testMinorCompactionWithDeleteVersion1() throws Exception {
164    Delete deleteVersion = new Delete(SECOND_ROW_BYTES);
165    deleteVersion.addColumns(fam2, COL2, 2);
166    /*
167     * compactionThreshold is 3. The table has 4 versions: 0, 1, 2, and 3. We delete versions 0 ...
168     * 2. So, we still have one remaining.
169     */
170    testMinorCompactionWithDelete(deleteVersion, 1);
171  }
172
173  @Test
174  public void testMinorCompactionWithDeleteVersion2() throws Exception {
175    Delete deleteVersion = new Delete(SECOND_ROW_BYTES);
176    deleteVersion.addColumn(fam2, COL2, 1);
177    /*
178     * the table has 4 versions: 0, 1, 2, and 3. We delete 1. Should have 3 remaining.
179     */
180    testMinorCompactionWithDelete(deleteVersion, 3);
181  }
182
183  /*
184   * A helper function to test the minor compaction algorithm. We check that the delete markers are
185   * left behind. Takes delete as an argument, which can be any delete (row, column, columnfamliy
186   * etc), that essentially deletes row2 and column2. row1 and column1 should be undeleted
187   */
188  private void testMinorCompactionWithDelete(Delete delete) throws Exception {
189    testMinorCompactionWithDelete(delete, 0);
190  }
191
192  private void testMinorCompactionWithDelete(Delete delete, int expectedResultsAfterDelete)
193    throws Exception {
194    Table loader = new RegionAsTable(r);
195    for (int i = 0; i < COMPACTION_THRESHOLD + 1; i++) {
196      HTestConst.addContent(loader, Bytes.toString(fam1), Bytes.toString(COL1), FIRST_ROW_BYTES,
197        THIRD_ROW_BYTES, i);
198      HTestConst.addContent(loader, Bytes.toString(fam1), Bytes.toString(COL2), FIRST_ROW_BYTES,
199        THIRD_ROW_BYTES, i);
200      HTestConst.addContent(loader, Bytes.toString(fam2), Bytes.toString(COL1), FIRST_ROW_BYTES,
201        THIRD_ROW_BYTES, i);
202      HTestConst.addContent(loader, Bytes.toString(fam2), Bytes.toString(COL2), FIRST_ROW_BYTES,
203        THIRD_ROW_BYTES, i);
204      r.flush(true);
205    }
206
207    Result result = r.get(new Get(FIRST_ROW_BYTES).addColumn(fam1, COL1).readVersions(100));
208    assertEquals(COMPACTION_THRESHOLD, result.size());
209    result = r.get(new Get(SECOND_ROW_BYTES).addColumn(fam2, COL2).readVersions(100));
210    assertEquals(COMPACTION_THRESHOLD, result.size());
211
212    // Now add deletes to memstore and then flush it. That will put us over
213    // the compaction threshold of 3 store files. Compacting these store files
214    // should result in a compacted store file that has no references to the
215    // deleted row.
216    r.delete(delete);
217
218    // Make sure that we have only deleted family2 from secondRowBytes
219    result = r.get(new Get(SECOND_ROW_BYTES).addColumn(fam2, COL2).readVersions(100));
220    assertEquals(expectedResultsAfterDelete, result.size());
221    // but we still have firstrow
222    result = r.get(new Get(FIRST_ROW_BYTES).addColumn(fam1, COL1).readVersions(100));
223    assertEquals(COMPACTION_THRESHOLD, result.size());
224
225    r.flush(true);
226    // should not change anything.
227    // Let us check again
228
229    // Make sure that we have only deleted family2 from secondRowBytes
230    result = r.get(new Get(SECOND_ROW_BYTES).addColumn(fam2, COL2).readVersions(100));
231    assertEquals(expectedResultsAfterDelete, result.size());
232    // but we still have firstrow
233    result = r.get(new Get(FIRST_ROW_BYTES).addColumn(fam1, COL1).readVersions(100));
234    assertEquals(COMPACTION_THRESHOLD, result.size());
235
236    // do a compaction
237    HStore store2 = r.getStore(fam2);
238    int numFiles1 = store2.getStorefiles().size();
239    assertTrue("Was expecting to see 4 store files", numFiles1 > COMPACTION_THRESHOLD); // > 3
240    Optional<CompactionContext> compaction = store2.requestCompaction();
241    assertTrue(compaction.isPresent());
242    store2.compact(compaction.get(), NoLimitThroughputController.INSTANCE, null); // = 3
243    int numFiles2 = store2.getStorefiles().size();
244    // Check that we did compact
245    assertTrue("Number of store files should go down", numFiles1 > numFiles2);
246    // Check that it was a minor compaction.
247    assertTrue("Was not supposed to be a major compaction", numFiles2 > 1);
248
249    // Make sure that we have only deleted family2 from secondRowBytes
250    result = r.get(new Get(SECOND_ROW_BYTES).addColumn(fam2, COL2).readVersions(100));
251    assertEquals(expectedResultsAfterDelete, result.size());
252    // but we still have firstrow
253    result = r.get(new Get(FIRST_ROW_BYTES).addColumn(fam1, COL1).readVersions(100));
254    assertEquals(COMPACTION_THRESHOLD, result.size());
255  }
256}