001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.apache.hadoop.hbase.HBaseTestingUtility.START_KEY_BYTES;
021import static org.apache.hadoop.hbase.HBaseTestingUtility.fam1;
022import static org.apache.hadoop.hbase.HBaseTestingUtility.fam2;
023import static org.junit.Assert.assertEquals;
024import static org.junit.Assert.assertTrue;
025
026import org.apache.hadoop.conf.Configuration;
027import org.apache.hadoop.hbase.HBaseClassTestRule;
028import org.apache.hadoop.hbase.HBaseTestingUtility;
029import org.apache.hadoop.hbase.HColumnDescriptor;
030import org.apache.hadoop.hbase.HConstants;
031import org.apache.hadoop.hbase.HTableDescriptor;
032import org.apache.hadoop.hbase.HTestConst;
033import org.apache.hadoop.hbase.TableName;
034import org.apache.hadoop.hbase.client.Delete;
035import org.apache.hadoop.hbase.client.Get;
036import org.apache.hadoop.hbase.client.Result;
037import org.apache.hadoop.hbase.client.Table;
038import org.apache.hadoop.hbase.testclassification.RegionServerTests;
039import org.apache.hadoop.hbase.testclassification.SmallTests;
040import org.apache.hadoop.hbase.util.Bytes;
041import org.apache.hadoop.hbase.wal.WAL;
042import org.junit.After;
043import org.junit.Before;
044import org.junit.ClassRule;
045import org.junit.Rule;
046import org.junit.Test;
047import org.junit.experimental.categories.Category;
048import org.junit.rules.TestName;
049import org.slf4j.Logger;
050import org.slf4j.LoggerFactory;
051
052/**
053 * Test minor compactions
054 */
055@Category({RegionServerTests.class, SmallTests.class})
056public class TestMinorCompaction {
057
058  @ClassRule
059  public static final HBaseClassTestRule CLASS_RULE =
060      HBaseClassTestRule.forClass(TestMinorCompaction.class);
061
062  @Rule public TestName name = new TestName();
063  private static final Logger LOG = LoggerFactory.getLogger(TestMinorCompaction.class.getName());
064  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
065  protected Configuration conf = UTIL.getConfiguration();
066
067  private HRegion r = null;
068  private HTableDescriptor htd = null;
069  private int compactionThreshold;
070  private byte[] firstRowBytes, secondRowBytes, thirdRowBytes;
071  final private byte[] col1, col2;
072
073  /** constructor */
074  public TestMinorCompaction() {
075    super();
076
077    // Set cache flush size to 1MB
078    conf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024*1024);
079    conf.setInt(HConstants.HREGION_MEMSTORE_BLOCK_MULTIPLIER, 100);
080    compactionThreshold = conf.getInt("hbase.hstore.compactionThreshold", 3);
081
082    firstRowBytes = START_KEY_BYTES;
083    secondRowBytes = START_KEY_BYTES.clone();
084    // Increment the least significant character so we get to next row.
085    secondRowBytes[START_KEY_BYTES.length - 1]++;
086    thirdRowBytes = START_KEY_BYTES.clone();
087    thirdRowBytes[START_KEY_BYTES.length - 1] =
088        (byte) (thirdRowBytes[START_KEY_BYTES.length - 1] + 2);
089    col1 = Bytes.toBytes("column1");
090    col2 = Bytes.toBytes("column2");
091  }
092
093  @Before
094  public void setUp() throws Exception {
095    this.htd = UTIL.createTableDescriptor(TableName.valueOf(name.getMethodName()),
096      HColumnDescriptor.DEFAULT_MIN_VERSIONS, 3, HConstants.FOREVER,
097      HColumnDescriptor.DEFAULT_KEEP_DELETED);
098    this.r = UTIL.createLocalHRegion(htd, null, null);
099  }
100
101  @After
102  public void tearDown() throws Exception {
103    WAL wal = ((HRegion)r).getWAL();
104    ((HRegion)r).close();
105    wal.close();
106  }
107
108  @Test
109  public void testMinorCompactionWithDeleteRow() throws Exception {
110    Delete deleteRow = new Delete(secondRowBytes);
111    testMinorCompactionWithDelete(deleteRow);
112  }
113
114  @Test
115  public void testMinorCompactionWithDeleteColumn1() throws Exception {
116    Delete dc = new Delete(secondRowBytes);
117    /* delete all timestamps in the column */
118    dc.addColumns(fam2, col2);
119    testMinorCompactionWithDelete(dc);
120  }
121
122  @Test
123  public void testMinorCompactionWithDeleteColumn2() throws Exception {
124    Delete dc = new Delete(secondRowBytes);
125    dc.addColumn(fam2, col2);
126    /* compactionThreshold is 3. The table has 4 versions: 0, 1, 2, and 3.
127     * we only delete the latest version. One might expect to see only
128     * versions 1 and 2. HBase differs, and gives us 0, 1 and 2.
129     * This is okay as well. Since there was no compaction done before the
130     * delete, version 0 seems to stay on.
131     */
132    testMinorCompactionWithDelete(dc, 3);
133  }
134
135  @Test
136  public void testMinorCompactionWithDeleteColumnFamily() throws Exception {
137    Delete deleteCF = new Delete(secondRowBytes);
138    deleteCF.addFamily(fam2);
139    testMinorCompactionWithDelete(deleteCF);
140  }
141
142  @Test
143  public void testMinorCompactionWithDeleteVersion1() throws Exception {
144    Delete deleteVersion = new Delete(secondRowBytes);
145    deleteVersion.addColumns(fam2, col2, 2);
146    /* compactionThreshold is 3. The table has 4 versions: 0, 1, 2, and 3.
147     * We delete versions 0 ... 2. So, we still have one remaining.
148     */
149    testMinorCompactionWithDelete(deleteVersion, 1);
150  }
151
152  @Test
153  public void testMinorCompactionWithDeleteVersion2() throws Exception {
154    Delete deleteVersion = new Delete(secondRowBytes);
155    deleteVersion.addColumn(fam2, col2, 1);
156    /*
157     * the table has 4 versions: 0, 1, 2, and 3.
158     * We delete 1.
159     * Should have 3 remaining.
160     */
161    testMinorCompactionWithDelete(deleteVersion, 3);
162  }
163
164  /*
165   * A helper function to test the minor compaction algorithm. We check that
166   * the delete markers are left behind. Takes delete as an argument, which
167   * can be any delete (row, column, columnfamliy etc), that essentially
168   * deletes row2 and column2. row1 and column1 should be undeleted
169   */
170  private void testMinorCompactionWithDelete(Delete delete) throws Exception {
171    testMinorCompactionWithDelete(delete, 0);
172  }
173
174  private void testMinorCompactionWithDelete(Delete delete, int expectedResultsAfterDelete)
175      throws Exception {
176    Table loader = new RegionAsTable(r);
177    for (int i = 0; i < compactionThreshold + 1; i++) {
178      HTestConst.addContent(loader, Bytes.toString(fam1), Bytes.toString(col1), firstRowBytes,
179        thirdRowBytes, i);
180      HTestConst.addContent(loader, Bytes.toString(fam1), Bytes.toString(col2), firstRowBytes,
181        thirdRowBytes, i);
182      HTestConst.addContent(loader, Bytes.toString(fam2), Bytes.toString(col1), firstRowBytes,
183        thirdRowBytes, i);
184      HTestConst.addContent(loader, Bytes.toString(fam2), Bytes.toString(col2), firstRowBytes,
185        thirdRowBytes, i);
186      r.flush(true);
187    }
188
189    Result result = r.get(new Get(firstRowBytes).addColumn(fam1, col1).readVersions(100));
190    assertEquals(compactionThreshold, result.size());
191    result = r.get(new Get(secondRowBytes).addColumn(fam2, col2).readVersions(100));
192    assertEquals(compactionThreshold, result.size());
193
194    // Now add deletes to memstore and then flush it.  That will put us over
195    // the compaction threshold of 3 store files.  Compacting these store files
196    // should result in a compacted store file that has no references to the
197    // deleted row.
198    r.delete(delete);
199
200    // Make sure that we have only deleted family2 from secondRowBytes
201    result = r.get(new Get(secondRowBytes).addColumn(fam2, col2).readVersions(100));
202    assertEquals(expectedResultsAfterDelete, result.size());
203    // but we still have firstrow
204    result = r.get(new Get(firstRowBytes).addColumn(fam1, col1).readVersions(100));
205    assertEquals(compactionThreshold, result.size());
206
207    r.flush(true);
208    // should not change anything.
209    // Let us check again
210
211    // Make sure that we have only deleted family2 from secondRowBytes
212    result = r.get(new Get(secondRowBytes).addColumn(fam2, col2).readVersions(100));
213    assertEquals(expectedResultsAfterDelete, result.size());
214    // but we still have firstrow
215    result = r.get(new Get(firstRowBytes).addColumn(fam1, col1).readVersions(100));
216    assertEquals(compactionThreshold, result.size());
217
218    // do a compaction
219    HStore store2 = r.getStore(fam2);
220    int numFiles1 = store2.getStorefiles().size();
221    assertTrue("Was expecting to see 4 store files", numFiles1 > compactionThreshold); // > 3
222    ((HStore)store2).compactRecentForTestingAssumingDefaultPolicy(compactionThreshold);   // = 3
223    int numFiles2 = store2.getStorefiles().size();
224    // Check that we did compact
225    assertTrue("Number of store files should go down", numFiles1 > numFiles2);
226    // Check that it was a minor compaction.
227    assertTrue("Was not supposed to be a major compaction", numFiles2 > 1);
228
229    // Make sure that we have only deleted family2 from secondRowBytes
230    result = r.get(new Get(secondRowBytes).addColumn(fam2, col2).readVersions(100));
231    assertEquals(expectedResultsAfterDelete, result.size());
232    // but we still have firstrow
233    result = r.get(new Get(firstRowBytes).addColumn(fam1, col1).readVersions(100));
234    assertEquals(compactionThreshold, result.size());
235  }
236}