001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.apache.hadoop.hbase.HBaseTestingUtility.START_KEY_BYTES;
021import static org.apache.hadoop.hbase.HBaseTestingUtility.fam1;
022import static org.apache.hadoop.hbase.HBaseTestingUtility.fam2;
023import static org.junit.Assert.assertEquals;
024import static org.junit.Assert.assertTrue;
025
026import org.apache.hadoop.conf.Configuration;
027import org.apache.hadoop.hbase.HBaseClassTestRule;
028import org.apache.hadoop.hbase.HBaseTestCase;
029import org.apache.hadoop.hbase.HBaseTestingUtility;
030import org.apache.hadoop.hbase.HConstants;
031import org.apache.hadoop.hbase.HTableDescriptor;
032import org.apache.hadoop.hbase.client.Delete;
033import org.apache.hadoop.hbase.client.Get;
034import org.apache.hadoop.hbase.client.Result;
035import org.apache.hadoop.hbase.client.Table;
036import org.apache.hadoop.hbase.testclassification.MediumTests;
037import org.apache.hadoop.hbase.testclassification.RegionServerTests;
038import org.apache.hadoop.hbase.util.Bytes;
039import org.apache.hadoop.hbase.wal.WAL;
040import org.junit.After;
041import org.junit.Before;
042import org.junit.ClassRule;
043import org.junit.Rule;
044import org.junit.Test;
045import org.junit.experimental.categories.Category;
046import org.junit.rules.TestName;
047import org.slf4j.Logger;
048import org.slf4j.LoggerFactory;
049
050/**
051 * Test minor compactions
052 */
053@Category({RegionServerTests.class, MediumTests.class})
054public class TestMinorCompaction {
055
056  @ClassRule
057  public static final HBaseClassTestRule CLASS_RULE =
058      HBaseClassTestRule.forClass(TestMinorCompaction.class);
059
060  @Rule public TestName name = new TestName();
061  private static final Logger LOG = LoggerFactory.getLogger(TestMinorCompaction.class.getName());
062  private static final HBaseTestingUtility UTIL = HBaseTestingUtility.createLocalHTU();
063  protected Configuration conf = UTIL.getConfiguration();
064
065  private HRegion r = null;
066  private HTableDescriptor htd = null;
067  private int compactionThreshold;
068  private byte[] firstRowBytes, secondRowBytes, thirdRowBytes;
069  final private byte[] col1, col2;
070
071  /** constructor */
072  public TestMinorCompaction() {
073    super();
074
075    // Set cache flush size to 1MB
076    conf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024*1024);
077    conf.setInt(HConstants.HREGION_MEMSTORE_BLOCK_MULTIPLIER, 100);
078    compactionThreshold = conf.getInt("hbase.hstore.compactionThreshold", 3);
079
080    firstRowBytes = START_KEY_BYTES;
081    secondRowBytes = START_KEY_BYTES.clone();
082    // Increment the least significant character so we get to next row.
083    secondRowBytes[START_KEY_BYTES.length - 1]++;
084    thirdRowBytes = START_KEY_BYTES.clone();
085    thirdRowBytes[START_KEY_BYTES.length - 1] =
086        (byte) (thirdRowBytes[START_KEY_BYTES.length - 1] + 2);
087    col1 = Bytes.toBytes("column1");
088    col2 = Bytes.toBytes("column2");
089  }
090
091  @Before
092  public void setUp() throws Exception {
093    this.htd = UTIL.createTableDescriptor(name.getMethodName());
094    this.r = UTIL.createLocalHRegion(htd, null, null);
095  }
096
097  @After
098  public void tearDown() throws Exception {
099    WAL wal = ((HRegion)r).getWAL();
100    ((HRegion)r).close();
101    wal.close();
102  }
103
104  @Test
105  public void testMinorCompactionWithDeleteRow() throws Exception {
106    Delete deleteRow = new Delete(secondRowBytes);
107    testMinorCompactionWithDelete(deleteRow);
108  }
109
110  @Test
111  public void testMinorCompactionWithDeleteColumn1() throws Exception {
112    Delete dc = new Delete(secondRowBytes);
113    /* delete all timestamps in the column */
114    dc.addColumns(fam2, col2);
115    testMinorCompactionWithDelete(dc);
116  }
117
118  @Test
119  public void testMinorCompactionWithDeleteColumn2() throws Exception {
120    Delete dc = new Delete(secondRowBytes);
121    dc.addColumn(fam2, col2);
122    /* compactionThreshold is 3. The table has 4 versions: 0, 1, 2, and 3.
123     * we only delete the latest version. One might expect to see only
124     * versions 1 and 2. HBase differs, and gives us 0, 1 and 2.
125     * This is okay as well. Since there was no compaction done before the
126     * delete, version 0 seems to stay on.
127     */
128    testMinorCompactionWithDelete(dc, 3);
129  }
130
131  @Test
132  public void testMinorCompactionWithDeleteColumnFamily() throws Exception {
133    Delete deleteCF = new Delete(secondRowBytes);
134    deleteCF.addFamily(fam2);
135    testMinorCompactionWithDelete(deleteCF);
136  }
137
138  @Test
139  public void testMinorCompactionWithDeleteVersion1() throws Exception {
140    Delete deleteVersion = new Delete(secondRowBytes);
141    deleteVersion.addColumns(fam2, col2, 2);
142    /* compactionThreshold is 3. The table has 4 versions: 0, 1, 2, and 3.
143     * We delete versions 0 ... 2. So, we still have one remaining.
144     */
145    testMinorCompactionWithDelete(deleteVersion, 1);
146  }
147
148  @Test
149  public void testMinorCompactionWithDeleteVersion2() throws Exception {
150    Delete deleteVersion = new Delete(secondRowBytes);
151    deleteVersion.addColumn(fam2, col2, 1);
152    /*
153     * the table has 4 versions: 0, 1, 2, and 3.
154     * We delete 1.
155     * Should have 3 remaining.
156     */
157    testMinorCompactionWithDelete(deleteVersion, 3);
158  }
159
160  /*
161   * A helper function to test the minor compaction algorithm. We check that
162   * the delete markers are left behind. Takes delete as an argument, which
163   * can be any delete (row, column, columnfamliy etc), that essentially
164   * deletes row2 and column2. row1 and column1 should be undeleted
165   */
166  private void testMinorCompactionWithDelete(Delete delete) throws Exception {
167    testMinorCompactionWithDelete(delete, 0);
168  }
169
170  private void testMinorCompactionWithDelete(Delete delete, int expectedResultsAfterDelete) throws Exception {
171    Table loader = new RegionAsTable(r);
172    for (int i = 0; i < compactionThreshold + 1; i++) {
173      HBaseTestCase.addContent(loader, Bytes.toString(fam1), Bytes.toString(col1), firstRowBytes,
174        thirdRowBytes, i);
175      HBaseTestCase.addContent(loader, Bytes.toString(fam1), Bytes.toString(col2), firstRowBytes,
176        thirdRowBytes, i);
177      HBaseTestCase.addContent(loader, Bytes.toString(fam2), Bytes.toString(col1), firstRowBytes,
178        thirdRowBytes, i);
179      HBaseTestCase.addContent(loader, Bytes.toString(fam2), Bytes.toString(col2), firstRowBytes,
180        thirdRowBytes, i);
181      r.flush(true);
182    }
183
184    Result result = r.get(new Get(firstRowBytes).addColumn(fam1, col1).setMaxVersions(100));
185    assertEquals(compactionThreshold, result.size());
186    result = r.get(new Get(secondRowBytes).addColumn(fam2, col2).setMaxVersions(100));
187    assertEquals(compactionThreshold, result.size());
188
189    // Now add deletes to memstore and then flush it.  That will put us over
190    // the compaction threshold of 3 store files.  Compacting these store files
191    // should result in a compacted store file that has no references to the
192    // deleted row.
193    r.delete(delete);
194
195    // Make sure that we have only deleted family2 from secondRowBytes
196    result = r.get(new Get(secondRowBytes).addColumn(fam2, col2).setMaxVersions(100));
197    assertEquals(expectedResultsAfterDelete, result.size());
198    // but we still have firstrow
199    result = r.get(new Get(firstRowBytes).addColumn(fam1, col1).setMaxVersions(100));
200    assertEquals(compactionThreshold, result.size());
201
202    r.flush(true);
203    // should not change anything.
204    // Let us check again
205
206    // Make sure that we have only deleted family2 from secondRowBytes
207    result = r.get(new Get(secondRowBytes).addColumn(fam2, col2).setMaxVersions(100));
208    assertEquals(expectedResultsAfterDelete, result.size());
209    // but we still have firstrow
210    result = r.get(new Get(firstRowBytes).addColumn(fam1, col1).setMaxVersions(100));
211    assertEquals(compactionThreshold, result.size());
212
213    // do a compaction
214    HStore store2 = r.getStore(fam2);
215    int numFiles1 = store2.getStorefiles().size();
216    assertTrue("Was expecting to see 4 store files", numFiles1 > compactionThreshold); // > 3
217    ((HStore)store2).compactRecentForTestingAssumingDefaultPolicy(compactionThreshold);   // = 3
218    int numFiles2 = store2.getStorefiles().size();
219    // Check that we did compact
220    assertTrue("Number of store files should go down", numFiles1 > numFiles2);
221    // Check that it was a minor compaction.
222    assertTrue("Was not supposed to be a major compaction", numFiles2 > 1);
223
224    // Make sure that we have only deleted family2 from secondRowBytes
225    result = r.get(new Get(secondRowBytes).addColumn(fam2, col2).setMaxVersions(100));
226    assertEquals(expectedResultsAfterDelete, result.size());
227    // but we still have firstrow
228    result = r.get(new Get(firstRowBytes).addColumn(fam1, col1).setMaxVersions(100));
229    assertEquals(compactionThreshold, result.size());
230  }
231}