001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.apache.hadoop.hbase.HBaseTestingUtility.START_KEY_BYTES;
021import static org.apache.hadoop.hbase.HBaseTestingUtility.fam1;
022import static org.apache.hadoop.hbase.HBaseTestingUtility.fam2;
023import static org.junit.Assert.assertEquals;
024import static org.junit.Assert.assertTrue;
025import org.apache.hadoop.conf.Configuration;
026import org.apache.hadoop.hbase.HBaseClassTestRule;
027import org.apache.hadoop.hbase.HBaseTestCase;
028import org.apache.hadoop.hbase.HBaseTestingUtility;
029import org.apache.hadoop.hbase.HConstants;
030import org.apache.hadoop.hbase.HTableDescriptor;
031import org.apache.hadoop.hbase.client.Delete;
032import org.apache.hadoop.hbase.client.Get;
033import org.apache.hadoop.hbase.client.Result;
034import org.apache.hadoop.hbase.client.Table;
035import org.apache.hadoop.hbase.testclassification.RegionServerTests;
036import org.apache.hadoop.hbase.testclassification.SmallTests;
037import org.apache.hadoop.hbase.util.Bytes;
038import org.apache.hadoop.hbase.wal.WAL;
039import org.junit.After;
040import org.junit.Before;
041import org.junit.ClassRule;
042import org.junit.Rule;
043import org.junit.Test;
044import org.junit.experimental.categories.Category;
045import org.junit.rules.TestName;
046import org.slf4j.Logger;
047import org.slf4j.LoggerFactory;
048
049/**
050 * Test minor compactions
051 */
052@Category({RegionServerTests.class, SmallTests.class})
053public class TestMinorCompaction {
054
055  @ClassRule
056  public static final HBaseClassTestRule CLASS_RULE =
057      HBaseClassTestRule.forClass(TestMinorCompaction.class);
058
059  @Rule public TestName name = new TestName();
060  private static final Logger LOG = LoggerFactory.getLogger(TestMinorCompaction.class.getName());
061  private static final HBaseTestingUtility UTIL = HBaseTestingUtility.createLocalHTU();
062  protected Configuration conf = UTIL.getConfiguration();
063
064  private HRegion r = null;
065  private HTableDescriptor htd = null;
066  private int compactionThreshold;
067  private byte[] firstRowBytes, secondRowBytes, thirdRowBytes;
068  final private byte[] col1, col2;
069
070  /** constructor */
071  public TestMinorCompaction() {
072    super();
073
074    // Set cache flush size to 1MB
075    conf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024*1024);
076    conf.setInt(HConstants.HREGION_MEMSTORE_BLOCK_MULTIPLIER, 100);
077    compactionThreshold = conf.getInt("hbase.hstore.compactionThreshold", 3);
078
079    firstRowBytes = START_KEY_BYTES;
080    secondRowBytes = START_KEY_BYTES.clone();
081    // Increment the least significant character so we get to next row.
082    secondRowBytes[START_KEY_BYTES.length - 1]++;
083    thirdRowBytes = START_KEY_BYTES.clone();
084    thirdRowBytes[START_KEY_BYTES.length - 1] =
085        (byte) (thirdRowBytes[START_KEY_BYTES.length - 1] + 2);
086    col1 = Bytes.toBytes("column1");
087    col2 = Bytes.toBytes("column2");
088  }
089
090  @Before
091  public void setUp() throws Exception {
092    this.htd = UTIL.createTableDescriptor(name.getMethodName());
093    this.r = UTIL.createLocalHRegion(htd, null, null);
094  }
095
096  @After
097  public void tearDown() throws Exception {
098    WAL wal = ((HRegion)r).getWAL();
099    ((HRegion)r).close();
100    wal.close();
101  }
102
103  @Test
104  public void testMinorCompactionWithDeleteRow() throws Exception {
105    Delete deleteRow = new Delete(secondRowBytes);
106    testMinorCompactionWithDelete(deleteRow);
107  }
108
109  @Test
110  public void testMinorCompactionWithDeleteColumn1() throws Exception {
111    Delete dc = new Delete(secondRowBytes);
112    /* delete all timestamps in the column */
113    dc.addColumns(fam2, col2);
114    testMinorCompactionWithDelete(dc);
115  }
116
117  @Test
118  public void testMinorCompactionWithDeleteColumn2() throws Exception {
119    Delete dc = new Delete(secondRowBytes);
120    dc.addColumn(fam2, col2);
121    /* compactionThreshold is 3. The table has 4 versions: 0, 1, 2, and 3.
122     * we only delete the latest version. One might expect to see only
123     * versions 1 and 2. HBase differs, and gives us 0, 1 and 2.
124     * This is okay as well. Since there was no compaction done before the
125     * delete, version 0 seems to stay on.
126     */
127    testMinorCompactionWithDelete(dc, 3);
128  }
129
130  @Test
131  public void testMinorCompactionWithDeleteColumnFamily() throws Exception {
132    Delete deleteCF = new Delete(secondRowBytes);
133    deleteCF.addFamily(fam2);
134    testMinorCompactionWithDelete(deleteCF);
135  }
136
137  @Test
138  public void testMinorCompactionWithDeleteVersion1() throws Exception {
139    Delete deleteVersion = new Delete(secondRowBytes);
140    deleteVersion.addColumns(fam2, col2, 2);
141    /* compactionThreshold is 3. The table has 4 versions: 0, 1, 2, and 3.
142     * We delete versions 0 ... 2. So, we still have one remaining.
143     */
144    testMinorCompactionWithDelete(deleteVersion, 1);
145  }
146
147  @Test
148  public void testMinorCompactionWithDeleteVersion2() throws Exception {
149    Delete deleteVersion = new Delete(secondRowBytes);
150    deleteVersion.addColumn(fam2, col2, 1);
151    /*
152     * the table has 4 versions: 0, 1, 2, and 3.
153     * We delete 1.
154     * Should have 3 remaining.
155     */
156    testMinorCompactionWithDelete(deleteVersion, 3);
157  }
158
159  /*
160   * A helper function to test the minor compaction algorithm. We check that
161   * the delete markers are left behind. Takes delete as an argument, which
162   * can be any delete (row, column, columnfamliy etc), that essentially
163   * deletes row2 and column2. row1 and column1 should be undeleted
164   */
165  private void testMinorCompactionWithDelete(Delete delete) throws Exception {
166    testMinorCompactionWithDelete(delete, 0);
167  }
168
169  private void testMinorCompactionWithDelete(Delete delete, int expectedResultsAfterDelete)
170      throws Exception {
171    Table loader = new RegionAsTable(r);
172    for (int i = 0; i < compactionThreshold + 1; i++) {
173      HBaseTestCase.addContent(loader, Bytes.toString(fam1), Bytes.toString(col1), firstRowBytes,
174        thirdRowBytes, i);
175      HBaseTestCase.addContent(loader, Bytes.toString(fam1), Bytes.toString(col2), firstRowBytes,
176        thirdRowBytes, i);
177      HBaseTestCase.addContent(loader, Bytes.toString(fam2), Bytes.toString(col1), firstRowBytes,
178        thirdRowBytes, i);
179      HBaseTestCase.addContent(loader, Bytes.toString(fam2), Bytes.toString(col2), firstRowBytes,
180        thirdRowBytes, i);
181      r.flush(true);
182    }
183
184    Result result = r.get(new Get(firstRowBytes).addColumn(fam1, col1).setMaxVersions(100));
185    assertEquals(compactionThreshold, result.size());
186    result = r.get(new Get(secondRowBytes).addColumn(fam2, col2).setMaxVersions(100));
187    assertEquals(compactionThreshold, result.size());
188
189    // Now add deletes to memstore and then flush it.  That will put us over
190    // the compaction threshold of 3 store files.  Compacting these store files
191    // should result in a compacted store file that has no references to the
192    // deleted row.
193    r.delete(delete);
194
195    // Make sure that we have only deleted family2 from secondRowBytes
196    result = r.get(new Get(secondRowBytes).addColumn(fam2, col2).setMaxVersions(100));
197    assertEquals(expectedResultsAfterDelete, result.size());
198    // but we still have firstrow
199    result = r.get(new Get(firstRowBytes).addColumn(fam1, col1).setMaxVersions(100));
200    assertEquals(compactionThreshold, result.size());
201
202    r.flush(true);
203    // should not change anything.
204    // Let us check again
205
206    // Make sure that we have only deleted family2 from secondRowBytes
207    result = r.get(new Get(secondRowBytes).addColumn(fam2, col2).setMaxVersions(100));
208    assertEquals(expectedResultsAfterDelete, result.size());
209    // but we still have firstrow
210    result = r.get(new Get(firstRowBytes).addColumn(fam1, col1).setMaxVersions(100));
211    assertEquals(compactionThreshold, result.size());
212
213    // do a compaction
214    HStore store2 = r.getStore(fam2);
215    int numFiles1 = store2.getStorefiles().size();
216    assertTrue("Was expecting to see 4 store files", numFiles1 > compactionThreshold); // > 3
217    ((HStore)store2).compactRecentForTestingAssumingDefaultPolicy(compactionThreshold);   // = 3
218    int numFiles2 = store2.getStorefiles().size();
219    // Check that we did compact
220    assertTrue("Number of store files should go down", numFiles1 > numFiles2);
221    // Check that it was a minor compaction.
222    assertTrue("Was not supposed to be a major compaction", numFiles2 > 1);
223
224    // Make sure that we have only deleted family2 from secondRowBytes
225    result = r.get(new Get(secondRowBytes).addColumn(fam2, col2).setMaxVersions(100));
226    assertEquals(expectedResultsAfterDelete, result.size());
227    // but we still have firstrow
228    result = r.get(new Get(firstRowBytes).addColumn(fam1, col1).setMaxVersions(100));
229    assertEquals(compactionThreshold, result.size());
230  }
231}