001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.regionserver; 019 020import static org.apache.hadoop.hbase.HBaseTestingUtility.START_KEY_BYTES; 021import static org.apache.hadoop.hbase.HBaseTestingUtility.fam1; 022import static org.apache.hadoop.hbase.HBaseTestingUtility.fam2; 023import static org.junit.Assert.assertEquals; 024import static org.junit.Assert.assertTrue; 025 026import org.apache.hadoop.conf.Configuration; 027import org.apache.hadoop.hbase.HBaseClassTestRule; 028import org.apache.hadoop.hbase.HBaseTestCase; 029import org.apache.hadoop.hbase.HBaseTestingUtility; 030import org.apache.hadoop.hbase.HConstants; 031import org.apache.hadoop.hbase.HTableDescriptor; 032import org.apache.hadoop.hbase.client.Delete; 033import org.apache.hadoop.hbase.client.Get; 034import org.apache.hadoop.hbase.client.Result; 035import org.apache.hadoop.hbase.client.Table; 036import org.apache.hadoop.hbase.testclassification.MediumTests; 037import org.apache.hadoop.hbase.testclassification.RegionServerTests; 038import org.apache.hadoop.hbase.util.Bytes; 039import org.apache.hadoop.hbase.wal.WAL; 040import org.junit.After; 041import org.junit.Before; 042import org.junit.ClassRule; 043import org.junit.Rule; 044import org.junit.Test; 045import org.junit.experimental.categories.Category; 046import org.junit.rules.TestName; 047import org.slf4j.Logger; 048import org.slf4j.LoggerFactory; 049 050/** 051 * Test minor compactions 052 */ 053@Category({RegionServerTests.class, MediumTests.class}) 054public class TestMinorCompaction { 055 056 @ClassRule 057 public static final HBaseClassTestRule CLASS_RULE = 058 HBaseClassTestRule.forClass(TestMinorCompaction.class); 059 060 @Rule public TestName name = new TestName(); 061 private static final Logger LOG = LoggerFactory.getLogger(TestMinorCompaction.class.getName()); 062 private static final HBaseTestingUtility UTIL = HBaseTestingUtility.createLocalHTU(); 063 protected Configuration conf = UTIL.getConfiguration(); 064 065 private HRegion r = null; 066 private HTableDescriptor htd = null; 067 private int compactionThreshold; 068 private byte[] firstRowBytes, secondRowBytes, thirdRowBytes; 069 final private byte[] col1, col2; 070 071 /** constructor */ 072 public TestMinorCompaction() { 073 super(); 074 075 // Set cache flush size to 1MB 076 conf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024*1024); 077 conf.setInt(HConstants.HREGION_MEMSTORE_BLOCK_MULTIPLIER, 100); 078 compactionThreshold = conf.getInt("hbase.hstore.compactionThreshold", 3); 079 080 firstRowBytes = START_KEY_BYTES; 081 secondRowBytes = START_KEY_BYTES.clone(); 082 // Increment the least significant character so we get to next row. 083 secondRowBytes[START_KEY_BYTES.length - 1]++; 084 thirdRowBytes = START_KEY_BYTES.clone(); 085 thirdRowBytes[START_KEY_BYTES.length - 1] = 086 (byte) (thirdRowBytes[START_KEY_BYTES.length - 1] + 2); 087 col1 = Bytes.toBytes("column1"); 088 col2 = Bytes.toBytes("column2"); 089 } 090 091 @Before 092 public void setUp() throws Exception { 093 this.htd = UTIL.createTableDescriptor(name.getMethodName()); 094 this.r = UTIL.createLocalHRegion(htd, null, null); 095 } 096 097 @After 098 public void tearDown() throws Exception { 099 WAL wal = ((HRegion)r).getWAL(); 100 ((HRegion)r).close(); 101 wal.close(); 102 } 103 104 @Test 105 public void testMinorCompactionWithDeleteRow() throws Exception { 106 Delete deleteRow = new Delete(secondRowBytes); 107 testMinorCompactionWithDelete(deleteRow); 108 } 109 110 @Test 111 public void testMinorCompactionWithDeleteColumn1() throws Exception { 112 Delete dc = new Delete(secondRowBytes); 113 /* delete all timestamps in the column */ 114 dc.addColumns(fam2, col2); 115 testMinorCompactionWithDelete(dc); 116 } 117 118 @Test 119 public void testMinorCompactionWithDeleteColumn2() throws Exception { 120 Delete dc = new Delete(secondRowBytes); 121 dc.addColumn(fam2, col2); 122 /* compactionThreshold is 3. The table has 4 versions: 0, 1, 2, and 3. 123 * we only delete the latest version. One might expect to see only 124 * versions 1 and 2. HBase differs, and gives us 0, 1 and 2. 125 * This is okay as well. Since there was no compaction done before the 126 * delete, version 0 seems to stay on. 127 */ 128 testMinorCompactionWithDelete(dc, 3); 129 } 130 131 @Test 132 public void testMinorCompactionWithDeleteColumnFamily() throws Exception { 133 Delete deleteCF = new Delete(secondRowBytes); 134 deleteCF.addFamily(fam2); 135 testMinorCompactionWithDelete(deleteCF); 136 } 137 138 @Test 139 public void testMinorCompactionWithDeleteVersion1() throws Exception { 140 Delete deleteVersion = new Delete(secondRowBytes); 141 deleteVersion.addColumns(fam2, col2, 2); 142 /* compactionThreshold is 3. The table has 4 versions: 0, 1, 2, and 3. 143 * We delete versions 0 ... 2. So, we still have one remaining. 144 */ 145 testMinorCompactionWithDelete(deleteVersion, 1); 146 } 147 148 @Test 149 public void testMinorCompactionWithDeleteVersion2() throws Exception { 150 Delete deleteVersion = new Delete(secondRowBytes); 151 deleteVersion.addColumn(fam2, col2, 1); 152 /* 153 * the table has 4 versions: 0, 1, 2, and 3. 154 * We delete 1. 155 * Should have 3 remaining. 156 */ 157 testMinorCompactionWithDelete(deleteVersion, 3); 158 } 159 160 /* 161 * A helper function to test the minor compaction algorithm. We check that 162 * the delete markers are left behind. Takes delete as an argument, which 163 * can be any delete (row, column, columnfamliy etc), that essentially 164 * deletes row2 and column2. row1 and column1 should be undeleted 165 */ 166 private void testMinorCompactionWithDelete(Delete delete) throws Exception { 167 testMinorCompactionWithDelete(delete, 0); 168 } 169 170 private void testMinorCompactionWithDelete(Delete delete, int expectedResultsAfterDelete) throws Exception { 171 Table loader = new RegionAsTable(r); 172 for (int i = 0; i < compactionThreshold + 1; i++) { 173 HBaseTestCase.addContent(loader, Bytes.toString(fam1), Bytes.toString(col1), firstRowBytes, 174 thirdRowBytes, i); 175 HBaseTestCase.addContent(loader, Bytes.toString(fam1), Bytes.toString(col2), firstRowBytes, 176 thirdRowBytes, i); 177 HBaseTestCase.addContent(loader, Bytes.toString(fam2), Bytes.toString(col1), firstRowBytes, 178 thirdRowBytes, i); 179 HBaseTestCase.addContent(loader, Bytes.toString(fam2), Bytes.toString(col2), firstRowBytes, 180 thirdRowBytes, i); 181 r.flush(true); 182 } 183 184 Result result = r.get(new Get(firstRowBytes).addColumn(fam1, col1).setMaxVersions(100)); 185 assertEquals(compactionThreshold, result.size()); 186 result = r.get(new Get(secondRowBytes).addColumn(fam2, col2).setMaxVersions(100)); 187 assertEquals(compactionThreshold, result.size()); 188 189 // Now add deletes to memstore and then flush it. That will put us over 190 // the compaction threshold of 3 store files. Compacting these store files 191 // should result in a compacted store file that has no references to the 192 // deleted row. 193 r.delete(delete); 194 195 // Make sure that we have only deleted family2 from secondRowBytes 196 result = r.get(new Get(secondRowBytes).addColumn(fam2, col2).setMaxVersions(100)); 197 assertEquals(expectedResultsAfterDelete, result.size()); 198 // but we still have firstrow 199 result = r.get(new Get(firstRowBytes).addColumn(fam1, col1).setMaxVersions(100)); 200 assertEquals(compactionThreshold, result.size()); 201 202 r.flush(true); 203 // should not change anything. 204 // Let us check again 205 206 // Make sure that we have only deleted family2 from secondRowBytes 207 result = r.get(new Get(secondRowBytes).addColumn(fam2, col2).setMaxVersions(100)); 208 assertEquals(expectedResultsAfterDelete, result.size()); 209 // but we still have firstrow 210 result = r.get(new Get(firstRowBytes).addColumn(fam1, col1).setMaxVersions(100)); 211 assertEquals(compactionThreshold, result.size()); 212 213 // do a compaction 214 HStore store2 = r.getStore(fam2); 215 int numFiles1 = store2.getStorefiles().size(); 216 assertTrue("Was expecting to see 4 store files", numFiles1 > compactionThreshold); // > 3 217 ((HStore)store2).compactRecentForTestingAssumingDefaultPolicy(compactionThreshold); // = 3 218 int numFiles2 = store2.getStorefiles().size(); 219 // Check that we did compact 220 assertTrue("Number of store files should go down", numFiles1 > numFiles2); 221 // Check that it was a minor compaction. 222 assertTrue("Was not supposed to be a major compaction", numFiles2 > 1); 223 224 // Make sure that we have only deleted family2 from secondRowBytes 225 result = r.get(new Get(secondRowBytes).addColumn(fam2, col2).setMaxVersions(100)); 226 assertEquals(expectedResultsAfterDelete, result.size()); 227 // but we still have firstrow 228 result = r.get(new Get(firstRowBytes).addColumn(fam1, col1).setMaxVersions(100)); 229 assertEquals(compactionThreshold, result.size()); 230 } 231}