001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.apache.hadoop.hbase.HBaseTestingUtil.START_KEY;
021import static org.apache.hadoop.hbase.HBaseTestingUtil.START_KEY_BYTES;
022import static org.apache.hadoop.hbase.HBaseTestingUtil.fam1;
023import static org.junit.jupiter.api.Assertions.assertEquals;
024import static org.junit.jupiter.api.Assertions.assertTrue;
025
026import java.io.IOException;
027import java.util.stream.Stream;
028import org.apache.hadoop.conf.Configuration;
029import org.apache.hadoop.hbase.Cell;
030import org.apache.hadoop.hbase.CellUtil;
031import org.apache.hadoop.hbase.HBaseTestingUtil;
032import org.apache.hadoop.hbase.HConstants;
033import org.apache.hadoop.hbase.HTestConst;
034import org.apache.hadoop.hbase.KeyValue;
035import org.apache.hadoop.hbase.TableName;
036import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
037import org.apache.hadoop.hbase.client.Delete;
038import org.apache.hadoop.hbase.client.Get;
039import org.apache.hadoop.hbase.client.Result;
040import org.apache.hadoop.hbase.client.Table;
041import org.apache.hadoop.hbase.client.TableDescriptor;
042import org.apache.hadoop.hbase.util.Bytes;
043import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
044import org.apache.hadoop.hbase.wal.WAL;
045import org.junit.jupiter.api.AfterEach;
046import org.junit.jupiter.api.BeforeEach;
047import org.junit.jupiter.api.TestInfo;
048import org.junit.jupiter.params.provider.Arguments;
049import org.slf4j.Logger;
050import org.slf4j.LoggerFactory;
051
052/**
053 * Base class for testing major compactions
054 */
055public abstract class MajorCompactionTestBase {
056
057  public static Stream<Arguments> parameters() {
058    return Stream.of("NONE", "BASIC", "EAGER").map(Arguments::of);
059  }
060
061  private static final Logger LOG =
062    LoggerFactory.getLogger(MajorCompactionTestBase.class.getName());
063  protected static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
064  protected Configuration conf = UTIL.getConfiguration();
065
066  protected String name;
067  protected final String compType;
068
069  protected HRegion r = null;
070  protected TableDescriptor htd = null;
071  protected static final byte[] COLUMN_FAMILY = fam1;
072  protected final byte[] STARTROW = Bytes.toBytes(START_KEY);
073  protected static final byte[] COLUMN_FAMILY_TEXT = COLUMN_FAMILY;
074  protected int compactionThreshold;
075  protected byte[] secondRowBytes, thirdRowBytes;
076  protected static final long MAX_FILES_TO_COMPACT = 10;
077
078  /** constructor */
079  protected MajorCompactionTestBase(String compType) {
080    this.compType = compType;
081  }
082
083  @BeforeEach
084  public void setUp(TestInfo testInfo) throws Exception {
085    this.name = testInfo.getTestMethod().get().getName();
086    // Set cache flush size to 1MB
087    conf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024 * 1024);
088    conf.setInt(HConstants.HREGION_MEMSTORE_BLOCK_MULTIPLIER, 100);
089    compactionThreshold = conf.getInt("hbase.hstore.compactionThreshold", 3);
090    conf.set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY, String.valueOf(compType));
091
092    secondRowBytes = START_KEY_BYTES.clone();
093    // Increment the least significant character so we get to next row.
094    secondRowBytes[START_KEY_BYTES.length - 1]++;
095    thirdRowBytes = START_KEY_BYTES.clone();
096    thirdRowBytes[START_KEY_BYTES.length - 1] =
097      (byte) (thirdRowBytes[START_KEY_BYTES.length - 1] + 2);
098    this.htd = UTIL.createTableDescriptor(
099      TableName.valueOf((name + "-" + compType).replace('[', 'i').replace(']', 'i')),
100      ColumnFamilyDescriptorBuilder.DEFAULT_MIN_VERSIONS, 3, HConstants.FOREVER,
101      ColumnFamilyDescriptorBuilder.DEFAULT_KEEP_DELETED);
102    this.r = UTIL.createLocalHRegion(htd, null, null);
103  }
104
105  @AfterEach
106  public void tearDown() throws Exception {
107    WAL wal = ((HRegion) r).getWAL();
108    ((HRegion) r).close();
109    wal.close();
110  }
111
112  protected final void majorCompaction() throws Exception {
113    createStoreFile(r);
114    for (int i = 0; i < compactionThreshold; i++) {
115      createStoreFile(r);
116    }
117    // Add more content.
118    HTestConst.addContent(new RegionAsTable(r), Bytes.toString(COLUMN_FAMILY));
119
120    // Now there are about 5 versions of each column.
121    // Default is that there only 3 (MAXVERSIONS) versions allowed per column.
122    //
123    // Assert == 3 when we ask for versions.
124    Result result = r.get(new Get(STARTROW).addFamily(COLUMN_FAMILY_TEXT).readVersions(100));
125    assertEquals(compactionThreshold, result.size());
126
127    r.flush(true);
128    r.compact(true);
129
130    // look at the second row
131    // Increment the least significant character so we get to next row.
132    byte[] secondRowBytes = START_KEY_BYTES.clone();
133    secondRowBytes[START_KEY_BYTES.length - 1]++;
134
135    // Always 3 versions if that is what max versions is.
136    result = r.get(new Get(secondRowBytes).addFamily(COLUMN_FAMILY_TEXT).readVersions(100));
137    LOG.debug(
138      "Row " + Bytes.toStringBinary(secondRowBytes) + " after " + "initial compaction: " + result);
139    assertEquals(compactionThreshold, result.size(),
140      "Invalid number of versions of row " + Bytes.toStringBinary(secondRowBytes) + ".");
141
142    // Now add deletes to memstore and then flush it.
143    // That will put us over
144    // the compaction threshold of 3 store files. Compacting these store files
145    // should result in a compacted store file that has no references to the
146    // deleted row.
147    LOG.debug("Adding deletes to memstore and flushing");
148    Delete delete = new Delete(secondRowBytes, EnvironmentEdgeManager.currentTime());
149    byte[][] famAndQf = { COLUMN_FAMILY, null };
150    delete.addFamily(famAndQf[0]);
151    r.delete(delete);
152
153    // Assert deleted.
154    result = r.get(new Get(secondRowBytes).addFamily(COLUMN_FAMILY_TEXT).readVersions(100));
155    assertTrue(result.isEmpty(), "Second row should have been deleted");
156
157    r.flush(true);
158
159    result = r.get(new Get(secondRowBytes).addFamily(COLUMN_FAMILY_TEXT).readVersions(100));
160    assertTrue(result.isEmpty(), "Second row should have been deleted");
161
162    // Add a bit of data and flush. Start adding at 'bbb'.
163    createSmallerStoreFile(this.r);
164    r.flush(true);
165    // Assert that the second row is still deleted.
166    result = r.get(new Get(secondRowBytes).addFamily(COLUMN_FAMILY_TEXT).readVersions(100));
167    assertTrue(result.isEmpty(), "Second row should still be deleted");
168
169    // Force major compaction.
170    r.compact(true);
171    assertEquals(1, r.getStore(COLUMN_FAMILY_TEXT).getStorefiles().size());
172
173    result = r.get(new Get(secondRowBytes).addFamily(COLUMN_FAMILY_TEXT).readVersions(100));
174    assertTrue(result.isEmpty(), "Second row should still be deleted");
175
176    // Make sure the store files do have some 'aaa' keys in them -- exactly 3.
177    // Also, that compacted store files do not have any secondRowBytes because
178    // they were deleted.
179    verifyCounts(3, 0);
180
181    // Multiple versions allowed for an entry, so the delete isn't enough
182    // Lower TTL and expire to ensure that all our entries have been wiped
183    final int ttl = 1000;
184    for (HStore store : r.getStores()) {
185      ScanInfo old = store.getScanInfo();
186      ScanInfo si = old.customize(old.getMaxVersions(), ttl, old.getKeepDeletedCells());
187      store.setScanInfo(si);
188    }
189    Thread.sleep(1000);
190
191    r.compact(true);
192    int count = count();
193    assertEquals(0, count, "Should not see anything after TTL has expired");
194  }
195
196  private void verifyCounts(int countRow1, int countRow2) throws Exception {
197    int count1 = 0;
198    int count2 = 0;
199    for (HStoreFile f : r.getStore(COLUMN_FAMILY_TEXT).getStorefiles()) {
200      try (StoreFileScanner scanner = f.getPreadScanner(false, Long.MAX_VALUE, 0, false)) {
201        scanner.seek(KeyValue.LOWESTKEY);
202        for (Cell cell;;) {
203          cell = scanner.next();
204          if (cell == null) {
205            break;
206          }
207          byte[] row = CellUtil.cloneRow(cell);
208          if (Bytes.equals(row, STARTROW)) {
209            count1++;
210          } else if (Bytes.equals(row, secondRowBytes)) {
211            count2++;
212          }
213        }
214      }
215    }
216    assertEquals(countRow1, count1);
217    assertEquals(countRow2, count2);
218  }
219
220  private int count() throws IOException {
221    int count = 0;
222    for (HStoreFile f : r.getStore(COLUMN_FAMILY_TEXT).getStorefiles()) {
223      try (StoreFileScanner scanner = f.getPreadScanner(false, Long.MAX_VALUE, 0, false)) {
224        scanner.seek(KeyValue.LOWESTKEY);
225        while (scanner.next() != null) {
226          count++;
227        }
228      }
229    }
230    return count;
231  }
232
233  protected final void createStoreFile(final HRegion region) throws IOException {
234    createStoreFile(region, Bytes.toString(COLUMN_FAMILY));
235  }
236
237  protected final void createStoreFile(final HRegion region, String family) throws IOException {
238    Table loader = new RegionAsTable(region);
239    HTestConst.addContent(loader, family);
240    region.flush(true);
241  }
242
243  protected final void createSmallerStoreFile(final HRegion region) throws IOException {
244    Table loader = new RegionAsTable(region);
245    HTestConst.addContent(loader, Bytes.toString(COLUMN_FAMILY), Bytes.toBytes("" + "bbb"), null);
246    region.flush(true);
247  }
248}