Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver.querymatcher;
019
020import static org.apache.hadoop.hbase.regionserver.querymatcher.ScanQueryMatcher.MatchCode.INCLUDE;
021import static org.apache.hadoop.hbase.regionserver.querymatcher.ScanQueryMatcher.MatchCode.SEEK_NEXT_COL;
022import static org.apache.hadoop.hbase.regionserver.querymatcher.ScanQueryMatcher.MatchCode.SKIP;
023import static org.junit.Assert.assertEquals;
024
025import java.io.IOException;
026import java.util.ArrayList;
027import java.util.List;
028import org.apache.hadoop.hbase.HBaseClassTestRule;
029import org.apache.hadoop.hbase.HConstants;
030import org.apache.hadoop.hbase.KeepDeletedCells;
031import org.apache.hadoop.hbase.KeyValue;
032import org.apache.hadoop.hbase.KeyValue.Type;
033import org.apache.hadoop.hbase.KeyValueUtil;
034import org.apache.hadoop.hbase.PrivateConstants;
035import org.apache.hadoop.hbase.regionserver.ScanInfo;
036import org.apache.hadoop.hbase.regionserver.ScanType;
037import org.apache.hadoop.hbase.regionserver.querymatcher.ScanQueryMatcher.MatchCode;
038import org.apache.hadoop.hbase.testclassification.RegionServerTests;
039import org.apache.hadoop.hbase.testclassification.SmallTests;
040import org.apache.hadoop.hbase.util.Bytes;
041import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
042import org.junit.ClassRule;
043import org.junit.Test;
044import org.junit.experimental.categories.Category;
045import org.slf4j.Logger;
046import org.slf4j.LoggerFactory;
047
048@Category({ RegionServerTests.class, SmallTests.class })
049public class TestCompactionScanQueryMatcher extends AbstractTestScanQueryMatcher {
050
051  @ClassRule
052  public static final HBaseClassTestRule CLASS_RULE =
053    HBaseClassTestRule.forClass(TestCompactionScanQueryMatcher.class);
054
055  private static final Logger LOG = LoggerFactory.getLogger(TestCompactionScanQueryMatcher.class);
056
057  @Test
058  public void testMatch_PartialRangeDropDeletes() throws Exception {
059    // Some ranges.
060    testDropDeletes(row2, row3, new byte[][] { row1, row2, row2, row3 }, INCLUDE, SKIP, SKIP,
061      INCLUDE);
062    testDropDeletes(row2, row3, new byte[][] { row1, row1, row2 }, INCLUDE, INCLUDE, SKIP);
063    testDropDeletes(row2, row3, new byte[][] { row2, row3, row3 }, SKIP, INCLUDE, INCLUDE);
064    testDropDeletes(row1, row3, new byte[][] { row1, row2, row3 }, SKIP, SKIP, INCLUDE);
065    // Open ranges.
066    testDropDeletes(HConstants.EMPTY_START_ROW, row3, new byte[][] { row1, row2, row3 }, SKIP, SKIP,
067      INCLUDE);
068    testDropDeletes(row2, HConstants.EMPTY_END_ROW, new byte[][] { row1, row2, row3 }, INCLUDE,
069      SKIP, SKIP);
070    testDropDeletes(HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
071      new byte[][] { row1, row2, row3, row3 }, SKIP, SKIP, SKIP, SKIP);
072
073    // No KVs in range.
074    testDropDeletes(row2, row3, new byte[][] { row1, row1, row3 }, INCLUDE, INCLUDE, INCLUDE);
075    testDropDeletes(row2, row3, new byte[][] { row3, row3 }, INCLUDE, INCLUDE);
076    testDropDeletes(row2, row3, new byte[][] { row1, row1 }, INCLUDE, INCLUDE);
077  }
078
079  /**
080   * Test redundant delete marker handling with COMPACT_RETAIN_DELETES. Cells are auto-generated
081   * from the given types with decrementing timestamps.
082   */
083  @Test
084  public void testSkipsRedundantDeleteMarkers() throws IOException {
085    // Interleaved DeleteColumn + Put. First DC included, put triggers SEEK_NEXT_COL.
086    assertRetainDeletes(new Type[] { Type.DeleteColumn, Type.Put, Type.DeleteColumn }, INCLUDE,
087      SEEK_NEXT_COL);
088
089    // Contiguous DeleteColumn. First included, rest redundant.
090    assertRetainDeletes(new Type[] { Type.DeleteColumn, Type.DeleteColumn, Type.DeleteColumn },
091      INCLUDE, SEEK_NEXT_COL, SEEK_NEXT_COL);
092
093    // Contiguous DeleteFamily. First included, rest redundant.
094    assertRetainDeletes(new Type[] { Type.DeleteFamily, Type.DeleteFamily, Type.DeleteFamily },
095      INCLUDE, SEEK_NEXT_COL, SEEK_NEXT_COL);
096
097    // DF + DFV interleaved. DF included, DFV redundant (SKIP because empty qualifier),
098    // older DF redundant (SEEK_NEXT_COL), older DFV redundant (SKIP).
099    assertRetainDeletes(new Type[] { Type.DeleteFamily, Type.DeleteFamilyVersion, Type.DeleteFamily,
100      Type.DeleteFamilyVersion }, INCLUDE, SKIP, SEEK_NEXT_COL, SKIP);
101
102    // Delete (version) covered by DeleteColumn.
103    assertRetainDeletes(new Type[] { Type.DeleteColumn, Type.Delete, Type.Delete, Type.Delete },
104      INCLUDE, SEEK_NEXT_COL, SEEK_NEXT_COL, SEEK_NEXT_COL);
105
106    // KEEP_DELETED_CELLS=TRUE: all markers retained.
107    assertRetainDeletes(KeepDeletedCells.TRUE,
108      new Type[] { Type.DeleteColumn, Type.DeleteColumn, Type.DeleteColumn }, INCLUDE, INCLUDE,
109      INCLUDE);
110  }
111
112  /**
113   * Redundant column-level deletes with empty qualifier must not seek past a subsequent
114   * DeleteFamily. getKeyForNextColumn treats empty qualifier as "no column" and returns
115   * SEEK_NEXT_ROW, which would skip the DF and all remaining cells in the row.
116   */
117  @Test
118  public void testEmptyQualifierDeleteDoesNotSkipDeleteFamily() throws IOException {
119    byte[] emptyQualifier = HConstants.EMPTY_BYTE_ARRAY;
120
121    // DC(empty) + DC(empty) redundant + DF must still be reachable.
122    assertRetainDeletes(emptyQualifier,
123      new Type[] { Type.DeleteColumn, Type.DeleteColumn, Type.DeleteFamily }, INCLUDE, SKIP,
124      INCLUDE);
125
126    // DC(empty) + Delete(empty) redundant + DF must still be reachable.
127    assertRetainDeletes(emptyQualifier,
128      new Type[] { Type.DeleteColumn, Type.Delete, Type.DeleteFamily }, INCLUDE, SKIP, INCLUDE);
129  }
130
131  private void assertRetainDeletes(Type[] types, MatchCode... expected) throws IOException {
132    assertRetainDeletes(KeepDeletedCells.FALSE, types, expected);
133  }
134
135  private void assertRetainDeletes(byte[] qualifier, Type[] types, MatchCode... expected)
136    throws IOException {
137    assertRetainDeletes(KeepDeletedCells.FALSE, qualifier, types, expected);
138  }
139
140  /**
141   * Build cells from the given types with decrementing timestamps (same ts for adjacent
142   * family-level and column-level types at the same position). Family-level types (DeleteFamily,
143   * DeleteFamilyVersion) use empty qualifier; others use col1.
144   */
145  private void assertRetainDeletes(KeepDeletedCells keepDeletedCells, Type[] types,
146    MatchCode... expected) throws IOException {
147    assertRetainDeletes(keepDeletedCells, null, types, expected);
148  }
149
150  /**
151   * Build cells from the given types with decrementing timestamps. If qualifier is null,
152   * family-level types use empty qualifier and others use col1. If qualifier is specified, all
153   * types use that qualifier.
154   */
155  private void assertRetainDeletes(KeepDeletedCells keepDeletedCells, byte[] qualifier,
156    Type[] types, MatchCode... expected) throws IOException {
157    long now = EnvironmentEdgeManager.currentTime();
158    ScanInfo scanInfo = new ScanInfo(this.conf, fam1, 0, 1, ttl, keepDeletedCells,
159      HConstants.DEFAULT_BLOCKSIZE, 0, rowComparator, false);
160    CompactionScanQueryMatcher qm = CompactionScanQueryMatcher.create(scanInfo,
161      ScanType.COMPACT_RETAIN_DELETES, 0L, PrivateConstants.OLDEST_TIMESTAMP,
162      PrivateConstants.OLDEST_TIMESTAMP, now, null, null, null);
163    qm.setToNewRow(KeyValueUtil.createFirstOnRow(row1));
164
165    long ts = now;
166    List<MatchCode> actual = new ArrayList<>(expected.length);
167    for (int i = 0; i < types.length; i++) {
168      byte[] qual;
169      if (qualifier != null) {
170        qual = qualifier;
171      } else {
172        boolean familyLevel = types[i] == Type.DeleteFamily || types[i] == Type.DeleteFamilyVersion;
173        qual = familyLevel ? HConstants.EMPTY_BYTE_ARRAY : col1;
174      }
175      KeyValue kv = types[i] == Type.Put
176        ? new KeyValue(row1, fam1, qual, ts, types[i], data)
177        : new KeyValue(row1, fam1, qual, ts, types[i]);
178      actual.add(qm.match(kv));
179      if (actual.size() >= expected.length) {
180        break;
181      }
182      // Decrement ts for next cell, but keep same ts when the next type has lower type code
183      // at the same logical position (e.g. DF then DFV at the same timestamp).
184      if (i + 1 < types.length && types[i + 1].getCode() < types[i].getCode()) {
185        continue;
186      }
187      ts--;
188    }
189    for (int i = 0; i < expected.length; i++) {
190      assertEquals("Mismatch at index " + i, expected[i], actual.get(i));
191    }
192  }
193
194  private void testDropDeletes(byte[] from, byte[] to, byte[][] rows, MatchCode... expected)
195    throws IOException {
196    long now = EnvironmentEdgeManager.currentTime();
197    // Set time to purge deletes to negative value to avoid it ever happening.
198    ScanInfo scanInfo = new ScanInfo(this.conf, fam2, 0, 1, ttl, KeepDeletedCells.FALSE,
199      HConstants.DEFAULT_BLOCKSIZE, -1L, rowComparator, false);
200
201    CompactionScanQueryMatcher qm =
202      CompactionScanQueryMatcher.create(scanInfo, ScanType.COMPACT_RETAIN_DELETES, Long.MAX_VALUE,
203        PrivateConstants.OLDEST_TIMESTAMP, PrivateConstants.OLDEST_TIMESTAMP, now, from, to, null);
204    List<ScanQueryMatcher.MatchCode> actual = new ArrayList<>(rows.length);
205    byte[] prevRow = null;
206    for (byte[] row : rows) {
207      if (prevRow == null || !Bytes.equals(prevRow, row)) {
208        qm.setToNewRow(KeyValueUtil.createFirstOnRow(row));
209        prevRow = row;
210      }
211      actual.add(qm.match(new KeyValue(row, fam2, null, now, Type.Delete)));
212    }
213
214    assertEquals(expected.length, actual.size());
215    for (int i = 0; i < expected.length; i++) {
216      LOG.debug("expected " + expected[i] + ", actual " + actual.get(i));
217      assertEquals(expected[i], actual.get(i));
218    }
219  }
220}