Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver.querymatcher;
019
020import static org.apache.hadoop.hbase.regionserver.querymatcher.ScanQueryMatcher.MatchCode.INCLUDE;
021import static org.apache.hadoop.hbase.regionserver.querymatcher.ScanQueryMatcher.MatchCode.SEEK_NEXT_COL;
022import static org.apache.hadoop.hbase.regionserver.querymatcher.ScanQueryMatcher.MatchCode.SKIP;
023import static org.junit.jupiter.api.Assertions.assertEquals;
024
025import java.io.IOException;
026import java.util.ArrayList;
027import java.util.List;
028import org.apache.hadoop.hbase.HConstants;
029import org.apache.hadoop.hbase.KeepDeletedCells;
030import org.apache.hadoop.hbase.KeyValue;
031import org.apache.hadoop.hbase.KeyValue.Type;
032import org.apache.hadoop.hbase.KeyValueUtil;
033import org.apache.hadoop.hbase.PrivateConstants;
034import org.apache.hadoop.hbase.regionserver.ScanInfo;
035import org.apache.hadoop.hbase.regionserver.ScanType;
036import org.apache.hadoop.hbase.regionserver.querymatcher.ScanQueryMatcher.MatchCode;
037import org.apache.hadoop.hbase.testclassification.RegionServerTests;
038import org.apache.hadoop.hbase.testclassification.SmallTests;
039import org.apache.hadoop.hbase.util.Bytes;
040import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
041import org.junit.jupiter.api.Tag;
042import org.junit.jupiter.api.Test;
043import org.slf4j.Logger;
044import org.slf4j.LoggerFactory;
045
046@Tag(RegionServerTests.TAG)
047@Tag(SmallTests.TAG)
048public class TestCompactionScanQueryMatcher extends AbstractTestScanQueryMatcher {
049
050  private static final Logger LOG = LoggerFactory.getLogger(TestCompactionScanQueryMatcher.class);
051
052  @Test
053  public void testMatch_PartialRangeDropDeletes() throws Exception {
054    // Some ranges.
055    testDropDeletes(row2, row3, new byte[][] { row1, row2, row2, row3 }, INCLUDE, SKIP, SKIP,
056      INCLUDE);
057    testDropDeletes(row2, row3, new byte[][] { row1, row1, row2 }, INCLUDE, INCLUDE, SKIP);
058    testDropDeletes(row2, row3, new byte[][] { row2, row3, row3 }, SKIP, INCLUDE, INCLUDE);
059    testDropDeletes(row1, row3, new byte[][] { row1, row2, row3 }, SKIP, SKIP, INCLUDE);
060    // Open ranges.
061    testDropDeletes(HConstants.EMPTY_START_ROW, row3, new byte[][] { row1, row2, row3 }, SKIP, SKIP,
062      INCLUDE);
063    testDropDeletes(row2, HConstants.EMPTY_END_ROW, new byte[][] { row1, row2, row3 }, INCLUDE,
064      SKIP, SKIP);
065    testDropDeletes(HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
066      new byte[][] { row1, row2, row3, row3 }, SKIP, SKIP, SKIP, SKIP);
067
068    // No KVs in range.
069    testDropDeletes(row2, row3, new byte[][] { row1, row1, row3 }, INCLUDE, INCLUDE, INCLUDE);
070    testDropDeletes(row2, row3, new byte[][] { row3, row3 }, INCLUDE, INCLUDE);
071    testDropDeletes(row2, row3, new byte[][] { row1, row1 }, INCLUDE, INCLUDE);
072  }
073
074  /**
075   * Test redundant delete marker handling with COMPACT_RETAIN_DELETES. Cells are auto-generated
076   * from the given types with decrementing timestamps.
077   */
078  @Test
079  public void testSkipsRedundantDeleteMarkers() throws IOException {
080    // Interleaved DeleteColumn + Put. First DC included, put triggers SEEK_NEXT_COL.
081    assertRetainDeletes(new Type[] { Type.DeleteColumn, Type.Put, Type.DeleteColumn }, INCLUDE,
082      SEEK_NEXT_COL);
083
084    // Contiguous DeleteColumn. First included, rest redundant.
085    assertRetainDeletes(new Type[] { Type.DeleteColumn, Type.DeleteColumn, Type.DeleteColumn },
086      INCLUDE, SEEK_NEXT_COL, SEEK_NEXT_COL);
087
088    // Contiguous DeleteFamily. First included, rest redundant.
089    assertRetainDeletes(new Type[] { Type.DeleteFamily, Type.DeleteFamily, Type.DeleteFamily },
090      INCLUDE, SEEK_NEXT_COL, SEEK_NEXT_COL);
091
092    // DF + DFV interleaved. DF included, DFV redundant (SKIP because empty qualifier),
093    // older DF redundant (SEEK_NEXT_COL), older DFV redundant (SKIP).
094    assertRetainDeletes(new Type[] { Type.DeleteFamily, Type.DeleteFamilyVersion, Type.DeleteFamily,
095      Type.DeleteFamilyVersion }, INCLUDE, SKIP, SEEK_NEXT_COL, SKIP);
096
097    // Delete (version) covered by DeleteColumn.
098    assertRetainDeletes(new Type[] { Type.DeleteColumn, Type.Delete, Type.Delete, Type.Delete },
099      INCLUDE, SEEK_NEXT_COL, SEEK_NEXT_COL, SEEK_NEXT_COL);
100
101    // KEEP_DELETED_CELLS=TRUE: all markers retained.
102    assertRetainDeletes(KeepDeletedCells.TRUE,
103      new Type[] { Type.DeleteColumn, Type.DeleteColumn, Type.DeleteColumn }, INCLUDE, INCLUDE,
104      INCLUDE);
105  }
106
107  /**
108   * Redundant column-level deletes with empty qualifier must not seek past a subsequent
109   * DeleteFamily. getKeyForNextColumn treats empty qualifier as "no column" and returns
110   * SEEK_NEXT_ROW, which would skip the DF and all remaining cells in the row.
111   */
112  @Test
113  public void testEmptyQualifierDeleteDoesNotSkipDeleteFamily() throws IOException {
114    byte[] emptyQualifier = HConstants.EMPTY_BYTE_ARRAY;
115
116    // DC(empty) + DC(empty) redundant + DF must still be reachable.
117    assertRetainDeletes(emptyQualifier,
118      new Type[] { Type.DeleteColumn, Type.DeleteColumn, Type.DeleteFamily }, INCLUDE, SKIP,
119      INCLUDE);
120
121    // DC(empty) + Delete(empty) redundant + DF must still be reachable.
122    assertRetainDeletes(emptyQualifier,
123      new Type[] { Type.DeleteColumn, Type.Delete, Type.DeleteFamily }, INCLUDE, SKIP, INCLUDE);
124  }
125
126  private void assertRetainDeletes(Type[] types, MatchCode... expected) throws IOException {
127    assertRetainDeletes(KeepDeletedCells.FALSE, types, expected);
128  }
129
130  private void assertRetainDeletes(byte[] qualifier, Type[] types, MatchCode... expected)
131    throws IOException {
132    assertRetainDeletes(KeepDeletedCells.FALSE, qualifier, types, expected);
133  }
134
135  /**
136   * Build cells from the given types with decrementing timestamps (same ts for adjacent
137   * family-level and column-level types at the same position). Family-level types (DeleteFamily,
138   * DeleteFamilyVersion) use empty qualifier; others use col1.
139   */
140  private void assertRetainDeletes(KeepDeletedCells keepDeletedCells, Type[] types,
141    MatchCode... expected) throws IOException {
142    assertRetainDeletes(keepDeletedCells, null, types, expected);
143  }
144
145  /**
146   * Build cells from the given types with decrementing timestamps. If qualifier is null,
147   * family-level types use empty qualifier and others use col1. If qualifier is specified, all
148   * types use that qualifier.
149   */
150  private void assertRetainDeletes(KeepDeletedCells keepDeletedCells, byte[] qualifier,
151    Type[] types, MatchCode... expected) throws IOException {
152    long now = EnvironmentEdgeManager.currentTime();
153    ScanInfo scanInfo = new ScanInfo(this.conf, fam1, 0, 1, ttl, keepDeletedCells,
154      HConstants.DEFAULT_BLOCKSIZE, 0, rowComparator, false);
155    CompactionScanQueryMatcher qm = CompactionScanQueryMatcher.create(scanInfo,
156      ScanType.COMPACT_RETAIN_DELETES, 0L, PrivateConstants.OLDEST_TIMESTAMP,
157      PrivateConstants.OLDEST_TIMESTAMP, now, null, null, null);
158    qm.setToNewRow(KeyValueUtil.createFirstOnRow(row1));
159
160    long ts = now;
161    List<MatchCode> actual = new ArrayList<>(expected.length);
162    for (int i = 0; i < types.length; i++) {
163      byte[] qual;
164      if (qualifier != null) {
165        qual = qualifier;
166      } else {
167        boolean familyLevel = types[i] == Type.DeleteFamily || types[i] == Type.DeleteFamilyVersion;
168        qual = familyLevel ? HConstants.EMPTY_BYTE_ARRAY : col1;
169      }
170      KeyValue kv = types[i] == Type.Put
171        ? new KeyValue(row1, fam1, qual, ts, types[i], data)
172        : new KeyValue(row1, fam1, qual, ts, types[i]);
173      actual.add(qm.match(kv));
174      if (actual.size() >= expected.length) {
175        break;
176      }
177      // Decrement ts for next cell, but keep same ts when the next type has lower type code
178      // at the same logical position (e.g. DF then DFV at the same timestamp).
179      if (i + 1 < types.length && types[i + 1].getCode() < types[i].getCode()) {
180        continue;
181      }
182      ts--;
183    }
184    for (int i = 0; i < expected.length; i++) {
185      assertEquals(expected[i], actual.get(i), "Mismatch at index " + i);
186    }
187  }
188
189  private void testDropDeletes(byte[] from, byte[] to, byte[][] rows, MatchCode... expected)
190    throws IOException {
191    long now = EnvironmentEdgeManager.currentTime();
192    // Set time to purge deletes to negative value to avoid it ever happening.
193    ScanInfo scanInfo = new ScanInfo(this.conf, fam2, 0, 1, ttl, KeepDeletedCells.FALSE,
194      HConstants.DEFAULT_BLOCKSIZE, -1L, rowComparator, false);
195
196    CompactionScanQueryMatcher qm =
197      CompactionScanQueryMatcher.create(scanInfo, ScanType.COMPACT_RETAIN_DELETES, Long.MAX_VALUE,
198        PrivateConstants.OLDEST_TIMESTAMP, PrivateConstants.OLDEST_TIMESTAMP, now, from, to, null);
199    List<ScanQueryMatcher.MatchCode> actual = new ArrayList<>(rows.length);
200    byte[] prevRow = null;
201    for (byte[] row : rows) {
202      if (prevRow == null || !Bytes.equals(prevRow, row)) {
203        qm.setToNewRow(KeyValueUtil.createFirstOnRow(row));
204        prevRow = row;
205      }
206      actual.add(qm.match(new KeyValue(row, fam2, null, now, Type.Delete)));
207    }
208
209    assertEquals(expected.length, actual.size());
210    for (int i = 0; i < expected.length; i++) {
211      LOG.debug("expected " + expected[i] + ", actual " + actual.get(i));
212      assertEquals(expected[i], actual.get(i));
213    }
214  }
215}