001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.io.hfile;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021
022import java.io.IOException;
023import java.util.ArrayList;
024import java.util.List;
025import java.util.Set;
026import java.util.stream.Stream;
027import org.apache.hadoop.conf.Configuration;
028import org.apache.hadoop.hbase.Cell;
029import org.apache.hadoop.hbase.HBaseParameterizedTestTemplate;
030import org.apache.hadoop.hbase.HBaseTestingUtil;
031import org.apache.hadoop.hbase.TableName;
032import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
033import org.apache.hadoop.hbase.client.Put;
034import org.apache.hadoop.hbase.client.RegionInfo;
035import org.apache.hadoop.hbase.client.RegionInfoBuilder;
036import org.apache.hadoop.hbase.client.Scan;
037import org.apache.hadoop.hbase.client.TableDescriptor;
038import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
039import org.apache.hadoop.hbase.regionserver.HRegion;
040import org.apache.hadoop.hbase.regionserver.InternalScanner;
041import org.apache.hadoop.hbase.testclassification.IOTests;
042import org.apache.hadoop.hbase.testclassification.LargeTests;
043import org.apache.hadoop.hbase.util.Bytes;
044import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
045import org.apache.hadoop.hbase.util.Threads;
046import org.junit.jupiter.api.Tag;
047import org.junit.jupiter.api.TestTemplate;
048import org.junit.jupiter.params.provider.Arguments;
049import org.slf4j.Logger;
050import org.slf4j.LoggerFactory;
051
052/**
053 * Test the optimization that does not scan files where all timestamps are expired.
054 */
055@HBaseParameterizedTestTemplate(name = "numFreshFiles={0}")
056@Tag(IOTests.TAG)
057@Tag(LargeTests.TAG)
058public class TestScannerSelectionUsingTTL {
059
060  private static final Logger LOG = LoggerFactory.getLogger(TestScannerSelectionUsingTTL.class);
061
062  private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
063  private static TableName TABLE = TableName.valueOf("myTable");
064  private static String FAMILY = "myCF";
065  private static byte[] FAMILY_BYTES = Bytes.toBytes(FAMILY);
066
067  private static final int TTL_SECONDS = 10;
068  private static final int TTL_MS = TTL_SECONDS * 1000;
069
070  private static final int NUM_EXPIRED_FILES = 2;
071  private static final int NUM_ROWS = 8;
072  private static final int NUM_COLS_PER_ROW = 5;
073
074  public final int numFreshFiles, totalNumFiles;
075
076  public static Stream<Arguments> parameters() {
077    List<Arguments> params = new ArrayList<>();
078    for (int numFreshFiles = 1; numFreshFiles <= 3; ++numFreshFiles) {
079      params.add(Arguments.of(numFreshFiles));
080    }
081    return params.stream();
082  }
083
084  public TestScannerSelectionUsingTTL(int numFreshFiles) {
085    this.numFreshFiles = numFreshFiles;
086    this.totalNumFiles = numFreshFiles + NUM_EXPIRED_FILES;
087  }
088
089  @TestTemplate
090  public void testScannerSelection() throws IOException {
091    Configuration conf = TEST_UTIL.getConfiguration();
092    conf.setBoolean("hbase.store.delete.expired.storefile", false);
093    LruBlockCache cache = (LruBlockCache) BlockCacheFactory.createBlockCache(conf);
094
095    TableDescriptor td = TableDescriptorBuilder.newBuilder(TABLE)
096      .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(FAMILY_BYTES)
097        .setMaxVersions(Integer.MAX_VALUE).setTimeToLive(TTL_SECONDS).build())
098      .build();
099    RegionInfo info = RegionInfoBuilder.newBuilder(TABLE).build();
100    HRegion region = HBaseTestingUtil.createRegionAndWAL(info,
101      TEST_UTIL.getDataTestDir(info.getEncodedName()), conf, td, cache);
102
103    long ts = EnvironmentEdgeManager.currentTime();
104    long version = 0; // make sure each new set of Put's have a new ts
105    for (int iFile = 0; iFile < totalNumFiles; ++iFile) {
106      if (iFile == NUM_EXPIRED_FILES) {
107        Threads.sleepWithoutInterrupt(TTL_MS);
108        version += TTL_MS;
109      }
110
111      for (int iRow = 0; iRow < NUM_ROWS; ++iRow) {
112        Put put = new Put(Bytes.toBytes("row" + iRow));
113        for (int iCol = 0; iCol < NUM_COLS_PER_ROW; ++iCol) {
114          put.addColumn(FAMILY_BYTES, Bytes.toBytes("col" + iCol), ts + version,
115            Bytes.toBytes("value" + iFile + "_" + iRow + "_" + iCol));
116        }
117        region.put(put);
118      }
119      region.flush(true);
120      version++;
121    }
122
123    Scan scan = new Scan().readVersions(Integer.MAX_VALUE);
124    cache.clearCache();
125    InternalScanner scanner = region.getScanner(scan);
126    List<Cell> results = new ArrayList<>();
127    final int expectedKVsPerRow = numFreshFiles * NUM_COLS_PER_ROW;
128    int numReturnedRows = 0;
129    LOG.info("Scanning the entire table");
130    while (scanner.next(results) || results.size() > 0) {
131      assertEquals(expectedKVsPerRow, results.size());
132      ++numReturnedRows;
133      results.clear();
134    }
135    assertEquals(NUM_ROWS, numReturnedRows);
136    Set<String> accessedFiles = cache.getCachedFileNamesForTest();
137    LOG.debug("Files accessed during scan: " + accessedFiles);
138
139    region.compact(false);
140
141    HBaseTestingUtil.closeRegionAndWAL(region);
142  }
143}