001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.io.hfile;
019
020import static org.junit.Assert.assertEquals;
021
022import java.io.IOException;
023import java.util.ArrayList;
024import java.util.Collection;
025import java.util.List;
026import java.util.Set;
027import org.apache.hadoop.conf.Configuration;
028import org.apache.hadoop.hbase.Cell;
029import org.apache.hadoop.hbase.HBaseClassTestRule;
030import org.apache.hadoop.hbase.HBaseTestingUtility;
031import org.apache.hadoop.hbase.TableName;
032import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
033import org.apache.hadoop.hbase.client.Put;
034import org.apache.hadoop.hbase.client.RegionInfo;
035import org.apache.hadoop.hbase.client.RegionInfoBuilder;
036import org.apache.hadoop.hbase.client.Scan;
037import org.apache.hadoop.hbase.client.TableDescriptor;
038import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
039import org.apache.hadoop.hbase.regionserver.HRegion;
040import org.apache.hadoop.hbase.regionserver.InternalScanner;
041import org.apache.hadoop.hbase.testclassification.IOTests;
042import org.apache.hadoop.hbase.testclassification.LargeTests;
043import org.apache.hadoop.hbase.util.Bytes;
044import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
045import org.apache.hadoop.hbase.util.Threads;
046import org.junit.ClassRule;
047import org.junit.Test;
048import org.junit.experimental.categories.Category;
049import org.junit.runner.RunWith;
050import org.junit.runners.Parameterized;
051import org.junit.runners.Parameterized.Parameters;
052import org.slf4j.Logger;
053import org.slf4j.LoggerFactory;
054
055/**
056 * Test the optimization that does not scan files where all timestamps are expired.
057 */
058@RunWith(Parameterized.class)
059@Category({ IOTests.class, LargeTests.class })
060public class TestScannerSelectionUsingTTL {
061
062  @ClassRule
063  public static final HBaseClassTestRule CLASS_RULE =
064    HBaseClassTestRule.forClass(TestScannerSelectionUsingTTL.class);
065
066  private static final Logger LOG = LoggerFactory.getLogger(TestScannerSelectionUsingTTL.class);
067
068  private static final HBaseTestingUtility TEST_UTIL = HBaseTestingUtility.createLocalHTU();
069  private static TableName TABLE = TableName.valueOf("myTable");
070  private static String FAMILY = "myCF";
071  private static byte[] FAMILY_BYTES = Bytes.toBytes(FAMILY);
072
073  private static final int TTL_SECONDS = 10;
074  private static final int TTL_MS = TTL_SECONDS * 1000;
075
076  private static final int NUM_EXPIRED_FILES = 2;
077  private static final int NUM_ROWS = 8;
078  private static final int NUM_COLS_PER_ROW = 5;
079
080  public final int numFreshFiles, totalNumFiles;
081
082  @Parameters
083  public static Collection<Object[]> parameters() {
084    List<Object[]> params = new ArrayList<>();
085    for (int numFreshFiles = 1; numFreshFiles <= 3; ++numFreshFiles) {
086      params.add(new Object[] { numFreshFiles });
087    }
088    return params;
089  }
090
091  public TestScannerSelectionUsingTTL(int numFreshFiles) {
092    this.numFreshFiles = numFreshFiles;
093    this.totalNumFiles = numFreshFiles + NUM_EXPIRED_FILES;
094  }
095
096  @Test
097  public void testScannerSelection() throws IOException {
098    Configuration conf = TEST_UTIL.getConfiguration();
099    conf.setBoolean("hbase.store.delete.expired.storefile", false);
100    LruBlockCache cache = (LruBlockCache) BlockCacheFactory.createBlockCache(conf);
101
102    TableDescriptor td = TableDescriptorBuilder.newBuilder(TABLE)
103      .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(FAMILY_BYTES)
104        .setMaxVersions(Integer.MAX_VALUE).setTimeToLive(TTL_SECONDS).build())
105      .build();
106    RegionInfo info = RegionInfoBuilder.newBuilder(TABLE).build();
107    HRegion region = HBaseTestingUtility.createRegionAndWAL(info,
108      TEST_UTIL.getDataTestDir(info.getEncodedName()), conf, td, cache);
109
110    long ts = EnvironmentEdgeManager.currentTime();
111    long version = 0; // make sure each new set of Put's have a new ts
112    for (int iFile = 0; iFile < totalNumFiles; ++iFile) {
113      if (iFile == NUM_EXPIRED_FILES) {
114        Threads.sleepWithoutInterrupt(TTL_MS);
115        version += TTL_MS;
116      }
117
118      for (int iRow = 0; iRow < NUM_ROWS; ++iRow) {
119        Put put = new Put(Bytes.toBytes("row" + iRow));
120        for (int iCol = 0; iCol < NUM_COLS_PER_ROW; ++iCol) {
121          put.addColumn(FAMILY_BYTES, Bytes.toBytes("col" + iCol), ts + version,
122            Bytes.toBytes("value" + iFile + "_" + iRow + "_" + iCol));
123        }
124        region.put(put);
125      }
126      region.flush(true);
127      version++;
128    }
129
130    Scan scan = new Scan().readVersions(Integer.MAX_VALUE);
131    cache.clearCache();
132    InternalScanner scanner = region.getScanner(scan);
133    List<Cell> results = new ArrayList<>();
134    final int expectedKVsPerRow = numFreshFiles * NUM_COLS_PER_ROW;
135    int numReturnedRows = 0;
136    LOG.info("Scanning the entire table");
137    while (scanner.next(results) || results.size() > 0) {
138      assertEquals(expectedKVsPerRow, results.size());
139      ++numReturnedRows;
140      results.clear();
141    }
142    assertEquals(NUM_ROWS, numReturnedRows);
143    Set<String> accessedFiles = cache.getCachedFileNamesForTest();
144    LOG.debug("Files accessed during scan: " + accessedFiles);
145
146    region.compact(false);
147
148    HBaseTestingUtility.closeRegionAndWAL(region);
149  }
150}