001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.io.hfile;
019
020import static org.junit.Assert.assertEquals;
021
022import java.io.IOException;
023import java.util.ArrayList;
024import java.util.Collection;
025import java.util.List;
026import java.util.Set;
027import org.apache.hadoop.conf.Configuration;
028import org.apache.hadoop.hbase.Cell;
029import org.apache.hadoop.hbase.HBaseClassTestRule;
030import org.apache.hadoop.hbase.HBaseTestingUtility;
031import org.apache.hadoop.hbase.TableName;
032import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
033import org.apache.hadoop.hbase.client.Put;
034import org.apache.hadoop.hbase.client.RegionInfo;
035import org.apache.hadoop.hbase.client.RegionInfoBuilder;
036import org.apache.hadoop.hbase.client.Scan;
037import org.apache.hadoop.hbase.client.TableDescriptor;
038import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
039import org.apache.hadoop.hbase.regionserver.HRegion;
040import org.apache.hadoop.hbase.regionserver.HStore;
041import org.apache.hadoop.hbase.regionserver.InternalScanner;
042import org.apache.hadoop.hbase.testclassification.IOTests;
043import org.apache.hadoop.hbase.testclassification.LargeTests;
044import org.apache.hadoop.hbase.util.Bytes;
045import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
046import org.apache.hadoop.hbase.util.Threads;
047import org.junit.ClassRule;
048import org.junit.Test;
049import org.junit.experimental.categories.Category;
050import org.junit.runner.RunWith;
051import org.junit.runners.Parameterized;
052import org.junit.runners.Parameterized.Parameters;
053import org.slf4j.Logger;
054import org.slf4j.LoggerFactory;
055
056/**
057 * Test the optimization that does not scan files where all timestamps are
058 * expired.
059 */
060@RunWith(Parameterized.class)
061@Category({IOTests.class, LargeTests.class})
062public class TestScannerSelectionUsingTTL {
063
064  @ClassRule
065  public static final HBaseClassTestRule CLASS_RULE =
066      HBaseClassTestRule.forClass(TestScannerSelectionUsingTTL.class);
067
068  private static final Logger LOG =
069      LoggerFactory.getLogger(TestScannerSelectionUsingTTL.class);
070
071  private static final HBaseTestingUtility TEST_UTIL = HBaseTestingUtility.createLocalHTU();
072  private static TableName TABLE = TableName.valueOf("myTable");
073  private static String FAMILY = "myCF";
074  private static byte[] FAMILY_BYTES = Bytes.toBytes(FAMILY);
075
076  private static final int TTL_SECONDS = 10;
077  private static final int TTL_MS = TTL_SECONDS * 1000;
078
079  private static final int NUM_EXPIRED_FILES = 2;
080  private static final int NUM_ROWS = 8;
081  private static final int NUM_COLS_PER_ROW = 5;
082
083  public final int numFreshFiles, totalNumFiles;
084
085  /** Whether we are specifying the exact files to compact */
086  private final boolean explicitCompaction;
087
088  @Parameters
089  public static Collection<Object[]> parameters() {
090    List<Object[]> params = new ArrayList<>();
091    for (int numFreshFiles = 1; numFreshFiles <= 3; ++numFreshFiles) {
092      for (boolean explicitCompaction : new boolean[] { false, true }) {
093        params.add(new Object[] { numFreshFiles, explicitCompaction });
094      }
095    }
096    return params;
097  }
098
099  public TestScannerSelectionUsingTTL(int numFreshFiles,
100      boolean explicitCompaction) {
101    this.numFreshFiles = numFreshFiles;
102    this.totalNumFiles = numFreshFiles + NUM_EXPIRED_FILES;
103    this.explicitCompaction = explicitCompaction;
104  }
105
106  @Test
107  public void testScannerSelection() throws IOException {
108    Configuration conf = TEST_UTIL.getConfiguration();
109    conf.setBoolean("hbase.store.delete.expired.storefile", false);
110    LruBlockCache cache = (LruBlockCache) BlockCacheFactory.createBlockCache(conf);
111
112    TableDescriptor td = TableDescriptorBuilder.newBuilder(TABLE).setColumnFamily(
113        ColumnFamilyDescriptorBuilder.newBuilder(FAMILY_BYTES).setMaxVersions(Integer.MAX_VALUE)
114            .setTimeToLive(TTL_SECONDS).build()).build();
115    RegionInfo info = RegionInfoBuilder.newBuilder(TABLE).build();
116    HRegion region = HBaseTestingUtility
117        .createRegionAndWAL(info, TEST_UTIL.getDataTestDir(info.getEncodedName()), conf, td, cache);
118
119    long ts = EnvironmentEdgeManager.currentTime();
120    long version = 0; //make sure each new set of Put's have a new ts
121    for (int iFile = 0; iFile < totalNumFiles; ++iFile) {
122      if (iFile == NUM_EXPIRED_FILES) {
123        Threads.sleepWithoutInterrupt(TTL_MS);
124        version += TTL_MS;
125      }
126
127      for (int iRow = 0; iRow < NUM_ROWS; ++iRow) {
128        Put put = new Put(Bytes.toBytes("row" + iRow));
129        for (int iCol = 0; iCol < NUM_COLS_PER_ROW; ++iCol) {
130          put.addColumn(FAMILY_BYTES, Bytes.toBytes("col" + iCol), ts + version,
131                  Bytes.toBytes("value" + iFile + "_" + iRow + "_" + iCol));
132        }
133        region.put(put);
134      }
135      region.flush(true);
136      version++;
137    }
138
139    Scan scan = new Scan().readVersions(Integer.MAX_VALUE);
140    cache.clearCache();
141    InternalScanner scanner = region.getScanner(scan);
142    List<Cell> results = new ArrayList<>();
143    final int expectedKVsPerRow = numFreshFiles * NUM_COLS_PER_ROW;
144    int numReturnedRows = 0;
145    LOG.info("Scanning the entire table");
146    while (scanner.next(results) || results.size() > 0) {
147      assertEquals(expectedKVsPerRow, results.size());
148      ++numReturnedRows;
149      results.clear();
150    }
151    assertEquals(NUM_ROWS, numReturnedRows);
152    Set<String> accessedFiles = cache.getCachedFileNamesForTest();
153    LOG.debug("Files accessed during scan: " + accessedFiles);
154
155    // Exercise both compaction codepaths.
156    if (explicitCompaction) {
157      HStore store = region.getStore(FAMILY_BYTES);
158      store.compactRecentForTestingAssumingDefaultPolicy(totalNumFiles);
159    } else {
160      region.compact(false);
161    }
162
163    HBaseTestingUtility.closeRegionAndWAL(region);
164  }
165}