001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver.compactions;
019
020import static org.apache.hadoop.hbase.regionserver.CustomTieringMultiFileWriter.CUSTOM_TIERING_TIME_RANGE;
021
022import java.io.IOException;
023import java.util.ArrayList;
024import java.util.Collection;
025import java.util.List;
026import org.apache.hadoop.conf.Configuration;
027import org.apache.hadoop.hbase.HDFSBlocksDistribution;
028import org.apache.hadoop.hbase.regionserver.HStoreFile;
029import org.apache.hadoop.hbase.regionserver.StoreConfigInformation;
030import org.apache.hadoop.hbase.regionserver.StoreUtils;
031import org.apache.hadoop.hbase.regionserver.TimeRangeTracker;
032import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
033import org.apache.yetus.audience.InterfaceAudience;
034import org.slf4j.Logger;
035import org.slf4j.LoggerFactory;
036
037/**
038 * Custom implementation of DateTieredCompactionPolicy that calculates compaction boundaries based
039 * on the <b>hbase.hstore.compaction.date.tiered.custom.age.limit.millis</b> configuration property
040 * and the TIERING_CELL_MIN/TIERING_CELL_MAX stored on metadata of each store file. This policy
041 * would produce either one or two tiers: - One tier if either all files data age are older than the
042 * configured age limit or all files data age are younger than the configured age limit. - Two tiers
043 * if files have both younger and older data than the configured age limit.
044 */
045@InterfaceAudience.Private
046public class CustomDateTieredCompactionPolicy extends DateTieredCompactionPolicy {
047
048  public static final String AGE_LIMIT_MILLIS =
049    "hbase.hstore.compaction.date.tiered.custom.age.limit.millis";
050
051  // Defaults to 10 years
052  public static final long DEFAULT_AGE_LIMIT_MILLIS =
053    (long) (10L * 365.25 * 24L * 60L * 60L * 1000L);
054
055  private static final Logger LOG = LoggerFactory.getLogger(CustomDateTieredCompactionPolicy.class);
056
057  private long cutOffTimestamp;
058
059  public CustomDateTieredCompactionPolicy(Configuration conf,
060    StoreConfigInformation storeConfigInfo) throws IOException {
061    super(conf, storeConfigInfo);
062    cutOffTimestamp = EnvironmentEdgeManager.currentTime()
063      - conf.getLong(AGE_LIMIT_MILLIS, DEFAULT_AGE_LIMIT_MILLIS);
064
065  }
066
067  @Override
068  protected List<Long> getCompactBoundariesForMajor(Collection<HStoreFile> filesToCompact,
069    long now) {
070    // CustomTieringMultiFileWriter#append buckets each cell into its tier by comparing against
071    // these boundaries directly, and only commits a file for a tier that actually received data.
072    // There is no need to traverse filesToCompact to inspect CUSTOM_TIERING_TIME_RANGE here:
073    // always offering the cutOffTimestamp boundary is sufficient and avoids missing it when a
074    // file lacks that metadata.
075    List<Long> boundaries = new ArrayList<>();
076    boundaries.add(Long.MIN_VALUE);
077    boundaries.add(cutOffTimestamp);
078    return boundaries;
079  }
080
081  @Override
082  public CompactionRequestImpl selectMinorCompaction(ArrayList<HStoreFile> candidateSelection,
083    boolean mayUseOffPeak, boolean mayBeStuck) throws IOException {
084    ArrayList<HStoreFile> filteredByPolicy = this.compactionPolicyPerWindow
085      .applyCompactionPolicy(candidateSelection, mayUseOffPeak, mayBeStuck);
086    return selectMajorCompaction(filteredByPolicy);
087  }
088
089  @Override
090  public boolean shouldPerformMajorCompaction(Collection<HStoreFile> filesToCompact)
091    throws IOException {
092    long lowTimestamp = StoreUtils.getLowestTimestamp(filesToCompact);
093    long now = EnvironmentEdgeManager.currentTime();
094    if (isMajorCompactionTime(filesToCompact, now, lowTimestamp)) {
095      long cfTTL = this.storeConfigInfo.getStoreFileTtl();
096      int countLower = 0;
097      int countHigher = 0;
098      HDFSBlocksDistribution hdfsBlocksDistribution = new HDFSBlocksDistribution();
099      for (HStoreFile f : filesToCompact) {
100        if (checkForTtl(cfTTL, f)) {
101          return true;
102        }
103        if (isMajorOrBulkloadResult(f, now - lowTimestamp)) {
104          return true;
105        }
106        byte[] timeRangeBytes = f.getMetadataValue(CUSTOM_TIERING_TIME_RANGE);
107        // this means this file has not been major compacted by manually triggered compaction at
108        // the time of enabling Custom Time Based Priority, so it needs compaction to have its rows
109        // separated according to the cutOffTimestamp.
110        if (timeRangeBytes == null) {
111          return true;
112        }
113        TimeRangeTracker timeRangeTracker = TimeRangeTracker.parseFrom(timeRangeBytes);
114        if (timeRangeTracker.getMin() < cutOffTimestamp) {
115          if (timeRangeTracker.getMax() > cutOffTimestamp) {
116            // Found at least one file crossing the cutOffTimestamp
117            return true;
118          } else {
119            countLower++;
120          }
121        } else {
122          countHigher++;
123        }
124        hdfsBlocksDistribution.add(f.getHDFSBlockDistribution());
125      }
126      // If we haven't found any files crossing the cutOffTimestamp, we have to check
127      // if there are at least more than one file on each tier and if so, perform compaction
128      if (countLower > 1 || countHigher > 1) {
129        return true;
130      }
131      return checkBlockLocality(hdfsBlocksDistribution);
132    }
133    return false;
134  }
135
136}