001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver.compactions;
019
020import static org.apache.hadoop.hbase.regionserver.StoreFileWriter.shouldEnableHistoricalCompactionFiles;
021
022import org.apache.hadoop.conf.Configuration;
023import org.apache.hadoop.hbase.HConstants;
024import org.apache.hadoop.hbase.client.RegionInfo;
025import org.apache.hadoop.hbase.conf.ConfigKey;
026import org.apache.hadoop.hbase.regionserver.StoreConfigInformation;
027import org.apache.hadoop.util.StringUtils;
028import org.apache.yetus.audience.InterfaceAudience;
029import org.slf4j.Logger;
030import org.slf4j.LoggerFactory;
031
032/**
033 * <p>
034 * Compaction configuration for a particular instance of HStore. Takes into account both global
035 * settings and ones set on the column family/store. Control knobs for default compaction algorithm:
036 * </p>
037 * <p>
038 * maxCompactSize - upper bound on file size to be included in minor compactions minCompactSize -
039 * lower bound below which compaction is selected without ratio test minFilesToCompact - lower bound
040 * on number of files in any minor compaction maxFilesToCompact - upper bound on number of files in
041 * any minor compaction compactionRatio - Ratio used for compaction minLocalityToForceCompact -
042 * Locality threshold for a store file to major compact (HBASE-11195)
043 * </p>
044 * Set parameter as "hbase.hstore.compaction.&lt;attribute&gt;"
045 */
046
047@InterfaceAudience.Private
048public class CompactionConfiguration {
049
050  private static final Logger LOG = LoggerFactory.getLogger(CompactionConfiguration.class);
051
052  public static final String HBASE_HSTORE_COMPACTION_RATIO_KEY =
053    ConfigKey.FLOAT("hbase.hstore.compaction.ratio");
054  public static final String HBASE_HSTORE_COMPACTION_RATIO_OFFPEAK_KEY =
055    ConfigKey.FLOAT("hbase.hstore.compaction.ratio.offpeak");
056  public static final String HBASE_HSTORE_COMPACTION_MIN_KEY_OLD =
057    ConfigKey.INT("hbase.hstore.compactionThreshold");
058  public static final String HBASE_HSTORE_COMPACTION_MIN_KEY =
059    ConfigKey.INT("hbase.hstore.compaction.min");
060  public static final String HBASE_HSTORE_COMPACTION_MIN_SIZE_KEY =
061    ConfigKey.LONG("hbase.hstore.compaction.min.size");
062  public static final String HBASE_HSTORE_COMPACTION_MAX_KEY =
063    ConfigKey.INT("hbase.hstore.compaction.max");
064  public static final String HBASE_HSTORE_COMPACTION_MAX_SIZE_KEY =
065    ConfigKey.LONG("hbase.hstore.compaction.max.size");
066  public static final String HBASE_HSTORE_COMPACTION_MAX_SIZE_OFFPEAK_KEY =
067    ConfigKey.LONG("hbase.hstore.compaction.max.size.offpeak");
068  public static final String HBASE_HSTORE_OFFPEAK_END_HOUR =
069    ConfigKey.INT("hbase.offpeak.end.hour");
070  public static final String HBASE_HSTORE_OFFPEAK_START_HOUR =
071    ConfigKey.INT("hbase.offpeak.start.hour");
072  public static final String HBASE_HSTORE_MIN_LOCALITY_TO_SKIP_MAJOR_COMPACT =
073    ConfigKey.FLOAT("hbase.hstore.min.locality.to.skip.major.compact");
074
075  public static final String HBASE_HFILE_COMPACTION_DISCHARGER_THREAD_COUNT =
076    ConfigKey.INT("hbase.hfile.compaction.discharger.thread.count");
077
078  /*
079   * The epoch time length for the windows we no longer compact
080   */
081  public static final String DATE_TIERED_MAX_AGE_MILLIS_KEY =
082    ConfigKey.LONG("hbase.hstore.compaction.date.tiered.max.storefile.age.millis");
083  public static final String DATE_TIERED_INCOMING_WINDOW_MIN_KEY =
084    ConfigKey.INT("hbase.hstore.compaction.date.tiered.incoming.window.min");
085  public static final String COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS_KEY = ConfigKey.CLASS(
086    "hbase.hstore.compaction.date.tiered.window.policy.class", RatioBasedCompactionPolicy.class);
087  public static final String DATE_TIERED_SINGLE_OUTPUT_FOR_MINOR_COMPACTION_KEY =
088    ConfigKey.BOOLEAN("hbase.hstore.compaction.date.tiered.single.output.for.minor.compaction");
089
090  private static final Class<
091    ? extends RatioBasedCompactionPolicy> DEFAULT_COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS =
092      ExploringCompactionPolicy.class;
093
094  public static final String DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS_KEY = ConfigKey.CLASS(
095    "hbase.hstore.compaction.date.tiered.window.factory.class", CompactionWindowFactory.class);
096
097  private static final Class<
098    ? extends CompactionWindowFactory> DEFAULT_DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS =
099      ExponentialCompactionWindowFactory.class;
100
101  public static final String DATE_TIERED_STORAGE_POLICY_ENABLE_KEY =
102    "hbase.hstore.compaction.date.tiered.storage.policy.enable";
103  public static final String DATE_TIERED_HOT_WINDOW_AGE_MILLIS_KEY =
104    ConfigKey.LONG("hbase.hstore.compaction.date.tiered.hot.window.age.millis");
105  public static final String DATE_TIERED_HOT_WINDOW_STORAGE_POLICY_KEY =
106    "hbase.hstore.compaction.date.tiered.hot.window.storage.policy";
107  public static final String DATE_TIERED_WARM_WINDOW_AGE_MILLIS_KEY =
108    ConfigKey.LONG("hbase.hstore.compaction.date.tiered.warm.window.age.millis");
109  public static final String DATE_TIERED_WARM_WINDOW_STORAGE_POLICY_KEY =
110    "hbase.hstore.compaction.date.tiered.warm.window.storage.policy";
111  /** Windows older than warm age belong to COLD_WINDOW **/
112  public static final String DATE_TIERED_COLD_WINDOW_STORAGE_POLICY_KEY =
113    "hbase.hstore.compaction.date.tiered.cold.window.storage.policy";
114
115  Configuration conf;
116  StoreConfigInformation storeConfigInfo;
117
118  private final double offPeakCompactionRatio;
119  /** Since all these properties can change online, they are volatile **/
120  private final long maxCompactSize;
121  private final long offPeakMaxCompactSize;
122  private final long minCompactSize;
123  /** This one can be update **/
124  private int minFilesToCompact;
125  private final int maxFilesToCompact;
126  private final double compactionRatio;
127  private final long throttlePoint;
128  private final long majorCompactionPeriod;
129  private final float majorCompactionJitter;
130  private final float minLocalityToForceCompact;
131  private final long dateTieredMaxStoreFileAgeMillis;
132  private final int dateTieredIncomingWindowMin;
133  private final String compactionPolicyForDateTieredWindow;
134  private final boolean dateTieredSingleOutputForMinorCompaction;
135  private final String dateTieredCompactionWindowFactory;
136  private final boolean dateTieredStoragePolicyEnable;
137  private long hotWindowAgeMillis;
138  private long warmWindowAgeMillis;
139  private String hotWindowStoragePolicy;
140  private String warmWindowStoragePolicy;
141  private String coldWindowStoragePolicy;
142
143  CompactionConfiguration(Configuration conf, StoreConfigInformation storeConfigInfo) {
144    this.conf = conf;
145    this.storeConfigInfo = storeConfigInfo;
146
147    maxCompactSize = conf.getLong(HBASE_HSTORE_COMPACTION_MAX_SIZE_KEY, Long.MAX_VALUE);
148    offPeakMaxCompactSize =
149      conf.getLong(HBASE_HSTORE_COMPACTION_MAX_SIZE_OFFPEAK_KEY, maxCompactSize);
150    minCompactSize =
151      conf.getLong(HBASE_HSTORE_COMPACTION_MIN_SIZE_KEY, storeConfigInfo.getMemStoreFlushSize());
152    minFilesToCompact = Math.max(2, conf.getInt(HBASE_HSTORE_COMPACTION_MIN_KEY,
153      conf.getInt(HBASE_HSTORE_COMPACTION_MIN_KEY_OLD, 3)));
154    if (shouldEnableHistoricalCompactionFiles(conf)) {
155      // If historical file writing is enabled, we bump up the min value by one as DualFileWriter
156      // compacts files into two files, live and historical, instead of one. This also eliminates
157      // infinite re-compaction when the min value is set to 2
158      minFilesToCompact += 1;
159    }
160    maxFilesToCompact = conf.getInt(HBASE_HSTORE_COMPACTION_MAX_KEY, 10);
161    compactionRatio = conf.getFloat(HBASE_HSTORE_COMPACTION_RATIO_KEY, 1.2F);
162    offPeakCompactionRatio = conf.getFloat(HBASE_HSTORE_COMPACTION_RATIO_OFFPEAK_KEY, 5.0F);
163
164    throttlePoint = conf.getLong("hbase.regionserver.thread.compaction.throttle",
165      2 * maxFilesToCompact * storeConfigInfo.getMemStoreFlushSize());
166    majorCompactionPeriod =
167      conf.getLong(HConstants.MAJOR_COMPACTION_PERIOD, HConstants.DEFAULT_MAJOR_COMPACTION_PERIOD);
168    majorCompactionJitter =
169      conf.getFloat(HConstants.MAJOR_COMPACTION_JITTER, HConstants.DEFAULT_MAJOR_COMPACTION_JITTER);
170    minLocalityToForceCompact = conf.getFloat(HBASE_HSTORE_MIN_LOCALITY_TO_SKIP_MAJOR_COMPACT, 0f);
171
172    dateTieredMaxStoreFileAgeMillis = conf.getLong(DATE_TIERED_MAX_AGE_MILLIS_KEY, Long.MAX_VALUE);
173    dateTieredIncomingWindowMin = conf.getInt(DATE_TIERED_INCOMING_WINDOW_MIN_KEY, 6);
174    compactionPolicyForDateTieredWindow =
175      conf.get(COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS_KEY,
176        DEFAULT_COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS.getName());
177    dateTieredSingleOutputForMinorCompaction =
178      conf.getBoolean(DATE_TIERED_SINGLE_OUTPUT_FOR_MINOR_COMPACTION_KEY, true);
179    this.dateTieredCompactionWindowFactory =
180      conf.get(DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS_KEY,
181        DEFAULT_DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS.getName());
182    // for Heterogeneous Storage
183    dateTieredStoragePolicyEnable = conf.getBoolean(DATE_TIERED_STORAGE_POLICY_ENABLE_KEY, false);
184    hotWindowAgeMillis = conf.getLong(DATE_TIERED_HOT_WINDOW_AGE_MILLIS_KEY, 86400000L);
185    hotWindowStoragePolicy = conf.get(DATE_TIERED_HOT_WINDOW_STORAGE_POLICY_KEY, "ALL_SSD");
186    warmWindowAgeMillis = conf.getLong(DATE_TIERED_WARM_WINDOW_AGE_MILLIS_KEY, 604800000L);
187    warmWindowStoragePolicy = conf.get(DATE_TIERED_WARM_WINDOW_STORAGE_POLICY_KEY, "ONE_SSD");
188    coldWindowStoragePolicy = conf.get(DATE_TIERED_COLD_WINDOW_STORAGE_POLICY_KEY, "HOT");
189    LOG.info(toString());
190  }
191
192  @Override
193  public String toString() {
194    return String.format(
195      "size [minCompactSize:%s, maxCompactSize:%s, offPeakMaxCompactSize:%s);"
196        + " files [minFilesToCompact:%d, maxFilesToCompact:%d);"
197        + " ratio %f; off-peak ratio %f; throttle point %d;"
198        + " major period %d, major jitter %f, min locality to compact %f;"
199        + " tiered compaction: max_age %d, incoming window min %d,"
200        + " compaction policy for tiered window %s, single output for minor %b,"
201        + " compaction window factory %s," + " region %s columnFamilyName %s",
202      StringUtils.byteDesc(minCompactSize), StringUtils.byteDesc(maxCompactSize),
203      StringUtils.byteDesc(offPeakMaxCompactSize), minFilesToCompact, maxFilesToCompact,
204      compactionRatio, offPeakCompactionRatio, throttlePoint, majorCompactionPeriod,
205      majorCompactionJitter, minLocalityToForceCompact, dateTieredMaxStoreFileAgeMillis,
206      dateTieredIncomingWindowMin, compactionPolicyForDateTieredWindow,
207      dateTieredSingleOutputForMinorCompaction, dateTieredCompactionWindowFactory,
208      RegionInfo.prettyPrint(storeConfigInfo.getRegionInfo().getEncodedName()),
209      storeConfigInfo.getColumnFamilyName());
210  }
211
212  /** Returns lower bound below which compaction is selected without ratio test */
213  public long getMinCompactSize() {
214    return minCompactSize;
215  }
216
217  /** Returns upper bound on file size to be included in minor compactions */
218  public long getMaxCompactSize() {
219    return maxCompactSize;
220  }
221
222  /** Returns lower bound on number of files to be included in minor compactions */
223  public int getMinFilesToCompact() {
224    return minFilesToCompact;
225  }
226
227  /**
228   * Set lower bound on number of files to be included in minor compactions
229   * @param threshold value to set to
230   */
231  public void setMinFilesToCompact(int threshold) {
232    minFilesToCompact = threshold;
233  }
234
235  /** Returns upper bound on number of files to be included in minor compactions */
236  public int getMaxFilesToCompact() {
237    return maxFilesToCompact;
238  }
239
240  /** Returns Ratio used for compaction */
241  public double getCompactionRatio() {
242    return compactionRatio;
243  }
244
245  /** Returns Off peak Ratio used for compaction */
246  public double getCompactionRatioOffPeak() {
247    return offPeakCompactionRatio;
248  }
249
250  /** Returns ThrottlePoint used for classifying small and large compactions */
251  public long getThrottlePoint() {
252    return throttlePoint;
253  }
254
255  /**
256   * @return Major compaction period from compaction. Major compactions are selected periodically
257   *         according to this parameter plus jitter
258   */
259  public long getMajorCompactionPeriod() {
260    return majorCompactionPeriod;
261  }
262
263  /**
264   * @return Major the jitter fraction, the fraction within which the major compaction period is
265   *         randomly chosen from the majorCompactionPeriod in each store.
266   */
267  public float getMajorCompactionJitter() {
268    return majorCompactionJitter;
269  }
270
271  /**
272   * @return Block locality ratio, the ratio at which we will include old regions with a single
273   *         store file for major compaction. Used to improve block locality for regions that
274   *         haven't had writes in a while but are still being read.
275   */
276  public float getMinLocalityToForceCompact() {
277    return minLocalityToForceCompact;
278  }
279
280  public long getOffPeakMaxCompactSize() {
281    return offPeakMaxCompactSize;
282  }
283
284  public long getMaxCompactSize(boolean mayUseOffpeak) {
285    if (mayUseOffpeak) {
286      return getOffPeakMaxCompactSize();
287    } else {
288      return getMaxCompactSize();
289    }
290  }
291
292  public long getDateTieredMaxStoreFileAgeMillis() {
293    return dateTieredMaxStoreFileAgeMillis;
294  }
295
296  public int getDateTieredIncomingWindowMin() {
297    return dateTieredIncomingWindowMin;
298  }
299
300  public String getCompactionPolicyForDateTieredWindow() {
301    return compactionPolicyForDateTieredWindow;
302  }
303
304  public boolean useDateTieredSingleOutputForMinorCompaction() {
305    return dateTieredSingleOutputForMinorCompaction;
306  }
307
308  public String getDateTieredCompactionWindowFactory() {
309    return dateTieredCompactionWindowFactory;
310  }
311
312  public boolean isDateTieredStoragePolicyEnable() {
313    return dateTieredStoragePolicyEnable;
314  }
315
316  public long getHotWindowAgeMillis() {
317    return hotWindowAgeMillis;
318  }
319
320  public long getWarmWindowAgeMillis() {
321    return warmWindowAgeMillis;
322  }
323
324  public String getHotWindowStoragePolicy() {
325    return hotWindowStoragePolicy.trim().toUpperCase();
326  }
327
328  public String getWarmWindowStoragePolicy() {
329    return warmWindowStoragePolicy.trim().toUpperCase();
330  }
331
332  public String getColdWindowStoragePolicy() {
333    return coldWindowStoragePolicy.trim().toUpperCase();
334  }
335}