001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver.compactions;
019
020import org.apache.hadoop.conf.Configuration;
021import org.apache.hadoop.hbase.HConstants;
022import org.apache.hadoop.hbase.client.RegionInfo;
023import org.apache.hadoop.hbase.regionserver.StoreConfigInformation;
024import org.apache.hadoop.util.StringUtils;
025import org.apache.yetus.audience.InterfaceAudience;
026import org.slf4j.Logger;
027import org.slf4j.LoggerFactory;
028
029/**
030 * <p>
031 * Compaction configuration for a particular instance of HStore. Takes into account both global
032 * settings and ones set on the column family/store. Control knobs for default compaction algorithm:
033 * </p>
034 * <p>
035 * maxCompactSize - upper bound on file size to be included in minor compactions minCompactSize -
036 * lower bound below which compaction is selected without ratio test minFilesToCompact - lower bound
037 * on number of files in any minor compaction maxFilesToCompact - upper bound on number of files in
038 * any minor compaction compactionRatio - Ratio used for compaction minLocalityToForceCompact -
039 * Locality threshold for a store file to major compact (HBASE-11195)
040 * </p>
041 * Set parameter as "hbase.hstore.compaction.&lt;attribute&gt;"
042 */
043
044@InterfaceAudience.Private
045public class CompactionConfiguration {
046
047  private static final Logger LOG = LoggerFactory.getLogger(CompactionConfiguration.class);
048
049  public static final String HBASE_HSTORE_COMPACTION_RATIO_KEY = "hbase.hstore.compaction.ratio";
050  public static final String HBASE_HSTORE_COMPACTION_RATIO_OFFPEAK_KEY =
051    "hbase.hstore.compaction.ratio.offpeak";
052  public static final String HBASE_HSTORE_COMPACTION_MIN_KEY_OLD =
053    "hbase.hstore.compactionThreshold";
054  public static final String HBASE_HSTORE_COMPACTION_MIN_KEY = "hbase.hstore.compaction.min";
055  public static final String HBASE_HSTORE_COMPACTION_MIN_SIZE_KEY =
056    "hbase.hstore.compaction.min.size";
057  public static final String HBASE_HSTORE_COMPACTION_MAX_KEY = "hbase.hstore.compaction.max";
058  public static final String HBASE_HSTORE_COMPACTION_MAX_SIZE_KEY =
059    "hbase.hstore.compaction.max.size";
060  public static final String HBASE_HSTORE_COMPACTION_MAX_SIZE_OFFPEAK_KEY =
061    "hbase.hstore.compaction.max.size.offpeak";
062  public static final String HBASE_HSTORE_OFFPEAK_END_HOUR = "hbase.offpeak.end.hour";
063  public static final String HBASE_HSTORE_OFFPEAK_START_HOUR = "hbase.offpeak.start.hour";
064  public static final String HBASE_HSTORE_MIN_LOCALITY_TO_SKIP_MAJOR_COMPACT =
065    "hbase.hstore.min.locality.to.skip.major.compact";
066
067  public static final String HBASE_HFILE_COMPACTION_DISCHARGER_THREAD_COUNT =
068    "hbase.hfile.compaction.discharger.thread.count";
069
070  /*
071   * The epoch time length for the windows we no longer compact
072   */
073  public static final String DATE_TIERED_MAX_AGE_MILLIS_KEY =
074    "hbase.hstore.compaction.date.tiered.max.storefile.age.millis";
075  public static final String DATE_TIERED_INCOMING_WINDOW_MIN_KEY =
076    "hbase.hstore.compaction.date.tiered.incoming.window.min";
077  public static final String COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS_KEY =
078    "hbase.hstore.compaction.date.tiered.window.policy.class";
079  public static final String DATE_TIERED_SINGLE_OUTPUT_FOR_MINOR_COMPACTION_KEY =
080    "hbase.hstore.compaction.date.tiered.single.output.for.minor.compaction";
081
082  private static final Class<
083    ? extends RatioBasedCompactionPolicy> DEFAULT_COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS =
084      ExploringCompactionPolicy.class;
085
086  public static final String DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS_KEY =
087    "hbase.hstore.compaction.date.tiered.window.factory.class";
088
089  private static final Class<
090    ? extends CompactionWindowFactory> DEFAULT_DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS =
091      ExponentialCompactionWindowFactory.class;
092
093  public static final String DATE_TIERED_STORAGE_POLICY_ENABLE_KEY =
094    "hbase.hstore.compaction.date.tiered.storage.policy.enable";
095  public static final String DATE_TIERED_HOT_WINDOW_AGE_MILLIS_KEY =
096    "hbase.hstore.compaction.date.tiered.hot.window.age.millis";
097  public static final String DATE_TIERED_HOT_WINDOW_STORAGE_POLICY_KEY =
098    "hbase.hstore.compaction.date.tiered.hot.window.storage.policy";
099  public static final String DATE_TIERED_WARM_WINDOW_AGE_MILLIS_KEY =
100    "hbase.hstore.compaction.date.tiered.warm.window.age.millis";
101  public static final String DATE_TIERED_WARM_WINDOW_STORAGE_POLICY_KEY =
102    "hbase.hstore.compaction.date.tiered.warm.window.storage.policy";
103  /** Windows older than warm age belong to COLD_WINDOW **/
104  public static final String DATE_TIERED_COLD_WINDOW_STORAGE_POLICY_KEY =
105    "hbase.hstore.compaction.date.tiered.cold.window.storage.policy";
106
107  Configuration conf;
108  StoreConfigInformation storeConfigInfo;
109
110  private final double offPeakCompactionRatio;
111  /** Since all these properties can change online, they are volatile **/
112  private final long maxCompactSize;
113  private final long offPeakMaxCompactSize;
114  private final long minCompactSize;
115  /** This one can be update **/
116  private int minFilesToCompact;
117  private final int maxFilesToCompact;
118  private final double compactionRatio;
119  private final long throttlePoint;
120  private final long majorCompactionPeriod;
121  private final float majorCompactionJitter;
122  private final float minLocalityToForceCompact;
123  private final long dateTieredMaxStoreFileAgeMillis;
124  private final int dateTieredIncomingWindowMin;
125  private final String compactionPolicyForDateTieredWindow;
126  private final boolean dateTieredSingleOutputForMinorCompaction;
127  private final String dateTieredCompactionWindowFactory;
128  private final boolean dateTieredStoragePolicyEnable;
129  private long hotWindowAgeMillis;
130  private long warmWindowAgeMillis;
131  private String hotWindowStoragePolicy;
132  private String warmWindowStoragePolicy;
133  private String coldWindowStoragePolicy;
134
135  CompactionConfiguration(Configuration conf, StoreConfigInformation storeConfigInfo) {
136    this.conf = conf;
137    this.storeConfigInfo = storeConfigInfo;
138
139    maxCompactSize = conf.getLong(HBASE_HSTORE_COMPACTION_MAX_SIZE_KEY, Long.MAX_VALUE);
140    offPeakMaxCompactSize =
141      conf.getLong(HBASE_HSTORE_COMPACTION_MAX_SIZE_OFFPEAK_KEY, maxCompactSize);
142    minCompactSize =
143      conf.getLong(HBASE_HSTORE_COMPACTION_MIN_SIZE_KEY, storeConfigInfo.getMemStoreFlushSize());
144    minFilesToCompact = Math.max(2, conf.getInt(HBASE_HSTORE_COMPACTION_MIN_KEY,
145      conf.getInt(HBASE_HSTORE_COMPACTION_MIN_KEY_OLD, 3)));
146    maxFilesToCompact = conf.getInt(HBASE_HSTORE_COMPACTION_MAX_KEY, 10);
147    compactionRatio = conf.getFloat(HBASE_HSTORE_COMPACTION_RATIO_KEY, 1.2F);
148    offPeakCompactionRatio = conf.getFloat(HBASE_HSTORE_COMPACTION_RATIO_OFFPEAK_KEY, 5.0F);
149
150    throttlePoint = conf.getLong("hbase.regionserver.thread.compaction.throttle",
151      2 * maxFilesToCompact * storeConfigInfo.getMemStoreFlushSize());
152    majorCompactionPeriod =
153      conf.getLong(HConstants.MAJOR_COMPACTION_PERIOD, HConstants.DEFAULT_MAJOR_COMPACTION_PERIOD);
154    majorCompactionJitter =
155      conf.getFloat(HConstants.MAJOR_COMPACTION_JITTER, HConstants.DEFAULT_MAJOR_COMPACTION_JITTER);
156    minLocalityToForceCompact = conf.getFloat(HBASE_HSTORE_MIN_LOCALITY_TO_SKIP_MAJOR_COMPACT, 0f);
157
158    dateTieredMaxStoreFileAgeMillis = conf.getLong(DATE_TIERED_MAX_AGE_MILLIS_KEY, Long.MAX_VALUE);
159    dateTieredIncomingWindowMin = conf.getInt(DATE_TIERED_INCOMING_WINDOW_MIN_KEY, 6);
160    compactionPolicyForDateTieredWindow =
161      conf.get(COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS_KEY,
162        DEFAULT_COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS.getName());
163    dateTieredSingleOutputForMinorCompaction =
164      conf.getBoolean(DATE_TIERED_SINGLE_OUTPUT_FOR_MINOR_COMPACTION_KEY, true);
165    this.dateTieredCompactionWindowFactory =
166      conf.get(DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS_KEY,
167        DEFAULT_DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS.getName());
168    // for Heterogeneous Storage
169    dateTieredStoragePolicyEnable = conf.getBoolean(DATE_TIERED_STORAGE_POLICY_ENABLE_KEY, false);
170    hotWindowAgeMillis = conf.getLong(DATE_TIERED_HOT_WINDOW_AGE_MILLIS_KEY, 86400000L);
171    hotWindowStoragePolicy = conf.get(DATE_TIERED_HOT_WINDOW_STORAGE_POLICY_KEY, "ALL_SSD");
172    warmWindowAgeMillis = conf.getLong(DATE_TIERED_WARM_WINDOW_AGE_MILLIS_KEY, 604800000L);
173    warmWindowStoragePolicy = conf.get(DATE_TIERED_WARM_WINDOW_STORAGE_POLICY_KEY, "ONE_SSD");
174    coldWindowStoragePolicy = conf.get(DATE_TIERED_COLD_WINDOW_STORAGE_POLICY_KEY, "HOT");
175    LOG.info(toString());
176  }
177
178  @Override
179  public String toString() {
180    return String.format(
181      "size [minCompactSize:%s, maxCompactSize:%s, offPeakMaxCompactSize:%s);"
182        + " files [minFilesToCompact:%d, maxFilesToCompact:%d);"
183        + " ratio %f; off-peak ratio %f; throttle point %d;"
184        + " major period %d, major jitter %f, min locality to compact %f;"
185        + " tiered compaction: max_age %d, incoming window min %d,"
186        + " compaction policy for tiered window %s, single output for minor %b,"
187        + " compaction window factory %s," + " region %s columnFamilyName %s",
188      StringUtils.byteDesc(minCompactSize), StringUtils.byteDesc(maxCompactSize),
189      StringUtils.byteDesc(offPeakMaxCompactSize), minFilesToCompact, maxFilesToCompact,
190      compactionRatio, offPeakCompactionRatio, throttlePoint, majorCompactionPeriod,
191      majorCompactionJitter, minLocalityToForceCompact, dateTieredMaxStoreFileAgeMillis,
192      dateTieredIncomingWindowMin, compactionPolicyForDateTieredWindow,
193      dateTieredSingleOutputForMinorCompaction, dateTieredCompactionWindowFactory,
194      RegionInfo.prettyPrint(storeConfigInfo.getRegionInfo().getEncodedName()),
195      storeConfigInfo.getColumnFamilyName());
196  }
197
198  /** Returns lower bound below which compaction is selected without ratio test */
199  public long getMinCompactSize() {
200    return minCompactSize;
201  }
202
203  /** Returns upper bound on file size to be included in minor compactions */
204  public long getMaxCompactSize() {
205    return maxCompactSize;
206  }
207
208  /** Returns lower bound on number of files to be included in minor compactions */
209  public int getMinFilesToCompact() {
210    return minFilesToCompact;
211  }
212
213  /**
214   * Set lower bound on number of files to be included in minor compactions
215   * @param threshold value to set to
216   */
217  public void setMinFilesToCompact(int threshold) {
218    minFilesToCompact = threshold;
219  }
220
221  /** Returns upper bound on number of files to be included in minor compactions */
222  public int getMaxFilesToCompact() {
223    return maxFilesToCompact;
224  }
225
226  /** Returns Ratio used for compaction */
227  public double getCompactionRatio() {
228    return compactionRatio;
229  }
230
231  /** Returns Off peak Ratio used for compaction */
232  public double getCompactionRatioOffPeak() {
233    return offPeakCompactionRatio;
234  }
235
236  /** Returns ThrottlePoint used for classifying small and large compactions */
237  public long getThrottlePoint() {
238    return throttlePoint;
239  }
240
241  /**
242   * @return Major compaction period from compaction. Major compactions are selected periodically
243   *         according to this parameter plus jitter
244   */
245  public long getMajorCompactionPeriod() {
246    return majorCompactionPeriod;
247  }
248
249  /**
250   * @return Major the jitter fraction, the fraction within which the major compaction period is
251   *         randomly chosen from the majorCompactionPeriod in each store.
252   */
253  public float getMajorCompactionJitter() {
254    return majorCompactionJitter;
255  }
256
257  /**
258   * @return Block locality ratio, the ratio at which we will include old regions with a single
259   *         store file for major compaction. Used to improve block locality for regions that
260   *         haven't had writes in a while but are still being read.
261   */
262  public float getMinLocalityToForceCompact() {
263    return minLocalityToForceCompact;
264  }
265
266  public long getOffPeakMaxCompactSize() {
267    return offPeakMaxCompactSize;
268  }
269
270  public long getMaxCompactSize(boolean mayUseOffpeak) {
271    if (mayUseOffpeak) {
272      return getOffPeakMaxCompactSize();
273    } else {
274      return getMaxCompactSize();
275    }
276  }
277
278  public long getDateTieredMaxStoreFileAgeMillis() {
279    return dateTieredMaxStoreFileAgeMillis;
280  }
281
282  public int getDateTieredIncomingWindowMin() {
283    return dateTieredIncomingWindowMin;
284  }
285
286  public String getCompactionPolicyForDateTieredWindow() {
287    return compactionPolicyForDateTieredWindow;
288  }
289
290  public boolean useDateTieredSingleOutputForMinorCompaction() {
291    return dateTieredSingleOutputForMinorCompaction;
292  }
293
294  public String getDateTieredCompactionWindowFactory() {
295    return dateTieredCompactionWindowFactory;
296  }
297
298  public boolean isDateTieredStoragePolicyEnable() {
299    return dateTieredStoragePolicyEnable;
300  }
301
302  public long getHotWindowAgeMillis() {
303    return hotWindowAgeMillis;
304  }
305
306  public long getWarmWindowAgeMillis() {
307    return warmWindowAgeMillis;
308  }
309
310  public String getHotWindowStoragePolicy() {
311    return hotWindowStoragePolicy.trim().toUpperCase();
312  }
313
314  public String getWarmWindowStoragePolicy() {
315    return warmWindowStoragePolicy.trim().toUpperCase();
316  }
317
318  public String getColdWindowStoragePolicy() {
319    return coldWindowStoragePolicy.trim().toUpperCase();
320  }
321}