001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver.compactions;
019
020import org.apache.hadoop.conf.Configuration;
021import org.apache.hadoop.hbase.HConstants;
022import org.apache.hadoop.hbase.client.RegionInfo;
023import org.apache.hadoop.hbase.regionserver.StoreConfigInformation;
024import org.apache.hadoop.util.StringUtils;
025import org.apache.yetus.audience.InterfaceAudience;
026import org.slf4j.Logger;
027import org.slf4j.LoggerFactory;
028
029/**
030 * <p>
031 * Compaction configuration for a particular instance of HStore. Takes into account both global
032 * settings and ones set on the column family/store. Control knobs for default compaction algorithm:
033 * </p>
034 * <p>
035 * maxCompactSize - upper bound on file size to be included in minor compactions minCompactSize -
036 * lower bound below which compaction is selected without ratio test minFilesToCompact - lower bound
037 * on number of files in any minor compaction maxFilesToCompact - upper bound on number of files in
038 * any minor compaction compactionRatio - Ratio used for compaction minLocalityToForceCompact -
039 * Locality threshold for a store file to major compact (HBASE-11195)
040 * </p>
041 * Set parameter as "hbase.hstore.compaction.&lt;attribute&gt;"
042 */
043
044@InterfaceAudience.Private
045public class CompactionConfiguration {
046
047  private static final Logger LOG = LoggerFactory.getLogger(CompactionConfiguration.class);
048
049  public static final String HBASE_HSTORE_COMPACTION_RATIO_KEY = "hbase.hstore.compaction.ratio";
050  public static final String HBASE_HSTORE_COMPACTION_RATIO_OFFPEAK_KEY =
051    "hbase.hstore.compaction.ratio.offpeak";
052  public static final String HBASE_HSTORE_COMPACTION_MIN_KEY = "hbase.hstore.compaction.min";
053  public static final String HBASE_HSTORE_COMPACTION_MIN_SIZE_KEY =
054    "hbase.hstore.compaction.min.size";
055  public static final String HBASE_HSTORE_COMPACTION_MAX_KEY = "hbase.hstore.compaction.max";
056  public static final String HBASE_HSTORE_COMPACTION_MAX_SIZE_KEY =
057    "hbase.hstore.compaction.max.size";
058  public static final String HBASE_HSTORE_COMPACTION_MAX_SIZE_OFFPEAK_KEY =
059    "hbase.hstore.compaction.max.size.offpeak";
060  public static final String HBASE_HSTORE_OFFPEAK_END_HOUR = "hbase.offpeak.end.hour";
061  public static final String HBASE_HSTORE_OFFPEAK_START_HOUR = "hbase.offpeak.start.hour";
062  public static final String HBASE_HSTORE_MIN_LOCALITY_TO_SKIP_MAJOR_COMPACT =
063    "hbase.hstore.min.locality.to.skip.major.compact";
064
065  public static final String HBASE_HFILE_COMPACTION_DISCHARGER_THREAD_COUNT =
066    "hbase.hfile.compaction.discharger.thread.count";
067
068  /*
069   * The epoch time length for the windows we no longer compact
070   */
071  public static final String DATE_TIERED_MAX_AGE_MILLIS_KEY =
072    "hbase.hstore.compaction.date.tiered.max.storefile.age.millis";
073  public static final String DATE_TIERED_INCOMING_WINDOW_MIN_KEY =
074    "hbase.hstore.compaction.date.tiered.incoming.window.min";
075  public static final String COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS_KEY =
076    "hbase.hstore.compaction.date.tiered.window.policy.class";
077  public static final String DATE_TIERED_SINGLE_OUTPUT_FOR_MINOR_COMPACTION_KEY =
078    "hbase.hstore.compaction.date.tiered.single.output.for.minor.compaction";
079
080  private static final Class<
081    ? extends RatioBasedCompactionPolicy> DEFAULT_COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS =
082      ExploringCompactionPolicy.class;
083
084  public static final String DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS_KEY =
085    "hbase.hstore.compaction.date.tiered.window.factory.class";
086
087  private static final Class<
088    ? extends CompactionWindowFactory> DEFAULT_DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS =
089      ExponentialCompactionWindowFactory.class;
090
091  public static final String DATE_TIERED_STORAGE_POLICY_ENABLE_KEY =
092    "hbase.hstore.compaction.date.tiered.storage.policy.enable";
093  public static final String DATE_TIERED_HOT_WINDOW_AGE_MILLIS_KEY =
094    "hbase.hstore.compaction.date.tiered.hot.window.age.millis";
095  public static final String DATE_TIERED_HOT_WINDOW_STORAGE_POLICY_KEY =
096    "hbase.hstore.compaction.date.tiered.hot.window.storage.policy";
097  public static final String DATE_TIERED_WARM_WINDOW_AGE_MILLIS_KEY =
098    "hbase.hstore.compaction.date.tiered.warm.window.age.millis";
099  public static final String DATE_TIERED_WARM_WINDOW_STORAGE_POLICY_KEY =
100    "hbase.hstore.compaction.date.tiered.warm.window.storage.policy";
101  /** Windows older than warm age belong to COLD_WINDOW **/
102  public static final String DATE_TIERED_COLD_WINDOW_STORAGE_POLICY_KEY =
103    "hbase.hstore.compaction.date.tiered.cold.window.storage.policy";
104
105  Configuration conf;
106  StoreConfigInformation storeConfigInfo;
107
108  private final double offPeakCompactionRatio;
109  /** Since all these properties can change online, they are volatile **/
110  private final long maxCompactSize;
111  private final long offPeakMaxCompactSize;
112  private final long minCompactSize;
113  /** This one can be update **/
114  private int minFilesToCompact;
115  private final int maxFilesToCompact;
116  private final double compactionRatio;
117  private final long throttlePoint;
118  private final long majorCompactionPeriod;
119  private final float majorCompactionJitter;
120  private final float minLocalityToForceCompact;
121  private final long dateTieredMaxStoreFileAgeMillis;
122  private final int dateTieredIncomingWindowMin;
123  private final String compactionPolicyForDateTieredWindow;
124  private final boolean dateTieredSingleOutputForMinorCompaction;
125  private final String dateTieredCompactionWindowFactory;
126  private final boolean dateTieredStoragePolicyEnable;
127  private long hotWindowAgeMillis;
128  private long warmWindowAgeMillis;
129  private String hotWindowStoragePolicy;
130  private String warmWindowStoragePolicy;
131  private String coldWindowStoragePolicy;
132
133  CompactionConfiguration(Configuration conf, StoreConfigInformation storeConfigInfo) {
134    this.conf = conf;
135    this.storeConfigInfo = storeConfigInfo;
136
137    maxCompactSize = conf.getLong(HBASE_HSTORE_COMPACTION_MAX_SIZE_KEY, Long.MAX_VALUE);
138    offPeakMaxCompactSize =
139      conf.getLong(HBASE_HSTORE_COMPACTION_MAX_SIZE_OFFPEAK_KEY, maxCompactSize);
140    minCompactSize =
141      conf.getLong(HBASE_HSTORE_COMPACTION_MIN_SIZE_KEY, storeConfigInfo.getMemStoreFlushSize());
142    minFilesToCompact = Math.max(2, conf.getInt(HBASE_HSTORE_COMPACTION_MIN_KEY,
143      /* old name */ conf.getInt("hbase.hstore.compactionThreshold", 3)));
144    maxFilesToCompact = conf.getInt(HBASE_HSTORE_COMPACTION_MAX_KEY, 10);
145    compactionRatio = conf.getFloat(HBASE_HSTORE_COMPACTION_RATIO_KEY, 1.2F);
146    offPeakCompactionRatio = conf.getFloat(HBASE_HSTORE_COMPACTION_RATIO_OFFPEAK_KEY, 5.0F);
147
148    throttlePoint = conf.getLong("hbase.regionserver.thread.compaction.throttle",
149      2 * maxFilesToCompact * storeConfigInfo.getMemStoreFlushSize());
150    majorCompactionPeriod =
151      conf.getLong(HConstants.MAJOR_COMPACTION_PERIOD, HConstants.DEFAULT_MAJOR_COMPACTION_PERIOD);
152    majorCompactionJitter =
153      conf.getFloat(HConstants.MAJOR_COMPACTION_JITTER, HConstants.DEFAULT_MAJOR_COMPACTION_JITTER);
154    minLocalityToForceCompact = conf.getFloat(HBASE_HSTORE_MIN_LOCALITY_TO_SKIP_MAJOR_COMPACT, 0f);
155
156    dateTieredMaxStoreFileAgeMillis = conf.getLong(DATE_TIERED_MAX_AGE_MILLIS_KEY, Long.MAX_VALUE);
157    dateTieredIncomingWindowMin = conf.getInt(DATE_TIERED_INCOMING_WINDOW_MIN_KEY, 6);
158    compactionPolicyForDateTieredWindow =
159      conf.get(COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS_KEY,
160        DEFAULT_COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS.getName());
161    dateTieredSingleOutputForMinorCompaction =
162      conf.getBoolean(DATE_TIERED_SINGLE_OUTPUT_FOR_MINOR_COMPACTION_KEY, true);
163    this.dateTieredCompactionWindowFactory =
164      conf.get(DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS_KEY,
165        DEFAULT_DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS.getName());
166    // for Heterogeneous Storage
167    dateTieredStoragePolicyEnable = conf.getBoolean(DATE_TIERED_STORAGE_POLICY_ENABLE_KEY, false);
168    hotWindowAgeMillis = conf.getLong(DATE_TIERED_HOT_WINDOW_AGE_MILLIS_KEY, 86400000L);
169    hotWindowStoragePolicy = conf.get(DATE_TIERED_HOT_WINDOW_STORAGE_POLICY_KEY, "ALL_SSD");
170    warmWindowAgeMillis = conf.getLong(DATE_TIERED_WARM_WINDOW_AGE_MILLIS_KEY, 604800000L);
171    warmWindowStoragePolicy = conf.get(DATE_TIERED_WARM_WINDOW_STORAGE_POLICY_KEY, "ONE_SSD");
172    coldWindowStoragePolicy = conf.get(DATE_TIERED_COLD_WINDOW_STORAGE_POLICY_KEY, "HOT");
173    LOG.info(toString());
174  }
175
176  @Override
177  public String toString() {
178    return String.format(
179      "size [minCompactSize:%s, maxCompactSize:%s, offPeakMaxCompactSize:%s);"
180        + " files [minFilesToCompact:%d, maxFilesToCompact:%d);"
181        + " ratio %f; off-peak ratio %f; throttle point %d;"
182        + " major period %d, major jitter %f, min locality to compact %f;"
183        + " tiered compaction: max_age %d, incoming window min %d,"
184        + " compaction policy for tiered window %s, single output for minor %b,"
185        + " compaction window factory %s," + " region %s columnFamilyName %s",
186      StringUtils.byteDesc(minCompactSize), StringUtils.byteDesc(maxCompactSize),
187      StringUtils.byteDesc(offPeakMaxCompactSize), minFilesToCompact, maxFilesToCompact,
188      compactionRatio, offPeakCompactionRatio, throttlePoint, majorCompactionPeriod,
189      majorCompactionJitter, minLocalityToForceCompact, dateTieredMaxStoreFileAgeMillis,
190      dateTieredIncomingWindowMin, compactionPolicyForDateTieredWindow,
191      dateTieredSingleOutputForMinorCompaction, dateTieredCompactionWindowFactory,
192      RegionInfo.prettyPrint(storeConfigInfo.getRegionInfo().getEncodedName()),
193      storeConfigInfo.getColumnFamilyName());
194  }
195
196  /** Returns lower bound below which compaction is selected without ratio test */
197  public long getMinCompactSize() {
198    return minCompactSize;
199  }
200
201  /** Returns upper bound on file size to be included in minor compactions */
202  public long getMaxCompactSize() {
203    return maxCompactSize;
204  }
205
206  /** Returns upper bound on number of files to be included in minor compactions */
207  public int getMinFilesToCompact() {
208    return minFilesToCompact;
209  }
210
211  /**
212   * Set upper bound on number of files to be included in minor compactions
213   * @param threshold value to set to
214   */
215  public void setMinFilesToCompact(int threshold) {
216    minFilesToCompact = threshold;
217  }
218
219  /** Returns upper bound on number of files to be included in minor compactions */
220  public int getMaxFilesToCompact() {
221    return maxFilesToCompact;
222  }
223
224  /** Returns Ratio used for compaction */
225  public double getCompactionRatio() {
226    return compactionRatio;
227  }
228
229  /** Returns Off peak Ratio used for compaction */
230  public double getCompactionRatioOffPeak() {
231    return offPeakCompactionRatio;
232  }
233
234  /** Returns ThrottlePoint used for classifying small and large compactions */
235  public long getThrottlePoint() {
236    return throttlePoint;
237  }
238
239  /**
240   * @return Major compaction period from compaction. Major compactions are selected periodically
241   *         according to this parameter plus jitter
242   */
243  public long getMajorCompactionPeriod() {
244    return majorCompactionPeriod;
245  }
246
247  /**
248   * @return Major the jitter fraction, the fraction within which the major compaction period is
249   *         randomly chosen from the majorCompactionPeriod in each store.
250   */
251  public float getMajorCompactionJitter() {
252    return majorCompactionJitter;
253  }
254
255  /**
256   * @return Block locality ratio, the ratio at which we will include old regions with a single
257   *         store file for major compaction. Used to improve block locality for regions that
258   *         haven't had writes in a while but are still being read.
259   */
260  public float getMinLocalityToForceCompact() {
261    return minLocalityToForceCompact;
262  }
263
264  public long getOffPeakMaxCompactSize() {
265    return offPeakMaxCompactSize;
266  }
267
268  public long getMaxCompactSize(boolean mayUseOffpeak) {
269    if (mayUseOffpeak) {
270      return getOffPeakMaxCompactSize();
271    } else {
272      return getMaxCompactSize();
273    }
274  }
275
276  public long getDateTieredMaxStoreFileAgeMillis() {
277    return dateTieredMaxStoreFileAgeMillis;
278  }
279
280  public int getDateTieredIncomingWindowMin() {
281    return dateTieredIncomingWindowMin;
282  }
283
284  public String getCompactionPolicyForDateTieredWindow() {
285    return compactionPolicyForDateTieredWindow;
286  }
287
288  public boolean useDateTieredSingleOutputForMinorCompaction() {
289    return dateTieredSingleOutputForMinorCompaction;
290  }
291
292  public String getDateTieredCompactionWindowFactory() {
293    return dateTieredCompactionWindowFactory;
294  }
295
296  public boolean isDateTieredStoragePolicyEnable() {
297    return dateTieredStoragePolicyEnable;
298  }
299
300  public long getHotWindowAgeMillis() {
301    return hotWindowAgeMillis;
302  }
303
304  public long getWarmWindowAgeMillis() {
305    return warmWindowAgeMillis;
306  }
307
308  public String getHotWindowStoragePolicy() {
309    return hotWindowStoragePolicy.trim().toUpperCase();
310  }
311
312  public String getWarmWindowStoragePolicy() {
313    return warmWindowStoragePolicy.trim().toUpperCase();
314  }
315
316  public String getColdWindowStoragePolicy() {
317    return coldWindowStoragePolicy.trim().toUpperCase();
318  }
319}