001/**
002 *
003 * Licensed to the Apache Software Foundation (ASF) under one
004 * or more contributor license agreements.  See the NOTICE file
005 * distributed with this work for additional information
006 * regarding copyright ownership.  The ASF licenses this file
007 * to you under the Apache License, Version 2.0 (the
008 * "License"); you may not use this file except in compliance
009 * with the License.  You may obtain a copy of the License at
010 *
011 *     http://www.apache.org/licenses/LICENSE-2.0
012 *
013 * Unless required by applicable law or agreed to in writing, software
014 * distributed under the License is distributed on an "AS IS" BASIS,
015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
016 * See the License for the specific language governing permissions and
017 * limitations under the License.
018 */
019
020package org.apache.hadoop.hbase.regionserver.compactions;
021
022import org.apache.hadoop.conf.Configuration;
023import org.apache.hadoop.hbase.HConstants;
024import org.apache.hadoop.hbase.client.RegionInfo;
025import org.apache.hadoop.util.StringUtils;
026import org.apache.yetus.audience.InterfaceAudience;
027import org.slf4j.Logger;
028import org.slf4j.LoggerFactory;
029import org.apache.hadoop.hbase.regionserver.StoreConfigInformation;
030
031/**
032 * <p>
033 * Compaction configuration for a particular instance of HStore.
034 * Takes into account both global settings and ones set on the column family/store.
035 * Control knobs for default compaction algorithm:
036 * </p>
037 * <p>
038 * maxCompactSize - upper bound on file size to be included in minor compactions
039 * minCompactSize - lower bound below which compaction is selected without ratio test
040 * minFilesToCompact - lower bound on number of files in any minor compaction
041 * maxFilesToCompact - upper bound on number of files in any minor compaction
042 * compactionRatio - Ratio used for compaction
043 * minLocalityToForceCompact - Locality threshold for a store file to major compact (HBASE-11195)
044 * </p>
045 * Set parameter as "hbase.hstore.compaction.&lt;attribute&gt;"
046 */
047
048@InterfaceAudience.Private
049public class CompactionConfiguration {
050
051  private static final Logger LOG = LoggerFactory.getLogger(CompactionConfiguration.class);
052
053  public static final String HBASE_HSTORE_COMPACTION_RATIO_KEY = "hbase.hstore.compaction.ratio";
054  public static final String HBASE_HSTORE_COMPACTION_RATIO_OFFPEAK_KEY =
055    "hbase.hstore.compaction.ratio.offpeak";
056  public static final String HBASE_HSTORE_COMPACTION_MIN_KEY = "hbase.hstore.compaction.min";
057  public static final String HBASE_HSTORE_COMPACTION_MIN_SIZE_KEY =
058    "hbase.hstore.compaction.min.size";
059  public static final String HBASE_HSTORE_COMPACTION_MAX_KEY = "hbase.hstore.compaction.max";
060  public static final String HBASE_HSTORE_COMPACTION_MAX_SIZE_KEY =
061    "hbase.hstore.compaction.max.size";
062  public static final String HBASE_HSTORE_COMPACTION_MAX_SIZE_OFFPEAK_KEY =
063      "hbase.hstore.compaction.max.size.offpeak";
064  public static final String HBASE_HSTORE_OFFPEAK_END_HOUR = "hbase.offpeak.end.hour";
065  public static final String HBASE_HSTORE_OFFPEAK_START_HOUR = "hbase.offpeak.start.hour";
066  public static final String HBASE_HSTORE_MIN_LOCALITY_TO_SKIP_MAJOR_COMPACT =
067      "hbase.hstore.min.locality.to.skip.major.compact";
068
069  public static final String HBASE_HFILE_COMPACTION_DISCHARGER_THREAD_COUNT =
070      "hbase.hfile.compaction.discharger.thread.count";
071
072  /*
073   * The epoch time length for the windows we no longer compact
074   */
075  public static final String DATE_TIERED_MAX_AGE_MILLIS_KEY =
076    "hbase.hstore.compaction.date.tiered.max.storefile.age.millis";
077  public static final String DATE_TIERED_INCOMING_WINDOW_MIN_KEY =
078    "hbase.hstore.compaction.date.tiered.incoming.window.min";
079  public static final String COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS_KEY =
080    "hbase.hstore.compaction.date.tiered.window.policy.class";
081  public static final String DATE_TIERED_SINGLE_OUTPUT_FOR_MINOR_COMPACTION_KEY =
082    "hbase.hstore.compaction.date.tiered.single.output.for.minor.compaction";
083
084  private static final Class<? extends RatioBasedCompactionPolicy>
085    DEFAULT_COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS = ExploringCompactionPolicy.class;
086
087  public static final String DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS_KEY =
088    "hbase.hstore.compaction.date.tiered.window.factory.class";
089
090  private static final Class<? extends CompactionWindowFactory>
091    DEFAULT_DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS = ExponentialCompactionWindowFactory.class;
092
093  public static final String DATE_TIERED_STORAGE_POLICY_ENABLE_KEY =
094    "hbase.hstore.compaction.date.tiered.storage.policy.enable";
095  public static final String DATE_TIERED_HOT_WINDOW_AGE_MILLIS_KEY =
096    "hbase.hstore.compaction.date.tiered.hot.window.age.millis";
097  public static final String DATE_TIERED_HOT_WINDOW_STORAGE_POLICY_KEY =
098    "hbase.hstore.compaction.date.tiered.hot.window.storage.policy";
099  public static final String DATE_TIERED_WARM_WINDOW_AGE_MILLIS_KEY =
100    "hbase.hstore.compaction.date.tiered.warm.window.age.millis";
101  public static final String DATE_TIERED_WARM_WINDOW_STORAGE_POLICY_KEY =
102    "hbase.hstore.compaction.date.tiered.warm.window.storage.policy";
103  /** Windows older than warm age belong to COLD_WINDOW **/
104  public static final String DATE_TIERED_COLD_WINDOW_STORAGE_POLICY_KEY =
105    "hbase.hstore.compaction.date.tiered.cold.window.storage.policy";
106
107  Configuration conf;
108  StoreConfigInformation storeConfigInfo;
109
110  private final double offPeakCompactionRatio;
111  /** Since all these properties can change online, they are volatile **/
112  private final long maxCompactSize;
113  private final long offPeakMaxCompactSize;
114  private final long minCompactSize;
115  /** This one can be update **/
116  private int minFilesToCompact;
117  private final int maxFilesToCompact;
118  private final double compactionRatio;
119  private final long throttlePoint;
120  private final long majorCompactionPeriod;
121  private final float majorCompactionJitter;
122  private final float minLocalityToForceCompact;
123  private final long dateTieredMaxStoreFileAgeMillis;
124  private final int dateTieredIncomingWindowMin;
125  private final String compactionPolicyForDateTieredWindow;
126  private final boolean dateTieredSingleOutputForMinorCompaction;
127  private final String dateTieredCompactionWindowFactory;
128  private final boolean dateTieredStoragePolicyEnable;
129  private long hotWindowAgeMillis;
130  private long warmWindowAgeMillis;
131  private String hotWindowStoragePolicy;
132  private String warmWindowStoragePolicy;
133  private String coldWindowStoragePolicy;
134
135  CompactionConfiguration(Configuration conf, StoreConfigInformation storeConfigInfo) {
136    this.conf = conf;
137    this.storeConfigInfo = storeConfigInfo;
138
139    maxCompactSize = conf.getLong(HBASE_HSTORE_COMPACTION_MAX_SIZE_KEY, Long.MAX_VALUE);
140    offPeakMaxCompactSize = conf.getLong(HBASE_HSTORE_COMPACTION_MAX_SIZE_OFFPEAK_KEY,
141      maxCompactSize);
142    minCompactSize = conf.getLong(HBASE_HSTORE_COMPACTION_MIN_SIZE_KEY,
143        storeConfigInfo.getMemStoreFlushSize());
144    minFilesToCompact = Math.max(2, conf.getInt(HBASE_HSTORE_COMPACTION_MIN_KEY,
145          /*old name*/ conf.getInt("hbase.hstore.compactionThreshold", 3)));
146    maxFilesToCompact = conf.getInt(HBASE_HSTORE_COMPACTION_MAX_KEY, 10);
147    compactionRatio = conf.getFloat(HBASE_HSTORE_COMPACTION_RATIO_KEY, 1.2F);
148    offPeakCompactionRatio = conf.getFloat(HBASE_HSTORE_COMPACTION_RATIO_OFFPEAK_KEY, 5.0F);
149
150    throttlePoint = conf.getLong("hbase.regionserver.thread.compaction.throttle",
151          2 * maxFilesToCompact * storeConfigInfo.getMemStoreFlushSize());
152    majorCompactionPeriod = conf.getLong(HConstants.MAJOR_COMPACTION_PERIOD,
153                                         HConstants.DEFAULT_MAJOR_COMPACTION_PERIOD);
154    majorCompactionJitter = conf.getFloat(HConstants.MAJOR_COMPACTION_JITTER,
155                                          HConstants.DEFAULT_MAJOR_COMPACTION_JITTER);
156    minLocalityToForceCompact = conf.getFloat(HBASE_HSTORE_MIN_LOCALITY_TO_SKIP_MAJOR_COMPACT, 0f);
157
158    dateTieredMaxStoreFileAgeMillis = conf.getLong(DATE_TIERED_MAX_AGE_MILLIS_KEY, Long.MAX_VALUE);
159    dateTieredIncomingWindowMin = conf.getInt(DATE_TIERED_INCOMING_WINDOW_MIN_KEY, 6);
160    compactionPolicyForDateTieredWindow = conf.get(
161      COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS_KEY,
162      DEFAULT_COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS.getName());
163    dateTieredSingleOutputForMinorCompaction = conf
164        .getBoolean(DATE_TIERED_SINGLE_OUTPUT_FOR_MINOR_COMPACTION_KEY, true);
165    this.dateTieredCompactionWindowFactory = conf.get(
166      DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS_KEY,
167      DEFAULT_DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS.getName());
168    // for Heterogeneous Storage
169    dateTieredStoragePolicyEnable = conf.getBoolean(DATE_TIERED_STORAGE_POLICY_ENABLE_KEY, false);
170    hotWindowAgeMillis = conf.getLong(DATE_TIERED_HOT_WINDOW_AGE_MILLIS_KEY, 86400000L);
171    hotWindowStoragePolicy = conf.get(DATE_TIERED_HOT_WINDOW_STORAGE_POLICY_KEY, "ALL_SSD");
172    warmWindowAgeMillis = conf.getLong(DATE_TIERED_WARM_WINDOW_AGE_MILLIS_KEY, 604800000L);
173    warmWindowStoragePolicy = conf.get(DATE_TIERED_WARM_WINDOW_STORAGE_POLICY_KEY, "ONE_SSD");
174    coldWindowStoragePolicy = conf.get(DATE_TIERED_COLD_WINDOW_STORAGE_POLICY_KEY, "HOT");
175    LOG.info(toString());
176  }
177
178  @Override
179  public String toString() {
180    return String.format(
181      "size [minCompactSize:%s, maxCompactSize:%s, offPeakMaxCompactSize:%s);"
182      + " files [minFilesToCompact:%d, maxFilesToCompact:%d);"
183      + " ratio %f; off-peak ratio %f; throttle point %d;"
184      + " major period %d, major jitter %f, min locality to compact %f;"
185      + " tiered compaction: max_age %d, incoming window min %d,"
186      + " compaction policy for tiered window %s, single output for minor %b,"
187      + " compaction window factory %s,"
188      + " region %s columnFamilyName %s",
189      StringUtils.byteDesc(minCompactSize),
190      StringUtils.byteDesc(maxCompactSize),
191      StringUtils.byteDesc(offPeakMaxCompactSize),
192      minFilesToCompact,
193      maxFilesToCompact,
194      compactionRatio,
195      offPeakCompactionRatio,
196      throttlePoint,
197      majorCompactionPeriod,
198      majorCompactionJitter,
199      minLocalityToForceCompact,
200      dateTieredMaxStoreFileAgeMillis,
201      dateTieredIncomingWindowMin,
202      compactionPolicyForDateTieredWindow,
203      dateTieredSingleOutputForMinorCompaction,
204      dateTieredCompactionWindowFactory,
205      RegionInfo.prettyPrint(storeConfigInfo.getRegionInfo().getEncodedName()),
206      storeConfigInfo.getColumnFamilyName()
207      );
208  }
209
210  /**
211   * @return lower bound below which compaction is selected without ratio test
212   */
213  public long getMinCompactSize() {
214    return minCompactSize;
215  }
216
217  /**
218   * @return upper bound on file size to be included in minor compactions
219   */
220  public long getMaxCompactSize() {
221    return maxCompactSize;
222  }
223
224  /**
225   * @return upper bound on number of files to be included in minor compactions
226   */
227  public int getMinFilesToCompact() {
228    return minFilesToCompact;
229  }
230
231  /**
232   * Set upper bound on number of files to be included in minor compactions
233   * @param threshold value to set to
234   */
235  public void setMinFilesToCompact(int threshold) {
236    minFilesToCompact = threshold;
237  }
238
239  /**
240   * @return upper bound on number of files to be included in minor compactions
241   */
242  public int getMaxFilesToCompact() {
243    return maxFilesToCompact;
244  }
245
246  /**
247   * @return Ratio used for compaction
248   */
249  public double getCompactionRatio() {
250    return compactionRatio;
251  }
252
253  /**
254   * @return Off peak Ratio used for compaction
255   */
256  public double getCompactionRatioOffPeak() {
257    return offPeakCompactionRatio;
258  }
259
260  /**
261   * @return ThrottlePoint used for classifying small and large compactions
262   */
263  public long getThrottlePoint() {
264    return throttlePoint;
265  }
266
267  /**
268   * @return Major compaction period from compaction.
269   *   Major compactions are selected periodically according to this parameter plus jitter
270   */
271  public long getMajorCompactionPeriod() {
272    return majorCompactionPeriod;
273  }
274
275  /**
276   * @return Major the jitter fraction, the fraction within which the major compaction
277   *    period is randomly chosen from the majorCompactionPeriod in each store.
278   */
279  public float getMajorCompactionJitter() {
280    return majorCompactionJitter;
281  }
282
283  /**
284   * @return Block locality ratio, the ratio at which we will include old regions with a single
285   *   store file for major compaction.  Used to improve block locality for regions that
286   *   haven't had writes in a while but are still being read.
287   */
288  public float getMinLocalityToForceCompact() {
289    return minLocalityToForceCompact;
290  }
291
292  public long getOffPeakMaxCompactSize() {
293    return offPeakMaxCompactSize;
294  }
295
296  public long getMaxCompactSize(boolean mayUseOffpeak) {
297    if (mayUseOffpeak) {
298      return getOffPeakMaxCompactSize();
299    } else {
300      return getMaxCompactSize();
301    }
302  }
303
304  public long getDateTieredMaxStoreFileAgeMillis() {
305    return dateTieredMaxStoreFileAgeMillis;
306  }
307
308  public int getDateTieredIncomingWindowMin() {
309    return dateTieredIncomingWindowMin;
310  }
311
312  public String getCompactionPolicyForDateTieredWindow() {
313    return compactionPolicyForDateTieredWindow;
314  }
315
316  public boolean useDateTieredSingleOutputForMinorCompaction() {
317    return dateTieredSingleOutputForMinorCompaction;
318  }
319
320  public String getDateTieredCompactionWindowFactory() {
321    return dateTieredCompactionWindowFactory;
322  }
323
324  public boolean isDateTieredStoragePolicyEnable() {
325    return dateTieredStoragePolicyEnable;
326  }
327
328  public long getHotWindowAgeMillis() {
329    return hotWindowAgeMillis;
330  }
331
332  public long getWarmWindowAgeMillis() {
333    return warmWindowAgeMillis;
334  }
335
336  public String getHotWindowStoragePolicy() {
337    return hotWindowStoragePolicy.trim().toUpperCase();
338  }
339
340  public String getWarmWindowStoragePolicy() {
341    return warmWindowStoragePolicy.trim().toUpperCase();
342  }
343
344  public String getColdWindowStoragePolicy() {
345    return coldWindowStoragePolicy.trim().toUpperCase();
346  }
347}