001/**
002 *
003 * Licensed to the Apache Software Foundation (ASF) under one
004 * or more contributor license agreements.  See the NOTICE file
005 * distributed with this work for additional information
006 * regarding copyright ownership.  The ASF licenses this file
007 * to you under the Apache License, Version 2.0 (the
008 * "License"); you may not use this file except in compliance
009 * with the License.  You may obtain a copy of the License at
010 *
011 *     http://www.apache.org/licenses/LICENSE-2.0
012 *
013 * Unless required by applicable law or agreed to in writing, software
014 * distributed under the License is distributed on an "AS IS" BASIS,
015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
016 * See the License for the specific language governing permissions and
017 * limitations under the License.
018 */
019
020package org.apache.hadoop.hbase.regionserver.compactions;
021
022import org.apache.hadoop.conf.Configuration;
023import org.apache.hadoop.hbase.HConstants;
024import org.apache.hadoop.hbase.client.RegionInfo;
025import org.apache.hadoop.util.StringUtils;
026import org.apache.yetus.audience.InterfaceAudience;
027import org.slf4j.Logger;
028import org.slf4j.LoggerFactory;
029import org.apache.hadoop.hbase.regionserver.StoreConfigInformation;
030
031/**
032 * <p>
033 * Compaction configuration for a particular instance of HStore.
034 * Takes into account both global settings and ones set on the column family/store.
035 * Control knobs for default compaction algorithm:
036 * </p>
037 * <p>
038 * maxCompactSize - upper bound on file size to be included in minor compactions
039 * minCompactSize - lower bound below which compaction is selected without ratio test
040 * minFilesToCompact - lower bound on number of files in any minor compaction
041 * maxFilesToCompact - upper bound on number of files in any minor compaction
042 * compactionRatio - Ratio used for compaction
043 * minLocalityToForceCompact - Locality threshold for a store file to major compact (HBASE-11195)
044 * </p>
045 * Set parameter as "hbase.hstore.compaction.&lt;attribute&gt;"
046 */
047
048@InterfaceAudience.Private
049public class CompactionConfiguration {
050
051  private static final Logger LOG = LoggerFactory.getLogger(CompactionConfiguration.class);
052
053  public static final String HBASE_HSTORE_COMPACTION_RATIO_KEY = "hbase.hstore.compaction.ratio";
054  public static final String HBASE_HSTORE_COMPACTION_RATIO_OFFPEAK_KEY =
055    "hbase.hstore.compaction.ratio.offpeak";
056  public static final String HBASE_HSTORE_COMPACTION_MIN_KEY = "hbase.hstore.compaction.min";
057  public static final String HBASE_HSTORE_COMPACTION_MIN_SIZE_KEY =
058    "hbase.hstore.compaction.min.size";
059  public static final String HBASE_HSTORE_COMPACTION_MAX_KEY = "hbase.hstore.compaction.max";
060  public static final String HBASE_HSTORE_COMPACTION_MAX_SIZE_KEY =
061    "hbase.hstore.compaction.max.size";
062  public static final String HBASE_HSTORE_COMPACTION_MAX_SIZE_OFFPEAK_KEY =
063      "hbase.hstore.compaction.max.size.offpeak";
064  public static final String HBASE_HSTORE_OFFPEAK_END_HOUR = "hbase.offpeak.end.hour";
065  public static final String HBASE_HSTORE_OFFPEAK_START_HOUR = "hbase.offpeak.start.hour";
066  public static final String HBASE_HSTORE_MIN_LOCALITY_TO_SKIP_MAJOR_COMPACT =
067      "hbase.hstore.min.locality.to.skip.major.compact";
068
069  public static final String HBASE_HFILE_COMPACTION_DISCHARGER_THREAD_COUNT =
070      "hbase.hfile.compaction.discharger.thread.count";
071
072  /*
073   * The epoch time length for the windows we no longer compact
074   */
075  public static final String DATE_TIERED_MAX_AGE_MILLIS_KEY =
076    "hbase.hstore.compaction.date.tiered.max.storefile.age.millis";
077  public static final String DATE_TIERED_INCOMING_WINDOW_MIN_KEY =
078    "hbase.hstore.compaction.date.tiered.incoming.window.min";
079  public static final String COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS_KEY =
080    "hbase.hstore.compaction.date.tiered.window.policy.class";
081  public static final String DATE_TIERED_SINGLE_OUTPUT_FOR_MINOR_COMPACTION_KEY =
082    "hbase.hstore.compaction.date.tiered.single.output.for.minor.compaction";
083
084  private static final Class<? extends RatioBasedCompactionPolicy>
085    DEFAULT_COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS = ExploringCompactionPolicy.class;
086
087  public static final String DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS_KEY =
088    "hbase.hstore.compaction.date.tiered.window.factory.class";
089
090  private static final Class<? extends CompactionWindowFactory>
091    DEFAULT_DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS = ExponentialCompactionWindowFactory.class;
092
093  Configuration conf;
094  StoreConfigInformation storeConfigInfo;
095
096  private final double offPeakCompactionRatio;
097  /** Since all these properties can change online, they are volatile **/
098  private final long maxCompactSize;
099  private final long offPeakMaxCompactSize;
100  private final long minCompactSize;
101  /** This one can be update **/
102  private int minFilesToCompact;
103  private final int maxFilesToCompact;
104  private final double compactionRatio;
105  private final long throttlePoint;
106  private final long majorCompactionPeriod;
107  private final float majorCompactionJitter;
108  private final float minLocalityToForceCompact;
109  private final long dateTieredMaxStoreFileAgeMillis;
110  private final int dateTieredIncomingWindowMin;
111  private final String compactionPolicyForDateTieredWindow;
112  private final boolean dateTieredSingleOutputForMinorCompaction;
113  private final String dateTieredCompactionWindowFactory;
114
115  CompactionConfiguration(Configuration conf, StoreConfigInformation storeConfigInfo) {
116    this.conf = conf;
117    this.storeConfigInfo = storeConfigInfo;
118
119    maxCompactSize = conf.getLong(HBASE_HSTORE_COMPACTION_MAX_SIZE_KEY, Long.MAX_VALUE);
120    offPeakMaxCompactSize = conf.getLong(HBASE_HSTORE_COMPACTION_MAX_SIZE_OFFPEAK_KEY,
121      maxCompactSize);
122    minCompactSize = conf.getLong(HBASE_HSTORE_COMPACTION_MIN_SIZE_KEY,
123        storeConfigInfo.getMemStoreFlushSize());
124    minFilesToCompact = Math.max(2, conf.getInt(HBASE_HSTORE_COMPACTION_MIN_KEY,
125          /*old name*/ conf.getInt("hbase.hstore.compactionThreshold", 3)));
126    maxFilesToCompact = conf.getInt(HBASE_HSTORE_COMPACTION_MAX_KEY, 10);
127    compactionRatio = conf.getFloat(HBASE_HSTORE_COMPACTION_RATIO_KEY, 1.2F);
128    offPeakCompactionRatio = conf.getFloat(HBASE_HSTORE_COMPACTION_RATIO_OFFPEAK_KEY, 5.0F);
129
130    throttlePoint = conf.getLong("hbase.regionserver.thread.compaction.throttle",
131          2 * maxFilesToCompact * storeConfigInfo.getMemStoreFlushSize());
132    majorCompactionPeriod = conf.getLong(HConstants.MAJOR_COMPACTION_PERIOD,
133                                         HConstants.DEFAULT_MAJOR_COMPACTION_PERIOD);
134    majorCompactionJitter = conf.getFloat(HConstants.MAJOR_COMPACTION_JITTER,
135                                          HConstants.DEFAULT_MAJOR_COMPACTION_JITTER);
136    minLocalityToForceCompact = conf.getFloat(HBASE_HSTORE_MIN_LOCALITY_TO_SKIP_MAJOR_COMPACT, 0f);
137
138    dateTieredMaxStoreFileAgeMillis = conf.getLong(DATE_TIERED_MAX_AGE_MILLIS_KEY, Long.MAX_VALUE);
139    dateTieredIncomingWindowMin = conf.getInt(DATE_TIERED_INCOMING_WINDOW_MIN_KEY, 6);
140    compactionPolicyForDateTieredWindow = conf.get(
141      COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS_KEY,
142      DEFAULT_COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS.getName());
143    dateTieredSingleOutputForMinorCompaction = conf
144        .getBoolean(DATE_TIERED_SINGLE_OUTPUT_FOR_MINOR_COMPACTION_KEY, true);
145    this.dateTieredCompactionWindowFactory = conf.get(
146      DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS_KEY,
147      DEFAULT_DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS.getName());
148    LOG.info(toString());
149  }
150
151  @Override
152  public String toString() {
153    return String.format(
154      "size [%s, %s, %s); files [%d, %d); ratio %f; off-peak ratio %f; throttle point %d;"
155      + " major period %d, major jitter %f, min locality to compact %f;"
156      + " tiered compaction: max_age %d, incoming window min %d,"
157      + " compaction policy for tiered window %s, single output for minor %b,"
158      + " compaction window factory %s,"
159      + " region %s columnFamilyName %s",
160      StringUtils.byteDesc(minCompactSize),
161      StringUtils.byteDesc(maxCompactSize),
162      StringUtils.byteDesc(offPeakMaxCompactSize),
163      minFilesToCompact,
164      maxFilesToCompact,
165      compactionRatio,
166      offPeakCompactionRatio,
167      throttlePoint,
168      majorCompactionPeriod,
169      majorCompactionJitter,
170      minLocalityToForceCompact,
171      dateTieredMaxStoreFileAgeMillis,
172      dateTieredIncomingWindowMin,
173      compactionPolicyForDateTieredWindow,
174      dateTieredSingleOutputForMinorCompaction,
175      dateTieredCompactionWindowFactory,
176      RegionInfo.prettyPrint(storeConfigInfo.getRegionInfo().getEncodedName()),
177      storeConfigInfo.getColumnFamilyName()
178      );
179  }
180
181  /**
182   * @return lower bound below which compaction is selected without ratio test
183   */
184  public long getMinCompactSize() {
185    return minCompactSize;
186  }
187
188  /**
189   * @return upper bound on file size to be included in minor compactions
190   */
191  public long getMaxCompactSize() {
192    return maxCompactSize;
193  }
194
195  /**
196   * @return upper bound on number of files to be included in minor compactions
197   */
198  public int getMinFilesToCompact() {
199    return minFilesToCompact;
200  }
201
202  /**
203   * Set upper bound on number of files to be included in minor compactions
204   * @param threshold value to set to
205   */
206  public void setMinFilesToCompact(int threshold) {
207    minFilesToCompact = threshold;
208  }
209
210  /**
211   * @return upper bound on number of files to be included in minor compactions
212   */
213  public int getMaxFilesToCompact() {
214    return maxFilesToCompact;
215  }
216
217  /**
218   * @return Ratio used for compaction
219   */
220  public double getCompactionRatio() {
221    return compactionRatio;
222  }
223
224  /**
225   * @return Off peak Ratio used for compaction
226   */
227  public double getCompactionRatioOffPeak() {
228    return offPeakCompactionRatio;
229  }
230
231  /**
232   * @return ThrottlePoint used for classifying small and large compactions
233   */
234  public long getThrottlePoint() {
235    return throttlePoint;
236  }
237
238  /**
239   * @return Major compaction period from compaction.
240   *   Major compactions are selected periodically according to this parameter plus jitter
241   */
242  public long getMajorCompactionPeriod() {
243    return majorCompactionPeriod;
244  }
245
246  /**
247   * @return Major the jitter fraction, the fraction within which the major compaction
248   *    period is randomly chosen from the majorCompactionPeriod in each store.
249   */
250  public float getMajorCompactionJitter() {
251    return majorCompactionJitter;
252  }
253
254  /**
255   * @return Block locality ratio, the ratio at which we will include old regions with a single
256   *   store file for major compaction.  Used to improve block locality for regions that
257   *   haven't had writes in a while but are still being read.
258   */
259  public float getMinLocalityToForceCompact() {
260    return minLocalityToForceCompact;
261  }
262
263  public long getOffPeakMaxCompactSize() {
264    return offPeakMaxCompactSize;
265  }
266
267  public long getMaxCompactSize(boolean mayUseOffpeak) {
268    if (mayUseOffpeak) {
269      return getOffPeakMaxCompactSize();
270    } else {
271      return getMaxCompactSize();
272    }
273  }
274
275  public long getDateTieredMaxStoreFileAgeMillis() {
276    return dateTieredMaxStoreFileAgeMillis;
277  }
278
279  public int getDateTieredIncomingWindowMin() {
280    return dateTieredIncomingWindowMin;
281  }
282
283  public String getCompactionPolicyForDateTieredWindow() {
284    return compactionPolicyForDateTieredWindow;
285  }
286
287  public boolean useDateTieredSingleOutputForMinorCompaction() {
288    return dateTieredSingleOutputForMinorCompaction;
289  }
290
291  public String getDateTieredCompactionWindowFactory() {
292    return dateTieredCompactionWindowFactory;
293  }
294}