001/**
002 *
003 * Licensed to the Apache Software Foundation (ASF) under one
004 * or more contributor license agreements.  See the NOTICE file
005 * distributed with this work for additional information
006 * regarding copyright ownership.  The ASF licenses this file
007 * to you under the Apache License, Version 2.0 (the
008 * "License"); you may not use this file except in compliance
009 * with the License.  You may obtain a copy of the License at
010 *
011 *     http://www.apache.org/licenses/LICENSE-2.0
012 *
013 * Unless required by applicable law or agreed to in writing, software
014 * distributed under the License is distributed on an "AS IS" BASIS,
015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
016 * See the License for the specific language governing permissions and
017 * limitations under the License.
018 */
019
020package org.apache.hadoop.hbase.regionserver.compactions;
021
022import org.apache.hadoop.conf.Configuration;
023import org.apache.hadoop.hbase.HConstants;
024import org.apache.hadoop.util.StringUtils;
025import org.apache.yetus.audience.InterfaceAudience;
026import org.slf4j.Logger;
027import org.slf4j.LoggerFactory;
028import org.apache.hadoop.hbase.regionserver.StoreConfigInformation;
029
030/**
031 * <p>
032 * Compaction configuration for a particular instance of HStore.
033 * Takes into account both global settings and ones set on the column family/store.
034 * Control knobs for default compaction algorithm:
035 * </p>
036 * <p>
037 * maxCompactSize - upper bound on file size to be included in minor compactions
038 * minCompactSize - lower bound below which compaction is selected without ratio test
039 * minFilesToCompact - lower bound on number of files in any minor compaction
040 * maxFilesToCompact - upper bound on number of files in any minor compaction
041 * compactionRatio - Ratio used for compaction
042 * minLocalityToForceCompact - Locality threshold for a store file to major compact (HBASE-11195)
043 * </p>
044 * Set parameter as "hbase.hstore.compaction.&lt;attribute&gt;"
045 */
046
047@InterfaceAudience.Private
048public class CompactionConfiguration {
049
050  private static final Logger LOG = LoggerFactory.getLogger(CompactionConfiguration.class);
051
052  public static final String HBASE_HSTORE_COMPACTION_RATIO_KEY = "hbase.hstore.compaction.ratio";
053  public static final String HBASE_HSTORE_COMPACTION_RATIO_OFFPEAK_KEY =
054    "hbase.hstore.compaction.ratio.offpeak";
055  public static final String HBASE_HSTORE_COMPACTION_MIN_KEY = "hbase.hstore.compaction.min";
056  public static final String HBASE_HSTORE_COMPACTION_MIN_SIZE_KEY =
057    "hbase.hstore.compaction.min.size";
058  public static final String HBASE_HSTORE_COMPACTION_MAX_KEY = "hbase.hstore.compaction.max";
059  public static final String HBASE_HSTORE_COMPACTION_MAX_SIZE_KEY =
060    "hbase.hstore.compaction.max.size";
061  public static final String HBASE_HSTORE_COMPACTION_MAX_SIZE_OFFPEAK_KEY =
062      "hbase.hstore.compaction.max.size.offpeak";
063  public static final String HBASE_HSTORE_OFFPEAK_END_HOUR = "hbase.offpeak.end.hour";
064  public static final String HBASE_HSTORE_OFFPEAK_START_HOUR = "hbase.offpeak.start.hour";
065  public static final String HBASE_HSTORE_MIN_LOCALITY_TO_SKIP_MAJOR_COMPACT =
066      "hbase.hstore.min.locality.to.skip.major.compact";
067
068  public static final String HBASE_HFILE_COMPACTION_DISCHARGER_THREAD_COUNT =
069      "hbase.hfile.compaction.discharger.thread.count";
070
071  /*
072   * The epoch time length for the windows we no longer compact
073   */
074  public static final String DATE_TIERED_MAX_AGE_MILLIS_KEY =
075    "hbase.hstore.compaction.date.tiered.max.storefile.age.millis";
076  public static final String DATE_TIERED_INCOMING_WINDOW_MIN_KEY =
077    "hbase.hstore.compaction.date.tiered.incoming.window.min";
078  public static final String COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS_KEY =
079    "hbase.hstore.compaction.date.tiered.window.policy.class";
080  public static final String DATE_TIERED_SINGLE_OUTPUT_FOR_MINOR_COMPACTION_KEY =
081    "hbase.hstore.compaction.date.tiered.single.output.for.minor.compaction";
082
083  private static final Class<? extends RatioBasedCompactionPolicy>
084    DEFAULT_COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS = ExploringCompactionPolicy.class;
085
086  public static final String DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS_KEY =
087    "hbase.hstore.compaction.date.tiered.window.factory.class";
088
089  private static final Class<? extends CompactionWindowFactory>
090    DEFAULT_DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS = ExponentialCompactionWindowFactory.class;
091
092  Configuration conf;
093  StoreConfigInformation storeConfigInfo;
094
095  private final double offPeakCompactionRatio;
096  /** Since all these properties can change online, they are volatile **/
097  private final long maxCompactSize;
098  private final long offPeakMaxCompactSize;
099  private final long minCompactSize;
100  /** This one can be update **/
101  private int minFilesToCompact;
102  private final int maxFilesToCompact;
103  private final double compactionRatio;
104  private final long throttlePoint;
105  private final long majorCompactionPeriod;
106  private final float majorCompactionJitter;
107  private final float minLocalityToForceCompact;
108  private final long dateTieredMaxStoreFileAgeMillis;
109  private final int dateTieredIncomingWindowMin;
110  private final String compactionPolicyForDateTieredWindow;
111  private final boolean dateTieredSingleOutputForMinorCompaction;
112  private final String dateTieredCompactionWindowFactory;
113
114  CompactionConfiguration(Configuration conf, StoreConfigInformation storeConfigInfo) {
115    this.conf = conf;
116    this.storeConfigInfo = storeConfigInfo;
117
118    maxCompactSize = conf.getLong(HBASE_HSTORE_COMPACTION_MAX_SIZE_KEY, Long.MAX_VALUE);
119    offPeakMaxCompactSize = conf.getLong(HBASE_HSTORE_COMPACTION_MAX_SIZE_OFFPEAK_KEY,
120      maxCompactSize);
121    minCompactSize = conf.getLong(HBASE_HSTORE_COMPACTION_MIN_SIZE_KEY,
122        storeConfigInfo.getMemStoreFlushSize());
123    minFilesToCompact = Math.max(2, conf.getInt(HBASE_HSTORE_COMPACTION_MIN_KEY,
124          /*old name*/ conf.getInt("hbase.hstore.compactionThreshold", 3)));
125    maxFilesToCompact = conf.getInt(HBASE_HSTORE_COMPACTION_MAX_KEY, 10);
126    compactionRatio = conf.getFloat(HBASE_HSTORE_COMPACTION_RATIO_KEY, 1.2F);
127    offPeakCompactionRatio = conf.getFloat(HBASE_HSTORE_COMPACTION_RATIO_OFFPEAK_KEY, 5.0F);
128
129    throttlePoint = conf.getLong("hbase.regionserver.thread.compaction.throttle",
130          2 * maxFilesToCompact * storeConfigInfo.getMemStoreFlushSize());
131    majorCompactionPeriod = conf.getLong(HConstants.MAJOR_COMPACTION_PERIOD,
132                                         HConstants.DEFAULT_MAJOR_COMPACTION_PERIOD);
133    majorCompactionJitter = conf.getFloat(HConstants.MAJOR_COMPACTION_JITTER,
134                                          HConstants.DEFAULT_MAJOR_COMPACTION_JITTER);
135    minLocalityToForceCompact = conf.getFloat(HBASE_HSTORE_MIN_LOCALITY_TO_SKIP_MAJOR_COMPACT, 0f);
136
137    dateTieredMaxStoreFileAgeMillis = conf.getLong(DATE_TIERED_MAX_AGE_MILLIS_KEY, Long.MAX_VALUE);
138    dateTieredIncomingWindowMin = conf.getInt(DATE_TIERED_INCOMING_WINDOW_MIN_KEY, 6);
139    compactionPolicyForDateTieredWindow = conf.get(
140      COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS_KEY,
141      DEFAULT_COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS.getName());
142    dateTieredSingleOutputForMinorCompaction = conf
143        .getBoolean(DATE_TIERED_SINGLE_OUTPUT_FOR_MINOR_COMPACTION_KEY, true);
144    this.dateTieredCompactionWindowFactory = conf.get(
145      DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS_KEY,
146      DEFAULT_DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS.getName());
147    LOG.info(toString());
148  }
149
150  @Override
151  public String toString() {
152    return String.format(
153      "size [%s, %s, %s); files [%d, %d); ratio %f; off-peak ratio %f; throttle point %d;"
154      + " major period %d, major jitter %f, min locality to compact %f;"
155      + " tiered compaction: max_age %d, incoming window min %d,"
156      + " compaction policy for tiered window %s, single output for minor %b,"
157      + " compaction window factory %s",
158      StringUtils.byteDesc(minCompactSize),
159      StringUtils.byteDesc(maxCompactSize),
160      StringUtils.byteDesc(offPeakMaxCompactSize),
161      minFilesToCompact,
162      maxFilesToCompact,
163      compactionRatio,
164      offPeakCompactionRatio,
165      throttlePoint,
166      majorCompactionPeriod,
167      majorCompactionJitter,
168      minLocalityToForceCompact,
169      dateTieredMaxStoreFileAgeMillis,
170      dateTieredIncomingWindowMin,
171      compactionPolicyForDateTieredWindow,
172      dateTieredSingleOutputForMinorCompaction,
173      dateTieredCompactionWindowFactory
174      );
175  }
176
177  /**
178   * @return lower bound below which compaction is selected without ratio test
179   */
180  public long getMinCompactSize() {
181    return minCompactSize;
182  }
183
184  /**
185   * @return upper bound on file size to be included in minor compactions
186   */
187  public long getMaxCompactSize() {
188    return maxCompactSize;
189  }
190
191  /**
192   * @return upper bound on number of files to be included in minor compactions
193   */
194  public int getMinFilesToCompact() {
195    return minFilesToCompact;
196  }
197
198  /**
199   * Set upper bound on number of files to be included in minor compactions
200   * @param threshold value to set to
201   */
202  public void setMinFilesToCompact(int threshold) {
203    minFilesToCompact = threshold;
204  }
205
206  /**
207   * @return upper bound on number of files to be included in minor compactions
208   */
209  public int getMaxFilesToCompact() {
210    return maxFilesToCompact;
211  }
212
213  /**
214   * @return Ratio used for compaction
215   */
216  public double getCompactionRatio() {
217    return compactionRatio;
218  }
219
220  /**
221   * @return Off peak Ratio used for compaction
222   */
223  public double getCompactionRatioOffPeak() {
224    return offPeakCompactionRatio;
225  }
226
227  /**
228   * @return ThrottlePoint used for classifying small and large compactions
229   */
230  public long getThrottlePoint() {
231    return throttlePoint;
232  }
233
234  /**
235   * @return Major compaction period from compaction.
236   *   Major compactions are selected periodically according to this parameter plus jitter
237   */
238  public long getMajorCompactionPeriod() {
239    return majorCompactionPeriod;
240  }
241
242  /**
243   * @return Major the jitter fraction, the fraction within which the major compaction
244   *    period is randomly chosen from the majorCompactionPeriod in each store.
245   */
246  public float getMajorCompactionJitter() {
247    return majorCompactionJitter;
248  }
249
250  /**
251   * @return Block locality ratio, the ratio at which we will include old regions with a single
252   *   store file for major compaction.  Used to improve block locality for regions that
253   *   haven't had writes in a while but are still being read.
254   */
255  public float getMinLocalityToForceCompact() {
256    return minLocalityToForceCompact;
257  }
258
259  public long getOffPeakMaxCompactSize() {
260    return offPeakMaxCompactSize;
261  }
262
263  public long getMaxCompactSize(boolean mayUseOffpeak) {
264    if (mayUseOffpeak) {
265      return getOffPeakMaxCompactSize();
266    } else {
267      return getMaxCompactSize();
268    }
269  }
270
271  public long getDateTieredMaxStoreFileAgeMillis() {
272    return dateTieredMaxStoreFileAgeMillis;
273  }
274
275  public int getDateTieredIncomingWindowMin() {
276    return dateTieredIncomingWindowMin;
277  }
278
279  public String getCompactionPolicyForDateTieredWindow() {
280    return compactionPolicyForDateTieredWindow;
281  }
282
283  public boolean useDateTieredSingleOutputForMinorCompaction() {
284    return dateTieredSingleOutputForMinorCompaction;
285  }
286
287  public String getDateTieredCompactionWindowFactory() {
288    return dateTieredCompactionWindowFactory;
289  }
290}