View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.regionserver.compactions;
21  
22  import org.apache.commons.logging.Log;
23  import org.apache.commons.logging.LogFactory;
24  import org.apache.hadoop.conf.Configuration;
25  import org.apache.hadoop.hbase.HConstants;
26  import org.apache.hadoop.hbase.classification.InterfaceAudience;
27  import org.apache.hadoop.hbase.regionserver.StoreConfigInformation;
28  
29  /**
30   * <p>
31   * Compaction configuration for a particular instance of HStore.
32   * Takes into account both global settings and ones set on the column family/store.
33   * Control knobs for default compaction algorithm:
34   * </p>
35   * <p>
36   * maxCompactSize - upper bound on file size to be included in minor compactions
37   * minCompactSize - lower bound below which compaction is selected without ratio test
38   * minFilesToCompact - lower bound on number of files in any minor compaction
39   * maxFilesToCompact - upper bound on number of files in any minor compaction
40   * compactionRatio - Ratio used for compaction
41   * minLocalityToForceCompact - Locality threshold for a store file to major compact (HBASE-11195)
42   * </p>
43   * Set parameter as "hbase.hstore.compaction.&lt;attribute&gt;"
44   */
45  
46  @InterfaceAudience.Private
47  public class CompactionConfiguration {
48  
49    private static final Log LOG = LogFactory.getLog(CompactionConfiguration.class);
50  
51    public static final String HBASE_HSTORE_COMPACTION_RATIO_KEY = "hbase.hstore.compaction.ratio";
52    public static final String HBASE_HSTORE_COMPACTION_RATIO_OFFPEAK_KEY =
53      "hbase.hstore.compaction.ratio.offpeak";
54    public static final String HBASE_HSTORE_COMPACTION_MIN_KEY = "hbase.hstore.compaction.min";
55    public static final String HBASE_HSTORE_COMPACTION_MIN_SIZE_KEY =
56      "hbase.hstore.compaction.min.size";
57    public static final String HBASE_HSTORE_COMPACTION_MAX_KEY = "hbase.hstore.compaction.max";
58    public static final String HBASE_HSTORE_COMPACTION_MAX_SIZE_KEY =
59      "hbase.hstore.compaction.max.size";
60    public static final String HBASE_HSTORE_COMPACTION_MAX_SIZE_OFFPEAK_KEY =
61        "hbase.hstore.compaction.max.size.offpeak";
62    public static final String HBASE_HSTORE_OFFPEAK_END_HOUR = "hbase.offpeak.end.hour";
63    public static final String HBASE_HSTORE_OFFPEAK_START_HOUR = "hbase.offpeak.start.hour";
64    public static final String HBASE_HSTORE_MIN_LOCALITY_TO_SKIP_MAJOR_COMPACT =
65        "hbase.hstore.min.locality.to.skip.major.compact";
66  
67    public static final String HBASE_HFILE_COMPACTION_DISCHARGER_THREAD_COUNT =
68        "hbase.hfile.compaction.discharger.thread.count";
69  
70    /*
71     * The epoch time length for the windows we no longer compact
72     */
73    public static final String DATE_TIERED_MAX_AGE_MILLIS_KEY =
74      "hbase.hstore.compaction.date.tiered.max.storefile.age.millis";
75    public static final String DATE_TIERED_INCOMING_WINDOW_MIN_KEY =
76      "hbase.hstore.compaction.date.tiered.incoming.window.min";
77    public static final String COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS_KEY =
78      "hbase.hstore.compaction.date.tiered.window.policy.class";
79    public static final String DATE_TIERED_SINGLE_OUTPUT_FOR_MINOR_COMPACTION_KEY =
80      "hbase.hstore.compaction.date.tiered.single.output.for.minor.compaction";
81  
82    private static final Class<? extends RatioBasedCompactionPolicy>
83      DEFAULT_COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS = ExploringCompactionPolicy.class;
84  
85    public static final String DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS_KEY =
86      "hbase.hstore.compaction.date.tiered.window.factory.class";
87  
88    private static final Class<? extends CompactionWindowFactory>
89      DEFAULT_DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS = ExponentialCompactionWindowFactory.class;
90  
91    Configuration conf;
92    StoreConfigInformation storeConfigInfo;
93  
94    private final double offPeakCompactionRatio;
95    /** Since all these properties can change online, they are volatile **/
96    private final long maxCompactSize;
97    private final long offPeakMaxCompactSize;
98    private final long minCompactSize;
99    /** This one can be update **/
100   private int minFilesToCompact;
101   private final int maxFilesToCompact;
102   private final double compactionRatio;
103   private final long throttlePoint;
104   private final long majorCompactionPeriod;
105   private final float majorCompactionJitter;
106   private final float minLocalityToForceCompact;
107   private final long dateTieredMaxStoreFileAgeMillis;
108   private final int dateTieredIncomingWindowMin;
109   private final String compactionPolicyForDateTieredWindow;
110   private final boolean dateTieredSingleOutputForMinorCompaction;
111   private final String dateTieredCompactionWindowFactory;
112 
113   CompactionConfiguration(Configuration conf, StoreConfigInformation storeConfigInfo) {
114     this.conf = conf;
115     this.storeConfigInfo = storeConfigInfo;
116 
117     maxCompactSize = conf.getLong(HBASE_HSTORE_COMPACTION_MAX_SIZE_KEY, Long.MAX_VALUE);
118     offPeakMaxCompactSize = conf.getLong(HBASE_HSTORE_COMPACTION_MAX_SIZE_OFFPEAK_KEY, 
119       maxCompactSize);      
120     minCompactSize = conf.getLong(HBASE_HSTORE_COMPACTION_MIN_SIZE_KEY,
121         storeConfigInfo.getMemstoreFlushSize());
122     minFilesToCompact = Math.max(2, conf.getInt(HBASE_HSTORE_COMPACTION_MIN_KEY,
123           /*old name*/ conf.getInt("hbase.hstore.compactionThreshold", 3)));
124     maxFilesToCompact = conf.getInt(HBASE_HSTORE_COMPACTION_MAX_KEY, 10);
125     compactionRatio = conf.getFloat(HBASE_HSTORE_COMPACTION_RATIO_KEY, 1.2F);
126     offPeakCompactionRatio = conf.getFloat(HBASE_HSTORE_COMPACTION_RATIO_OFFPEAK_KEY, 5.0F);
127 
128     throttlePoint = conf.getLong("hbase.regionserver.thread.compaction.throttle",
129           2 * maxFilesToCompact * storeConfigInfo.getMemstoreFlushSize());
130     majorCompactionPeriod = conf.getLong(HConstants.MAJOR_COMPACTION_PERIOD, 1000*60*60*24*7);
131     // Make it 0.5 so jitter has us fall evenly either side of when the compaction should run
132     majorCompactionJitter = conf.getFloat("hbase.hregion.majorcompaction.jitter", 0.50F);
133     minLocalityToForceCompact = conf.getFloat(HBASE_HSTORE_MIN_LOCALITY_TO_SKIP_MAJOR_COMPACT, 0f);
134 
135     dateTieredMaxStoreFileAgeMillis = conf.getLong(DATE_TIERED_MAX_AGE_MILLIS_KEY, Long.MAX_VALUE);
136     dateTieredIncomingWindowMin = conf.getInt(DATE_TIERED_INCOMING_WINDOW_MIN_KEY, 6);
137     compactionPolicyForDateTieredWindow = conf.get(
138       COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS_KEY,
139       DEFAULT_COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS.getName());
140     dateTieredSingleOutputForMinorCompaction = conf
141         .getBoolean(DATE_TIERED_SINGLE_OUTPUT_FOR_MINOR_COMPACTION_KEY, true);
142     this.dateTieredCompactionWindowFactory = conf.get(
143       DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS_KEY,
144       DEFAULT_DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS.getName());
145     LOG.info(this);
146   }
147 
148   @Override
149   public String toString() {
150     return String.format(
151       "size [%d, %d, %d); files [%d, %d); ratio %f; off-peak ratio %f; throttle point %d;"
152       + " major period %d, major jitter %f, min locality to compact %f;"
153       + " tiered compaction: max_age %d, incoming window min %d,"
154       + " compaction policy for tiered window %s, single output for minor %b,"
155       + " compaction window factory %s",
156       minCompactSize,
157       maxCompactSize,
158       offPeakMaxCompactSize,
159       minFilesToCompact,
160       maxFilesToCompact,
161       compactionRatio,
162       offPeakCompactionRatio,
163       throttlePoint,
164       majorCompactionPeriod,
165       majorCompactionJitter,
166       minLocalityToForceCompact,
167       dateTieredMaxStoreFileAgeMillis,
168       dateTieredIncomingWindowMin,
169       compactionPolicyForDateTieredWindow,
170       dateTieredSingleOutputForMinorCompaction,
171       dateTieredCompactionWindowFactory
172       );
173   }
174 
175   /**
176    * @return lower bound below which compaction is selected without ratio test
177    */
178   public long getMinCompactSize() {
179     return minCompactSize;
180   }
181 
182   /**
183    * @return upper bound on file size to be included in minor compactions
184    */
185   public long getMaxCompactSize() {
186     return maxCompactSize;
187   }
188 
189   /**
190    * @return upper bound on number of files to be included in minor compactions
191    */
192   public int getMinFilesToCompact() {
193     return minFilesToCompact;
194   }
195 
196   /**
197    * Set upper bound on number of files to be included in minor compactions
198    * @param threshold value to set to
199    */
200   public void setMinFilesToCompact(int threshold) {
201     minFilesToCompact = threshold;
202   }
203 
204   /**
205    * @return upper bound on number of files to be included in minor compactions
206    */
207   public int getMaxFilesToCompact() {
208     return maxFilesToCompact;
209   }
210 
211   /**
212    * @return Ratio used for compaction
213    */
214   public double getCompactionRatio() {
215     return compactionRatio;
216   }
217 
218   /**
219    * @return Off peak Ratio used for compaction
220    */
221   public double getCompactionRatioOffPeak() {
222     return offPeakCompactionRatio;
223   }
224 
225   /**
226    * @return ThrottlePoint used for classifying small and large compactions
227    */
228   public long getThrottlePoint() {
229     return throttlePoint;
230   }
231 
232   /**
233    * @return Major compaction period from compaction.
234    *   Major compactions are selected periodically according to this parameter plus jitter
235    */
236   public long getMajorCompactionPeriod() {
237     return majorCompactionPeriod;
238   }
239 
240   /**
241    * @return Major the jitter fraction, the fraction within which the major compaction
242    *    period is randomly chosen from the majorCompactionPeriod in each store.
243    */
244   public float getMajorCompactionJitter() {
245     return majorCompactionJitter;
246   }
247 
248   /**
249    * @return Block locality ratio, the ratio at which we will include old regions with a single
250    *   store file for major compaction.  Used to improve block locality for regions that
251    *   haven't had writes in a while but are still being read.
252    */
253   public float getMinLocalityToForceCompact() {
254     return minLocalityToForceCompact;
255   }
256 
257   public long getOffPeakMaxCompactSize() {
258     return offPeakMaxCompactSize;
259   }
260 
261   public long getMaxCompactSize(boolean mayUseOffpeak) {
262     if (mayUseOffpeak) {
263       return getOffPeakMaxCompactSize();
264     } else {
265       return getMaxCompactSize();
266     }
267   }
268 
269   public long getDateTieredMaxStoreFileAgeMillis() {
270     return dateTieredMaxStoreFileAgeMillis;
271   }
272 
273   public int getDateTieredIncomingWindowMin() {
274     return dateTieredIncomingWindowMin;
275   }
276 
277   public String getCompactionPolicyForDateTieredWindow() {
278     return compactionPolicyForDateTieredWindow;
279   }
280 
281   public boolean useDateTieredSingleOutputForMinorCompaction() {
282     return dateTieredSingleOutputForMinorCompaction;
283   }
284 
285   public String getDateTieredCompactionWindowFactory() {
286     return dateTieredCompactionWindowFactory;
287   }
288 }