View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.regionserver.compactions;
21  
22  import org.apache.commons.logging.Log;
23  import org.apache.commons.logging.LogFactory;
24  import org.apache.hadoop.hbase.classification.InterfaceAudience;
25  import org.apache.hadoop.conf.Configuration;
26  import org.apache.hadoop.hbase.HConstants;
27  import org.apache.hadoop.hbase.regionserver.StoreConfigInformation;
28  
29  /**
30   * <p>
31   * Compaction configuration for a particular instance of HStore.
32   * Takes into account both global settings and ones set on the column family/store.
33   * Control knobs for default compaction algorithm:
34   * </p>
35   * <p>
36   * maxCompactSize - upper bound on file size to be included in minor compactions
37   * minCompactSize - lower bound below which compaction is selected without ratio test
38   * minFilesToCompact - lower bound on number of files in any minor compaction
39   * maxFilesToCompact - upper bound on number of files in any minor compaction
40   * compactionRatio - Ratio used for compaction
41   * minLocalityToForceCompact - Locality threshold for a store file to major compact (HBASE-11195)
42   * </p>
43   * Set parameter as "hbase.hstore.compaction.&lt;attribute&gt;"
44   */
45  
46  @InterfaceAudience.Private
47  public class CompactionConfiguration {
48  
49    private static final Log LOG = LogFactory.getLog(CompactionConfiguration.class);
50  
51    public static final String HBASE_HSTORE_COMPACTION_RATIO_KEY = "hbase.hstore.compaction.ratio";
52    public static final String HBASE_HSTORE_COMPACTION_RATIO_OFFPEAK_KEY =
53      "hbase.hstore.compaction.ratio.offpeak";
54    public static final String HBASE_HSTORE_COMPACTION_MIN_KEY = "hbase.hstore.compaction.min";
55    public static final String HBASE_HSTORE_COMPACTION_MIN_SIZE_KEY =
56      "hbase.hstore.compaction.min.size";
57    public static final String HBASE_HSTORE_COMPACTION_MAX_KEY = "hbase.hstore.compaction.max";
58    public static final String HBASE_HSTORE_COMPACTION_MAX_SIZE_KEY =
59      "hbase.hstore.compaction.max.size";
60    public static final String HBASE_HSTORE_COMPACTION_MAX_SIZE_OFFPEAK_KEY =
61        "hbase.hstore.compaction.max.size.offpeak";
62    public static final String HBASE_HSTORE_OFFPEAK_END_HOUR = "hbase.offpeak.end.hour";
63    public static final String HBASE_HSTORE_OFFPEAK_START_HOUR = "hbase.offpeak.start.hour";
64    public static final String HBASE_HSTORE_MIN_LOCALITY_TO_SKIP_MAJOR_COMPACT =
65        "hbase.hstore.min.locality.to.skip.major.compact";
66  
67    public static final String HBASE_HFILE_COMPACTION_DISCHARGER_THREAD_COUNT =
68        "hbase.hfile.compaction.discharger.thread.count";
69  
70    Configuration conf;
71    StoreConfigInformation storeConfigInfo;
72  
73    private final double offPeakCompactionRatio;
74    /** Since all these properties can change online, they are volatile **/
75    private final long maxCompactSize;
76    private final long offPeakMaxCompactSize;
77    private final long minCompactSize;
78    private final int minFilesToCompact;
79    private final int maxFilesToCompact;
80    private final double compactionRatio;
81    private final long throttlePoint;
82    private final long majorCompactionPeriod;
83    private final float majorCompactionJitter;
84    private final float minLocalityToForceCompact;
85  
86    CompactionConfiguration(Configuration conf, StoreConfigInformation storeConfigInfo) {
87      this.conf = conf;
88      this.storeConfigInfo = storeConfigInfo;
89  
90      maxCompactSize = conf.getLong(HBASE_HSTORE_COMPACTION_MAX_SIZE_KEY, Long.MAX_VALUE);
91      offPeakMaxCompactSize = conf.getLong(HBASE_HSTORE_COMPACTION_MAX_SIZE_OFFPEAK_KEY, 
92        maxCompactSize);      
93      minCompactSize = conf.getLong(HBASE_HSTORE_COMPACTION_MIN_SIZE_KEY,
94          storeConfigInfo.getMemstoreFlushSize());
95      minFilesToCompact = Math.max(2, conf.getInt(HBASE_HSTORE_COMPACTION_MIN_KEY,
96            /*old name*/ conf.getInt("hbase.hstore.compactionThreshold", 3)));
97      maxFilesToCompact = conf.getInt(HBASE_HSTORE_COMPACTION_MAX_KEY, 10);
98      compactionRatio = conf.getFloat(HBASE_HSTORE_COMPACTION_RATIO_KEY, 1.2F);
99      offPeakCompactionRatio = conf.getFloat(HBASE_HSTORE_COMPACTION_RATIO_OFFPEAK_KEY, 5.0F);
100 
101     throttlePoint = conf.getLong("hbase.regionserver.thread.compaction.throttle",
102           2 * maxFilesToCompact * storeConfigInfo.getMemstoreFlushSize());
103     majorCompactionPeriod = conf.getLong(HConstants.MAJOR_COMPACTION_PERIOD, 1000*60*60*24*7);
104     // Make it 0.5 so jitter has us fall evenly either side of when the compaction should run
105     majorCompactionJitter = conf.getFloat("hbase.hregion.majorcompaction.jitter", 0.50F);
106     minLocalityToForceCompact = conf.getFloat(HBASE_HSTORE_MIN_LOCALITY_TO_SKIP_MAJOR_COMPACT, 0f);
107     LOG.info(this);
108   }
109 
110   @Override
111   public String toString() {
112     return String.format(
113       "size [%d, %d, %d); files [%d, %d); ratio %f; off-peak ratio %f; throttle point %d;"
114       + " major period %d, major jitter %f, min locality to compact %f",
115       minCompactSize,
116       maxCompactSize,
117       offPeakMaxCompactSize,
118       minFilesToCompact,
119       maxFilesToCompact,
120       compactionRatio,
121       offPeakCompactionRatio,
122       throttlePoint,
123       majorCompactionPeriod,
124       majorCompactionJitter,
125       minLocalityToForceCompact);
126   }
127 
128   /**
129    * @return lower bound below which compaction is selected without ratio test
130    */
131   public long getMinCompactSize() {
132     return minCompactSize;
133   }
134 
135   /**
136    * @return upper bound on file size to be included in minor compactions
137    */
138   public long getMaxCompactSize() {
139     return maxCompactSize;
140   }
141 
142   /**
143    * @return upper bound on number of files to be included in minor compactions
144    */
145   public int getMinFilesToCompact() {
146     return minFilesToCompact;
147   }
148 
149   /**
150    * @return upper bound on number of files to be included in minor compactions
151    */
152   public int getMaxFilesToCompact() {
153     return maxFilesToCompact;
154   }
155 
156   /**
157    * @return Ratio used for compaction
158    */
159   public double getCompactionRatio() {
160     return compactionRatio;
161   }
162 
163   /**
164    * @return Off peak Ratio used for compaction
165    */
166   public double getCompactionRatioOffPeak() {
167     return offPeakCompactionRatio;
168   }
169 
170   /**
171    * @return ThrottlePoint used for classifying small and large compactions
172    */
173   public long getThrottlePoint() {
174     return throttlePoint;
175   }
176 
177   /**
178    * @return Major compaction period from compaction.
179    * Major compactions are selected periodically according to this parameter plus jitter
180    */
181   public long getMajorCompactionPeriod() {
182     return majorCompactionPeriod;
183   }
184 
185   /**
186    * @return Major the jitter fraction, the fraction within which the major compaction
187    *  period is randomly chosen from the majorCompactionPeriod in each store.
188    */
189   public float getMajorCompactionJitter() {
190     return majorCompactionJitter;
191   }
192 
193   /**
194    * @return Block locality ratio, the ratio at which we will include old regions with a single
195    * store file for major compaction.  Used to improve block locality for regions that
196    * haven't had writes in a while but are still being read.
197    */
198   public float getMinLocalityToForceCompact() {
199     return minLocalityToForceCompact;
200   }
201 
202   public long getOffPeakMaxCompactSize() {
203     return offPeakMaxCompactSize;
204   }
205   
206   public long getMaxCompactSize(boolean mayUseOffpeak) {
207     if (mayUseOffpeak) {
208       return getOffPeakMaxCompactSize();
209     } else {
210       return getMaxCompactSize();
211     }
212   }
213 }