001/** 002 * 003 * Licensed to the Apache Software Foundation (ASF) under one 004 * or more contributor license agreements. See the NOTICE file 005 * distributed with this work for additional information 006 * regarding copyright ownership. The ASF licenses this file 007 * to you under the Apache License, Version 2.0 (the 008 * "License"); you may not use this file except in compliance 009 * with the License. You may obtain a copy of the License at 010 * 011 * http://www.apache.org/licenses/LICENSE-2.0 012 * 013 * Unless required by applicable law or agreed to in writing, software 014 * distributed under the License is distributed on an "AS IS" BASIS, 015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 016 * See the License for the specific language governing permissions and 017 * limitations under the License. 018 */ 019 020package org.apache.hadoop.hbase.regionserver.compactions; 021 022import org.apache.hadoop.conf.Configuration; 023import org.apache.hadoop.hbase.HConstants; 024import org.apache.hadoop.hbase.client.RegionInfo; 025import org.apache.hadoop.util.StringUtils; 026import org.apache.yetus.audience.InterfaceAudience; 027import org.slf4j.Logger; 028import org.slf4j.LoggerFactory; 029import org.apache.hadoop.hbase.regionserver.StoreConfigInformation; 030 031/** 032 * <p> 033 * Compaction configuration for a particular instance of HStore. 034 * Takes into account both global settings and ones set on the column family/store. 035 * Control knobs for default compaction algorithm: 036 * </p> 037 * <p> 038 * maxCompactSize - upper bound on file size to be included in minor compactions 039 * minCompactSize - lower bound below which compaction is selected without ratio test 040 * minFilesToCompact - lower bound on number of files in any minor compaction 041 * maxFilesToCompact - upper bound on number of files in any minor compaction 042 * compactionRatio - Ratio used for compaction 043 * minLocalityToForceCompact - Locality threshold for a store file to major compact (HBASE-11195) 044 * </p> 045 * Set parameter as "hbase.hstore.compaction.<attribute>" 046 */ 047 048@InterfaceAudience.Private 049public class CompactionConfiguration { 050 051 private static final Logger LOG = LoggerFactory.getLogger(CompactionConfiguration.class); 052 053 public static final String HBASE_HSTORE_COMPACTION_RATIO_KEY = "hbase.hstore.compaction.ratio"; 054 public static final String HBASE_HSTORE_COMPACTION_RATIO_OFFPEAK_KEY = 055 "hbase.hstore.compaction.ratio.offpeak"; 056 public static final String HBASE_HSTORE_COMPACTION_MIN_KEY = "hbase.hstore.compaction.min"; 057 public static final String HBASE_HSTORE_COMPACTION_MIN_SIZE_KEY = 058 "hbase.hstore.compaction.min.size"; 059 public static final String HBASE_HSTORE_COMPACTION_MAX_KEY = "hbase.hstore.compaction.max"; 060 public static final String HBASE_HSTORE_COMPACTION_MAX_SIZE_KEY = 061 "hbase.hstore.compaction.max.size"; 062 public static final String HBASE_HSTORE_COMPACTION_MAX_SIZE_OFFPEAK_KEY = 063 "hbase.hstore.compaction.max.size.offpeak"; 064 public static final String HBASE_HSTORE_OFFPEAK_END_HOUR = "hbase.offpeak.end.hour"; 065 public static final String HBASE_HSTORE_OFFPEAK_START_HOUR = "hbase.offpeak.start.hour"; 066 public static final String HBASE_HSTORE_MIN_LOCALITY_TO_SKIP_MAJOR_COMPACT = 067 "hbase.hstore.min.locality.to.skip.major.compact"; 068 069 public static final String HBASE_HFILE_COMPACTION_DISCHARGER_THREAD_COUNT = 070 "hbase.hfile.compaction.discharger.thread.count"; 071 072 /* 073 * The epoch time length for the windows we no longer compact 074 */ 075 public static final String DATE_TIERED_MAX_AGE_MILLIS_KEY = 076 "hbase.hstore.compaction.date.tiered.max.storefile.age.millis"; 077 public static final String DATE_TIERED_INCOMING_WINDOW_MIN_KEY = 078 "hbase.hstore.compaction.date.tiered.incoming.window.min"; 079 public static final String COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS_KEY = 080 "hbase.hstore.compaction.date.tiered.window.policy.class"; 081 public static final String DATE_TIERED_SINGLE_OUTPUT_FOR_MINOR_COMPACTION_KEY = 082 "hbase.hstore.compaction.date.tiered.single.output.for.minor.compaction"; 083 084 private static final Class<? extends RatioBasedCompactionPolicy> 085 DEFAULT_COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS = ExploringCompactionPolicy.class; 086 087 public static final String DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS_KEY = 088 "hbase.hstore.compaction.date.tiered.window.factory.class"; 089 090 private static final Class<? extends CompactionWindowFactory> 091 DEFAULT_DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS = ExponentialCompactionWindowFactory.class; 092 093 public static final String DATE_TIERED_STORAGE_POLICY_ENABLE_KEY = 094 "hbase.hstore.compaction.date.tiered.storage.policy.enable"; 095 public static final String DATE_TIERED_HOT_WINDOW_AGE_MILLIS_KEY = 096 "hbase.hstore.compaction.date.tiered.hot.window.age.millis"; 097 public static final String DATE_TIERED_HOT_WINDOW_STORAGE_POLICY_KEY = 098 "hbase.hstore.compaction.date.tiered.hot.window.storage.policy"; 099 public static final String DATE_TIERED_WARM_WINDOW_AGE_MILLIS_KEY = 100 "hbase.hstore.compaction.date.tiered.warm.window.age.millis"; 101 public static final String DATE_TIERED_WARM_WINDOW_STORAGE_POLICY_KEY = 102 "hbase.hstore.compaction.date.tiered.warm.window.storage.policy"; 103 /** Windows older than warm age belong to COLD_WINDOW **/ 104 public static final String DATE_TIERED_COLD_WINDOW_STORAGE_POLICY_KEY = 105 "hbase.hstore.compaction.date.tiered.cold.window.storage.policy"; 106 107 Configuration conf; 108 StoreConfigInformation storeConfigInfo; 109 110 private final double offPeakCompactionRatio; 111 /** Since all these properties can change online, they are volatile **/ 112 private final long maxCompactSize; 113 private final long offPeakMaxCompactSize; 114 private final long minCompactSize; 115 /** This one can be update **/ 116 private int minFilesToCompact; 117 private final int maxFilesToCompact; 118 private final double compactionRatio; 119 private final long throttlePoint; 120 private final long majorCompactionPeriod; 121 private final float majorCompactionJitter; 122 private final float minLocalityToForceCompact; 123 private final long dateTieredMaxStoreFileAgeMillis; 124 private final int dateTieredIncomingWindowMin; 125 private final String compactionPolicyForDateTieredWindow; 126 private final boolean dateTieredSingleOutputForMinorCompaction; 127 private final String dateTieredCompactionWindowFactory; 128 private final boolean dateTieredStoragePolicyEnable; 129 private long hotWindowAgeMillis; 130 private long warmWindowAgeMillis; 131 private String hotWindowStoragePolicy; 132 private String warmWindowStoragePolicy; 133 private String coldWindowStoragePolicy; 134 135 CompactionConfiguration(Configuration conf, StoreConfigInformation storeConfigInfo) { 136 this.conf = conf; 137 this.storeConfigInfo = storeConfigInfo; 138 139 maxCompactSize = conf.getLong(HBASE_HSTORE_COMPACTION_MAX_SIZE_KEY, Long.MAX_VALUE); 140 offPeakMaxCompactSize = conf.getLong(HBASE_HSTORE_COMPACTION_MAX_SIZE_OFFPEAK_KEY, 141 maxCompactSize); 142 minCompactSize = conf.getLong(HBASE_HSTORE_COMPACTION_MIN_SIZE_KEY, 143 storeConfigInfo.getMemStoreFlushSize()); 144 minFilesToCompact = Math.max(2, conf.getInt(HBASE_HSTORE_COMPACTION_MIN_KEY, 145 /*old name*/ conf.getInt("hbase.hstore.compactionThreshold", 3))); 146 maxFilesToCompact = conf.getInt(HBASE_HSTORE_COMPACTION_MAX_KEY, 10); 147 compactionRatio = conf.getFloat(HBASE_HSTORE_COMPACTION_RATIO_KEY, 1.2F); 148 offPeakCompactionRatio = conf.getFloat(HBASE_HSTORE_COMPACTION_RATIO_OFFPEAK_KEY, 5.0F); 149 150 throttlePoint = conf.getLong("hbase.regionserver.thread.compaction.throttle", 151 2 * maxFilesToCompact * storeConfigInfo.getMemStoreFlushSize()); 152 majorCompactionPeriod = conf.getLong(HConstants.MAJOR_COMPACTION_PERIOD, 153 HConstants.DEFAULT_MAJOR_COMPACTION_PERIOD); 154 majorCompactionJitter = conf.getFloat(HConstants.MAJOR_COMPACTION_JITTER, 155 HConstants.DEFAULT_MAJOR_COMPACTION_JITTER); 156 minLocalityToForceCompact = conf.getFloat(HBASE_HSTORE_MIN_LOCALITY_TO_SKIP_MAJOR_COMPACT, 0f); 157 158 dateTieredMaxStoreFileAgeMillis = conf.getLong(DATE_TIERED_MAX_AGE_MILLIS_KEY, Long.MAX_VALUE); 159 dateTieredIncomingWindowMin = conf.getInt(DATE_TIERED_INCOMING_WINDOW_MIN_KEY, 6); 160 compactionPolicyForDateTieredWindow = conf.get( 161 COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS_KEY, 162 DEFAULT_COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS.getName()); 163 dateTieredSingleOutputForMinorCompaction = conf 164 .getBoolean(DATE_TIERED_SINGLE_OUTPUT_FOR_MINOR_COMPACTION_KEY, true); 165 this.dateTieredCompactionWindowFactory = conf.get( 166 DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS_KEY, 167 DEFAULT_DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS.getName()); 168 // for Heterogeneous Storage 169 dateTieredStoragePolicyEnable = conf.getBoolean(DATE_TIERED_STORAGE_POLICY_ENABLE_KEY, false); 170 hotWindowAgeMillis = conf.getLong(DATE_TIERED_HOT_WINDOW_AGE_MILLIS_KEY, 86400000L); 171 hotWindowStoragePolicy = conf.get(DATE_TIERED_HOT_WINDOW_STORAGE_POLICY_KEY, "ALL_SSD"); 172 warmWindowAgeMillis = conf.getLong(DATE_TIERED_WARM_WINDOW_AGE_MILLIS_KEY, 604800000L); 173 warmWindowStoragePolicy = conf.get(DATE_TIERED_WARM_WINDOW_STORAGE_POLICY_KEY, "ONE_SSD"); 174 coldWindowStoragePolicy = conf.get(DATE_TIERED_COLD_WINDOW_STORAGE_POLICY_KEY, "HOT"); 175 LOG.info(toString()); 176 } 177 178 @Override 179 public String toString() { 180 return String.format( 181 "size [minCompactSize:%s, maxCompactSize:%s, offPeakMaxCompactSize:%s);" 182 + " files [minFilesToCompact:%d, maxFilesToCompact:%d);" 183 + " ratio %f; off-peak ratio %f; throttle point %d;" 184 + " major period %d, major jitter %f, min locality to compact %f;" 185 + " tiered compaction: max_age %d, incoming window min %d," 186 + " compaction policy for tiered window %s, single output for minor %b," 187 + " compaction window factory %s," 188 + " region %s columnFamilyName %s", 189 StringUtils.byteDesc(minCompactSize), 190 StringUtils.byteDesc(maxCompactSize), 191 StringUtils.byteDesc(offPeakMaxCompactSize), 192 minFilesToCompact, 193 maxFilesToCompact, 194 compactionRatio, 195 offPeakCompactionRatio, 196 throttlePoint, 197 majorCompactionPeriod, 198 majorCompactionJitter, 199 minLocalityToForceCompact, 200 dateTieredMaxStoreFileAgeMillis, 201 dateTieredIncomingWindowMin, 202 compactionPolicyForDateTieredWindow, 203 dateTieredSingleOutputForMinorCompaction, 204 dateTieredCompactionWindowFactory, 205 RegionInfo.prettyPrint(storeConfigInfo.getRegionInfo().getEncodedName()), 206 storeConfigInfo.getColumnFamilyName() 207 ); 208 } 209 210 /** 211 * @return lower bound below which compaction is selected without ratio test 212 */ 213 public long getMinCompactSize() { 214 return minCompactSize; 215 } 216 217 /** 218 * @return upper bound on file size to be included in minor compactions 219 */ 220 public long getMaxCompactSize() { 221 return maxCompactSize; 222 } 223 224 /** 225 * @return upper bound on number of files to be included in minor compactions 226 */ 227 public int getMinFilesToCompact() { 228 return minFilesToCompact; 229 } 230 231 /** 232 * Set upper bound on number of files to be included in minor compactions 233 * @param threshold value to set to 234 */ 235 public void setMinFilesToCompact(int threshold) { 236 minFilesToCompact = threshold; 237 } 238 239 /** 240 * @return upper bound on number of files to be included in minor compactions 241 */ 242 public int getMaxFilesToCompact() { 243 return maxFilesToCompact; 244 } 245 246 /** 247 * @return Ratio used for compaction 248 */ 249 public double getCompactionRatio() { 250 return compactionRatio; 251 } 252 253 /** 254 * @return Off peak Ratio used for compaction 255 */ 256 public double getCompactionRatioOffPeak() { 257 return offPeakCompactionRatio; 258 } 259 260 /** 261 * @return ThrottlePoint used for classifying small and large compactions 262 */ 263 public long getThrottlePoint() { 264 return throttlePoint; 265 } 266 267 /** 268 * @return Major compaction period from compaction. 269 * Major compactions are selected periodically according to this parameter plus jitter 270 */ 271 public long getMajorCompactionPeriod() { 272 return majorCompactionPeriod; 273 } 274 275 /** 276 * @return Major the jitter fraction, the fraction within which the major compaction 277 * period is randomly chosen from the majorCompactionPeriod in each store. 278 */ 279 public float getMajorCompactionJitter() { 280 return majorCompactionJitter; 281 } 282 283 /** 284 * @return Block locality ratio, the ratio at which we will include old regions with a single 285 * store file for major compaction. Used to improve block locality for regions that 286 * haven't had writes in a while but are still being read. 287 */ 288 public float getMinLocalityToForceCompact() { 289 return minLocalityToForceCompact; 290 } 291 292 public long getOffPeakMaxCompactSize() { 293 return offPeakMaxCompactSize; 294 } 295 296 public long getMaxCompactSize(boolean mayUseOffpeak) { 297 if (mayUseOffpeak) { 298 return getOffPeakMaxCompactSize(); 299 } else { 300 return getMaxCompactSize(); 301 } 302 } 303 304 public long getDateTieredMaxStoreFileAgeMillis() { 305 return dateTieredMaxStoreFileAgeMillis; 306 } 307 308 public int getDateTieredIncomingWindowMin() { 309 return dateTieredIncomingWindowMin; 310 } 311 312 public String getCompactionPolicyForDateTieredWindow() { 313 return compactionPolicyForDateTieredWindow; 314 } 315 316 public boolean useDateTieredSingleOutputForMinorCompaction() { 317 return dateTieredSingleOutputForMinorCompaction; 318 } 319 320 public String getDateTieredCompactionWindowFactory() { 321 return dateTieredCompactionWindowFactory; 322 } 323 324 public boolean isDateTieredStoragePolicyEnable() { 325 return dateTieredStoragePolicyEnable; 326 } 327 328 public long getHotWindowAgeMillis() { 329 return hotWindowAgeMillis; 330 } 331 332 public long getWarmWindowAgeMillis() { 333 return warmWindowAgeMillis; 334 } 335 336 public String getHotWindowStoragePolicy() { 337 return hotWindowStoragePolicy.trim().toUpperCase(); 338 } 339 340 public String getWarmWindowStoragePolicy() { 341 return warmWindowStoragePolicy.trim().toUpperCase(); 342 } 343 344 public String getColdWindowStoragePolicy() { 345 return coldWindowStoragePolicy.trim().toUpperCase(); 346 } 347}