001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.regionserver.compactions; 019 020import org.apache.hadoop.conf.Configuration; 021import org.apache.hadoop.hbase.HConstants; 022import org.apache.hadoop.hbase.client.RegionInfo; 023import org.apache.hadoop.hbase.regionserver.StoreConfigInformation; 024import org.apache.hadoop.util.StringUtils; 025import org.apache.yetus.audience.InterfaceAudience; 026import org.slf4j.Logger; 027import org.slf4j.LoggerFactory; 028 029/** 030 * <p> 031 * Compaction configuration for a particular instance of HStore. Takes into account both global 032 * settings and ones set on the column family/store. Control knobs for default compaction algorithm: 033 * </p> 034 * <p> 035 * maxCompactSize - upper bound on file size to be included in minor compactions minCompactSize - 036 * lower bound below which compaction is selected without ratio test minFilesToCompact - lower bound 037 * on number of files in any minor compaction maxFilesToCompact - upper bound on number of files in 038 * any minor compaction compactionRatio - Ratio used for compaction minLocalityToForceCompact - 039 * Locality threshold for a store file to major compact (HBASE-11195) 040 * </p> 041 * Set parameter as "hbase.hstore.compaction.<attribute>" 042 */ 043 044@InterfaceAudience.Private 045public class CompactionConfiguration { 046 047 private static final Logger LOG = LoggerFactory.getLogger(CompactionConfiguration.class); 048 049 public static final String HBASE_HSTORE_COMPACTION_RATIO_KEY = "hbase.hstore.compaction.ratio"; 050 public static final String HBASE_HSTORE_COMPACTION_RATIO_OFFPEAK_KEY = 051 "hbase.hstore.compaction.ratio.offpeak"; 052 public static final String HBASE_HSTORE_COMPACTION_MIN_KEY_OLD = 053 "hbase.hstore.compactionThreshold"; 054 public static final String HBASE_HSTORE_COMPACTION_MIN_KEY = "hbase.hstore.compaction.min"; 055 public static final String HBASE_HSTORE_COMPACTION_MIN_SIZE_KEY = 056 "hbase.hstore.compaction.min.size"; 057 public static final String HBASE_HSTORE_COMPACTION_MAX_KEY = "hbase.hstore.compaction.max"; 058 public static final String HBASE_HSTORE_COMPACTION_MAX_SIZE_KEY = 059 "hbase.hstore.compaction.max.size"; 060 public static final String HBASE_HSTORE_COMPACTION_MAX_SIZE_OFFPEAK_KEY = 061 "hbase.hstore.compaction.max.size.offpeak"; 062 public static final String HBASE_HSTORE_OFFPEAK_END_HOUR = "hbase.offpeak.end.hour"; 063 public static final String HBASE_HSTORE_OFFPEAK_START_HOUR = "hbase.offpeak.start.hour"; 064 public static final String HBASE_HSTORE_MIN_LOCALITY_TO_SKIP_MAJOR_COMPACT = 065 "hbase.hstore.min.locality.to.skip.major.compact"; 066 067 public static final String HBASE_HFILE_COMPACTION_DISCHARGER_THREAD_COUNT = 068 "hbase.hfile.compaction.discharger.thread.count"; 069 070 /* 071 * The epoch time length for the windows we no longer compact 072 */ 073 public static final String DATE_TIERED_MAX_AGE_MILLIS_KEY = 074 "hbase.hstore.compaction.date.tiered.max.storefile.age.millis"; 075 public static final String DATE_TIERED_INCOMING_WINDOW_MIN_KEY = 076 "hbase.hstore.compaction.date.tiered.incoming.window.min"; 077 public static final String COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS_KEY = 078 "hbase.hstore.compaction.date.tiered.window.policy.class"; 079 public static final String DATE_TIERED_SINGLE_OUTPUT_FOR_MINOR_COMPACTION_KEY = 080 "hbase.hstore.compaction.date.tiered.single.output.for.minor.compaction"; 081 082 private static final Class< 083 ? extends RatioBasedCompactionPolicy> DEFAULT_COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS = 084 ExploringCompactionPolicy.class; 085 086 public static final String DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS_KEY = 087 "hbase.hstore.compaction.date.tiered.window.factory.class"; 088 089 private static final Class< 090 ? extends CompactionWindowFactory> DEFAULT_DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS = 091 ExponentialCompactionWindowFactory.class; 092 093 public static final String DATE_TIERED_STORAGE_POLICY_ENABLE_KEY = 094 "hbase.hstore.compaction.date.tiered.storage.policy.enable"; 095 public static final String DATE_TIERED_HOT_WINDOW_AGE_MILLIS_KEY = 096 "hbase.hstore.compaction.date.tiered.hot.window.age.millis"; 097 public static final String DATE_TIERED_HOT_WINDOW_STORAGE_POLICY_KEY = 098 "hbase.hstore.compaction.date.tiered.hot.window.storage.policy"; 099 public static final String DATE_TIERED_WARM_WINDOW_AGE_MILLIS_KEY = 100 "hbase.hstore.compaction.date.tiered.warm.window.age.millis"; 101 public static final String DATE_TIERED_WARM_WINDOW_STORAGE_POLICY_KEY = 102 "hbase.hstore.compaction.date.tiered.warm.window.storage.policy"; 103 /** Windows older than warm age belong to COLD_WINDOW **/ 104 public static final String DATE_TIERED_COLD_WINDOW_STORAGE_POLICY_KEY = 105 "hbase.hstore.compaction.date.tiered.cold.window.storage.policy"; 106 107 Configuration conf; 108 StoreConfigInformation storeConfigInfo; 109 110 private final double offPeakCompactionRatio; 111 /** Since all these properties can change online, they are volatile **/ 112 private final long maxCompactSize; 113 private final long offPeakMaxCompactSize; 114 private final long minCompactSize; 115 /** This one can be update **/ 116 private int minFilesToCompact; 117 private final int maxFilesToCompact; 118 private final double compactionRatio; 119 private final long throttlePoint; 120 private final long majorCompactionPeriod; 121 private final float majorCompactionJitter; 122 private final float minLocalityToForceCompact; 123 private final long dateTieredMaxStoreFileAgeMillis; 124 private final int dateTieredIncomingWindowMin; 125 private final String compactionPolicyForDateTieredWindow; 126 private final boolean dateTieredSingleOutputForMinorCompaction; 127 private final String dateTieredCompactionWindowFactory; 128 private final boolean dateTieredStoragePolicyEnable; 129 private long hotWindowAgeMillis; 130 private long warmWindowAgeMillis; 131 private String hotWindowStoragePolicy; 132 private String warmWindowStoragePolicy; 133 private String coldWindowStoragePolicy; 134 135 CompactionConfiguration(Configuration conf, StoreConfigInformation storeConfigInfo) { 136 this.conf = conf; 137 this.storeConfigInfo = storeConfigInfo; 138 139 maxCompactSize = conf.getLong(HBASE_HSTORE_COMPACTION_MAX_SIZE_KEY, Long.MAX_VALUE); 140 offPeakMaxCompactSize = 141 conf.getLong(HBASE_HSTORE_COMPACTION_MAX_SIZE_OFFPEAK_KEY, maxCompactSize); 142 minCompactSize = 143 conf.getLong(HBASE_HSTORE_COMPACTION_MIN_SIZE_KEY, storeConfigInfo.getMemStoreFlushSize()); 144 minFilesToCompact = Math.max(2, conf.getInt(HBASE_HSTORE_COMPACTION_MIN_KEY, 145 conf.getInt(HBASE_HSTORE_COMPACTION_MIN_KEY_OLD, 3))); 146 maxFilesToCompact = conf.getInt(HBASE_HSTORE_COMPACTION_MAX_KEY, 10); 147 compactionRatio = conf.getFloat(HBASE_HSTORE_COMPACTION_RATIO_KEY, 1.2F); 148 offPeakCompactionRatio = conf.getFloat(HBASE_HSTORE_COMPACTION_RATIO_OFFPEAK_KEY, 5.0F); 149 150 throttlePoint = conf.getLong("hbase.regionserver.thread.compaction.throttle", 151 2 * maxFilesToCompact * storeConfigInfo.getMemStoreFlushSize()); 152 majorCompactionPeriod = 153 conf.getLong(HConstants.MAJOR_COMPACTION_PERIOD, HConstants.DEFAULT_MAJOR_COMPACTION_PERIOD); 154 majorCompactionJitter = 155 conf.getFloat(HConstants.MAJOR_COMPACTION_JITTER, HConstants.DEFAULT_MAJOR_COMPACTION_JITTER); 156 minLocalityToForceCompact = conf.getFloat(HBASE_HSTORE_MIN_LOCALITY_TO_SKIP_MAJOR_COMPACT, 0f); 157 158 dateTieredMaxStoreFileAgeMillis = conf.getLong(DATE_TIERED_MAX_AGE_MILLIS_KEY, Long.MAX_VALUE); 159 dateTieredIncomingWindowMin = conf.getInt(DATE_TIERED_INCOMING_WINDOW_MIN_KEY, 6); 160 compactionPolicyForDateTieredWindow = 161 conf.get(COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS_KEY, 162 DEFAULT_COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS.getName()); 163 dateTieredSingleOutputForMinorCompaction = 164 conf.getBoolean(DATE_TIERED_SINGLE_OUTPUT_FOR_MINOR_COMPACTION_KEY, true); 165 this.dateTieredCompactionWindowFactory = 166 conf.get(DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS_KEY, 167 DEFAULT_DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS.getName()); 168 // for Heterogeneous Storage 169 dateTieredStoragePolicyEnable = conf.getBoolean(DATE_TIERED_STORAGE_POLICY_ENABLE_KEY, false); 170 hotWindowAgeMillis = conf.getLong(DATE_TIERED_HOT_WINDOW_AGE_MILLIS_KEY, 86400000L); 171 hotWindowStoragePolicy = conf.get(DATE_TIERED_HOT_WINDOW_STORAGE_POLICY_KEY, "ALL_SSD"); 172 warmWindowAgeMillis = conf.getLong(DATE_TIERED_WARM_WINDOW_AGE_MILLIS_KEY, 604800000L); 173 warmWindowStoragePolicy = conf.get(DATE_TIERED_WARM_WINDOW_STORAGE_POLICY_KEY, "ONE_SSD"); 174 coldWindowStoragePolicy = conf.get(DATE_TIERED_COLD_WINDOW_STORAGE_POLICY_KEY, "HOT"); 175 LOG.info(toString()); 176 } 177 178 @Override 179 public String toString() { 180 return String.format( 181 "size [minCompactSize:%s, maxCompactSize:%s, offPeakMaxCompactSize:%s);" 182 + " files [minFilesToCompact:%d, maxFilesToCompact:%d);" 183 + " ratio %f; off-peak ratio %f; throttle point %d;" 184 + " major period %d, major jitter %f, min locality to compact %f;" 185 + " tiered compaction: max_age %d, incoming window min %d," 186 + " compaction policy for tiered window %s, single output for minor %b," 187 + " compaction window factory %s," + " region %s columnFamilyName %s", 188 StringUtils.byteDesc(minCompactSize), StringUtils.byteDesc(maxCompactSize), 189 StringUtils.byteDesc(offPeakMaxCompactSize), minFilesToCompact, maxFilesToCompact, 190 compactionRatio, offPeakCompactionRatio, throttlePoint, majorCompactionPeriod, 191 majorCompactionJitter, minLocalityToForceCompact, dateTieredMaxStoreFileAgeMillis, 192 dateTieredIncomingWindowMin, compactionPolicyForDateTieredWindow, 193 dateTieredSingleOutputForMinorCompaction, dateTieredCompactionWindowFactory, 194 RegionInfo.prettyPrint(storeConfigInfo.getRegionInfo().getEncodedName()), 195 storeConfigInfo.getColumnFamilyName()); 196 } 197 198 /** Returns lower bound below which compaction is selected without ratio test */ 199 public long getMinCompactSize() { 200 return minCompactSize; 201 } 202 203 /** Returns upper bound on file size to be included in minor compactions */ 204 public long getMaxCompactSize() { 205 return maxCompactSize; 206 } 207 208 /** Returns lower bound on number of files to be included in minor compactions */ 209 public int getMinFilesToCompact() { 210 return minFilesToCompact; 211 } 212 213 /** 214 * Set lower bound on number of files to be included in minor compactions 215 * @param threshold value to set to 216 */ 217 public void setMinFilesToCompact(int threshold) { 218 minFilesToCompact = threshold; 219 } 220 221 /** Returns upper bound on number of files to be included in minor compactions */ 222 public int getMaxFilesToCompact() { 223 return maxFilesToCompact; 224 } 225 226 /** Returns Ratio used for compaction */ 227 public double getCompactionRatio() { 228 return compactionRatio; 229 } 230 231 /** Returns Off peak Ratio used for compaction */ 232 public double getCompactionRatioOffPeak() { 233 return offPeakCompactionRatio; 234 } 235 236 /** Returns ThrottlePoint used for classifying small and large compactions */ 237 public long getThrottlePoint() { 238 return throttlePoint; 239 } 240 241 /** 242 * @return Major compaction period from compaction. Major compactions are selected periodically 243 * according to this parameter plus jitter 244 */ 245 public long getMajorCompactionPeriod() { 246 return majorCompactionPeriod; 247 } 248 249 /** 250 * @return Major the jitter fraction, the fraction within which the major compaction period is 251 * randomly chosen from the majorCompactionPeriod in each store. 252 */ 253 public float getMajorCompactionJitter() { 254 return majorCompactionJitter; 255 } 256 257 /** 258 * @return Block locality ratio, the ratio at which we will include old regions with a single 259 * store file for major compaction. Used to improve block locality for regions that 260 * haven't had writes in a while but are still being read. 261 */ 262 public float getMinLocalityToForceCompact() { 263 return minLocalityToForceCompact; 264 } 265 266 public long getOffPeakMaxCompactSize() { 267 return offPeakMaxCompactSize; 268 } 269 270 public long getMaxCompactSize(boolean mayUseOffpeak) { 271 if (mayUseOffpeak) { 272 return getOffPeakMaxCompactSize(); 273 } else { 274 return getMaxCompactSize(); 275 } 276 } 277 278 public long getDateTieredMaxStoreFileAgeMillis() { 279 return dateTieredMaxStoreFileAgeMillis; 280 } 281 282 public int getDateTieredIncomingWindowMin() { 283 return dateTieredIncomingWindowMin; 284 } 285 286 public String getCompactionPolicyForDateTieredWindow() { 287 return compactionPolicyForDateTieredWindow; 288 } 289 290 public boolean useDateTieredSingleOutputForMinorCompaction() { 291 return dateTieredSingleOutputForMinorCompaction; 292 } 293 294 public String getDateTieredCompactionWindowFactory() { 295 return dateTieredCompactionWindowFactory; 296 } 297 298 public boolean isDateTieredStoragePolicyEnable() { 299 return dateTieredStoragePolicyEnable; 300 } 301 302 public long getHotWindowAgeMillis() { 303 return hotWindowAgeMillis; 304 } 305 306 public long getWarmWindowAgeMillis() { 307 return warmWindowAgeMillis; 308 } 309 310 public String getHotWindowStoragePolicy() { 311 return hotWindowStoragePolicy.trim().toUpperCase(); 312 } 313 314 public String getWarmWindowStoragePolicy() { 315 return warmWindowStoragePolicy.trim().toUpperCase(); 316 } 317 318 public String getColdWindowStoragePolicy() { 319 return coldWindowStoragePolicy.trim().toUpperCase(); 320 } 321}