001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.regionserver.compactions; 019 020import org.apache.hadoop.conf.Configuration; 021import org.apache.hadoop.hbase.HConstants; 022import org.apache.hadoop.hbase.client.RegionInfo; 023import org.apache.hadoop.hbase.regionserver.StoreConfigInformation; 024import org.apache.hadoop.util.StringUtils; 025import org.apache.yetus.audience.InterfaceAudience; 026import org.slf4j.Logger; 027import org.slf4j.LoggerFactory; 028 029/** 030 * <p> 031 * Compaction configuration for a particular instance of HStore. Takes into account both global 032 * settings and ones set on the column family/store. Control knobs for default compaction algorithm: 033 * </p> 034 * <p> 035 * maxCompactSize - upper bound on file size to be included in minor compactions minCompactSize - 036 * lower bound below which compaction is selected without ratio test minFilesToCompact - lower bound 037 * on number of files in any minor compaction maxFilesToCompact - upper bound on number of files in 038 * any minor compaction compactionRatio - Ratio used for compaction minLocalityToForceCompact - 039 * Locality threshold for a store file to major compact (HBASE-11195) 040 * </p> 041 * Set parameter as "hbase.hstore.compaction.<attribute>" 042 */ 043 044@InterfaceAudience.Private 045public class CompactionConfiguration { 046 047 private static final Logger LOG = LoggerFactory.getLogger(CompactionConfiguration.class); 048 049 public static final String HBASE_HSTORE_COMPACTION_RATIO_KEY = "hbase.hstore.compaction.ratio"; 050 public static final String HBASE_HSTORE_COMPACTION_RATIO_OFFPEAK_KEY = 051 "hbase.hstore.compaction.ratio.offpeak"; 052 public static final String HBASE_HSTORE_COMPACTION_MIN_KEY = "hbase.hstore.compaction.min"; 053 public static final String HBASE_HSTORE_COMPACTION_MIN_SIZE_KEY = 054 "hbase.hstore.compaction.min.size"; 055 public static final String HBASE_HSTORE_COMPACTION_MAX_KEY = "hbase.hstore.compaction.max"; 056 public static final String HBASE_HSTORE_COMPACTION_MAX_SIZE_KEY = 057 "hbase.hstore.compaction.max.size"; 058 public static final String HBASE_HSTORE_COMPACTION_MAX_SIZE_OFFPEAK_KEY = 059 "hbase.hstore.compaction.max.size.offpeak"; 060 public static final String HBASE_HSTORE_OFFPEAK_END_HOUR = "hbase.offpeak.end.hour"; 061 public static final String HBASE_HSTORE_OFFPEAK_START_HOUR = "hbase.offpeak.start.hour"; 062 public static final String HBASE_HSTORE_MIN_LOCALITY_TO_SKIP_MAJOR_COMPACT = 063 "hbase.hstore.min.locality.to.skip.major.compact"; 064 065 public static final String HBASE_HFILE_COMPACTION_DISCHARGER_THREAD_COUNT = 066 "hbase.hfile.compaction.discharger.thread.count"; 067 068 /* 069 * The epoch time length for the windows we no longer compact 070 */ 071 public static final String DATE_TIERED_MAX_AGE_MILLIS_KEY = 072 "hbase.hstore.compaction.date.tiered.max.storefile.age.millis"; 073 public static final String DATE_TIERED_INCOMING_WINDOW_MIN_KEY = 074 "hbase.hstore.compaction.date.tiered.incoming.window.min"; 075 public static final String COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS_KEY = 076 "hbase.hstore.compaction.date.tiered.window.policy.class"; 077 public static final String DATE_TIERED_SINGLE_OUTPUT_FOR_MINOR_COMPACTION_KEY = 078 "hbase.hstore.compaction.date.tiered.single.output.for.minor.compaction"; 079 080 private static final Class< 081 ? extends RatioBasedCompactionPolicy> DEFAULT_COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS = 082 ExploringCompactionPolicy.class; 083 084 public static final String DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS_KEY = 085 "hbase.hstore.compaction.date.tiered.window.factory.class"; 086 087 private static final Class< 088 ? extends CompactionWindowFactory> DEFAULT_DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS = 089 ExponentialCompactionWindowFactory.class; 090 091 public static final String DATE_TIERED_STORAGE_POLICY_ENABLE_KEY = 092 "hbase.hstore.compaction.date.tiered.storage.policy.enable"; 093 public static final String DATE_TIERED_HOT_WINDOW_AGE_MILLIS_KEY = 094 "hbase.hstore.compaction.date.tiered.hot.window.age.millis"; 095 public static final String DATE_TIERED_HOT_WINDOW_STORAGE_POLICY_KEY = 096 "hbase.hstore.compaction.date.tiered.hot.window.storage.policy"; 097 public static final String DATE_TIERED_WARM_WINDOW_AGE_MILLIS_KEY = 098 "hbase.hstore.compaction.date.tiered.warm.window.age.millis"; 099 public static final String DATE_TIERED_WARM_WINDOW_STORAGE_POLICY_KEY = 100 "hbase.hstore.compaction.date.tiered.warm.window.storage.policy"; 101 /** Windows older than warm age belong to COLD_WINDOW **/ 102 public static final String DATE_TIERED_COLD_WINDOW_STORAGE_POLICY_KEY = 103 "hbase.hstore.compaction.date.tiered.cold.window.storage.policy"; 104 105 Configuration conf; 106 StoreConfigInformation storeConfigInfo; 107 108 private final double offPeakCompactionRatio; 109 /** Since all these properties can change online, they are volatile **/ 110 private final long maxCompactSize; 111 private final long offPeakMaxCompactSize; 112 private final long minCompactSize; 113 /** This one can be update **/ 114 private int minFilesToCompact; 115 private final int maxFilesToCompact; 116 private final double compactionRatio; 117 private final long throttlePoint; 118 private final long majorCompactionPeriod; 119 private final float majorCompactionJitter; 120 private final float minLocalityToForceCompact; 121 private final long dateTieredMaxStoreFileAgeMillis; 122 private final int dateTieredIncomingWindowMin; 123 private final String compactionPolicyForDateTieredWindow; 124 private final boolean dateTieredSingleOutputForMinorCompaction; 125 private final String dateTieredCompactionWindowFactory; 126 private final boolean dateTieredStoragePolicyEnable; 127 private long hotWindowAgeMillis; 128 private long warmWindowAgeMillis; 129 private String hotWindowStoragePolicy; 130 private String warmWindowStoragePolicy; 131 private String coldWindowStoragePolicy; 132 133 CompactionConfiguration(Configuration conf, StoreConfigInformation storeConfigInfo) { 134 this.conf = conf; 135 this.storeConfigInfo = storeConfigInfo; 136 137 maxCompactSize = conf.getLong(HBASE_HSTORE_COMPACTION_MAX_SIZE_KEY, Long.MAX_VALUE); 138 offPeakMaxCompactSize = 139 conf.getLong(HBASE_HSTORE_COMPACTION_MAX_SIZE_OFFPEAK_KEY, maxCompactSize); 140 minCompactSize = 141 conf.getLong(HBASE_HSTORE_COMPACTION_MIN_SIZE_KEY, storeConfigInfo.getMemStoreFlushSize()); 142 minFilesToCompact = Math.max(2, conf.getInt(HBASE_HSTORE_COMPACTION_MIN_KEY, 143 /* old name */ conf.getInt("hbase.hstore.compactionThreshold", 3))); 144 maxFilesToCompact = conf.getInt(HBASE_HSTORE_COMPACTION_MAX_KEY, 10); 145 compactionRatio = conf.getFloat(HBASE_HSTORE_COMPACTION_RATIO_KEY, 1.2F); 146 offPeakCompactionRatio = conf.getFloat(HBASE_HSTORE_COMPACTION_RATIO_OFFPEAK_KEY, 5.0F); 147 148 throttlePoint = conf.getLong("hbase.regionserver.thread.compaction.throttle", 149 2 * maxFilesToCompact * storeConfigInfo.getMemStoreFlushSize()); 150 majorCompactionPeriod = 151 conf.getLong(HConstants.MAJOR_COMPACTION_PERIOD, HConstants.DEFAULT_MAJOR_COMPACTION_PERIOD); 152 majorCompactionJitter = 153 conf.getFloat(HConstants.MAJOR_COMPACTION_JITTER, HConstants.DEFAULT_MAJOR_COMPACTION_JITTER); 154 minLocalityToForceCompact = conf.getFloat(HBASE_HSTORE_MIN_LOCALITY_TO_SKIP_MAJOR_COMPACT, 0f); 155 156 dateTieredMaxStoreFileAgeMillis = conf.getLong(DATE_TIERED_MAX_AGE_MILLIS_KEY, Long.MAX_VALUE); 157 dateTieredIncomingWindowMin = conf.getInt(DATE_TIERED_INCOMING_WINDOW_MIN_KEY, 6); 158 compactionPolicyForDateTieredWindow = 159 conf.get(COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS_KEY, 160 DEFAULT_COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS.getName()); 161 dateTieredSingleOutputForMinorCompaction = 162 conf.getBoolean(DATE_TIERED_SINGLE_OUTPUT_FOR_MINOR_COMPACTION_KEY, true); 163 this.dateTieredCompactionWindowFactory = 164 conf.get(DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS_KEY, 165 DEFAULT_DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS.getName()); 166 // for Heterogeneous Storage 167 dateTieredStoragePolicyEnable = conf.getBoolean(DATE_TIERED_STORAGE_POLICY_ENABLE_KEY, false); 168 hotWindowAgeMillis = conf.getLong(DATE_TIERED_HOT_WINDOW_AGE_MILLIS_KEY, 86400000L); 169 hotWindowStoragePolicy = conf.get(DATE_TIERED_HOT_WINDOW_STORAGE_POLICY_KEY, "ALL_SSD"); 170 warmWindowAgeMillis = conf.getLong(DATE_TIERED_WARM_WINDOW_AGE_MILLIS_KEY, 604800000L); 171 warmWindowStoragePolicy = conf.get(DATE_TIERED_WARM_WINDOW_STORAGE_POLICY_KEY, "ONE_SSD"); 172 coldWindowStoragePolicy = conf.get(DATE_TIERED_COLD_WINDOW_STORAGE_POLICY_KEY, "HOT"); 173 LOG.info(toString()); 174 } 175 176 @Override 177 public String toString() { 178 return String.format( 179 "size [minCompactSize:%s, maxCompactSize:%s, offPeakMaxCompactSize:%s);" 180 + " files [minFilesToCompact:%d, maxFilesToCompact:%d);" 181 + " ratio %f; off-peak ratio %f; throttle point %d;" 182 + " major period %d, major jitter %f, min locality to compact %f;" 183 + " tiered compaction: max_age %d, incoming window min %d," 184 + " compaction policy for tiered window %s, single output for minor %b," 185 + " compaction window factory %s," + " region %s columnFamilyName %s", 186 StringUtils.byteDesc(minCompactSize), StringUtils.byteDesc(maxCompactSize), 187 StringUtils.byteDesc(offPeakMaxCompactSize), minFilesToCompact, maxFilesToCompact, 188 compactionRatio, offPeakCompactionRatio, throttlePoint, majorCompactionPeriod, 189 majorCompactionJitter, minLocalityToForceCompact, dateTieredMaxStoreFileAgeMillis, 190 dateTieredIncomingWindowMin, compactionPolicyForDateTieredWindow, 191 dateTieredSingleOutputForMinorCompaction, dateTieredCompactionWindowFactory, 192 RegionInfo.prettyPrint(storeConfigInfo.getRegionInfo().getEncodedName()), 193 storeConfigInfo.getColumnFamilyName()); 194 } 195 196 /** Returns lower bound below which compaction is selected without ratio test */ 197 public long getMinCompactSize() { 198 return minCompactSize; 199 } 200 201 /** Returns upper bound on file size to be included in minor compactions */ 202 public long getMaxCompactSize() { 203 return maxCompactSize; 204 } 205 206 /** Returns upper bound on number of files to be included in minor compactions */ 207 public int getMinFilesToCompact() { 208 return minFilesToCompact; 209 } 210 211 /** 212 * Set upper bound on number of files to be included in minor compactions 213 * @param threshold value to set to 214 */ 215 public void setMinFilesToCompact(int threshold) { 216 minFilesToCompact = threshold; 217 } 218 219 /** Returns upper bound on number of files to be included in minor compactions */ 220 public int getMaxFilesToCompact() { 221 return maxFilesToCompact; 222 } 223 224 /** Returns Ratio used for compaction */ 225 public double getCompactionRatio() { 226 return compactionRatio; 227 } 228 229 /** Returns Off peak Ratio used for compaction */ 230 public double getCompactionRatioOffPeak() { 231 return offPeakCompactionRatio; 232 } 233 234 /** Returns ThrottlePoint used for classifying small and large compactions */ 235 public long getThrottlePoint() { 236 return throttlePoint; 237 } 238 239 /** 240 * @return Major compaction period from compaction. Major compactions are selected periodically 241 * according to this parameter plus jitter 242 */ 243 public long getMajorCompactionPeriod() { 244 return majorCompactionPeriod; 245 } 246 247 /** 248 * @return Major the jitter fraction, the fraction within which the major compaction period is 249 * randomly chosen from the majorCompactionPeriod in each store. 250 */ 251 public float getMajorCompactionJitter() { 252 return majorCompactionJitter; 253 } 254 255 /** 256 * @return Block locality ratio, the ratio at which we will include old regions with a single 257 * store file for major compaction. Used to improve block locality for regions that 258 * haven't had writes in a while but are still being read. 259 */ 260 public float getMinLocalityToForceCompact() { 261 return minLocalityToForceCompact; 262 } 263 264 public long getOffPeakMaxCompactSize() { 265 return offPeakMaxCompactSize; 266 } 267 268 public long getMaxCompactSize(boolean mayUseOffpeak) { 269 if (mayUseOffpeak) { 270 return getOffPeakMaxCompactSize(); 271 } else { 272 return getMaxCompactSize(); 273 } 274 } 275 276 public long getDateTieredMaxStoreFileAgeMillis() { 277 return dateTieredMaxStoreFileAgeMillis; 278 } 279 280 public int getDateTieredIncomingWindowMin() { 281 return dateTieredIncomingWindowMin; 282 } 283 284 public String getCompactionPolicyForDateTieredWindow() { 285 return compactionPolicyForDateTieredWindow; 286 } 287 288 public boolean useDateTieredSingleOutputForMinorCompaction() { 289 return dateTieredSingleOutputForMinorCompaction; 290 } 291 292 public String getDateTieredCompactionWindowFactory() { 293 return dateTieredCompactionWindowFactory; 294 } 295 296 public boolean isDateTieredStoragePolicyEnable() { 297 return dateTieredStoragePolicyEnable; 298 } 299 300 public long getHotWindowAgeMillis() { 301 return hotWindowAgeMillis; 302 } 303 304 public long getWarmWindowAgeMillis() { 305 return warmWindowAgeMillis; 306 } 307 308 public String getHotWindowStoragePolicy() { 309 return hotWindowStoragePolicy.trim().toUpperCase(); 310 } 311 312 public String getWarmWindowStoragePolicy() { 313 return warmWindowStoragePolicy.trim().toUpperCase(); 314 } 315 316 public String getColdWindowStoragePolicy() { 317 return coldWindowStoragePolicy.trim().toUpperCase(); 318 } 319}