001/** 002 * 003 * Licensed to the Apache Software Foundation (ASF) under one 004 * or more contributor license agreements. See the NOTICE file 005 * distributed with this work for additional information 006 * regarding copyright ownership. The ASF licenses this file 007 * to you under the Apache License, Version 2.0 (the 008 * "License"); you may not use this file except in compliance 009 * with the License. You may obtain a copy of the License at 010 * 011 * http://www.apache.org/licenses/LICENSE-2.0 012 * 013 * Unless required by applicable law or agreed to in writing, software 014 * distributed under the License is distributed on an "AS IS" BASIS, 015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 016 * See the License for the specific language governing permissions and 017 * limitations under the License. 018 */ 019 020package org.apache.hadoop.hbase.regionserver.compactions; 021 022import org.apache.hadoop.conf.Configuration; 023import org.apache.hadoop.hbase.HConstants; 024import org.apache.hadoop.hbase.client.RegionInfo; 025import org.apache.hadoop.util.StringUtils; 026import org.apache.yetus.audience.InterfaceAudience; 027import org.slf4j.Logger; 028import org.slf4j.LoggerFactory; 029import org.apache.hadoop.hbase.regionserver.StoreConfigInformation; 030 031/** 032 * <p> 033 * Compaction configuration for a particular instance of HStore. 034 * Takes into account both global settings and ones set on the column family/store. 035 * Control knobs for default compaction algorithm: 036 * </p> 037 * <p> 038 * maxCompactSize - upper bound on file size to be included in minor compactions 039 * minCompactSize - lower bound below which compaction is selected without ratio test 040 * minFilesToCompact - lower bound on number of files in any minor compaction 041 * maxFilesToCompact - upper bound on number of files in any minor compaction 042 * compactionRatio - Ratio used for compaction 043 * minLocalityToForceCompact - Locality threshold for a store file to major compact (HBASE-11195) 044 * </p> 045 * Set parameter as "hbase.hstore.compaction.<attribute>" 046 */ 047 048@InterfaceAudience.Private 049public class CompactionConfiguration { 050 051 private static final Logger LOG = LoggerFactory.getLogger(CompactionConfiguration.class); 052 053 public static final String HBASE_HSTORE_COMPACTION_RATIO_KEY = "hbase.hstore.compaction.ratio"; 054 public static final String HBASE_HSTORE_COMPACTION_RATIO_OFFPEAK_KEY = 055 "hbase.hstore.compaction.ratio.offpeak"; 056 public static final String HBASE_HSTORE_COMPACTION_MIN_KEY = "hbase.hstore.compaction.min"; 057 public static final String HBASE_HSTORE_COMPACTION_MIN_SIZE_KEY = 058 "hbase.hstore.compaction.min.size"; 059 public static final String HBASE_HSTORE_COMPACTION_MAX_KEY = "hbase.hstore.compaction.max"; 060 public static final String HBASE_HSTORE_COMPACTION_MAX_SIZE_KEY = 061 "hbase.hstore.compaction.max.size"; 062 public static final String HBASE_HSTORE_COMPACTION_MAX_SIZE_OFFPEAK_KEY = 063 "hbase.hstore.compaction.max.size.offpeak"; 064 public static final String HBASE_HSTORE_OFFPEAK_END_HOUR = "hbase.offpeak.end.hour"; 065 public static final String HBASE_HSTORE_OFFPEAK_START_HOUR = "hbase.offpeak.start.hour"; 066 public static final String HBASE_HSTORE_MIN_LOCALITY_TO_SKIP_MAJOR_COMPACT = 067 "hbase.hstore.min.locality.to.skip.major.compact"; 068 069 public static final String HBASE_HFILE_COMPACTION_DISCHARGER_THREAD_COUNT = 070 "hbase.hfile.compaction.discharger.thread.count"; 071 072 /* 073 * The epoch time length for the windows we no longer compact 074 */ 075 public static final String DATE_TIERED_MAX_AGE_MILLIS_KEY = 076 "hbase.hstore.compaction.date.tiered.max.storefile.age.millis"; 077 public static final String DATE_TIERED_INCOMING_WINDOW_MIN_KEY = 078 "hbase.hstore.compaction.date.tiered.incoming.window.min"; 079 public static final String COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS_KEY = 080 "hbase.hstore.compaction.date.tiered.window.policy.class"; 081 public static final String DATE_TIERED_SINGLE_OUTPUT_FOR_MINOR_COMPACTION_KEY = 082 "hbase.hstore.compaction.date.tiered.single.output.for.minor.compaction"; 083 084 private static final Class<? extends RatioBasedCompactionPolicy> 085 DEFAULT_COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS = ExploringCompactionPolicy.class; 086 087 public static final String DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS_KEY = 088 "hbase.hstore.compaction.date.tiered.window.factory.class"; 089 090 private static final Class<? extends CompactionWindowFactory> 091 DEFAULT_DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS = ExponentialCompactionWindowFactory.class; 092 093 Configuration conf; 094 StoreConfigInformation storeConfigInfo; 095 096 private final double offPeakCompactionRatio; 097 /** Since all these properties can change online, they are volatile **/ 098 private final long maxCompactSize; 099 private final long offPeakMaxCompactSize; 100 private final long minCompactSize; 101 /** This one can be update **/ 102 private int minFilesToCompact; 103 private final int maxFilesToCompact; 104 private final double compactionRatio; 105 private final long throttlePoint; 106 private final long majorCompactionPeriod; 107 private final float majorCompactionJitter; 108 private final float minLocalityToForceCompact; 109 private final long dateTieredMaxStoreFileAgeMillis; 110 private final int dateTieredIncomingWindowMin; 111 private final String compactionPolicyForDateTieredWindow; 112 private final boolean dateTieredSingleOutputForMinorCompaction; 113 private final String dateTieredCompactionWindowFactory; 114 115 CompactionConfiguration(Configuration conf, StoreConfigInformation storeConfigInfo) { 116 this.conf = conf; 117 this.storeConfigInfo = storeConfigInfo; 118 119 maxCompactSize = conf.getLong(HBASE_HSTORE_COMPACTION_MAX_SIZE_KEY, Long.MAX_VALUE); 120 offPeakMaxCompactSize = conf.getLong(HBASE_HSTORE_COMPACTION_MAX_SIZE_OFFPEAK_KEY, 121 maxCompactSize); 122 minCompactSize = conf.getLong(HBASE_HSTORE_COMPACTION_MIN_SIZE_KEY, 123 storeConfigInfo.getMemStoreFlushSize()); 124 minFilesToCompact = Math.max(2, conf.getInt(HBASE_HSTORE_COMPACTION_MIN_KEY, 125 /*old name*/ conf.getInt("hbase.hstore.compactionThreshold", 3))); 126 maxFilesToCompact = conf.getInt(HBASE_HSTORE_COMPACTION_MAX_KEY, 10); 127 compactionRatio = conf.getFloat(HBASE_HSTORE_COMPACTION_RATIO_KEY, 1.2F); 128 offPeakCompactionRatio = conf.getFloat(HBASE_HSTORE_COMPACTION_RATIO_OFFPEAK_KEY, 5.0F); 129 130 throttlePoint = conf.getLong("hbase.regionserver.thread.compaction.throttle", 131 2 * maxFilesToCompact * storeConfigInfo.getMemStoreFlushSize()); 132 majorCompactionPeriod = conf.getLong(HConstants.MAJOR_COMPACTION_PERIOD, 133 HConstants.DEFAULT_MAJOR_COMPACTION_PERIOD); 134 majorCompactionJitter = conf.getFloat(HConstants.MAJOR_COMPACTION_JITTER, 135 HConstants.DEFAULT_MAJOR_COMPACTION_JITTER); 136 minLocalityToForceCompact = conf.getFloat(HBASE_HSTORE_MIN_LOCALITY_TO_SKIP_MAJOR_COMPACT, 0f); 137 138 dateTieredMaxStoreFileAgeMillis = conf.getLong(DATE_TIERED_MAX_AGE_MILLIS_KEY, Long.MAX_VALUE); 139 dateTieredIncomingWindowMin = conf.getInt(DATE_TIERED_INCOMING_WINDOW_MIN_KEY, 6); 140 compactionPolicyForDateTieredWindow = conf.get( 141 COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS_KEY, 142 DEFAULT_COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS.getName()); 143 dateTieredSingleOutputForMinorCompaction = conf 144 .getBoolean(DATE_TIERED_SINGLE_OUTPUT_FOR_MINOR_COMPACTION_KEY, true); 145 this.dateTieredCompactionWindowFactory = conf.get( 146 DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS_KEY, 147 DEFAULT_DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS.getName()); 148 LOG.info(toString()); 149 } 150 151 @Override 152 public String toString() { 153 return String.format( 154 "size [%s, %s, %s); files [%d, %d); ratio %f; off-peak ratio %f; throttle point %d;" 155 + " major period %d, major jitter %f, min locality to compact %f;" 156 + " tiered compaction: max_age %d, incoming window min %d," 157 + " compaction policy for tiered window %s, single output for minor %b," 158 + " compaction window factory %s," 159 + " region %s columnFamilyName %s", 160 StringUtils.byteDesc(minCompactSize), 161 StringUtils.byteDesc(maxCompactSize), 162 StringUtils.byteDesc(offPeakMaxCompactSize), 163 minFilesToCompact, 164 maxFilesToCompact, 165 compactionRatio, 166 offPeakCompactionRatio, 167 throttlePoint, 168 majorCompactionPeriod, 169 majorCompactionJitter, 170 minLocalityToForceCompact, 171 dateTieredMaxStoreFileAgeMillis, 172 dateTieredIncomingWindowMin, 173 compactionPolicyForDateTieredWindow, 174 dateTieredSingleOutputForMinorCompaction, 175 dateTieredCompactionWindowFactory, 176 RegionInfo.prettyPrint(storeConfigInfo.getRegionInfo().getEncodedName()), 177 storeConfigInfo.getColumnFamilyName() 178 ); 179 } 180 181 /** 182 * @return lower bound below which compaction is selected without ratio test 183 */ 184 public long getMinCompactSize() { 185 return minCompactSize; 186 } 187 188 /** 189 * @return upper bound on file size to be included in minor compactions 190 */ 191 public long getMaxCompactSize() { 192 return maxCompactSize; 193 } 194 195 /** 196 * @return upper bound on number of files to be included in minor compactions 197 */ 198 public int getMinFilesToCompact() { 199 return minFilesToCompact; 200 } 201 202 /** 203 * Set upper bound on number of files to be included in minor compactions 204 * @param threshold value to set to 205 */ 206 public void setMinFilesToCompact(int threshold) { 207 minFilesToCompact = threshold; 208 } 209 210 /** 211 * @return upper bound on number of files to be included in minor compactions 212 */ 213 public int getMaxFilesToCompact() { 214 return maxFilesToCompact; 215 } 216 217 /** 218 * @return Ratio used for compaction 219 */ 220 public double getCompactionRatio() { 221 return compactionRatio; 222 } 223 224 /** 225 * @return Off peak Ratio used for compaction 226 */ 227 public double getCompactionRatioOffPeak() { 228 return offPeakCompactionRatio; 229 } 230 231 /** 232 * @return ThrottlePoint used for classifying small and large compactions 233 */ 234 public long getThrottlePoint() { 235 return throttlePoint; 236 } 237 238 /** 239 * @return Major compaction period from compaction. 240 * Major compactions are selected periodically according to this parameter plus jitter 241 */ 242 public long getMajorCompactionPeriod() { 243 return majorCompactionPeriod; 244 } 245 246 /** 247 * @return Major the jitter fraction, the fraction within which the major compaction 248 * period is randomly chosen from the majorCompactionPeriod in each store. 249 */ 250 public float getMajorCompactionJitter() { 251 return majorCompactionJitter; 252 } 253 254 /** 255 * @return Block locality ratio, the ratio at which we will include old regions with a single 256 * store file for major compaction. Used to improve block locality for regions that 257 * haven't had writes in a while but are still being read. 258 */ 259 public float getMinLocalityToForceCompact() { 260 return minLocalityToForceCompact; 261 } 262 263 public long getOffPeakMaxCompactSize() { 264 return offPeakMaxCompactSize; 265 } 266 267 public long getMaxCompactSize(boolean mayUseOffpeak) { 268 if (mayUseOffpeak) { 269 return getOffPeakMaxCompactSize(); 270 } else { 271 return getMaxCompactSize(); 272 } 273 } 274 275 public long getDateTieredMaxStoreFileAgeMillis() { 276 return dateTieredMaxStoreFileAgeMillis; 277 } 278 279 public int getDateTieredIncomingWindowMin() { 280 return dateTieredIncomingWindowMin; 281 } 282 283 public String getCompactionPolicyForDateTieredWindow() { 284 return compactionPolicyForDateTieredWindow; 285 } 286 287 public boolean useDateTieredSingleOutputForMinorCompaction() { 288 return dateTieredSingleOutputForMinorCompaction; 289 } 290 291 public String getDateTieredCompactionWindowFactory() { 292 return dateTieredCompactionWindowFactory; 293 } 294}