1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21 import static org.apache.hadoop.hbase.regionserver.HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY;
22 import static org.apache.hadoop.hbase.regionserver.HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY;
23 import static org.apache.hadoop.hbase.HConstants.HFILE_BLOCK_CACHE_SIZE_KEY;
24 import static org.apache.hadoop.hbase.regionserver.HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY;
25 import static org.apache.hadoop.hbase.regionserver.HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY;
26
27 import org.apache.commons.logging.Log;
28 import org.apache.commons.logging.LogFactory;
29 import org.apache.hadoop.hbase.classification.InterfaceAudience;
30 import org.apache.hadoop.conf.Configuration;
31 import org.apache.hadoop.hbase.HConstants;
32 import org.apache.hadoop.hbase.io.util.HeapMemorySizeUtil;
33 import org.apache.hadoop.hbase.regionserver.HeapMemoryManager.TunerContext;
34 import org.apache.hadoop.hbase.regionserver.HeapMemoryManager.TunerResult;
35 import org.apache.hadoop.hbase.util.RollingStatCalculator;
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70 @InterfaceAudience.Private
71 class DefaultHeapMemoryTuner implements HeapMemoryTuner {
72 public static final String MAX_STEP_KEY = "hbase.regionserver.heapmemory.autotuner.step.max";
73 public static final String MIN_STEP_KEY = "hbase.regionserver.heapmemory.autotuner.step.min";
74 public static final String SUFFICIENT_MEMORY_LEVEL_KEY =
75 "hbase.regionserver.heapmemory.autotuner.sufficient.memory.level";
76 public static final String LOOKUP_PERIODS_KEY =
77 "hbase.regionserver.heapmemory.autotuner.lookup.periods";
78 public static final String NUM_PERIODS_TO_IGNORE =
79 "hbase.regionserver.heapmemory.autotuner.ignored.periods";
80
81 public static final float DEFAULT_MAX_STEP_VALUE = 0.04f;
82
83 public static final float DEFAULT_MIN_STEP_VALUE = 0.00125f;
84
85
86 public static final float DEFAULT_SUFFICIENT_MEMORY_LEVEL_VALUE = 0.5f;
87
88
89 public static final int DEFAULT_LOOKUP_PERIODS = 60;
90 public static final int DEFAULT_NUM_PERIODS_IGNORED = 60;
91 private static final TunerResult NO_OP_TUNER_RESULT = new TunerResult(false);
92
93
94 private static final double TUNER_STEP_EPS = 1e-6;
95
96 private Log LOG = LogFactory.getLog(DefaultHeapMemoryTuner.class);
97 private TunerResult TUNER_RESULT = new TunerResult(true);
98 private Configuration conf;
99 private float sufficientMemoryLevel = DEFAULT_SUFFICIENT_MEMORY_LEVEL_VALUE;
100 private float maximumStepSize = DEFAULT_MAX_STEP_VALUE;
101 private float minimumStepSize = DEFAULT_MIN_STEP_VALUE;
102 private int tunerLookupPeriods = DEFAULT_LOOKUP_PERIODS;
103 private int numPeriodsToIgnore = DEFAULT_NUM_PERIODS_IGNORED;
104
105
106 private int ignoreInitialPeriods = 0;
107
108 private float globalMemStorePercentMinRange;
109 private float globalMemStorePercentMaxRange;
110 private float blockCachePercentMinRange;
111 private float blockCachePercentMaxRange;
112
113 private RollingStatCalculator rollingStatsForCacheMisses;
114 private RollingStatCalculator rollingStatsForFlushes;
115 private RollingStatCalculator rollingStatsForEvictions;
116 private RollingStatCalculator rollingStatsForTunerSteps;
117
118 private float step = DEFAULT_MAX_STEP_VALUE;
119 private StepDirection prevTuneDirection = StepDirection.NEUTRAL;
120
121
122
123 private double decayingTunerStepSizeSum = 0;
124
125 @Override
126 public TunerResult tune(TunerContext context) {
127 float curMemstoreSize = context.getCurMemStoreSize();
128 float curBlockCacheSize = context.getCurBlockCacheSize();
129 addToRollingStats(context);
130
131 if (ignoreInitialPeriods < numPeriodsToIgnore) {
132
133 ignoreInitialPeriods++;
134 rollingStatsForTunerSteps.insertDataValue(0);
135 return NO_OP_TUNER_RESULT;
136 }
137 StepDirection newTuneDirection = getTuneDirection(context);
138
139 float newMemstoreSize;
140 float newBlockCacheSize;
141
142
143
144
145
146 if (prevTuneDirection == StepDirection.NEUTRAL
147 && newTuneDirection != StepDirection.NEUTRAL
148 && rollingStatsForTunerSteps.getDeviation() < TUNER_STEP_EPS) {
149
150
151
152 step = maximumStepSize;
153 } else if ((newTuneDirection == StepDirection.INCREASE_MEMSTORE_SIZE
154 && decayingTunerStepSizeSum < 0) ||
155 (newTuneDirection == StepDirection.INCREASE_BLOCK_CACHE_SIZE
156 && decayingTunerStepSizeSum > 0)) {
157
158
159 step = step/2.00f;
160 }
161 if (step < minimumStepSize) {
162
163 LOG.debug("Tuner step size is too low; we will not perform any tuning this time.");
164 step = 0.0f;
165 newTuneDirection = StepDirection.NEUTRAL;
166 }
167
168 float globalMemstoreLowerMark = HeapMemorySizeUtil.getGlobalMemStoreLowerMark(conf,
169 curMemstoreSize);
170
171
172 float minMemstoreSize = ((globalMemstoreLowerMark + 1) * curMemstoreSize) / 2.00f;
173
174 switch (newTuneDirection) {
175 case INCREASE_BLOCK_CACHE_SIZE:
176 if (curMemstoreSize - step < minMemstoreSize) {
177 step = curMemstoreSize - minMemstoreSize;
178 }
179 newMemstoreSize = curMemstoreSize - step;
180 newBlockCacheSize = curBlockCacheSize + step;
181 rollingStatsForTunerSteps.insertDataValue(-(int)(step*100000));
182 decayingTunerStepSizeSum = (decayingTunerStepSizeSum - step)/2.00f;
183 break;
184 case INCREASE_MEMSTORE_SIZE:
185 newBlockCacheSize = curBlockCacheSize - step;
186 newMemstoreSize = curMemstoreSize + step;
187 rollingStatsForTunerSteps.insertDataValue((int)(step*100000));
188 decayingTunerStepSizeSum = (decayingTunerStepSizeSum + step)/2.00f;
189 break;
190 default:
191 prevTuneDirection = StepDirection.NEUTRAL;
192 rollingStatsForTunerSteps.insertDataValue(0);
193 decayingTunerStepSizeSum = (decayingTunerStepSizeSum)/2.00f;
194 return NO_OP_TUNER_RESULT;
195 }
196
197 if (newMemstoreSize > globalMemStorePercentMaxRange) {
198 newMemstoreSize = globalMemStorePercentMaxRange;
199 } else if (newMemstoreSize < globalMemStorePercentMinRange) {
200 newMemstoreSize = globalMemStorePercentMinRange;
201 }
202 if (newBlockCacheSize > blockCachePercentMaxRange) {
203 newBlockCacheSize = blockCachePercentMaxRange;
204 } else if (newBlockCacheSize < blockCachePercentMinRange) {
205 newBlockCacheSize = blockCachePercentMinRange;
206 }
207 TUNER_RESULT.setBlockCacheSize(newBlockCacheSize);
208 TUNER_RESULT.setMemstoreSize(newMemstoreSize);
209 prevTuneDirection = newTuneDirection;
210 return TUNER_RESULT;
211 }
212
213
214
215
216
217
218 private StepDirection getTuneDirection(TunerContext context) {
219 StepDirection newTuneDirection = StepDirection.NEUTRAL;
220 long blockedFlushCount = context.getBlockedFlushCount();
221 long unblockedFlushCount = context.getUnblockedFlushCount();
222 long evictCount = context.getEvictCount();
223 long cacheMissCount = context.getCacheMissCount();
224 long totalFlushCount = blockedFlushCount+unblockedFlushCount;
225 float curMemstoreSize = context.getCurMemStoreSize();
226 float curBlockCacheSize = context.getCurBlockCacheSize();
227 StringBuilder tunerLog = new StringBuilder();
228
229
230 boolean earlyMemstoreSufficientCheck = totalFlushCount == 0
231 || context.getCurMemStoreUsed() < curMemstoreSize * sufficientMemoryLevel;
232 boolean earlyBlockCacheSufficientCheck = evictCount == 0 ||
233 context.getCurBlockCacheUsed() < curBlockCacheSize * sufficientMemoryLevel;
234 if (earlyMemstoreSufficientCheck && earlyBlockCacheSufficientCheck) {
235
236 newTuneDirection = StepDirection.NEUTRAL;
237 } else if (earlyMemstoreSufficientCheck) {
238
239 newTuneDirection = StepDirection.INCREASE_BLOCK_CACHE_SIZE;
240 } else if (earlyBlockCacheSufficientCheck) {
241
242 newTuneDirection = StepDirection.INCREASE_MEMSTORE_SIZE;
243 } else {
244
245
246 boolean isReverting = false;
247 switch (prevTuneDirection) {
248
249
250
251 case INCREASE_BLOCK_CACHE_SIZE:
252 if ((double)evictCount > rollingStatsForEvictions.getMean() ||
253 (double)totalFlushCount > rollingStatsForFlushes.getMean() +
254 rollingStatsForFlushes.getDeviation()/2.00) {
255
256
257 newTuneDirection = StepDirection.INCREASE_MEMSTORE_SIZE;
258 tunerLog.append("We will revert previous tuning");
259 if ((double)evictCount > rollingStatsForEvictions.getMean()) {
260 tunerLog.append(" because we could not decrease evictions sufficiently.");
261 } else {
262 tunerLog.append(" because the number of flushes rose significantly.");
263 }
264 isReverting = true;
265 }
266 break;
267 case INCREASE_MEMSTORE_SIZE:
268 if ((double)totalFlushCount > rollingStatsForFlushes.getMean() ||
269 (double)evictCount > rollingStatsForEvictions.getMean() +
270 rollingStatsForEvictions.getDeviation()/2.00) {
271
272
273 newTuneDirection = StepDirection.INCREASE_BLOCK_CACHE_SIZE;
274 tunerLog.append("We will revert previous tuning");
275 if ((double)totalFlushCount > rollingStatsForFlushes.getMean()) {
276 tunerLog.append(" because we could not decrease flushes sufficiently.");
277 } else {
278 tunerLog.append(" because number of evictions rose significantly.");
279 }
280 isReverting = true;
281 }
282 break;
283 default:
284
285 break;
286 }
287
288 if (!isReverting){
289
290
291
292
293
294
295
296
297
298 if ((double)cacheMissCount < rollingStatsForCacheMisses.getMean() -
299 rollingStatsForCacheMisses.getDeviation()*0.80 &&
300 (double)totalFlushCount < rollingStatsForFlushes.getMean() -
301 rollingStatsForFlushes.getDeviation()*0.80) {
302
303 newTuneDirection = StepDirection.NEUTRAL;
304 } else if ((double)cacheMissCount > rollingStatsForCacheMisses.getMean() +
305 rollingStatsForCacheMisses.getDeviation()*0.80 &&
306 (double)totalFlushCount < rollingStatsForFlushes.getMean() -
307 rollingStatsForFlushes.getDeviation()*0.80) {
308
309 newTuneDirection = StepDirection.INCREASE_BLOCK_CACHE_SIZE;
310 tunerLog.append(
311 "Going to increase block cache size due to increase in number of cache misses.");
312 } else if ((double)cacheMissCount < rollingStatsForCacheMisses.getMean() -
313 rollingStatsForCacheMisses.getDeviation()*0.80 &&
314 (double)totalFlushCount > rollingStatsForFlushes.getMean() +
315 rollingStatsForFlushes.getDeviation()*0.80) {
316
317 newTuneDirection = StepDirection.INCREASE_MEMSTORE_SIZE;
318 tunerLog.append("Going to increase memstore size due to increase in number of flushes.");
319 } else if (blockedFlushCount > 0 && prevTuneDirection == StepDirection.NEUTRAL) {
320
321 newTuneDirection = StepDirection.INCREASE_MEMSTORE_SIZE;
322 tunerLog.append("Going to increase memstore size due to"
323 + blockedFlushCount + " blocked flushes.");
324 } else {
325
326 tunerLog.append("Going to do nothing because we "
327 + "could not determine best tuning direction");
328 newTuneDirection = StepDirection.NEUTRAL;
329 }
330 }
331 }
332 if (LOG.isDebugEnabled()) {
333 LOG.debug(tunerLog.toString());
334 }
335 return newTuneDirection;
336 }
337
338
339
340
341
342 private void addToRollingStats(TunerContext context) {
343 rollingStatsForCacheMisses.insertDataValue(context.getCacheMissCount());
344 rollingStatsForFlushes.insertDataValue(context.getBlockedFlushCount() +
345 context.getUnblockedFlushCount());
346 rollingStatsForEvictions.insertDataValue(context.getEvictCount());
347 }
348
349 @Override
350 public Configuration getConf() {
351 return this.conf;
352 }
353
354 @Override
355 public void setConf(Configuration conf) {
356 this.conf = conf;
357 this.maximumStepSize = conf.getFloat(MAX_STEP_KEY, DEFAULT_MAX_STEP_VALUE);
358 this.minimumStepSize = conf.getFloat(MIN_STEP_KEY, DEFAULT_MIN_STEP_VALUE);
359 this.step = this.maximumStepSize;
360 this.sufficientMemoryLevel = conf.getFloat(SUFFICIENT_MEMORY_LEVEL_KEY,
361 DEFAULT_SUFFICIENT_MEMORY_LEVEL_VALUE);
362 this.tunerLookupPeriods = conf.getInt(LOOKUP_PERIODS_KEY, DEFAULT_LOOKUP_PERIODS);
363 this.blockCachePercentMinRange = conf.getFloat(BLOCK_CACHE_SIZE_MIN_RANGE_KEY,
364 conf.getFloat(HFILE_BLOCK_CACHE_SIZE_KEY, HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT));
365 this.blockCachePercentMaxRange = conf.getFloat(BLOCK_CACHE_SIZE_MAX_RANGE_KEY,
366 conf.getFloat(HFILE_BLOCK_CACHE_SIZE_KEY, HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT));
367 this.globalMemStorePercentMinRange = conf.getFloat(MEMSTORE_SIZE_MIN_RANGE_KEY,
368 HeapMemorySizeUtil.getGlobalMemStorePercent(conf, false));
369 this.globalMemStorePercentMaxRange = conf.getFloat(MEMSTORE_SIZE_MAX_RANGE_KEY,
370 HeapMemorySizeUtil.getGlobalMemStorePercent(conf, false));
371
372 this.numPeriodsToIgnore = conf.getInt(NUM_PERIODS_TO_IGNORE, this.tunerLookupPeriods);
373 this.rollingStatsForCacheMisses = new RollingStatCalculator(this.tunerLookupPeriods);
374 this.rollingStatsForFlushes = new RollingStatCalculator(this.tunerLookupPeriods);
375 this.rollingStatsForEvictions = new RollingStatCalculator(this.tunerLookupPeriods);
376 this.rollingStatsForTunerSteps = new RollingStatCalculator(this.tunerLookupPeriods);
377 }
378
379 private enum StepDirection{
380
381 INCREASE_BLOCK_CACHE_SIZE,
382
383 INCREASE_MEMSTORE_SIZE,
384
385 NEUTRAL
386 }
387 }