1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.io.util;
19
20 import java.lang.management.ManagementFactory;
21 import java.lang.management.MemoryUsage;
22
23 import org.apache.commons.logging.Log;
24 import org.apache.commons.logging.LogFactory;
25 import org.apache.hadoop.hbase.classification.InterfaceAudience;
26 import org.apache.hadoop.conf.Configuration;
27 import org.apache.hadoop.hbase.HConstants;
28
29 @InterfaceAudience.Private
30 public class HeapMemorySizeUtil {
31
32 public static final String MEMSTORE_SIZE_KEY = "hbase.regionserver.global.memstore.size";
33 public static final String MEMSTORE_SIZE_OLD_KEY =
34 "hbase.regionserver.global.memstore.upperLimit";
35 public static final String MEMSTORE_SIZE_LOWER_LIMIT_KEY =
36 "hbase.regionserver.global.memstore.size.lower.limit";
37 public static final String MEMSTORE_SIZE_LOWER_LIMIT_OLD_KEY =
38 "hbase.regionserver.global.memstore.lowerLimit";
39
40 public static final float DEFAULT_MEMSTORE_SIZE = 0.4f;
41
42 public static final float DEFAULT_MEMSTORE_SIZE_LOWER_LIMIT = 0.95f;
43
44 private static final Log LOG = LogFactory.getLog(HeapMemorySizeUtil.class);
45
46 private static final int CONVERT_TO_PERCENTAGE = 100;
47
48 private static final String JVM_HEAP_EXCEPTION = "Got an exception while attempting to read " +
49 "information about the JVM heap. Please submit this log information in a bug report and " +
50 "include your JVM settings, specifically the GC in use and any -XX options. Consider " +
51 "restarting the service.";
52
53
54
55
56
57
58
59 public static MemoryUsage safeGetHeapMemoryUsage() {
60 MemoryUsage usage = null;
61 try {
62 usage = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
63 } catch (RuntimeException exception) {
64 LOG.warn(JVM_HEAP_EXCEPTION, exception);
65 }
66 return usage;
67 }
68
69
70
71
72
73
74 public static void checkForClusterFreeMemoryLimit(Configuration conf) {
75 if (conf.get(MEMSTORE_SIZE_OLD_KEY) != null) {
76 LOG.warn(MEMSTORE_SIZE_OLD_KEY + " is deprecated by " + MEMSTORE_SIZE_KEY);
77 }
78 float globalMemstoreSize = getGlobalMemStorePercent(conf, false);
79 int gml = (int)(globalMemstoreSize * CONVERT_TO_PERCENTAGE);
80 float blockCacheUpperLimit = getBlockCacheHeapPercent(conf);
81 int bcul = (int)(blockCacheUpperLimit * CONVERT_TO_PERCENTAGE);
82 if (CONVERT_TO_PERCENTAGE - (gml + bcul)
83 < (int)(CONVERT_TO_PERCENTAGE *
84 HConstants.HBASE_CLUSTER_MINIMUM_MEMORY_THRESHOLD)) {
85 throw new RuntimeException("Current heap configuration for MemStore and BlockCache exceeds "
86 + "the threshold required for successful cluster operation. "
87 + "The combined value cannot exceed 0.8. Please check "
88 + "the settings for hbase.regionserver.global.memstore.size and "
89 + "hfile.block.cache.size in your configuration. "
90 + "hbase.regionserver.global.memstore.size is " + globalMemstoreSize
91 + " hfile.block.cache.size is " + blockCacheUpperLimit);
92 }
93 }
94
95
96
97
98
99
100 public static float getGlobalMemStorePercent(final Configuration c, final boolean logInvalid) {
101 float limit = c.getFloat(MEMSTORE_SIZE_KEY,
102 c.getFloat(MEMSTORE_SIZE_OLD_KEY, DEFAULT_MEMSTORE_SIZE));
103 if (limit > 0.8f || limit <= 0.0f) {
104 if (logInvalid) {
105 LOG.warn("Setting global memstore limit to default of " + DEFAULT_MEMSTORE_SIZE
106 + " because supplied value outside allowed range of (0 -> 0.8]");
107 }
108 limit = DEFAULT_MEMSTORE_SIZE;
109 }
110 return limit;
111 }
112
113
114
115
116
117 public static float getGlobalMemStoreLowerMark(final Configuration conf, float globalMemStorePercent) {
118 String lowMarkPercentStr = conf.get(MEMSTORE_SIZE_LOWER_LIMIT_KEY);
119 if (lowMarkPercentStr != null) {
120 float lowMarkPercent = Float.parseFloat(lowMarkPercentStr);
121 if (lowMarkPercent > 1.0f) {
122 LOG.error("Bad configuration value for " + MEMSTORE_SIZE_LOWER_LIMIT_KEY + ": " +
123 lowMarkPercent + ". Using 1.0f instead.");
124 lowMarkPercent = 1.0f;
125 }
126 return lowMarkPercent;
127 }
128 String lowerWaterMarkOldValStr = conf.get(MEMSTORE_SIZE_LOWER_LIMIT_OLD_KEY);
129 if (lowerWaterMarkOldValStr != null) {
130 LOG.warn(MEMSTORE_SIZE_LOWER_LIMIT_OLD_KEY + " is deprecated. Instead use "
131 + MEMSTORE_SIZE_LOWER_LIMIT_KEY);
132 float lowerWaterMarkOldVal = Float.parseFloat(lowerWaterMarkOldValStr);
133 if (lowerWaterMarkOldVal > globalMemStorePercent) {
134 lowerWaterMarkOldVal = globalMemStorePercent;
135 LOG.error("Value of " + MEMSTORE_SIZE_LOWER_LIMIT_OLD_KEY + " (" + lowerWaterMarkOldVal
136 + ") is greater than global memstore limit (" + globalMemStorePercent + ") set by "
137 + MEMSTORE_SIZE_KEY + "/" + MEMSTORE_SIZE_OLD_KEY + ". Setting memstore lower limit "
138 + "to " + globalMemStorePercent);
139 }
140 return lowerWaterMarkOldVal / globalMemStorePercent;
141 }
142 return DEFAULT_MEMSTORE_SIZE_LOWER_LIMIT;
143 }
144
145
146
147
148
149 public static float getBlockCacheHeapPercent(final Configuration conf) {
150
151 float l1CachePercent = conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY,
152 HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
153 float l2CachePercent = getL2BlockCacheHeapPercent(conf);
154 return l1CachePercent + l2CachePercent;
155 }
156
157
158
159
160
161 public static float getL2BlockCacheHeapPercent(Configuration conf) {
162 float l2CachePercent = 0.0F;
163 String bucketCacheIOEngineName = conf.get(HConstants.BUCKET_CACHE_IOENGINE_KEY, null);
164
165 if (bucketCacheIOEngineName != null && bucketCacheIOEngineName.startsWith("heap")) {
166 float bucketCachePercentage = conf.getFloat(HConstants.BUCKET_CACHE_SIZE_KEY, 0F);
167 long max = -1L;
168 final MemoryUsage usage = safeGetHeapMemoryUsage();
169 if (usage != null) {
170 max = usage.getMax();
171 }
172 l2CachePercent = bucketCachePercentage < 1 ? bucketCachePercentage
173 : (bucketCachePercentage * 1024 * 1024) / max;
174 }
175 return l2CachePercent;
176 }
177
178
179
180
181
182
183 public static long getLruCacheSize(final Configuration conf) {
184 float cachePercentage = conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY,
185 HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
186 if (cachePercentage <= 0.0001f) {
187 return -1;
188 }
189 if (cachePercentage > 1.0) {
190 throw new IllegalArgumentException(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY +
191 " must be between 0.0 and 1.0, and not > 1.0");
192 }
193 long max = -1L;
194 final MemoryUsage usage = safeGetHeapMemoryUsage();
195 if (usage != null) {
196 max = usage.getMax();
197 }
198
199
200 return (long) (max * cachePercentage);
201 }
202
203
204
205
206
207 public static long getBucketCacheSize(final Configuration conf) {
208 final float bucketCachePercentage = conf.getFloat(HConstants.BUCKET_CACHE_SIZE_KEY, 0F);
209 long bucketCacheSize;
210
211 if (bucketCachePercentage < 1) {
212 long max = -1L;
213 final MemoryUsage usage = safeGetHeapMemoryUsage();
214 if (usage != null) {
215 max = usage.getMax();
216 }
217 bucketCacheSize = (long)(max * bucketCachePercentage);
218
219 } else {
220 bucketCacheSize = (long)(bucketCachePercentage * 1024 * 1024);
221 }
222 return bucketCacheSize;
223 }
224
225 }