View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.io.util;
19  
20  import java.lang.management.ManagementFactory;
21  import java.lang.management.MemoryUsage;
22  
23  import org.apache.commons.logging.Log;
24  import org.apache.commons.logging.LogFactory;
25  import org.apache.hadoop.hbase.classification.InterfaceAudience;
26  import org.apache.hadoop.conf.Configuration;
27  import org.apache.hadoop.hbase.HConstants;
28  
29  @InterfaceAudience.Private
30  public class HeapMemorySizeUtil {
31  
32    public static final String MEMSTORE_SIZE_KEY = "hbase.regionserver.global.memstore.size";
33    public static final String MEMSTORE_SIZE_OLD_KEY =
34        "hbase.regionserver.global.memstore.upperLimit";
35    public static final String MEMSTORE_SIZE_LOWER_LIMIT_KEY =
36        "hbase.regionserver.global.memstore.size.lower.limit";
37    public static final String MEMSTORE_SIZE_LOWER_LIMIT_OLD_KEY =
38        "hbase.regionserver.global.memstore.lowerLimit";
39  
40    public static final float DEFAULT_MEMSTORE_SIZE = 0.4f;
41    // Default lower water mark limit is 95% size of memstore size.
42    public static final float DEFAULT_MEMSTORE_SIZE_LOWER_LIMIT = 0.95f;
43  
44    private static final Log LOG = LogFactory.getLog(HeapMemorySizeUtil.class);
45    // a constant to convert a fraction to a percentage
46    private static final int CONVERT_TO_PERCENTAGE = 100;
47  
48    /**
49     * Checks whether we have enough heap memory left out after portion for Memstore and Block cache.
50     * We need atleast 20% of heap left out for other RS functions.
51     * @param conf
52     */
53    public static void checkForClusterFreeMemoryLimit(Configuration conf) {
54      if (conf.get(MEMSTORE_SIZE_OLD_KEY) != null) {
55        LOG.warn(MEMSTORE_SIZE_OLD_KEY + " is deprecated by " + MEMSTORE_SIZE_KEY);
56      }
57      float globalMemstoreSize = getGlobalMemStorePercent(conf, false);
58      int gml = (int)(globalMemstoreSize * CONVERT_TO_PERCENTAGE);
59      float blockCacheUpperLimit = getBlockCacheHeapPercent(conf);
60      int bcul = (int)(blockCacheUpperLimit * CONVERT_TO_PERCENTAGE);
61      if (CONVERT_TO_PERCENTAGE - (gml + bcul)
62              < (int)(CONVERT_TO_PERCENTAGE *
63                      HConstants.HBASE_CLUSTER_MINIMUM_MEMORY_THRESHOLD)) {
64        throw new RuntimeException("Current heap configuration for MemStore and BlockCache exceeds "
65            + "the threshold required for successful cluster operation. "
66            + "The combined value cannot exceed 0.8. Please check "
67            + "the settings for hbase.regionserver.global.memstore.size and "
68            + "hfile.block.cache.size in your configuration. "
69            + "hbase.regionserver.global.memstore.size is " + globalMemstoreSize
70            + " hfile.block.cache.size is " + blockCacheUpperLimit);
71      }
72    }
73  
74    /**
75     * Retrieve global memstore configured size as percentage of total heap.
76     * @param c
77     * @param logInvalid
78     */
79    public static float getGlobalMemStorePercent(final Configuration c, final boolean logInvalid) {
80      float limit = c.getFloat(MEMSTORE_SIZE_KEY,
81          c.getFloat(MEMSTORE_SIZE_OLD_KEY, DEFAULT_MEMSTORE_SIZE));
82      if (limit > 0.8f || limit < 0.05f) {
83        if (logInvalid) {
84          LOG.warn("Setting global memstore limit to default of " + DEFAULT_MEMSTORE_SIZE
85              + " because supplied value outside allowed range of 0.05 -> 0.8");
86        }
87        limit = DEFAULT_MEMSTORE_SIZE;
88      }
89      return limit;
90    }
91  
92    /**
93     * Retrieve configured size for global memstore lower water mark as percentage of total heap.
94     * @param c
95     * @param globalMemStorePercent
96     */
97    public static float getGlobalMemStoreLowerMark(final Configuration c, float globalMemStorePercent) {
98      String lowMarkPercentStr = c.get(MEMSTORE_SIZE_LOWER_LIMIT_KEY);
99      if (lowMarkPercentStr != null) {
100       return Float.parseFloat(lowMarkPercentStr);
101     }
102     String lowerWaterMarkOldValStr = c.get(MEMSTORE_SIZE_LOWER_LIMIT_OLD_KEY);
103     if (lowerWaterMarkOldValStr != null) {
104       LOG.warn(MEMSTORE_SIZE_LOWER_LIMIT_OLD_KEY + " is deprecated. Instead use "
105           + MEMSTORE_SIZE_LOWER_LIMIT_KEY);
106       float lowerWaterMarkOldVal = Float.parseFloat(lowerWaterMarkOldValStr);
107       if (lowerWaterMarkOldVal > globalMemStorePercent) {
108         lowerWaterMarkOldVal = globalMemStorePercent;
109         LOG.info("Setting globalMemStoreLimitLowMark == globalMemStoreLimit " + "because supplied "
110             + MEMSTORE_SIZE_LOWER_LIMIT_OLD_KEY + " was > " + MEMSTORE_SIZE_OLD_KEY);
111       }
112       return lowerWaterMarkOldVal / globalMemStorePercent;
113     }
114     return DEFAULT_MEMSTORE_SIZE_LOWER_LIMIT;
115   }
116 
117   /**
118    * Retrieve configured size for on heap block cache as percentage of total heap.
119    * @param conf
120    */
121   public static float getBlockCacheHeapPercent(final Configuration conf) {
122     // L1 block cache is always on heap
123     float l1CachePercent = conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY,
124         HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
125     float l2CachePercent = getL2BlockCacheHeapPercent(conf);
126     return l1CachePercent + l2CachePercent;
127   }
128 
129   /**
130    * @param conf
131    * @return The on heap size for L2 block cache.
132    */
133   public static float getL2BlockCacheHeapPercent(Configuration conf) {
134     float l2CachePercent = 0.0F;
135     String bucketCacheIOEngineName = conf.get(HConstants.BUCKET_CACHE_IOENGINE_KEY, null);
136     // L2 block cache can be on heap when IOEngine is "heap"
137     if (bucketCacheIOEngineName != null && bucketCacheIOEngineName.startsWith("heap")) {
138       float bucketCachePercentage = conf.getFloat(HConstants.BUCKET_CACHE_SIZE_KEY, 0F);
139       MemoryUsage mu = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
140       l2CachePercent = bucketCachePercentage < 1 ? bucketCachePercentage
141           : (bucketCachePercentage * 1024 * 1024) / mu.getMax();
142     }
143     return l2CachePercent;
144   }
145 }