View Javadoc

1   /**
2    * Copyright The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  
21  package org.apache.hadoop.hbase.util;
22  
23  import java.lang.reflect.Field;
24  import java.lang.reflect.Modifier;
25  import java.util.concurrent.ConcurrentHashMap;
26  import java.util.concurrent.ConcurrentSkipListMap;
27  
28  import org.apache.commons.logging.Log;
29  import org.apache.commons.logging.LogFactory;
30  import org.apache.hadoop.hbase.classification.InterfaceAudience;
31  
32  /**
33   * Class for determining the "size" of a class, an attempt to calculate the
34   * actual bytes that an object of this class will occupy in memory
35   *
36   * The core of this class is taken from the Derby project
37   */
38  @InterfaceAudience.Private
39  public class ClassSize {
40    private static final Log LOG = LogFactory.getLog(ClassSize.class);
41  
42    /** Array overhead */
43    public static final int ARRAY;
44
45    /** Overhead for ArrayList(0) */
46    public static final int ARRAYLIST;
47
48    /** Overhead for ByteBuffer */
49    public static final int BYTE_BUFFER;
50
51    /** Overhead for an Integer */
52    public static final int INTEGER;
53
54    /** Overhead for entry in map */
55    public static final int MAP_ENTRY;
56
57    /** Object overhead is minimum 2 * reference size (8 bytes on 64-bit) */
58    public static final int OBJECT;
59
60    /** Reference size is 8 bytes on 64-bit, 4 bytes on 32-bit */
61    public static final int REFERENCE;
62
63    /** String overhead */
64    public static final int STRING;
65
66    /** Overhead for TreeMap */
67    public static final int TREEMAP;
68
69    /** Overhead for ConcurrentHashMap */
70    public static final int CONCURRENT_HASHMAP;
71
72    /** Overhead for ConcurrentHashMap.Entry */
73    public static final int CONCURRENT_HASHMAP_ENTRY;
74
75    /** Overhead for ConcurrentHashMap.Segment */
76    public static final int CONCURRENT_HASHMAP_SEGMENT;
77
78    /** Overhead for ConcurrentSkipListMap */
79    public static final int CONCURRENT_SKIPLISTMAP;
80
81    /** Overhead for ConcurrentSkipListMap Entry */
82    public static final int CONCURRENT_SKIPLISTMAP_ENTRY;
83
84    /** Overhead for ReentrantReadWriteLock */
85    public static final int REENTRANT_LOCK;
86
87    /** Overhead for AtomicLong */
88    public static final int ATOMIC_LONG;
89
90    /** Overhead for AtomicInteger */
91    public static final int ATOMIC_INTEGER;
92
93    /** Overhead for AtomicBoolean */
94    public static final int ATOMIC_BOOLEAN;
95
96    /** Overhead for CopyOnWriteArraySet */
97    public static final int COPYONWRITE_ARRAYSET;
98
99    /** Overhead for CopyOnWriteArrayList */
100   public static final int COPYONWRITE_ARRAYLIST;
101
102   /** Overhead for timerange */
103   public static final int TIMERANGE;
104
105   /** Overhead for TimeRangeTracker */
106   public static final int TIMERANGE_TRACKER;
107
108   /** Overhead for CellSkipListSet */
109   public static final int CELL_SKIPLIST_SET;
110
111   public static final int STORE_SERVICES;
112 
113   /* Are we running on jdk7? */
114   private static final boolean JDK7;
115   static {
116     final String version = System.getProperty("java.version");
117     // Verify String looks like this: 1.6.0_29
118     if (version == null || !version.matches("\\d\\.\\d\\..*")) {
119       throw new RuntimeException("Unexpected version format: " + version);
120     }
121     // Convert char to int
122     int major = (int)(version.charAt(0) - '0');
123     int minor = (int)(version.charAt(2) - '0');
124     JDK7 = major == 1 && minor == 7;
125   }
126
127   /**
128    * MemoryLayout abstracts details about the JVM object layout. Default implementation is used in
129    * case Unsafe is not available.
130    */
131   private static class MemoryLayout {
132     int headerSize() {
133       return 2 * oopSize();
134     }
135
136     int arrayHeaderSize() {
137       return (int) align(3 * oopSize());
138     }
139
140     /**
141      * Return the size of an "ordinary object pointer". Either 4 or 8, depending on 32/64 bit,
142      * and CompressedOops
143      */
144     int oopSize() {
145       return is32BitJVM() ? 4 : 8;
146     }
147 
148     /**
149      * Aligns a number to 8.
150      * @param num number to align to 8
151      * @return smallest number >= input that is a multiple of 8
152      */
153     public long align(long num) {
154       //The 7 comes from that the alignSize is 8 which is the number of bytes
155       //stored and sent together
156       return  ((num + 7) >> 3) << 3;
157     }
158 
159     long sizeOf(byte[] b, int len) {
160       return align(arrayHeaderSize() + len);
161     }
162   }
163
164   /**
165    * UnsafeLayout uses Unsafe to guesstimate the object-layout related parameters like object header
166    * sizes and oop sizes
167    * See HBASE-15950.
168    */
169   private static class UnsafeLayout extends MemoryLayout {
170     @SuppressWarnings("unused")
171     private static final class HeaderSize {
172       private byte a;
173     }
174
175     public UnsafeLayout() {
176     }
177
178     @Override
179     int headerSize() {
180       try {
181         return (int) UnsafeAccess.theUnsafe.objectFieldOffset(
182           HeaderSize.class.getDeclaredField("a"));
183       } catch (NoSuchFieldException | SecurityException e) {
184         LOG.error(e);
185       }
186       return super.headerSize();
187     }
188 
189     @Override
190     int arrayHeaderSize() {
191       return UnsafeAccess.theUnsafe.arrayBaseOffset(byte[].class);
192     }
193
194     @Override
195     @SuppressWarnings("static-access")
196     int oopSize() {
197       // Unsafe.addressSize() returns 8, even with CompressedOops. This is how many bytes each
198       // element is allocated in an Object[].
199       return UnsafeAccess.theUnsafe.ARRAY_OBJECT_INDEX_SCALE;
200     }
201 
202     @Override
203     @SuppressWarnings("static-access")
204     long sizeOf(byte[] b, int len) {
205       return align(arrayHeaderSize() + len * UnsafeAccess.theUnsafe.ARRAY_BYTE_INDEX_SCALE);
206     }
207   }
208
209   private static MemoryLayout getMemoryLayout() {
210     // Have a safeguard in case Unsafe estimate is wrong. This is static context, there is
211     // no configuration, so we look at System property.
212     String enabled = System.getProperty("hbase.memorylayout.use.unsafe");
213     if (UnsafeAvailChecker.isAvailable() && (enabled == null || Boolean.parseBoolean(enabled))) {
214       LOG.debug("Using Unsafe to estimate memory layout");
215       return new UnsafeLayout();
216     }
217     LOG.debug("Not using Unsafe to estimate memory layout");
218     return new MemoryLayout();
219   }
220
221   private static final MemoryLayout memoryLayout = getMemoryLayout();
222
223   /**
224    * Method for reading the arc settings and setting overheads according
225    * to 32-bit or 64-bit architecture.
226    */
227   static {
228     REFERENCE = memoryLayout.oopSize();
229
230     OBJECT = memoryLayout.headerSize();
231
232     ARRAY = memoryLayout.arrayHeaderSize();
233
234     ARRAYLIST = align(OBJECT + REFERENCE + (2 * Bytes.SIZEOF_INT)) + align(ARRAY);
235
236     //noinspection PointlessArithmeticExpression
237     BYTE_BUFFER = align(OBJECT + REFERENCE +
238         (5 * Bytes.SIZEOF_INT) +
239         (3 * Bytes.SIZEOF_BOOLEAN) + Bytes.SIZEOF_LONG) + align(ARRAY);
240
241     INTEGER = align(OBJECT + Bytes.SIZEOF_INT);
242
243     MAP_ENTRY = align(OBJECT + 5 * REFERENCE + Bytes.SIZEOF_BOOLEAN);
244
245     TREEMAP = align(OBJECT + (2 * Bytes.SIZEOF_INT) + 7 * REFERENCE);
246
247     // STRING is different size in jdk6 and jdk7. Just use what we estimate as size rather than
248     // have a conditional on whether jdk7.
249     STRING = (int) estimateBase(String.class, false);
250
251     // CONCURRENT_HASHMAP is different size in jdk6 and jdk7; it looks like its different between
252     // 23.6-b03 and 23.0-b21. Just use what we estimate as size rather than have a conditional on
253     // whether jdk7.
254     CONCURRENT_HASHMAP = (int) estimateBase(ConcurrentHashMap.class, false);
255
256     CONCURRENT_HASHMAP_ENTRY = align(REFERENCE + OBJECT + (3 * REFERENCE) +
257         (2 * Bytes.SIZEOF_INT));
258
259     CONCURRENT_HASHMAP_SEGMENT = align(REFERENCE + OBJECT +
260         (3 * Bytes.SIZEOF_INT) + Bytes.SIZEOF_FLOAT + ARRAY);
261
262     // The size changes from jdk7 to jdk8, estimate the size rather than use a conditional
263     CONCURRENT_SKIPLISTMAP = (int) estimateBase(ConcurrentSkipListMap.class, false);
264
265     CONCURRENT_SKIPLISTMAP_ENTRY =
266         align(OBJECT + (3 * REFERENCE)) + /* one node per entry */
267         align((OBJECT + (3 * REFERENCE))/2); /* one index per two entries */
268
269     REENTRANT_LOCK = align(OBJECT + (3 * REFERENCE));
270
271     ATOMIC_LONG = align(OBJECT + Bytes.SIZEOF_LONG);
272
273     ATOMIC_INTEGER = align(OBJECT + Bytes.SIZEOF_INT);
274
275     ATOMIC_BOOLEAN = align(OBJECT + Bytes.SIZEOF_BOOLEAN);
276
277     COPYONWRITE_ARRAYSET = align(OBJECT + REFERENCE);
278
279     COPYONWRITE_ARRAYLIST = align(OBJECT + (2 * REFERENCE) + ARRAY);
280
281     TIMERANGE = align(ClassSize.OBJECT + Bytes.SIZEOF_LONG * 2 + Bytes.SIZEOF_BOOLEAN);
282
283     TIMERANGE_TRACKER = align(ClassSize.OBJECT + Bytes.SIZEOF_LONG * 2);
284
285     CELL_SKIPLIST_SET = align(OBJECT + REFERENCE);
286
287     STORE_SERVICES = align(OBJECT + REFERENCE + ATOMIC_LONG);
288   }
289
290   /**
291    * The estimate of the size of a class instance depends on whether the JVM
292    * uses 32 or 64 bit addresses, that is it depends on the size of an object
293    * reference. It is a linear function of the size of a reference, e.g.
294    * 24 + 5*r where r is the size of a reference (usually 4 or 8 bytes).
295    *
296    * This method returns the coefficients of the linear function, e.g. {24, 5}
297    * in the above example.
298    *
299    * @param cl A class whose instance size is to be estimated
300    * @param debug debug flag
301    * @return an array of 3 integers. The first integer is the size of the
302    * primitives, the second the number of arrays and the third the number of
303    * references.
304    */
305   @SuppressWarnings("unchecked")
306   private static int [] getSizeCoefficients(Class cl, boolean debug) {
307     int primitives = 0;
308     int arrays = 0;
309     int references = 0;
310     int index = 0;
311
312     for ( ; null != cl; cl = cl.getSuperclass()) {
313       Field[] field = cl.getDeclaredFields();
314       if (null != field) {
315         for (Field aField : field) {
316           if (Modifier.isStatic(aField.getModifiers())) continue;
317           Class fieldClass = aField.getType();
318           if (fieldClass.isArray()) {
319             arrays++;
320             references++;
321           } else if (!fieldClass.isPrimitive()) {
322             references++;
323           } else {// Is simple primitive
324             String name = fieldClass.getName();
325
326             if (name.equals("int") || name.equals("I"))
327               primitives += Bytes.SIZEOF_INT;
328             else if (name.equals("long") || name.equals("J"))
329               primitives += Bytes.SIZEOF_LONG;
330             else if (name.equals("boolean") || name.equals("Z"))
331               primitives += Bytes.SIZEOF_BOOLEAN;
332             else if (name.equals("short") || name.equals("S"))
333               primitives += Bytes.SIZEOF_SHORT;
334             else if (name.equals("byte") || name.equals("B"))
335               primitives += Bytes.SIZEOF_BYTE;
336             else if (name.equals("char") || name.equals("C"))
337               primitives += Bytes.SIZEOF_CHAR;
338             else if (name.equals("float") || name.equals("F"))
339               primitives += Bytes.SIZEOF_FLOAT;
340             else if (name.equals("double") || name.equals("D"))
341               primitives += Bytes.SIZEOF_DOUBLE;
342           }
343           if (debug) {
344             if (LOG.isDebugEnabled()) {
345               LOG.debug("" + index + " " + aField.getName() + " " + aField.getType());
346             }
347           }
348           index++;
349         }
350       }
351     }
352     return new int [] {primitives, arrays, references};
353   }
354
355   /**
356    * Estimate the static space taken up by a class instance given the
357    * coefficients returned by getSizeCoefficients.
358    *
359    * @param coeff the coefficients
360    *
361    * @param debug debug flag
362    * @return the size estimate, in bytes
363    */
364   private static long estimateBaseFromCoefficients(int [] coeff, boolean debug) {
365     long prealign_size = OBJECT + coeff[0] + coeff[2] * REFERENCE;
366
367     // Round up to a multiple of 8
368     long size = align(prealign_size) + align(coeff[1] * ARRAY);
369     if (debug) {
370       if (LOG.isDebugEnabled()) {
371         LOG.debug("Primitives=" + coeff[0] + ", arrays=" + coeff[1] +
372             ", references=" + coeff[2] + ", refSize " + REFERENCE +
373             ", size=" + size + ", prealign_size=" + prealign_size);
374       }
375     }
376     return size;
377   }
378
379   /**
380    * Estimate the static space taken up by the fields of a class. This includes
381    * the space taken up by by references (the pointer) but not by the referenced
382    * object. So the estimated size of an array field does not depend on the size
383    * of the array. Similarly the size of an object (reference) field does not
384    * depend on the object.
385    *
386    * @param cl class
387    * @param debug debug flag
388    * @return the size estimate in bytes.
389    */
390   @SuppressWarnings("unchecked")
391   public static long estimateBase(Class cl, boolean debug) {
392     return estimateBaseFromCoefficients( getSizeCoefficients(cl, debug), debug);
393   }
394
395   /**
396    * Aligns a number to 8.
397    * @param num number to align to 8
398    * @return smallest number &gt;= input that is a multiple of 8
399    */
400   public static int align(int num) {
401     return (int)(align((long)num));
402   }
403
404   /**
405    * Aligns a number to 8.
406    * @param num number to align to 8
407    * @return smallest number &gt;= input that is a multiple of 8
408    */
409   public static long align(long num) {
410     return memoryLayout.align(num);
411   }
412
413   /**
414    * Determines if we are running in a 32-bit JVM. Some unit tests need to
415    * know this too.
416    */
417   public static boolean is32BitJVM() {
418     final String model = System.getProperty("sun.arch.data.model");
419     return model != null && model.equals("32");
420   }
421
422   public static long sizeOf(byte[] b, int len) {
423     return memoryLayout.sizeOf(b, len);
424   }
425
426 }
427