View Javadoc

1   /**
2    * Copyright The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  
21  package org.apache.hadoop.hbase.util;
22  
23  import java.lang.reflect.Field;
24  import java.lang.reflect.Modifier;
25  import java.util.concurrent.ConcurrentHashMap;
26  import java.util.concurrent.ConcurrentSkipListMap;
27  
28  import org.apache.commons.logging.Log;
29  import org.apache.commons.logging.LogFactory;
30  import org.apache.hadoop.hbase.classification.InterfaceAudience;
31  
32
33  /**
34   * Class for determining the "size" of a class, an attempt to calculate the
35   * actual bytes that an object of this class will occupy in memory
36   *
37   * The core of this class is taken from the Derby project
38   */
39  @InterfaceAudience.Private
40  public class ClassSize {
41    private static final Log LOG = LogFactory.getLog(ClassSize.class);
42
43    /** Array overhead */
44    public static final int ARRAY;
45
46    /** Overhead for ArrayList(0) */
47    public static final int ARRAYLIST;
48
49    /** Overhead for LinkedList(0) */
50    public static final int LINKEDLIST;
51
52    /** Overhead for a single entry in LinkedList */
53    public static final int LINKEDLIST_ENTRY;
54
55    /** Overhead for ByteBuffer */
56    public static final int BYTE_BUFFER;
57
58    /** Overhead for an Integer */
59    public static final int INTEGER;
60
61    /** Overhead for entry in map */
62    public static final int MAP_ENTRY;
63
64    /** Object overhead is minimum 2 * reference size (8 bytes on 64-bit) */
65    public static final int OBJECT;
66
67    /** Reference size is 8 bytes on 64-bit, 4 bytes on 32-bit */
68    public static final int REFERENCE;
69
70    /** String overhead */
71    public static final int STRING;
72
73    /** Overhead for TreeMap */
74    public static final int TREEMAP;
75
76    /** Overhead for ConcurrentHashMap */
77    public static final int CONCURRENT_HASHMAP;
78
79    /** Overhead for ConcurrentHashMap.Entry */
80    public static final int CONCURRENT_HASHMAP_ENTRY;
81
82    /** Overhead for ConcurrentHashMap.Segment */
83    public static final int CONCURRENT_HASHMAP_SEGMENT;
84
85    /** Overhead for ConcurrentSkipListMap */
86    public static final int CONCURRENT_SKIPLISTMAP;
87
88    /** Overhead for ConcurrentSkipListMap Entry */
89    public static final int CONCURRENT_SKIPLISTMAP_ENTRY;
90
91    /** Overhead for CellArrayMap */
92    public static final int CELL_ARRAY_MAP;
93
94    /** Overhead for Cell Array Entry */
95    public static final int CELL_ARRAY_MAP_ENTRY;
96
97    /** Overhead for ReentrantReadWriteLock */
98    public static final int REENTRANT_LOCK;
99
100   /** Overhead for AtomicLong */
101   public static final int ATOMIC_LONG;
102
103   /** Overhead for AtomicInteger */
104   public static final int ATOMIC_INTEGER;
105
106   /** Overhead for AtomicBoolean */
107   public static final int ATOMIC_BOOLEAN;
108
109   /** Overhead for AtomicReference */
110   public static final int ATOMIC_REFERENCE;
111
112   /** Overhead for CopyOnWriteArraySet */
113   public static final int COPYONWRITE_ARRAYSET;
114
115   /** Overhead for CopyOnWriteArrayList */
116   public static final int COPYONWRITE_ARRAYLIST;
117
118   /** Overhead for timerange */
119   public static final int TIMERANGE;
120
121   /** Overhead for TimeRangeTracker */
122   public static final int TIMERANGE_TRACKER;
123
124   /** Overhead for CellSkipListSet */
125   public static final int CELL_SET;
126
127   public static final int STORE_SERVICES;
128
129   /* Are we running on jdk7? */
130   private static final boolean JDK7;
131   static {
132     final String version = System.getProperty("java.version");
133     // Verify String looks like this: 1.6.0_29
134     if (version == null || !version.matches("\\d\\.\\d\\..*")) {
135       throw new RuntimeException("Unexpected version format: " + version);
136     }
137     // Convert char to int
138     int major = (int)(version.charAt(0) - '0');
139     int minor = (int)(version.charAt(2) - '0');
140     JDK7 = major == 1 && minor == 7;
141   }
142
143   /**
144    * MemoryLayout abstracts details about the JVM object layout. Default implementation is used in
145    * case Unsafe is not available.
146    */
147   private static class MemoryLayout {
148     int headerSize() {
149       return 2 * oopSize();
150     }
151
152     int arrayHeaderSize() {
153       return (int) align(3 * oopSize());
154     }
155
156     /**
157      * Return the size of an "ordinary object pointer". Either 4 or 8, depending on 32/64 bit,
158      * and CompressedOops
159      */
160     int oopSize() {
161       return is32BitJVM() ? 4 : 8;
162     }
163
164     /**
165      * Aligns a number to 8.
166      * @param num number to align to 8
167      * @return smallest number >= input that is a multiple of 8
168      */
169     public long align(long num) {
170       //The 7 comes from that the alignSize is 8 which is the number of bytes
171       //stored and sent together
172       return  ((num + 7) >> 3) << 3;
173     }
174
175     long sizeOf(byte[] b, int len) {
176       return align(arrayHeaderSize() + len);
177     }
178   }
179
180   /**
181    * UnsafeLayout uses Unsafe to guesstimate the object-layout related parameters like object header
182    * sizes and oop sizes
183    * See HBASE-15950.
184    */
185   private static class UnsafeLayout extends MemoryLayout {
186     @SuppressWarnings("unused")
187     private static final class HeaderSize {
188       private byte a;
189     }
190
191     public UnsafeLayout() {
192     }
193
194     @Override
195     int headerSize() {
196       try {
197         return (int) UnsafeAccess.theUnsafe.objectFieldOffset(
198           HeaderSize.class.getDeclaredField("a"));
199       } catch (NoSuchFieldException | SecurityException e) {
200         LOG.error(e);
201       }
202       return super.headerSize();
203     }
204
205     @Override
206     int arrayHeaderSize() {
207       return UnsafeAccess.theUnsafe.arrayBaseOffset(byte[].class);
208     }
209
210     @Override
211     @SuppressWarnings("static-access")
212     int oopSize() {
213       // Unsafe.addressSize() returns 8, even with CompressedOops. This is how many bytes each
214       // element is allocated in an Object[].
215       return UnsafeAccess.theUnsafe.ARRAY_OBJECT_INDEX_SCALE;
216     }
217
218     @Override
219     @SuppressWarnings("static-access")
220     long sizeOf(byte[] b, int len) {
221       return align(arrayHeaderSize() + len * UnsafeAccess.theUnsafe.ARRAY_BYTE_INDEX_SCALE);
222     }
223   }
224
225   private static MemoryLayout getMemoryLayout() {
226     // Have a safeguard in case Unsafe estimate is wrong. This is static context, there is
227     // no configuration, so we look at System property.
228     String enabled = System.getProperty("hbase.memorylayout.use.unsafe");
229     if (UnsafeAvailChecker.isAvailable() && (enabled == null || Boolean.parseBoolean(enabled))) {
230       LOG.debug("Using Unsafe to estimate memory layout");
231       return new UnsafeLayout();
232     }
233     LOG.debug("Not using Unsafe to estimate memory layout");
234     return new MemoryLayout();
235   }
236
237   private static final MemoryLayout memoryLayout = getMemoryLayout();
238
239   /**
240    * Method for reading the arc settings and setting overheads according
241    * to 32-bit or 64-bit architecture.
242    */
243   static {
244     REFERENCE = memoryLayout.oopSize();
245
246     OBJECT = memoryLayout.headerSize();
247
248     ARRAY = memoryLayout.arrayHeaderSize();
249
250     ARRAYLIST = align(OBJECT + REFERENCE + (2 * Bytes.SIZEOF_INT)) + align(ARRAY);
251
252     LINKEDLIST = align(OBJECT + (2 * Bytes.SIZEOF_INT) + (2 * REFERENCE));
253
254     LINKEDLIST_ENTRY = align(OBJECT + (2 * REFERENCE));
255
256     //noinspection PointlessArithmeticExpression
257     BYTE_BUFFER = align(OBJECT + REFERENCE +
258         (5 * Bytes.SIZEOF_INT) +
259         (3 * Bytes.SIZEOF_BOOLEAN) + Bytes.SIZEOF_LONG) + align(ARRAY);
260
261     INTEGER = align(OBJECT + Bytes.SIZEOF_INT);
262
263     MAP_ENTRY = align(OBJECT + 5 * REFERENCE + Bytes.SIZEOF_BOOLEAN);
264
265     TREEMAP = align(OBJECT + (2 * Bytes.SIZEOF_INT) + 7 * REFERENCE);
266
267     // STRING is different size in jdk6 and jdk7. Just use what we estimate as size rather than
268     // have a conditional on whether jdk7.
269     STRING = (int) estimateBase(String.class, false);
270
271     // CONCURRENT_HASHMAP is different size in jdk6 and jdk7; it looks like its different between
272     // 23.6-b03 and 23.0-b21. Just use what we estimate as size rather than have a conditional on
273     // whether jdk7.
274     CONCURRENT_HASHMAP = (int) estimateBase(ConcurrentHashMap.class, false);
275
276     CONCURRENT_HASHMAP_ENTRY = align(REFERENCE + OBJECT + (3 * REFERENCE) +
277         (2 * Bytes.SIZEOF_INT));
278
279     CONCURRENT_HASHMAP_SEGMENT = align(REFERENCE + OBJECT +
280         (3 * Bytes.SIZEOF_INT) + Bytes.SIZEOF_FLOAT + ARRAY);
281
282     // The size changes from jdk7 to jdk8, estimate the size rather than use a conditional
283     CONCURRENT_SKIPLISTMAP = (int) estimateBase(ConcurrentSkipListMap.class, false);
284
285     // CELL_ARRAY_MAP is the size of an instance of CellArrayMap class, which extends
286     // CellFlatMap class. CellArrayMap object containing a ref to an Array, so
287     // OBJECT + REFERENCE + ARRAY
288     // CellFlatMap object contains two integers, one boolean and one reference to object, so
289     // 2*INT + BOOLEAN + REFERENCE
290     CELL_ARRAY_MAP = align(OBJECT + 2*Bytes.SIZEOF_INT + Bytes.SIZEOF_BOOLEAN
291         + ARRAY + 2*REFERENCE);
292
293     CONCURRENT_SKIPLISTMAP_ENTRY = align(
294         align(OBJECT + (3 * REFERENCE)) + /* one node per entry */
295         align((OBJECT + (3 * REFERENCE))/2)); /* one index per two entries */
296
297     // REFERENCE in the CellArrayMap all the rest is counted in KeyValue.heapSize()
298     CELL_ARRAY_MAP_ENTRY = align(REFERENCE);
299
300     REENTRANT_LOCK = align(OBJECT + (3 * REFERENCE));
301
302     ATOMIC_LONG = align(OBJECT + Bytes.SIZEOF_LONG);
303
304     ATOMIC_INTEGER = align(OBJECT + Bytes.SIZEOF_INT);
305
306     ATOMIC_BOOLEAN = align(OBJECT + Bytes.SIZEOF_BOOLEAN);
307
308     ATOMIC_REFERENCE = align(OBJECT + REFERENCE);
309
310     COPYONWRITE_ARRAYSET = align(OBJECT + REFERENCE);
311
312     COPYONWRITE_ARRAYLIST = align(OBJECT + (2 * REFERENCE) + ARRAY);
313
314     TIMERANGE = align(ClassSize.OBJECT + Bytes.SIZEOF_LONG * 2 + Bytes.SIZEOF_BOOLEAN);
315
316     TIMERANGE_TRACKER = align(ClassSize.OBJECT + Bytes.SIZEOF_LONG * 2);
317
318     CELL_SET = align(OBJECT + REFERENCE);
319
320     STORE_SERVICES = align(OBJECT + REFERENCE + ATOMIC_LONG);
321   }
322
323   /**
324    * The estimate of the size of a class instance depends on whether the JVM
325    * uses 32 or 64 bit addresses, that is it depends on the size of an object
326    * reference. It is a linear function of the size of a reference, e.g.
327    * 24 + 5*r where r is the size of a reference (usually 4 or 8 bytes).
328    *
329    * This method returns the coefficients of the linear function, e.g. {24, 5}
330    * in the above example.
331    *
332    * @param cl A class whose instance size is to be estimated
333    * @param debug debug flag
334    * @return an array of 3 integers. The first integer is the size of the
335    * primitives, the second the number of arrays and the third the number of
336    * references.
337    */
338   @SuppressWarnings("unchecked")
339   private static int [] getSizeCoefficients(Class cl, boolean debug) {
340     int primitives = 0;
341     int arrays = 0;
342     int references = 0;
343     int index = 0;
344
345     for ( ; null != cl; cl = cl.getSuperclass()) {
346       Field[] field = cl.getDeclaredFields();
347       if (null != field) {
348         for (Field aField : field) {
349           if (Modifier.isStatic(aField.getModifiers())) continue;
350           Class fieldClass = aField.getType();
351           if (fieldClass.isArray()) {
352             arrays++;
353             references++;
354           } else if (!fieldClass.isPrimitive()) {
355             references++;
356           } else {// Is simple primitive
357             String name = fieldClass.getName();
358
359             if (name.equals("int") || name.equals("I"))
360               primitives += Bytes.SIZEOF_INT;
361             else if (name.equals("long") || name.equals("J"))
362               primitives += Bytes.SIZEOF_LONG;
363             else if (name.equals("boolean") || name.equals("Z"))
364               primitives += Bytes.SIZEOF_BOOLEAN;
365             else if (name.equals("short") || name.equals("S"))
366               primitives += Bytes.SIZEOF_SHORT;
367             else if (name.equals("byte") || name.equals("B"))
368               primitives += Bytes.SIZEOF_BYTE;
369             else if (name.equals("char") || name.equals("C"))
370               primitives += Bytes.SIZEOF_CHAR;
371             else if (name.equals("float") || name.equals("F"))
372               primitives += Bytes.SIZEOF_FLOAT;
373             else if (name.equals("double") || name.equals("D"))
374               primitives += Bytes.SIZEOF_DOUBLE;
375           }
376           if (debug) {
377             if (LOG.isDebugEnabled()) {
378               LOG.debug("" + index + " " + aField.getName() + " " + aField.getType());
379             }
380           }
381           index++;
382         }
383       }
384     }
385     return new int [] {primitives, arrays, references};
386   }
387
388   /**
389    * Estimate the static space taken up by a class instance given the
390    * coefficients returned by getSizeCoefficients.
391    *
392    * @param coeff the coefficients
393    *
394    * @param debug debug flag
395    * @return the size estimate, in bytes
396    */
397   private static long estimateBaseFromCoefficients(int [] coeff, boolean debug) {
398     long prealign_size = OBJECT + coeff[0] + coeff[2] * REFERENCE;
399
400     // Round up to a multiple of 8
401     long size = align(prealign_size) + align(coeff[1] * ARRAY);
402     if (debug) {
403       if (LOG.isDebugEnabled()) {
404         LOG.debug("Primitives=" + coeff[0] + ", arrays=" + coeff[1] +
405             ", references=" + coeff[2] + ", refSize " + REFERENCE +
406             ", size=" + size + ", prealign_size=" + prealign_size);
407       }
408     }
409     return size;
410   }
411
412   /**
413    * Estimate the static space taken up by the fields of a class. This includes
414    * the space taken up by by references (the pointer) but not by the referenced
415    * object. So the estimated size of an array field does not depend on the size
416    * of the array. Similarly the size of an object (reference) field does not
417    * depend on the object.
418    *
419    * @param cl class
420    * @param debug debug flag
421    * @return the size estimate in bytes.
422    */
423   @SuppressWarnings("unchecked")
424   public static long estimateBase(Class cl, boolean debug) {
425     return estimateBaseFromCoefficients( getSizeCoefficients(cl, debug), debug);
426   }
427
428   /**
429    * Aligns a number to 8.
430    * @param num number to align to 8
431    * @return smallest number &gt;= input that is a multiple of 8
432    */
433   public static int align(int num) {
434     return (int)(align((long)num));
435   }
436
437   /**
438    * Aligns a number to 8.
439    * @param num number to align to 8
440    * @return smallest number &gt;= input that is a multiple of 8
441    */
442   public static long align(long num) {
443     return memoryLayout.align(num);
444   }
445
446   /**
447    * Determines if we are running in a 32-bit JVM. Some unit tests need to
448    * know this too.
449    */
450   public static boolean is32BitJVM() {
451     final String model = System.getProperty("sun.arch.data.model");
452     return model != null && model.equals("32");
453   }
454
455   public static long sizeOf(byte[] b, int len) {
456     return memoryLayout.sizeOf(b, len);
457   }
458
459 }
460