001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.io;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertTrue;
022
023import java.io.IOException;
024import java.lang.management.ManagementFactory;
025import java.lang.management.RuntimeMXBean;
026import java.nio.ByteBuffer;
027import java.util.ArrayList;
028import java.util.LinkedList;
029import java.util.Map;
030import java.util.TreeMap;
031import java.util.concurrent.ConcurrentHashMap;
032import java.util.concurrent.ConcurrentSkipListMap;
033import java.util.concurrent.CopyOnWriteArrayList;
034import java.util.concurrent.CopyOnWriteArraySet;
035import java.util.concurrent.atomic.AtomicBoolean;
036import java.util.concurrent.atomic.AtomicInteger;
037import java.util.concurrent.atomic.AtomicLong;
038import java.util.concurrent.atomic.AtomicReference;
039import java.util.concurrent.locks.ReentrantReadWriteLock;
040import org.apache.hadoop.hbase.HBaseClassTestRule;
041import org.apache.hadoop.hbase.KeyValue;
042import org.apache.hadoop.hbase.client.Delete;
043import org.apache.hadoop.hbase.client.Mutation;
044import org.apache.hadoop.hbase.client.Put;
045import org.apache.hadoop.hbase.io.hfile.BlockCacheKey;
046import org.apache.hadoop.hbase.io.hfile.LruBlockCache;
047import org.apache.hadoop.hbase.io.hfile.LruCachedBlock;
048import org.apache.hadoop.hbase.regionserver.CSLMImmutableSegment;
049import org.apache.hadoop.hbase.regionserver.CellArrayImmutableSegment;
050import org.apache.hadoop.hbase.regionserver.CellArrayMap;
051import org.apache.hadoop.hbase.regionserver.CellSet;
052import org.apache.hadoop.hbase.regionserver.CompactingMemStore;
053import org.apache.hadoop.hbase.regionserver.CompactionPipeline;
054import org.apache.hadoop.hbase.regionserver.DefaultMemStore;
055import org.apache.hadoop.hbase.regionserver.HRegion;
056import org.apache.hadoop.hbase.regionserver.HStore;
057import org.apache.hadoop.hbase.regionserver.ImmutableSegment;
058import org.apache.hadoop.hbase.regionserver.MemStoreCompactor;
059import org.apache.hadoop.hbase.regionserver.MutableSegment;
060import org.apache.hadoop.hbase.regionserver.Segment;
061import org.apache.hadoop.hbase.regionserver.TimeRangeTracker.NonSyncTimeRangeTracker;
062import org.apache.hadoop.hbase.regionserver.TimeRangeTracker.SyncTimeRangeTracker;
063import org.apache.hadoop.hbase.testclassification.IOTests;
064import org.apache.hadoop.hbase.testclassification.SmallTests;
065import org.apache.hadoop.hbase.util.ClassSize;
066import org.junit.BeforeClass;
067import org.junit.ClassRule;
068import org.junit.Test;
069import org.junit.experimental.categories.Category;
070import org.slf4j.Logger;
071import org.slf4j.LoggerFactory;
072
073/**
074 * Testing the sizing that HeapSize offers and compares to the size given by
075 * ClassSize.
076 */
077@Category({IOTests.class, SmallTests.class})
078public class TestHeapSize  {
079
080  @ClassRule
081  public static final HBaseClassTestRule CLASS_RULE =
082      HBaseClassTestRule.forClass(TestHeapSize.class);
083
084  private static final Logger LOG = LoggerFactory.getLogger(TestHeapSize.class);
085  // List of classes implementing HeapSize
086  // BatchOperation, BatchUpdate, BlockIndex, Entry, Entry<K,V>, HStoreKey
087  // KeyValue, LruBlockCache, Put, WALKey
088
089  @BeforeClass
090  public static void beforeClass() throws Exception {
091    // Print detail on jvm so we know what is different should below test fail.
092    RuntimeMXBean b = ManagementFactory.getRuntimeMXBean();
093    LOG.info("name=" + b.getName());
094    LOG.info("specname=" + b.getSpecName());
095    LOG.info("specvendor=" + b.getSpecVendor());
096    LOG.info("vmname=" + b.getVmName());
097    LOG.info("vmversion=" + b.getVmVersion());
098    LOG.info("vmvendor=" + b.getVmVendor());
099    Map<String, String> p = b.getSystemProperties();
100    LOG.info("properties=" + p);
101  }
102
103  /**
104   * Test our hard-coded sizing of native java objects
105   */
106  @Test
107  public void testNativeSizes() throws IOException {
108    Class<?> cl;
109    long expected;
110    long actual;
111
112    // ArrayList
113    cl = ArrayList.class;
114    expected = ClassSize.estimateBase(cl, false);
115    actual = ClassSize.ARRAYLIST;
116    if(expected != actual) {
117      ClassSize.estimateBase(cl, true);
118      assertEquals(expected, actual);
119    }
120
121    // ByteBuffer
122    cl = ByteBuffer.class;
123    expected = ClassSize.estimateBase(cl, false);
124    actual = ClassSize.BYTE_BUFFER;
125    if(expected != actual) {
126      ClassSize.estimateBase(cl, true);
127      assertEquals(expected, actual);
128    }
129
130    // Integer
131    cl = Integer.class;
132    expected = ClassSize.estimateBase(cl, false);
133    actual = ClassSize.INTEGER;
134    if(expected != actual) {
135      ClassSize.estimateBase(cl, true);
136      assertEquals(expected, actual);
137    }
138
139    // Map.Entry
140    // Interface is public, all others are not.  Hard to size via ClassSize
141//    cl = Map.Entry.class;
142//    expected = ClassSize.estimateBase(cl, false);
143//    actual = ClassSize.MAP_ENTRY;
144//    if(expected != actual) {
145//      ClassSize.estimateBase(cl, true);
146//      assertEquals(expected, actual);
147//    }
148
149    // Object
150    cl = Object.class;
151    expected = ClassSize.estimateBase(cl, false);
152    actual = ClassSize.align(ClassSize.OBJECT);
153    if(expected != actual) {
154      ClassSize.estimateBase(cl, true);
155      assertEquals(expected, actual);
156    }
157
158    // TreeMap
159    cl = TreeMap.class;
160    expected = ClassSize.estimateBase(cl, false);
161    actual = ClassSize.TREEMAP;
162    if(expected != actual) {
163      ClassSize.estimateBase(cl, true);
164      assertEquals(expected, actual);
165    }
166
167    // String
168    cl = String.class;
169    expected = ClassSize.estimateBase(cl, false);
170    actual = ClassSize.STRING;
171    if(expected != actual) {
172      ClassSize.estimateBase(cl, true);
173      assertEquals(expected, actual);
174    }
175
176    // ConcurrentHashMap
177    cl = ConcurrentHashMap.class;
178    expected = ClassSize.estimateBase(cl, false);
179    actual = ClassSize.CONCURRENT_HASHMAP;
180    if(expected != actual) {
181      ClassSize.estimateBase(cl, true);
182      assertEquals(expected, actual);
183    }
184
185    // ConcurrentSkipListMap
186    cl = ConcurrentSkipListMap.class;
187    expected = ClassSize.estimateBase(cl, false);
188    actual = ClassSize.CONCURRENT_SKIPLISTMAP;
189    if(expected != actual) {
190      ClassSize.estimateBase(cl, true);
191      assertEquals(expected, actual);
192    }
193
194    // CellArrayMap
195    cl = CellArrayMap.class;
196    expected = ClassSize.estimateBase(cl, false);
197    actual = ClassSize.CELL_ARRAY_MAP;
198    if(expected != actual) {
199      ClassSize.estimateBase(cl, true);
200      assertEquals(expected, actual);
201    }
202
203    // ReentrantReadWriteLock
204    cl = ReentrantReadWriteLock.class;
205    expected = ClassSize.estimateBase(cl, false);
206    actual = ClassSize.REENTRANT_LOCK;
207    if(expected != actual) {
208      ClassSize.estimateBase(cl, true);
209      assertEquals(expected, actual);
210    }
211
212    // AtomicLong
213    cl = AtomicLong.class;
214    expected = ClassSize.estimateBase(cl, false);
215    actual = ClassSize.ATOMIC_LONG;
216    if(expected != actual) {
217      ClassSize.estimateBase(cl, true);
218      assertEquals(expected, actual);
219    }
220
221    // AtomicInteger
222    cl = AtomicInteger.class;
223    expected = ClassSize.estimateBase(cl, false);
224    actual = ClassSize.ATOMIC_INTEGER;
225    if(expected != actual) {
226      ClassSize.estimateBase(cl, true);
227      assertEquals(expected, actual);
228    }
229
230    // AtomicBoolean
231    cl = AtomicBoolean.class;
232    expected = ClassSize.estimateBase(cl, false);
233    actual = ClassSize.ATOMIC_BOOLEAN;
234    if(expected != actual) {
235      ClassSize.estimateBase(cl, true);
236      assertEquals(expected, actual);
237    }
238
239    // CopyOnWriteArraySet
240    cl = CopyOnWriteArraySet.class;
241    expected = ClassSize.estimateBase(cl, false);
242    actual = ClassSize.COPYONWRITE_ARRAYSET;
243    if(expected != actual) {
244      ClassSize.estimateBase(cl, true);
245      assertEquals(expected, actual);
246    }
247
248    // CopyOnWriteArrayList
249    cl = CopyOnWriteArrayList.class;
250    expected = ClassSize.estimateBase(cl, false);
251    actual = ClassSize.COPYONWRITE_ARRAYLIST;
252    if(expected != actual) {
253      ClassSize.estimateBase(cl, true);
254      assertEquals(expected, actual);
255    }
256
257    // SyncTimeRangeTracker
258    cl = SyncTimeRangeTracker.class;
259    expected = ClassSize.estimateBase(cl, false);
260    actual = ClassSize.SYNC_TIMERANGE_TRACKER;
261    if (expected != actual) {
262      ClassSize.estimateBase(cl, true);
263      assertEquals(expected, actual);
264    }
265
266    // NonSyncTimeRangeTracker
267    cl = NonSyncTimeRangeTracker.class;
268    expected = ClassSize.estimateBase(cl, false);
269    actual = ClassSize.NON_SYNC_TIMERANGE_TRACKER;
270    if (expected != actual) {
271      ClassSize.estimateBase(cl, true);
272      assertEquals(expected, actual);
273    }
274
275    // CellSet
276    cl = CellSet.class;
277    expected = ClassSize.estimateBase(cl, false);
278    actual = ClassSize.CELL_SET;
279    if (expected != actual) {
280      ClassSize.estimateBase(cl, true);
281      assertEquals(expected, actual);
282    }
283  }
284
285  /**
286   * Testing the classes that implements HeapSize and are a part of 0.20.
287   * Some are not tested here for example BlockIndex which is tested in
288   * TestHFile since it is a non public class
289   * @throws IOException
290   */
291  @Test
292  public void testSizes() throws IOException {
293    Class<?> cl;
294    long expected;
295    long actual;
296
297    //KeyValue
298    cl = KeyValue.class;
299    expected = ClassSize.estimateBase(cl, false);
300    KeyValue kv = new KeyValue();
301    actual = kv.heapSize();
302    if(expected != actual) {
303      ClassSize.estimateBase(cl, true);
304      assertEquals(expected, actual);
305    }
306
307    //LruBlockCache Overhead
308    cl = LruBlockCache.class;
309    actual = LruBlockCache.CACHE_FIXED_OVERHEAD;
310    expected = ClassSize.estimateBase(cl, false);
311    if(expected != actual) {
312      ClassSize.estimateBase(cl, true);
313      assertEquals(expected, actual);
314    }
315
316    // CachedBlock Fixed Overhead
317    // We really need "deep" sizing but ClassSize does not do this.
318    // Perhaps we should do all these more in this style....
319    cl = LruCachedBlock.class;
320    actual = LruCachedBlock.PER_BLOCK_OVERHEAD;
321    expected = ClassSize.estimateBase(cl, false);
322    expected += ClassSize.estimateBase(String.class, false);
323    expected += ClassSize.estimateBase(ByteBuffer.class, false);
324    if(expected != actual) {
325      ClassSize.estimateBase(cl, true);
326      ClassSize.estimateBase(String.class, true);
327      ClassSize.estimateBase(ByteBuffer.class, true);
328      assertEquals(expected, actual);
329    }
330
331    // DefaultMemStore Overhead
332    cl = DefaultMemStore.class;
333    actual = DefaultMemStore.FIXED_OVERHEAD;
334    expected = ClassSize.estimateBase(cl, false);
335    if(expected != actual) {
336      ClassSize.estimateBase(cl, true);
337      assertEquals(expected, actual);
338    }
339
340    // DefaultMemStore Deep Overhead
341    actual = DefaultMemStore.DEEP_OVERHEAD;
342    expected = ClassSize.estimateBase(cl, false);
343    if (expected != actual) {
344      ClassSize.estimateBase(cl, true);
345      assertEquals(expected, actual);
346    }
347
348    // CompactingMemStore Deep Overhead
349    cl = CompactingMemStore.class;
350    actual = CompactingMemStore.DEEP_OVERHEAD;
351    expected = ClassSize.estimateBase(cl, false);
352    expected += ClassSize.estimateBase(AtomicBoolean.class, false);
353    expected += ClassSize.estimateBase(AtomicBoolean.class, false);
354    expected += ClassSize.estimateBase(CompactionPipeline.class, false);
355    expected += ClassSize.estimateBase(LinkedList.class, false); //inside CompactionPipeline
356    expected += ClassSize.estimateBase(LinkedList.class, false); //inside CompactionPipeline
357    expected += ClassSize.estimateBase(MemStoreCompactor.class, false);
358    expected += ClassSize.estimateBase(AtomicBoolean.class, false);// inside MemStoreCompactor
359    if (expected != actual) {
360      ClassSize.estimateBase(cl, true);
361      ClassSize.estimateBase(AtomicBoolean.class, true);
362      ClassSize.estimateBase(AtomicBoolean.class, true);
363      ClassSize.estimateBase(CompactionPipeline.class, true);
364      ClassSize.estimateBase(LinkedList.class, true);
365      ClassSize.estimateBase(LinkedList.class, true);
366      ClassSize.estimateBase(MemStoreCompactor.class, true);
367      ClassSize.estimateBase(AtomicBoolean.class, true);
368      assertEquals(expected, actual);
369    }
370
371    // Segment Deep overhead
372    cl = Segment.class;
373    actual = Segment.DEEP_OVERHEAD;
374    expected = ClassSize.estimateBase(cl, false);
375    expected += 2 * ClassSize.estimateBase(AtomicLong.class, false);
376    expected += ClassSize.estimateBase(AtomicReference.class, false);
377    expected += ClassSize.estimateBase(CellSet.class, false);
378    if (expected != actual) {
379      ClassSize.estimateBase(cl, true);
380      ClassSize.estimateBase(AtomicLong.class, true);
381      ClassSize.estimateBase(AtomicReference.class, true);
382      ClassSize.estimateBase(CellSet.class, true);
383      assertEquals(expected, actual);
384    }
385
386    // MutableSegment Deep overhead
387    cl = MutableSegment.class;
388    actual = MutableSegment.DEEP_OVERHEAD;
389    expected = ClassSize.estimateBase(cl, false);
390    expected += 2 * ClassSize.estimateBase(AtomicLong.class, false);
391    expected += ClassSize.estimateBase(AtomicReference.class, false);
392    expected += ClassSize.estimateBase(CellSet.class, false);
393    expected += ClassSize.estimateBase(SyncTimeRangeTracker.class, false);
394    expected += ClassSize.estimateBase(ConcurrentSkipListMap.class, false);
395    if (expected != actual) {
396      ClassSize.estimateBase(cl, true);
397      ClassSize.estimateBase(AtomicLong.class, true);
398      ClassSize.estimateBase(AtomicLong.class, true);
399      ClassSize.estimateBase(AtomicReference.class, true);
400      ClassSize.estimateBase(CellSet.class, true);
401      ClassSize.estimateBase(SyncTimeRangeTracker.class, true);
402      ClassSize.estimateBase(ConcurrentSkipListMap.class, true);
403      assertEquals(expected, actual);
404    }
405
406    // ImmutableSegments Deep overhead
407    cl = ImmutableSegment.class;
408    actual = ImmutableSegment.DEEP_OVERHEAD;
409    expected = ClassSize.estimateBase(cl, false);
410    expected += 2 * ClassSize.estimateBase(AtomicLong.class, false);
411    expected += ClassSize.estimateBase(AtomicReference.class, false);
412    expected += ClassSize.estimateBase(CellSet.class, false);
413    expected += ClassSize.estimateBase(NonSyncTimeRangeTracker.class, false);
414    if (expected != actual) {
415      ClassSize.estimateBase(cl, true);
416      ClassSize.estimateBase(AtomicLong.class, true);
417      ClassSize.estimateBase(AtomicLong.class, true);
418      ClassSize.estimateBase(AtomicReference.class, true);
419      ClassSize.estimateBase(CellSet.class, true);
420      ClassSize.estimateBase(NonSyncTimeRangeTracker.class, true);
421      assertEquals(expected, actual);
422    }
423
424    cl = CSLMImmutableSegment.class;
425    actual = CSLMImmutableSegment.DEEP_OVERHEAD_CSLM;
426    expected = ClassSize.estimateBase(cl, false);
427    expected += 2 * ClassSize.estimateBase(AtomicLong.class, false);
428    expected += ClassSize.estimateBase(AtomicReference.class, false);
429    expected += ClassSize.estimateBase(CellSet.class, false);
430    expected += ClassSize.estimateBase(NonSyncTimeRangeTracker.class, false);
431    expected += ClassSize.estimateBase(ConcurrentSkipListMap.class, false);
432    if (expected != actual) {
433      ClassSize.estimateBase(cl, true);
434      ClassSize.estimateBase(AtomicLong.class, true);
435      ClassSize.estimateBase(AtomicLong.class, true);
436      ClassSize.estimateBase(AtomicReference.class, true);
437      ClassSize.estimateBase(CellSet.class, true);
438      ClassSize.estimateBase(NonSyncTimeRangeTracker.class, true);
439      ClassSize.estimateBase(ConcurrentSkipListMap.class, true);
440      assertEquals(expected, actual);
441    }
442    cl = CellArrayImmutableSegment.class;
443    actual = CellArrayImmutableSegment.DEEP_OVERHEAD_CAM;
444    expected = ClassSize.estimateBase(cl, false);
445    expected += 2 * ClassSize.estimateBase(AtomicLong.class, false);
446    expected += ClassSize.estimateBase(AtomicReference.class, false);
447    expected += ClassSize.estimateBase(CellSet.class, false);
448    expected += ClassSize.estimateBase(NonSyncTimeRangeTracker.class, false);
449    expected += ClassSize.estimateBase(CellArrayMap.class, false);
450    if (expected != actual) {
451      ClassSize.estimateBase(cl, true);
452      ClassSize.estimateBase(AtomicLong.class, true);
453      ClassSize.estimateBase(AtomicLong.class, true);
454      ClassSize.estimateBase(AtomicReference.class, true);
455      ClassSize.estimateBase(CellSet.class, true);
456      ClassSize.estimateBase(NonSyncTimeRangeTracker.class, true);
457      ClassSize.estimateBase(CellArrayMap.class, true);
458      assertEquals(expected, actual);
459    }
460
461    // Store Overhead
462    cl = HStore.class;
463    actual = HStore.FIXED_OVERHEAD;
464    expected = ClassSize.estimateBase(cl, false);
465    if(expected != actual) {
466      ClassSize.estimateBase(cl, true);
467      assertEquals(expected, actual);
468    }
469
470    // Region Overhead
471    cl = HRegion.class;
472    actual = HRegion.FIXED_OVERHEAD;
473    expected = ClassSize.estimateBase(cl, false);
474    if (expected != actual) {
475      ClassSize.estimateBase(cl, true);
476      assertEquals(expected, actual);
477    }
478
479    // Block cache key overhead. Only tests fixed overhead as estimating heap
480    // size of strings is hard.
481    cl = BlockCacheKey.class;
482    actual = BlockCacheKey.FIXED_OVERHEAD;
483    expected  = ClassSize.estimateBase(cl, false);
484    if (expected != actual) {
485      ClassSize.estimateBase(cl, true);
486      assertEquals(expected, actual);
487    }
488
489    // Currently NOT testing Deep Overheads of many of these classes.
490    // Deep overheads cover a vast majority of stuff, but will not be 100%
491    // accurate because it's unclear when we're referencing stuff that's already
492    // accounted for.  But we have satisfied our two core requirements.
493    // Sizing is quite accurate now, and our tests will throw errors if
494    // any of these classes are modified without updating overhead sizes.
495  }
496
497  @Test
498  public void testMutations(){
499    Class<?> cl;
500    long expected;
501    long actual;
502
503    cl = TimeRange.class;
504    actual = ClassSize.TIMERANGE;
505    expected  = ClassSize.estimateBase(cl, false);
506    if (expected != actual) {
507      ClassSize.estimateBase(cl, true);
508      assertEquals(expected, actual);
509    }
510
511    byte[] row = new byte[] { 0 };
512    cl = Put.class;
513    actual = Mutation.MUTATION_OVERHEAD + ClassSize.align(ClassSize.ARRAY);
514    expected = ClassSize.estimateBase(cl, false);
515    //The actual TreeMap is not included in the above calculation
516    expected += ClassSize.align(ClassSize.TREEMAP);
517    expected += ClassSize.align(ClassSize.INTEGER); // priority
518    if (expected != actual) {
519      ClassSize.estimateBase(cl, true);
520      assertEquals(expected, actual);
521    }
522
523    cl = Delete.class;
524    actual = Mutation.MUTATION_OVERHEAD + ClassSize.align(ClassSize.ARRAY);
525    expected  = ClassSize.estimateBase(cl, false);
526    //The actual TreeMap is not included in the above calculation
527    expected += ClassSize.align(ClassSize.TREEMAP);
528    expected += ClassSize.align(ClassSize.INTEGER); // priority
529    if (expected != actual) {
530      ClassSize.estimateBase(cl, true);
531      assertEquals(expected, actual);
532    }
533  }
534
535  @Test
536  public void testReferenceSize() {
537    LOG.info("ClassSize.REFERENCE is " + ClassSize.REFERENCE);
538    // oop should be either 4 or 8
539    assertTrue(ClassSize.REFERENCE == 4 || ClassSize.REFERENCE == 8);
540  }
541
542  @Test
543  public void testObjectSize() throws IOException {
544    LOG.info("header:" + ClassSize.OBJECT);
545    LOG.info("array header:" + ClassSize.ARRAY);
546
547    if (ClassSize.is32BitJVM()) {
548      assertEquals(ClassSize.OBJECT, 8);
549    } else {
550      assertTrue(ClassSize.OBJECT == 12 || ClassSize.OBJECT == 16); // depending on CompressedOops
551    }
552    if (ClassSize.useUnsafeLayout()) {
553      assertEquals(ClassSize.ARRAY, ClassSize.OBJECT + 4);
554    } else {
555      assertEquals(ClassSize.ARRAY, ClassSize.OBJECT + 8);
556    }
557  }
558}
559