001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.io;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertTrue;
022
023import java.io.IOException;
024import java.lang.management.ManagementFactory;
025import java.lang.management.RuntimeMXBean;
026import java.nio.ByteBuffer;
027import java.util.ArrayList;
028import java.util.LinkedList;
029import java.util.Map;
030import java.util.TreeMap;
031import java.util.concurrent.ConcurrentHashMap;
032import java.util.concurrent.ConcurrentSkipListMap;
033import java.util.concurrent.CopyOnWriteArrayList;
034import java.util.concurrent.CopyOnWriteArraySet;
035import java.util.concurrent.atomic.AtomicBoolean;
036import java.util.concurrent.atomic.AtomicInteger;
037import java.util.concurrent.atomic.AtomicLong;
038import java.util.concurrent.atomic.AtomicReference;
039import java.util.concurrent.locks.ReentrantReadWriteLock;
040import org.apache.hadoop.hbase.HBaseClassTestRule;
041import org.apache.hadoop.hbase.KeyValue;
042import org.apache.hadoop.hbase.client.Delete;
043import org.apache.hadoop.hbase.client.Mutation;
044import org.apache.hadoop.hbase.client.Put;
045import org.apache.hadoop.hbase.io.hfile.BlockCacheKey;
046import org.apache.hadoop.hbase.io.hfile.LruBlockCache;
047import org.apache.hadoop.hbase.io.hfile.LruCachedBlock;
048import org.apache.hadoop.hbase.regionserver.CSLMImmutableSegment;
049import org.apache.hadoop.hbase.regionserver.CellArrayImmutableSegment;
050import org.apache.hadoop.hbase.regionserver.CellArrayMap;
051import org.apache.hadoop.hbase.regionserver.CellSet;
052import org.apache.hadoop.hbase.regionserver.CompactingMemStore;
053import org.apache.hadoop.hbase.regionserver.CompactionPipeline;
054import org.apache.hadoop.hbase.regionserver.DefaultMemStore;
055import org.apache.hadoop.hbase.regionserver.HRegion;
056import org.apache.hadoop.hbase.regionserver.HStore;
057import org.apache.hadoop.hbase.regionserver.ImmutableSegment;
058import org.apache.hadoop.hbase.regionserver.MemStoreCompactor;
059import org.apache.hadoop.hbase.regionserver.MutableSegment;
060import org.apache.hadoop.hbase.regionserver.Segment;
061import org.apache.hadoop.hbase.regionserver.TimeRangeTracker.NonSyncTimeRangeTracker;
062import org.apache.hadoop.hbase.regionserver.TimeRangeTracker.SyncTimeRangeTracker;
063import org.apache.hadoop.hbase.regionserver.throttle.StoreHotnessProtector;
064import org.apache.hadoop.hbase.testclassification.IOTests;
065import org.apache.hadoop.hbase.testclassification.SmallTests;
066import org.apache.hadoop.hbase.util.ClassSize;
067import org.junit.BeforeClass;
068import org.junit.ClassRule;
069import org.junit.Test;
070import org.junit.experimental.categories.Category;
071import org.slf4j.Logger;
072import org.slf4j.LoggerFactory;
073
074/**
075 * Testing the sizing that HeapSize offers and compares to the size given by
076 * ClassSize.
077 */
078@Category({IOTests.class, SmallTests.class})
079public class TestHeapSize  {
080
081  @ClassRule
082  public static final HBaseClassTestRule CLASS_RULE =
083      HBaseClassTestRule.forClass(TestHeapSize.class);
084
085  private static final Logger LOG = LoggerFactory.getLogger(TestHeapSize.class);
086  // List of classes implementing HeapSize
087  // BatchOperation, BatchUpdate, BlockIndex, Entry, Entry<K,V>, HStoreKey
088  // KeyValue, LruBlockCache, Put, WALKey
089
090  @BeforeClass
091  public static void beforeClass() throws Exception {
092    // Print detail on jvm so we know what is different should below test fail.
093    RuntimeMXBean b = ManagementFactory.getRuntimeMXBean();
094    LOG.info("name=" + b.getName());
095    LOG.info("specname=" + b.getSpecName());
096    LOG.info("specvendor=" + b.getSpecVendor());
097    LOG.info("vmname=" + b.getVmName());
098    LOG.info("vmversion=" + b.getVmVersion());
099    LOG.info("vmvendor=" + b.getVmVendor());
100    Map<String, String> p = b.getSystemProperties();
101    LOG.info("properties=" + p);
102  }
103
104  /**
105   * Test our hard-coded sizing of native java objects
106   */
107  @Test
108  public void testNativeSizes() throws IOException {
109    Class<?> cl;
110    long expected;
111    long actual;
112
113    // ArrayList
114    cl = ArrayList.class;
115    expected = ClassSize.estimateBase(cl, false);
116    actual = ClassSize.ARRAYLIST;
117    if(expected != actual) {
118      ClassSize.estimateBase(cl, true);
119      assertEquals(expected, actual);
120    }
121
122    // ByteBuffer
123    cl = ByteBuffer.class;
124    expected = ClassSize.estimateBase(cl, false);
125    actual = ClassSize.BYTE_BUFFER;
126    if(expected != actual) {
127      ClassSize.estimateBase(cl, true);
128      assertEquals(expected, actual);
129    }
130
131    // Integer
132    cl = Integer.class;
133    expected = ClassSize.estimateBase(cl, false);
134    actual = ClassSize.INTEGER;
135    if(expected != actual) {
136      ClassSize.estimateBase(cl, true);
137      assertEquals(expected, actual);
138    }
139
140    // Map.Entry
141    // Interface is public, all others are not.  Hard to size via ClassSize
142//    cl = Map.Entry.class;
143//    expected = ClassSize.estimateBase(cl, false);
144//    actual = ClassSize.MAP_ENTRY;
145//    if(expected != actual) {
146//      ClassSize.estimateBase(cl, true);
147//      assertEquals(expected, actual);
148//    }
149
150    // Object
151    cl = Object.class;
152    expected = ClassSize.estimateBase(cl, false);
153    actual = ClassSize.align(ClassSize.OBJECT);
154    if(expected != actual) {
155      ClassSize.estimateBase(cl, true);
156      assertEquals(expected, actual);
157    }
158
159    // TreeMap
160    cl = TreeMap.class;
161    expected = ClassSize.estimateBase(cl, false);
162    actual = ClassSize.TREEMAP;
163    if(expected != actual) {
164      ClassSize.estimateBase(cl, true);
165      assertEquals(expected, actual);
166    }
167
168    // String
169    cl = String.class;
170    expected = ClassSize.estimateBase(cl, false);
171    actual = ClassSize.STRING;
172    if(expected != actual) {
173      ClassSize.estimateBase(cl, true);
174      assertEquals(expected, actual);
175    }
176
177    // ConcurrentHashMap
178    cl = ConcurrentHashMap.class;
179    expected = ClassSize.estimateBase(cl, false);
180    actual = ClassSize.CONCURRENT_HASHMAP;
181    if(expected != actual) {
182      ClassSize.estimateBase(cl, true);
183      assertEquals(expected, actual);
184    }
185
186    // ConcurrentSkipListMap
187    cl = ConcurrentSkipListMap.class;
188    expected = ClassSize.estimateBase(cl, false);
189    actual = ClassSize.CONCURRENT_SKIPLISTMAP;
190    if(expected != actual) {
191      ClassSize.estimateBase(cl, true);
192      assertEquals(expected, actual);
193    }
194
195    // CellArrayMap
196    cl = CellArrayMap.class;
197    expected = ClassSize.estimateBase(cl, false);
198    actual = ClassSize.CELL_ARRAY_MAP;
199    if(expected != actual) {
200      ClassSize.estimateBase(cl, true);
201      assertEquals(expected, actual);
202    }
203
204    // ReentrantReadWriteLock
205    cl = ReentrantReadWriteLock.class;
206    expected = ClassSize.estimateBase(cl, false);
207    actual = ClassSize.REENTRANT_LOCK;
208    if(expected != actual) {
209      ClassSize.estimateBase(cl, true);
210      assertEquals(expected, actual);
211    }
212
213    // AtomicLong
214    cl = AtomicLong.class;
215    expected = ClassSize.estimateBase(cl, false);
216    actual = ClassSize.ATOMIC_LONG;
217    if(expected != actual) {
218      ClassSize.estimateBase(cl, true);
219      assertEquals(expected, actual);
220    }
221
222    // AtomicInteger
223    cl = AtomicInteger.class;
224    expected = ClassSize.estimateBase(cl, false);
225    actual = ClassSize.ATOMIC_INTEGER;
226    if(expected != actual) {
227      ClassSize.estimateBase(cl, true);
228      assertEquals(expected, actual);
229    }
230
231    // AtomicBoolean
232    cl = AtomicBoolean.class;
233    expected = ClassSize.estimateBase(cl, false);
234    actual = ClassSize.ATOMIC_BOOLEAN;
235    if(expected != actual) {
236      ClassSize.estimateBase(cl, true);
237      assertEquals(expected, actual);
238    }
239
240    // CopyOnWriteArraySet
241    cl = CopyOnWriteArraySet.class;
242    expected = ClassSize.estimateBase(cl, false);
243    actual = ClassSize.COPYONWRITE_ARRAYSET;
244    if(expected != actual) {
245      ClassSize.estimateBase(cl, true);
246      assertEquals(expected, actual);
247    }
248
249    // CopyOnWriteArrayList
250    cl = CopyOnWriteArrayList.class;
251    expected = ClassSize.estimateBase(cl, false);
252    actual = ClassSize.COPYONWRITE_ARRAYLIST;
253    if(expected != actual) {
254      ClassSize.estimateBase(cl, true);
255      assertEquals(expected, actual);
256    }
257
258    // SyncTimeRangeTracker
259    cl = SyncTimeRangeTracker.class;
260    expected = ClassSize.estimateBase(cl, false);
261    actual = ClassSize.SYNC_TIMERANGE_TRACKER;
262    if (expected != actual) {
263      ClassSize.estimateBase(cl, true);
264      assertEquals(expected, actual);
265    }
266
267    // NonSyncTimeRangeTracker
268    cl = NonSyncTimeRangeTracker.class;
269    expected = ClassSize.estimateBase(cl, false);
270    actual = ClassSize.NON_SYNC_TIMERANGE_TRACKER;
271    if (expected != actual) {
272      ClassSize.estimateBase(cl, true);
273      assertEquals(expected, actual);
274    }
275
276    // CellSet
277    cl = CellSet.class;
278    expected = ClassSize.estimateBase(cl, false);
279    actual = ClassSize.CELL_SET;
280    if (expected != actual) {
281      ClassSize.estimateBase(cl, true);
282      assertEquals(expected, actual);
283    }
284  }
285
286  /**
287   * Testing the classes that implements HeapSize and are a part of 0.20.
288   * Some are not tested here for example BlockIndex which is tested in
289   * TestHFile since it is a non public class
290   * @throws IOException
291   */
292  @Test
293  public void testSizes() throws IOException {
294    Class<?> cl;
295    long expected;
296    long actual;
297
298    //KeyValue
299    cl = KeyValue.class;
300    expected = ClassSize.estimateBase(cl, false);
301    KeyValue kv = new KeyValue();
302    actual = kv.heapSize();
303    if(expected != actual) {
304      ClassSize.estimateBase(cl, true);
305      assertEquals(expected, actual);
306    }
307
308    //LruBlockCache Overhead
309    cl = LruBlockCache.class;
310    actual = LruBlockCache.CACHE_FIXED_OVERHEAD;
311    expected = ClassSize.estimateBase(cl, false);
312    if(expected != actual) {
313      ClassSize.estimateBase(cl, true);
314      assertEquals(expected, actual);
315    }
316
317    // CachedBlock Fixed Overhead
318    // We really need "deep" sizing but ClassSize does not do this.
319    // Perhaps we should do all these more in this style....
320    cl = LruCachedBlock.class;
321    actual = LruCachedBlock.PER_BLOCK_OVERHEAD;
322    expected = ClassSize.estimateBase(cl, false);
323    expected += ClassSize.estimateBase(String.class, false);
324    expected += ClassSize.estimateBase(ByteBuffer.class, false);
325    if(expected != actual) {
326      ClassSize.estimateBase(cl, true);
327      ClassSize.estimateBase(String.class, true);
328      ClassSize.estimateBase(ByteBuffer.class, true);
329      assertEquals(expected, actual);
330    }
331
332    // DefaultMemStore Overhead
333    cl = DefaultMemStore.class;
334    actual = DefaultMemStore.FIXED_OVERHEAD;
335    expected = ClassSize.estimateBase(cl, false);
336    if(expected != actual) {
337      ClassSize.estimateBase(cl, true);
338      assertEquals(expected, actual);
339    }
340
341    // DefaultMemStore Deep Overhead
342    actual = DefaultMemStore.DEEP_OVERHEAD;
343    expected = ClassSize.estimateBase(cl, false);
344    if (expected != actual) {
345      ClassSize.estimateBase(cl, true);
346      assertEquals(expected, actual);
347    }
348
349    // CompactingMemStore Deep Overhead
350    cl = CompactingMemStore.class;
351    actual = CompactingMemStore.DEEP_OVERHEAD;
352    expected = ClassSize.estimateBase(cl, false);
353    expected += ClassSize.estimateBase(AtomicBoolean.class, false);
354    expected += ClassSize.estimateBase(AtomicBoolean.class, false);
355    expected += ClassSize.estimateBase(CompactionPipeline.class, false);
356    expected += ClassSize.estimateBase(LinkedList.class, false); //inside CompactionPipeline
357    expected += ClassSize.estimateBase(LinkedList.class, false); //inside CompactionPipeline
358    expected += ClassSize.estimateBase(MemStoreCompactor.class, false);
359    expected += ClassSize.estimateBase(AtomicBoolean.class, false);// inside MemStoreCompactor
360    if (expected != actual) {
361      ClassSize.estimateBase(cl, true);
362      ClassSize.estimateBase(AtomicBoolean.class, true);
363      ClassSize.estimateBase(AtomicBoolean.class, true);
364      ClassSize.estimateBase(CompactionPipeline.class, true);
365      ClassSize.estimateBase(LinkedList.class, true);
366      ClassSize.estimateBase(LinkedList.class, true);
367      ClassSize.estimateBase(MemStoreCompactor.class, true);
368      ClassSize.estimateBase(AtomicBoolean.class, true);
369      assertEquals(expected, actual);
370    }
371
372    // Segment Deep overhead
373    cl = Segment.class;
374    actual = Segment.DEEP_OVERHEAD;
375    expected = ClassSize.estimateBase(cl, false);
376    expected += 2 * ClassSize.estimateBase(AtomicLong.class, false);
377    expected += ClassSize.estimateBase(AtomicReference.class, false);
378    expected += ClassSize.estimateBase(CellSet.class, false);
379    if (expected != actual) {
380      ClassSize.estimateBase(cl, true);
381      ClassSize.estimateBase(AtomicLong.class, true);
382      ClassSize.estimateBase(AtomicReference.class, true);
383      ClassSize.estimateBase(CellSet.class, true);
384      assertEquals(expected, actual);
385    }
386
387    // MutableSegment Deep overhead
388    cl = MutableSegment.class;
389    actual = MutableSegment.DEEP_OVERHEAD;
390    expected = ClassSize.estimateBase(cl, false);
391    expected += 2 * ClassSize.estimateBase(AtomicLong.class, false);
392    expected += ClassSize.estimateBase(AtomicReference.class, false);
393    expected += ClassSize.estimateBase(CellSet.class, false);
394    expected += ClassSize.estimateBase(SyncTimeRangeTracker.class, false);
395    expected += ClassSize.estimateBase(ConcurrentSkipListMap.class, false);
396    if (expected != actual) {
397      ClassSize.estimateBase(cl, true);
398      ClassSize.estimateBase(AtomicLong.class, true);
399      ClassSize.estimateBase(AtomicLong.class, true);
400      ClassSize.estimateBase(AtomicReference.class, true);
401      ClassSize.estimateBase(CellSet.class, true);
402      ClassSize.estimateBase(SyncTimeRangeTracker.class, true);
403      ClassSize.estimateBase(ConcurrentSkipListMap.class, true);
404      assertEquals(expected, actual);
405    }
406
407    // ImmutableSegments Deep overhead
408    cl = ImmutableSegment.class;
409    actual = ImmutableSegment.DEEP_OVERHEAD;
410    expected = ClassSize.estimateBase(cl, false);
411    expected += 2 * ClassSize.estimateBase(AtomicLong.class, false);
412    expected += ClassSize.estimateBase(AtomicReference.class, false);
413    expected += ClassSize.estimateBase(CellSet.class, false);
414    expected += ClassSize.estimateBase(NonSyncTimeRangeTracker.class, false);
415    if (expected != actual) {
416      ClassSize.estimateBase(cl, true);
417      ClassSize.estimateBase(AtomicLong.class, true);
418      ClassSize.estimateBase(AtomicLong.class, true);
419      ClassSize.estimateBase(AtomicReference.class, true);
420      ClassSize.estimateBase(CellSet.class, true);
421      ClassSize.estimateBase(NonSyncTimeRangeTracker.class, true);
422      assertEquals(expected, actual);
423    }
424
425    cl = CSLMImmutableSegment.class;
426    actual = CSLMImmutableSegment.DEEP_OVERHEAD_CSLM;
427    expected = ClassSize.estimateBase(cl, false);
428    expected += 2 * ClassSize.estimateBase(AtomicLong.class, false);
429    expected += ClassSize.estimateBase(AtomicReference.class, false);
430    expected += ClassSize.estimateBase(CellSet.class, false);
431    expected += ClassSize.estimateBase(NonSyncTimeRangeTracker.class, false);
432    expected += ClassSize.estimateBase(ConcurrentSkipListMap.class, false);
433    if (expected != actual) {
434      ClassSize.estimateBase(cl, true);
435      ClassSize.estimateBase(AtomicLong.class, true);
436      ClassSize.estimateBase(AtomicLong.class, true);
437      ClassSize.estimateBase(AtomicReference.class, true);
438      ClassSize.estimateBase(CellSet.class, true);
439      ClassSize.estimateBase(NonSyncTimeRangeTracker.class, true);
440      ClassSize.estimateBase(ConcurrentSkipListMap.class, true);
441      assertEquals(expected, actual);
442    }
443    cl = CellArrayImmutableSegment.class;
444    actual = CellArrayImmutableSegment.DEEP_OVERHEAD_CAM;
445    expected = ClassSize.estimateBase(cl, false);
446    expected += 2 * ClassSize.estimateBase(AtomicLong.class, false);
447    expected += ClassSize.estimateBase(AtomicReference.class, false);
448    expected += ClassSize.estimateBase(CellSet.class, false);
449    expected += ClassSize.estimateBase(NonSyncTimeRangeTracker.class, false);
450    expected += ClassSize.estimateBase(CellArrayMap.class, false);
451    if (expected != actual) {
452      ClassSize.estimateBase(cl, true);
453      ClassSize.estimateBase(AtomicLong.class, true);
454      ClassSize.estimateBase(AtomicLong.class, true);
455      ClassSize.estimateBase(AtomicReference.class, true);
456      ClassSize.estimateBase(CellSet.class, true);
457      ClassSize.estimateBase(NonSyncTimeRangeTracker.class, true);
458      ClassSize.estimateBase(CellArrayMap.class, true);
459      assertEquals(expected, actual);
460    }
461
462    // Store Overhead
463    cl = HStore.class;
464    actual = HStore.FIXED_OVERHEAD;
465    expected = ClassSize.estimateBase(cl, false);
466    if(expected != actual) {
467      ClassSize.estimateBase(cl, true);
468      assertEquals(expected, actual);
469    }
470
471    // Region Overhead
472    cl = HRegion.class;
473    actual = HRegion.FIXED_OVERHEAD;
474    expected = ClassSize.estimateBase(cl, false);
475    if (expected != actual) {
476      ClassSize.estimateBase(cl, true);
477      assertEquals(expected, actual);
478    }
479
480    cl = StoreHotnessProtector.class;
481    actual = StoreHotnessProtector.FIXED_SIZE;
482    expected = ClassSize.estimateBase(cl, false);
483    if (expected != actual) {
484      ClassSize.estimateBase(cl, true);
485      assertEquals(expected, actual);
486    }
487
488    // Block cache key overhead. Only tests fixed overhead as estimating heap
489    // size of strings is hard.
490    cl = BlockCacheKey.class;
491    actual = BlockCacheKey.FIXED_OVERHEAD;
492    expected  = ClassSize.estimateBase(cl, false);
493    if (expected != actual) {
494      ClassSize.estimateBase(cl, true);
495      assertEquals(expected, actual);
496    }
497
498    // Currently NOT testing Deep Overheads of many of these classes.
499    // Deep overheads cover a vast majority of stuff, but will not be 100%
500    // accurate because it's unclear when we're referencing stuff that's already
501    // accounted for.  But we have satisfied our two core requirements.
502    // Sizing is quite accurate now, and our tests will throw errors if
503    // any of these classes are modified without updating overhead sizes.
504  }
505
506  @Test
507  public void testMutations(){
508    Class<?> cl;
509    long expected;
510    long actual;
511
512    cl = TimeRange.class;
513    actual = ClassSize.TIMERANGE;
514    expected  = ClassSize.estimateBase(cl, false);
515    if (expected != actual) {
516      ClassSize.estimateBase(cl, true);
517      assertEquals(expected, actual);
518    }
519
520    byte[] row = new byte[] { 0 };
521    cl = Put.class;
522    actual = Mutation.MUTATION_OVERHEAD + ClassSize.align(ClassSize.ARRAY);
523    expected = ClassSize.estimateBase(cl, false);
524    //The actual TreeMap is not included in the above calculation
525    expected += ClassSize.align(ClassSize.TREEMAP);
526    expected += ClassSize.align(ClassSize.INTEGER); // priority
527    if (expected != actual) {
528      ClassSize.estimateBase(cl, true);
529      assertEquals(expected, actual);
530    }
531
532    cl = Delete.class;
533    actual = Mutation.MUTATION_OVERHEAD + ClassSize.align(ClassSize.ARRAY);
534    expected  = ClassSize.estimateBase(cl, false);
535    //The actual TreeMap is not included in the above calculation
536    expected += ClassSize.align(ClassSize.TREEMAP);
537    expected += ClassSize.align(ClassSize.INTEGER); // priority
538    if (expected != actual) {
539      ClassSize.estimateBase(cl, true);
540      assertEquals(expected, actual);
541    }
542  }
543
544  @Test
545  public void testReferenceSize() {
546    LOG.info("ClassSize.REFERENCE is " + ClassSize.REFERENCE);
547    // oop should be either 4 or 8
548    assertTrue(ClassSize.REFERENCE == 4 || ClassSize.REFERENCE == 8);
549  }
550
551  @Test
552  public void testObjectSize() throws IOException {
553    LOG.info("header:" + ClassSize.OBJECT);
554    LOG.info("array header:" + ClassSize.ARRAY);
555
556    if (ClassSize.is32BitJVM()) {
557      assertEquals(ClassSize.OBJECT, 8);
558    } else {
559      assertTrue(ClassSize.OBJECT == 12 || ClassSize.OBJECT == 16); // depending on CompressedOops
560    }
561    if (ClassSize.useUnsafeLayout()) {
562      assertEquals(ClassSize.ARRAY, ClassSize.OBJECT + 4);
563    } else {
564      assertEquals(ClassSize.ARRAY, ClassSize.OBJECT + 8);
565    }
566  }
567}
568