001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertTrue;
023import static org.junit.Assert.fail;
024import java.io.IOException;
025import java.lang.management.ManagementFactory;
026import java.util.Iterator;
027import java.util.List;
028
029import org.apache.hadoop.conf.Configuration;
030import org.apache.hadoop.fs.FileSystem;
031import org.apache.hadoop.hbase.ChoreService;
032import org.apache.hadoop.hbase.CoordinatedStateManager;
033import org.apache.hadoop.hbase.HBaseClassTestRule;
034import org.apache.hadoop.hbase.HBaseConfiguration;
035import org.apache.hadoop.hbase.HBaseTestingUtil;
036import org.apache.hadoop.hbase.HConstants;
037import org.apache.hadoop.hbase.Server;
038import org.apache.hadoop.hbase.ServerName;
039import org.apache.hadoop.hbase.Waiter;
040import org.apache.hadoop.hbase.client.AsyncClusterConnection;
041import org.apache.hadoop.hbase.client.Connection;
042import org.apache.hadoop.hbase.io.hfile.BlockCache;
043import org.apache.hadoop.hbase.io.hfile.BlockCacheKey;
044import org.apache.hadoop.hbase.io.hfile.CacheStats;
045import org.apache.hadoop.hbase.io.hfile.Cacheable;
046import org.apache.hadoop.hbase.io.hfile.CachedBlock;
047import org.apache.hadoop.hbase.io.hfile.ResizableBlockCache;
048import org.apache.hadoop.hbase.io.util.MemorySizeUtil;
049import org.apache.hadoop.hbase.regionserver.HeapMemoryManager.TunerContext;
050import org.apache.hadoop.hbase.regionserver.HeapMemoryManager.TunerResult;
051import org.apache.hadoop.hbase.testclassification.MediumTests;
052import org.apache.hadoop.hbase.testclassification.RegionServerTests;
053import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
054import org.junit.ClassRule;
055import org.junit.Test;
056import org.junit.experimental.categories.Category;
057
058@Category({RegionServerTests.class, MediumTests.class})
059public class TestHeapMemoryManager {
060
061  @ClassRule
062  public static final HBaseClassTestRule CLASS_RULE =
063      HBaseClassTestRule.forClass(TestHeapMemoryManager.class);
064
065  private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
066
067  private long maxHeapSize = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax();
068
069  @Test
070  public void testAutoTunerShouldBeOffWhenMaxMinRangesForMemstoreIsNotGiven() throws Exception {
071    Configuration conf = HBaseConfiguration.create();
072    conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_KEY, 0.02f);
073    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.75f);
074    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.03f);
075    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf);
076    HeapMemoryManager manager = new HeapMemoryManager(new BlockCacheStub(0),
077        new MemstoreFlusherStub(0), new RegionServerStub(conf),
078        regionServerAccounting);
079    assertFalse(manager.isTunerOn());
080  }
081
082  @Test
083  public void testAutoTunerShouldBeOffWhenMaxMinRangesForBlockCacheIsNotGiven() throws Exception {
084    Configuration conf = HBaseConfiguration.create();
085    conf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.02f);
086    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
087    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.03f);
088    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf);
089    HeapMemoryManager manager = new HeapMemoryManager(new BlockCacheStub(0),
090        new MemstoreFlusherStub(0), new RegionServerStub(conf),
091        regionServerAccounting);
092    assertFalse(manager.isTunerOn());
093  }
094
095  @Test
096  public void testWhenMemstoreAndBlockCacheMaxMinChecksFails() throws Exception {
097    BlockCacheStub blockCache = new BlockCacheStub(0);
098    Configuration conf = HBaseConfiguration.create();
099    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
100    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.06f);
101    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf);
102    MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub(0);
103    try {
104      new HeapMemoryManager(blockCache, memStoreFlusher,
105          new RegionServerStub(conf), regionServerAccounting);
106      fail();
107    } catch (RuntimeException e) {
108    }
109    conf = HBaseConfiguration.create();
110    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.2f);
111    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
112    try {
113      new HeapMemoryManager(blockCache, memStoreFlusher,
114          new RegionServerStub(conf), regionServerAccounting);
115      fail();
116    } catch (RuntimeException e) {
117    }
118  }
119
120  @Test
121  public void testWhenClusterIsWriteHeavyWithEmptyMemstore() throws Exception {
122    Configuration conf = HBaseConfiguration.create();
123    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
124    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f);
125    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
126    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f);
127    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
128    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
129    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf);
130    MemstoreFlusherStub memStoreFlusher =
131        new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
132    // Empty block cache and memstore
133    blockCache.setTestBlockSize(0);
134    regionServerAccounting.setTestMemstoreSize(0);
135    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
136    // Let the system start with default values for memstore heap and block cache size.
137    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
138        new RegionServerStub(conf), regionServerAccounting);
139    long oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
140    long oldBlockCacheSize = blockCache.maxSize;
141    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
142    heapMemoryManager.start(choreService);
143    memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_HIGHER_MARK;
144    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
145    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
146    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
147    memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK;
148    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
149    // Allow the tuner to run once and do necessary memory up
150    Thread.sleep(1500);
151    // No changes should be made by tuner as we already have lot of empty space
152    assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
153    assertEquals(oldBlockCacheSize, blockCache.maxSize);
154  }
155
156  @Test
157  public void testHeapMemoryManagerWhenOffheapFlushesHappenUnderReadHeavyCase() throws Exception {
158    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
159    Configuration conf = HBaseConfiguration.create();
160    conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_LOWER_LIMIT_KEY, 0.7f);
161    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
162    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f);
163    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
164    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f);
165    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
166    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
167    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf, true);
168    MemstoreFlusherStub memStoreFlusher =
169        new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
170    // Empty memstore and but nearly filled block cache
171    blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8));
172    regionServerAccounting.setTestMemstoreSize(0);
173    // Let the system start with default values for memstore heap and block cache size.
174    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
175        new RegionServerStub(conf), regionServerAccounting);
176    long oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
177    long oldBlockCacheSize = blockCache.maxSize;
178    float maxStepValue = DefaultHeapMemoryTuner.DEFAULT_MIN_STEP_VALUE;
179    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
180    heapMemoryManager.start(choreService);
181    blockCache.evictBlock(null);
182    blockCache.evictBlock(null);
183    blockCache.evictBlock(null);
184    // do some offheap flushes also. So there should be decrease in memstore but
185    // not as that when we don't have offheap flushes
186    memStoreFlusher.flushType = FlushType.ABOVE_OFFHEAP_HIGHER_MARK;
187    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
188    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
189    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
190    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
191    // Allow the tuner to run once and do necessary memory up
192    waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
193    assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
194    assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize);
195    oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
196    oldBlockCacheSize = blockCache.maxSize;
197    // Do some more evictions before the next run of HeapMemoryTuner
198    blockCache.evictBlock(null);
199    // Allow the tuner to run once and do necessary memory up
200    waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
201    assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
202    assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize);
203  }
204
205  @Test
206  public void testHeapMemoryManagerWithOffheapMemstoreAndMixedWorkload() throws Exception {
207    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
208    Configuration conf = HBaseConfiguration.create();
209    conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_LOWER_LIMIT_KEY, 0.7f);
210    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
211    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f);
212    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
213    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f);
214    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
215    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
216    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf, true);
217    MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
218    // Empty memstore and but nearly filled block cache
219    blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8));
220    regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8));
221    // Let the system start with default values for memstore heap and block cache size.
222    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
223        new RegionServerStub(conf), regionServerAccounting);
224    long oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
225    long oldBlockCacheSize = blockCache.maxSize;
226    float maxStepValue = DefaultHeapMemoryTuner.DEFAULT_MIN_STEP_VALUE;
227    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
228    heapMemoryManager.start(choreService);
229    blockCache.evictBlock(null);
230    blockCache.evictBlock(null);
231    blockCache.evictBlock(null);
232    // do some offheap flushes also. So there should be decrease in memstore but
233    // not as that when we don't have offheap flushes
234    memStoreFlusher.flushType = FlushType.ABOVE_OFFHEAP_HIGHER_MARK;
235    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
236    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
237    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
238    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
239    // Allow the tuner to run once and do necessary memory up
240    waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
241    assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
242    assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize);
243    oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
244    oldBlockCacheSize = blockCache.maxSize;
245    // change memstore size
246    // regionServerAccounting.setTestMemstoreSize((long)(maxHeapSize * 0.4 * 0.8));
247    // The memstore size would have decreased. Now again do some flushes and ensure the
248    // flushes are due to onheap overhead. This should once again call for increase in
249    // memstore size but that increase should be to the safe size
250    memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_HIGHER_MARK;
251    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
252    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
253    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
254    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
255    // Allow the tuner to run once and do necessary memory up
256    waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
257    assertHeapSpaceDelta(maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
258    assertHeapSpaceDelta(-maxStepValue, oldBlockCacheSize, blockCache.maxSize);
259  }
260
261  @Test
262  public void testWhenClusterIsReadHeavyWithEmptyBlockCache() throws Exception {
263    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
264    Configuration conf = HBaseConfiguration.create();
265    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
266    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f);
267    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
268    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f);
269    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
270    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
271    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf);
272    MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
273    // Empty block cache and memstore
274    blockCache.setTestBlockSize(0);
275    regionServerAccounting.setTestMemstoreSize(0);
276    // Let the system start with default values for memstore heap and block cache size.
277    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
278        new RegionServerStub(conf), regionServerAccounting);
279    long oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
280    long oldBlockCacheSize = blockCache.maxSize;
281    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
282    heapMemoryManager.start(choreService);
283    blockCache.evictBlock(null);
284    blockCache.evictBlock(null);
285    blockCache.evictBlock(null);
286    // Allow the tuner to run once and do necessary memory up
287    Thread.sleep(1500);
288    // No changes should be made by tuner as we already have lot of empty space
289    assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
290    assertEquals(oldBlockCacheSize, blockCache.maxSize);
291  }
292
293  @Test
294  public void testWhenClusterIsWriteHeavy() throws Exception {
295    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
296    Configuration conf = HBaseConfiguration.create();
297    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
298    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f);
299    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
300    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f);
301    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
302    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
303    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf);
304    MemstoreFlusherStub memStoreFlusher =
305        new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
306    // Empty block cache and but nearly filled memstore
307    blockCache.setTestBlockSize(0);
308    regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8));
309    // Let the system start with default values for memstore heap and block cache size.
310    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
311        new RegionServerStub(conf), regionServerAccounting);
312    long oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
313    long oldBlockCacheSize = blockCache.maxSize;
314    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
315    heapMemoryManager.start(choreService);
316    memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK;
317    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
318    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
319    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
320    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
321    // Allow the tuner to run once and do necessary memory up
322    waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
323    assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldMemstoreHeapSize,
324        memStoreFlusher.memstoreSize);
325    assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldBlockCacheSize,
326        blockCache.maxSize);
327    oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
328    oldBlockCacheSize = blockCache.maxSize;
329    // Do some more flushes before the next run of HeapMemoryTuner
330    memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK;
331    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
332    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
333    // Allow the tuner to run once and do necessary memory up
334    waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
335    assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldMemstoreHeapSize,
336        memStoreFlusher.memstoreSize);
337    assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldBlockCacheSize,
338        blockCache.maxSize);
339  }
340
341  @Test
342  public void testWhenClusterIsWriteHeavyWithOffheapMemstore() throws Exception {
343    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
344    Configuration conf = HBaseConfiguration.create();
345    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
346    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f);
347    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
348    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f);
349    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
350    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
351    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf);
352    MemstoreFlusherStub memStoreFlusher =
353        new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
354    // Empty block cache and but nearly filled memstore
355    blockCache.setTestBlockSize(0);
356    regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8));
357    // Let the system start with default values for memstore heap and block cache size.
358    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
359        new RegionServerStub(conf), regionServerAccounting);
360    long oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
361    long oldBlockCacheSize = blockCache.maxSize;
362    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
363    heapMemoryManager.start(choreService);
364    // this should not change anything with onheap memstore
365    memStoreFlusher.flushType = FlushType.ABOVE_OFFHEAP_HIGHER_MARK;
366    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
367    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
368    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
369    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
370    // Allow the tuner to run once and do necessary memory up
371    Thread.sleep(1500);
372    // No changes should be made by tuner as we already have lot of empty space
373    assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
374    assertEquals(oldBlockCacheSize, blockCache.maxSize);
375  }
376
377  @Test
378  public void testWhenClusterIsReadHeavy() throws Exception {
379    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
380    Configuration conf = HBaseConfiguration.create();
381    conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_LOWER_LIMIT_KEY, 0.7f);
382    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
383    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f);
384    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
385    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f);
386    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
387    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
388    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf);
389    MemstoreFlusherStub memStoreFlusher =
390        new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
391    // Empty memstore and but nearly filled block cache
392    blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8));
393    regionServerAccounting.setTestMemstoreSize(0);
394    // Let the system start with default values for memstore heap and block cache size.
395    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
396        new RegionServerStub(conf), new RegionServerAccountingStub(conf));
397    long oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
398    long oldBlockCacheSize = blockCache.maxSize;
399    long oldMemstoreLowerMarkSize = 7 * oldMemstoreHeapSize / 10;
400    long maxTuneSize = oldMemstoreHeapSize -  (oldMemstoreLowerMarkSize + oldMemstoreHeapSize) / 2;
401    float maxStepValue = (maxTuneSize * 1.0f) / oldMemstoreHeapSize;
402    maxStepValue = maxStepValue > DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE ?
403        DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE:maxStepValue;
404    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
405    heapMemoryManager.start(choreService);
406    blockCache.evictBlock(null);
407    blockCache.evictBlock(null);
408    blockCache.evictBlock(null);
409    // Allow the tuner to run once and do necessary memory up
410    waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
411    assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
412    assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize);
413    oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
414    oldBlockCacheSize = blockCache.maxSize;
415    oldMemstoreLowerMarkSize = 7 * oldMemstoreHeapSize / 10;
416    maxTuneSize = oldMemstoreHeapSize -  (oldMemstoreLowerMarkSize + oldMemstoreHeapSize) / 2;
417    maxStepValue = (maxTuneSize * 1.0f) / oldMemstoreHeapSize;
418    maxStepValue = maxStepValue > DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE ?
419        DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE:maxStepValue;
420    // Do some more evictions before the next run of HeapMemoryTuner
421    blockCache.evictBlock(null);
422    // Allow the tuner to run once and do necessary memory up
423    waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
424    assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
425    assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize);
426  }
427
428  @Test
429  public void testWhenClusterIsHavingMoreWritesThanReads() throws Exception {
430    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
431    Configuration conf = HBaseConfiguration.create();
432    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
433    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f);
434    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
435    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f);
436    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
437    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
438    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf);
439    MemstoreFlusherStub memStoreFlusher =
440        new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
441    // Both memstore and block cache are nearly filled
442    blockCache.setTestBlockSize(0);
443    regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8));
444    blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8));
445    // Let the system start with default values for memstore heap and block cache size.
446    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
447        new RegionServerStub(conf), regionServerAccounting);
448    long oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
449    long oldBlockCacheSize = blockCache.maxSize;
450    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
451    heapMemoryManager.start(choreService);
452    memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK;
453    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
454    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
455    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
456    blockCache.evictBlock(null);
457    // Allow the tuner to run once and do necessary memory up
458    Thread.sleep(1500);
459    // No changes should happen as there is undefined increase in flushes and evictions
460    assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
461    assertEquals(oldBlockCacheSize, blockCache.maxSize);
462    // Do some more flushes before the next run of HeapMemoryTuner
463    memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK;
464    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
465    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
466    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
467    // Allow the tuner to run once and do necessary memory up
468    waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
469    assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldMemstoreHeapSize,
470        memStoreFlusher.memstoreSize);
471    assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldBlockCacheSize,
472        blockCache.maxSize);
473  }
474
475  @Test
476  public void testBlockedFlushesIncreaseMemstoreInSteadyState() throws Exception {
477    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
478    MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
479    Configuration conf = HBaseConfiguration.create();
480    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
481    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f);
482    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
483    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f);
484    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
485    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
486    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf);
487    // Both memstore and block cache are nearly filled
488    blockCache.setTestBlockSize(0);
489    regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8));
490    blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8));
491    // Let the system start with default values for memstore heap and block cache size.
492    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
493        new RegionServerStub(conf), regionServerAccounting);
494    long oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
495    long oldBlockCacheSize = blockCache.maxSize;
496    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
497    heapMemoryManager.start(choreService);
498    memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK;
499    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
500    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
501    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
502    blockCache.evictBlock(null);
503    blockCache.evictBlock(null);
504    // Allow the tuner to run once and do necessary memory up
505    Thread.sleep(1500);
506    // No changes should happen as there is undefined increase in flushes and evictions
507    assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
508    assertEquals(oldBlockCacheSize, blockCache.maxSize);
509    // Flushes that block updates
510    memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_HIGHER_MARK;
511    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
512    blockCache.evictBlock(null);
513    blockCache.evictBlock(null);
514    blockCache.evictBlock(null);
515    blockCache.evictBlock(null);
516    // Allow the tuner to run once and do necessary memory up
517    Thread.sleep(1500);
518    assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldMemstoreHeapSize,
519        memStoreFlusher.memstoreSize);
520    assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldBlockCacheSize,
521        blockCache.maxSize);
522  }
523
524  @Test
525  public void testPluggingInHeapMemoryTuner() throws Exception {
526    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
527    MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
528    Configuration conf = HBaseConfiguration.create();
529    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.78f);
530    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.05f);
531    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.75f);
532    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.02f);
533    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
534    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
535    conf.setClass(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_CLASS, CustomHeapMemoryTuner.class,
536        HeapMemoryTuner.class);
537    // Let the system start with default values for memstore heap and block cache size.
538    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
539        new RegionServerStub(conf), new RegionServerAccountingStub(conf));
540    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
541    heapMemoryManager.start(choreService);
542    // Now we wants to be in write mode. Set bigger memstore size from CustomHeapMemoryTuner
543    CustomHeapMemoryTuner.memstoreSize = 0.78f;
544    CustomHeapMemoryTuner.blockCacheSize = 0.02f;
545    // Allow the tuner to run once and do necessary memory up
546    waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
547    assertHeapSpace(0.78f, memStoreFlusher.memstoreSize);// Memstore
548    assertHeapSpace(0.02f, blockCache.maxSize);// BlockCache
549    // Now we wants to be in read mode. Set bigger memstore size from CustomHeapMemoryTuner
550    CustomHeapMemoryTuner.blockCacheSize = 0.75f;
551    CustomHeapMemoryTuner.memstoreSize = 0.05f;
552    // Allow the tuner to run once and do necessary memory up
553    waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
554    assertHeapSpace(0.75f, blockCache.maxSize);// BlockCache
555    assertHeapSpace(0.05f, memStoreFlusher.memstoreSize);// Memstore
556  }
557
558  @Test
559  public void testWhenSizeGivenByHeapTunerGoesOutsideRange() throws Exception {
560    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
561    MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
562    Configuration conf = HBaseConfiguration.create();
563    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.7f);
564    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.1f);
565    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
566    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.1f);
567    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
568    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
569    conf.setClass(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_CLASS, CustomHeapMemoryTuner.class,
570        HeapMemoryTuner.class);
571    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
572        new RegionServerStub(conf), new RegionServerAccountingStub(conf));
573    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
574    heapMemoryManager.start(choreService);
575    CustomHeapMemoryTuner.memstoreSize = 0.78f;
576    CustomHeapMemoryTuner.blockCacheSize = 0.02f;
577    Thread.sleep(1500); // Allow the tuner to run once and do necessary memory up
578    // Even if the tuner says to set the memstore to 78%, HBase makes it as 70% as that is the
579    // upper bound. Same with block cache as 10% is the lower bound.
580    assertHeapSpace(0.7f, memStoreFlusher.memstoreSize);
581    assertHeapSpace(0.1f, blockCache.maxSize);
582  }
583
584  @Test
585  public void testWhenCombinedHeapSizesFromTunerGoesOutSideMaxLimit() throws Exception {
586    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
587    MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
588    Configuration conf = HBaseConfiguration.create();
589    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.7f);
590    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.1f);
591    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
592    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.1f);
593    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
594    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
595    conf.setClass(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_CLASS, CustomHeapMemoryTuner.class,
596        HeapMemoryTuner.class);
597    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
598        new RegionServerStub(conf), new RegionServerAccountingStub(conf));
599    long oldMemstoreSize = memStoreFlusher.memstoreSize;
600    long oldBlockCacheSize = blockCache.maxSize;
601    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
602    heapMemoryManager.start(choreService);
603    CustomHeapMemoryTuner.memstoreSize = 0.7f;
604    CustomHeapMemoryTuner.blockCacheSize = 0.3f;
605    // Allow the tuner to run once and do necessary memory up
606    Thread.sleep(1500);
607    assertEquals(oldMemstoreSize, memStoreFlusher.memstoreSize);
608    assertEquals(oldBlockCacheSize, blockCache.maxSize);
609  }
610
611  private void assertHeapSpace(float expectedHeapPercentage, long currentHeapSpace) {
612    long expected = (long) (this.maxHeapSize * expectedHeapPercentage);
613    assertEquals(expected, currentHeapSpace);
614  }
615
616  private void assertHeapSpaceDelta(double expectedDeltaPercent, long oldHeapSpace,
617      long newHeapSpace) {
618    double expctedMinDelta = (double) (this.maxHeapSize * expectedDeltaPercent);
619    // Tolerable error
620    double error = 0.95;
621    if (expectedDeltaPercent > 0) {
622      assertTrue(expctedMinDelta*error <= (double)(newHeapSpace - oldHeapSpace));
623      assertTrue(expctedMinDelta/error >= (double)(newHeapSpace - oldHeapSpace));
624    } else {
625      assertTrue(-expctedMinDelta*error <= (double)(oldHeapSpace - newHeapSpace));
626      assertTrue(-expctedMinDelta/error >= (double)(oldHeapSpace - newHeapSpace));
627    }
628  }
629
630
631  private void waitForTune(final MemstoreFlusherStub memStoreFlusher,
632                           final long oldMemstoreHeapSize) throws Exception {
633    // Allow the tuner to run once and do necessary memory up
634    UTIL.waitFor(10000, new Waiter.Predicate<Exception>() {
635      @Override
636      public boolean evaluate() throws Exception {
637        return oldMemstoreHeapSize != memStoreFlusher.memstoreSize;
638      }
639    });
640  }
641
642  private static class BlockCacheStub implements ResizableBlockCache {
643    CacheStats stats = new CacheStats("test");
644    long maxSize = 0;
645    private long testBlockSize = 0;
646
647    public BlockCacheStub(long size){
648      this.maxSize = size;
649    }
650
651    @Override
652    public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory) {
653
654    }
655
656    @Override
657    public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf) {
658
659    }
660
661    @Override
662    public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching, boolean repeat,
663        boolean updateCacheMetrics) {
664      return null;
665    }
666
667    @Override
668    public boolean evictBlock(BlockCacheKey cacheKey) {
669      stats.evicted(0, cacheKey != null ? cacheKey.isPrimary() : true);
670      return false;
671    }
672
673    @Override
674    public int evictBlocksByHfileName(String hfileName) {
675      stats.evicted(0, true); // Just assuming only one block for file here.
676      return 0;
677    }
678
679    @Override
680    public CacheStats getStats() {
681      return this.stats;
682    }
683
684    @Override
685    public void shutdown() {
686
687    }
688
689    @Override
690    public long size() {
691      return 0;
692    }
693
694    @Override
695    public long getMaxSize() {
696      return 0;
697    }
698
699    @Override
700    public long getFreeSize() {
701      return 0;
702    }
703
704    @Override
705    public long getCurrentSize() {
706      return this.testBlockSize;
707    }
708
709    @Override
710    public long getCurrentDataSize() {
711      return 0;
712    }
713
714    @Override
715    public long getBlockCount() {
716      return 0;
717    }
718
719    @Override
720    public long getDataBlockCount() {
721      return 0;
722    }
723
724    @Override
725    public void setMaxSize(long size) {
726      this.maxSize = size;
727    }
728
729    @Override
730    public Iterator<CachedBlock> iterator() {
731      return null;
732    }
733
734    @Override
735    public BlockCache[] getBlockCaches() {
736      return null;
737    }
738
739    public void setTestBlockSize(long testBlockSize) {
740      this.testBlockSize = testBlockSize;
741    }
742  }
743
744  private static class MemstoreFlusherStub implements FlushRequester {
745
746    long memstoreSize;
747
748    FlushRequestListener listener;
749
750    FlushType flushType = FlushType.NORMAL;
751
752    public MemstoreFlusherStub(long memstoreSize) {
753      this.memstoreSize = memstoreSize;
754    }
755
756    @Override
757    public boolean requestFlush(HRegion region, FlushLifeCycleTracker tracker) {
758      this.listener.flushRequested(flushType, region);
759      return true;
760    }
761
762    @Override
763    public boolean requestFlush(HRegion region, List<byte[]> families,
764        FlushLifeCycleTracker tracker) {
765      return true;
766    }
767
768    @Override
769    public boolean requestDelayedFlush(HRegion region, long delay) {
770      return true;
771    }
772
773    @Override
774    public void registerFlushRequestListener(FlushRequestListener listener) {
775      this.listener = listener;
776    }
777
778    @Override
779    public boolean unregisterFlushRequestListener(FlushRequestListener listener) {
780      return false;
781    }
782
783    @Override
784    public void setGlobalMemStoreLimit(long globalMemStoreSize) {
785      this.memstoreSize = globalMemStoreSize;
786    }
787  }
788
789  private static class RegionServerStub implements Server {
790    private Configuration conf;
791    private boolean stopped = false;
792
793    public RegionServerStub(Configuration conf) {
794      this.conf = conf;
795    }
796
797    @Override
798    public void abort(String why, Throwable e) {
799
800    }
801
802    @Override
803    public boolean isAborted() {
804      return false;
805    }
806
807    @Override
808    public void stop(String why) {
809      this.stopped = true;
810    }
811
812    @Override
813    public boolean isStopped() {
814      return this.stopped;
815    }
816
817    @Override
818    public Configuration getConfiguration() {
819      return this.conf;
820    }
821
822    @Override
823    public ZKWatcher getZooKeeper() {
824      return null;
825    }
826
827    @Override
828    public CoordinatedStateManager getCoordinatedStateManager() {
829      return null;
830    }
831
832    @Override
833    public Connection getConnection() {
834      return null;
835    }
836
837    @Override
838    public ServerName getServerName() {
839      return ServerName.valueOf("server1",4000,12345);
840    }
841
842    @Override
843    public ChoreService getChoreService() {
844      return null;
845    }
846
847    @Override
848    public FileSystem getFileSystem() {
849      return null;
850    }
851
852    @Override
853    public boolean isStopping() {
854      return false;
855    }
856
857    @Override
858    public Connection createConnection(Configuration conf) throws IOException {
859      return null;
860    }
861
862    @Override
863    public AsyncClusterConnection getAsyncClusterConnection() {
864      return null;
865    }
866  }
867
868  static class CustomHeapMemoryTuner implements HeapMemoryTuner {
869    static float blockCacheSize = 0.4f;
870    static float memstoreSize = 0.4f;
871
872    @Override
873    public Configuration getConf() {
874      return null;
875    }
876
877    @Override
878    public void setConf(Configuration arg0) {
879
880    }
881
882    @Override
883    public TunerResult tune(TunerContext context) {
884      TunerResult result = new TunerResult(true);
885      result.setBlockCacheSize(blockCacheSize);
886      result.setMemStoreSize(memstoreSize);
887      return result;
888    }
889  }
890
891  private static class RegionServerAccountingStub extends RegionServerAccounting {
892    boolean offheap;
893
894    public RegionServerAccountingStub(Configuration conf) {
895      super(conf);
896    }
897
898    public RegionServerAccountingStub(Configuration conf, boolean offheap) {
899      super(conf);
900      this.offheap = offheap;
901    }
902
903    private long testMemstoreSize = 0;
904
905    @Override
906    public long getGlobalMemStoreDataSize() {
907      return testMemstoreSize;
908    }
909
910    @Override
911    public long getGlobalMemStoreHeapSize() {
912      return testMemstoreSize;
913    }
914
915    @Override
916    public boolean isOffheap() {
917      return offheap;
918    }
919
920    public void setTestMemstoreSize(long testMemstoreSize) {
921      this.testMemstoreSize = testMemstoreSize;
922    }
923  }
924}