001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertTrue;
023import static org.junit.Assert.fail;
024
025import java.io.IOException;
026import java.lang.management.ManagementFactory;
027import java.util.Iterator;
028import java.util.List;
029import org.apache.hadoop.conf.Configuration;
030import org.apache.hadoop.fs.FileSystem;
031import org.apache.hadoop.hbase.ChoreService;
032import org.apache.hadoop.hbase.CoordinatedStateManager;
033import org.apache.hadoop.hbase.HBaseClassTestRule;
034import org.apache.hadoop.hbase.HBaseConfiguration;
035import org.apache.hadoop.hbase.HBaseTestingUtility;
036import org.apache.hadoop.hbase.HConstants;
037import org.apache.hadoop.hbase.Server;
038import org.apache.hadoop.hbase.ServerName;
039import org.apache.hadoop.hbase.Waiter;
040import org.apache.hadoop.hbase.client.ClusterConnection;
041import org.apache.hadoop.hbase.client.Connection;
042import org.apache.hadoop.hbase.io.hfile.BlockCache;
043import org.apache.hadoop.hbase.io.hfile.BlockCacheKey;
044import org.apache.hadoop.hbase.io.hfile.CacheStats;
045import org.apache.hadoop.hbase.io.hfile.Cacheable;
046import org.apache.hadoop.hbase.io.hfile.CachedBlock;
047import org.apache.hadoop.hbase.io.hfile.ResizableBlockCache;
048import org.apache.hadoop.hbase.io.util.MemorySizeUtil;
049import org.apache.hadoop.hbase.regionserver.HeapMemoryManager.TunerContext;
050import org.apache.hadoop.hbase.regionserver.HeapMemoryManager.TunerResult;
051import org.apache.hadoop.hbase.testclassification.MediumTests;
052import org.apache.hadoop.hbase.testclassification.RegionServerTests;
053import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
054import org.junit.ClassRule;
055import org.junit.Test;
056import org.junit.experimental.categories.Category;
057
058@Category({ RegionServerTests.class, MediumTests.class })
059public class TestHeapMemoryManager {
060
061  @ClassRule
062  public static final HBaseClassTestRule CLASS_RULE =
063    HBaseClassTestRule.forClass(TestHeapMemoryManager.class);
064
065  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
066
067  private long maxHeapSize = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax();
068
069  @Test
070  public void testAutoTunerShouldBeOffWhenMaxMinRangesForMemstoreIsNotGiven() throws Exception {
071    Configuration conf = HBaseConfiguration.create();
072    conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_KEY, 0.02f);
073    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.75f);
074    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.03f);
075    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf);
076    HeapMemoryManager manager = new HeapMemoryManager(new BlockCacheStub(0),
077      new MemstoreFlusherStub(0), new RegionServerStub(conf), regionServerAccounting);
078    assertFalse(manager.isTunerOn());
079  }
080
081  @Test
082  public void testAutoTunerShouldBeOffWhenMaxMinRangesForBlockCacheIsNotGiven() throws Exception {
083    Configuration conf = HBaseConfiguration.create();
084    conf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.02f);
085    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
086    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.03f);
087    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf);
088    HeapMemoryManager manager = new HeapMemoryManager(new BlockCacheStub(0),
089      new MemstoreFlusherStub(0), new RegionServerStub(conf), regionServerAccounting);
090    assertFalse(manager.isTunerOn());
091  }
092
093  @Test
094  public void testWhenMemstoreAndBlockCacheMaxMinChecksFails() throws Exception {
095    BlockCacheStub blockCache = new BlockCacheStub(0);
096    Configuration conf = HBaseConfiguration.create();
097    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
098    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.06f);
099    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf);
100    MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub(0);
101    try {
102      new HeapMemoryManager(blockCache, memStoreFlusher, new RegionServerStub(conf),
103        regionServerAccounting);
104      fail();
105    } catch (RuntimeException e) {
106    }
107    conf = HBaseConfiguration.create();
108    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.2f);
109    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
110    try {
111      new HeapMemoryManager(blockCache, memStoreFlusher, new RegionServerStub(conf),
112        regionServerAccounting);
113      fail();
114    } catch (RuntimeException e) {
115    }
116  }
117
118  @Test
119  public void testWhenClusterIsWriteHeavyWithEmptyMemstore() throws Exception {
120    Configuration conf = HBaseConfiguration.create();
121    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
122    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f);
123    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
124    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f);
125    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
126    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
127    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf);
128    MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
129    // Empty block cache and memstore
130    blockCache.setTestBlockSize(0);
131    regionServerAccounting.setTestMemstoreSize(0);
132    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
133    // Let the system start with default values for memstore heap and block cache size.
134    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
135      new RegionServerStub(conf), regionServerAccounting);
136    long oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
137    long oldBlockCacheSize = blockCache.maxSize;
138    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
139    heapMemoryManager.start(choreService);
140    memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_HIGHER_MARK;
141    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
142    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
143    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
144    memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK;
145    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
146    // Allow the tuner to run once and do necessary memory up
147    Thread.sleep(1500);
148    // No changes should be made by tuner as we already have lot of empty space
149    assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
150    assertEquals(oldBlockCacheSize, blockCache.maxSize);
151  }
152
153  @Test
154  public void testHeapMemoryManagerWhenOffheapFlushesHappenUnderReadHeavyCase() throws Exception {
155    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
156    Configuration conf = HBaseConfiguration.create();
157    conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_LOWER_LIMIT_KEY, 0.7f);
158    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
159    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f);
160    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
161    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f);
162    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
163    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
164    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf, true);
165    MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
166    // Empty memstore and but nearly filled block cache
167    blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8));
168    regionServerAccounting.setTestMemstoreSize(0);
169    // Let the system start with default values for memstore heap and block cache size.
170    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
171      new RegionServerStub(conf), regionServerAccounting);
172    long oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
173    long oldBlockCacheSize = blockCache.maxSize;
174    float maxStepValue = DefaultHeapMemoryTuner.DEFAULT_MIN_STEP_VALUE;
175    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
176    heapMemoryManager.start(choreService);
177    blockCache.evictBlock(null);
178    blockCache.evictBlock(null);
179    blockCache.evictBlock(null);
180    // do some offheap flushes also. So there should be decrease in memstore but
181    // not as that when we don't have offheap flushes
182    memStoreFlusher.flushType = FlushType.ABOVE_OFFHEAP_HIGHER_MARK;
183    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
184    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
185    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
186    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
187    // Allow the tuner to run once and do necessary memory up
188    waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
189    assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
190    assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize);
191    oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
192    oldBlockCacheSize = blockCache.maxSize;
193    // Do some more evictions before the next run of HeapMemoryTuner
194    blockCache.evictBlock(null);
195    // Allow the tuner to run once and do necessary memory up
196    waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
197    assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
198    assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize);
199  }
200
201  @Test
202  public void testHeapMemoryManagerWithOffheapMemstoreAndMixedWorkload() throws Exception {
203    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
204    Configuration conf = HBaseConfiguration.create();
205    conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_LOWER_LIMIT_KEY, 0.7f);
206    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
207    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f);
208    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
209    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f);
210    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
211    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
212    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf, true);
213    MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
214    // Empty memstore and but nearly filled block cache
215    blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8));
216    regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8));
217    // Let the system start with default values for memstore heap and block cache size.
218    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
219      new RegionServerStub(conf), regionServerAccounting);
220    long oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
221    long oldBlockCacheSize = blockCache.maxSize;
222    float maxStepValue = DefaultHeapMemoryTuner.DEFAULT_MIN_STEP_VALUE;
223    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
224    heapMemoryManager.start(choreService);
225    blockCache.evictBlock(null);
226    blockCache.evictBlock(null);
227    blockCache.evictBlock(null);
228    // do some offheap flushes also. So there should be decrease in memstore but
229    // not as that when we don't have offheap flushes
230    memStoreFlusher.flushType = FlushType.ABOVE_OFFHEAP_HIGHER_MARK;
231    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
232    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
233    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
234    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
235    // Allow the tuner to run once and do necessary memory up
236    waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
237    assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
238    assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize);
239    oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
240    oldBlockCacheSize = blockCache.maxSize;
241    // change memstore size
242    // regionServerAccounting.setTestMemstoreSize((long)(maxHeapSize * 0.4 * 0.8));
243    // The memstore size would have decreased. Now again do some flushes and ensure the
244    // flushes are due to onheap overhead. This should once again call for increase in
245    // memstore size but that increase should be to the safe size
246    memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_HIGHER_MARK;
247    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
248    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
249    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
250    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
251    // Allow the tuner to run once and do necessary memory up
252    waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
253    assertHeapSpaceDelta(maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
254    assertHeapSpaceDelta(-maxStepValue, oldBlockCacheSize, blockCache.maxSize);
255  }
256
257  @Test
258  public void testWhenClusterIsReadHeavyWithEmptyBlockCache() throws Exception {
259    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
260    Configuration conf = HBaseConfiguration.create();
261    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
262    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f);
263    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
264    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f);
265    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
266    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
267    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf);
268    MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
269    // Empty block cache and memstore
270    blockCache.setTestBlockSize(0);
271    regionServerAccounting.setTestMemstoreSize(0);
272    // Let the system start with default values for memstore heap and block cache size.
273    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
274      new RegionServerStub(conf), regionServerAccounting);
275    long oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
276    long oldBlockCacheSize = blockCache.maxSize;
277    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
278    heapMemoryManager.start(choreService);
279    blockCache.evictBlock(null);
280    blockCache.evictBlock(null);
281    blockCache.evictBlock(null);
282    // Allow the tuner to run once and do necessary memory up
283    Thread.sleep(1500);
284    // No changes should be made by tuner as we already have lot of empty space
285    assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
286    assertEquals(oldBlockCacheSize, blockCache.maxSize);
287  }
288
289  @Test
290  public void testWhenClusterIsWriteHeavy() throws Exception {
291    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
292    Configuration conf = HBaseConfiguration.create();
293    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
294    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f);
295    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
296    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f);
297    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
298    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
299    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf);
300    MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
301    // Empty block cache and but nearly filled memstore
302    blockCache.setTestBlockSize(0);
303    regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8));
304    // Let the system start with default values for memstore heap and block cache size.
305    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
306      new RegionServerStub(conf), regionServerAccounting);
307    long oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
308    long oldBlockCacheSize = blockCache.maxSize;
309    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
310    heapMemoryManager.start(choreService);
311    memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK;
312    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
313    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
314    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
315    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
316    // Allow the tuner to run once and do necessary memory up
317    waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
318    assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldMemstoreHeapSize,
319      memStoreFlusher.memstoreSize);
320    assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldBlockCacheSize,
321      blockCache.maxSize);
322    oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
323    oldBlockCacheSize = blockCache.maxSize;
324    // Do some more flushes before the next run of HeapMemoryTuner
325    memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK;
326    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
327    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
328    // Allow the tuner to run once and do necessary memory up
329    waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
330    assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldMemstoreHeapSize,
331      memStoreFlusher.memstoreSize);
332    assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldBlockCacheSize,
333      blockCache.maxSize);
334  }
335
336  @Test
337  public void testWhenClusterIsWriteHeavyWithOffheapMemstore() throws Exception {
338    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
339    Configuration conf = HBaseConfiguration.create();
340    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
341    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f);
342    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
343    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f);
344    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
345    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
346    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf);
347    MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
348    // Empty block cache and but nearly filled memstore
349    blockCache.setTestBlockSize(0);
350    regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8));
351    // Let the system start with default values for memstore heap and block cache size.
352    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
353      new RegionServerStub(conf), regionServerAccounting);
354    long oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
355    long oldBlockCacheSize = blockCache.maxSize;
356    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
357    heapMemoryManager.start(choreService);
358    // this should not change anything with onheap memstore
359    memStoreFlusher.flushType = FlushType.ABOVE_OFFHEAP_HIGHER_MARK;
360    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
361    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
362    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
363    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
364    // Allow the tuner to run once and do necessary memory up
365    Thread.sleep(1500);
366    // No changes should be made by tuner as we already have lot of empty space
367    assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
368    assertEquals(oldBlockCacheSize, blockCache.maxSize);
369  }
370
371  @Test
372  public void testWhenClusterIsReadHeavy() throws Exception {
373    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
374    Configuration conf = HBaseConfiguration.create();
375    conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_LOWER_LIMIT_KEY, 0.7f);
376    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
377    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f);
378    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
379    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f);
380    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
381    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
382    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf);
383    MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
384    // Empty memstore and but nearly filled block cache
385    blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8));
386    regionServerAccounting.setTestMemstoreSize(0);
387    // Let the system start with default values for memstore heap and block cache size.
388    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
389      new RegionServerStub(conf), new RegionServerAccountingStub(conf));
390    long oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
391    long oldBlockCacheSize = blockCache.maxSize;
392    long oldMemstoreLowerMarkSize = 7 * oldMemstoreHeapSize / 10;
393    long maxTuneSize = oldMemstoreHeapSize - (oldMemstoreLowerMarkSize + oldMemstoreHeapSize) / 2;
394    float maxStepValue = (maxTuneSize * 1.0f) / oldMemstoreHeapSize;
395    maxStepValue = maxStepValue > DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE
396      ? DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE
397      : maxStepValue;
398    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
399    heapMemoryManager.start(choreService);
400    blockCache.evictBlock(null);
401    blockCache.evictBlock(null);
402    blockCache.evictBlock(null);
403    // Allow the tuner to run once and do necessary memory up
404    waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
405    assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
406    assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize);
407    oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
408    oldBlockCacheSize = blockCache.maxSize;
409    oldMemstoreLowerMarkSize = 7 * oldMemstoreHeapSize / 10;
410    maxTuneSize = oldMemstoreHeapSize - (oldMemstoreLowerMarkSize + oldMemstoreHeapSize) / 2;
411    maxStepValue = (maxTuneSize * 1.0f) / oldMemstoreHeapSize;
412    maxStepValue = maxStepValue > DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE
413      ? DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE
414      : maxStepValue;
415    // Do some more evictions before the next run of HeapMemoryTuner
416    blockCache.evictBlock(null);
417    // Allow the tuner to run once and do necessary memory up
418    waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
419    assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
420    assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize);
421  }
422
423  @Test
424  public void testWhenClusterIsHavingMoreWritesThanReads() throws Exception {
425    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
426    Configuration conf = HBaseConfiguration.create();
427    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
428    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f);
429    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
430    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f);
431    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
432    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
433    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf);
434    MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
435    // Both memstore and block cache are nearly filled
436    blockCache.setTestBlockSize(0);
437    regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8));
438    blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8));
439    // Let the system start with default values for memstore heap and block cache size.
440    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
441      new RegionServerStub(conf), regionServerAccounting);
442    long oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
443    long oldBlockCacheSize = blockCache.maxSize;
444    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
445    heapMemoryManager.start(choreService);
446    memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK;
447    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
448    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
449    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
450    blockCache.evictBlock(null);
451    // Allow the tuner to run once and do necessary memory up
452    Thread.sleep(1500);
453    // No changes should happen as there is undefined increase in flushes and evictions
454    assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
455    assertEquals(oldBlockCacheSize, blockCache.maxSize);
456    // Do some more flushes before the next run of HeapMemoryTuner
457    memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK;
458    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
459    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
460    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
461    // Allow the tuner to run once and do necessary memory up
462    waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
463    assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldMemstoreHeapSize,
464      memStoreFlusher.memstoreSize);
465    assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldBlockCacheSize,
466      blockCache.maxSize);
467  }
468
469  @Test
470  public void testBlockedFlushesIncreaseMemstoreInSteadyState() throws Exception {
471    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
472    MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
473    Configuration conf = HBaseConfiguration.create();
474    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
475    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f);
476    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
477    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f);
478    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
479    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
480    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf);
481    // Both memstore and block cache are nearly filled
482    blockCache.setTestBlockSize(0);
483    regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8));
484    blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8));
485    // Let the system start with default values for memstore heap and block cache size.
486    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
487      new RegionServerStub(conf), regionServerAccounting);
488    long oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
489    long oldBlockCacheSize = blockCache.maxSize;
490    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
491    heapMemoryManager.start(choreService);
492    memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK;
493    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
494    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
495    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
496    blockCache.evictBlock(null);
497    blockCache.evictBlock(null);
498    // Allow the tuner to run once and do necessary memory up
499    Thread.sleep(1500);
500    // No changes should happen as there is undefined increase in flushes and evictions
501    assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
502    assertEquals(oldBlockCacheSize, blockCache.maxSize);
503    // Flushes that block updates
504    memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_HIGHER_MARK;
505    memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
506    blockCache.evictBlock(null);
507    blockCache.evictBlock(null);
508    blockCache.evictBlock(null);
509    blockCache.evictBlock(null);
510    // Allow the tuner to run once and do necessary memory up
511    Thread.sleep(1500);
512    assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldMemstoreHeapSize,
513      memStoreFlusher.memstoreSize);
514    assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldBlockCacheSize,
515      blockCache.maxSize);
516  }
517
518  @Test
519  public void testPluggingInHeapMemoryTuner() throws Exception {
520    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
521    MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
522    Configuration conf = HBaseConfiguration.create();
523    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.78f);
524    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.05f);
525    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.75f);
526    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.02f);
527    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
528    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
529    conf.setClass(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_CLASS, CustomHeapMemoryTuner.class,
530      HeapMemoryTuner.class);
531    // Let the system start with default values for memstore heap and block cache size.
532    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
533      new RegionServerStub(conf), new RegionServerAccountingStub(conf));
534    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
535    heapMemoryManager.start(choreService);
536    // Now we wants to be in write mode. Set bigger memstore size from CustomHeapMemoryTuner
537    CustomHeapMemoryTuner.memstoreSize = 0.78f;
538    CustomHeapMemoryTuner.blockCacheSize = 0.02f;
539    // Allow the tuner to run once and do necessary memory up
540    waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
541    assertHeapSpace(0.78f, memStoreFlusher.memstoreSize);// Memstore
542    assertHeapSpace(0.02f, blockCache.maxSize);// BlockCache
543    // Now we wants to be in read mode. Set bigger memstore size from CustomHeapMemoryTuner
544    CustomHeapMemoryTuner.blockCacheSize = 0.75f;
545    CustomHeapMemoryTuner.memstoreSize = 0.05f;
546    // Allow the tuner to run once and do necessary memory up
547    waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
548    assertHeapSpace(0.75f, blockCache.maxSize);// BlockCache
549    assertHeapSpace(0.05f, memStoreFlusher.memstoreSize);// Memstore
550  }
551
552  @Test
553  public void testWhenSizeGivenByHeapTunerGoesOutsideRange() throws Exception {
554    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
555    MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
556    Configuration conf = HBaseConfiguration.create();
557    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.7f);
558    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.1f);
559    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
560    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.1f);
561    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
562    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
563    conf.setClass(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_CLASS, CustomHeapMemoryTuner.class,
564      HeapMemoryTuner.class);
565    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
566      new RegionServerStub(conf), new RegionServerAccountingStub(conf));
567    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
568    heapMemoryManager.start(choreService);
569    CustomHeapMemoryTuner.memstoreSize = 0.78f;
570    CustomHeapMemoryTuner.blockCacheSize = 0.02f;
571    Thread.sleep(1500); // Allow the tuner to run once and do necessary memory up
572    // Even if the tuner says to set the memstore to 78%, HBase makes it as 70% as that is the
573    // upper bound. Same with block cache as 10% is the lower bound.
574    assertHeapSpace(0.7f, memStoreFlusher.memstoreSize);
575    assertHeapSpace(0.1f, blockCache.maxSize);
576  }
577
578  @Test
579  public void testWhenCombinedHeapSizesFromTunerGoesOutSideMaxLimit() throws Exception {
580    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
581    MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
582    Configuration conf = HBaseConfiguration.create();
583    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.7f);
584    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.1f);
585    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
586    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.1f);
587    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
588    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
589    conf.setClass(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_CLASS, CustomHeapMemoryTuner.class,
590      HeapMemoryTuner.class);
591    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
592      new RegionServerStub(conf), new RegionServerAccountingStub(conf));
593    long oldMemstoreSize = memStoreFlusher.memstoreSize;
594    long oldBlockCacheSize = blockCache.maxSize;
595    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
596    heapMemoryManager.start(choreService);
597    CustomHeapMemoryTuner.memstoreSize = 0.7f;
598    CustomHeapMemoryTuner.blockCacheSize = 0.3f;
599    // Allow the tuner to run once and do necessary memory up
600    Thread.sleep(1500);
601    assertEquals(oldMemstoreSize, memStoreFlusher.memstoreSize);
602    assertEquals(oldBlockCacheSize, blockCache.maxSize);
603  }
604
605  private void assertHeapSpace(float expectedHeapPercentage, long currentHeapSpace) {
606    long expected = (long) (this.maxHeapSize * expectedHeapPercentage);
607    assertEquals(expected, currentHeapSpace);
608  }
609
610  private void assertHeapSpaceDelta(double expectedDeltaPercent, long oldHeapSpace,
611    long newHeapSpace) {
612    double expctedMinDelta = (double) (this.maxHeapSize * expectedDeltaPercent);
613    // Tolerable error
614    double error = 0.95;
615    if (expectedDeltaPercent > 0) {
616      assertTrue(expctedMinDelta * error <= (double) (newHeapSpace - oldHeapSpace));
617      assertTrue(expctedMinDelta / error >= (double) (newHeapSpace - oldHeapSpace));
618    } else {
619      assertTrue(-expctedMinDelta * error <= (double) (oldHeapSpace - newHeapSpace));
620      assertTrue(-expctedMinDelta / error >= (double) (oldHeapSpace - newHeapSpace));
621    }
622  }
623
624  private void waitForTune(final MemstoreFlusherStub memStoreFlusher,
625    final long oldMemstoreHeapSize) throws Exception {
626    // Allow the tuner to run once and do necessary memory up
627    UTIL.waitFor(10000, new Waiter.Predicate<Exception>() {
628      @Override
629      public boolean evaluate() throws Exception {
630        return oldMemstoreHeapSize != memStoreFlusher.memstoreSize;
631      }
632    });
633  }
634
635  private static class BlockCacheStub implements ResizableBlockCache {
636    CacheStats stats = new CacheStats("test");
637    long maxSize = 0;
638    private long testBlockSize = 0;
639
640    public BlockCacheStub(long size) {
641      this.maxSize = size;
642    }
643
644    @Override
645    public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory) {
646
647    }
648
649    @Override
650    public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf) {
651
652    }
653
654    @Override
655    public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching, boolean repeat,
656      boolean updateCacheMetrics) {
657      return null;
658    }
659
660    @Override
661    public boolean evictBlock(BlockCacheKey cacheKey) {
662      stats.evicted(0, cacheKey != null ? cacheKey.isPrimary() : true);
663      return false;
664    }
665
666    @Override
667    public int evictBlocksByHfileName(String hfileName) {
668      stats.evicted(0, true); // Just assuming only one block for file here.
669      return 0;
670    }
671
672    @Override
673    public CacheStats getStats() {
674      return this.stats;
675    }
676
677    @Override
678    public void shutdown() {
679
680    }
681
682    @Override
683    public long size() {
684      return 0;
685    }
686
687    @Override
688    public long getMaxSize() {
689      return 0;
690    }
691
692    @Override
693    public long getFreeSize() {
694      return 0;
695    }
696
697    @Override
698    public long getCurrentSize() {
699      return this.testBlockSize;
700    }
701
702    @Override
703    public long getCurrentDataSize() {
704      return 0;
705    }
706
707    @Override
708    public long getBlockCount() {
709      return 0;
710    }
711
712    @Override
713    public long getDataBlockCount() {
714      return 0;
715    }
716
717    @Override
718    public void setMaxSize(long size) {
719      this.maxSize = size;
720    }
721
722    @Override
723    public Iterator<CachedBlock> iterator() {
724      return null;
725    }
726
727    @Override
728    public BlockCache[] getBlockCaches() {
729      return null;
730    }
731
732    public void setTestBlockSize(long testBlockSize) {
733      this.testBlockSize = testBlockSize;
734    }
735  }
736
737  private static class MemstoreFlusherStub implements FlushRequester {
738
739    long memstoreSize;
740
741    FlushRequestListener listener;
742
743    FlushType flushType = FlushType.NORMAL;
744
745    public MemstoreFlusherStub(long memstoreSize) {
746      this.memstoreSize = memstoreSize;
747    }
748
749    @Override
750    public boolean requestFlush(HRegion region, FlushLifeCycleTracker tracker) {
751      this.listener.flushRequested(flushType, region);
752      return true;
753    }
754
755    @Override
756    public boolean requestFlush(HRegion region, List<byte[]> families,
757      FlushLifeCycleTracker tracker) {
758      return true;
759    }
760
761    @Override
762    public boolean requestDelayedFlush(HRegion region, long delay) {
763      return true;
764    }
765
766    @Override
767    public void registerFlushRequestListener(FlushRequestListener listener) {
768      this.listener = listener;
769    }
770
771    @Override
772    public boolean unregisterFlushRequestListener(FlushRequestListener listener) {
773      return false;
774    }
775
776    @Override
777    public void setGlobalMemStoreLimit(long globalMemStoreSize) {
778      this.memstoreSize = globalMemStoreSize;
779    }
780  }
781
782  private static class RegionServerStub implements Server {
783    private Configuration conf;
784    private boolean stopped = false;
785
786    public RegionServerStub(Configuration conf) {
787      this.conf = conf;
788    }
789
790    @Override
791    public void abort(String why, Throwable e) {
792
793    }
794
795    @Override
796    public boolean isAborted() {
797      return false;
798    }
799
800    @Override
801    public void stop(String why) {
802      this.stopped = true;
803    }
804
805    @Override
806    public boolean isStopped() {
807      return this.stopped;
808    }
809
810    @Override
811    public Configuration getConfiguration() {
812      return this.conf;
813    }
814
815    @Override
816    public ZKWatcher getZooKeeper() {
817      return null;
818    }
819
820    @Override
821    public CoordinatedStateManager getCoordinatedStateManager() {
822      return null;
823    }
824
825    @Override
826    public ClusterConnection getConnection() {
827      return null;
828    }
829
830    @Override
831    public ServerName getServerName() {
832      return ServerName.valueOf("server1", 4000, 12345);
833    }
834
835    @Override
836    public ChoreService getChoreService() {
837      return null;
838    }
839
840    @Override
841    public ClusterConnection getClusterConnection() {
842      // TODO Auto-generated method stub
843      return null;
844    }
845
846    @Override
847    public FileSystem getFileSystem() {
848      return null;
849    }
850
851    @Override
852    public boolean isStopping() {
853      return false;
854    }
855
856    @Override
857    public Connection createConnection(Configuration conf) throws IOException {
858      return null;
859    }
860  }
861
862  static class CustomHeapMemoryTuner implements HeapMemoryTuner {
863    static float blockCacheSize = 0.4f;
864    static float memstoreSize = 0.4f;
865
866    @Override
867    public Configuration getConf() {
868      return null;
869    }
870
871    @Override
872    public void setConf(Configuration arg0) {
873
874    }
875
876    @Override
877    public TunerResult tune(TunerContext context) {
878      TunerResult result = new TunerResult(true);
879      result.setBlockCacheSize(blockCacheSize);
880      result.setMemStoreSize(memstoreSize);
881      return result;
882    }
883  }
884
885  private static class RegionServerAccountingStub extends RegionServerAccounting {
886    boolean offheap;
887
888    public RegionServerAccountingStub(Configuration conf) {
889      super(conf);
890    }
891
892    public RegionServerAccountingStub(Configuration conf, boolean offheap) {
893      super(conf);
894      this.offheap = offheap;
895    }
896
897    private long testMemstoreSize = 0;
898
899    @Override
900    public long getGlobalMemStoreDataSize() {
901      return testMemstoreSize;
902    }
903
904    @Override
905    public long getGlobalMemStoreHeapSize() {
906      return testMemstoreSize;
907    }
908
909    @Override
910    public boolean isOffheap() {
911      return offheap;
912    }
913
914    public void setTestMemstoreSize(long testMemstoreSize) {
915      this.testMemstoreSize = testMemstoreSize;
916    }
917  }
918}