001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertTrue;
023import static org.junit.Assert.fail;
024
025import java.io.IOException;
026import java.lang.management.ManagementFactory;
027import java.util.Iterator;
028import org.apache.hadoop.conf.Configuration;
029import org.apache.hadoop.fs.FileSystem;
030import org.apache.hadoop.hbase.ChoreService;
031import org.apache.hadoop.hbase.CoordinatedStateManager;
032import org.apache.hadoop.hbase.HBaseClassTestRule;
033import org.apache.hadoop.hbase.HBaseConfiguration;
034import org.apache.hadoop.hbase.HBaseTestingUtility;
035import org.apache.hadoop.hbase.HConstants;
036import org.apache.hadoop.hbase.Server;
037import org.apache.hadoop.hbase.ServerName;
038import org.apache.hadoop.hbase.Waiter;
039import org.apache.hadoop.hbase.client.ClusterConnection;
040import org.apache.hadoop.hbase.client.Connection;
041import org.apache.hadoop.hbase.io.hfile.BlockCache;
042import org.apache.hadoop.hbase.io.hfile.BlockCacheKey;
043import org.apache.hadoop.hbase.io.hfile.CacheStats;
044import org.apache.hadoop.hbase.io.hfile.Cacheable;
045import org.apache.hadoop.hbase.io.hfile.CachedBlock;
046import org.apache.hadoop.hbase.io.hfile.ResizableBlockCache;
047import org.apache.hadoop.hbase.io.util.MemorySizeUtil;
048import org.apache.hadoop.hbase.regionserver.HeapMemoryManager.TunerContext;
049import org.apache.hadoop.hbase.regionserver.HeapMemoryManager.TunerResult;
050import org.apache.hadoop.hbase.testclassification.RegionServerTests;
051import org.apache.hadoop.hbase.testclassification.SmallTests;
052import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
053import org.junit.ClassRule;
054import org.junit.Test;
055import org.junit.experimental.categories.Category;
056
057@Category({RegionServerTests.class, SmallTests.class})
058public class TestHeapMemoryManager {
059
060  @ClassRule
061  public static final HBaseClassTestRule CLASS_RULE =
062      HBaseClassTestRule.forClass(TestHeapMemoryManager.class);
063
064  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
065
066  private long maxHeapSize = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax();
067
068  @Test
069  public void testAutoTunerShouldBeOffWhenMaxMinRangesForMemstoreIsNotGiven() throws Exception {
070    Configuration conf = HBaseConfiguration.create();
071    conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_KEY, 0.02f);
072    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.75f);
073    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.03f);
074    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf);
075    HeapMemoryManager manager = new HeapMemoryManager(new BlockCacheStub(0),
076        new MemstoreFlusherStub(0), new RegionServerStub(conf),
077        regionServerAccounting);
078    assertFalse(manager.isTunerOn());
079  }
080
081  @Test
082  public void testAutoTunerShouldBeOffWhenMaxMinRangesForBlockCacheIsNotGiven() throws Exception {
083    Configuration conf = HBaseConfiguration.create();
084    conf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.02f);
085    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
086    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.03f);
087    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf);
088    HeapMemoryManager manager = new HeapMemoryManager(new BlockCacheStub(0),
089        new MemstoreFlusherStub(0), new RegionServerStub(conf),
090        regionServerAccounting);
091    assertFalse(manager.isTunerOn());
092  }
093
094  @Test
095  public void testWhenMemstoreAndBlockCacheMaxMinChecksFails() throws Exception {
096    BlockCacheStub blockCache = new BlockCacheStub(0);
097    Configuration conf = HBaseConfiguration.create();
098    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
099    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.06f);
100    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf);
101    MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub(0);
102    try {
103      new HeapMemoryManager(blockCache, memStoreFlusher,
104          new RegionServerStub(conf), regionServerAccounting);
105      fail();
106    } catch (RuntimeException e) {
107    }
108    conf = HBaseConfiguration.create();
109    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.2f);
110    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
111    try {
112      new HeapMemoryManager(blockCache, memStoreFlusher,
113          new RegionServerStub(conf), regionServerAccounting);
114      fail();
115    } catch (RuntimeException e) {
116    }
117  }
118
119  @Test
120  public void testWhenClusterIsWriteHeavyWithEmptyMemstore() throws Exception {
121    Configuration conf = HBaseConfiguration.create();
122    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
123    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f);
124    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
125    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f);
126    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
127    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
128    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf);
129    MemstoreFlusherStub memStoreFlusher =
130        new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
131    // Empty block cache and memstore
132    blockCache.setTestBlockSize(0);
133    regionServerAccounting.setTestMemstoreSize(0);
134    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
135    // Let the system start with default values for memstore heap and block cache size.
136    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
137        new RegionServerStub(conf), regionServerAccounting);
138    long oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
139    long oldBlockCacheSize = blockCache.maxSize;
140    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
141    heapMemoryManager.start(choreService);
142    memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_HIGHER_MARK;
143    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
144    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
145    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
146    memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK;
147    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
148    // Allow the tuner to run once and do necessary memory up
149    Thread.sleep(1500);
150    // No changes should be made by tuner as we already have lot of empty space
151    assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
152    assertEquals(oldBlockCacheSize, blockCache.maxSize);
153  }
154
155  @Test
156  public void testHeapMemoryManagerWhenOffheapFlushesHappenUnderReadHeavyCase() throws Exception {
157    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
158    Configuration conf = HBaseConfiguration.create();
159    conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_LOWER_LIMIT_KEY, 0.7f);
160    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
161    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f);
162    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
163    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f);
164    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
165    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
166    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf, true);
167    MemstoreFlusherStub memStoreFlusher =
168        new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
169    // Empty memstore and but nearly filled block cache
170    blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8));
171    regionServerAccounting.setTestMemstoreSize(0);
172    // Let the system start with default values for memstore heap and block cache size.
173    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
174        new RegionServerStub(conf), regionServerAccounting);
175    long oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
176    long oldBlockCacheSize = blockCache.maxSize;
177    float maxStepValue = DefaultHeapMemoryTuner.DEFAULT_MIN_STEP_VALUE;
178    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
179    heapMemoryManager.start(choreService);
180    blockCache.evictBlock(null);
181    blockCache.evictBlock(null);
182    blockCache.evictBlock(null);
183    // do some offheap flushes also. So there should be decrease in memstore but
184    // not as that when we don't have offheap flushes
185    memStoreFlusher.flushType = FlushType.ABOVE_OFFHEAP_HIGHER_MARK;
186    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
187    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
188    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
189    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
190    // Allow the tuner to run once and do necessary memory up
191    waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
192    assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
193    assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize);
194    oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
195    oldBlockCacheSize = blockCache.maxSize;
196    // Do some more evictions before the next run of HeapMemoryTuner
197    blockCache.evictBlock(null);
198    // Allow the tuner to run once and do necessary memory up
199    waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
200    assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
201    assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize);
202  }
203
204  @Test
205  public void testHeapMemoryManagerWithOffheapMemstoreAndMixedWorkload() throws Exception {
206    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
207    Configuration conf = HBaseConfiguration.create();
208    conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_LOWER_LIMIT_KEY, 0.7f);
209    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
210    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f);
211    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
212    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f);
213    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
214    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
215    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf, true);
216    MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
217    // Empty memstore and but nearly filled block cache
218    blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8));
219    regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8));
220    // Let the system start with default values for memstore heap and block cache size.
221    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
222        new RegionServerStub(conf), regionServerAccounting);
223    long oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
224    long oldBlockCacheSize = blockCache.maxSize;
225    float maxStepValue = DefaultHeapMemoryTuner.DEFAULT_MIN_STEP_VALUE;
226    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
227    heapMemoryManager.start(choreService);
228    blockCache.evictBlock(null);
229    blockCache.evictBlock(null);
230    blockCache.evictBlock(null);
231    // do some offheap flushes also. So there should be decrease in memstore but
232    // not as that when we don't have offheap flushes
233    memStoreFlusher.flushType = FlushType.ABOVE_OFFHEAP_HIGHER_MARK;
234    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
235    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
236    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
237    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
238    // Allow the tuner to run once and do necessary memory up
239    waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
240    assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
241    assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize);
242    oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
243    oldBlockCacheSize = blockCache.maxSize;
244    // change memstore size
245    // regionServerAccounting.setTestMemstoreSize((long)(maxHeapSize * 0.4 * 0.8));
246    // The memstore size would have decreased. Now again do some flushes and ensure the
247    // flushes are due to onheap overhead. This should once again call for increase in
248    // memstore size but that increase should be to the safe size
249    memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_HIGHER_MARK;
250    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
251    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
252    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
253    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
254    // Allow the tuner to run once and do necessary memory up
255    waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
256    assertHeapSpaceDelta(maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
257    assertHeapSpaceDelta(-maxStepValue, oldBlockCacheSize, blockCache.maxSize);
258  }
259
260  @Test
261  public void testWhenClusterIsReadHeavyWithEmptyBlockCache() throws Exception {
262    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
263    Configuration conf = HBaseConfiguration.create();
264    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
265    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f);
266    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
267    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f);
268    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
269    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
270    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf);
271    MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
272    // Empty block cache and memstore
273    blockCache.setTestBlockSize(0);
274    regionServerAccounting.setTestMemstoreSize(0);
275    // Let the system start with default values for memstore heap and block cache size.
276    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
277        new RegionServerStub(conf), regionServerAccounting);
278    long oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
279    long oldBlockCacheSize = blockCache.maxSize;
280    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
281    heapMemoryManager.start(choreService);
282    blockCache.evictBlock(null);
283    blockCache.evictBlock(null);
284    blockCache.evictBlock(null);
285    // Allow the tuner to run once and do necessary memory up
286    Thread.sleep(1500);
287    // No changes should be made by tuner as we already have lot of empty space
288    assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
289    assertEquals(oldBlockCacheSize, blockCache.maxSize);
290  }
291
292  @Test
293  public void testWhenClusterIsWriteHeavy() throws Exception {
294    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
295    Configuration conf = HBaseConfiguration.create();
296    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
297    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f);
298    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
299    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f);
300    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
301    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
302    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf);
303    MemstoreFlusherStub memStoreFlusher =
304        new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
305    // Empty block cache and but nearly filled memstore
306    blockCache.setTestBlockSize(0);
307    regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8));
308    // Let the system start with default values for memstore heap and block cache size.
309    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
310        new RegionServerStub(conf), regionServerAccounting);
311    long oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
312    long oldBlockCacheSize = blockCache.maxSize;
313    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
314    heapMemoryManager.start(choreService);
315    memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK;
316    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
317    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
318    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
319    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
320    // Allow the tuner to run once and do necessary memory up
321    waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
322    assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldMemstoreHeapSize,
323        memStoreFlusher.memstoreSize);
324    assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldBlockCacheSize,
325        blockCache.maxSize);
326    oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
327    oldBlockCacheSize = blockCache.maxSize;
328    // Do some more flushes before the next run of HeapMemoryTuner
329    memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK;
330    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
331    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
332    // Allow the tuner to run once and do necessary memory up
333    waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
334    assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldMemstoreHeapSize,
335        memStoreFlusher.memstoreSize);
336    assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldBlockCacheSize,
337        blockCache.maxSize);
338  }
339
340  @Test
341  public void testWhenClusterIsWriteHeavyWithOffheapMemstore() throws Exception {
342    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
343    Configuration conf = HBaseConfiguration.create();
344    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
345    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f);
346    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
347    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f);
348    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
349    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
350    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf);
351    MemstoreFlusherStub memStoreFlusher =
352        new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
353    // Empty block cache and but nearly filled memstore
354    blockCache.setTestBlockSize(0);
355    regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8));
356    // Let the system start with default values for memstore heap and block cache size.
357    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
358        new RegionServerStub(conf), regionServerAccounting);
359    long oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
360    long oldBlockCacheSize = blockCache.maxSize;
361    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
362    heapMemoryManager.start(choreService);
363    // this should not change anything with onheap memstore
364    memStoreFlusher.flushType = FlushType.ABOVE_OFFHEAP_HIGHER_MARK;
365    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
366    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
367    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
368    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
369    // Allow the tuner to run once and do necessary memory up
370    Thread.sleep(1500);
371    // No changes should be made by tuner as we already have lot of empty space
372    assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
373    assertEquals(oldBlockCacheSize, blockCache.maxSize);
374  }
375
376  @Test
377  public void testWhenClusterIsReadHeavy() throws Exception {
378    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
379    Configuration conf = HBaseConfiguration.create();
380    conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_LOWER_LIMIT_KEY, 0.7f);
381    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
382    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f);
383    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
384    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f);
385    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
386    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
387    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf);
388    MemstoreFlusherStub memStoreFlusher =
389        new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
390    // Empty memstore and but nearly filled block cache
391    blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8));
392    regionServerAccounting.setTestMemstoreSize(0);
393    // Let the system start with default values for memstore heap and block cache size.
394    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
395        new RegionServerStub(conf), new RegionServerAccountingStub(conf));
396    long oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
397    long oldBlockCacheSize = blockCache.maxSize;
398    long oldMemstoreLowerMarkSize = 7 * oldMemstoreHeapSize / 10;
399    long maxTuneSize = oldMemstoreHeapSize -  (oldMemstoreLowerMarkSize + oldMemstoreHeapSize) / 2;
400    float maxStepValue = (maxTuneSize * 1.0f) / oldMemstoreHeapSize;
401    maxStepValue = maxStepValue > DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE ?
402        DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE:maxStepValue;
403    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
404    heapMemoryManager.start(choreService);
405    blockCache.evictBlock(null);
406    blockCache.evictBlock(null);
407    blockCache.evictBlock(null);
408    // Allow the tuner to run once and do necessary memory up
409    waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
410    assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
411    assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize);
412    oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
413    oldBlockCacheSize = blockCache.maxSize;
414    oldMemstoreLowerMarkSize = 7 * oldMemstoreHeapSize / 10;
415    maxTuneSize = oldMemstoreHeapSize -  (oldMemstoreLowerMarkSize + oldMemstoreHeapSize) / 2;
416    maxStepValue = (maxTuneSize * 1.0f) / oldMemstoreHeapSize;
417    maxStepValue = maxStepValue > DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE ?
418        DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE:maxStepValue;
419    // Do some more evictions before the next run of HeapMemoryTuner
420    blockCache.evictBlock(null);
421    // Allow the tuner to run once and do necessary memory up
422    waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
423    assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
424    assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize);
425  }
426
427  @Test
428  public void testWhenClusterIsHavingMoreWritesThanReads() throws Exception {
429    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
430    Configuration conf = HBaseConfiguration.create();
431    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
432    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f);
433    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
434    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f);
435    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
436    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
437    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf);
438    MemstoreFlusherStub memStoreFlusher =
439        new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
440    // Both memstore and block cache are nearly filled
441    blockCache.setTestBlockSize(0);
442    regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8));
443    blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8));
444    // Let the system start with default values for memstore heap and block cache size.
445    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
446        new RegionServerStub(conf), regionServerAccounting);
447    long oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
448    long oldBlockCacheSize = blockCache.maxSize;
449    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
450    heapMemoryManager.start(choreService);
451    memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK;
452    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
453    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
454    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
455    blockCache.evictBlock(null);
456    // Allow the tuner to run once and do necessary memory up
457    Thread.sleep(1500);
458    // No changes should happen as there is undefined increase in flushes and evictions
459    assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
460    assertEquals(oldBlockCacheSize, blockCache.maxSize);
461    // Do some more flushes before the next run of HeapMemoryTuner
462    memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK;
463    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
464    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
465    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
466    // Allow the tuner to run once and do necessary memory up
467    waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
468    assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldMemstoreHeapSize,
469        memStoreFlusher.memstoreSize);
470    assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldBlockCacheSize,
471        blockCache.maxSize);
472  }
473
474  @Test
475  public void testBlockedFlushesIncreaseMemstoreInSteadyState() throws Exception {
476    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
477    MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
478    Configuration conf = HBaseConfiguration.create();
479    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
480    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f);
481    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
482    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f);
483    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
484    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
485    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf);
486    // Both memstore and block cache are nearly filled
487    blockCache.setTestBlockSize(0);
488    regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8));
489    blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8));
490    // Let the system start with default values for memstore heap and block cache size.
491    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
492        new RegionServerStub(conf), regionServerAccounting);
493    long oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
494    long oldBlockCacheSize = blockCache.maxSize;
495    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
496    heapMemoryManager.start(choreService);
497    memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK;
498    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
499    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
500    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
501    blockCache.evictBlock(null);
502    blockCache.evictBlock(null);
503    // Allow the tuner to run once and do necessary memory up
504    Thread.sleep(1500);
505    // No changes should happen as there is undefined increase in flushes and evictions
506    assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
507    assertEquals(oldBlockCacheSize, blockCache.maxSize);
508    // Flushes that block updates
509    memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_HIGHER_MARK;
510    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
511    blockCache.evictBlock(null);
512    blockCache.evictBlock(null);
513    blockCache.evictBlock(null);
514    blockCache.evictBlock(null);
515    // Allow the tuner to run once and do necessary memory up
516    Thread.sleep(1500);
517    assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldMemstoreHeapSize,
518        memStoreFlusher.memstoreSize);
519    assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldBlockCacheSize,
520        blockCache.maxSize);
521  }
522
523  @Test
524  public void testPluggingInHeapMemoryTuner() throws Exception {
525    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
526    MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
527    Configuration conf = HBaseConfiguration.create();
528    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.78f);
529    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.05f);
530    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.75f);
531    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.02f);
532    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
533    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
534    conf.setClass(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_CLASS, CustomHeapMemoryTuner.class,
535        HeapMemoryTuner.class);
536    // Let the system start with default values for memstore heap and block cache size.
537    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
538        new RegionServerStub(conf), new RegionServerAccountingStub(conf));
539    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
540    heapMemoryManager.start(choreService);
541    // Now we wants to be in write mode. Set bigger memstore size from CustomHeapMemoryTuner
542    CustomHeapMemoryTuner.memstoreSize = 0.78f;
543    CustomHeapMemoryTuner.blockCacheSize = 0.02f;
544    // Allow the tuner to run once and do necessary memory up
545    waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
546    assertHeapSpace(0.78f, memStoreFlusher.memstoreSize);// Memstore
547    assertHeapSpace(0.02f, blockCache.maxSize);// BlockCache
548    // Now we wants to be in read mode. Set bigger memstore size from CustomHeapMemoryTuner
549    CustomHeapMemoryTuner.blockCacheSize = 0.75f;
550    CustomHeapMemoryTuner.memstoreSize = 0.05f;
551    // Allow the tuner to run once and do necessary memory up
552    waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
553    assertHeapSpace(0.75f, blockCache.maxSize);// BlockCache
554    assertHeapSpace(0.05f, memStoreFlusher.memstoreSize);// Memstore
555  }
556
557  @Test
558  public void testWhenSizeGivenByHeapTunerGoesOutsideRange() throws Exception {
559    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
560    MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
561    Configuration conf = HBaseConfiguration.create();
562    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.7f);
563    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.1f);
564    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
565    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.1f);
566    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
567    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
568    conf.setClass(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_CLASS, CustomHeapMemoryTuner.class,
569        HeapMemoryTuner.class);
570    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
571        new RegionServerStub(conf), new RegionServerAccountingStub(conf));
572    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
573    heapMemoryManager.start(choreService);
574    CustomHeapMemoryTuner.memstoreSize = 0.78f;
575    CustomHeapMemoryTuner.blockCacheSize = 0.02f;
576    Thread.sleep(1500); // Allow the tuner to run once and do necessary memory up
577    // Even if the tuner says to set the memstore to 78%, HBase makes it as 70% as that is the
578    // upper bound. Same with block cache as 10% is the lower bound.
579    assertHeapSpace(0.7f, memStoreFlusher.memstoreSize);
580    assertHeapSpace(0.1f, blockCache.maxSize);
581  }
582
583  @Test
584  public void testWhenCombinedHeapSizesFromTunerGoesOutSideMaxLimit() throws Exception {
585    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
586    MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
587    Configuration conf = HBaseConfiguration.create();
588    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.7f);
589    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.1f);
590    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
591    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.1f);
592    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
593    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
594    conf.setClass(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_CLASS, CustomHeapMemoryTuner.class,
595        HeapMemoryTuner.class);
596    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
597        new RegionServerStub(conf), new RegionServerAccountingStub(conf));
598    long oldMemstoreSize = memStoreFlusher.memstoreSize;
599    long oldBlockCacheSize = blockCache.maxSize;
600    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
601    heapMemoryManager.start(choreService);
602    CustomHeapMemoryTuner.memstoreSize = 0.7f;
603    CustomHeapMemoryTuner.blockCacheSize = 0.3f;
604    // Allow the tuner to run once and do necessary memory up
605    Thread.sleep(1500);
606    assertEquals(oldMemstoreSize, memStoreFlusher.memstoreSize);
607    assertEquals(oldBlockCacheSize, blockCache.maxSize);
608  }
609
610  private void assertHeapSpace(float expectedHeapPercentage, long currentHeapSpace) {
611    long expected = (long) (this.maxHeapSize * expectedHeapPercentage);
612    assertEquals(expected, currentHeapSpace);
613  }
614
615  private void assertHeapSpaceDelta(double expectedDeltaPercent, long oldHeapSpace,
616      long newHeapSpace) {
617    double expctedMinDelta = (double) (this.maxHeapSize * expectedDeltaPercent);
618    // Tolerable error
619    double error = 0.95;
620    if (expectedDeltaPercent > 0) {
621      assertTrue(expctedMinDelta*error <= (double)(newHeapSpace - oldHeapSpace));
622      assertTrue(expctedMinDelta/error >= (double)(newHeapSpace - oldHeapSpace));
623    } else {
624      assertTrue(-expctedMinDelta*error <= (double)(oldHeapSpace - newHeapSpace));
625      assertTrue(-expctedMinDelta/error >= (double)(oldHeapSpace - newHeapSpace));
626    }
627  }
628
629
630  private void waitForTune(final MemstoreFlusherStub memStoreFlusher,
631                           final long oldMemstoreHeapSize) throws Exception {
632    // Allow the tuner to run once and do necessary memory up
633    UTIL.waitFor(10000, new Waiter.Predicate<Exception>() {
634      @Override
635      public boolean evaluate() throws Exception {
636        return oldMemstoreHeapSize != memStoreFlusher.memstoreSize;
637      }
638    });
639  }
640
641  private static class BlockCacheStub implements ResizableBlockCache {
642    CacheStats stats = new CacheStats("test");
643    long maxSize = 0;
644    private long testBlockSize = 0;
645
646    public BlockCacheStub(long size){
647      this.maxSize = size;
648    }
649
650    @Override
651    public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory) {
652
653    }
654
655    @Override
656    public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf) {
657
658    }
659
660    @Override
661    public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching, boolean repeat,
662        boolean updateCacheMetrics) {
663      return null;
664    }
665
666    @Override
667    public boolean evictBlock(BlockCacheKey cacheKey) {
668      stats.evicted(0, cacheKey != null ? cacheKey.isPrimary() : true);
669      return false;
670    }
671
672    @Override
673    public int evictBlocksByHfileName(String hfileName) {
674      stats.evicted(0, true); // Just assuming only one block for file here.
675      return 0;
676    }
677
678    @Override
679    public CacheStats getStats() {
680      return this.stats;
681    }
682
683    @Override
684    public void shutdown() {
685
686    }
687
688    @Override
689    public long size() {
690      return 0;
691    }
692
693    @Override
694    public long getMaxSize() {
695      return 0;
696    }
697
698    @Override
699    public long getFreeSize() {
700      return 0;
701    }
702
703    @Override
704    public long getCurrentSize() {
705      return this.testBlockSize;
706    }
707
708    @Override
709    public long getCurrentDataSize() {
710      return 0;
711    }
712
713    @Override
714    public long getBlockCount() {
715      return 0;
716    }
717
718    @Override
719    public long getDataBlockCount() {
720      return 0;
721    }
722
723    @Override
724    public void setMaxSize(long size) {
725      this.maxSize = size;
726    }
727
728    @Override
729    public Iterator<CachedBlock> iterator() {
730      return null;
731    }
732
733    @Override
734    public BlockCache[] getBlockCaches() {
735      return null;
736    }
737
738    @Override
739    public void returnBlock(BlockCacheKey cacheKey, Cacheable buf) {
740    }
741
742    public void setTestBlockSize(long testBlockSize) {
743      this.testBlockSize = testBlockSize;
744    }
745  }
746
747  private static class MemstoreFlusherStub implements FlushRequester {
748
749    long memstoreSize;
750
751    FlushRequestListener listener;
752
753    FlushType flushType = FlushType.NORMAL;
754
755    public MemstoreFlusherStub(long memstoreSize) {
756      this.memstoreSize = memstoreSize;
757    }
758
759    @Override
760    public boolean requestFlush(HRegion region, boolean forceFlushAllStores,
761        FlushLifeCycleTracker tracker) {
762      this.listener.flushRequested(flushType, region);
763      return true;
764    }
765
766    @Override
767    public boolean requestDelayedFlush(HRegion region, long delay, boolean forceFlushAllStores) {
768      return true;
769    }
770
771    @Override
772    public void registerFlushRequestListener(FlushRequestListener listener) {
773      this.listener = listener;
774    }
775
776    @Override
777    public boolean unregisterFlushRequestListener(FlushRequestListener listener) {
778      return false;
779    }
780
781    @Override
782    public void setGlobalMemStoreLimit(long globalMemStoreSize) {
783      this.memstoreSize = globalMemStoreSize;
784    }
785  }
786
787  private static class RegionServerStub implements Server {
788    private Configuration conf;
789    private boolean stopped = false;
790
791    public RegionServerStub(Configuration conf) {
792      this.conf = conf;
793    }
794
795    @Override
796    public void abort(String why, Throwable e) {
797
798    }
799
800    @Override
801    public boolean isAborted() {
802      return false;
803    }
804
805    @Override
806    public void stop(String why) {
807      this.stopped = true;
808    }
809
810    @Override
811    public boolean isStopped() {
812      return this.stopped;
813    }
814
815    @Override
816    public Configuration getConfiguration() {
817      return this.conf;
818    }
819
820    @Override
821    public ZKWatcher getZooKeeper() {
822      return null;
823    }
824
825    @Override
826    public CoordinatedStateManager getCoordinatedStateManager() {
827      return null;
828    }
829
830    @Override
831    public ClusterConnection getConnection() {
832      return null;
833    }
834
835    @Override
836    public ServerName getServerName() {
837      return ServerName.valueOf("server1",4000,12345);
838    }
839
840    @Override
841    public ChoreService getChoreService() {
842      return null;
843    }
844
845    @Override
846    public ClusterConnection getClusterConnection() {
847      // TODO Auto-generated method stub
848      return null;
849    }
850
851    @Override
852    public FileSystem getFileSystem() {
853      return null;
854    }
855
856    @Override
857    public boolean isStopping() {
858      return false;
859    }
860
861    @Override
862    public Connection createConnection(Configuration conf) throws IOException {
863      return null;
864    }
865  }
866
867  static class CustomHeapMemoryTuner implements HeapMemoryTuner {
868    static float blockCacheSize = 0.4f;
869    static float memstoreSize = 0.4f;
870
871    @Override
872    public Configuration getConf() {
873      return null;
874    }
875
876    @Override
877    public void setConf(Configuration arg0) {
878
879    }
880
881    @Override
882    public TunerResult tune(TunerContext context) {
883      TunerResult result = new TunerResult(true);
884      result.setBlockCacheSize(blockCacheSize);
885      result.setMemStoreSize(memstoreSize);
886      return result;
887    }
888  }
889
890  private static class RegionServerAccountingStub extends RegionServerAccounting {
891    boolean offheap;
892
893    public RegionServerAccountingStub(Configuration conf) {
894      super(conf);
895    }
896
897    public RegionServerAccountingStub(Configuration conf, boolean offheap) {
898      super(conf);
899      this.offheap = offheap;
900    }
901
902    private long testMemstoreSize = 0;
903
904    @Override
905    public long getGlobalMemStoreDataSize() {
906      return testMemstoreSize;
907    }
908
909    @Override
910    public long getGlobalMemStoreHeapSize() {
911      return testMemstoreSize;
912    }
913
914    @Override
915    public boolean isOffheap() {
916      return offheap;
917    }
918
919    public void setTestMemstoreSize(long testMemstoreSize) {
920      this.testMemstoreSize = testMemstoreSize;
921    }
922  }
923}