001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertTrue;
023import static org.junit.Assert.fail;
024import java.io.IOException;
025import java.lang.management.ManagementFactory;
026import java.util.Iterator;
027import org.apache.hadoop.conf.Configuration;
028import org.apache.hadoop.fs.FileSystem;
029import org.apache.hadoop.hbase.ChoreService;
030import org.apache.hadoop.hbase.CoordinatedStateManager;
031import org.apache.hadoop.hbase.HBaseClassTestRule;
032import org.apache.hadoop.hbase.HBaseConfiguration;
033import org.apache.hadoop.hbase.HBaseTestingUtility;
034import org.apache.hadoop.hbase.HConstants;
035import org.apache.hadoop.hbase.Server;
036import org.apache.hadoop.hbase.ServerName;
037import org.apache.hadoop.hbase.Waiter;
038import org.apache.hadoop.hbase.client.ClusterConnection;
039import org.apache.hadoop.hbase.client.Connection;
040import org.apache.hadoop.hbase.io.hfile.BlockCache;
041import org.apache.hadoop.hbase.io.hfile.BlockCacheKey;
042import org.apache.hadoop.hbase.io.hfile.CacheStats;
043import org.apache.hadoop.hbase.io.hfile.Cacheable;
044import org.apache.hadoop.hbase.io.hfile.CachedBlock;
045import org.apache.hadoop.hbase.io.hfile.ResizableBlockCache;
046import org.apache.hadoop.hbase.io.util.MemorySizeUtil;
047import org.apache.hadoop.hbase.regionserver.HeapMemoryManager.TunerContext;
048import org.apache.hadoop.hbase.regionserver.HeapMemoryManager.TunerResult;
049import org.apache.hadoop.hbase.testclassification.MediumTests;
050import org.apache.hadoop.hbase.testclassification.RegionServerTests;
051import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
052import org.junit.ClassRule;
053import org.junit.Test;
054import org.junit.experimental.categories.Category;
055
056@Category({RegionServerTests.class, MediumTests.class})
057public class TestHeapMemoryManager {
058
059  @ClassRule
060  public static final HBaseClassTestRule CLASS_RULE =
061      HBaseClassTestRule.forClass(TestHeapMemoryManager.class);
062
063  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
064
065  private long maxHeapSize = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax();
066
067  @Test
068  public void testAutoTunerShouldBeOffWhenMaxMinRangesForMemstoreIsNotGiven() throws Exception {
069    Configuration conf = HBaseConfiguration.create();
070    conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_KEY, 0.02f);
071    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.75f);
072    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.03f);
073    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf);
074    HeapMemoryManager manager = new HeapMemoryManager(new BlockCacheStub(0),
075        new MemstoreFlusherStub(0), new RegionServerStub(conf),
076        regionServerAccounting);
077    assertFalse(manager.isTunerOn());
078  }
079
080  @Test
081  public void testAutoTunerShouldBeOffWhenMaxMinRangesForBlockCacheIsNotGiven() throws Exception {
082    Configuration conf = HBaseConfiguration.create();
083    conf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.02f);
084    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
085    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.03f);
086    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf);
087    HeapMemoryManager manager = new HeapMemoryManager(new BlockCacheStub(0),
088        new MemstoreFlusherStub(0), new RegionServerStub(conf),
089        regionServerAccounting);
090    assertFalse(manager.isTunerOn());
091  }
092
093  @Test
094  public void testWhenMemstoreAndBlockCacheMaxMinChecksFails() throws Exception {
095    BlockCacheStub blockCache = new BlockCacheStub(0);
096    Configuration conf = HBaseConfiguration.create();
097    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
098    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.06f);
099    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf);
100    MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub(0);
101    try {
102      new HeapMemoryManager(blockCache, memStoreFlusher,
103          new RegionServerStub(conf), regionServerAccounting);
104      fail();
105    } catch (RuntimeException e) {
106    }
107    conf = HBaseConfiguration.create();
108    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.2f);
109    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
110    try {
111      new HeapMemoryManager(blockCache, memStoreFlusher,
112          new RegionServerStub(conf), regionServerAccounting);
113      fail();
114    } catch (RuntimeException e) {
115    }
116  }
117
118  @Test
119  public void testWhenClusterIsWriteHeavyWithEmptyMemstore() throws Exception {
120    Configuration conf = HBaseConfiguration.create();
121    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
122    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f);
123    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
124    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f);
125    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
126    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
127    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf);
128    MemstoreFlusherStub memStoreFlusher =
129        new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
130    // Empty block cache and memstore
131    blockCache.setTestBlockSize(0);
132    regionServerAccounting.setTestMemstoreSize(0);
133    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
134    // Let the system start with default values for memstore heap and block cache size.
135    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
136        new RegionServerStub(conf), regionServerAccounting);
137    long oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
138    long oldBlockCacheSize = blockCache.maxSize;
139    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
140    heapMemoryManager.start(choreService);
141    memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_HIGHER_MARK;
142    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
143    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
144    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
145    memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK;
146    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
147    // Allow the tuner to run once and do necessary memory up
148    Thread.sleep(1500);
149    // No changes should be made by tuner as we already have lot of empty space
150    assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
151    assertEquals(oldBlockCacheSize, blockCache.maxSize);
152  }
153
154  @Test
155  public void testHeapMemoryManagerWhenOffheapFlushesHappenUnderReadHeavyCase() throws Exception {
156    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
157    Configuration conf = HBaseConfiguration.create();
158    conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_LOWER_LIMIT_KEY, 0.7f);
159    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
160    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f);
161    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
162    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f);
163    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
164    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
165    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf, true);
166    MemstoreFlusherStub memStoreFlusher =
167        new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
168    // Empty memstore and but nearly filled block cache
169    blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8));
170    regionServerAccounting.setTestMemstoreSize(0);
171    // Let the system start with default values for memstore heap and block cache size.
172    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
173        new RegionServerStub(conf), regionServerAccounting);
174    long oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
175    long oldBlockCacheSize = blockCache.maxSize;
176    float maxStepValue = DefaultHeapMemoryTuner.DEFAULT_MIN_STEP_VALUE;
177    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
178    heapMemoryManager.start(choreService);
179    blockCache.evictBlock(null);
180    blockCache.evictBlock(null);
181    blockCache.evictBlock(null);
182    // do some offheap flushes also. So there should be decrease in memstore but
183    // not as that when we don't have offheap flushes
184    memStoreFlusher.flushType = FlushType.ABOVE_OFFHEAP_HIGHER_MARK;
185    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
186    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
187    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
188    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
189    // Allow the tuner to run once and do necessary memory up
190    waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
191    assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
192    assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize);
193    oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
194    oldBlockCacheSize = blockCache.maxSize;
195    // Do some more evictions before the next run of HeapMemoryTuner
196    blockCache.evictBlock(null);
197    // Allow the tuner to run once and do necessary memory up
198    waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
199    assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
200    assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize);
201  }
202
203  @Test
204  public void testHeapMemoryManagerWithOffheapMemstoreAndMixedWorkload() throws Exception {
205    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
206    Configuration conf = HBaseConfiguration.create();
207    conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_LOWER_LIMIT_KEY, 0.7f);
208    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
209    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f);
210    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
211    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f);
212    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
213    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
214    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf, true);
215    MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
216    // Empty memstore and but nearly filled block cache
217    blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8));
218    regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8));
219    // Let the system start with default values for memstore heap and block cache size.
220    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
221        new RegionServerStub(conf), regionServerAccounting);
222    long oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
223    long oldBlockCacheSize = blockCache.maxSize;
224    float maxStepValue = DefaultHeapMemoryTuner.DEFAULT_MIN_STEP_VALUE;
225    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
226    heapMemoryManager.start(choreService);
227    blockCache.evictBlock(null);
228    blockCache.evictBlock(null);
229    blockCache.evictBlock(null);
230    // do some offheap flushes also. So there should be decrease in memstore but
231    // not as that when we don't have offheap flushes
232    memStoreFlusher.flushType = FlushType.ABOVE_OFFHEAP_HIGHER_MARK;
233    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
234    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
235    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
236    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
237    // Allow the tuner to run once and do necessary memory up
238    waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
239    assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
240    assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize);
241    oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
242    oldBlockCacheSize = blockCache.maxSize;
243    // change memstore size
244    // regionServerAccounting.setTestMemstoreSize((long)(maxHeapSize * 0.4 * 0.8));
245    // The memstore size would have decreased. Now again do some flushes and ensure the
246    // flushes are due to onheap overhead. This should once again call for increase in
247    // memstore size but that increase should be to the safe size
248    memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_HIGHER_MARK;
249    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
250    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
251    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
252    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
253    // Allow the tuner to run once and do necessary memory up
254    waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
255    assertHeapSpaceDelta(maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
256    assertHeapSpaceDelta(-maxStepValue, oldBlockCacheSize, blockCache.maxSize);
257  }
258
259  @Test
260  public void testWhenClusterIsReadHeavyWithEmptyBlockCache() throws Exception {
261    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
262    Configuration conf = HBaseConfiguration.create();
263    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
264    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f);
265    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
266    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f);
267    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
268    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
269    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf);
270    MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
271    // Empty block cache and memstore
272    blockCache.setTestBlockSize(0);
273    regionServerAccounting.setTestMemstoreSize(0);
274    // Let the system start with default values for memstore heap and block cache size.
275    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
276        new RegionServerStub(conf), regionServerAccounting);
277    long oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
278    long oldBlockCacheSize = blockCache.maxSize;
279    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
280    heapMemoryManager.start(choreService);
281    blockCache.evictBlock(null);
282    blockCache.evictBlock(null);
283    blockCache.evictBlock(null);
284    // Allow the tuner to run once and do necessary memory up
285    Thread.sleep(1500);
286    // No changes should be made by tuner as we already have lot of empty space
287    assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
288    assertEquals(oldBlockCacheSize, blockCache.maxSize);
289  }
290
291  @Test
292  public void testWhenClusterIsWriteHeavy() throws Exception {
293    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
294    Configuration conf = HBaseConfiguration.create();
295    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
296    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f);
297    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
298    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f);
299    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
300    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
301    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf);
302    MemstoreFlusherStub memStoreFlusher =
303        new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
304    // Empty block cache and but nearly filled memstore
305    blockCache.setTestBlockSize(0);
306    regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8));
307    // Let the system start with default values for memstore heap and block cache size.
308    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
309        new RegionServerStub(conf), regionServerAccounting);
310    long oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
311    long oldBlockCacheSize = blockCache.maxSize;
312    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
313    heapMemoryManager.start(choreService);
314    memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK;
315    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
316    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
317    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
318    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
319    // Allow the tuner to run once and do necessary memory up
320    waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
321    assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldMemstoreHeapSize,
322        memStoreFlusher.memstoreSize);
323    assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldBlockCacheSize,
324        blockCache.maxSize);
325    oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
326    oldBlockCacheSize = blockCache.maxSize;
327    // Do some more flushes before the next run of HeapMemoryTuner
328    memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK;
329    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
330    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
331    // Allow the tuner to run once and do necessary memory up
332    waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
333    assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldMemstoreHeapSize,
334        memStoreFlusher.memstoreSize);
335    assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldBlockCacheSize,
336        blockCache.maxSize);
337  }
338
339  @Test
340  public void testWhenClusterIsWriteHeavyWithOffheapMemstore() throws Exception {
341    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
342    Configuration conf = HBaseConfiguration.create();
343    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
344    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f);
345    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
346    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f);
347    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
348    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
349    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf);
350    MemstoreFlusherStub memStoreFlusher =
351        new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
352    // Empty block cache and but nearly filled memstore
353    blockCache.setTestBlockSize(0);
354    regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8));
355    // Let the system start with default values for memstore heap and block cache size.
356    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
357        new RegionServerStub(conf), regionServerAccounting);
358    long oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
359    long oldBlockCacheSize = blockCache.maxSize;
360    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
361    heapMemoryManager.start(choreService);
362    // this should not change anything with onheap memstore
363    memStoreFlusher.flushType = FlushType.ABOVE_OFFHEAP_HIGHER_MARK;
364    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
365    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
366    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
367    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
368    // Allow the tuner to run once and do necessary memory up
369    Thread.sleep(1500);
370    // No changes should be made by tuner as we already have lot of empty space
371    assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
372    assertEquals(oldBlockCacheSize, blockCache.maxSize);
373  }
374
375  @Test
376  public void testWhenClusterIsReadHeavy() throws Exception {
377    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
378    Configuration conf = HBaseConfiguration.create();
379    conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_LOWER_LIMIT_KEY, 0.7f);
380    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
381    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f);
382    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
383    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f);
384    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
385    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
386    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf);
387    MemstoreFlusherStub memStoreFlusher =
388        new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
389    // Empty memstore and but nearly filled block cache
390    blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8));
391    regionServerAccounting.setTestMemstoreSize(0);
392    // Let the system start with default values for memstore heap and block cache size.
393    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
394        new RegionServerStub(conf), new RegionServerAccountingStub(conf));
395    long oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
396    long oldBlockCacheSize = blockCache.maxSize;
397    long oldMemstoreLowerMarkSize = 7 * oldMemstoreHeapSize / 10;
398    long maxTuneSize = oldMemstoreHeapSize -  (oldMemstoreLowerMarkSize + oldMemstoreHeapSize) / 2;
399    float maxStepValue = (maxTuneSize * 1.0f) / oldMemstoreHeapSize;
400    maxStepValue = maxStepValue > DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE ?
401        DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE:maxStepValue;
402    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
403    heapMemoryManager.start(choreService);
404    blockCache.evictBlock(null);
405    blockCache.evictBlock(null);
406    blockCache.evictBlock(null);
407    // Allow the tuner to run once and do necessary memory up
408    waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
409    assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
410    assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize);
411    oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
412    oldBlockCacheSize = blockCache.maxSize;
413    oldMemstoreLowerMarkSize = 7 * oldMemstoreHeapSize / 10;
414    maxTuneSize = oldMemstoreHeapSize -  (oldMemstoreLowerMarkSize + oldMemstoreHeapSize) / 2;
415    maxStepValue = (maxTuneSize * 1.0f) / oldMemstoreHeapSize;
416    maxStepValue = maxStepValue > DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE ?
417        DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE:maxStepValue;
418    // Do some more evictions before the next run of HeapMemoryTuner
419    blockCache.evictBlock(null);
420    // Allow the tuner to run once and do necessary memory up
421    waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
422    assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
423    assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize);
424  }
425
426  @Test
427  public void testWhenClusterIsHavingMoreWritesThanReads() throws Exception {
428    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
429    Configuration conf = HBaseConfiguration.create();
430    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
431    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f);
432    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
433    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f);
434    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
435    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
436    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf);
437    MemstoreFlusherStub memStoreFlusher =
438        new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
439    // Both memstore and block cache are nearly filled
440    blockCache.setTestBlockSize(0);
441    regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8));
442    blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8));
443    // Let the system start with default values for memstore heap and block cache size.
444    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
445        new RegionServerStub(conf), regionServerAccounting);
446    long oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
447    long oldBlockCacheSize = blockCache.maxSize;
448    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
449    heapMemoryManager.start(choreService);
450    memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK;
451    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
452    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
453    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
454    blockCache.evictBlock(null);
455    // Allow the tuner to run once and do necessary memory up
456    Thread.sleep(1500);
457    // No changes should happen as there is undefined increase in flushes and evictions
458    assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
459    assertEquals(oldBlockCacheSize, blockCache.maxSize);
460    // Do some more flushes before the next run of HeapMemoryTuner
461    memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK;
462    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
463    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
464    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
465    // Allow the tuner to run once and do necessary memory up
466    waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
467    assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldMemstoreHeapSize,
468        memStoreFlusher.memstoreSize);
469    assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldBlockCacheSize,
470        blockCache.maxSize);
471  }
472
473  @Test
474  public void testBlockedFlushesIncreaseMemstoreInSteadyState() throws Exception {
475    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
476    MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
477    Configuration conf = HBaseConfiguration.create();
478    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
479    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f);
480    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
481    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f);
482    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
483    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
484    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf);
485    // Both memstore and block cache are nearly filled
486    blockCache.setTestBlockSize(0);
487    regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8));
488    blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8));
489    // Let the system start with default values for memstore heap and block cache size.
490    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
491        new RegionServerStub(conf), regionServerAccounting);
492    long oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
493    long oldBlockCacheSize = blockCache.maxSize;
494    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
495    heapMemoryManager.start(choreService);
496    memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK;
497    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
498    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
499    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
500    blockCache.evictBlock(null);
501    blockCache.evictBlock(null);
502    // Allow the tuner to run once and do necessary memory up
503    Thread.sleep(1500);
504    // No changes should happen as there is undefined increase in flushes and evictions
505    assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
506    assertEquals(oldBlockCacheSize, blockCache.maxSize);
507    // Flushes that block updates
508    memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_HIGHER_MARK;
509    memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY);
510    blockCache.evictBlock(null);
511    blockCache.evictBlock(null);
512    blockCache.evictBlock(null);
513    blockCache.evictBlock(null);
514    // Allow the tuner to run once and do necessary memory up
515    Thread.sleep(1500);
516    assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldMemstoreHeapSize,
517        memStoreFlusher.memstoreSize);
518    assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldBlockCacheSize,
519        blockCache.maxSize);
520  }
521
522  @Test
523  public void testPluggingInHeapMemoryTuner() throws Exception {
524    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
525    MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
526    Configuration conf = HBaseConfiguration.create();
527    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.78f);
528    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.05f);
529    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.75f);
530    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.02f);
531    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
532    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
533    conf.setClass(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_CLASS, CustomHeapMemoryTuner.class,
534        HeapMemoryTuner.class);
535    // Let the system start with default values for memstore heap and block cache size.
536    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
537        new RegionServerStub(conf), new RegionServerAccountingStub(conf));
538    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
539    heapMemoryManager.start(choreService);
540    // Now we wants to be in write mode. Set bigger memstore size from CustomHeapMemoryTuner
541    CustomHeapMemoryTuner.memstoreSize = 0.78f;
542    CustomHeapMemoryTuner.blockCacheSize = 0.02f;
543    // Allow the tuner to run once and do necessary memory up
544    waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
545    assertHeapSpace(0.78f, memStoreFlusher.memstoreSize);// Memstore
546    assertHeapSpace(0.02f, blockCache.maxSize);// BlockCache
547    // Now we wants to be in read mode. Set bigger memstore size from CustomHeapMemoryTuner
548    CustomHeapMemoryTuner.blockCacheSize = 0.75f;
549    CustomHeapMemoryTuner.memstoreSize = 0.05f;
550    // Allow the tuner to run once and do necessary memory up
551    waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
552    assertHeapSpace(0.75f, blockCache.maxSize);// BlockCache
553    assertHeapSpace(0.05f, memStoreFlusher.memstoreSize);// Memstore
554  }
555
556  @Test
557  public void testWhenSizeGivenByHeapTunerGoesOutsideRange() throws Exception {
558    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
559    MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
560    Configuration conf = HBaseConfiguration.create();
561    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.7f);
562    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.1f);
563    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
564    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.1f);
565    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
566    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
567    conf.setClass(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_CLASS, CustomHeapMemoryTuner.class,
568        HeapMemoryTuner.class);
569    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
570        new RegionServerStub(conf), new RegionServerAccountingStub(conf));
571    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
572    heapMemoryManager.start(choreService);
573    CustomHeapMemoryTuner.memstoreSize = 0.78f;
574    CustomHeapMemoryTuner.blockCacheSize = 0.02f;
575    Thread.sleep(1500); // Allow the tuner to run once and do necessary memory up
576    // Even if the tuner says to set the memstore to 78%, HBase makes it as 70% as that is the
577    // upper bound. Same with block cache as 10% is the lower bound.
578    assertHeapSpace(0.7f, memStoreFlusher.memstoreSize);
579    assertHeapSpace(0.1f, blockCache.maxSize);
580  }
581
582  @Test
583  public void testWhenCombinedHeapSizesFromTunerGoesOutSideMaxLimit() throws Exception {
584    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
585    MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
586    Configuration conf = HBaseConfiguration.create();
587    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.7f);
588    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.1f);
589    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
590    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.1f);
591    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
592    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
593    conf.setClass(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_CLASS, CustomHeapMemoryTuner.class,
594        HeapMemoryTuner.class);
595    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
596        new RegionServerStub(conf), new RegionServerAccountingStub(conf));
597    long oldMemstoreSize = memStoreFlusher.memstoreSize;
598    long oldBlockCacheSize = blockCache.maxSize;
599    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
600    heapMemoryManager.start(choreService);
601    CustomHeapMemoryTuner.memstoreSize = 0.7f;
602    CustomHeapMemoryTuner.blockCacheSize = 0.3f;
603    // Allow the tuner to run once and do necessary memory up
604    Thread.sleep(1500);
605    assertEquals(oldMemstoreSize, memStoreFlusher.memstoreSize);
606    assertEquals(oldBlockCacheSize, blockCache.maxSize);
607  }
608
609  private void assertHeapSpace(float expectedHeapPercentage, long currentHeapSpace) {
610    long expected = (long) (this.maxHeapSize * expectedHeapPercentage);
611    assertEquals(expected, currentHeapSpace);
612  }
613
614  private void assertHeapSpaceDelta(double expectedDeltaPercent, long oldHeapSpace,
615      long newHeapSpace) {
616    double expctedMinDelta = (double) (this.maxHeapSize * expectedDeltaPercent);
617    // Tolerable error
618    double error = 0.95;
619    if (expectedDeltaPercent > 0) {
620      assertTrue(expctedMinDelta*error <= (double)(newHeapSpace - oldHeapSpace));
621      assertTrue(expctedMinDelta/error >= (double)(newHeapSpace - oldHeapSpace));
622    } else {
623      assertTrue(-expctedMinDelta*error <= (double)(oldHeapSpace - newHeapSpace));
624      assertTrue(-expctedMinDelta/error >= (double)(oldHeapSpace - newHeapSpace));
625    }
626  }
627
628
629  private void waitForTune(final MemstoreFlusherStub memStoreFlusher,
630                           final long oldMemstoreHeapSize) throws Exception {
631    // Allow the tuner to run once and do necessary memory up
632    UTIL.waitFor(10000, new Waiter.Predicate<Exception>() {
633      @Override
634      public boolean evaluate() throws Exception {
635        return oldMemstoreHeapSize != memStoreFlusher.memstoreSize;
636      }
637    });
638  }
639
640  private static class BlockCacheStub implements ResizableBlockCache {
641    CacheStats stats = new CacheStats("test");
642    long maxSize = 0;
643    private long testBlockSize = 0;
644
645    public BlockCacheStub(long size){
646      this.maxSize = size;
647    }
648
649    @Override
650    public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory) {
651
652    }
653
654    @Override
655    public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf) {
656
657    }
658
659    @Override
660    public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching, boolean repeat,
661        boolean updateCacheMetrics) {
662      return null;
663    }
664
665    @Override
666    public boolean evictBlock(BlockCacheKey cacheKey) {
667      stats.evicted(0, cacheKey != null ? cacheKey.isPrimary() : true);
668      return false;
669    }
670
671    @Override
672    public int evictBlocksByHfileName(String hfileName) {
673      stats.evicted(0, true); // Just assuming only one block for file here.
674      return 0;
675    }
676
677    @Override
678    public CacheStats getStats() {
679      return this.stats;
680    }
681
682    @Override
683    public void shutdown() {
684
685    }
686
687    @Override
688    public long size() {
689      return 0;
690    }
691
692    @Override
693    public long getMaxSize() {
694      return 0;
695    }
696
697    @Override
698    public long getFreeSize() {
699      return 0;
700    }
701
702    @Override
703    public long getCurrentSize() {
704      return this.testBlockSize;
705    }
706
707    @Override
708    public long getCurrentDataSize() {
709      return 0;
710    }
711
712    @Override
713    public long getBlockCount() {
714      return 0;
715    }
716
717    @Override
718    public long getDataBlockCount() {
719      return 0;
720    }
721
722    @Override
723    public void setMaxSize(long size) {
724      this.maxSize = size;
725    }
726
727    @Override
728    public Iterator<CachedBlock> iterator() {
729      return null;
730    }
731
732    @Override
733    public BlockCache[] getBlockCaches() {
734      return null;
735    }
736
737    public void setTestBlockSize(long testBlockSize) {
738      this.testBlockSize = testBlockSize;
739    }
740  }
741
742  private static class MemstoreFlusherStub implements FlushRequester {
743
744    long memstoreSize;
745
746    FlushRequestListener listener;
747
748    FlushType flushType = FlushType.NORMAL;
749
750    public MemstoreFlusherStub(long memstoreSize) {
751      this.memstoreSize = memstoreSize;
752    }
753
754    @Override
755    public boolean requestFlush(HRegion region, boolean forceFlushAllStores,
756        FlushLifeCycleTracker tracker) {
757      this.listener.flushRequested(flushType, region);
758      return true;
759    }
760
761    @Override
762    public boolean requestDelayedFlush(HRegion region, long delay, boolean forceFlushAllStores) {
763      return true;
764    }
765
766    @Override
767    public void registerFlushRequestListener(FlushRequestListener listener) {
768      this.listener = listener;
769    }
770
771    @Override
772    public boolean unregisterFlushRequestListener(FlushRequestListener listener) {
773      return false;
774    }
775
776    @Override
777    public void setGlobalMemStoreLimit(long globalMemStoreSize) {
778      this.memstoreSize = globalMemStoreSize;
779    }
780  }
781
782  private static class RegionServerStub implements Server {
783    private Configuration conf;
784    private boolean stopped = false;
785
786    public RegionServerStub(Configuration conf) {
787      this.conf = conf;
788    }
789
790    @Override
791    public void abort(String why, Throwable e) {
792
793    }
794
795    @Override
796    public boolean isAborted() {
797      return false;
798    }
799
800    @Override
801    public void stop(String why) {
802      this.stopped = true;
803    }
804
805    @Override
806    public boolean isStopped() {
807      return this.stopped;
808    }
809
810    @Override
811    public Configuration getConfiguration() {
812      return this.conf;
813    }
814
815    @Override
816    public ZKWatcher getZooKeeper() {
817      return null;
818    }
819
820    @Override
821    public CoordinatedStateManager getCoordinatedStateManager() {
822      return null;
823    }
824
825    @Override
826    public ClusterConnection getConnection() {
827      return null;
828    }
829
830    @Override
831    public ServerName getServerName() {
832      return ServerName.valueOf("server1",4000,12345);
833    }
834
835    @Override
836    public ChoreService getChoreService() {
837      return null;
838    }
839
840    @Override
841    public ClusterConnection getClusterConnection() {
842      // TODO Auto-generated method stub
843      return null;
844    }
845
846    @Override
847    public FileSystem getFileSystem() {
848      return null;
849    }
850
851    @Override
852    public boolean isStopping() {
853      return false;
854    }
855
856    @Override
857    public Connection createConnection(Configuration conf) throws IOException {
858      return null;
859    }
860  }
861
862  static class CustomHeapMemoryTuner implements HeapMemoryTuner {
863    static float blockCacheSize = 0.4f;
864    static float memstoreSize = 0.4f;
865
866    @Override
867    public Configuration getConf() {
868      return null;
869    }
870
871    @Override
872    public void setConf(Configuration arg0) {
873
874    }
875
876    @Override
877    public TunerResult tune(TunerContext context) {
878      TunerResult result = new TunerResult(true);
879      result.setBlockCacheSize(blockCacheSize);
880      result.setMemStoreSize(memstoreSize);
881      return result;
882    }
883  }
884
885  private static class RegionServerAccountingStub extends RegionServerAccounting {
886    boolean offheap;
887
888    public RegionServerAccountingStub(Configuration conf) {
889      super(conf);
890    }
891
892    public RegionServerAccountingStub(Configuration conf, boolean offheap) {
893      super(conf);
894      this.offheap = offheap;
895    }
896
897    private long testMemstoreSize = 0;
898
899    @Override
900    public long getGlobalMemStoreDataSize() {
901      return testMemstoreSize;
902    }
903
904    @Override
905    public long getGlobalMemStoreHeapSize() {
906      return testMemstoreSize;
907    }
908
909    @Override
910    public boolean isOffheap() {
911      return offheap;
912    }
913
914    public void setTestMemstoreSize(long testMemstoreSize) {
915      this.testMemstoreSize = testMemstoreSize;
916    }
917  }
918}