001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.regionserver; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertFalse; 022import static org.junit.Assert.assertTrue; 023import static org.junit.Assert.fail; 024import java.io.IOException; 025import java.lang.management.ManagementFactory; 026import java.util.Iterator; 027import org.apache.hadoop.conf.Configuration; 028import org.apache.hadoop.fs.FileSystem; 029import org.apache.hadoop.hbase.ChoreService; 030import org.apache.hadoop.hbase.CoordinatedStateManager; 031import org.apache.hadoop.hbase.HBaseClassTestRule; 032import org.apache.hadoop.hbase.HBaseConfiguration; 033import org.apache.hadoop.hbase.HBaseTestingUtility; 034import org.apache.hadoop.hbase.HConstants; 035import org.apache.hadoop.hbase.Server; 036import org.apache.hadoop.hbase.ServerName; 037import org.apache.hadoop.hbase.Waiter; 038import org.apache.hadoop.hbase.client.ClusterConnection; 039import org.apache.hadoop.hbase.client.Connection; 040import org.apache.hadoop.hbase.io.hfile.BlockCache; 041import org.apache.hadoop.hbase.io.hfile.BlockCacheKey; 042import org.apache.hadoop.hbase.io.hfile.CacheStats; 043import org.apache.hadoop.hbase.io.hfile.Cacheable; 044import org.apache.hadoop.hbase.io.hfile.CachedBlock; 045import org.apache.hadoop.hbase.io.hfile.ResizableBlockCache; 046import org.apache.hadoop.hbase.io.util.MemorySizeUtil; 047import org.apache.hadoop.hbase.regionserver.HeapMemoryManager.TunerContext; 048import org.apache.hadoop.hbase.regionserver.HeapMemoryManager.TunerResult; 049import org.apache.hadoop.hbase.testclassification.MediumTests; 050import org.apache.hadoop.hbase.testclassification.RegionServerTests; 051import org.apache.hadoop.hbase.zookeeper.ZKWatcher; 052import org.junit.ClassRule; 053import org.junit.Test; 054import org.junit.experimental.categories.Category; 055 056@Category({RegionServerTests.class, MediumTests.class}) 057public class TestHeapMemoryManager { 058 059 @ClassRule 060 public static final HBaseClassTestRule CLASS_RULE = 061 HBaseClassTestRule.forClass(TestHeapMemoryManager.class); 062 063 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); 064 065 private long maxHeapSize = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax(); 066 067 @Test 068 public void testAutoTunerShouldBeOffWhenMaxMinRangesForMemstoreIsNotGiven() throws Exception { 069 Configuration conf = HBaseConfiguration.create(); 070 conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_KEY, 0.02f); 071 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.75f); 072 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.03f); 073 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 074 HeapMemoryManager manager = new HeapMemoryManager(new BlockCacheStub(0), 075 new MemstoreFlusherStub(0), new RegionServerStub(conf), 076 regionServerAccounting); 077 assertFalse(manager.isTunerOn()); 078 } 079 080 @Test 081 public void testAutoTunerShouldBeOffWhenMaxMinRangesForBlockCacheIsNotGiven() throws Exception { 082 Configuration conf = HBaseConfiguration.create(); 083 conf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.02f); 084 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 085 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.03f); 086 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 087 HeapMemoryManager manager = new HeapMemoryManager(new BlockCacheStub(0), 088 new MemstoreFlusherStub(0), new RegionServerStub(conf), 089 regionServerAccounting); 090 assertFalse(manager.isTunerOn()); 091 } 092 093 @Test 094 public void testWhenMemstoreAndBlockCacheMaxMinChecksFails() throws Exception { 095 BlockCacheStub blockCache = new BlockCacheStub(0); 096 Configuration conf = HBaseConfiguration.create(); 097 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 098 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.06f); 099 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 100 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub(0); 101 try { 102 new HeapMemoryManager(blockCache, memStoreFlusher, 103 new RegionServerStub(conf), regionServerAccounting); 104 fail(); 105 } catch (RuntimeException e) { 106 } 107 conf = HBaseConfiguration.create(); 108 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.2f); 109 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 110 try { 111 new HeapMemoryManager(blockCache, memStoreFlusher, 112 new RegionServerStub(conf), regionServerAccounting); 113 fail(); 114 } catch (RuntimeException e) { 115 } 116 } 117 118 @Test 119 public void testWhenClusterIsWriteHeavyWithEmptyMemstore() throws Exception { 120 Configuration conf = HBaseConfiguration.create(); 121 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 122 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 123 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 124 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 125 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 126 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 127 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 128 MemstoreFlusherStub memStoreFlusher = 129 new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 130 // Empty block cache and memstore 131 blockCache.setTestBlockSize(0); 132 regionServerAccounting.setTestMemstoreSize(0); 133 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 134 // Let the system start with default values for memstore heap and block cache size. 135 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 136 new RegionServerStub(conf), regionServerAccounting); 137 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 138 long oldBlockCacheSize = blockCache.maxSize; 139 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 140 heapMemoryManager.start(choreService); 141 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_HIGHER_MARK; 142 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 143 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 144 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 145 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK; 146 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 147 // Allow the tuner to run once and do necessary memory up 148 Thread.sleep(1500); 149 // No changes should be made by tuner as we already have lot of empty space 150 assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 151 assertEquals(oldBlockCacheSize, blockCache.maxSize); 152 } 153 154 @Test 155 public void testHeapMemoryManagerWhenOffheapFlushesHappenUnderReadHeavyCase() throws Exception { 156 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 157 Configuration conf = HBaseConfiguration.create(); 158 conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_LOWER_LIMIT_KEY, 0.7f); 159 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 160 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 161 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 162 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 163 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 164 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 165 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf, true); 166 MemstoreFlusherStub memStoreFlusher = 167 new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 168 // Empty memstore and but nearly filled block cache 169 blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8)); 170 regionServerAccounting.setTestMemstoreSize(0); 171 // Let the system start with default values for memstore heap and block cache size. 172 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 173 new RegionServerStub(conf), regionServerAccounting); 174 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 175 long oldBlockCacheSize = blockCache.maxSize; 176 float maxStepValue = DefaultHeapMemoryTuner.DEFAULT_MIN_STEP_VALUE; 177 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 178 heapMemoryManager.start(choreService); 179 blockCache.evictBlock(null); 180 blockCache.evictBlock(null); 181 blockCache.evictBlock(null); 182 // do some offheap flushes also. So there should be decrease in memstore but 183 // not as that when we don't have offheap flushes 184 memStoreFlusher.flushType = FlushType.ABOVE_OFFHEAP_HIGHER_MARK; 185 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 186 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 187 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 188 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 189 // Allow the tuner to run once and do necessary memory up 190 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 191 assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 192 assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize); 193 oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 194 oldBlockCacheSize = blockCache.maxSize; 195 // Do some more evictions before the next run of HeapMemoryTuner 196 blockCache.evictBlock(null); 197 // Allow the tuner to run once and do necessary memory up 198 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 199 assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 200 assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize); 201 } 202 203 @Test 204 public void testHeapMemoryManagerWithOffheapMemstoreAndMixedWorkload() throws Exception { 205 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 206 Configuration conf = HBaseConfiguration.create(); 207 conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_LOWER_LIMIT_KEY, 0.7f); 208 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 209 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 210 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 211 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 212 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 213 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 214 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf, true); 215 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 216 // Empty memstore and but nearly filled block cache 217 blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8)); 218 regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8)); 219 // Let the system start with default values for memstore heap and block cache size. 220 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 221 new RegionServerStub(conf), regionServerAccounting); 222 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 223 long oldBlockCacheSize = blockCache.maxSize; 224 float maxStepValue = DefaultHeapMemoryTuner.DEFAULT_MIN_STEP_VALUE; 225 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 226 heapMemoryManager.start(choreService); 227 blockCache.evictBlock(null); 228 blockCache.evictBlock(null); 229 blockCache.evictBlock(null); 230 // do some offheap flushes also. So there should be decrease in memstore but 231 // not as that when we don't have offheap flushes 232 memStoreFlusher.flushType = FlushType.ABOVE_OFFHEAP_HIGHER_MARK; 233 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 234 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 235 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 236 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 237 // Allow the tuner to run once and do necessary memory up 238 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 239 assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 240 assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize); 241 oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 242 oldBlockCacheSize = blockCache.maxSize; 243 // change memstore size 244 // regionServerAccounting.setTestMemstoreSize((long)(maxHeapSize * 0.4 * 0.8)); 245 // The memstore size would have decreased. Now again do some flushes and ensure the 246 // flushes are due to onheap overhead. This should once again call for increase in 247 // memstore size but that increase should be to the safe size 248 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_HIGHER_MARK; 249 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 250 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 251 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 252 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 253 // Allow the tuner to run once and do necessary memory up 254 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 255 assertHeapSpaceDelta(maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 256 assertHeapSpaceDelta(-maxStepValue, oldBlockCacheSize, blockCache.maxSize); 257 } 258 259 @Test 260 public void testWhenClusterIsReadHeavyWithEmptyBlockCache() throws Exception { 261 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 262 Configuration conf = HBaseConfiguration.create(); 263 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 264 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 265 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 266 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 267 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 268 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 269 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 270 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 271 // Empty block cache and memstore 272 blockCache.setTestBlockSize(0); 273 regionServerAccounting.setTestMemstoreSize(0); 274 // Let the system start with default values for memstore heap and block cache size. 275 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 276 new RegionServerStub(conf), regionServerAccounting); 277 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 278 long oldBlockCacheSize = blockCache.maxSize; 279 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 280 heapMemoryManager.start(choreService); 281 blockCache.evictBlock(null); 282 blockCache.evictBlock(null); 283 blockCache.evictBlock(null); 284 // Allow the tuner to run once and do necessary memory up 285 Thread.sleep(1500); 286 // No changes should be made by tuner as we already have lot of empty space 287 assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 288 assertEquals(oldBlockCacheSize, blockCache.maxSize); 289 } 290 291 @Test 292 public void testWhenClusterIsWriteHeavy() throws Exception { 293 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 294 Configuration conf = HBaseConfiguration.create(); 295 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 296 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 297 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 298 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 299 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 300 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 301 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 302 MemstoreFlusherStub memStoreFlusher = 303 new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 304 // Empty block cache and but nearly filled memstore 305 blockCache.setTestBlockSize(0); 306 regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8)); 307 // Let the system start with default values for memstore heap and block cache size. 308 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 309 new RegionServerStub(conf), regionServerAccounting); 310 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 311 long oldBlockCacheSize = blockCache.maxSize; 312 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 313 heapMemoryManager.start(choreService); 314 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK; 315 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 316 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 317 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 318 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 319 // Allow the tuner to run once and do necessary memory up 320 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 321 assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldMemstoreHeapSize, 322 memStoreFlusher.memstoreSize); 323 assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldBlockCacheSize, 324 blockCache.maxSize); 325 oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 326 oldBlockCacheSize = blockCache.maxSize; 327 // Do some more flushes before the next run of HeapMemoryTuner 328 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK; 329 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 330 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 331 // Allow the tuner to run once and do necessary memory up 332 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 333 assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldMemstoreHeapSize, 334 memStoreFlusher.memstoreSize); 335 assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldBlockCacheSize, 336 blockCache.maxSize); 337 } 338 339 @Test 340 public void testWhenClusterIsWriteHeavyWithOffheapMemstore() throws Exception { 341 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 342 Configuration conf = HBaseConfiguration.create(); 343 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 344 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 345 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 346 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 347 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 348 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 349 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 350 MemstoreFlusherStub memStoreFlusher = 351 new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 352 // Empty block cache and but nearly filled memstore 353 blockCache.setTestBlockSize(0); 354 regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8)); 355 // Let the system start with default values for memstore heap and block cache size. 356 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 357 new RegionServerStub(conf), regionServerAccounting); 358 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 359 long oldBlockCacheSize = blockCache.maxSize; 360 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 361 heapMemoryManager.start(choreService); 362 // this should not change anything with onheap memstore 363 memStoreFlusher.flushType = FlushType.ABOVE_OFFHEAP_HIGHER_MARK; 364 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 365 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 366 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 367 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 368 // Allow the tuner to run once and do necessary memory up 369 Thread.sleep(1500); 370 // No changes should be made by tuner as we already have lot of empty space 371 assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 372 assertEquals(oldBlockCacheSize, blockCache.maxSize); 373 } 374 375 @Test 376 public void testWhenClusterIsReadHeavy() throws Exception { 377 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 378 Configuration conf = HBaseConfiguration.create(); 379 conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_LOWER_LIMIT_KEY, 0.7f); 380 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 381 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 382 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 383 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 384 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 385 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 386 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 387 MemstoreFlusherStub memStoreFlusher = 388 new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 389 // Empty memstore and but nearly filled block cache 390 blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8)); 391 regionServerAccounting.setTestMemstoreSize(0); 392 // Let the system start with default values for memstore heap and block cache size. 393 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 394 new RegionServerStub(conf), new RegionServerAccountingStub(conf)); 395 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 396 long oldBlockCacheSize = blockCache.maxSize; 397 long oldMemstoreLowerMarkSize = 7 * oldMemstoreHeapSize / 10; 398 long maxTuneSize = oldMemstoreHeapSize - (oldMemstoreLowerMarkSize + oldMemstoreHeapSize) / 2; 399 float maxStepValue = (maxTuneSize * 1.0f) / oldMemstoreHeapSize; 400 maxStepValue = maxStepValue > DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE ? 401 DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE:maxStepValue; 402 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 403 heapMemoryManager.start(choreService); 404 blockCache.evictBlock(null); 405 blockCache.evictBlock(null); 406 blockCache.evictBlock(null); 407 // Allow the tuner to run once and do necessary memory up 408 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 409 assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 410 assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize); 411 oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 412 oldBlockCacheSize = blockCache.maxSize; 413 oldMemstoreLowerMarkSize = 7 * oldMemstoreHeapSize / 10; 414 maxTuneSize = oldMemstoreHeapSize - (oldMemstoreLowerMarkSize + oldMemstoreHeapSize) / 2; 415 maxStepValue = (maxTuneSize * 1.0f) / oldMemstoreHeapSize; 416 maxStepValue = maxStepValue > DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE ? 417 DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE:maxStepValue; 418 // Do some more evictions before the next run of HeapMemoryTuner 419 blockCache.evictBlock(null); 420 // Allow the tuner to run once and do necessary memory up 421 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 422 assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 423 assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize); 424 } 425 426 @Test 427 public void testWhenClusterIsHavingMoreWritesThanReads() throws Exception { 428 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 429 Configuration conf = HBaseConfiguration.create(); 430 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 431 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 432 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 433 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 434 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 435 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 436 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 437 MemstoreFlusherStub memStoreFlusher = 438 new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 439 // Both memstore and block cache are nearly filled 440 blockCache.setTestBlockSize(0); 441 regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8)); 442 blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8)); 443 // Let the system start with default values for memstore heap and block cache size. 444 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 445 new RegionServerStub(conf), regionServerAccounting); 446 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 447 long oldBlockCacheSize = blockCache.maxSize; 448 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 449 heapMemoryManager.start(choreService); 450 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK; 451 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 452 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 453 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 454 blockCache.evictBlock(null); 455 // Allow the tuner to run once and do necessary memory up 456 Thread.sleep(1500); 457 // No changes should happen as there is undefined increase in flushes and evictions 458 assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 459 assertEquals(oldBlockCacheSize, blockCache.maxSize); 460 // Do some more flushes before the next run of HeapMemoryTuner 461 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK; 462 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 463 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 464 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 465 // Allow the tuner to run once and do necessary memory up 466 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 467 assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldMemstoreHeapSize, 468 memStoreFlusher.memstoreSize); 469 assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldBlockCacheSize, 470 blockCache.maxSize); 471 } 472 473 @Test 474 public void testBlockedFlushesIncreaseMemstoreInSteadyState() throws Exception { 475 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 476 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 477 Configuration conf = HBaseConfiguration.create(); 478 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 479 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 480 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 481 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 482 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 483 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 484 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 485 // Both memstore and block cache are nearly filled 486 blockCache.setTestBlockSize(0); 487 regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8)); 488 blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8)); 489 // Let the system start with default values for memstore heap and block cache size. 490 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 491 new RegionServerStub(conf), regionServerAccounting); 492 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 493 long oldBlockCacheSize = blockCache.maxSize; 494 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 495 heapMemoryManager.start(choreService); 496 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK; 497 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 498 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 499 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 500 blockCache.evictBlock(null); 501 blockCache.evictBlock(null); 502 // Allow the tuner to run once and do necessary memory up 503 Thread.sleep(1500); 504 // No changes should happen as there is undefined increase in flushes and evictions 505 assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 506 assertEquals(oldBlockCacheSize, blockCache.maxSize); 507 // Flushes that block updates 508 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_HIGHER_MARK; 509 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 510 blockCache.evictBlock(null); 511 blockCache.evictBlock(null); 512 blockCache.evictBlock(null); 513 blockCache.evictBlock(null); 514 // Allow the tuner to run once and do necessary memory up 515 Thread.sleep(1500); 516 assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldMemstoreHeapSize, 517 memStoreFlusher.memstoreSize); 518 assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldBlockCacheSize, 519 blockCache.maxSize); 520 } 521 522 @Test 523 public void testPluggingInHeapMemoryTuner() throws Exception { 524 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 525 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 526 Configuration conf = HBaseConfiguration.create(); 527 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.78f); 528 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.05f); 529 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.75f); 530 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.02f); 531 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 532 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 533 conf.setClass(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_CLASS, CustomHeapMemoryTuner.class, 534 HeapMemoryTuner.class); 535 // Let the system start with default values for memstore heap and block cache size. 536 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 537 new RegionServerStub(conf), new RegionServerAccountingStub(conf)); 538 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 539 heapMemoryManager.start(choreService); 540 // Now we wants to be in write mode. Set bigger memstore size from CustomHeapMemoryTuner 541 CustomHeapMemoryTuner.memstoreSize = 0.78f; 542 CustomHeapMemoryTuner.blockCacheSize = 0.02f; 543 // Allow the tuner to run once and do necessary memory up 544 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 545 assertHeapSpace(0.78f, memStoreFlusher.memstoreSize);// Memstore 546 assertHeapSpace(0.02f, blockCache.maxSize);// BlockCache 547 // Now we wants to be in read mode. Set bigger memstore size from CustomHeapMemoryTuner 548 CustomHeapMemoryTuner.blockCacheSize = 0.75f; 549 CustomHeapMemoryTuner.memstoreSize = 0.05f; 550 // Allow the tuner to run once and do necessary memory up 551 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 552 assertHeapSpace(0.75f, blockCache.maxSize);// BlockCache 553 assertHeapSpace(0.05f, memStoreFlusher.memstoreSize);// Memstore 554 } 555 556 @Test 557 public void testWhenSizeGivenByHeapTunerGoesOutsideRange() throws Exception { 558 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 559 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 560 Configuration conf = HBaseConfiguration.create(); 561 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.7f); 562 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.1f); 563 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 564 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.1f); 565 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 566 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 567 conf.setClass(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_CLASS, CustomHeapMemoryTuner.class, 568 HeapMemoryTuner.class); 569 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 570 new RegionServerStub(conf), new RegionServerAccountingStub(conf)); 571 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 572 heapMemoryManager.start(choreService); 573 CustomHeapMemoryTuner.memstoreSize = 0.78f; 574 CustomHeapMemoryTuner.blockCacheSize = 0.02f; 575 Thread.sleep(1500); // Allow the tuner to run once and do necessary memory up 576 // Even if the tuner says to set the memstore to 78%, HBase makes it as 70% as that is the 577 // upper bound. Same with block cache as 10% is the lower bound. 578 assertHeapSpace(0.7f, memStoreFlusher.memstoreSize); 579 assertHeapSpace(0.1f, blockCache.maxSize); 580 } 581 582 @Test 583 public void testWhenCombinedHeapSizesFromTunerGoesOutSideMaxLimit() throws Exception { 584 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 585 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 586 Configuration conf = HBaseConfiguration.create(); 587 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.7f); 588 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.1f); 589 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 590 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.1f); 591 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 592 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 593 conf.setClass(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_CLASS, CustomHeapMemoryTuner.class, 594 HeapMemoryTuner.class); 595 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 596 new RegionServerStub(conf), new RegionServerAccountingStub(conf)); 597 long oldMemstoreSize = memStoreFlusher.memstoreSize; 598 long oldBlockCacheSize = blockCache.maxSize; 599 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 600 heapMemoryManager.start(choreService); 601 CustomHeapMemoryTuner.memstoreSize = 0.7f; 602 CustomHeapMemoryTuner.blockCacheSize = 0.3f; 603 // Allow the tuner to run once and do necessary memory up 604 Thread.sleep(1500); 605 assertEquals(oldMemstoreSize, memStoreFlusher.memstoreSize); 606 assertEquals(oldBlockCacheSize, blockCache.maxSize); 607 } 608 609 private void assertHeapSpace(float expectedHeapPercentage, long currentHeapSpace) { 610 long expected = (long) (this.maxHeapSize * expectedHeapPercentage); 611 assertEquals(expected, currentHeapSpace); 612 } 613 614 private void assertHeapSpaceDelta(double expectedDeltaPercent, long oldHeapSpace, 615 long newHeapSpace) { 616 double expctedMinDelta = (double) (this.maxHeapSize * expectedDeltaPercent); 617 // Tolerable error 618 double error = 0.95; 619 if (expectedDeltaPercent > 0) { 620 assertTrue(expctedMinDelta*error <= (double)(newHeapSpace - oldHeapSpace)); 621 assertTrue(expctedMinDelta/error >= (double)(newHeapSpace - oldHeapSpace)); 622 } else { 623 assertTrue(-expctedMinDelta*error <= (double)(oldHeapSpace - newHeapSpace)); 624 assertTrue(-expctedMinDelta/error >= (double)(oldHeapSpace - newHeapSpace)); 625 } 626 } 627 628 629 private void waitForTune(final MemstoreFlusherStub memStoreFlusher, 630 final long oldMemstoreHeapSize) throws Exception { 631 // Allow the tuner to run once and do necessary memory up 632 UTIL.waitFor(10000, new Waiter.Predicate<Exception>() { 633 @Override 634 public boolean evaluate() throws Exception { 635 return oldMemstoreHeapSize != memStoreFlusher.memstoreSize; 636 } 637 }); 638 } 639 640 private static class BlockCacheStub implements ResizableBlockCache { 641 CacheStats stats = new CacheStats("test"); 642 long maxSize = 0; 643 private long testBlockSize = 0; 644 645 public BlockCacheStub(long size){ 646 this.maxSize = size; 647 } 648 649 @Override 650 public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory) { 651 652 } 653 654 @Override 655 public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf) { 656 657 } 658 659 @Override 660 public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching, boolean repeat, 661 boolean updateCacheMetrics) { 662 return null; 663 } 664 665 @Override 666 public boolean evictBlock(BlockCacheKey cacheKey) { 667 stats.evicted(0, cacheKey != null ? cacheKey.isPrimary() : true); 668 return false; 669 } 670 671 @Override 672 public int evictBlocksByHfileName(String hfileName) { 673 stats.evicted(0, true); // Just assuming only one block for file here. 674 return 0; 675 } 676 677 @Override 678 public CacheStats getStats() { 679 return this.stats; 680 } 681 682 @Override 683 public void shutdown() { 684 685 } 686 687 @Override 688 public long size() { 689 return 0; 690 } 691 692 @Override 693 public long getMaxSize() { 694 return 0; 695 } 696 697 @Override 698 public long getFreeSize() { 699 return 0; 700 } 701 702 @Override 703 public long getCurrentSize() { 704 return this.testBlockSize; 705 } 706 707 @Override 708 public long getCurrentDataSize() { 709 return 0; 710 } 711 712 @Override 713 public long getBlockCount() { 714 return 0; 715 } 716 717 @Override 718 public long getDataBlockCount() { 719 return 0; 720 } 721 722 @Override 723 public void setMaxSize(long size) { 724 this.maxSize = size; 725 } 726 727 @Override 728 public Iterator<CachedBlock> iterator() { 729 return null; 730 } 731 732 @Override 733 public BlockCache[] getBlockCaches() { 734 return null; 735 } 736 737 public void setTestBlockSize(long testBlockSize) { 738 this.testBlockSize = testBlockSize; 739 } 740 } 741 742 private static class MemstoreFlusherStub implements FlushRequester { 743 744 long memstoreSize; 745 746 FlushRequestListener listener; 747 748 FlushType flushType = FlushType.NORMAL; 749 750 public MemstoreFlusherStub(long memstoreSize) { 751 this.memstoreSize = memstoreSize; 752 } 753 754 @Override 755 public boolean requestFlush(HRegion region, boolean forceFlushAllStores, 756 FlushLifeCycleTracker tracker) { 757 this.listener.flushRequested(flushType, region); 758 return true; 759 } 760 761 @Override 762 public boolean requestDelayedFlush(HRegion region, long delay, boolean forceFlushAllStores) { 763 return true; 764 } 765 766 @Override 767 public void registerFlushRequestListener(FlushRequestListener listener) { 768 this.listener = listener; 769 } 770 771 @Override 772 public boolean unregisterFlushRequestListener(FlushRequestListener listener) { 773 return false; 774 } 775 776 @Override 777 public void setGlobalMemStoreLimit(long globalMemStoreSize) { 778 this.memstoreSize = globalMemStoreSize; 779 } 780 } 781 782 private static class RegionServerStub implements Server { 783 private Configuration conf; 784 private boolean stopped = false; 785 786 public RegionServerStub(Configuration conf) { 787 this.conf = conf; 788 } 789 790 @Override 791 public void abort(String why, Throwable e) { 792 793 } 794 795 @Override 796 public boolean isAborted() { 797 return false; 798 } 799 800 @Override 801 public void stop(String why) { 802 this.stopped = true; 803 } 804 805 @Override 806 public boolean isStopped() { 807 return this.stopped; 808 } 809 810 @Override 811 public Configuration getConfiguration() { 812 return this.conf; 813 } 814 815 @Override 816 public ZKWatcher getZooKeeper() { 817 return null; 818 } 819 820 @Override 821 public CoordinatedStateManager getCoordinatedStateManager() { 822 return null; 823 } 824 825 @Override 826 public ClusterConnection getConnection() { 827 return null; 828 } 829 830 @Override 831 public ServerName getServerName() { 832 return ServerName.valueOf("server1",4000,12345); 833 } 834 835 @Override 836 public ChoreService getChoreService() { 837 return null; 838 } 839 840 @Override 841 public ClusterConnection getClusterConnection() { 842 // TODO Auto-generated method stub 843 return null; 844 } 845 846 @Override 847 public FileSystem getFileSystem() { 848 return null; 849 } 850 851 @Override 852 public boolean isStopping() { 853 return false; 854 } 855 856 @Override 857 public Connection createConnection(Configuration conf) throws IOException { 858 return null; 859 } 860 } 861 862 static class CustomHeapMemoryTuner implements HeapMemoryTuner { 863 static float blockCacheSize = 0.4f; 864 static float memstoreSize = 0.4f; 865 866 @Override 867 public Configuration getConf() { 868 return null; 869 } 870 871 @Override 872 public void setConf(Configuration arg0) { 873 874 } 875 876 @Override 877 public TunerResult tune(TunerContext context) { 878 TunerResult result = new TunerResult(true); 879 result.setBlockCacheSize(blockCacheSize); 880 result.setMemStoreSize(memstoreSize); 881 return result; 882 } 883 } 884 885 private static class RegionServerAccountingStub extends RegionServerAccounting { 886 boolean offheap; 887 888 public RegionServerAccountingStub(Configuration conf) { 889 super(conf); 890 } 891 892 public RegionServerAccountingStub(Configuration conf, boolean offheap) { 893 super(conf); 894 this.offheap = offheap; 895 } 896 897 private long testMemstoreSize = 0; 898 899 @Override 900 public long getGlobalMemStoreDataSize() { 901 return testMemstoreSize; 902 } 903 904 @Override 905 public long getGlobalMemStoreHeapSize() { 906 return testMemstoreSize; 907 } 908 909 @Override 910 public boolean isOffheap() { 911 return offheap; 912 } 913 914 public void setTestMemstoreSize(long testMemstoreSize) { 915 this.testMemstoreSize = testMemstoreSize; 916 } 917 } 918}