001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.regionserver; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertFalse; 022import static org.junit.Assert.assertTrue; 023import static org.junit.Assert.fail; 024 025import java.io.IOException; 026import java.lang.management.ManagementFactory; 027import java.util.Iterator; 028import org.apache.hadoop.conf.Configuration; 029import org.apache.hadoop.fs.FileSystem; 030import org.apache.hadoop.hbase.ChoreService; 031import org.apache.hadoop.hbase.CoordinatedStateManager; 032import org.apache.hadoop.hbase.HBaseClassTestRule; 033import org.apache.hadoop.hbase.HBaseConfiguration; 034import org.apache.hadoop.hbase.HBaseTestingUtility; 035import org.apache.hadoop.hbase.HConstants; 036import org.apache.hadoop.hbase.Server; 037import org.apache.hadoop.hbase.ServerName; 038import org.apache.hadoop.hbase.Waiter; 039import org.apache.hadoop.hbase.client.ClusterConnection; 040import org.apache.hadoop.hbase.client.Connection; 041import org.apache.hadoop.hbase.io.hfile.BlockCache; 042import org.apache.hadoop.hbase.io.hfile.BlockCacheKey; 043import org.apache.hadoop.hbase.io.hfile.CacheStats; 044import org.apache.hadoop.hbase.io.hfile.Cacheable; 045import org.apache.hadoop.hbase.io.hfile.CachedBlock; 046import org.apache.hadoop.hbase.io.hfile.ResizableBlockCache; 047import org.apache.hadoop.hbase.io.util.MemorySizeUtil; 048import org.apache.hadoop.hbase.regionserver.HeapMemoryManager.TunerContext; 049import org.apache.hadoop.hbase.regionserver.HeapMemoryManager.TunerResult; 050import org.apache.hadoop.hbase.testclassification.RegionServerTests; 051import org.apache.hadoop.hbase.testclassification.SmallTests; 052import org.apache.hadoop.hbase.zookeeper.ZKWatcher; 053import org.junit.ClassRule; 054import org.junit.Test; 055import org.junit.experimental.categories.Category; 056 057@Category({RegionServerTests.class, SmallTests.class}) 058public class TestHeapMemoryManager { 059 060 @ClassRule 061 public static final HBaseClassTestRule CLASS_RULE = 062 HBaseClassTestRule.forClass(TestHeapMemoryManager.class); 063 064 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); 065 066 private long maxHeapSize = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax(); 067 068 @Test 069 public void testAutoTunerShouldBeOffWhenMaxMinRangesForMemstoreIsNotGiven() throws Exception { 070 Configuration conf = HBaseConfiguration.create(); 071 conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_KEY, 0.02f); 072 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.75f); 073 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.03f); 074 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 075 HeapMemoryManager manager = new HeapMemoryManager(new BlockCacheStub(0), 076 new MemstoreFlusherStub(0), new RegionServerStub(conf), 077 regionServerAccounting); 078 assertFalse(manager.isTunerOn()); 079 } 080 081 @Test 082 public void testAutoTunerShouldBeOffWhenMaxMinRangesForBlockCacheIsNotGiven() throws Exception { 083 Configuration conf = HBaseConfiguration.create(); 084 conf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.02f); 085 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 086 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.03f); 087 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 088 HeapMemoryManager manager = new HeapMemoryManager(new BlockCacheStub(0), 089 new MemstoreFlusherStub(0), new RegionServerStub(conf), 090 regionServerAccounting); 091 assertFalse(manager.isTunerOn()); 092 } 093 094 @Test 095 public void testWhenMemstoreAndBlockCacheMaxMinChecksFails() throws Exception { 096 BlockCacheStub blockCache = new BlockCacheStub(0); 097 Configuration conf = HBaseConfiguration.create(); 098 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 099 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.06f); 100 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 101 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub(0); 102 try { 103 new HeapMemoryManager(blockCache, memStoreFlusher, 104 new RegionServerStub(conf), regionServerAccounting); 105 fail(); 106 } catch (RuntimeException e) { 107 } 108 conf = HBaseConfiguration.create(); 109 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.2f); 110 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 111 try { 112 new HeapMemoryManager(blockCache, memStoreFlusher, 113 new RegionServerStub(conf), regionServerAccounting); 114 fail(); 115 } catch (RuntimeException e) { 116 } 117 } 118 119 @Test 120 public void testWhenClusterIsWriteHeavyWithEmptyMemstore() throws Exception { 121 Configuration conf = HBaseConfiguration.create(); 122 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 123 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 124 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 125 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 126 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 127 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 128 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 129 MemstoreFlusherStub memStoreFlusher = 130 new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 131 // Empty block cache and memstore 132 blockCache.setTestBlockSize(0); 133 regionServerAccounting.setTestMemstoreSize(0); 134 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 135 // Let the system start with default values for memstore heap and block cache size. 136 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 137 new RegionServerStub(conf), regionServerAccounting); 138 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 139 long oldBlockCacheSize = blockCache.maxSize; 140 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 141 heapMemoryManager.start(choreService); 142 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_HIGHER_MARK; 143 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 144 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 145 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 146 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK; 147 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 148 // Allow the tuner to run once and do necessary memory up 149 Thread.sleep(1500); 150 // No changes should be made by tuner as we already have lot of empty space 151 assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 152 assertEquals(oldBlockCacheSize, blockCache.maxSize); 153 } 154 155 @Test 156 public void testHeapMemoryManagerWhenOffheapFlushesHappenUnderReadHeavyCase() throws Exception { 157 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 158 Configuration conf = HBaseConfiguration.create(); 159 conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_LOWER_LIMIT_KEY, 0.7f); 160 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 161 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 162 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 163 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 164 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 165 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 166 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf, true); 167 MemstoreFlusherStub memStoreFlusher = 168 new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 169 // Empty memstore and but nearly filled block cache 170 blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8)); 171 regionServerAccounting.setTestMemstoreSize(0); 172 // Let the system start with default values for memstore heap and block cache size. 173 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 174 new RegionServerStub(conf), regionServerAccounting); 175 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 176 long oldBlockCacheSize = blockCache.maxSize; 177 float maxStepValue = DefaultHeapMemoryTuner.DEFAULT_MIN_STEP_VALUE; 178 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 179 heapMemoryManager.start(choreService); 180 blockCache.evictBlock(null); 181 blockCache.evictBlock(null); 182 blockCache.evictBlock(null); 183 // do some offheap flushes also. So there should be decrease in memstore but 184 // not as that when we don't have offheap flushes 185 memStoreFlusher.flushType = FlushType.ABOVE_OFFHEAP_HIGHER_MARK; 186 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 187 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 188 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 189 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 190 // Allow the tuner to run once and do necessary memory up 191 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 192 assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 193 assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize); 194 oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 195 oldBlockCacheSize = blockCache.maxSize; 196 // Do some more evictions before the next run of HeapMemoryTuner 197 blockCache.evictBlock(null); 198 // Allow the tuner to run once and do necessary memory up 199 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 200 assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 201 assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize); 202 } 203 204 @Test 205 public void testHeapMemoryManagerWithOffheapMemstoreAndMixedWorkload() throws Exception { 206 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 207 Configuration conf = HBaseConfiguration.create(); 208 conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_LOWER_LIMIT_KEY, 0.7f); 209 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 210 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 211 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 212 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 213 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 214 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 215 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf, true); 216 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 217 // Empty memstore and but nearly filled block cache 218 blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8)); 219 regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8)); 220 // Let the system start with default values for memstore heap and block cache size. 221 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 222 new RegionServerStub(conf), regionServerAccounting); 223 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 224 long oldBlockCacheSize = blockCache.maxSize; 225 float maxStepValue = DefaultHeapMemoryTuner.DEFAULT_MIN_STEP_VALUE; 226 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 227 heapMemoryManager.start(choreService); 228 blockCache.evictBlock(null); 229 blockCache.evictBlock(null); 230 blockCache.evictBlock(null); 231 // do some offheap flushes also. So there should be decrease in memstore but 232 // not as that when we don't have offheap flushes 233 memStoreFlusher.flushType = FlushType.ABOVE_OFFHEAP_HIGHER_MARK; 234 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 235 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 236 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 237 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 238 // Allow the tuner to run once and do necessary memory up 239 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 240 assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 241 assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize); 242 oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 243 oldBlockCacheSize = blockCache.maxSize; 244 // change memstore size 245 // regionServerAccounting.setTestMemstoreSize((long)(maxHeapSize * 0.4 * 0.8)); 246 // The memstore size would have decreased. Now again do some flushes and ensure the 247 // flushes are due to onheap overhead. This should once again call for increase in 248 // memstore size but that increase should be to the safe size 249 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_HIGHER_MARK; 250 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 251 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 252 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 253 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 254 // Allow the tuner to run once and do necessary memory up 255 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 256 assertHeapSpaceDelta(maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 257 assertHeapSpaceDelta(-maxStepValue, oldBlockCacheSize, blockCache.maxSize); 258 } 259 260 @Test 261 public void testWhenClusterIsReadHeavyWithEmptyBlockCache() throws Exception { 262 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 263 Configuration conf = HBaseConfiguration.create(); 264 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 265 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 266 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 267 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 268 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 269 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 270 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 271 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 272 // Empty block cache and memstore 273 blockCache.setTestBlockSize(0); 274 regionServerAccounting.setTestMemstoreSize(0); 275 // Let the system start with default values for memstore heap and block cache size. 276 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 277 new RegionServerStub(conf), regionServerAccounting); 278 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 279 long oldBlockCacheSize = blockCache.maxSize; 280 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 281 heapMemoryManager.start(choreService); 282 blockCache.evictBlock(null); 283 blockCache.evictBlock(null); 284 blockCache.evictBlock(null); 285 // Allow the tuner to run once and do necessary memory up 286 Thread.sleep(1500); 287 // No changes should be made by tuner as we already have lot of empty space 288 assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 289 assertEquals(oldBlockCacheSize, blockCache.maxSize); 290 } 291 292 @Test 293 public void testWhenClusterIsWriteHeavy() throws Exception { 294 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 295 Configuration conf = HBaseConfiguration.create(); 296 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 297 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 298 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 299 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 300 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 301 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 302 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 303 MemstoreFlusherStub memStoreFlusher = 304 new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 305 // Empty block cache and but nearly filled memstore 306 blockCache.setTestBlockSize(0); 307 regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8)); 308 // Let the system start with default values for memstore heap and block cache size. 309 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 310 new RegionServerStub(conf), regionServerAccounting); 311 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 312 long oldBlockCacheSize = blockCache.maxSize; 313 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 314 heapMemoryManager.start(choreService); 315 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK; 316 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 317 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 318 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 319 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 320 // Allow the tuner to run once and do necessary memory up 321 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 322 assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldMemstoreHeapSize, 323 memStoreFlusher.memstoreSize); 324 assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldBlockCacheSize, 325 blockCache.maxSize); 326 oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 327 oldBlockCacheSize = blockCache.maxSize; 328 // Do some more flushes before the next run of HeapMemoryTuner 329 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK; 330 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 331 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 332 // Allow the tuner to run once and do necessary memory up 333 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 334 assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldMemstoreHeapSize, 335 memStoreFlusher.memstoreSize); 336 assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldBlockCacheSize, 337 blockCache.maxSize); 338 } 339 340 @Test 341 public void testWhenClusterIsWriteHeavyWithOffheapMemstore() throws Exception { 342 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 343 Configuration conf = HBaseConfiguration.create(); 344 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 345 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 346 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 347 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 348 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 349 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 350 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 351 MemstoreFlusherStub memStoreFlusher = 352 new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 353 // Empty block cache and but nearly filled memstore 354 blockCache.setTestBlockSize(0); 355 regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8)); 356 // Let the system start with default values for memstore heap and block cache size. 357 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 358 new RegionServerStub(conf), regionServerAccounting); 359 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 360 long oldBlockCacheSize = blockCache.maxSize; 361 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 362 heapMemoryManager.start(choreService); 363 // this should not change anything with onheap memstore 364 memStoreFlusher.flushType = FlushType.ABOVE_OFFHEAP_HIGHER_MARK; 365 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 366 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 367 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 368 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 369 // Allow the tuner to run once and do necessary memory up 370 Thread.sleep(1500); 371 // No changes should be made by tuner as we already have lot of empty space 372 assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 373 assertEquals(oldBlockCacheSize, blockCache.maxSize); 374 } 375 376 @Test 377 public void testWhenClusterIsReadHeavy() throws Exception { 378 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 379 Configuration conf = HBaseConfiguration.create(); 380 conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_LOWER_LIMIT_KEY, 0.7f); 381 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 382 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 383 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 384 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 385 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 386 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 387 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 388 MemstoreFlusherStub memStoreFlusher = 389 new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 390 // Empty memstore and but nearly filled block cache 391 blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8)); 392 regionServerAccounting.setTestMemstoreSize(0); 393 // Let the system start with default values for memstore heap and block cache size. 394 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 395 new RegionServerStub(conf), new RegionServerAccountingStub(conf)); 396 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 397 long oldBlockCacheSize = blockCache.maxSize; 398 long oldMemstoreLowerMarkSize = 7 * oldMemstoreHeapSize / 10; 399 long maxTuneSize = oldMemstoreHeapSize - (oldMemstoreLowerMarkSize + oldMemstoreHeapSize) / 2; 400 float maxStepValue = (maxTuneSize * 1.0f) / oldMemstoreHeapSize; 401 maxStepValue = maxStepValue > DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE ? 402 DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE:maxStepValue; 403 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 404 heapMemoryManager.start(choreService); 405 blockCache.evictBlock(null); 406 blockCache.evictBlock(null); 407 blockCache.evictBlock(null); 408 // Allow the tuner to run once and do necessary memory up 409 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 410 assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 411 assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize); 412 oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 413 oldBlockCacheSize = blockCache.maxSize; 414 oldMemstoreLowerMarkSize = 7 * oldMemstoreHeapSize / 10; 415 maxTuneSize = oldMemstoreHeapSize - (oldMemstoreLowerMarkSize + oldMemstoreHeapSize) / 2; 416 maxStepValue = (maxTuneSize * 1.0f) / oldMemstoreHeapSize; 417 maxStepValue = maxStepValue > DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE ? 418 DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE:maxStepValue; 419 // Do some more evictions before the next run of HeapMemoryTuner 420 blockCache.evictBlock(null); 421 // Allow the tuner to run once and do necessary memory up 422 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 423 assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 424 assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize); 425 } 426 427 @Test 428 public void testWhenClusterIsHavingMoreWritesThanReads() throws Exception { 429 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 430 Configuration conf = HBaseConfiguration.create(); 431 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 432 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 433 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 434 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 435 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 436 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 437 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 438 MemstoreFlusherStub memStoreFlusher = 439 new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 440 // Both memstore and block cache are nearly filled 441 blockCache.setTestBlockSize(0); 442 regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8)); 443 blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8)); 444 // Let the system start with default values for memstore heap and block cache size. 445 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 446 new RegionServerStub(conf), regionServerAccounting); 447 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 448 long oldBlockCacheSize = blockCache.maxSize; 449 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 450 heapMemoryManager.start(choreService); 451 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK; 452 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 453 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 454 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 455 blockCache.evictBlock(null); 456 // Allow the tuner to run once and do necessary memory up 457 Thread.sleep(1500); 458 // No changes should happen as there is undefined increase in flushes and evictions 459 assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 460 assertEquals(oldBlockCacheSize, blockCache.maxSize); 461 // Do some more flushes before the next run of HeapMemoryTuner 462 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK; 463 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 464 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 465 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 466 // Allow the tuner to run once and do necessary memory up 467 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 468 assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldMemstoreHeapSize, 469 memStoreFlusher.memstoreSize); 470 assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldBlockCacheSize, 471 blockCache.maxSize); 472 } 473 474 @Test 475 public void testBlockedFlushesIncreaseMemstoreInSteadyState() throws Exception { 476 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 477 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 478 Configuration conf = HBaseConfiguration.create(); 479 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 480 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 481 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 482 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 483 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 484 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 485 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 486 // Both memstore and block cache are nearly filled 487 blockCache.setTestBlockSize(0); 488 regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8)); 489 blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8)); 490 // Let the system start with default values for memstore heap and block cache size. 491 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 492 new RegionServerStub(conf), regionServerAccounting); 493 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 494 long oldBlockCacheSize = blockCache.maxSize; 495 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 496 heapMemoryManager.start(choreService); 497 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK; 498 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 499 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 500 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 501 blockCache.evictBlock(null); 502 blockCache.evictBlock(null); 503 // Allow the tuner to run once and do necessary memory up 504 Thread.sleep(1500); 505 // No changes should happen as there is undefined increase in flushes and evictions 506 assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 507 assertEquals(oldBlockCacheSize, blockCache.maxSize); 508 // Flushes that block updates 509 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_HIGHER_MARK; 510 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 511 blockCache.evictBlock(null); 512 blockCache.evictBlock(null); 513 blockCache.evictBlock(null); 514 blockCache.evictBlock(null); 515 // Allow the tuner to run once and do necessary memory up 516 Thread.sleep(1500); 517 assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldMemstoreHeapSize, 518 memStoreFlusher.memstoreSize); 519 assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldBlockCacheSize, 520 blockCache.maxSize); 521 } 522 523 @Test 524 public void testPluggingInHeapMemoryTuner() throws Exception { 525 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 526 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 527 Configuration conf = HBaseConfiguration.create(); 528 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.78f); 529 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.05f); 530 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.75f); 531 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.02f); 532 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 533 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 534 conf.setClass(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_CLASS, CustomHeapMemoryTuner.class, 535 HeapMemoryTuner.class); 536 // Let the system start with default values for memstore heap and block cache size. 537 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 538 new RegionServerStub(conf), new RegionServerAccountingStub(conf)); 539 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 540 heapMemoryManager.start(choreService); 541 // Now we wants to be in write mode. Set bigger memstore size from CustomHeapMemoryTuner 542 CustomHeapMemoryTuner.memstoreSize = 0.78f; 543 CustomHeapMemoryTuner.blockCacheSize = 0.02f; 544 // Allow the tuner to run once and do necessary memory up 545 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 546 assertHeapSpace(0.78f, memStoreFlusher.memstoreSize);// Memstore 547 assertHeapSpace(0.02f, blockCache.maxSize);// BlockCache 548 // Now we wants to be in read mode. Set bigger memstore size from CustomHeapMemoryTuner 549 CustomHeapMemoryTuner.blockCacheSize = 0.75f; 550 CustomHeapMemoryTuner.memstoreSize = 0.05f; 551 // Allow the tuner to run once and do necessary memory up 552 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 553 assertHeapSpace(0.75f, blockCache.maxSize);// BlockCache 554 assertHeapSpace(0.05f, memStoreFlusher.memstoreSize);// Memstore 555 } 556 557 @Test 558 public void testWhenSizeGivenByHeapTunerGoesOutsideRange() throws Exception { 559 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 560 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 561 Configuration conf = HBaseConfiguration.create(); 562 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.7f); 563 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.1f); 564 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 565 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.1f); 566 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 567 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 568 conf.setClass(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_CLASS, CustomHeapMemoryTuner.class, 569 HeapMemoryTuner.class); 570 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 571 new RegionServerStub(conf), new RegionServerAccountingStub(conf)); 572 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 573 heapMemoryManager.start(choreService); 574 CustomHeapMemoryTuner.memstoreSize = 0.78f; 575 CustomHeapMemoryTuner.blockCacheSize = 0.02f; 576 Thread.sleep(1500); // Allow the tuner to run once and do necessary memory up 577 // Even if the tuner says to set the memstore to 78%, HBase makes it as 70% as that is the 578 // upper bound. Same with block cache as 10% is the lower bound. 579 assertHeapSpace(0.7f, memStoreFlusher.memstoreSize); 580 assertHeapSpace(0.1f, blockCache.maxSize); 581 } 582 583 @Test 584 public void testWhenCombinedHeapSizesFromTunerGoesOutSideMaxLimit() throws Exception { 585 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 586 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 587 Configuration conf = HBaseConfiguration.create(); 588 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.7f); 589 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.1f); 590 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 591 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.1f); 592 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 593 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 594 conf.setClass(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_CLASS, CustomHeapMemoryTuner.class, 595 HeapMemoryTuner.class); 596 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 597 new RegionServerStub(conf), new RegionServerAccountingStub(conf)); 598 long oldMemstoreSize = memStoreFlusher.memstoreSize; 599 long oldBlockCacheSize = blockCache.maxSize; 600 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 601 heapMemoryManager.start(choreService); 602 CustomHeapMemoryTuner.memstoreSize = 0.7f; 603 CustomHeapMemoryTuner.blockCacheSize = 0.3f; 604 // Allow the tuner to run once and do necessary memory up 605 Thread.sleep(1500); 606 assertEquals(oldMemstoreSize, memStoreFlusher.memstoreSize); 607 assertEquals(oldBlockCacheSize, blockCache.maxSize); 608 } 609 610 private void assertHeapSpace(float expectedHeapPercentage, long currentHeapSpace) { 611 long expected = (long) (this.maxHeapSize * expectedHeapPercentage); 612 assertEquals(expected, currentHeapSpace); 613 } 614 615 private void assertHeapSpaceDelta(double expectedDeltaPercent, long oldHeapSpace, 616 long newHeapSpace) { 617 double expctedMinDelta = (double) (this.maxHeapSize * expectedDeltaPercent); 618 // Tolerable error 619 double error = 0.95; 620 if (expectedDeltaPercent > 0) { 621 assertTrue(expctedMinDelta*error <= (double)(newHeapSpace - oldHeapSpace)); 622 assertTrue(expctedMinDelta/error >= (double)(newHeapSpace - oldHeapSpace)); 623 } else { 624 assertTrue(-expctedMinDelta*error <= (double)(oldHeapSpace - newHeapSpace)); 625 assertTrue(-expctedMinDelta/error >= (double)(oldHeapSpace - newHeapSpace)); 626 } 627 } 628 629 630 private void waitForTune(final MemstoreFlusherStub memStoreFlusher, 631 final long oldMemstoreHeapSize) throws Exception { 632 // Allow the tuner to run once and do necessary memory up 633 UTIL.waitFor(10000, new Waiter.Predicate<Exception>() { 634 @Override 635 public boolean evaluate() throws Exception { 636 return oldMemstoreHeapSize != memStoreFlusher.memstoreSize; 637 } 638 }); 639 } 640 641 private static class BlockCacheStub implements ResizableBlockCache { 642 CacheStats stats = new CacheStats("test"); 643 long maxSize = 0; 644 private long testBlockSize = 0; 645 646 public BlockCacheStub(long size){ 647 this.maxSize = size; 648 } 649 650 @Override 651 public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory) { 652 653 } 654 655 @Override 656 public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf) { 657 658 } 659 660 @Override 661 public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching, boolean repeat, 662 boolean updateCacheMetrics) { 663 return null; 664 } 665 666 @Override 667 public boolean evictBlock(BlockCacheKey cacheKey) { 668 stats.evicted(0, cacheKey != null ? cacheKey.isPrimary() : true); 669 return false; 670 } 671 672 @Override 673 public int evictBlocksByHfileName(String hfileName) { 674 stats.evicted(0, true); // Just assuming only one block for file here. 675 return 0; 676 } 677 678 @Override 679 public CacheStats getStats() { 680 return this.stats; 681 } 682 683 @Override 684 public void shutdown() { 685 686 } 687 688 @Override 689 public long size() { 690 return 0; 691 } 692 693 @Override 694 public long getMaxSize() { 695 return 0; 696 } 697 698 @Override 699 public long getFreeSize() { 700 return 0; 701 } 702 703 @Override 704 public long getCurrentSize() { 705 return this.testBlockSize; 706 } 707 708 @Override 709 public long getCurrentDataSize() { 710 return 0; 711 } 712 713 @Override 714 public long getBlockCount() { 715 return 0; 716 } 717 718 @Override 719 public long getDataBlockCount() { 720 return 0; 721 } 722 723 @Override 724 public void setMaxSize(long size) { 725 this.maxSize = size; 726 } 727 728 @Override 729 public Iterator<CachedBlock> iterator() { 730 return null; 731 } 732 733 @Override 734 public BlockCache[] getBlockCaches() { 735 return null; 736 } 737 738 @Override 739 public void returnBlock(BlockCacheKey cacheKey, Cacheable buf) { 740 } 741 742 public void setTestBlockSize(long testBlockSize) { 743 this.testBlockSize = testBlockSize; 744 } 745 } 746 747 private static class MemstoreFlusherStub implements FlushRequester { 748 749 long memstoreSize; 750 751 FlushRequestListener listener; 752 753 FlushType flushType = FlushType.NORMAL; 754 755 public MemstoreFlusherStub(long memstoreSize) { 756 this.memstoreSize = memstoreSize; 757 } 758 759 @Override 760 public boolean requestFlush(HRegion region, boolean forceFlushAllStores, 761 FlushLifeCycleTracker tracker) { 762 this.listener.flushRequested(flushType, region); 763 return true; 764 } 765 766 @Override 767 public boolean requestDelayedFlush(HRegion region, long delay, boolean forceFlushAllStores) { 768 return true; 769 } 770 771 @Override 772 public void registerFlushRequestListener(FlushRequestListener listener) { 773 this.listener = listener; 774 } 775 776 @Override 777 public boolean unregisterFlushRequestListener(FlushRequestListener listener) { 778 return false; 779 } 780 781 @Override 782 public void setGlobalMemStoreLimit(long globalMemStoreSize) { 783 this.memstoreSize = globalMemStoreSize; 784 } 785 } 786 787 private static class RegionServerStub implements Server { 788 private Configuration conf; 789 private boolean stopped = false; 790 791 public RegionServerStub(Configuration conf) { 792 this.conf = conf; 793 } 794 795 @Override 796 public void abort(String why, Throwable e) { 797 798 } 799 800 @Override 801 public boolean isAborted() { 802 return false; 803 } 804 805 @Override 806 public void stop(String why) { 807 this.stopped = true; 808 } 809 810 @Override 811 public boolean isStopped() { 812 return this.stopped; 813 } 814 815 @Override 816 public Configuration getConfiguration() { 817 return this.conf; 818 } 819 820 @Override 821 public ZKWatcher getZooKeeper() { 822 return null; 823 } 824 825 @Override 826 public CoordinatedStateManager getCoordinatedStateManager() { 827 return null; 828 } 829 830 @Override 831 public ClusterConnection getConnection() { 832 return null; 833 } 834 835 @Override 836 public ServerName getServerName() { 837 return ServerName.valueOf("server1",4000,12345); 838 } 839 840 @Override 841 public ChoreService getChoreService() { 842 return null; 843 } 844 845 @Override 846 public ClusterConnection getClusterConnection() { 847 // TODO Auto-generated method stub 848 return null; 849 } 850 851 @Override 852 public FileSystem getFileSystem() { 853 return null; 854 } 855 856 @Override 857 public boolean isStopping() { 858 return false; 859 } 860 861 @Override 862 public Connection createConnection(Configuration conf) throws IOException { 863 return null; 864 } 865 } 866 867 static class CustomHeapMemoryTuner implements HeapMemoryTuner { 868 static float blockCacheSize = 0.4f; 869 static float memstoreSize = 0.4f; 870 871 @Override 872 public Configuration getConf() { 873 return null; 874 } 875 876 @Override 877 public void setConf(Configuration arg0) { 878 879 } 880 881 @Override 882 public TunerResult tune(TunerContext context) { 883 TunerResult result = new TunerResult(true); 884 result.setBlockCacheSize(blockCacheSize); 885 result.setMemStoreSize(memstoreSize); 886 return result; 887 } 888 } 889 890 private static class RegionServerAccountingStub extends RegionServerAccounting { 891 boolean offheap; 892 893 public RegionServerAccountingStub(Configuration conf) { 894 super(conf); 895 } 896 897 public RegionServerAccountingStub(Configuration conf, boolean offheap) { 898 super(conf); 899 this.offheap = offheap; 900 } 901 902 private long testMemstoreSize = 0; 903 904 @Override 905 public long getGlobalMemStoreDataSize() { 906 return testMemstoreSize; 907 } 908 909 @Override 910 public long getGlobalMemStoreHeapSize() { 911 return testMemstoreSize; 912 } 913 914 @Override 915 public boolean isOffheap() { 916 return offheap; 917 } 918 919 public void setTestMemstoreSize(long testMemstoreSize) { 920 this.testMemstoreSize = testMemstoreSize; 921 } 922 } 923}