001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.regionserver; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertFalse; 022import static org.junit.Assert.assertTrue; 023import static org.junit.Assert.fail; 024 025import java.io.IOException; 026import java.lang.management.ManagementFactory; 027import java.util.Iterator; 028import java.util.List; 029import org.apache.hadoop.conf.Configuration; 030import org.apache.hadoop.fs.FileSystem; 031import org.apache.hadoop.hbase.ChoreService; 032import org.apache.hadoop.hbase.CoordinatedStateManager; 033import org.apache.hadoop.hbase.HBaseClassTestRule; 034import org.apache.hadoop.hbase.HBaseConfiguration; 035import org.apache.hadoop.hbase.HBaseTestingUtil; 036import org.apache.hadoop.hbase.HConstants; 037import org.apache.hadoop.hbase.Server; 038import org.apache.hadoop.hbase.ServerName; 039import org.apache.hadoop.hbase.Waiter; 040import org.apache.hadoop.hbase.client.AsyncClusterConnection; 041import org.apache.hadoop.hbase.client.Connection; 042import org.apache.hadoop.hbase.io.hfile.BlockCache; 043import org.apache.hadoop.hbase.io.hfile.BlockCacheKey; 044import org.apache.hadoop.hbase.io.hfile.CacheStats; 045import org.apache.hadoop.hbase.io.hfile.Cacheable; 046import org.apache.hadoop.hbase.io.hfile.CachedBlock; 047import org.apache.hadoop.hbase.io.hfile.ResizableBlockCache; 048import org.apache.hadoop.hbase.io.util.MemorySizeUtil; 049import org.apache.hadoop.hbase.keymeta.KeyManagementService; 050import org.apache.hadoop.hbase.regionserver.HeapMemoryManager.TunerContext; 051import org.apache.hadoop.hbase.regionserver.HeapMemoryManager.TunerResult; 052import org.apache.hadoop.hbase.testclassification.MediumTests; 053import org.apache.hadoop.hbase.testclassification.RegionServerTests; 054import org.apache.hadoop.hbase.zookeeper.ZKWatcher; 055import org.junit.ClassRule; 056import org.junit.Test; 057import org.junit.experimental.categories.Category; 058 059@Category({ RegionServerTests.class, MediumTests.class }) 060public class TestHeapMemoryManager { 061 062 @ClassRule 063 public static final HBaseClassTestRule CLASS_RULE = 064 HBaseClassTestRule.forClass(TestHeapMemoryManager.class); 065 066 private static final HBaseTestingUtil UTIL = new HBaseTestingUtil(); 067 068 private long maxHeapSize = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax(); 069 070 @Test 071 public void testAutoTunerShouldBeOffWhenMaxMinRangesForMemstoreIsNotGiven() throws Exception { 072 Configuration conf = HBaseConfiguration.create(); 073 conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_KEY, 0.02f); 074 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.75f); 075 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.03f); 076 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 077 HeapMemoryManager manager = new HeapMemoryManager(new BlockCacheStub(0), 078 new MemstoreFlusherStub(0), new RegionServerStub(conf), regionServerAccounting); 079 assertFalse(manager.isTunerOn()); 080 } 081 082 @Test 083 public void testAutoTunerShouldBeOffWhenMaxMinRangesForBlockCacheIsNotGiven() throws Exception { 084 Configuration conf = HBaseConfiguration.create(); 085 conf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.02f); 086 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 087 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.03f); 088 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 089 HeapMemoryManager manager = new HeapMemoryManager(new BlockCacheStub(0), 090 new MemstoreFlusherStub(0), new RegionServerStub(conf), regionServerAccounting); 091 assertFalse(manager.isTunerOn()); 092 } 093 094 @Test 095 public void testWhenMemstoreAndBlockCacheMaxMinChecksFails() throws Exception { 096 BlockCacheStub blockCache = new BlockCacheStub(0); 097 Configuration conf = HBaseConfiguration.create(); 098 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 099 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.06f); 100 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 101 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub(0); 102 try { 103 new HeapMemoryManager(blockCache, memStoreFlusher, new RegionServerStub(conf), 104 regionServerAccounting); 105 fail(); 106 } catch (RuntimeException e) { 107 } 108 conf = HBaseConfiguration.create(); 109 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.2f); 110 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 111 try { 112 new HeapMemoryManager(blockCache, memStoreFlusher, new RegionServerStub(conf), 113 regionServerAccounting); 114 fail(); 115 } catch (RuntimeException e) { 116 } 117 } 118 119 @Test 120 public void testWhenClusterIsWriteHeavyWithEmptyMemstore() throws Exception { 121 Configuration conf = HBaseConfiguration.create(); 122 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 123 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 124 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 125 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 126 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 127 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 128 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 129 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 130 // Empty block cache and memstore 131 blockCache.setTestBlockSize(0); 132 regionServerAccounting.setTestMemstoreSize(0); 133 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 134 // Let the system start with default values for memstore heap and block cache size. 135 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 136 new RegionServerStub(conf), regionServerAccounting); 137 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 138 long oldBlockCacheSize = blockCache.maxSize; 139 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 140 heapMemoryManager.start(choreService); 141 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_HIGHER_MARK; 142 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 143 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 144 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 145 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK; 146 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 147 // Allow the tuner to run once and do necessary memory up 148 Thread.sleep(1500); 149 // No changes should be made by tuner as we already have lot of empty space 150 assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 151 assertEquals(oldBlockCacheSize, blockCache.maxSize); 152 } 153 154 @Test 155 public void testHeapMemoryManagerWhenOffheapFlushesHappenUnderReadHeavyCase() throws Exception { 156 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 157 Configuration conf = HBaseConfiguration.create(); 158 conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_LOWER_LIMIT_KEY, 0.7f); 159 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 160 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 161 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 162 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 163 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 164 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 165 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf, true); 166 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 167 // Empty memstore and but nearly filled block cache 168 blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8)); 169 regionServerAccounting.setTestMemstoreSize(0); 170 // Let the system start with default values for memstore heap and block cache size. 171 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 172 new RegionServerStub(conf), regionServerAccounting); 173 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 174 long oldBlockCacheSize = blockCache.maxSize; 175 float maxStepValue = DefaultHeapMemoryTuner.DEFAULT_MIN_STEP_VALUE; 176 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 177 heapMemoryManager.start(choreService); 178 blockCache.evictBlock(null); 179 blockCache.evictBlock(null); 180 blockCache.evictBlock(null); 181 // do some offheap flushes also. So there should be decrease in memstore but 182 // not as that when we don't have offheap flushes 183 memStoreFlusher.flushType = FlushType.ABOVE_OFFHEAP_HIGHER_MARK; 184 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 185 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 186 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 187 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 188 // Allow the tuner to run once and do necessary memory up 189 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 190 assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 191 assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize); 192 oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 193 oldBlockCacheSize = blockCache.maxSize; 194 // Do some more evictions before the next run of HeapMemoryTuner 195 blockCache.evictBlock(null); 196 // Allow the tuner to run once and do necessary memory up 197 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 198 assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 199 assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize); 200 } 201 202 @Test 203 public void testHeapMemoryManagerWithOffheapMemstoreAndMixedWorkload() throws Exception { 204 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 205 Configuration conf = HBaseConfiguration.create(); 206 conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_LOWER_LIMIT_KEY, 0.7f); 207 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 208 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 209 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 210 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 211 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 212 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 213 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf, true); 214 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 215 // Empty memstore and but nearly filled block cache 216 blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8)); 217 regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8)); 218 // Let the system start with default values for memstore heap and block cache size. 219 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 220 new RegionServerStub(conf), regionServerAccounting); 221 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 222 long oldBlockCacheSize = blockCache.maxSize; 223 float maxStepValue = DefaultHeapMemoryTuner.DEFAULT_MIN_STEP_VALUE; 224 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 225 heapMemoryManager.start(choreService); 226 blockCache.evictBlock(null); 227 blockCache.evictBlock(null); 228 blockCache.evictBlock(null); 229 // do some offheap flushes also. So there should be decrease in memstore but 230 // not as that when we don't have offheap flushes 231 memStoreFlusher.flushType = FlushType.ABOVE_OFFHEAP_HIGHER_MARK; 232 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 233 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 234 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 235 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 236 // Allow the tuner to run once and do necessary memory up 237 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 238 assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 239 assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize); 240 oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 241 oldBlockCacheSize = blockCache.maxSize; 242 // change memstore size 243 // regionServerAccounting.setTestMemstoreSize((long)(maxHeapSize * 0.4 * 0.8)); 244 // The memstore size would have decreased. Now again do some flushes and ensure the 245 // flushes are due to onheap overhead. This should once again call for increase in 246 // memstore size but that increase should be to the safe size 247 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_HIGHER_MARK; 248 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 249 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 250 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 251 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 252 // Allow the tuner to run once and do necessary memory up 253 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 254 assertHeapSpaceDelta(maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 255 assertHeapSpaceDelta(-maxStepValue, oldBlockCacheSize, blockCache.maxSize); 256 } 257 258 @Test 259 public void testWhenClusterIsReadHeavyWithEmptyBlockCache() throws Exception { 260 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 261 Configuration conf = HBaseConfiguration.create(); 262 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 263 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 264 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 265 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 266 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 267 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 268 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 269 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 270 // Empty block cache and memstore 271 blockCache.setTestBlockSize(0); 272 regionServerAccounting.setTestMemstoreSize(0); 273 // Let the system start with default values for memstore heap and block cache size. 274 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 275 new RegionServerStub(conf), regionServerAccounting); 276 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 277 long oldBlockCacheSize = blockCache.maxSize; 278 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 279 heapMemoryManager.start(choreService); 280 blockCache.evictBlock(null); 281 blockCache.evictBlock(null); 282 blockCache.evictBlock(null); 283 // Allow the tuner to run once and do necessary memory up 284 Thread.sleep(1500); 285 // No changes should be made by tuner as we already have lot of empty space 286 assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 287 assertEquals(oldBlockCacheSize, blockCache.maxSize); 288 } 289 290 @Test 291 public void testWhenClusterIsWriteHeavy() throws Exception { 292 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 293 Configuration conf = HBaseConfiguration.create(); 294 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 295 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 296 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 297 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 298 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 299 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 300 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 301 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 302 // Empty block cache and but nearly filled memstore 303 blockCache.setTestBlockSize(0); 304 regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8)); 305 // Let the system start with default values for memstore heap and block cache size. 306 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 307 new RegionServerStub(conf), regionServerAccounting); 308 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 309 long oldBlockCacheSize = blockCache.maxSize; 310 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 311 heapMemoryManager.start(choreService); 312 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK; 313 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 314 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 315 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 316 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 317 // Allow the tuner to run once and do necessary memory up 318 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 319 assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldMemstoreHeapSize, 320 memStoreFlusher.memstoreSize); 321 assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldBlockCacheSize, 322 blockCache.maxSize); 323 oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 324 oldBlockCacheSize = blockCache.maxSize; 325 // Do some more flushes before the next run of HeapMemoryTuner 326 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK; 327 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 328 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 329 // Allow the tuner to run once and do necessary memory up 330 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 331 assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldMemstoreHeapSize, 332 memStoreFlusher.memstoreSize); 333 assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldBlockCacheSize, 334 blockCache.maxSize); 335 } 336 337 @Test 338 public void testWhenClusterIsWriteHeavyWithOffheapMemstore() throws Exception { 339 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 340 Configuration conf = HBaseConfiguration.create(); 341 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 342 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 343 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 344 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 345 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 346 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 347 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 348 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 349 // Empty block cache and but nearly filled memstore 350 blockCache.setTestBlockSize(0); 351 regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8)); 352 // Let the system start with default values for memstore heap and block cache size. 353 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 354 new RegionServerStub(conf), regionServerAccounting); 355 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 356 long oldBlockCacheSize = blockCache.maxSize; 357 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 358 heapMemoryManager.start(choreService); 359 // this should not change anything with onheap memstore 360 memStoreFlusher.flushType = FlushType.ABOVE_OFFHEAP_HIGHER_MARK; 361 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 362 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 363 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 364 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 365 // Allow the tuner to run once and do necessary memory up 366 Thread.sleep(1500); 367 // No changes should be made by tuner as we already have lot of empty space 368 assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 369 assertEquals(oldBlockCacheSize, blockCache.maxSize); 370 } 371 372 @Test 373 public void testWhenClusterIsReadHeavy() throws Exception { 374 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 375 Configuration conf = HBaseConfiguration.create(); 376 conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_LOWER_LIMIT_KEY, 0.7f); 377 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 378 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 379 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 380 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 381 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 382 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 383 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 384 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 385 // Empty memstore and but nearly filled block cache 386 blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8)); 387 regionServerAccounting.setTestMemstoreSize(0); 388 // Let the system start with default values for memstore heap and block cache size. 389 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 390 new RegionServerStub(conf), new RegionServerAccountingStub(conf)); 391 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 392 long oldBlockCacheSize = blockCache.maxSize; 393 long oldMemstoreLowerMarkSize = 7 * oldMemstoreHeapSize / 10; 394 long maxTuneSize = oldMemstoreHeapSize - (oldMemstoreLowerMarkSize + oldMemstoreHeapSize) / 2; 395 float maxStepValue = (maxTuneSize * 1.0f) / oldMemstoreHeapSize; 396 maxStepValue = maxStepValue > DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE 397 ? DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE 398 : maxStepValue; 399 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 400 heapMemoryManager.start(choreService); 401 blockCache.evictBlock(null); 402 blockCache.evictBlock(null); 403 blockCache.evictBlock(null); 404 // Allow the tuner to run once and do necessary memory up 405 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 406 assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 407 assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize); 408 oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 409 oldBlockCacheSize = blockCache.maxSize; 410 oldMemstoreLowerMarkSize = 7 * oldMemstoreHeapSize / 10; 411 maxTuneSize = oldMemstoreHeapSize - (oldMemstoreLowerMarkSize + oldMemstoreHeapSize) / 2; 412 maxStepValue = (maxTuneSize * 1.0f) / oldMemstoreHeapSize; 413 maxStepValue = maxStepValue > DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE 414 ? DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE 415 : maxStepValue; 416 // Do some more evictions before the next run of HeapMemoryTuner 417 blockCache.evictBlock(null); 418 // Allow the tuner to run once and do necessary memory up 419 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 420 assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 421 assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize); 422 } 423 424 @Test 425 public void testWhenClusterIsHavingMoreWritesThanReads() throws Exception { 426 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 427 Configuration conf = HBaseConfiguration.create(); 428 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 429 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 430 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 431 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 432 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 433 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 434 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 435 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 436 // Both memstore and block cache are nearly filled 437 blockCache.setTestBlockSize(0); 438 regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8)); 439 blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8)); 440 // Let the system start with default values for memstore heap and block cache size. 441 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 442 new RegionServerStub(conf), regionServerAccounting); 443 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 444 long oldBlockCacheSize = blockCache.maxSize; 445 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 446 heapMemoryManager.start(choreService); 447 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK; 448 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 449 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 450 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 451 blockCache.evictBlock(null); 452 // Allow the tuner to run once and do necessary memory up 453 Thread.sleep(1500); 454 // No changes should happen as there is undefined increase in flushes and evictions 455 assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 456 assertEquals(oldBlockCacheSize, blockCache.maxSize); 457 // Do some more flushes before the next run of HeapMemoryTuner 458 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK; 459 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 460 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 461 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 462 // Allow the tuner to run once and do necessary memory up 463 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 464 assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldMemstoreHeapSize, 465 memStoreFlusher.memstoreSize); 466 assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldBlockCacheSize, 467 blockCache.maxSize); 468 } 469 470 @Test 471 public void testBlockedFlushesIncreaseMemstoreInSteadyState() throws Exception { 472 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 473 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 474 Configuration conf = HBaseConfiguration.create(); 475 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 476 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 477 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 478 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 479 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 480 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 481 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 482 // Both memstore and block cache are nearly filled 483 blockCache.setTestBlockSize(0); 484 regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8)); 485 blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8)); 486 // Let the system start with default values for memstore heap and block cache size. 487 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 488 new RegionServerStub(conf), regionServerAccounting); 489 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 490 long oldBlockCacheSize = blockCache.maxSize; 491 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 492 heapMemoryManager.start(choreService); 493 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK; 494 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 495 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 496 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 497 blockCache.evictBlock(null); 498 blockCache.evictBlock(null); 499 // Allow the tuner to run once and do necessary memory up 500 Thread.sleep(1500); 501 // No changes should happen as there is undefined increase in flushes and evictions 502 assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 503 assertEquals(oldBlockCacheSize, blockCache.maxSize); 504 // Flushes that block updates 505 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_HIGHER_MARK; 506 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 507 blockCache.evictBlock(null); 508 blockCache.evictBlock(null); 509 blockCache.evictBlock(null); 510 blockCache.evictBlock(null); 511 // Allow the tuner to run once and do necessary memory up 512 Thread.sleep(1500); 513 assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldMemstoreHeapSize, 514 memStoreFlusher.memstoreSize); 515 assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldBlockCacheSize, 516 blockCache.maxSize); 517 } 518 519 @Test 520 public void testPluggingInHeapMemoryTuner() throws Exception { 521 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 522 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 523 Configuration conf = HBaseConfiguration.create(); 524 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.78f); 525 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.05f); 526 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.75f); 527 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.02f); 528 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 529 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 530 conf.setClass(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_CLASS, CustomHeapMemoryTuner.class, 531 HeapMemoryTuner.class); 532 // Let the system start with default values for memstore heap and block cache size. 533 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 534 new RegionServerStub(conf), new RegionServerAccountingStub(conf)); 535 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 536 heapMemoryManager.start(choreService); 537 // Now we wants to be in write mode. Set bigger memstore size from CustomHeapMemoryTuner 538 CustomHeapMemoryTuner.memstoreSize = 0.78f; 539 CustomHeapMemoryTuner.blockCacheSize = 0.02f; 540 // Allow the tuner to run once and do necessary memory up 541 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 542 assertHeapSpace(0.78f, memStoreFlusher.memstoreSize);// Memstore 543 assertHeapSpace(0.02f, blockCache.maxSize);// BlockCache 544 // Now we wants to be in read mode. Set bigger memstore size from CustomHeapMemoryTuner 545 CustomHeapMemoryTuner.blockCacheSize = 0.75f; 546 CustomHeapMemoryTuner.memstoreSize = 0.05f; 547 // Allow the tuner to run once and do necessary memory up 548 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 549 assertHeapSpace(0.75f, blockCache.maxSize);// BlockCache 550 assertHeapSpace(0.05f, memStoreFlusher.memstoreSize);// Memstore 551 } 552 553 @Test 554 public void testWhenSizeGivenByHeapTunerGoesOutsideRange() throws Exception { 555 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 556 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 557 Configuration conf = HBaseConfiguration.create(); 558 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.7f); 559 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.1f); 560 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 561 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.1f); 562 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 563 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 564 conf.setClass(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_CLASS, CustomHeapMemoryTuner.class, 565 HeapMemoryTuner.class); 566 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 567 new RegionServerStub(conf), new RegionServerAccountingStub(conf)); 568 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 569 heapMemoryManager.start(choreService); 570 CustomHeapMemoryTuner.memstoreSize = 0.78f; 571 CustomHeapMemoryTuner.blockCacheSize = 0.02f; 572 Thread.sleep(1500); // Allow the tuner to run once and do necessary memory up 573 // Even if the tuner says to set the memstore to 78%, HBase makes it as 70% as that is the 574 // upper bound. Same with block cache as 10% is the lower bound. 575 assertHeapSpace(0.7f, memStoreFlusher.memstoreSize); 576 assertHeapSpace(0.1f, blockCache.maxSize); 577 } 578 579 @Test 580 public void testWhenCombinedHeapSizesFromTunerGoesOutSideMaxLimit() throws Exception { 581 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 582 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 583 Configuration conf = HBaseConfiguration.create(); 584 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.7f); 585 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.1f); 586 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 587 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.1f); 588 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 589 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 590 conf.setClass(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_CLASS, CustomHeapMemoryTuner.class, 591 HeapMemoryTuner.class); 592 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 593 new RegionServerStub(conf), new RegionServerAccountingStub(conf)); 594 long oldMemstoreSize = memStoreFlusher.memstoreSize; 595 long oldBlockCacheSize = blockCache.maxSize; 596 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 597 heapMemoryManager.start(choreService); 598 CustomHeapMemoryTuner.memstoreSize = 0.7f; 599 CustomHeapMemoryTuner.blockCacheSize = 0.3f; 600 // Allow the tuner to run once and do necessary memory up 601 Thread.sleep(1500); 602 assertEquals(oldMemstoreSize, memStoreFlusher.memstoreSize); 603 assertEquals(oldBlockCacheSize, blockCache.maxSize); 604 } 605 606 private void assertHeapSpace(float expectedHeapPercentage, long currentHeapSpace) { 607 long expected = (long) (this.maxHeapSize * expectedHeapPercentage); 608 assertEquals(expected, currentHeapSpace); 609 } 610 611 private void assertHeapSpaceDelta(double expectedDeltaPercent, long oldHeapSpace, 612 long newHeapSpace) { 613 double expctedMinDelta = (double) (this.maxHeapSize * expectedDeltaPercent); 614 // Tolerable error 615 double error = 0.95; 616 if (expectedDeltaPercent > 0) { 617 assertTrue(expctedMinDelta * error <= (double) (newHeapSpace - oldHeapSpace)); 618 assertTrue(expctedMinDelta / error >= (double) (newHeapSpace - oldHeapSpace)); 619 } else { 620 assertTrue(-expctedMinDelta * error <= (double) (oldHeapSpace - newHeapSpace)); 621 assertTrue(-expctedMinDelta / error >= (double) (oldHeapSpace - newHeapSpace)); 622 } 623 } 624 625 private void waitForTune(final MemstoreFlusherStub memStoreFlusher, 626 final long oldMemstoreHeapSize) throws Exception { 627 // Allow the tuner to run once and do necessary memory up 628 UTIL.waitFor(10000, new Waiter.Predicate<Exception>() { 629 @Override 630 public boolean evaluate() throws Exception { 631 return oldMemstoreHeapSize != memStoreFlusher.memstoreSize; 632 } 633 }); 634 } 635 636 private static class BlockCacheStub implements ResizableBlockCache { 637 CacheStats stats = new CacheStats("test"); 638 long maxSize = 0; 639 private long testBlockSize = 0; 640 641 public BlockCacheStub(long size) { 642 this.maxSize = size; 643 } 644 645 @Override 646 public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory) { 647 648 } 649 650 @Override 651 public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf) { 652 653 } 654 655 @Override 656 public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching, boolean repeat, 657 boolean updateCacheMetrics) { 658 return null; 659 } 660 661 @Override 662 public boolean evictBlock(BlockCacheKey cacheKey) { 663 stats.evicted(0, cacheKey != null ? cacheKey.isPrimary() : true); 664 return false; 665 } 666 667 @Override 668 public int evictBlocksByHfileName(String hfileName) { 669 stats.evicted(0, true); // Just assuming only one block for file here. 670 return 0; 671 } 672 673 @Override 674 public CacheStats getStats() { 675 return this.stats; 676 } 677 678 @Override 679 public void shutdown() { 680 681 } 682 683 @Override 684 public long size() { 685 return 0; 686 } 687 688 @Override 689 public long getMaxSize() { 690 return 0; 691 } 692 693 @Override 694 public long getFreeSize() { 695 return 0; 696 } 697 698 @Override 699 public long getCurrentSize() { 700 return this.testBlockSize; 701 } 702 703 @Override 704 public long getCurrentDataSize() { 705 return 0; 706 } 707 708 @Override 709 public long getBlockCount() { 710 return 0; 711 } 712 713 @Override 714 public long getDataBlockCount() { 715 return 0; 716 } 717 718 @Override 719 public void setMaxSize(long size) { 720 this.maxSize = size; 721 } 722 723 @Override 724 public Iterator<CachedBlock> iterator() { 725 return null; 726 } 727 728 @Override 729 public BlockCache[] getBlockCaches() { 730 return null; 731 } 732 733 public void setTestBlockSize(long testBlockSize) { 734 this.testBlockSize = testBlockSize; 735 } 736 } 737 738 private static class MemstoreFlusherStub implements FlushRequester { 739 740 long memstoreSize; 741 742 FlushRequestListener listener; 743 744 FlushType flushType = FlushType.NORMAL; 745 746 public MemstoreFlusherStub(long memstoreSize) { 747 this.memstoreSize = memstoreSize; 748 } 749 750 @Override 751 public boolean requestFlush(HRegion region, FlushLifeCycleTracker tracker) { 752 this.listener.flushRequested(flushType, region); 753 return true; 754 } 755 756 @Override 757 public boolean requestFlush(HRegion region, List<byte[]> families, 758 FlushLifeCycleTracker tracker) { 759 return true; 760 } 761 762 @Override 763 public boolean requestDelayedFlush(HRegion region, long delay) { 764 return true; 765 } 766 767 @Override 768 public void registerFlushRequestListener(FlushRequestListener listener) { 769 this.listener = listener; 770 } 771 772 @Override 773 public boolean unregisterFlushRequestListener(FlushRequestListener listener) { 774 return false; 775 } 776 777 @Override 778 public void setGlobalMemStoreLimit(long globalMemStoreSize) { 779 this.memstoreSize = globalMemStoreSize; 780 } 781 } 782 783 private static class RegionServerStub implements Server { 784 private Configuration conf; 785 private boolean stopped = false; 786 787 public RegionServerStub(Configuration conf) { 788 this.conf = conf; 789 } 790 791 @Override 792 public void abort(String why, Throwable e) { 793 794 } 795 796 @Override 797 public boolean isAborted() { 798 return false; 799 } 800 801 @Override 802 public void stop(String why) { 803 this.stopped = true; 804 } 805 806 @Override 807 public boolean isStopped() { 808 return this.stopped; 809 } 810 811 @Override 812 public Configuration getConfiguration() { 813 return this.conf; 814 } 815 816 @Override 817 public ZKWatcher getZooKeeper() { 818 return null; 819 } 820 821 @Override 822 public CoordinatedStateManager getCoordinatedStateManager() { 823 return null; 824 } 825 826 @Override 827 public Connection getConnection() { 828 return null; 829 } 830 831 @Override 832 public ServerName getServerName() { 833 return ServerName.valueOf("server1", 4000, 12345); 834 } 835 836 @Override 837 public ChoreService getChoreService() { 838 return null; 839 } 840 841 @Override 842 public FileSystem getFileSystem() { 843 return null; 844 } 845 846 @Override 847 public boolean isStopping() { 848 return false; 849 } 850 851 @Override 852 public Connection createConnection(Configuration conf) throws IOException { 853 return null; 854 } 855 856 @Override 857 public AsyncClusterConnection getAsyncClusterConnection() { 858 return null; 859 } 860 861 @Override 862 public KeyManagementService getKeyManagementService() { 863 return null; 864 } 865 } 866 867 static class CustomHeapMemoryTuner implements HeapMemoryTuner { 868 static float blockCacheSize = 0.4f; 869 static float memstoreSize = 0.4f; 870 871 @Override 872 public Configuration getConf() { 873 return null; 874 } 875 876 @Override 877 public void setConf(Configuration arg0) { 878 879 } 880 881 @Override 882 public TunerResult tune(TunerContext context) { 883 TunerResult result = new TunerResult(true); 884 result.setBlockCacheSize(blockCacheSize); 885 result.setMemStoreSize(memstoreSize); 886 return result; 887 } 888 } 889 890 private static class RegionServerAccountingStub extends RegionServerAccounting { 891 boolean offheap; 892 893 public RegionServerAccountingStub(Configuration conf) { 894 super(conf); 895 } 896 897 public RegionServerAccountingStub(Configuration conf, boolean offheap) { 898 super(conf); 899 this.offheap = offheap; 900 } 901 902 private long testMemstoreSize = 0; 903 904 @Override 905 public long getGlobalMemStoreDataSize() { 906 return testMemstoreSize; 907 } 908 909 @Override 910 public long getGlobalMemStoreHeapSize() { 911 return testMemstoreSize; 912 } 913 914 @Override 915 public boolean isOffheap() { 916 return offheap; 917 } 918 919 public void setTestMemstoreSize(long testMemstoreSize) { 920 this.testMemstoreSize = testMemstoreSize; 921 } 922 } 923}