001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.regionserver; 019 020import static org.junit.jupiter.api.Assertions.assertEquals; 021import static org.junit.jupiter.api.Assertions.assertFalse; 022import static org.junit.jupiter.api.Assertions.assertTrue; 023import static org.junit.jupiter.api.Assertions.fail; 024 025import java.io.IOException; 026import java.lang.management.ManagementFactory; 027import java.util.Iterator; 028import java.util.List; 029import org.apache.hadoop.conf.Configuration; 030import org.apache.hadoop.fs.FileSystem; 031import org.apache.hadoop.hbase.ChoreService; 032import org.apache.hadoop.hbase.CoordinatedStateManager; 033import org.apache.hadoop.hbase.HBaseConfiguration; 034import org.apache.hadoop.hbase.HBaseTestingUtil; 035import org.apache.hadoop.hbase.HConstants; 036import org.apache.hadoop.hbase.Server; 037import org.apache.hadoop.hbase.ServerName; 038import org.apache.hadoop.hbase.Waiter; 039import org.apache.hadoop.hbase.client.AsyncClusterConnection; 040import org.apache.hadoop.hbase.client.Connection; 041import org.apache.hadoop.hbase.io.hfile.BlockCache; 042import org.apache.hadoop.hbase.io.hfile.BlockCacheKey; 043import org.apache.hadoop.hbase.io.hfile.CacheStats; 044import org.apache.hadoop.hbase.io.hfile.Cacheable; 045import org.apache.hadoop.hbase.io.hfile.CachedBlock; 046import org.apache.hadoop.hbase.io.hfile.ResizableBlockCache; 047import org.apache.hadoop.hbase.io.util.MemorySizeUtil; 048import org.apache.hadoop.hbase.keymeta.KeyManagementService; 049import org.apache.hadoop.hbase.regionserver.HeapMemoryManager.TunerContext; 050import org.apache.hadoop.hbase.regionserver.HeapMemoryManager.TunerResult; 051import org.apache.hadoop.hbase.testclassification.MediumTests; 052import org.apache.hadoop.hbase.testclassification.RegionServerTests; 053import org.apache.hadoop.hbase.zookeeper.ZKWatcher; 054import org.junit.jupiter.api.Tag; 055import org.junit.jupiter.api.Test; 056 057@Tag(RegionServerTests.TAG) 058@Tag(MediumTests.TAG) 059public class TestHeapMemoryManager { 060 061 private static final HBaseTestingUtil UTIL = new HBaseTestingUtil(); 062 063 private long maxHeapSize = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax(); 064 065 @Test 066 public void testAutoTunerShouldBeOffWhenMaxMinRangesForMemstoreIsNotGiven() throws Exception { 067 Configuration conf = HBaseConfiguration.create(); 068 conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_KEY, 0.02f); 069 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.75f); 070 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.03f); 071 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 072 HeapMemoryManager manager = new HeapMemoryManager(new BlockCacheStub(0), 073 new MemstoreFlusherStub(0), new RegionServerStub(conf), regionServerAccounting); 074 assertFalse(manager.isTunerOn()); 075 } 076 077 @Test 078 public void testAutoTunerShouldBeOffWhenMaxMinRangesForBlockCacheIsNotGiven() throws Exception { 079 Configuration conf = HBaseConfiguration.create(); 080 conf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.02f); 081 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 082 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.03f); 083 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 084 HeapMemoryManager manager = new HeapMemoryManager(new BlockCacheStub(0), 085 new MemstoreFlusherStub(0), new RegionServerStub(conf), regionServerAccounting); 086 assertFalse(manager.isTunerOn()); 087 } 088 089 @Test 090 public void testWhenMemstoreAndBlockCacheMaxMinChecksFails() throws Exception { 091 BlockCacheStub blockCache = new BlockCacheStub(0); 092 Configuration conf = HBaseConfiguration.create(); 093 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 094 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.06f); 095 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 096 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub(0); 097 try { 098 new HeapMemoryManager(blockCache, memStoreFlusher, new RegionServerStub(conf), 099 regionServerAccounting); 100 fail(); 101 } catch (RuntimeException e) { 102 } 103 conf = HBaseConfiguration.create(); 104 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.2f); 105 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 106 try { 107 new HeapMemoryManager(blockCache, memStoreFlusher, new RegionServerStub(conf), 108 regionServerAccounting); 109 fail(); 110 } catch (RuntimeException e) { 111 } 112 } 113 114 @Test 115 public void testWhenClusterIsWriteHeavyWithEmptyMemstore() throws Exception { 116 Configuration conf = HBaseConfiguration.create(); 117 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 118 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 119 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 120 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 121 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 122 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 123 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 124 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 125 // Empty block cache and memstore 126 blockCache.setTestBlockSize(0); 127 regionServerAccounting.setTestMemstoreSize(0); 128 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 129 // Let the system start with default values for memstore heap and block cache size. 130 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 131 new RegionServerStub(conf), regionServerAccounting); 132 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 133 long oldBlockCacheSize = blockCache.maxSize; 134 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 135 heapMemoryManager.start(choreService); 136 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_HIGHER_MARK; 137 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 138 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 139 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 140 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK; 141 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 142 // Allow the tuner to run once and do necessary memory up 143 Thread.sleep(1500); 144 // No changes should be made by tuner as we already have lot of empty space 145 assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 146 assertEquals(oldBlockCacheSize, blockCache.maxSize); 147 } 148 149 @Test 150 public void testHeapMemoryManagerWhenOffheapFlushesHappenUnderReadHeavyCase() throws Exception { 151 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 152 Configuration conf = HBaseConfiguration.create(); 153 conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_LOWER_LIMIT_KEY, 0.7f); 154 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 155 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 156 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 157 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 158 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 159 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 160 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf, true); 161 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 162 // Empty memstore and but nearly filled block cache 163 blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8)); 164 regionServerAccounting.setTestMemstoreSize(0); 165 // Let the system start with default values for memstore heap and block cache size. 166 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 167 new RegionServerStub(conf), regionServerAccounting); 168 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 169 long oldBlockCacheSize = blockCache.maxSize; 170 float maxStepValue = DefaultHeapMemoryTuner.DEFAULT_MIN_STEP_VALUE; 171 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 172 heapMemoryManager.start(choreService); 173 blockCache.evictBlock(null); 174 blockCache.evictBlock(null); 175 blockCache.evictBlock(null); 176 // do some offheap flushes also. So there should be decrease in memstore but 177 // not as that when we don't have offheap flushes 178 memStoreFlusher.flushType = FlushType.ABOVE_OFFHEAP_HIGHER_MARK; 179 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 180 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 181 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 182 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 183 // Allow the tuner to run once and do necessary memory up 184 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 185 assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 186 assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize); 187 oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 188 oldBlockCacheSize = blockCache.maxSize; 189 // Do some more evictions before the next run of HeapMemoryTuner 190 blockCache.evictBlock(null); 191 // Allow the tuner to run once and do necessary memory up 192 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 193 assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 194 assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize); 195 } 196 197 @Test 198 public void testHeapMemoryManagerWithOffheapMemstoreAndMixedWorkload() throws Exception { 199 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 200 Configuration conf = HBaseConfiguration.create(); 201 conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_LOWER_LIMIT_KEY, 0.7f); 202 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 203 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 204 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 205 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 206 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 207 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 208 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf, true); 209 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 210 // Empty memstore and but nearly filled block cache 211 blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8)); 212 regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8)); 213 // Let the system start with default values for memstore heap and block cache size. 214 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 215 new RegionServerStub(conf), regionServerAccounting); 216 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 217 long oldBlockCacheSize = blockCache.maxSize; 218 float maxStepValue = DefaultHeapMemoryTuner.DEFAULT_MIN_STEP_VALUE; 219 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 220 heapMemoryManager.start(choreService); 221 blockCache.evictBlock(null); 222 blockCache.evictBlock(null); 223 blockCache.evictBlock(null); 224 // do some offheap flushes also. So there should be decrease in memstore but 225 // not as that when we don't have offheap flushes 226 memStoreFlusher.flushType = FlushType.ABOVE_OFFHEAP_HIGHER_MARK; 227 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 228 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 229 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 230 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 231 // Allow the tuner to run once and do necessary memory up 232 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 233 assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 234 assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize); 235 oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 236 oldBlockCacheSize = blockCache.maxSize; 237 // change memstore size 238 // regionServerAccounting.setTestMemstoreSize((long)(maxHeapSize * 0.4 * 0.8)); 239 // The memstore size would have decreased. Now again do some flushes and ensure the 240 // flushes are due to onheap overhead. This should once again call for increase in 241 // memstore size but that increase should be to the safe size 242 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_HIGHER_MARK; 243 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 244 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 245 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 246 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 247 // Allow the tuner to run once and do necessary memory up 248 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 249 assertHeapSpaceDelta(maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 250 assertHeapSpaceDelta(-maxStepValue, oldBlockCacheSize, blockCache.maxSize); 251 } 252 253 @Test 254 public void testWhenClusterIsReadHeavyWithEmptyBlockCache() throws Exception { 255 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 256 Configuration conf = HBaseConfiguration.create(); 257 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 258 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 259 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 260 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 261 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 262 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 263 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 264 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 265 // Empty block cache and memstore 266 blockCache.setTestBlockSize(0); 267 regionServerAccounting.setTestMemstoreSize(0); 268 // Let the system start with default values for memstore heap and block cache size. 269 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 270 new RegionServerStub(conf), regionServerAccounting); 271 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 272 long oldBlockCacheSize = blockCache.maxSize; 273 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 274 heapMemoryManager.start(choreService); 275 blockCache.evictBlock(null); 276 blockCache.evictBlock(null); 277 blockCache.evictBlock(null); 278 // Allow the tuner to run once and do necessary memory up 279 Thread.sleep(1500); 280 // No changes should be made by tuner as we already have lot of empty space 281 assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 282 assertEquals(oldBlockCacheSize, blockCache.maxSize); 283 } 284 285 @Test 286 public void testWhenClusterIsWriteHeavy() throws Exception { 287 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 288 Configuration conf = HBaseConfiguration.create(); 289 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 290 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 291 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 292 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 293 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 294 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 295 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 296 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 297 // Empty block cache and but nearly filled memstore 298 blockCache.setTestBlockSize(0); 299 regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8)); 300 // Let the system start with default values for memstore heap and block cache size. 301 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 302 new RegionServerStub(conf), regionServerAccounting); 303 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 304 long oldBlockCacheSize = blockCache.maxSize; 305 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 306 heapMemoryManager.start(choreService); 307 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK; 308 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 309 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 310 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 311 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 312 // Allow the tuner to run once and do necessary memory up 313 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 314 assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldMemstoreHeapSize, 315 memStoreFlusher.memstoreSize); 316 assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldBlockCacheSize, 317 blockCache.maxSize); 318 oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 319 oldBlockCacheSize = blockCache.maxSize; 320 // Do some more flushes before the next run of HeapMemoryTuner 321 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK; 322 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 323 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 324 // Allow the tuner to run once and do necessary memory up 325 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 326 assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldMemstoreHeapSize, 327 memStoreFlusher.memstoreSize); 328 assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldBlockCacheSize, 329 blockCache.maxSize); 330 } 331 332 @Test 333 public void testWhenClusterIsWriteHeavyWithOffheapMemstore() throws Exception { 334 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 335 Configuration conf = HBaseConfiguration.create(); 336 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 337 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 338 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 339 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 340 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 341 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 342 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 343 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 344 // Empty block cache and but nearly filled memstore 345 blockCache.setTestBlockSize(0); 346 regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8)); 347 // Let the system start with default values for memstore heap and block cache size. 348 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 349 new RegionServerStub(conf), regionServerAccounting); 350 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 351 long oldBlockCacheSize = blockCache.maxSize; 352 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 353 heapMemoryManager.start(choreService); 354 // this should not change anything with onheap memstore 355 memStoreFlusher.flushType = FlushType.ABOVE_OFFHEAP_HIGHER_MARK; 356 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 357 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 358 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 359 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 360 // Allow the tuner to run once and do necessary memory up 361 Thread.sleep(1500); 362 // No changes should be made by tuner as we already have lot of empty space 363 assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 364 assertEquals(oldBlockCacheSize, blockCache.maxSize); 365 } 366 367 @Test 368 public void testWhenClusterIsReadHeavy() throws Exception { 369 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 370 Configuration conf = HBaseConfiguration.create(); 371 conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_LOWER_LIMIT_KEY, 0.7f); 372 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 373 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 374 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 375 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 376 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 377 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 378 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 379 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 380 // Empty memstore and but nearly filled block cache 381 blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8)); 382 regionServerAccounting.setTestMemstoreSize(0); 383 // Let the system start with default values for memstore heap and block cache size. 384 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 385 new RegionServerStub(conf), new RegionServerAccountingStub(conf)); 386 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 387 long oldBlockCacheSize = blockCache.maxSize; 388 long oldMemstoreLowerMarkSize = 7 * oldMemstoreHeapSize / 10; 389 long maxTuneSize = oldMemstoreHeapSize - (oldMemstoreLowerMarkSize + oldMemstoreHeapSize) / 2; 390 float maxStepValue = (maxTuneSize * 1.0f) / oldMemstoreHeapSize; 391 maxStepValue = maxStepValue > DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE 392 ? DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE 393 : maxStepValue; 394 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 395 heapMemoryManager.start(choreService); 396 blockCache.evictBlock(null); 397 blockCache.evictBlock(null); 398 blockCache.evictBlock(null); 399 // Allow the tuner to run once and do necessary memory up 400 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 401 assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 402 assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize); 403 oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 404 oldBlockCacheSize = blockCache.maxSize; 405 oldMemstoreLowerMarkSize = 7 * oldMemstoreHeapSize / 10; 406 maxTuneSize = oldMemstoreHeapSize - (oldMemstoreLowerMarkSize + oldMemstoreHeapSize) / 2; 407 maxStepValue = (maxTuneSize * 1.0f) / oldMemstoreHeapSize; 408 maxStepValue = maxStepValue > DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE 409 ? DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE 410 : maxStepValue; 411 // Do some more evictions before the next run of HeapMemoryTuner 412 blockCache.evictBlock(null); 413 // Allow the tuner to run once and do necessary memory up 414 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 415 assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 416 assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize); 417 } 418 419 @Test 420 public void testWhenClusterIsHavingMoreWritesThanReads() throws Exception { 421 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 422 Configuration conf = HBaseConfiguration.create(); 423 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 424 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 425 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 426 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 427 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 428 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 429 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 430 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 431 // Both memstore and block cache are nearly filled 432 blockCache.setTestBlockSize(0); 433 regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8)); 434 blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8)); 435 // Let the system start with default values for memstore heap and block cache size. 436 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 437 new RegionServerStub(conf), regionServerAccounting); 438 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 439 long oldBlockCacheSize = blockCache.maxSize; 440 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 441 heapMemoryManager.start(choreService); 442 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK; 443 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 444 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 445 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 446 blockCache.evictBlock(null); 447 // Allow the tuner to run once and do necessary memory up 448 Thread.sleep(1500); 449 // No changes should happen as there is undefined increase in flushes and evictions 450 assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 451 assertEquals(oldBlockCacheSize, blockCache.maxSize); 452 // Do some more flushes before the next run of HeapMemoryTuner 453 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK; 454 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 455 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 456 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 457 // Allow the tuner to run once and do necessary memory up 458 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 459 assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldMemstoreHeapSize, 460 memStoreFlusher.memstoreSize); 461 assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldBlockCacheSize, 462 blockCache.maxSize); 463 } 464 465 @Test 466 public void testBlockedFlushesIncreaseMemstoreInSteadyState() throws Exception { 467 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 468 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 469 Configuration conf = HBaseConfiguration.create(); 470 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 471 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 472 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 473 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 474 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 475 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 476 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 477 // Both memstore and block cache are nearly filled 478 blockCache.setTestBlockSize(0); 479 regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8)); 480 blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8)); 481 // Let the system start with default values for memstore heap and block cache size. 482 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 483 new RegionServerStub(conf), regionServerAccounting); 484 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 485 long oldBlockCacheSize = blockCache.maxSize; 486 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 487 heapMemoryManager.start(choreService); 488 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK; 489 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 490 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 491 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 492 blockCache.evictBlock(null); 493 blockCache.evictBlock(null); 494 // Allow the tuner to run once and do necessary memory up 495 Thread.sleep(1500); 496 // No changes should happen as there is undefined increase in flushes and evictions 497 assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 498 assertEquals(oldBlockCacheSize, blockCache.maxSize); 499 // Flushes that block updates 500 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_HIGHER_MARK; 501 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 502 blockCache.evictBlock(null); 503 blockCache.evictBlock(null); 504 blockCache.evictBlock(null); 505 blockCache.evictBlock(null); 506 // Allow the tuner to run once and do necessary memory up 507 Thread.sleep(1500); 508 assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldMemstoreHeapSize, 509 memStoreFlusher.memstoreSize); 510 assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldBlockCacheSize, 511 blockCache.maxSize); 512 } 513 514 @Test 515 public void testPluggingInHeapMemoryTuner() throws Exception { 516 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 517 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 518 Configuration conf = HBaseConfiguration.create(); 519 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.78f); 520 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.05f); 521 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.75f); 522 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.02f); 523 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 524 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 525 conf.setClass(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_CLASS, CustomHeapMemoryTuner.class, 526 HeapMemoryTuner.class); 527 // Let the system start with default values for memstore heap and block cache size. 528 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 529 new RegionServerStub(conf), new RegionServerAccountingStub(conf)); 530 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 531 heapMemoryManager.start(choreService); 532 // Now we wants to be in write mode. Set bigger memstore size from CustomHeapMemoryTuner 533 CustomHeapMemoryTuner.memstoreSize = 0.78f; 534 CustomHeapMemoryTuner.blockCacheSize = 0.02f; 535 // Allow the tuner to run once and do necessary memory up 536 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 537 assertHeapSpace(0.78f, memStoreFlusher.memstoreSize);// Memstore 538 assertHeapSpace(0.02f, blockCache.maxSize);// BlockCache 539 // Now we wants to be in read mode. Set bigger memstore size from CustomHeapMemoryTuner 540 CustomHeapMemoryTuner.blockCacheSize = 0.75f; 541 CustomHeapMemoryTuner.memstoreSize = 0.05f; 542 // Allow the tuner to run once and do necessary memory up 543 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 544 assertHeapSpace(0.75f, blockCache.maxSize);// BlockCache 545 assertHeapSpace(0.05f, memStoreFlusher.memstoreSize);// Memstore 546 } 547 548 @Test 549 public void testWhenSizeGivenByHeapTunerGoesOutsideRange() throws Exception { 550 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 551 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 552 Configuration conf = HBaseConfiguration.create(); 553 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.7f); 554 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.1f); 555 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 556 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.1f); 557 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 558 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 559 conf.setClass(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_CLASS, CustomHeapMemoryTuner.class, 560 HeapMemoryTuner.class); 561 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 562 new RegionServerStub(conf), new RegionServerAccountingStub(conf)); 563 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 564 heapMemoryManager.start(choreService); 565 CustomHeapMemoryTuner.memstoreSize = 0.78f; 566 CustomHeapMemoryTuner.blockCacheSize = 0.02f; 567 Thread.sleep(1500); // Allow the tuner to run once and do necessary memory up 568 // Even if the tuner says to set the memstore to 78%, HBase makes it as 70% as that is the 569 // upper bound. Same with block cache as 10% is the lower bound. 570 assertHeapSpace(0.7f, memStoreFlusher.memstoreSize); 571 assertHeapSpace(0.1f, blockCache.maxSize); 572 } 573 574 @Test 575 public void testWhenCombinedHeapSizesFromTunerGoesOutSideMaxLimit() throws Exception { 576 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 577 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 578 Configuration conf = HBaseConfiguration.create(); 579 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.7f); 580 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.1f); 581 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 582 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.1f); 583 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 584 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 585 conf.setClass(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_CLASS, CustomHeapMemoryTuner.class, 586 HeapMemoryTuner.class); 587 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 588 new RegionServerStub(conf), new RegionServerAccountingStub(conf)); 589 long oldMemstoreSize = memStoreFlusher.memstoreSize; 590 long oldBlockCacheSize = blockCache.maxSize; 591 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 592 heapMemoryManager.start(choreService); 593 CustomHeapMemoryTuner.memstoreSize = 0.7f; 594 CustomHeapMemoryTuner.blockCacheSize = 0.3f; 595 // Allow the tuner to run once and do necessary memory up 596 Thread.sleep(1500); 597 assertEquals(oldMemstoreSize, memStoreFlusher.memstoreSize); 598 assertEquals(oldBlockCacheSize, blockCache.maxSize); 599 } 600 601 private void assertHeapSpace(float expectedHeapPercentage, long currentHeapSpace) { 602 long expected = (long) (this.maxHeapSize * expectedHeapPercentage); 603 assertEquals(expected, currentHeapSpace); 604 } 605 606 private void assertHeapSpaceDelta(double expectedDeltaPercent, long oldHeapSpace, 607 long newHeapSpace) { 608 double expctedMinDelta = (double) (this.maxHeapSize * expectedDeltaPercent); 609 // Tolerable error 610 double error = 0.95; 611 if (expectedDeltaPercent > 0) { 612 assertTrue(expctedMinDelta * error <= (double) (newHeapSpace - oldHeapSpace)); 613 assertTrue(expctedMinDelta / error >= (double) (newHeapSpace - oldHeapSpace)); 614 } else { 615 assertTrue(-expctedMinDelta * error <= (double) (oldHeapSpace - newHeapSpace)); 616 assertTrue(-expctedMinDelta / error >= (double) (oldHeapSpace - newHeapSpace)); 617 } 618 } 619 620 private void waitForTune(final MemstoreFlusherStub memStoreFlusher, 621 final long oldMemstoreHeapSize) throws Exception { 622 // Allow the tuner to run once and do necessary memory up 623 UTIL.waitFor(10000, new Waiter.Predicate<Exception>() { 624 @Override 625 public boolean evaluate() throws Exception { 626 return oldMemstoreHeapSize != memStoreFlusher.memstoreSize; 627 } 628 }); 629 } 630 631 private static class BlockCacheStub implements ResizableBlockCache { 632 CacheStats stats = new CacheStats("test"); 633 long maxSize = 0; 634 private long testBlockSize = 0; 635 636 public BlockCacheStub(long size) { 637 this.maxSize = size; 638 } 639 640 @Override 641 public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory) { 642 643 } 644 645 @Override 646 public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf) { 647 648 } 649 650 @Override 651 public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching, boolean repeat, 652 boolean updateCacheMetrics) { 653 return null; 654 } 655 656 @Override 657 public boolean evictBlock(BlockCacheKey cacheKey) { 658 stats.evicted(0, cacheKey != null ? cacheKey.isPrimary() : true); 659 return false; 660 } 661 662 @Override 663 public int evictBlocksByHfileName(String hfileName) { 664 stats.evicted(0, true); // Just assuming only one block for file here. 665 return 0; 666 } 667 668 @Override 669 public CacheStats getStats() { 670 return this.stats; 671 } 672 673 @Override 674 public void shutdown() { 675 676 } 677 678 @Override 679 public long size() { 680 return 0; 681 } 682 683 @Override 684 public long getMaxSize() { 685 return 0; 686 } 687 688 @Override 689 public long getFreeSize() { 690 return 0; 691 } 692 693 @Override 694 public long getCurrentSize() { 695 return this.testBlockSize; 696 } 697 698 @Override 699 public long getCurrentDataSize() { 700 return 0; 701 } 702 703 @Override 704 public long getBlockCount() { 705 return 0; 706 } 707 708 @Override 709 public long getDataBlockCount() { 710 return 0; 711 } 712 713 @Override 714 public void setMaxSize(long size) { 715 this.maxSize = size; 716 } 717 718 @Override 719 public Iterator<CachedBlock> iterator() { 720 return null; 721 } 722 723 @Override 724 public BlockCache[] getBlockCaches() { 725 return null; 726 } 727 728 public void setTestBlockSize(long testBlockSize) { 729 this.testBlockSize = testBlockSize; 730 } 731 } 732 733 private static class MemstoreFlusherStub implements FlushRequester { 734 735 long memstoreSize; 736 737 FlushRequestListener listener; 738 739 FlushType flushType = FlushType.NORMAL; 740 741 public MemstoreFlusherStub(long memstoreSize) { 742 this.memstoreSize = memstoreSize; 743 } 744 745 @Override 746 public boolean requestFlush(HRegion region, FlushLifeCycleTracker tracker) { 747 this.listener.flushRequested(flushType, region); 748 return true; 749 } 750 751 @Override 752 public boolean requestFlush(HRegion region, List<byte[]> families, 753 FlushLifeCycleTracker tracker) { 754 return true; 755 } 756 757 @Override 758 public boolean requestDelayedFlush(HRegion region, long delay) { 759 return true; 760 } 761 762 @Override 763 public void registerFlushRequestListener(FlushRequestListener listener) { 764 this.listener = listener; 765 } 766 767 @Override 768 public boolean unregisterFlushRequestListener(FlushRequestListener listener) { 769 return false; 770 } 771 772 @Override 773 public void setGlobalMemStoreLimit(long globalMemStoreSize) { 774 this.memstoreSize = globalMemStoreSize; 775 } 776 } 777 778 private static class RegionServerStub implements Server { 779 private Configuration conf; 780 private boolean stopped = false; 781 782 public RegionServerStub(Configuration conf) { 783 this.conf = conf; 784 } 785 786 @Override 787 public void abort(String why, Throwable e) { 788 789 } 790 791 @Override 792 public boolean isAborted() { 793 return false; 794 } 795 796 @Override 797 public void stop(String why) { 798 this.stopped = true; 799 } 800 801 @Override 802 public boolean isStopped() { 803 return this.stopped; 804 } 805 806 @Override 807 public Configuration getConfiguration() { 808 return this.conf; 809 } 810 811 @Override 812 public ZKWatcher getZooKeeper() { 813 return null; 814 } 815 816 @Override 817 public CoordinatedStateManager getCoordinatedStateManager() { 818 return null; 819 } 820 821 @Override 822 public Connection getConnection() { 823 return null; 824 } 825 826 @Override 827 public ServerName getServerName() { 828 return ServerName.valueOf("server1", 4000, 12345); 829 } 830 831 @Override 832 public ChoreService getChoreService() { 833 return null; 834 } 835 836 @Override 837 public FileSystem getFileSystem() { 838 return null; 839 } 840 841 @Override 842 public boolean isStopping() { 843 return false; 844 } 845 846 @Override 847 public Connection createConnection(Configuration conf) throws IOException { 848 return null; 849 } 850 851 @Override 852 public AsyncClusterConnection getAsyncClusterConnection() { 853 return null; 854 } 855 856 @Override 857 public KeyManagementService getKeyManagementService() { 858 return null; 859 } 860 } 861 862 static class CustomHeapMemoryTuner implements HeapMemoryTuner { 863 static float blockCacheSize = 0.4f; 864 static float memstoreSize = 0.4f; 865 866 @Override 867 public Configuration getConf() { 868 return null; 869 } 870 871 @Override 872 public void setConf(Configuration arg0) { 873 874 } 875 876 @Override 877 public TunerResult tune(TunerContext context) { 878 TunerResult result = new TunerResult(true); 879 result.setBlockCacheSize(blockCacheSize); 880 result.setMemStoreSize(memstoreSize); 881 return result; 882 } 883 } 884 885 private static class RegionServerAccountingStub extends RegionServerAccounting { 886 boolean offheap; 887 888 public RegionServerAccountingStub(Configuration conf) { 889 super(conf); 890 } 891 892 public RegionServerAccountingStub(Configuration conf, boolean offheap) { 893 super(conf); 894 this.offheap = offheap; 895 } 896 897 private long testMemstoreSize = 0; 898 899 @Override 900 public long getGlobalMemStoreDataSize() { 901 return testMemstoreSize; 902 } 903 904 @Override 905 public long getGlobalMemStoreHeapSize() { 906 return testMemstoreSize; 907 } 908 909 @Override 910 public boolean isOffheap() { 911 return offheap; 912 } 913 914 public void setTestMemstoreSize(long testMemstoreSize) { 915 this.testMemstoreSize = testMemstoreSize; 916 } 917 } 918}