001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.regionserver; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertFalse; 022import static org.junit.Assert.assertTrue; 023import static org.junit.Assert.fail; 024 025import java.io.IOException; 026import java.lang.management.ManagementFactory; 027import java.util.Iterator; 028import org.apache.hadoop.conf.Configuration; 029import org.apache.hadoop.fs.FileSystem; 030import org.apache.hadoop.hbase.ChoreService; 031import org.apache.hadoop.hbase.CoordinatedStateManager; 032import org.apache.hadoop.hbase.HBaseClassTestRule; 033import org.apache.hadoop.hbase.HBaseConfiguration; 034import org.apache.hadoop.hbase.HBaseTestingUtility; 035import org.apache.hadoop.hbase.HConstants; 036import org.apache.hadoop.hbase.Server; 037import org.apache.hadoop.hbase.ServerName; 038import org.apache.hadoop.hbase.Waiter; 039import org.apache.hadoop.hbase.client.ClusterConnection; 040import org.apache.hadoop.hbase.client.Connection; 041import org.apache.hadoop.hbase.io.hfile.BlockCache; 042import org.apache.hadoop.hbase.io.hfile.BlockCacheKey; 043import org.apache.hadoop.hbase.io.hfile.CacheStats; 044import org.apache.hadoop.hbase.io.hfile.Cacheable; 045import org.apache.hadoop.hbase.io.hfile.CachedBlock; 046import org.apache.hadoop.hbase.io.hfile.ResizableBlockCache; 047import org.apache.hadoop.hbase.io.util.MemorySizeUtil; 048import org.apache.hadoop.hbase.regionserver.HeapMemoryManager.TunerContext; 049import org.apache.hadoop.hbase.regionserver.HeapMemoryManager.TunerResult; 050import org.apache.hadoop.hbase.testclassification.RegionServerTests; 051import org.apache.hadoop.hbase.testclassification.SmallTests; 052import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; 053import org.apache.hadoop.hbase.zookeeper.ZKWatcher; 054import org.junit.ClassRule; 055import org.junit.Test; 056import org.junit.experimental.categories.Category; 057 058@Category({RegionServerTests.class, SmallTests.class}) 059public class TestHeapMemoryManager { 060 061 @ClassRule 062 public static final HBaseClassTestRule CLASS_RULE = 063 HBaseClassTestRule.forClass(TestHeapMemoryManager.class); 064 065 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); 066 067 private long maxHeapSize = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax(); 068 069 @Test 070 public void testAutoTunerShouldBeOffWhenMaxMinRangesForMemstoreIsNotGiven() throws Exception { 071 Configuration conf = HBaseConfiguration.create(); 072 conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_KEY, 0.02f); 073 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.75f); 074 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.03f); 075 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 076 HeapMemoryManager manager = new HeapMemoryManager(new BlockCacheStub(0), 077 new MemstoreFlusherStub(0), new RegionServerStub(conf), 078 regionServerAccounting); 079 assertFalse(manager.isTunerOn()); 080 } 081 082 @Test 083 public void testAutoTunerShouldBeOffWhenMaxMinRangesForBlockCacheIsNotGiven() throws Exception { 084 Configuration conf = HBaseConfiguration.create(); 085 conf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.02f); 086 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 087 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.03f); 088 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 089 HeapMemoryManager manager = new HeapMemoryManager(new BlockCacheStub(0), 090 new MemstoreFlusherStub(0), new RegionServerStub(conf), 091 regionServerAccounting); 092 assertFalse(manager.isTunerOn()); 093 } 094 095 @Test 096 public void testWhenMemstoreAndBlockCacheMaxMinChecksFails() throws Exception { 097 BlockCacheStub blockCache = new BlockCacheStub(0); 098 Configuration conf = HBaseConfiguration.create(); 099 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 100 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.06f); 101 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 102 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub(0); 103 try { 104 new HeapMemoryManager(blockCache, memStoreFlusher, 105 new RegionServerStub(conf), regionServerAccounting); 106 fail(); 107 } catch (RuntimeException e) { 108 } 109 conf = HBaseConfiguration.create(); 110 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.2f); 111 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 112 try { 113 new HeapMemoryManager(blockCache, memStoreFlusher, 114 new RegionServerStub(conf), regionServerAccounting); 115 fail(); 116 } catch (RuntimeException e) { 117 } 118 } 119 120 @Test 121 public void testWhenClusterIsWriteHeavyWithEmptyMemstore() throws Exception { 122 Configuration conf = HBaseConfiguration.create(); 123 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 124 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 125 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 126 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 127 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 128 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 129 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 130 MemstoreFlusherStub memStoreFlusher = 131 new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 132 // Empty block cache and memstore 133 blockCache.setTestBlockSize(0); 134 regionServerAccounting.setTestMemstoreSize(0); 135 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 136 // Let the system start with default values for memstore heap and block cache size. 137 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 138 new RegionServerStub(conf), regionServerAccounting); 139 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 140 long oldBlockCacheSize = blockCache.maxSize; 141 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 142 heapMemoryManager.start(choreService); 143 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_HIGHER_MARK; 144 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 145 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 146 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 147 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK; 148 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 149 // Allow the tuner to run once and do necessary memory up 150 Thread.sleep(1500); 151 // No changes should be made by tuner as we already have lot of empty space 152 assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 153 assertEquals(oldBlockCacheSize, blockCache.maxSize); 154 } 155 156 @Test 157 public void testHeapMemoryManagerWhenOffheapFlushesHappenUnderReadHeavyCase() throws Exception { 158 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 159 Configuration conf = HBaseConfiguration.create(); 160 conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_LOWER_LIMIT_KEY, 0.7f); 161 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 162 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 163 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 164 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 165 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 166 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 167 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf, true); 168 MemstoreFlusherStub memStoreFlusher = 169 new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 170 // Empty memstore and but nearly filled block cache 171 blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8)); 172 regionServerAccounting.setTestMemstoreSize(0); 173 // Let the system start with default values for memstore heap and block cache size. 174 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 175 new RegionServerStub(conf), regionServerAccounting); 176 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 177 long oldBlockCacheSize = blockCache.maxSize; 178 float maxStepValue = DefaultHeapMemoryTuner.DEFAULT_MIN_STEP_VALUE; 179 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 180 heapMemoryManager.start(choreService); 181 blockCache.evictBlock(null); 182 blockCache.evictBlock(null); 183 blockCache.evictBlock(null); 184 // do some offheap flushes also. So there should be decrease in memstore but 185 // not as that when we don't have offheap flushes 186 memStoreFlusher.flushType = FlushType.ABOVE_OFFHEAP_HIGHER_MARK; 187 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 188 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 189 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 190 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 191 // Allow the tuner to run once and do necessary memory up 192 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 193 assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 194 assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize); 195 oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 196 oldBlockCacheSize = blockCache.maxSize; 197 // Do some more evictions before the next run of HeapMemoryTuner 198 blockCache.evictBlock(null); 199 // Allow the tuner to run once and do necessary memory up 200 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 201 assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 202 assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize); 203 } 204 205 @Test 206 public void testHeapMemoryManagerWithOffheapMemstoreAndMixedWorkload() throws Exception { 207 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 208 Configuration conf = HBaseConfiguration.create(); 209 conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_LOWER_LIMIT_KEY, 0.7f); 210 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 211 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 212 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 213 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 214 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 215 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 216 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf, true); 217 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 218 // Empty memstore and but nearly filled block cache 219 blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8)); 220 regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8)); 221 // Let the system start with default values for memstore heap and block cache size. 222 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 223 new RegionServerStub(conf), regionServerAccounting); 224 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 225 long oldBlockCacheSize = blockCache.maxSize; 226 float maxStepValue = DefaultHeapMemoryTuner.DEFAULT_MIN_STEP_VALUE; 227 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 228 heapMemoryManager.start(choreService); 229 blockCache.evictBlock(null); 230 blockCache.evictBlock(null); 231 blockCache.evictBlock(null); 232 // do some offheap flushes also. So there should be decrease in memstore but 233 // not as that when we don't have offheap flushes 234 memStoreFlusher.flushType = FlushType.ABOVE_OFFHEAP_HIGHER_MARK; 235 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 236 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 237 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 238 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 239 // Allow the tuner to run once and do necessary memory up 240 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 241 assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 242 assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize); 243 oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 244 oldBlockCacheSize = blockCache.maxSize; 245 // change memstore size 246 // regionServerAccounting.setTestMemstoreSize((long)(maxHeapSize * 0.4 * 0.8)); 247 // The memstore size would have decreased. Now again do some flushes and ensure the 248 // flushes are due to onheap overhead. This should once again call for increase in 249 // memstore size but that increase should be to the safe size 250 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_HIGHER_MARK; 251 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 252 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 253 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 254 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 255 // Allow the tuner to run once and do necessary memory up 256 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 257 assertHeapSpaceDelta(maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 258 assertHeapSpaceDelta(-maxStepValue, oldBlockCacheSize, blockCache.maxSize); 259 } 260 261 @Test 262 public void testWhenClusterIsReadHeavyWithEmptyBlockCache() throws Exception { 263 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 264 Configuration conf = HBaseConfiguration.create(); 265 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 266 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 267 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 268 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 269 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 270 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 271 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 272 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 273 // Empty block cache and memstore 274 blockCache.setTestBlockSize(0); 275 regionServerAccounting.setTestMemstoreSize(0); 276 // Let the system start with default values for memstore heap and block cache size. 277 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 278 new RegionServerStub(conf), regionServerAccounting); 279 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 280 long oldBlockCacheSize = blockCache.maxSize; 281 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 282 heapMemoryManager.start(choreService); 283 blockCache.evictBlock(null); 284 blockCache.evictBlock(null); 285 blockCache.evictBlock(null); 286 // Allow the tuner to run once and do necessary memory up 287 Thread.sleep(1500); 288 // No changes should be made by tuner as we already have lot of empty space 289 assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 290 assertEquals(oldBlockCacheSize, blockCache.maxSize); 291 } 292 293 @Test 294 public void testWhenClusterIsWriteHeavy() throws Exception { 295 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 296 Configuration conf = HBaseConfiguration.create(); 297 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 298 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 299 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 300 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 301 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 302 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 303 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 304 MemstoreFlusherStub memStoreFlusher = 305 new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 306 // Empty block cache and but nearly filled memstore 307 blockCache.setTestBlockSize(0); 308 regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8)); 309 // Let the system start with default values for memstore heap and block cache size. 310 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 311 new RegionServerStub(conf), regionServerAccounting); 312 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 313 long oldBlockCacheSize = blockCache.maxSize; 314 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 315 heapMemoryManager.start(choreService); 316 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK; 317 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 318 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 319 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 320 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 321 // Allow the tuner to run once and do necessary memory up 322 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 323 assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldMemstoreHeapSize, 324 memStoreFlusher.memstoreSize); 325 assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldBlockCacheSize, 326 blockCache.maxSize); 327 oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 328 oldBlockCacheSize = blockCache.maxSize; 329 // Do some more flushes before the next run of HeapMemoryTuner 330 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK; 331 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 332 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 333 // Allow the tuner to run once and do necessary memory up 334 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 335 assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldMemstoreHeapSize, 336 memStoreFlusher.memstoreSize); 337 assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldBlockCacheSize, 338 blockCache.maxSize); 339 } 340 341 @Test 342 public void testWhenClusterIsWriteHeavyWithOffheapMemstore() throws Exception { 343 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 344 Configuration conf = HBaseConfiguration.create(); 345 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 346 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 347 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 348 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 349 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 350 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 351 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 352 MemstoreFlusherStub memStoreFlusher = 353 new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 354 // Empty block cache and but nearly filled memstore 355 blockCache.setTestBlockSize(0); 356 regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8)); 357 // Let the system start with default values for memstore heap and block cache size. 358 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 359 new RegionServerStub(conf), regionServerAccounting); 360 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 361 long oldBlockCacheSize = blockCache.maxSize; 362 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 363 heapMemoryManager.start(choreService); 364 // this should not change anything with onheap memstore 365 memStoreFlusher.flushType = FlushType.ABOVE_OFFHEAP_HIGHER_MARK; 366 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 367 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 368 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 369 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 370 // Allow the tuner to run once and do necessary memory up 371 Thread.sleep(1500); 372 // No changes should be made by tuner as we already have lot of empty space 373 assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 374 assertEquals(oldBlockCacheSize, blockCache.maxSize); 375 } 376 377 @Test 378 public void testWhenClusterIsReadHeavy() throws Exception { 379 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 380 Configuration conf = HBaseConfiguration.create(); 381 conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_LOWER_LIMIT_KEY, 0.7f); 382 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 383 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 384 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 385 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 386 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 387 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 388 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 389 MemstoreFlusherStub memStoreFlusher = 390 new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 391 // Empty memstore and but nearly filled block cache 392 blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8)); 393 regionServerAccounting.setTestMemstoreSize(0); 394 // Let the system start with default values for memstore heap and block cache size. 395 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 396 new RegionServerStub(conf), new RegionServerAccountingStub(conf)); 397 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 398 long oldBlockCacheSize = blockCache.maxSize; 399 long oldMemstoreLowerMarkSize = 7 * oldMemstoreHeapSize / 10; 400 long maxTuneSize = oldMemstoreHeapSize - (oldMemstoreLowerMarkSize + oldMemstoreHeapSize) / 2; 401 float maxStepValue = (maxTuneSize * 1.0f) / oldMemstoreHeapSize; 402 maxStepValue = maxStepValue > DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE ? 403 DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE:maxStepValue; 404 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 405 heapMemoryManager.start(choreService); 406 blockCache.evictBlock(null); 407 blockCache.evictBlock(null); 408 blockCache.evictBlock(null); 409 // Allow the tuner to run once and do necessary memory up 410 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 411 assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 412 assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize); 413 oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 414 oldBlockCacheSize = blockCache.maxSize; 415 oldMemstoreLowerMarkSize = 7 * oldMemstoreHeapSize / 10; 416 maxTuneSize = oldMemstoreHeapSize - (oldMemstoreLowerMarkSize + oldMemstoreHeapSize) / 2; 417 maxStepValue = (maxTuneSize * 1.0f) / oldMemstoreHeapSize; 418 maxStepValue = maxStepValue > DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE ? 419 DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE:maxStepValue; 420 // Do some more evictions before the next run of HeapMemoryTuner 421 blockCache.evictBlock(null); 422 // Allow the tuner to run once and do necessary memory up 423 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 424 assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 425 assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize); 426 } 427 428 @Test 429 public void testWhenClusterIsHavingMoreWritesThanReads() throws Exception { 430 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 431 Configuration conf = HBaseConfiguration.create(); 432 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 433 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 434 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 435 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 436 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 437 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 438 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 439 MemstoreFlusherStub memStoreFlusher = 440 new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 441 // Both memstore and block cache are nearly filled 442 blockCache.setTestBlockSize(0); 443 regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8)); 444 blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8)); 445 // Let the system start with default values for memstore heap and block cache size. 446 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 447 new RegionServerStub(conf), regionServerAccounting); 448 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 449 long oldBlockCacheSize = blockCache.maxSize; 450 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 451 heapMemoryManager.start(choreService); 452 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK; 453 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 454 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 455 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 456 blockCache.evictBlock(null); 457 // Allow the tuner to run once and do necessary memory up 458 Thread.sleep(1500); 459 // No changes should happen as there is undefined increase in flushes and evictions 460 assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 461 assertEquals(oldBlockCacheSize, blockCache.maxSize); 462 // Do some more flushes before the next run of HeapMemoryTuner 463 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK; 464 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 465 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 466 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 467 // Allow the tuner to run once and do necessary memory up 468 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 469 assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldMemstoreHeapSize, 470 memStoreFlusher.memstoreSize); 471 assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldBlockCacheSize, 472 blockCache.maxSize); 473 } 474 475 @Test 476 public void testBlockedFlushesIncreaseMemstoreInSteadyState() throws Exception { 477 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 478 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 479 Configuration conf = HBaseConfiguration.create(); 480 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 481 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 482 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 483 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 484 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 485 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 486 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 487 // Both memstore and block cache are nearly filled 488 blockCache.setTestBlockSize(0); 489 regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8)); 490 blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8)); 491 // Let the system start with default values for memstore heap and block cache size. 492 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 493 new RegionServerStub(conf), regionServerAccounting); 494 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 495 long oldBlockCacheSize = blockCache.maxSize; 496 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 497 heapMemoryManager.start(choreService); 498 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK; 499 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 500 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 501 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 502 blockCache.evictBlock(null); 503 blockCache.evictBlock(null); 504 // Allow the tuner to run once and do necessary memory up 505 Thread.sleep(1500); 506 // No changes should happen as there is undefined increase in flushes and evictions 507 assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 508 assertEquals(oldBlockCacheSize, blockCache.maxSize); 509 // Flushes that block updates 510 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_HIGHER_MARK; 511 memStoreFlusher.requestFlush(null, false, FlushLifeCycleTracker.DUMMY); 512 blockCache.evictBlock(null); 513 blockCache.evictBlock(null); 514 blockCache.evictBlock(null); 515 blockCache.evictBlock(null); 516 // Allow the tuner to run once and do necessary memory up 517 Thread.sleep(1500); 518 assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldMemstoreHeapSize, 519 memStoreFlusher.memstoreSize); 520 assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldBlockCacheSize, 521 blockCache.maxSize); 522 } 523 524 @Test 525 public void testPluggingInHeapMemoryTuner() throws Exception { 526 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 527 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 528 Configuration conf = HBaseConfiguration.create(); 529 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.78f); 530 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.05f); 531 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.75f); 532 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.02f); 533 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 534 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 535 conf.setClass(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_CLASS, CustomHeapMemoryTuner.class, 536 HeapMemoryTuner.class); 537 // Let the system start with default values for memstore heap and block cache size. 538 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 539 new RegionServerStub(conf), new RegionServerAccountingStub(conf)); 540 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 541 heapMemoryManager.start(choreService); 542 // Now we wants to be in write mode. Set bigger memstore size from CustomHeapMemoryTuner 543 CustomHeapMemoryTuner.memstoreSize = 0.78f; 544 CustomHeapMemoryTuner.blockCacheSize = 0.02f; 545 // Allow the tuner to run once and do necessary memory up 546 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 547 assertHeapSpace(0.78f, memStoreFlusher.memstoreSize);// Memstore 548 assertHeapSpace(0.02f, blockCache.maxSize);// BlockCache 549 // Now we wants to be in read mode. Set bigger memstore size from CustomHeapMemoryTuner 550 CustomHeapMemoryTuner.blockCacheSize = 0.75f; 551 CustomHeapMemoryTuner.memstoreSize = 0.05f; 552 // Allow the tuner to run once and do necessary memory up 553 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 554 assertHeapSpace(0.75f, blockCache.maxSize);// BlockCache 555 assertHeapSpace(0.05f, memStoreFlusher.memstoreSize);// Memstore 556 } 557 558 @Test 559 public void testWhenSizeGivenByHeapTunerGoesOutsideRange() throws Exception { 560 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 561 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 562 Configuration conf = HBaseConfiguration.create(); 563 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.7f); 564 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.1f); 565 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 566 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.1f); 567 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 568 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 569 conf.setClass(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_CLASS, CustomHeapMemoryTuner.class, 570 HeapMemoryTuner.class); 571 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 572 new RegionServerStub(conf), new RegionServerAccountingStub(conf)); 573 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 574 heapMemoryManager.start(choreService); 575 CustomHeapMemoryTuner.memstoreSize = 0.78f; 576 CustomHeapMemoryTuner.blockCacheSize = 0.02f; 577 Thread.sleep(1500); // Allow the tuner to run once and do necessary memory up 578 // Even if the tuner says to set the memstore to 78%, HBase makes it as 70% as that is the 579 // upper bound. Same with block cache as 10% is the lower bound. 580 assertHeapSpace(0.7f, memStoreFlusher.memstoreSize); 581 assertHeapSpace(0.1f, blockCache.maxSize); 582 } 583 584 @Test 585 public void testWhenCombinedHeapSizesFromTunerGoesOutSideMaxLimit() throws Exception { 586 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 587 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 588 Configuration conf = HBaseConfiguration.create(); 589 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.7f); 590 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.1f); 591 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 592 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.1f); 593 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 594 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 595 conf.setClass(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_CLASS, CustomHeapMemoryTuner.class, 596 HeapMemoryTuner.class); 597 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 598 new RegionServerStub(conf), new RegionServerAccountingStub(conf)); 599 long oldMemstoreSize = memStoreFlusher.memstoreSize; 600 long oldBlockCacheSize = blockCache.maxSize; 601 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 602 heapMemoryManager.start(choreService); 603 CustomHeapMemoryTuner.memstoreSize = 0.7f; 604 CustomHeapMemoryTuner.blockCacheSize = 0.3f; 605 // Allow the tuner to run once and do necessary memory up 606 Thread.sleep(1500); 607 assertEquals(oldMemstoreSize, memStoreFlusher.memstoreSize); 608 assertEquals(oldBlockCacheSize, blockCache.maxSize); 609 } 610 611 private void assertHeapSpace(float expectedHeapPercentage, long currentHeapSpace) { 612 long expected = (long) (this.maxHeapSize * expectedHeapPercentage); 613 assertEquals(expected, currentHeapSpace); 614 } 615 616 private void assertHeapSpaceDelta(double expectedDeltaPercent, long oldHeapSpace, 617 long newHeapSpace) { 618 double expctedMinDelta = (double) (this.maxHeapSize * expectedDeltaPercent); 619 // Tolerable error 620 double error = 0.95; 621 if (expectedDeltaPercent > 0) { 622 assertTrue(expctedMinDelta*error <= (double)(newHeapSpace - oldHeapSpace)); 623 assertTrue(expctedMinDelta/error >= (double)(newHeapSpace - oldHeapSpace)); 624 } else { 625 assertTrue(-expctedMinDelta*error <= (double)(oldHeapSpace - newHeapSpace)); 626 assertTrue(-expctedMinDelta/error >= (double)(oldHeapSpace - newHeapSpace)); 627 } 628 } 629 630 631 private void waitForTune(final MemstoreFlusherStub memStoreFlusher, 632 final long oldMemstoreHeapSize) throws Exception { 633 // Allow the tuner to run once and do necessary memory up 634 UTIL.waitFor(10000, new Waiter.Predicate<Exception>() { 635 @Override 636 public boolean evaluate() throws Exception { 637 return oldMemstoreHeapSize != memStoreFlusher.memstoreSize; 638 } 639 }); 640 } 641 642 private static class BlockCacheStub implements ResizableBlockCache { 643 CacheStats stats = new CacheStats("test"); 644 long maxSize = 0; 645 private long testBlockSize = 0; 646 647 public BlockCacheStub(long size){ 648 this.maxSize = size; 649 } 650 651 @Override 652 public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory) { 653 654 } 655 656 @Override 657 public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf) { 658 659 } 660 661 @Override 662 public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching, boolean repeat, 663 boolean updateCacheMetrics) { 664 return null; 665 } 666 667 @Override 668 public boolean evictBlock(BlockCacheKey cacheKey) { 669 stats.evicted(0, cacheKey != null ? cacheKey.isPrimary() : true); 670 return false; 671 } 672 673 @Override 674 public int evictBlocksByHfileName(String hfileName) { 675 stats.evicted(0, true); // Just assuming only one block for file here. 676 return 0; 677 } 678 679 @Override 680 public CacheStats getStats() { 681 return this.stats; 682 } 683 684 @Override 685 public void shutdown() { 686 687 } 688 689 @Override 690 public long size() { 691 return 0; 692 } 693 694 @Override 695 public long getMaxSize() { 696 return 0; 697 } 698 699 @Override 700 public long getFreeSize() { 701 return 0; 702 } 703 704 @Override 705 public long getCurrentSize() { 706 return this.testBlockSize; 707 } 708 709 @Override 710 public long getCurrentDataSize() { 711 return 0; 712 } 713 714 @Override 715 public long getBlockCount() { 716 return 0; 717 } 718 719 @Override 720 public long getDataBlockCount() { 721 return 0; 722 } 723 724 @Override 725 public void setMaxSize(long size) { 726 this.maxSize = size; 727 } 728 729 @Override 730 public Iterator<CachedBlock> iterator() { 731 return null; 732 } 733 734 @Override 735 public BlockCache[] getBlockCaches() { 736 return null; 737 } 738 739 @Override 740 public void returnBlock(BlockCacheKey cacheKey, Cacheable buf) { 741 } 742 743 public void setTestBlockSize(long testBlockSize) { 744 this.testBlockSize = testBlockSize; 745 } 746 } 747 748 private static class MemstoreFlusherStub implements FlushRequester { 749 750 long memstoreSize; 751 752 FlushRequestListener listener; 753 754 FlushType flushType = FlushType.NORMAL; 755 756 public MemstoreFlusherStub(long memstoreSize) { 757 this.memstoreSize = memstoreSize; 758 } 759 760 @Override 761 public void requestFlush(HRegion region, boolean forceFlushAllStores, 762 FlushLifeCycleTracker tracker) { 763 this.listener.flushRequested(flushType, region); 764 } 765 766 @Override 767 public void requestDelayedFlush(HRegion region, long delay, boolean forceFlushAllStores) { 768 769 } 770 771 @Override 772 public void registerFlushRequestListener(FlushRequestListener listener) { 773 this.listener = listener; 774 } 775 776 @Override 777 public boolean unregisterFlushRequestListener(FlushRequestListener listener) { 778 return false; 779 } 780 781 @Override 782 public void setGlobalMemStoreLimit(long globalMemStoreSize) { 783 this.memstoreSize = globalMemStoreSize; 784 } 785 } 786 787 private static class RegionServerStub implements Server { 788 private Configuration conf; 789 private boolean stopped = false; 790 791 public RegionServerStub(Configuration conf) { 792 this.conf = conf; 793 } 794 795 @Override 796 public void abort(String why, Throwable e) { 797 798 } 799 800 @Override 801 public boolean isAborted() { 802 return false; 803 } 804 805 @Override 806 public void stop(String why) { 807 this.stopped = true; 808 } 809 810 @Override 811 public boolean isStopped() { 812 return this.stopped; 813 } 814 815 @Override 816 public Configuration getConfiguration() { 817 return this.conf; 818 } 819 820 @Override 821 public ZKWatcher getZooKeeper() { 822 return null; 823 } 824 825 @Override 826 public CoordinatedStateManager getCoordinatedStateManager() { 827 return null; 828 } 829 830 @Override 831 public ClusterConnection getConnection() { 832 return null; 833 } 834 835 @Override 836 public MetaTableLocator getMetaTableLocator() { 837 return null; 838 } 839 840 @Override 841 public ServerName getServerName() { 842 return ServerName.valueOf("server1",4000,12345); 843 } 844 845 @Override 846 public ChoreService getChoreService() { 847 return null; 848 } 849 850 @Override 851 public ClusterConnection getClusterConnection() { 852 // TODO Auto-generated method stub 853 return null; 854 } 855 856 @Override 857 public FileSystem getFileSystem() { 858 return null; 859 } 860 861 @Override 862 public boolean isStopping() { 863 return false; 864 } 865 866 @Override 867 public Connection createConnection(Configuration conf) throws IOException { 868 return null; 869 } 870 } 871 872 static class CustomHeapMemoryTuner implements HeapMemoryTuner { 873 static float blockCacheSize = 0.4f; 874 static float memstoreSize = 0.4f; 875 876 @Override 877 public Configuration getConf() { 878 return null; 879 } 880 881 @Override 882 public void setConf(Configuration arg0) { 883 884 } 885 886 @Override 887 public TunerResult tune(TunerContext context) { 888 TunerResult result = new TunerResult(true); 889 result.setBlockCacheSize(blockCacheSize); 890 result.setMemStoreSize(memstoreSize); 891 return result; 892 } 893 } 894 895 private static class RegionServerAccountingStub extends RegionServerAccounting { 896 boolean offheap; 897 898 public RegionServerAccountingStub(Configuration conf) { 899 super(conf); 900 } 901 902 public RegionServerAccountingStub(Configuration conf, boolean offheap) { 903 super(conf); 904 this.offheap = offheap; 905 } 906 907 private long testMemstoreSize = 0; 908 909 @Override 910 public long getGlobalMemStoreDataSize() { 911 return testMemstoreSize; 912 } 913 914 @Override 915 public long getGlobalMemStoreHeapSize() { 916 return testMemstoreSize; 917 } 918 919 @Override 920 public boolean isOffheap() { 921 return offheap; 922 } 923 924 public void setTestMemstoreSize(long testMemstoreSize) { 925 this.testMemstoreSize = testMemstoreSize; 926 } 927 } 928}