001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.regionserver; 019 020import static org.junit.jupiter.api.Assertions.assertFalse; 021import static org.junit.jupiter.api.Assertions.assertNotNull; 022import static org.junit.jupiter.api.Assertions.assertTrue; 023import static org.mockito.Mockito.mock; 024import static org.mockito.Mockito.when; 025 026import org.apache.hadoop.conf.Configuration; 027import org.apache.hadoop.hbase.CompatibilityFactory; 028import org.apache.hadoop.hbase.metrics.MetricRegistries; 029import org.apache.hadoop.hbase.quotas.RpcThrottlingException; 030import org.apache.hadoop.hbase.regionserver.metrics.MetricsTableRequests; 031import org.apache.hadoop.hbase.test.MetricsAssertHelper; 032import org.apache.hadoop.hbase.testclassification.RegionServerTests; 033import org.apache.hadoop.hbase.testclassification.SmallTests; 034import org.apache.hadoop.hbase.util.JvmPauseMonitor; 035import org.junit.jupiter.api.AfterEach; 036import org.junit.jupiter.api.BeforeAll; 037import org.junit.jupiter.api.BeforeEach; 038import org.junit.jupiter.api.Tag; 039import org.junit.jupiter.api.Test; 040 041/** 042 * Unit test version of rs metrics tests. 043 */ 044@Tag(RegionServerTests.TAG) 045@Tag(SmallTests.TAG) 046public class TestMetricsRegionServer { 047 048 public static MetricsAssertHelper HELPER = 049 CompatibilityFactory.getInstance(MetricsAssertHelper.class); 050 051 private MetricsRegionServerWrapperStub wrapper; 052 private MetricsRegionServer rsm; 053 private MetricsRegionServerSource serverSource; 054 055 @BeforeAll 056 public static void classSetUp() { 057 HELPER.init(); 058 } 059 060 @BeforeEach 061 public void setUp() { 062 wrapper = new MetricsRegionServerWrapperStub(); 063 rsm = new MetricsRegionServer(wrapper, new Configuration(false), null); 064 serverSource = rsm.getMetricsSource(); 065 } 066 067 @AfterEach 068 public void tearDown() { 069 // Clean up global registries after each test to avoid interference 070 MetricRegistries.global().clear(); 071 } 072 073 @Test 074 public void testWrapperSource() { 075 HELPER.assertTag("serverName", "test", serverSource); 076 HELPER.assertTag("clusterId", "tClusterId", serverSource); 077 HELPER.assertTag("zookeeperQuorum", "zk", serverSource); 078 HELPER.assertGauge("regionServerStartTime", 100, serverSource); 079 HELPER.assertGauge("regionCount", 101, serverSource); 080 HELPER.assertGauge("storeCount", 2, serverSource); 081 HELPER.assertGauge("maxStoreFileCount", 23, serverSource); 082 HELPER.assertGauge("maxStoreFileAge", 2, serverSource); 083 HELPER.assertGauge("minStoreFileAge", 2, serverSource); 084 HELPER.assertGauge("avgStoreFileAge", 2, serverSource); 085 HELPER.assertGauge("numReferenceFiles", 2, serverSource); 086 HELPER.assertGauge("hlogFileCount", 10, serverSource); 087 HELPER.assertGauge("hlogFileSize", 1024000, serverSource); 088 HELPER.assertGauge("storeFileCount", 300, serverSource); 089 HELPER.assertGauge("memstoreSize", 1025, serverSource); 090 HELPER.assertGauge("memstoreHeapSize", 500, serverSource); 091 HELPER.assertGauge("memstoreOffHeapSize", 600, serverSource); 092 HELPER.assertGauge("storeFileSize", 1900, serverSource); 093 HELPER.assertGauge("storeFileSizeGrowthRate", 50.0, serverSource); 094 HELPER.assertCounter("totalRequestCount", 899, serverSource); 095 HELPER.assertCounter("totalRowActionRequestCount", 096 HELPER.getCounter("readRequestCount", serverSource) 097 + HELPER.getCounter("writeRequestCount", serverSource), 098 serverSource); 099 HELPER.assertCounter("readRequestCount", 997, serverSource); 100 HELPER.assertCounter("cpRequestCount", 998, serverSource); 101 HELPER.assertCounter("filteredReadRequestCount", 1997, serverSource); 102 HELPER.assertCounter("writeRequestCount", 707, serverSource); 103 HELPER.assertCounter("checkMutateFailedCount", 401, serverSource); 104 HELPER.assertCounter("checkMutatePassedCount", 405, serverSource); 105 HELPER.assertGauge("storeFileIndexSize", 406, serverSource); 106 HELPER.assertGauge("staticIndexSize", 407, serverSource); 107 HELPER.assertGauge("staticBloomSize", 408, serverSource); 108 HELPER.assertGauge("mutationsWithoutWALCount", 409, serverSource); 109 HELPER.assertGauge("mutationsWithoutWALSize", 410, serverSource); 110 HELPER.assertCounter("bloomFilterRequestsCount", 411, serverSource); 111 HELPER.assertCounter("bloomFilterNegativeResultsCount", 412, serverSource); 112 HELPER.assertCounter("bloomFilterEligibleRequestsCount", 413, serverSource); 113 HELPER.assertGauge("percentFilesLocal", 99, serverSource); 114 HELPER.assertGauge("percentFilesLocalSecondaryRegions", 99, serverSource); 115 HELPER.assertGauge("compactionQueueLength", 411, serverSource); 116 HELPER.assertGauge("flushQueueLength", 412, serverSource); 117 HELPER.assertGauge("blockCacheFreeSize", 413, serverSource); 118 HELPER.assertGauge("blockCacheCount", 414, serverSource); 119 HELPER.assertGauge("blockCacheDataBlockCount", 300, serverSource); 120 HELPER.assertGauge("blockCacheSize", 415, serverSource); 121 HELPER.assertCounter("blockCacheHitCount", 416, serverSource); 122 HELPER.assertCounter("blockCacheHitCachingCount", 16, serverSource); 123 HELPER.assertCounter("blockCacheMissCount", 417, serverSource); 124 HELPER.assertCounter("blockCacheMissCachingCount", 17, serverSource); 125 HELPER.assertCounter("blockCacheEvictionCount", 418, serverSource); 126 HELPER.assertGauge("blockCacheCountHitPercent", 98, serverSource); 127 HELPER.assertGauge("blockCacheExpressHitPercent", 97, serverSource); 128 HELPER.assertCounter("blockCacheFailedInsertionCount", 36, serverSource); 129 HELPER.assertGauge("l1CacheFreeSize", 100, serverSource); 130 HELPER.assertGauge("l1CacheSize", 123, serverSource); 131 HELPER.assertGauge("l1CacheCount", 50, serverSource); 132 HELPER.assertCounter("l1CacheEvictionCount", 1000, serverSource); 133 HELPER.assertGauge("l1CacheHitCount", 200, serverSource); 134 HELPER.assertGauge("l1CacheHitCachingCount", 200, serverSource); 135 HELPER.assertGauge("l1CacheMissCount", 100, serverSource); 136 HELPER.assertGauge("l1CacheMissCachingCount", 100, serverSource); 137 HELPER.assertGauge("l1CacheHitRatio", 80, serverSource); 138 HELPER.assertGauge("l1CacheHitCachingRatio", 80, serverSource); 139 HELPER.assertGauge("l1CacheMissRatio", 20, serverSource); 140 HELPER.assertGauge("l1CacheMissCachingRatio", 20, serverSource); 141 HELPER.assertGauge("l2CacheFreeSize", 200, serverSource); 142 HELPER.assertGauge("l2CacheSize", 456, serverSource); 143 HELPER.assertGauge("l2CacheCount", 75, serverSource); 144 HELPER.assertCounter("l2CacheEvictionCount", 2000, serverSource); 145 HELPER.assertGauge("l2CacheHitCount", 800, serverSource); 146 HELPER.assertGauge("l2CacheHitCachingCount", 800, serverSource); 147 HELPER.assertGauge("l2CacheMissCount", 200, serverSource); 148 HELPER.assertGauge("l2CacheMissCachingCount", 200, serverSource); 149 HELPER.assertGauge("l2CacheHitRatio", 90, serverSource); 150 HELPER.assertGauge("l2CacheHitCachingRatio", 90, serverSource); 151 HELPER.assertGauge("l2CacheMissRatio", 10, serverSource); 152 HELPER.assertGauge("l2CacheMissCachingRatio", 10, serverSource); 153 HELPER.assertCounter("updatesBlockedTime", 419, serverSource); 154 } 155 156 @Test 157 public void testConstuctor() { 158 assertNotNull(rsm.getMetricsSource(), "There should be a hadoop1/hadoop2 metrics source"); 159 assertNotNull(rsm.getRegionServerWrapper(), 160 "The RegionServerMetricsWrapper should be accessable"); 161 } 162 163 @Test 164 public void testSlowCount() { 165 HRegion region = mock(HRegion.class); 166 MetricsTableRequests metricsTableRequests = mock(MetricsTableRequests.class); 167 when(region.getMetricsTableRequests()).thenReturn(metricsTableRequests); 168 when(metricsTableRequests.isEnableTableLatenciesMetrics()).thenReturn(false); 169 when(metricsTableRequests.isEnabTableQueryMeterMetrics()).thenReturn(false); 170 for (int i = 0; i < 12; i++) { 171 rsm.updateAppend(region, 12, 120); 172 rsm.updateAppend(region, 1002, 10020); 173 } 174 for (int i = 0; i < 13; i++) { 175 rsm.updateDeleteBatch(region, 13); 176 rsm.updateDeleteBatch(region, 1003); 177 } 178 for (int i = 0; i < 14; i++) { 179 rsm.updateGet(region, 14, 140); 180 rsm.updateGet(region, 1004, 10040); 181 } 182 for (int i = 0; i < 15; i++) { 183 rsm.updateIncrement(region, 15, 150); 184 rsm.updateIncrement(region, 1005, 10050); 185 } 186 for (int i = 0; i < 16; i++) { 187 rsm.updatePutBatch(region, 16); 188 rsm.updatePutBatch(region, 1006); 189 } 190 191 for (int i = 0; i < 17; i++) { 192 rsm.updatePut(region, 17); 193 rsm.updateDelete(region, 17); 194 rsm.updatePut(region, 1006); 195 rsm.updateDelete(region, 1003); 196 rsm.updateCheckAndDelete(region, 17); 197 rsm.updateCheckAndPut(region, 17); 198 rsm.updateCheckAndMutate(region, 17, 170); 199 } 200 201 HELPER.assertCounter("blockBytesScannedCount", 420090, serverSource); 202 HELPER.assertCounter("appendNumOps", 24, serverSource); 203 HELPER.assertCounter("appendBlockBytesScannedNumOps", 24, serverSource); 204 HELPER.assertCounter("deleteBatchNumOps", 26, serverSource); 205 HELPER.assertCounter("getNumOps", 28, serverSource); 206 HELPER.assertCounter("getBlockBytesScannedNumOps", 28, serverSource); 207 HELPER.assertCounter("incrementNumOps", 30, serverSource); 208 HELPER.assertCounter("incrementBlockBytesScannedNumOps", 30, serverSource); 209 HELPER.assertCounter("putBatchNumOps", 32, serverSource); 210 HELPER.assertCounter("putNumOps", 34, serverSource); 211 HELPER.assertCounter("deleteNumOps", 34, serverSource); 212 HELPER.assertCounter("checkAndDeleteNumOps", 17, serverSource); 213 HELPER.assertCounter("checkAndPutNumOps", 17, serverSource); 214 HELPER.assertCounter("checkAndMutateNumOps", 17, serverSource); 215 HELPER.assertCounter("checkAndMutateBlockBytesScannedNumOps", 17, serverSource); 216 217 HELPER.assertCounter("slowAppendCount", 12, serverSource); 218 HELPER.assertCounter("slowDeleteCount", 17, serverSource); 219 HELPER.assertCounter("slowGetCount", 14, serverSource); 220 HELPER.assertCounter("slowIncrementCount", 15, serverSource); 221 HELPER.assertCounter("slowPutCount", 17, serverSource); 222 } 223 224 @Test 225 public void testFlush() { 226 rsm.updateFlush(null, 1, 2, 3); 227 HELPER.assertCounter("flushTime_num_ops", 1, serverSource); 228 HELPER.assertCounter("flushMemstoreSize_num_ops", 1, serverSource); 229 HELPER.assertCounter("flushOutputSize_num_ops", 1, serverSource); 230 HELPER.assertCounter("flushedMemstoreBytes", 2, serverSource); 231 HELPER.assertCounter("flushedOutputBytes", 3, serverSource); 232 233 rsm.updateFlush(null, 10, 20, 30); 234 HELPER.assertCounter("flushTimeNumOps", 2, serverSource); 235 HELPER.assertCounter("flushMemstoreSize_num_ops", 2, serverSource); 236 HELPER.assertCounter("flushOutputSize_num_ops", 2, serverSource); 237 HELPER.assertCounter("flushedMemstoreBytes", 22, serverSource); 238 HELPER.assertCounter("flushedOutputBytes", 33, serverSource); 239 } 240 241 @Test 242 public void testCompaction() { 243 rsm.updateCompaction(null, false, 1, 2, 3, 4, 5); 244 HELPER.assertCounter("compactionTime_num_ops", 1, serverSource); 245 HELPER.assertCounter("compactionInputFileCount_num_ops", 1, serverSource); 246 HELPER.assertCounter("compactionInputSize_num_ops", 1, serverSource); 247 HELPER.assertCounter("compactionOutputFileCount_num_ops", 1, serverSource); 248 HELPER.assertCounter("compactedInputBytes", 4, serverSource); 249 HELPER.assertCounter("compactedoutputBytes", 5, serverSource); 250 251 rsm.updateCompaction(null, false, 10, 20, 30, 40, 50); 252 HELPER.assertCounter("compactionTime_num_ops", 2, serverSource); 253 HELPER.assertCounter("compactionInputFileCount_num_ops", 2, serverSource); 254 HELPER.assertCounter("compactionInputSize_num_ops", 2, serverSource); 255 HELPER.assertCounter("compactionOutputFileCount_num_ops", 2, serverSource); 256 HELPER.assertCounter("compactedInputBytes", 44, serverSource); 257 HELPER.assertCounter("compactedoutputBytes", 55, serverSource); 258 259 // do major compaction 260 rsm.updateCompaction(null, true, 100, 200, 300, 400, 500); 261 262 HELPER.assertCounter("compactionTime_num_ops", 3, serverSource); 263 HELPER.assertCounter("compactionInputFileCount_num_ops", 3, serverSource); 264 HELPER.assertCounter("compactionInputSize_num_ops", 3, serverSource); 265 HELPER.assertCounter("compactionOutputFileCount_num_ops", 3, serverSource); 266 HELPER.assertCounter("compactedInputBytes", 444, serverSource); 267 HELPER.assertCounter("compactedoutputBytes", 555, serverSource); 268 269 HELPER.assertCounter("majorCompactionTime_num_ops", 1, serverSource); 270 HELPER.assertCounter("majorCompactionInputFileCount_num_ops", 1, serverSource); 271 HELPER.assertCounter("majorCompactionInputSize_num_ops", 1, serverSource); 272 HELPER.assertCounter("majorCompactionOutputFileCount_num_ops", 1, serverSource); 273 HELPER.assertCounter("majorCompactedInputBytes", 400, serverSource); 274 HELPER.assertCounter("majorCompactedoutputBytes", 500, serverSource); 275 } 276 277 @Test 278 public void testPauseMonitor() { 279 Configuration conf = new Configuration(); 280 conf.setLong(JvmPauseMonitor.INFO_THRESHOLD_KEY, 1000L); 281 conf.setLong(JvmPauseMonitor.WARN_THRESHOLD_KEY, 10000L); 282 JvmPauseMonitor monitor = new JvmPauseMonitor(conf, serverSource); 283 monitor.updateMetrics(1500, false); 284 HELPER.assertCounter("pauseInfoThresholdExceeded", 1, serverSource); 285 HELPER.assertCounter("pauseWarnThresholdExceeded", 0, serverSource); 286 HELPER.assertCounter("pauseTimeWithoutGc_num_ops", 1, serverSource); 287 HELPER.assertCounter("pauseTimeWithGc_num_ops", 0, serverSource); 288 monitor.updateMetrics(15000, true); 289 HELPER.assertCounter("pauseInfoThresholdExceeded", 1, serverSource); 290 HELPER.assertCounter("pauseWarnThresholdExceeded", 1, serverSource); 291 HELPER.assertCounter("pauseTimeWithoutGc_num_ops", 1, serverSource); 292 HELPER.assertCounter("pauseTimeWithGc_num_ops", 1, serverSource); 293 } 294 295 @Test 296 public void testScannerMetrics() { 297 HELPER.assertCounter("scannerLeaseExpiredCount", 0, serverSource); 298 rsm.incrScannerLeaseExpired(); 299 HELPER.assertCounter("scannerLeaseExpiredCount", 1, serverSource); 300 HELPER.assertGauge("activeScanners", 0, serverSource); 301 } 302 303 @Test 304 public void testTableQueryMeterSwitch() { 305 HRegion region = mock(HRegion.class); 306 MetricsTableRequests metricsTableRequests = mock(MetricsTableRequests.class); 307 when(region.getMetricsTableRequests()).thenReturn(metricsTableRequests); 308 when(metricsTableRequests.isEnableTableLatenciesMetrics()).thenReturn(false); 309 when(metricsTableRequests.isEnabTableQueryMeterMetrics()).thenReturn(false); 310 Configuration conf = new Configuration(false); 311 // disable 312 rsm.updateReadQueryMeter(region, 500L); 313 assertFalse(HELPER.checkGaugeExists("ServerReadQueryPerSecond_count", serverSource)); 314 rsm.updateWriteQueryMeter(region, 500L); 315 assertFalse(HELPER.checkGaugeExists("ServerWriteQueryPerSecond_count", serverSource)); 316 317 // enable 318 conf.setBoolean(MetricsRegionServer.RS_ENABLE_SERVER_QUERY_METER_METRICS_KEY, true); 319 rsm = new MetricsRegionServer(wrapper, conf, null); 320 serverSource = rsm.getMetricsSource(); 321 rsm.updateReadQueryMeter(region, 500L); 322 assertTrue(HELPER.checkGaugeExists("ServerWriteQueryPerSecond_count", serverSource)); 323 HELPER.assertGauge("ServerReadQueryPerSecond_count", 500L, serverSource); 324 assertTrue(HELPER.checkGaugeExists("ServerWriteQueryPerSecond_count", serverSource)); 325 rsm.updateWriteQueryMeter(region, 500L); 326 HELPER.assertGauge("ServerWriteQueryPerSecond_count", 500L, serverSource); 327 } 328 329 @Test 330 public void testThrottleExceptionMetricsIntegration() { 331 // Record different types of throttle exceptions 332 rsm.recordThrottleException(RpcThrottlingException.Type.NumRequestsExceeded, "alice", "users"); 333 rsm.recordThrottleException(RpcThrottlingException.Type.WriteSizeExceeded, "bob", "logs"); 334 rsm.recordThrottleException(RpcThrottlingException.Type.ReadSizeExceeded, "charlie", 335 "metadata"); 336 337 // Record the same exception multiple times to test increment 338 rsm.recordThrottleException(RpcThrottlingException.Type.NumRequestsExceeded, "alice", "users"); 339 rsm.recordThrottleException(RpcThrottlingException.Type.NumRequestsExceeded, "alice", "users"); 340 341 // Verify the specific counters were created and have correct values using HELPER 342 HELPER.assertCounter("RpcThrottlingException_Type_NumRequestsExceeded_User_alice_Table_users", 343 3L, serverSource); 344 HELPER.assertCounter("RpcThrottlingException_Type_WriteSizeExceeded_User_bob_Table_logs", 1L, 345 serverSource); 346 HELPER.assertCounter("RpcThrottlingException_Type_ReadSizeExceeded_User_charlie_Table_metadata", 347 1L, serverSource); 348 349 // Test metric name sanitization through the integration 350 rsm.recordThrottleException(RpcThrottlingException.Type.RequestSizeExceeded, 351 "user.with@special", "table:with,problematic=chars"); 352 HELPER.assertCounter( 353 "RpcThrottlingException_Type_RequestSizeExceeded_User_user.with@special_Table_table_with_problematic_chars", 354 1L, serverSource); 355 356 // Test null handling through the integration 357 rsm.recordThrottleException(RpcThrottlingException.Type.ReadCapacityUnitExceeded, null, null); 358 HELPER.assertCounter( 359 "RpcThrottlingException_Type_ReadCapacityUnitExceeded_User_unknown_Table_unknown", 1L, 360 serverSource); 361 } 362 363 @Test 364 public void testSplitRequest() { 365 rsm.incrSplitRequest(null); 366 HELPER.assertCounter("splitRequestCount", 1, serverSource); 367 } 368}