001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.regionserver; 019 020import java.io.IOException; 021import java.util.ArrayList; 022import java.util.Collection; 023import java.util.Collections; 024import java.util.List; 025import java.util.Map; 026import java.util.OptionalDouble; 027import java.util.OptionalLong; 028import java.util.concurrent.ConcurrentHashMap; 029import java.util.concurrent.ScheduledExecutorService; 030import java.util.concurrent.TimeUnit; 031import java.util.stream.Collectors; 032import org.apache.commons.lang3.StringUtils; 033import org.apache.hadoop.hbase.CompatibilitySingletonFactory; 034import org.apache.hadoop.hbase.HConstants; 035import org.apache.hadoop.hbase.HDFSBlocksDistribution; 036import org.apache.hadoop.hbase.ServerName; 037import org.apache.hadoop.hbase.client.RegionInfo; 038import org.apache.hadoop.hbase.io.ByteBuffAllocator; 039import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper; 040import org.apache.hadoop.hbase.io.asyncfs.monitor.ExcludeDatanodeManager; 041import org.apache.hadoop.hbase.io.hfile.BlockCache; 042import org.apache.hadoop.hbase.io.hfile.CacheStats; 043import org.apache.hadoop.hbase.io.hfile.CombinedBlockCache; 044import org.apache.hadoop.hbase.mob.MobFileCache; 045import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource; 046import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; 047import org.apache.hadoop.hbase.util.FSUtils; 048import org.apache.hadoop.hbase.wal.WALProvider; 049import org.apache.hadoop.hbase.zookeeper.ZKWatcher; 050import org.apache.hadoop.hdfs.DFSHedgedReadMetrics; 051import org.apache.hadoop.metrics2.MetricsExecutor; 052import org.apache.yetus.audience.InterfaceAudience; 053import org.slf4j.Logger; 054import org.slf4j.LoggerFactory; 055 056/** 057 * Impl for exposing HRegionServer Information through Hadoop's metrics 2 system. 058 */ 059@InterfaceAudience.Private 060class MetricsRegionServerWrapperImpl implements MetricsRegionServerWrapper { 061 062 private static final Logger LOG = LoggerFactory.getLogger(MetricsRegionServerWrapperImpl.class); 063 064 private final HRegionServer regionServer; 065 private final MetricsWALSource metricsWALSource; 066 private final ByteBuffAllocator allocator; 067 068 private BlockCache blockCache; 069 private MobFileCache mobFileCache; 070 private CacheStats cacheStats; 071 private CacheStats l1Stats = null; 072 private CacheStats l2Stats = null; 073 074 private volatile long numStores = 0; 075 private volatile long numWALFiles = 0; 076 private volatile long walFileSize = 0; 077 private volatile long numStoreFiles = 0; 078 private volatile long memstoreSize = 0; 079 private volatile long onHeapMemstoreSize = 0; 080 private volatile long offHeapMemstoreSize = 0; 081 private volatile long storeFileSize = 0; 082 private volatile double storeFileSizeGrowthRate = 0; 083 private volatile long maxStoreFileAge = 0; 084 private volatile long minStoreFileAge = 0; 085 private volatile long avgStoreFileAge = 0; 086 private volatile long numReferenceFiles = 0; 087 private volatile double requestsPerSecond = 0.0; 088 private volatile long readRequestsCount = 0; 089 private volatile double readRequestsRatePerSecond = 0; 090 private volatile long cpRequestsCount = 0; 091 private volatile long filteredReadRequestsCount = 0; 092 private volatile long writeRequestsCount = 0; 093 private volatile double writeRequestsRatePerSecond = 0; 094 private volatile long checkAndMutateChecksFailed = 0; 095 private volatile long checkAndMutateChecksPassed = 0; 096 private volatile long storefileIndexSize = 0; 097 private volatile long totalStaticIndexSize = 0; 098 private volatile long totalStaticBloomSize = 0; 099 private volatile long numMutationsWithoutWAL = 0; 100 private volatile long dataInMemoryWithoutWAL = 0; 101 private volatile double percentFileLocal = 0; 102 private volatile double percentFileLocalSecondaryRegions = 0; 103 private volatile long flushedCellsCount = 0; 104 private volatile long compactedCellsCount = 0; 105 private volatile long majorCompactedCellsCount = 0; 106 private volatile long flushedCellsSize = 0; 107 private volatile long compactedCellsSize = 0; 108 private volatile long majorCompactedCellsSize = 0; 109 private volatile long cellsCountCompactedToMob = 0; 110 private volatile long cellsCountCompactedFromMob = 0; 111 private volatile long cellsSizeCompactedToMob = 0; 112 private volatile long cellsSizeCompactedFromMob = 0; 113 private volatile long mobFlushCount = 0; 114 private volatile long mobFlushedCellsCount = 0; 115 private volatile long mobFlushedCellsSize = 0; 116 private volatile long mobScanCellsCount = 0; 117 private volatile long mobScanCellsSize = 0; 118 private volatile long mobFileCacheAccessCount = 0; 119 private volatile long mobFileCacheMissCount = 0; 120 private volatile double mobFileCacheHitRatio = 0; 121 private volatile long mobFileCacheEvictedCount = 0; 122 private volatile long mobFileCacheCount = 0; 123 private volatile long blockedRequestsCount = 0L; 124 private volatile long averageRegionSize = 0L; 125 protected final Map<String, ArrayList<Long>> requestsCountCache = 126 new ConcurrentHashMap<String, ArrayList<Long>>(); 127 128 private ScheduledExecutorService executor; 129 private Runnable runnable; 130 private long period; 131 132 /** 133 * Can be null if not on hdfs. 134 */ 135 private DFSHedgedReadMetrics dfsHedgedReadMetrics; 136 137 private final ExcludeDatanodeManager excludeDatanodeManager; 138 139 public MetricsRegionServerWrapperImpl(final HRegionServer regionServer) { 140 this.regionServer = regionServer; 141 initBlockCache(); 142 initMobFileCache(); 143 this.excludeDatanodeManager = this.regionServer.getWalFactory().getExcludeDatanodeManager(); 144 145 this.period = regionServer.getConfiguration().getLong(HConstants.REGIONSERVER_METRICS_PERIOD, 146 HConstants.DEFAULT_REGIONSERVER_METRICS_PERIOD); 147 148 this.executor = CompatibilitySingletonFactory.getInstance(MetricsExecutor.class).getExecutor(); 149 this.runnable = new RegionServerMetricsWrapperRunnable(); 150 this.executor.scheduleWithFixedDelay(this.runnable, this.period, this.period, 151 TimeUnit.MILLISECONDS); 152 this.metricsWALSource = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class); 153 this.allocator = regionServer.getRpcServer().getByteBuffAllocator(); 154 155 try { 156 this.dfsHedgedReadMetrics = FSUtils.getDFSHedgedReadMetrics(regionServer.getConfiguration()); 157 } catch (IOException e) { 158 LOG.warn("Failed to get hedged metrics", e); 159 } 160 if (LOG.isInfoEnabled()) { 161 LOG.info("Computing regionserver metrics every " + this.period + " milliseconds"); 162 } 163 } 164 165 private void initBlockCache() { 166 this.blockCache = this.regionServer.getBlockCache().orElse(null); 167 this.cacheStats = this.blockCache != null ? this.blockCache.getStats() : null; 168 if (this.cacheStats != null) { 169 if (this.cacheStats instanceof CombinedBlockCache.CombinedCacheStats) { 170 l1Stats = ((CombinedBlockCache.CombinedCacheStats) this.cacheStats).getLruCacheStats(); 171 l2Stats = ((CombinedBlockCache.CombinedCacheStats) this.cacheStats).getBucketCacheStats(); 172 } else { 173 l1Stats = this.cacheStats; 174 } 175 } 176 } 177 178 /** 179 * Initializes the mob file cache. 180 */ 181 private void initMobFileCache() { 182 this.mobFileCache = this.regionServer.getMobFileCache().orElse(null); 183 } 184 185 @Override 186 public String getClusterId() { 187 return regionServer.getClusterId(); 188 } 189 190 @Override 191 public long getStartCode() { 192 return regionServer.getStartcode(); 193 } 194 195 @Override 196 public String getZookeeperQuorum() { 197 ZKWatcher zk = regionServer.getZooKeeper(); 198 if (zk == null) { 199 return ""; 200 } 201 return zk.getQuorum(); 202 } 203 204 @Override 205 public String getCoprocessors() { 206 String[] coprocessors = regionServer.getRegionServerCoprocessors(); 207 if (coprocessors == null || coprocessors.length == 0) { 208 return ""; 209 } 210 return StringUtils.join(coprocessors, ", "); 211 } 212 213 @Override 214 public String getServerName() { 215 ServerName serverName = regionServer.getServerName(); 216 if (serverName == null) { 217 return ""; 218 } 219 return serverName.getServerName(); 220 } 221 222 @Override 223 public long getNumOnlineRegions() { 224 Collection<HRegion> onlineRegionsLocalContext = regionServer.getOnlineRegionsLocalContext(); 225 if (onlineRegionsLocalContext == null) { 226 return 0; 227 } 228 return onlineRegionsLocalContext.size(); 229 } 230 231 @Override 232 public long getTotalRequestCount() { 233 return regionServer.getRpcServices().requestCount.sum(); 234 } 235 236 @Override 237 public long getTotalRowActionRequestCount() { 238 return readRequestsCount + writeRequestsCount; 239 } 240 241 @Override 242 public int getSplitQueueSize() { 243 final CompactSplit compactSplit = regionServer.getCompactSplitThread(); 244 return compactSplit == null ? 0 : compactSplit.getSplitQueueSize(); 245 } 246 247 @Override 248 public int getCompactionQueueSize() { 249 final CompactSplit compactSplit = regionServer.getCompactSplitThread(); 250 return compactSplit == null ? 0 : compactSplit.getCompactionQueueSize(); 251 } 252 253 @Override 254 public int getSmallCompactionQueueSize() { 255 final CompactSplit compactSplit = regionServer.getCompactSplitThread(); 256 return compactSplit == null ? 0 : compactSplit.getSmallCompactionQueueSize(); 257 } 258 259 @Override 260 public int getLargeCompactionQueueSize() { 261 final CompactSplit compactSplit = regionServer.getCompactSplitThread(); 262 return compactSplit == null ? 0 : compactSplit.getLargeCompactionQueueSize(); 263 } 264 265 @Override 266 public int getFlushQueueSize() { 267 // If there is no flusher there should be no queue. 268 if (this.regionServer.getMemStoreFlusher() == null) { 269 return 0; 270 } 271 return this.regionServer.getMemStoreFlusher().getFlushQueueSize(); 272 } 273 274 @Override 275 public long getBlockCacheCount() { 276 return this.blockCache != null ? this.blockCache.getBlockCount() : 0L; 277 } 278 279 @Override 280 public long getMemStoreLimit() { 281 return this.regionServer.getRegionServerAccounting().getGlobalMemStoreLimit(); 282 } 283 284 @Override 285 public long getOnHeapMemStoreLimit() { 286 return this.regionServer.getRegionServerAccounting().getGlobalOnHeapMemStoreLimit(); 287 } 288 289 @Override 290 public long getOffHeapMemStoreLimit() { 291 return this.regionServer.getRegionServerAccounting().getGlobalOffHeapMemStoreLimit(); 292 } 293 294 @Override 295 public long getBlockCacheSize() { 296 return this.blockCache != null ? this.blockCache.getCurrentSize() : 0L; 297 } 298 299 @Override 300 public long getBlockCacheFreeSize() { 301 return this.blockCache != null ? this.blockCache.getFreeSize() : 0L; 302 } 303 304 @Override 305 public long getBlockCacheHitCount() { 306 return this.cacheStats != null ? this.cacheStats.getHitCount() : 0L; 307 } 308 309 @Override 310 public long getBlockCachePrimaryHitCount() { 311 return this.cacheStats != null ? this.cacheStats.getPrimaryHitCount() : 0L; 312 } 313 314 @Override 315 public long getBlockCacheMissCount() { 316 return this.cacheStats != null ? this.cacheStats.getMissCount() : 0L; 317 } 318 319 @Override 320 public long getBlockCachePrimaryMissCount() { 321 return this.cacheStats != null ? this.cacheStats.getPrimaryMissCount() : 0L; 322 } 323 324 @Override 325 public long getBlockCacheEvictedCount() { 326 return this.cacheStats != null ? this.cacheStats.getEvictedCount() : 0L; 327 } 328 329 @Override 330 public long getBlockCachePrimaryEvictedCount() { 331 return this.cacheStats != null ? this.cacheStats.getPrimaryEvictedCount() : 0L; 332 } 333 334 @Override 335 public double getBlockCacheHitPercent() { 336 double ratio = this.cacheStats != null ? this.cacheStats.getHitRatio() : 0.0; 337 if (Double.isNaN(ratio)) { 338 ratio = 0; 339 } 340 return (ratio * 100); 341 } 342 343 @Override 344 public double getBlockCacheHitCachingPercent() { 345 double ratio = this.cacheStats != null ? this.cacheStats.getHitCachingRatio() : 0.0; 346 if (Double.isNaN(ratio)) { 347 ratio = 0; 348 } 349 return (ratio * 100); 350 } 351 352 @Override 353 public long getBlockCacheFailedInsertions() { 354 return this.cacheStats != null ? this.cacheStats.getFailedInserts() : 0L; 355 } 356 357 @Override 358 public long getL1CacheHitCount() { 359 return this.l1Stats != null ? this.l1Stats.getHitCount() : 0L; 360 } 361 362 @Override 363 public long getL1CacheMissCount() { 364 return this.l1Stats != null ? this.l1Stats.getMissCount() : 0L; 365 } 366 367 @Override 368 public double getL1CacheHitRatio() { 369 return this.l1Stats != null ? this.l1Stats.getHitRatio() : 0.0; 370 } 371 372 @Override 373 public double getL1CacheMissRatio() { 374 return this.l1Stats != null ? this.l1Stats.getMissRatio() : 0.0; 375 } 376 377 @Override 378 public long getL2CacheHitCount() { 379 return this.l2Stats != null ? this.l2Stats.getHitCount() : 0L; 380 } 381 382 @Override 383 public long getL2CacheMissCount() { 384 return this.l2Stats != null ? this.l2Stats.getMissCount() : 0L; 385 } 386 387 @Override 388 public double getL2CacheHitRatio() { 389 return this.l2Stats != null ? this.l2Stats.getHitRatio() : 0.0; 390 } 391 392 @Override 393 public double getL2CacheMissRatio() { 394 return this.l2Stats != null ? this.l2Stats.getMissRatio() : 0.0; 395 } 396 397 @Override 398 public void forceRecompute() { 399 this.runnable.run(); 400 } 401 402 @Override 403 public long getNumStores() { 404 return numStores; 405 } 406 407 @Override 408 public long getNumWALFiles() { 409 return numWALFiles; 410 } 411 412 @Override 413 public long getWALFileSize() { 414 return walFileSize; 415 } 416 417 @Override 418 public List<String> getWALExcludeDNs() { 419 if (excludeDatanodeManager == null) { 420 return Collections.emptyList(); 421 } 422 return excludeDatanodeManager.getExcludeDNs().entrySet().stream() 423 .map(e -> e.getKey().toString() + ", " + e.getValue()).collect(Collectors.toList()); 424 } 425 426 @Override 427 public long getNumWALSlowAppend() { 428 return metricsWALSource.getSlowAppendCount(); 429 } 430 431 @Override 432 public long getNumStoreFiles() { 433 return numStoreFiles; 434 } 435 436 @Override 437 public long getMaxStoreFileAge() { 438 return maxStoreFileAge; 439 } 440 441 @Override 442 public long getMinStoreFileAge() { 443 return minStoreFileAge; 444 } 445 446 @Override 447 public long getAvgStoreFileAge() { 448 return avgStoreFileAge; 449 } 450 451 @Override 452 public long getNumReferenceFiles() { 453 return numReferenceFiles; 454 } 455 456 @Override 457 public long getMemStoreSize() { 458 return memstoreSize; 459 } 460 461 @Override 462 public long getOnHeapMemStoreSize() { 463 return onHeapMemstoreSize; 464 } 465 466 @Override 467 public long getOffHeapMemStoreSize() { 468 return offHeapMemstoreSize; 469 } 470 471 @Override 472 public long getStoreFileSize() { 473 return storeFileSize; 474 } 475 476 @Override 477 public double getStoreFileSizeGrowthRate() { 478 return storeFileSizeGrowthRate; 479 } 480 481 @Override 482 public double getRequestsPerSecond() { 483 return requestsPerSecond; 484 } 485 486 @Override 487 public long getReadRequestsCount() { 488 return readRequestsCount; 489 } 490 491 @Override 492 public long getCpRequestsCount() { 493 return cpRequestsCount; 494 } 495 496 @Override 497 public double getReadRequestsRatePerSecond() { 498 return readRequestsRatePerSecond; 499 } 500 501 @Override 502 public long getFilteredReadRequestsCount() { 503 return filteredReadRequestsCount; 504 } 505 506 @Override 507 public long getWriteRequestsCount() { 508 return writeRequestsCount; 509 } 510 511 @Override 512 public double getWriteRequestsRatePerSecond() { 513 return writeRequestsRatePerSecond; 514 } 515 516 @Override 517 public long getRpcGetRequestsCount() { 518 return regionServer.getRpcServices().rpcGetRequestCount.sum(); 519 } 520 521 @Override 522 public long getRpcScanRequestsCount() { 523 return regionServer.getRpcServices().rpcScanRequestCount.sum(); 524 } 525 526 @Override 527 public long getRpcFullScanRequestsCount() { 528 return regionServer.getRpcServices().rpcFullScanRequestCount.sum(); 529 } 530 531 @Override 532 public long getRpcMultiRequestsCount() { 533 return regionServer.getRpcServices().rpcMultiRequestCount.sum(); 534 } 535 536 @Override 537 public long getRpcMutateRequestsCount() { 538 return regionServer.getRpcServices().rpcMutateRequestCount.sum(); 539 } 540 541 @Override 542 public long getCheckAndMutateChecksFailed() { 543 return checkAndMutateChecksFailed; 544 } 545 546 @Override 547 public long getCheckAndMutateChecksPassed() { 548 return checkAndMutateChecksPassed; 549 } 550 551 @Override 552 public long getStoreFileIndexSize() { 553 return storefileIndexSize; 554 } 555 556 @Override 557 public long getTotalStaticIndexSize() { 558 return totalStaticIndexSize; 559 } 560 561 @Override 562 public long getTotalStaticBloomSize() { 563 return totalStaticBloomSize; 564 } 565 566 @Override 567 public long getNumMutationsWithoutWAL() { 568 return numMutationsWithoutWAL; 569 } 570 571 @Override 572 public long getDataInMemoryWithoutWAL() { 573 return dataInMemoryWithoutWAL; 574 } 575 576 @Override 577 public double getPercentFileLocal() { 578 return percentFileLocal; 579 } 580 581 @Override 582 public double getPercentFileLocalSecondaryRegions() { 583 return percentFileLocalSecondaryRegions; 584 } 585 586 @Override 587 public long getUpdatesBlockedTime() { 588 if (this.regionServer.getMemStoreFlusher() == null) { 589 return 0; 590 } 591 return this.regionServer.getMemStoreFlusher().getUpdatesBlockedMsHighWater().sum(); 592 } 593 594 @Override 595 public long getFlushedCellsCount() { 596 return flushedCellsCount; 597 } 598 599 @Override 600 public long getCompactedCellsCount() { 601 return compactedCellsCount; 602 } 603 604 @Override 605 public long getMajorCompactedCellsCount() { 606 return majorCompactedCellsCount; 607 } 608 609 @Override 610 public long getFlushedCellsSize() { 611 return flushedCellsSize; 612 } 613 614 @Override 615 public long getCompactedCellsSize() { 616 return compactedCellsSize; 617 } 618 619 @Override 620 public long getMajorCompactedCellsSize() { 621 return majorCompactedCellsSize; 622 } 623 624 @Override 625 public long getCellsCountCompactedFromMob() { 626 return cellsCountCompactedFromMob; 627 } 628 629 @Override 630 public long getCellsCountCompactedToMob() { 631 return cellsCountCompactedToMob; 632 } 633 634 @Override 635 public long getCellsSizeCompactedFromMob() { 636 return cellsSizeCompactedFromMob; 637 } 638 639 @Override 640 public long getCellsSizeCompactedToMob() { 641 return cellsSizeCompactedToMob; 642 } 643 644 @Override 645 public long getMobFlushCount() { 646 return mobFlushCount; 647 } 648 649 @Override 650 public long getMobFlushedCellsCount() { 651 return mobFlushedCellsCount; 652 } 653 654 @Override 655 public long getMobFlushedCellsSize() { 656 return mobFlushedCellsSize; 657 } 658 659 @Override 660 public long getMobScanCellsCount() { 661 return mobScanCellsCount; 662 } 663 664 @Override 665 public long getMobScanCellsSize() { 666 return mobScanCellsSize; 667 } 668 669 @Override 670 public long getMobFileCacheAccessCount() { 671 return mobFileCacheAccessCount; 672 } 673 674 @Override 675 public long getMobFileCacheMissCount() { 676 return mobFileCacheMissCount; 677 } 678 679 @Override 680 public long getMobFileCacheCount() { 681 return mobFileCacheCount; 682 } 683 684 @Override 685 public long getMobFileCacheEvictedCount() { 686 return mobFileCacheEvictedCount; 687 } 688 689 @Override 690 public double getMobFileCacheHitPercent() { 691 return mobFileCacheHitRatio * 100; 692 } 693 694 @Override 695 public int getActiveScanners() { 696 return regionServer.getRpcServices().getScannersCount(); 697 } 698 699 /** 700 * This is the runnable that will be executed on the executor every PERIOD number of seconds It 701 * will take metrics/numbers from all of the regions and use them to compute point in time 702 * metrics. 703 */ 704 public class RegionServerMetricsWrapperRunnable implements Runnable { 705 706 private long lastRan = 0; 707 private long lastStoreFileSize = 0; 708 709 @Override 710 synchronized public void run() { 711 try { 712 HDFSBlocksDistribution hdfsBlocksDistribution = new HDFSBlocksDistribution(); 713 HDFSBlocksDistribution hdfsBlocksDistributionSecondaryRegions = 714 new HDFSBlocksDistribution(); 715 716 long tempNumStores = 0, tempNumStoreFiles = 0, tempStoreFileSize = 0; 717 long tempMemstoreSize = 0, tempOnHeapMemstoreSize = 0, tempOffHeapMemstoreSize = 0; 718 long tempMaxStoreFileAge = 0, tempNumReferenceFiles = 0; 719 long avgAgeNumerator = 0, numHFiles = 0; 720 long tempMinStoreFileAge = Long.MAX_VALUE; 721 long tempFilteredReadRequestsCount = 0, tempCpRequestsCount = 0; 722 long tempCheckAndMutateChecksFailed = 0; 723 long tempCheckAndMutateChecksPassed = 0; 724 long tempStorefileIndexSize = 0; 725 long tempTotalStaticIndexSize = 0; 726 long tempTotalStaticBloomSize = 0; 727 long tempNumMutationsWithoutWAL = 0; 728 long tempDataInMemoryWithoutWAL = 0; 729 double tempPercentFileLocal = 0; 730 double tempPercentFileLocalSecondaryRegions = 0; 731 long tempFlushedCellsCount = 0; 732 long tempCompactedCellsCount = 0; 733 long tempMajorCompactedCellsCount = 0; 734 long tempFlushedCellsSize = 0; 735 long tempCompactedCellsSize = 0; 736 long tempMajorCompactedCellsSize = 0; 737 long tempCellsCountCompactedToMob = 0; 738 long tempCellsCountCompactedFromMob = 0; 739 long tempCellsSizeCompactedToMob = 0; 740 long tempCellsSizeCompactedFromMob = 0; 741 long tempMobFlushCount = 0; 742 long tempMobFlushedCellsCount = 0; 743 long tempMobFlushedCellsSize = 0; 744 long tempMobScanCellsCount = 0; 745 long tempMobScanCellsSize = 0; 746 long tempBlockedRequestsCount = 0; 747 int regionCount = 0; 748 749 long tempReadRequestsCount = 0; 750 long tempWriteRequestsCount = 0; 751 long currentReadRequestsCount = 0; 752 long currentWriteRequestsCount = 0; 753 long lastReadRequestsCount = 0; 754 long lastWriteRequestsCount = 0; 755 long readRequestsDelta = 0; 756 long writeRequestsDelta = 0; 757 long totalReadRequestsDelta = 0; 758 long totalWriteRequestsDelta = 0; 759 String encodedRegionName; 760 for (HRegion r : regionServer.getOnlineRegionsLocalContext()) { 761 encodedRegionName = r.getRegionInfo().getEncodedName(); 762 currentReadRequestsCount = r.getReadRequestsCount(); 763 currentWriteRequestsCount = r.getWriteRequestsCount(); 764 if (requestsCountCache.containsKey(encodedRegionName)) { 765 lastReadRequestsCount = requestsCountCache.get(encodedRegionName).get(0); 766 lastWriteRequestsCount = requestsCountCache.get(encodedRegionName).get(1); 767 readRequestsDelta = currentReadRequestsCount - lastReadRequestsCount; 768 writeRequestsDelta = currentWriteRequestsCount - lastWriteRequestsCount; 769 totalReadRequestsDelta += readRequestsDelta; 770 totalWriteRequestsDelta += writeRequestsDelta; 771 // Update cache for our next comparision 772 requestsCountCache.get(encodedRegionName).set(0, currentReadRequestsCount); 773 requestsCountCache.get(encodedRegionName).set(1, currentWriteRequestsCount); 774 } else { 775 // List[0] -> readRequestCount 776 // List[1] -> writeRequestCount 777 ArrayList<Long> requests = new ArrayList<Long>(2); 778 requests.add(currentReadRequestsCount); 779 requests.add(currentWriteRequestsCount); 780 requestsCountCache.put(encodedRegionName, requests); 781 totalReadRequestsDelta += currentReadRequestsCount; 782 totalWriteRequestsDelta += currentWriteRequestsCount; 783 } 784 tempReadRequestsCount += r.getReadRequestsCount(); 785 tempWriteRequestsCount += r.getWriteRequestsCount(); 786 tempNumMutationsWithoutWAL += r.getNumMutationsWithoutWAL(); 787 tempDataInMemoryWithoutWAL += r.getDataInMemoryWithoutWAL(); 788 tempCpRequestsCount += r.getCpRequestsCount(); 789 tempFilteredReadRequestsCount += r.getFilteredReadRequestsCount(); 790 tempCheckAndMutateChecksFailed += r.getCheckAndMutateChecksFailed(); 791 tempCheckAndMutateChecksPassed += r.getCheckAndMutateChecksPassed(); 792 tempBlockedRequestsCount += r.getBlockedRequestsCount(); 793 List<? extends Store> storeList = r.getStores(); 794 tempNumStores += storeList.size(); 795 for (Store store : storeList) { 796 tempNumStoreFiles += store.getStorefilesCount(); 797 tempMemstoreSize += store.getMemStoreSize().getDataSize(); 798 tempOnHeapMemstoreSize += store.getMemStoreSize().getHeapSize(); 799 tempOffHeapMemstoreSize += store.getMemStoreSize().getOffHeapSize(); 800 tempStoreFileSize += store.getStorefilesSize(); 801 802 OptionalLong storeMaxStoreFileAge = store.getMaxStoreFileAge(); 803 if ( 804 storeMaxStoreFileAge.isPresent() 805 && storeMaxStoreFileAge.getAsLong() > tempMaxStoreFileAge 806 ) { 807 tempMaxStoreFileAge = storeMaxStoreFileAge.getAsLong(); 808 } 809 810 OptionalLong storeMinStoreFileAge = store.getMinStoreFileAge(); 811 if ( 812 storeMinStoreFileAge.isPresent() 813 && storeMinStoreFileAge.getAsLong() < tempMinStoreFileAge 814 ) { 815 tempMinStoreFileAge = storeMinStoreFileAge.getAsLong(); 816 } 817 818 long storeHFiles = store.getNumHFiles(); 819 numHFiles += storeHFiles; 820 tempNumReferenceFiles += store.getNumReferenceFiles(); 821 822 OptionalDouble storeAvgStoreFileAge = store.getAvgStoreFileAge(); 823 if (storeAvgStoreFileAge.isPresent()) { 824 avgAgeNumerator = 825 (long) (avgAgeNumerator + storeAvgStoreFileAge.getAsDouble() * storeHFiles); 826 } 827 828 tempStorefileIndexSize += store.getStorefilesRootLevelIndexSize(); 829 tempTotalStaticBloomSize += store.getTotalStaticBloomSize(); 830 tempTotalStaticIndexSize += store.getTotalStaticIndexSize(); 831 tempFlushedCellsCount += store.getFlushedCellsCount(); 832 tempCompactedCellsCount += store.getCompactedCellsCount(); 833 tempMajorCompactedCellsCount += store.getMajorCompactedCellsCount(); 834 tempFlushedCellsSize += store.getFlushedCellsSize(); 835 tempCompactedCellsSize += store.getCompactedCellsSize(); 836 tempMajorCompactedCellsSize += store.getMajorCompactedCellsSize(); 837 if (store instanceof HMobStore) { 838 HMobStore mobStore = (HMobStore) store; 839 tempCellsCountCompactedToMob += mobStore.getCellsCountCompactedToMob(); 840 tempCellsCountCompactedFromMob += mobStore.getCellsCountCompactedFromMob(); 841 tempCellsSizeCompactedToMob += mobStore.getCellsSizeCompactedToMob(); 842 tempCellsSizeCompactedFromMob += mobStore.getCellsSizeCompactedFromMob(); 843 tempMobFlushCount += mobStore.getMobFlushCount(); 844 tempMobFlushedCellsCount += mobStore.getMobFlushedCellsCount(); 845 tempMobFlushedCellsSize += mobStore.getMobFlushedCellsSize(); 846 tempMobScanCellsCount += mobStore.getMobScanCellsCount(); 847 tempMobScanCellsSize += mobStore.getMobScanCellsSize(); 848 } 849 } 850 851 HDFSBlocksDistribution distro = r.getHDFSBlocksDistribution(); 852 hdfsBlocksDistribution.add(distro); 853 if (r.getRegionInfo().getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) { 854 hdfsBlocksDistributionSecondaryRegions.add(distro); 855 } 856 regionCount++; 857 } 858 859 float localityIndex = 860 hdfsBlocksDistribution.getBlockLocalityIndex(regionServer.getServerName().getHostname()); 861 tempPercentFileLocal = Double.isNaN(tempBlockedRequestsCount) ? 0 : (localityIndex * 100); 862 863 float localityIndexSecondaryRegions = hdfsBlocksDistributionSecondaryRegions 864 .getBlockLocalityIndex(regionServer.getServerName().getHostname()); 865 tempPercentFileLocalSecondaryRegions = 866 Double.isNaN(localityIndexSecondaryRegions) ? 0 : (localityIndexSecondaryRegions * 100); 867 868 // Compute the number of requests per second 869 long currentTime = EnvironmentEdgeManager.currentTime(); 870 871 // assume that it took PERIOD seconds to start the executor. 872 // this is a guess but it's a pretty good one. 873 if (lastRan == 0) { 874 lastRan = currentTime - period; 875 } 876 // If we've time traveled keep the last requests per second. 877 if ((currentTime - lastRan) > 0) { 878 requestsPerSecond = 879 (totalReadRequestsDelta + totalWriteRequestsDelta) / ((currentTime - lastRan) / 1000.0); 880 881 double readRequestsRatePerMilliSecond = (double) totalReadRequestsDelta / period; 882 double writeRequestsRatePerMilliSecond = (double) totalWriteRequestsDelta / period; 883 884 readRequestsRatePerSecond = readRequestsRatePerMilliSecond * 1000.0; 885 writeRequestsRatePerSecond = writeRequestsRatePerMilliSecond * 1000.0; 886 887 long intervalStoreFileSize = tempStoreFileSize - lastStoreFileSize; 888 storeFileSizeGrowthRate = (double) intervalStoreFileSize * 1000.0 / period; 889 890 lastStoreFileSize = tempStoreFileSize; 891 } 892 893 lastRan = currentTime; 894 895 final WALProvider provider = regionServer.getWalFactory().getWALProvider(); 896 final WALProvider metaProvider = regionServer.getWalFactory().getMetaWALProvider(); 897 numWALFiles = (provider == null ? 0 : provider.getNumLogFiles()) 898 + (metaProvider == null ? 0 : metaProvider.getNumLogFiles()); 899 walFileSize = (provider == null ? 0 : provider.getLogFileSize()) 900 + (metaProvider == null ? 0 : metaProvider.getLogFileSize()); 901 // Copy over computed values so that no thread sees half computed values. 902 numStores = tempNumStores; 903 numStoreFiles = tempNumStoreFiles; 904 memstoreSize = tempMemstoreSize; 905 onHeapMemstoreSize = tempOnHeapMemstoreSize; 906 offHeapMemstoreSize = tempOffHeapMemstoreSize; 907 storeFileSize = tempStoreFileSize; 908 maxStoreFileAge = tempMaxStoreFileAge; 909 if (regionCount > 0) { 910 averageRegionSize = (memstoreSize + storeFileSize) / regionCount; 911 } 912 if (tempMinStoreFileAge != Long.MAX_VALUE) { 913 minStoreFileAge = tempMinStoreFileAge; 914 } 915 916 if (numHFiles != 0) { 917 avgStoreFileAge = avgAgeNumerator / numHFiles; 918 } 919 920 numReferenceFiles = tempNumReferenceFiles; 921 readRequestsCount = tempReadRequestsCount; 922 cpRequestsCount = tempCpRequestsCount; 923 filteredReadRequestsCount = tempFilteredReadRequestsCount; 924 writeRequestsCount = tempWriteRequestsCount; 925 checkAndMutateChecksFailed = tempCheckAndMutateChecksFailed; 926 checkAndMutateChecksPassed = tempCheckAndMutateChecksPassed; 927 storefileIndexSize = tempStorefileIndexSize; 928 totalStaticIndexSize = tempTotalStaticIndexSize; 929 totalStaticBloomSize = tempTotalStaticBloomSize; 930 numMutationsWithoutWAL = tempNumMutationsWithoutWAL; 931 dataInMemoryWithoutWAL = tempDataInMemoryWithoutWAL; 932 percentFileLocal = tempPercentFileLocal; 933 percentFileLocalSecondaryRegions = tempPercentFileLocalSecondaryRegions; 934 flushedCellsCount = tempFlushedCellsCount; 935 compactedCellsCount = tempCompactedCellsCount; 936 majorCompactedCellsCount = tempMajorCompactedCellsCount; 937 flushedCellsSize = tempFlushedCellsSize; 938 compactedCellsSize = tempCompactedCellsSize; 939 majorCompactedCellsSize = tempMajorCompactedCellsSize; 940 cellsCountCompactedToMob = tempCellsCountCompactedToMob; 941 cellsCountCompactedFromMob = tempCellsCountCompactedFromMob; 942 cellsSizeCompactedToMob = tempCellsSizeCompactedToMob; 943 cellsSizeCompactedFromMob = tempCellsSizeCompactedFromMob; 944 mobFlushCount = tempMobFlushCount; 945 mobFlushedCellsCount = tempMobFlushedCellsCount; 946 mobFlushedCellsSize = tempMobFlushedCellsSize; 947 mobScanCellsCount = tempMobScanCellsCount; 948 mobScanCellsSize = tempMobScanCellsSize; 949 mobFileCacheAccessCount = mobFileCache != null ? mobFileCache.getAccessCount() : 0L; 950 mobFileCacheMissCount = mobFileCache != null ? mobFileCache.getMissCount() : 0L; 951 mobFileCacheHitRatio = mobFileCache != null ? mobFileCache.getHitRatio() : 0.0; 952 if (Double.isNaN(mobFileCacheHitRatio)) { 953 mobFileCacheHitRatio = 0.0; 954 } 955 mobFileCacheEvictedCount = mobFileCache != null ? mobFileCache.getEvictedFileCount() : 0L; 956 mobFileCacheCount = mobFileCache != null ? mobFileCache.getCacheSize() : 0; 957 blockedRequestsCount = tempBlockedRequestsCount; 958 } catch (Throwable e) { 959 LOG.warn("Caught exception! Will suppress and retry.", e); 960 } 961 } 962 } 963 964 @Override 965 public long getHedgedReadOps() { 966 return this.dfsHedgedReadMetrics == null ? 0 : this.dfsHedgedReadMetrics.getHedgedReadOps(); 967 } 968 969 @Override 970 public long getHedgedReadWins() { 971 return this.dfsHedgedReadMetrics == null ? 0 : this.dfsHedgedReadMetrics.getHedgedReadWins(); 972 } 973 974 @Override 975 public long getHedgedReadOpsInCurThread() { 976 return this.dfsHedgedReadMetrics == null 977 ? 0 978 : this.dfsHedgedReadMetrics.getHedgedReadOpsInCurThread(); 979 } 980 981 @Override 982 public long getTotalBytesRead() { 983 return FSDataInputStreamWrapper.getTotalBytesRead(); 984 } 985 986 @Override 987 public long getLocalBytesRead() { 988 return FSDataInputStreamWrapper.getLocalBytesRead(); 989 } 990 991 @Override 992 public long getShortCircuitBytesRead() { 993 return FSDataInputStreamWrapper.getShortCircuitBytesRead(); 994 } 995 996 @Override 997 public long getZeroCopyBytesRead() { 998 return FSDataInputStreamWrapper.getZeroCopyBytesRead(); 999 } 1000 1001 @Override 1002 public long getBlockedRequestsCount() { 1003 return blockedRequestsCount; 1004 } 1005 1006 @Override 1007 public long getAverageRegionSize() { 1008 return averageRegionSize; 1009 } 1010 1011 @Override 1012 public long getDataMissCount() { 1013 return this.cacheStats != null ? this.cacheStats.getDataMissCount() : 0L; 1014 } 1015 1016 @Override 1017 public long getLeafIndexMissCount() { 1018 return this.cacheStats != null ? this.cacheStats.getLeafIndexMissCount() : 0L; 1019 } 1020 1021 @Override 1022 public long getBloomChunkMissCount() { 1023 return this.cacheStats != null ? this.cacheStats.getBloomChunkMissCount() : 0L; 1024 } 1025 1026 @Override 1027 public long getMetaMissCount() { 1028 return this.cacheStats != null ? this.cacheStats.getMetaMissCount() : 0L; 1029 } 1030 1031 @Override 1032 public long getRootIndexMissCount() { 1033 return this.cacheStats != null ? this.cacheStats.getRootIndexMissCount() : 0L; 1034 } 1035 1036 @Override 1037 public long getIntermediateIndexMissCount() { 1038 return this.cacheStats != null ? this.cacheStats.getIntermediateIndexMissCount() : 0L; 1039 } 1040 1041 @Override 1042 public long getFileInfoMissCount() { 1043 return this.cacheStats != null ? this.cacheStats.getFileInfoMissCount() : 0L; 1044 } 1045 1046 @Override 1047 public long getGeneralBloomMetaMissCount() { 1048 return this.cacheStats != null ? this.cacheStats.getGeneralBloomMetaMissCount() : 0L; 1049 } 1050 1051 @Override 1052 public long getDeleteFamilyBloomMissCount() { 1053 return this.cacheStats != null ? this.cacheStats.getDeleteFamilyBloomMissCount() : 0L; 1054 } 1055 1056 @Override 1057 public long getTrailerMissCount() { 1058 return this.cacheStats != null ? this.cacheStats.getTrailerMissCount() : 0L; 1059 } 1060 1061 @Override 1062 public long getDataHitCount() { 1063 return this.cacheStats != null ? this.cacheStats.getDataHitCount() : 0L; 1064 } 1065 1066 @Override 1067 public long getLeafIndexHitCount() { 1068 return this.cacheStats != null ? this.cacheStats.getLeafIndexHitCount() : 0L; 1069 } 1070 1071 @Override 1072 public long getBloomChunkHitCount() { 1073 return this.cacheStats != null ? this.cacheStats.getBloomChunkHitCount() : 0L; 1074 } 1075 1076 @Override 1077 public long getMetaHitCount() { 1078 return this.cacheStats != null ? this.cacheStats.getMetaHitCount() : 0L; 1079 } 1080 1081 @Override 1082 public long getRootIndexHitCount() { 1083 return this.cacheStats != null ? this.cacheStats.getRootIndexHitCount() : 0L; 1084 } 1085 1086 @Override 1087 public long getIntermediateIndexHitCount() { 1088 return this.cacheStats != null ? this.cacheStats.getIntermediateIndexHitCount() : 0L; 1089 } 1090 1091 @Override 1092 public long getFileInfoHitCount() { 1093 return this.cacheStats != null ? this.cacheStats.getFileInfoHitCount() : 0L; 1094 } 1095 1096 @Override 1097 public long getGeneralBloomMetaHitCount() { 1098 return this.cacheStats != null ? this.cacheStats.getGeneralBloomMetaHitCount() : 0L; 1099 } 1100 1101 @Override 1102 public long getDeleteFamilyBloomHitCount() { 1103 return this.cacheStats != null ? this.cacheStats.getDeleteFamilyBloomHitCount() : 0L; 1104 } 1105 1106 @Override 1107 public long getTrailerHitCount() { 1108 return this.cacheStats != null ? this.cacheStats.getTrailerHitCount() : 0L; 1109 } 1110 1111 @Override 1112 public long getByteBuffAllocatorHeapAllocationBytes() { 1113 return ByteBuffAllocator.getHeapAllocationBytes(allocator, ByteBuffAllocator.HEAP); 1114 } 1115 1116 @Override 1117 public long getByteBuffAllocatorPoolAllocationBytes() { 1118 return this.allocator.getPoolAllocationBytes(); 1119 } 1120 1121 @Override 1122 public double getByteBuffAllocatorHeapAllocRatio() { 1123 return ByteBuffAllocator.getHeapAllocationRatio(allocator, ByteBuffAllocator.HEAP); 1124 } 1125 1126 @Override 1127 public long getByteBuffAllocatorTotalBufferCount() { 1128 return this.allocator.getTotalBufferCount(); 1129 } 1130 1131 @Override 1132 public long getByteBuffAllocatorUsedBufferCount() { 1133 return this.allocator.getUsedBufferCount(); 1134 } 1135}