001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import java.io.IOException;
021import java.util.ArrayList;
022import java.util.Collection;
023import java.util.Collections;
024import java.util.List;
025import java.util.Map;
026import java.util.OptionalDouble;
027import java.util.OptionalLong;
028import java.util.concurrent.ConcurrentHashMap;
029import java.util.concurrent.ScheduledExecutorService;
030import java.util.concurrent.TimeUnit;
031import java.util.stream.Collectors;
032import org.apache.commons.lang3.StringUtils;
033import org.apache.hadoop.hbase.CompatibilitySingletonFactory;
034import org.apache.hadoop.hbase.HConstants;
035import org.apache.hadoop.hbase.HDFSBlocksDistribution;
036import org.apache.hadoop.hbase.ServerName;
037import org.apache.hadoop.hbase.client.RegionInfo;
038import org.apache.hadoop.hbase.io.ByteBuffAllocator;
039import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
040import org.apache.hadoop.hbase.io.asyncfs.monitor.ExcludeDatanodeManager;
041import org.apache.hadoop.hbase.io.hfile.BlockCache;
042import org.apache.hadoop.hbase.io.hfile.CacheStats;
043import org.apache.hadoop.hbase.io.hfile.CombinedBlockCache;
044import org.apache.hadoop.hbase.mob.MobFileCache;
045import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource;
046import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
047import org.apache.hadoop.hbase.util.FSUtils;
048import org.apache.hadoop.hbase.wal.WALProvider;
049import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
050import org.apache.hadoop.hdfs.DFSHedgedReadMetrics;
051import org.apache.hadoop.metrics2.MetricsExecutor;
052import org.apache.yetus.audience.InterfaceAudience;
053import org.slf4j.Logger;
054import org.slf4j.LoggerFactory;
055
056/**
057 * Impl for exposing HRegionServer Information through Hadoop's metrics 2 system.
058 */
059@InterfaceAudience.Private
060class MetricsRegionServerWrapperImpl implements MetricsRegionServerWrapper {
061
062  private static final Logger LOG = LoggerFactory.getLogger(MetricsRegionServerWrapperImpl.class);
063
064  private final HRegionServer regionServer;
065  private final MetricsWALSource metricsWALSource;
066  private final ByteBuffAllocator allocator;
067
068  private BlockCache blockCache;
069  private MobFileCache mobFileCache;
070  private CacheStats cacheStats;
071  private CacheStats l1Stats = null;
072  private CacheStats l2Stats = null;
073
074  private volatile long numStores = 0;
075  private volatile long numWALFiles = 0;
076  private volatile long walFileSize = 0;
077  private volatile long numStoreFiles = 0;
078  private volatile long memstoreSize = 0;
079  private volatile long onHeapMemstoreSize = 0;
080  private volatile long offHeapMemstoreSize = 0;
081  private volatile long storeFileSize = 0;
082  private volatile double storeFileSizeGrowthRate = 0;
083  private volatile long maxStoreFileAge = 0;
084  private volatile long minStoreFileAge = 0;
085  private volatile long avgStoreFileAge = 0;
086  private volatile long numReferenceFiles = 0;
087  private volatile double requestsPerSecond = 0.0;
088  private volatile long readRequestsCount = 0;
089  private volatile double readRequestsRatePerSecond = 0;
090  private volatile long cpRequestsCount = 0;
091  private volatile long filteredReadRequestsCount = 0;
092  private volatile long writeRequestsCount = 0;
093  private volatile double writeRequestsRatePerSecond = 0;
094  private volatile long checkAndMutateChecksFailed = 0;
095  private volatile long checkAndMutateChecksPassed = 0;
096  private volatile long storefileIndexSize = 0;
097  private volatile long totalStaticIndexSize = 0;
098  private volatile long totalStaticBloomSize = 0;
099  private volatile long numMutationsWithoutWAL = 0;
100  private volatile long dataInMemoryWithoutWAL = 0;
101  private volatile double percentFileLocal = 0;
102  private volatile double percentFileLocalSecondaryRegions = 0;
103  private volatile long flushedCellsCount = 0;
104  private volatile long compactedCellsCount = 0;
105  private volatile long majorCompactedCellsCount = 0;
106  private volatile long flushedCellsSize = 0;
107  private volatile long compactedCellsSize = 0;
108  private volatile long majorCompactedCellsSize = 0;
109  private volatile long cellsCountCompactedToMob = 0;
110  private volatile long cellsCountCompactedFromMob = 0;
111  private volatile long cellsSizeCompactedToMob = 0;
112  private volatile long cellsSizeCompactedFromMob = 0;
113  private volatile long mobFlushCount = 0;
114  private volatile long mobFlushedCellsCount = 0;
115  private volatile long mobFlushedCellsSize = 0;
116  private volatile long mobScanCellsCount = 0;
117  private volatile long mobScanCellsSize = 0;
118  private volatile long mobFileCacheAccessCount = 0;
119  private volatile long mobFileCacheMissCount = 0;
120  private volatile double mobFileCacheHitRatio = 0;
121  private volatile long mobFileCacheEvictedCount = 0;
122  private volatile long mobFileCacheCount = 0;
123  private volatile long blockedRequestsCount = 0L;
124  private volatile long averageRegionSize = 0L;
125  protected final Map<String, ArrayList<Long>> requestsCountCache =
126    new ConcurrentHashMap<String, ArrayList<Long>>();
127
128  private ScheduledExecutorService executor;
129  private Runnable runnable;
130  private long period;
131
132  /**
133   * Can be null if not on hdfs.
134   */
135  private DFSHedgedReadMetrics dfsHedgedReadMetrics;
136
137  private final ExcludeDatanodeManager excludeDatanodeManager;
138
139  public MetricsRegionServerWrapperImpl(final HRegionServer regionServer) {
140    this.regionServer = regionServer;
141    initBlockCache();
142    initMobFileCache();
143    this.excludeDatanodeManager = this.regionServer.getWalFactory().getExcludeDatanodeManager();
144
145    this.period = regionServer.getConfiguration().getLong(HConstants.REGIONSERVER_METRICS_PERIOD,
146      HConstants.DEFAULT_REGIONSERVER_METRICS_PERIOD);
147
148    this.executor = CompatibilitySingletonFactory.getInstance(MetricsExecutor.class).getExecutor();
149    this.runnable = new RegionServerMetricsWrapperRunnable();
150    this.executor.scheduleWithFixedDelay(this.runnable, this.period, this.period,
151      TimeUnit.MILLISECONDS);
152    this.metricsWALSource = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
153    this.allocator = regionServer.getRpcServer().getByteBuffAllocator();
154
155    try {
156      this.dfsHedgedReadMetrics = FSUtils.getDFSHedgedReadMetrics(regionServer.getConfiguration());
157    } catch (IOException e) {
158      LOG.warn("Failed to get hedged metrics", e);
159    }
160    if (LOG.isInfoEnabled()) {
161      LOG.info("Computing regionserver metrics every " + this.period + " milliseconds");
162    }
163  }
164
165  private void initBlockCache() {
166    this.blockCache = this.regionServer.getBlockCache().orElse(null);
167    this.cacheStats = this.blockCache != null ? this.blockCache.getStats() : null;
168    if (this.cacheStats != null) {
169      if (this.cacheStats instanceof CombinedBlockCache.CombinedCacheStats) {
170        l1Stats = ((CombinedBlockCache.CombinedCacheStats) this.cacheStats).getLruCacheStats();
171        l2Stats = ((CombinedBlockCache.CombinedCacheStats) this.cacheStats).getBucketCacheStats();
172      } else {
173        l1Stats = this.cacheStats;
174      }
175    }
176  }
177
178  /**
179   * Initializes the mob file cache.
180   */
181  private void initMobFileCache() {
182    this.mobFileCache = this.regionServer.getMobFileCache().orElse(null);
183  }
184
185  @Override
186  public String getClusterId() {
187    return regionServer.getClusterId();
188  }
189
190  @Override
191  public long getStartCode() {
192    return regionServer.getStartcode();
193  }
194
195  @Override
196  public String getZookeeperQuorum() {
197    ZKWatcher zk = regionServer.getZooKeeper();
198    if (zk == null) {
199      return "";
200    }
201    return zk.getQuorum();
202  }
203
204  @Override
205  public String getCoprocessors() {
206    String[] coprocessors = regionServer.getRegionServerCoprocessors();
207    if (coprocessors == null || coprocessors.length == 0) {
208      return "";
209    }
210    return StringUtils.join(coprocessors, ", ");
211  }
212
213  @Override
214  public String getServerName() {
215    ServerName serverName = regionServer.getServerName();
216    if (serverName == null) {
217      return "";
218    }
219    return serverName.getServerName();
220  }
221
222  @Override
223  public long getNumOnlineRegions() {
224    Collection<HRegion> onlineRegionsLocalContext = regionServer.getOnlineRegionsLocalContext();
225    if (onlineRegionsLocalContext == null) {
226      return 0;
227    }
228    return onlineRegionsLocalContext.size();
229  }
230
231  @Override
232  public long getTotalRequestCount() {
233    return regionServer.getRpcServices().requestCount.sum();
234  }
235
236  @Override
237  public long getTotalRowActionRequestCount() {
238    return readRequestsCount + writeRequestsCount;
239  }
240
241  @Override
242  public int getSplitQueueSize() {
243    final CompactSplit compactSplit = regionServer.getCompactSplitThread();
244    return compactSplit == null ? 0 : compactSplit.getSplitQueueSize();
245  }
246
247  @Override
248  public int getCompactionQueueSize() {
249    final CompactSplit compactSplit = regionServer.getCompactSplitThread();
250    return compactSplit == null ? 0 : compactSplit.getCompactionQueueSize();
251  }
252
253  @Override
254  public int getSmallCompactionQueueSize() {
255    final CompactSplit compactSplit = regionServer.getCompactSplitThread();
256    return compactSplit == null ? 0 : compactSplit.getSmallCompactionQueueSize();
257  }
258
259  @Override
260  public int getLargeCompactionQueueSize() {
261    final CompactSplit compactSplit = regionServer.getCompactSplitThread();
262    return compactSplit == null ? 0 : compactSplit.getLargeCompactionQueueSize();
263  }
264
265  @Override
266  public int getFlushQueueSize() {
267    // If there is no flusher there should be no queue.
268    if (this.regionServer.getMemStoreFlusher() == null) {
269      return 0;
270    }
271    return this.regionServer.getMemStoreFlusher().getFlushQueueSize();
272  }
273
274  @Override
275  public long getBlockCacheCount() {
276    return this.blockCache != null ? this.blockCache.getBlockCount() : 0L;
277  }
278
279  @Override
280  public long getMemStoreLimit() {
281    return this.regionServer.getRegionServerAccounting().getGlobalMemStoreLimit();
282  }
283
284  @Override
285  public long getOnHeapMemStoreLimit() {
286    return this.regionServer.getRegionServerAccounting().getGlobalOnHeapMemStoreLimit();
287  }
288
289  @Override
290  public long getOffHeapMemStoreLimit() {
291    return this.regionServer.getRegionServerAccounting().getGlobalOffHeapMemStoreLimit();
292  }
293
294  @Override
295  public long getBlockCacheSize() {
296    return this.blockCache != null ? this.blockCache.getCurrentSize() : 0L;
297  }
298
299  @Override
300  public long getBlockCacheFreeSize() {
301    return this.blockCache != null ? this.blockCache.getFreeSize() : 0L;
302  }
303
304  @Override
305  public long getBlockCacheHitCount() {
306    return this.cacheStats != null ? this.cacheStats.getHitCount() : 0L;
307  }
308
309  @Override
310  public long getBlockCachePrimaryHitCount() {
311    return this.cacheStats != null ? this.cacheStats.getPrimaryHitCount() : 0L;
312  }
313
314  @Override
315  public long getBlockCacheMissCount() {
316    return this.cacheStats != null ? this.cacheStats.getMissCount() : 0L;
317  }
318
319  @Override
320  public long getBlockCachePrimaryMissCount() {
321    return this.cacheStats != null ? this.cacheStats.getPrimaryMissCount() : 0L;
322  }
323
324  @Override
325  public long getBlockCacheEvictedCount() {
326    return this.cacheStats != null ? this.cacheStats.getEvictedCount() : 0L;
327  }
328
329  @Override
330  public long getBlockCachePrimaryEvictedCount() {
331    return this.cacheStats != null ? this.cacheStats.getPrimaryEvictedCount() : 0L;
332  }
333
334  @Override
335  public double getBlockCacheHitPercent() {
336    double ratio = this.cacheStats != null ? this.cacheStats.getHitRatio() : 0.0;
337    if (Double.isNaN(ratio)) {
338      ratio = 0;
339    }
340    return (ratio * 100);
341  }
342
343  @Override
344  public double getBlockCacheHitCachingPercent() {
345    double ratio = this.cacheStats != null ? this.cacheStats.getHitCachingRatio() : 0.0;
346    if (Double.isNaN(ratio)) {
347      ratio = 0;
348    }
349    return (ratio * 100);
350  }
351
352  @Override
353  public long getBlockCacheFailedInsertions() {
354    return this.cacheStats != null ? this.cacheStats.getFailedInserts() : 0L;
355  }
356
357  @Override
358  public long getL1CacheHitCount() {
359    return this.l1Stats != null ? this.l1Stats.getHitCount() : 0L;
360  }
361
362  @Override
363  public long getL1CacheMissCount() {
364    return this.l1Stats != null ? this.l1Stats.getMissCount() : 0L;
365  }
366
367  @Override
368  public double getL1CacheHitRatio() {
369    return this.l1Stats != null ? this.l1Stats.getHitRatio() : 0.0;
370  }
371
372  @Override
373  public double getL1CacheMissRatio() {
374    return this.l1Stats != null ? this.l1Stats.getMissRatio() : 0.0;
375  }
376
377  @Override
378  public long getL2CacheHitCount() {
379    return this.l2Stats != null ? this.l2Stats.getHitCount() : 0L;
380  }
381
382  @Override
383  public long getL2CacheMissCount() {
384    return this.l2Stats != null ? this.l2Stats.getMissCount() : 0L;
385  }
386
387  @Override
388  public double getL2CacheHitRatio() {
389    return this.l2Stats != null ? this.l2Stats.getHitRatio() : 0.0;
390  }
391
392  @Override
393  public double getL2CacheMissRatio() {
394    return this.l2Stats != null ? this.l2Stats.getMissRatio() : 0.0;
395  }
396
397  @Override
398  public void forceRecompute() {
399    this.runnable.run();
400  }
401
402  @Override
403  public long getNumStores() {
404    return numStores;
405  }
406
407  @Override
408  public long getNumWALFiles() {
409    return numWALFiles;
410  }
411
412  @Override
413  public long getWALFileSize() {
414    return walFileSize;
415  }
416
417  @Override
418  public List<String> getWALExcludeDNs() {
419    if (excludeDatanodeManager == null) {
420      return Collections.emptyList();
421    }
422    return excludeDatanodeManager.getExcludeDNs().entrySet().stream()
423      .map(e -> e.getKey().toString() + ", " + e.getValue()).collect(Collectors.toList());
424  }
425
426  @Override
427  public long getNumWALSlowAppend() {
428    return metricsWALSource.getSlowAppendCount();
429  }
430
431  @Override
432  public long getNumStoreFiles() {
433    return numStoreFiles;
434  }
435
436  @Override
437  public long getMaxStoreFileAge() {
438    return maxStoreFileAge;
439  }
440
441  @Override
442  public long getMinStoreFileAge() {
443    return minStoreFileAge;
444  }
445
446  @Override
447  public long getAvgStoreFileAge() {
448    return avgStoreFileAge;
449  }
450
451  @Override
452  public long getNumReferenceFiles() {
453    return numReferenceFiles;
454  }
455
456  @Override
457  public long getMemStoreSize() {
458    return memstoreSize;
459  }
460
461  @Override
462  public long getOnHeapMemStoreSize() {
463    return onHeapMemstoreSize;
464  }
465
466  @Override
467  public long getOffHeapMemStoreSize() {
468    return offHeapMemstoreSize;
469  }
470
471  @Override
472  public long getStoreFileSize() {
473    return storeFileSize;
474  }
475
476  @Override
477  public double getStoreFileSizeGrowthRate() {
478    return storeFileSizeGrowthRate;
479  }
480
481  @Override
482  public double getRequestsPerSecond() {
483    return requestsPerSecond;
484  }
485
486  @Override
487  public long getReadRequestsCount() {
488    return readRequestsCount;
489  }
490
491  @Override
492  public long getCpRequestsCount() {
493    return cpRequestsCount;
494  }
495
496  @Override
497  public double getReadRequestsRatePerSecond() {
498    return readRequestsRatePerSecond;
499  }
500
501  @Override
502  public long getFilteredReadRequestsCount() {
503    return filteredReadRequestsCount;
504  }
505
506  @Override
507  public long getWriteRequestsCount() {
508    return writeRequestsCount;
509  }
510
511  @Override
512  public double getWriteRequestsRatePerSecond() {
513    return writeRequestsRatePerSecond;
514  }
515
516  @Override
517  public long getRpcGetRequestsCount() {
518    return regionServer.getRpcServices().rpcGetRequestCount.sum();
519  }
520
521  @Override
522  public long getRpcScanRequestsCount() {
523    return regionServer.getRpcServices().rpcScanRequestCount.sum();
524  }
525
526  @Override
527  public long getRpcFullScanRequestsCount() {
528    return regionServer.getRpcServices().rpcFullScanRequestCount.sum();
529  }
530
531  @Override
532  public long getRpcMultiRequestsCount() {
533    return regionServer.getRpcServices().rpcMultiRequestCount.sum();
534  }
535
536  @Override
537  public long getRpcMutateRequestsCount() {
538    return regionServer.getRpcServices().rpcMutateRequestCount.sum();
539  }
540
541  @Override
542  public long getCheckAndMutateChecksFailed() {
543    return checkAndMutateChecksFailed;
544  }
545
546  @Override
547  public long getCheckAndMutateChecksPassed() {
548    return checkAndMutateChecksPassed;
549  }
550
551  @Override
552  public long getStoreFileIndexSize() {
553    return storefileIndexSize;
554  }
555
556  @Override
557  public long getTotalStaticIndexSize() {
558    return totalStaticIndexSize;
559  }
560
561  @Override
562  public long getTotalStaticBloomSize() {
563    return totalStaticBloomSize;
564  }
565
566  @Override
567  public long getNumMutationsWithoutWAL() {
568    return numMutationsWithoutWAL;
569  }
570
571  @Override
572  public long getDataInMemoryWithoutWAL() {
573    return dataInMemoryWithoutWAL;
574  }
575
576  @Override
577  public double getPercentFileLocal() {
578    return percentFileLocal;
579  }
580
581  @Override
582  public double getPercentFileLocalSecondaryRegions() {
583    return percentFileLocalSecondaryRegions;
584  }
585
586  @Override
587  public long getUpdatesBlockedTime() {
588    if (this.regionServer.getMemStoreFlusher() == null) {
589      return 0;
590    }
591    return this.regionServer.getMemStoreFlusher().getUpdatesBlockedMsHighWater().sum();
592  }
593
594  @Override
595  public long getFlushedCellsCount() {
596    return flushedCellsCount;
597  }
598
599  @Override
600  public long getCompactedCellsCount() {
601    return compactedCellsCount;
602  }
603
604  @Override
605  public long getMajorCompactedCellsCount() {
606    return majorCompactedCellsCount;
607  }
608
609  @Override
610  public long getFlushedCellsSize() {
611    return flushedCellsSize;
612  }
613
614  @Override
615  public long getCompactedCellsSize() {
616    return compactedCellsSize;
617  }
618
619  @Override
620  public long getMajorCompactedCellsSize() {
621    return majorCompactedCellsSize;
622  }
623
624  @Override
625  public long getCellsCountCompactedFromMob() {
626    return cellsCountCompactedFromMob;
627  }
628
629  @Override
630  public long getCellsCountCompactedToMob() {
631    return cellsCountCompactedToMob;
632  }
633
634  @Override
635  public long getCellsSizeCompactedFromMob() {
636    return cellsSizeCompactedFromMob;
637  }
638
639  @Override
640  public long getCellsSizeCompactedToMob() {
641    return cellsSizeCompactedToMob;
642  }
643
644  @Override
645  public long getMobFlushCount() {
646    return mobFlushCount;
647  }
648
649  @Override
650  public long getMobFlushedCellsCount() {
651    return mobFlushedCellsCount;
652  }
653
654  @Override
655  public long getMobFlushedCellsSize() {
656    return mobFlushedCellsSize;
657  }
658
659  @Override
660  public long getMobScanCellsCount() {
661    return mobScanCellsCount;
662  }
663
664  @Override
665  public long getMobScanCellsSize() {
666    return mobScanCellsSize;
667  }
668
669  @Override
670  public long getMobFileCacheAccessCount() {
671    return mobFileCacheAccessCount;
672  }
673
674  @Override
675  public long getMobFileCacheMissCount() {
676    return mobFileCacheMissCount;
677  }
678
679  @Override
680  public long getMobFileCacheCount() {
681    return mobFileCacheCount;
682  }
683
684  @Override
685  public long getMobFileCacheEvictedCount() {
686    return mobFileCacheEvictedCount;
687  }
688
689  @Override
690  public double getMobFileCacheHitPercent() {
691    return mobFileCacheHitRatio * 100;
692  }
693
694  @Override
695  public int getActiveScanners() {
696    return regionServer.getRpcServices().getScannersCount();
697  }
698
699  /**
700   * This is the runnable that will be executed on the executor every PERIOD number of seconds It
701   * will take metrics/numbers from all of the regions and use them to compute point in time
702   * metrics.
703   */
704  public class RegionServerMetricsWrapperRunnable implements Runnable {
705
706    private long lastRan = 0;
707    private long lastStoreFileSize = 0;
708
709    @Override
710    synchronized public void run() {
711      try {
712        HDFSBlocksDistribution hdfsBlocksDistribution = new HDFSBlocksDistribution();
713        HDFSBlocksDistribution hdfsBlocksDistributionSecondaryRegions =
714          new HDFSBlocksDistribution();
715
716        long tempNumStores = 0, tempNumStoreFiles = 0, tempStoreFileSize = 0;
717        long tempMemstoreSize = 0, tempOnHeapMemstoreSize = 0, tempOffHeapMemstoreSize = 0;
718        long tempMaxStoreFileAge = 0, tempNumReferenceFiles = 0;
719        long avgAgeNumerator = 0, numHFiles = 0;
720        long tempMinStoreFileAge = Long.MAX_VALUE;
721        long tempFilteredReadRequestsCount = 0, tempCpRequestsCount = 0;
722        long tempCheckAndMutateChecksFailed = 0;
723        long tempCheckAndMutateChecksPassed = 0;
724        long tempStorefileIndexSize = 0;
725        long tempTotalStaticIndexSize = 0;
726        long tempTotalStaticBloomSize = 0;
727        long tempNumMutationsWithoutWAL = 0;
728        long tempDataInMemoryWithoutWAL = 0;
729        double tempPercentFileLocal = 0;
730        double tempPercentFileLocalSecondaryRegions = 0;
731        long tempFlushedCellsCount = 0;
732        long tempCompactedCellsCount = 0;
733        long tempMajorCompactedCellsCount = 0;
734        long tempFlushedCellsSize = 0;
735        long tempCompactedCellsSize = 0;
736        long tempMajorCompactedCellsSize = 0;
737        long tempCellsCountCompactedToMob = 0;
738        long tempCellsCountCompactedFromMob = 0;
739        long tempCellsSizeCompactedToMob = 0;
740        long tempCellsSizeCompactedFromMob = 0;
741        long tempMobFlushCount = 0;
742        long tempMobFlushedCellsCount = 0;
743        long tempMobFlushedCellsSize = 0;
744        long tempMobScanCellsCount = 0;
745        long tempMobScanCellsSize = 0;
746        long tempBlockedRequestsCount = 0;
747        int regionCount = 0;
748
749        long tempReadRequestsCount = 0;
750        long tempWriteRequestsCount = 0;
751        long currentReadRequestsCount = 0;
752        long currentWriteRequestsCount = 0;
753        long lastReadRequestsCount = 0;
754        long lastWriteRequestsCount = 0;
755        long readRequestsDelta = 0;
756        long writeRequestsDelta = 0;
757        long totalReadRequestsDelta = 0;
758        long totalWriteRequestsDelta = 0;
759        String encodedRegionName;
760        for (HRegion r : regionServer.getOnlineRegionsLocalContext()) {
761          encodedRegionName = r.getRegionInfo().getEncodedName();
762          currentReadRequestsCount = r.getReadRequestsCount();
763          currentWriteRequestsCount = r.getWriteRequestsCount();
764          if (requestsCountCache.containsKey(encodedRegionName)) {
765            lastReadRequestsCount = requestsCountCache.get(encodedRegionName).get(0);
766            lastWriteRequestsCount = requestsCountCache.get(encodedRegionName).get(1);
767            readRequestsDelta = currentReadRequestsCount - lastReadRequestsCount;
768            writeRequestsDelta = currentWriteRequestsCount - lastWriteRequestsCount;
769            totalReadRequestsDelta += readRequestsDelta;
770            totalWriteRequestsDelta += writeRequestsDelta;
771            // Update cache for our next comparision
772            requestsCountCache.get(encodedRegionName).set(0, currentReadRequestsCount);
773            requestsCountCache.get(encodedRegionName).set(1, currentWriteRequestsCount);
774          } else {
775            // List[0] -> readRequestCount
776            // List[1] -> writeRequestCount
777            ArrayList<Long> requests = new ArrayList<Long>(2);
778            requests.add(currentReadRequestsCount);
779            requests.add(currentWriteRequestsCount);
780            requestsCountCache.put(encodedRegionName, requests);
781            totalReadRequestsDelta += currentReadRequestsCount;
782            totalWriteRequestsDelta += currentWriteRequestsCount;
783          }
784          tempReadRequestsCount += r.getReadRequestsCount();
785          tempWriteRequestsCount += r.getWriteRequestsCount();
786          tempNumMutationsWithoutWAL += r.getNumMutationsWithoutWAL();
787          tempDataInMemoryWithoutWAL += r.getDataInMemoryWithoutWAL();
788          tempCpRequestsCount += r.getCpRequestsCount();
789          tempFilteredReadRequestsCount += r.getFilteredReadRequestsCount();
790          tempCheckAndMutateChecksFailed += r.getCheckAndMutateChecksFailed();
791          tempCheckAndMutateChecksPassed += r.getCheckAndMutateChecksPassed();
792          tempBlockedRequestsCount += r.getBlockedRequestsCount();
793          List<? extends Store> storeList = r.getStores();
794          tempNumStores += storeList.size();
795          for (Store store : storeList) {
796            tempNumStoreFiles += store.getStorefilesCount();
797            tempMemstoreSize += store.getMemStoreSize().getDataSize();
798            tempOnHeapMemstoreSize += store.getMemStoreSize().getHeapSize();
799            tempOffHeapMemstoreSize += store.getMemStoreSize().getOffHeapSize();
800            tempStoreFileSize += store.getStorefilesSize();
801
802            OptionalLong storeMaxStoreFileAge = store.getMaxStoreFileAge();
803            if (
804              storeMaxStoreFileAge.isPresent()
805                && storeMaxStoreFileAge.getAsLong() > tempMaxStoreFileAge
806            ) {
807              tempMaxStoreFileAge = storeMaxStoreFileAge.getAsLong();
808            }
809
810            OptionalLong storeMinStoreFileAge = store.getMinStoreFileAge();
811            if (
812              storeMinStoreFileAge.isPresent()
813                && storeMinStoreFileAge.getAsLong() < tempMinStoreFileAge
814            ) {
815              tempMinStoreFileAge = storeMinStoreFileAge.getAsLong();
816            }
817
818            long storeHFiles = store.getNumHFiles();
819            numHFiles += storeHFiles;
820            tempNumReferenceFiles += store.getNumReferenceFiles();
821
822            OptionalDouble storeAvgStoreFileAge = store.getAvgStoreFileAge();
823            if (storeAvgStoreFileAge.isPresent()) {
824              avgAgeNumerator =
825                (long) (avgAgeNumerator + storeAvgStoreFileAge.getAsDouble() * storeHFiles);
826            }
827
828            tempStorefileIndexSize += store.getStorefilesRootLevelIndexSize();
829            tempTotalStaticBloomSize += store.getTotalStaticBloomSize();
830            tempTotalStaticIndexSize += store.getTotalStaticIndexSize();
831            tempFlushedCellsCount += store.getFlushedCellsCount();
832            tempCompactedCellsCount += store.getCompactedCellsCount();
833            tempMajorCompactedCellsCount += store.getMajorCompactedCellsCount();
834            tempFlushedCellsSize += store.getFlushedCellsSize();
835            tempCompactedCellsSize += store.getCompactedCellsSize();
836            tempMajorCompactedCellsSize += store.getMajorCompactedCellsSize();
837            if (store instanceof HMobStore) {
838              HMobStore mobStore = (HMobStore) store;
839              tempCellsCountCompactedToMob += mobStore.getCellsCountCompactedToMob();
840              tempCellsCountCompactedFromMob += mobStore.getCellsCountCompactedFromMob();
841              tempCellsSizeCompactedToMob += mobStore.getCellsSizeCompactedToMob();
842              tempCellsSizeCompactedFromMob += mobStore.getCellsSizeCompactedFromMob();
843              tempMobFlushCount += mobStore.getMobFlushCount();
844              tempMobFlushedCellsCount += mobStore.getMobFlushedCellsCount();
845              tempMobFlushedCellsSize += mobStore.getMobFlushedCellsSize();
846              tempMobScanCellsCount += mobStore.getMobScanCellsCount();
847              tempMobScanCellsSize += mobStore.getMobScanCellsSize();
848            }
849          }
850
851          HDFSBlocksDistribution distro = r.getHDFSBlocksDistribution();
852          hdfsBlocksDistribution.add(distro);
853          if (r.getRegionInfo().getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) {
854            hdfsBlocksDistributionSecondaryRegions.add(distro);
855          }
856          regionCount++;
857        }
858
859        float localityIndex =
860          hdfsBlocksDistribution.getBlockLocalityIndex(regionServer.getServerName().getHostname());
861        tempPercentFileLocal = Double.isNaN(tempBlockedRequestsCount) ? 0 : (localityIndex * 100);
862
863        float localityIndexSecondaryRegions = hdfsBlocksDistributionSecondaryRegions
864          .getBlockLocalityIndex(regionServer.getServerName().getHostname());
865        tempPercentFileLocalSecondaryRegions =
866          Double.isNaN(localityIndexSecondaryRegions) ? 0 : (localityIndexSecondaryRegions * 100);
867
868        // Compute the number of requests per second
869        long currentTime = EnvironmentEdgeManager.currentTime();
870
871        // assume that it took PERIOD seconds to start the executor.
872        // this is a guess but it's a pretty good one.
873        if (lastRan == 0) {
874          lastRan = currentTime - period;
875        }
876        // If we've time traveled keep the last requests per second.
877        if ((currentTime - lastRan) > 0) {
878          requestsPerSecond =
879            (totalReadRequestsDelta + totalWriteRequestsDelta) / ((currentTime - lastRan) / 1000.0);
880
881          double readRequestsRatePerMilliSecond = (double) totalReadRequestsDelta / period;
882          double writeRequestsRatePerMilliSecond = (double) totalWriteRequestsDelta / period;
883
884          readRequestsRatePerSecond = readRequestsRatePerMilliSecond * 1000.0;
885          writeRequestsRatePerSecond = writeRequestsRatePerMilliSecond * 1000.0;
886
887          long intervalStoreFileSize = tempStoreFileSize - lastStoreFileSize;
888          storeFileSizeGrowthRate = (double) intervalStoreFileSize * 1000.0 / period;
889
890          lastStoreFileSize = tempStoreFileSize;
891        }
892
893        lastRan = currentTime;
894
895        final WALProvider provider = regionServer.getWalFactory().getWALProvider();
896        final WALProvider metaProvider = regionServer.getWalFactory().getMetaWALProvider();
897        numWALFiles = (provider == null ? 0 : provider.getNumLogFiles())
898          + (metaProvider == null ? 0 : metaProvider.getNumLogFiles());
899        walFileSize = (provider == null ? 0 : provider.getLogFileSize())
900          + (metaProvider == null ? 0 : metaProvider.getLogFileSize());
901        // Copy over computed values so that no thread sees half computed values.
902        numStores = tempNumStores;
903        numStoreFiles = tempNumStoreFiles;
904        memstoreSize = tempMemstoreSize;
905        onHeapMemstoreSize = tempOnHeapMemstoreSize;
906        offHeapMemstoreSize = tempOffHeapMemstoreSize;
907        storeFileSize = tempStoreFileSize;
908        maxStoreFileAge = tempMaxStoreFileAge;
909        if (regionCount > 0) {
910          averageRegionSize = (memstoreSize + storeFileSize) / regionCount;
911        }
912        if (tempMinStoreFileAge != Long.MAX_VALUE) {
913          minStoreFileAge = tempMinStoreFileAge;
914        }
915
916        if (numHFiles != 0) {
917          avgStoreFileAge = avgAgeNumerator / numHFiles;
918        }
919
920        numReferenceFiles = tempNumReferenceFiles;
921        readRequestsCount = tempReadRequestsCount;
922        cpRequestsCount = tempCpRequestsCount;
923        filteredReadRequestsCount = tempFilteredReadRequestsCount;
924        writeRequestsCount = tempWriteRequestsCount;
925        checkAndMutateChecksFailed = tempCheckAndMutateChecksFailed;
926        checkAndMutateChecksPassed = tempCheckAndMutateChecksPassed;
927        storefileIndexSize = tempStorefileIndexSize;
928        totalStaticIndexSize = tempTotalStaticIndexSize;
929        totalStaticBloomSize = tempTotalStaticBloomSize;
930        numMutationsWithoutWAL = tempNumMutationsWithoutWAL;
931        dataInMemoryWithoutWAL = tempDataInMemoryWithoutWAL;
932        percentFileLocal = tempPercentFileLocal;
933        percentFileLocalSecondaryRegions = tempPercentFileLocalSecondaryRegions;
934        flushedCellsCount = tempFlushedCellsCount;
935        compactedCellsCount = tempCompactedCellsCount;
936        majorCompactedCellsCount = tempMajorCompactedCellsCount;
937        flushedCellsSize = tempFlushedCellsSize;
938        compactedCellsSize = tempCompactedCellsSize;
939        majorCompactedCellsSize = tempMajorCompactedCellsSize;
940        cellsCountCompactedToMob = tempCellsCountCompactedToMob;
941        cellsCountCompactedFromMob = tempCellsCountCompactedFromMob;
942        cellsSizeCompactedToMob = tempCellsSizeCompactedToMob;
943        cellsSizeCompactedFromMob = tempCellsSizeCompactedFromMob;
944        mobFlushCount = tempMobFlushCount;
945        mobFlushedCellsCount = tempMobFlushedCellsCount;
946        mobFlushedCellsSize = tempMobFlushedCellsSize;
947        mobScanCellsCount = tempMobScanCellsCount;
948        mobScanCellsSize = tempMobScanCellsSize;
949        mobFileCacheAccessCount = mobFileCache != null ? mobFileCache.getAccessCount() : 0L;
950        mobFileCacheMissCount = mobFileCache != null ? mobFileCache.getMissCount() : 0L;
951        mobFileCacheHitRatio = mobFileCache != null ? mobFileCache.getHitRatio() : 0.0;
952        if (Double.isNaN(mobFileCacheHitRatio)) {
953          mobFileCacheHitRatio = 0.0;
954        }
955        mobFileCacheEvictedCount = mobFileCache != null ? mobFileCache.getEvictedFileCount() : 0L;
956        mobFileCacheCount = mobFileCache != null ? mobFileCache.getCacheSize() : 0;
957        blockedRequestsCount = tempBlockedRequestsCount;
958      } catch (Throwable e) {
959        LOG.warn("Caught exception! Will suppress and retry.", e);
960      }
961    }
962  }
963
964  @Override
965  public long getHedgedReadOps() {
966    return this.dfsHedgedReadMetrics == null ? 0 : this.dfsHedgedReadMetrics.getHedgedReadOps();
967  }
968
969  @Override
970  public long getHedgedReadWins() {
971    return this.dfsHedgedReadMetrics == null ? 0 : this.dfsHedgedReadMetrics.getHedgedReadWins();
972  }
973
974  @Override
975  public long getHedgedReadOpsInCurThread() {
976    return this.dfsHedgedReadMetrics == null
977      ? 0
978      : this.dfsHedgedReadMetrics.getHedgedReadOpsInCurThread();
979  }
980
981  @Override
982  public long getTotalBytesRead() {
983    return FSDataInputStreamWrapper.getTotalBytesRead();
984  }
985
986  @Override
987  public long getLocalBytesRead() {
988    return FSDataInputStreamWrapper.getLocalBytesRead();
989  }
990
991  @Override
992  public long getShortCircuitBytesRead() {
993    return FSDataInputStreamWrapper.getShortCircuitBytesRead();
994  }
995
996  @Override
997  public long getZeroCopyBytesRead() {
998    return FSDataInputStreamWrapper.getZeroCopyBytesRead();
999  }
1000
1001  @Override
1002  public long getBlockedRequestsCount() {
1003    return blockedRequestsCount;
1004  }
1005
1006  @Override
1007  public long getAverageRegionSize() {
1008    return averageRegionSize;
1009  }
1010
1011  @Override
1012  public long getDataMissCount() {
1013    return this.cacheStats != null ? this.cacheStats.getDataMissCount() : 0L;
1014  }
1015
1016  @Override
1017  public long getLeafIndexMissCount() {
1018    return this.cacheStats != null ? this.cacheStats.getLeafIndexMissCount() : 0L;
1019  }
1020
1021  @Override
1022  public long getBloomChunkMissCount() {
1023    return this.cacheStats != null ? this.cacheStats.getBloomChunkMissCount() : 0L;
1024  }
1025
1026  @Override
1027  public long getMetaMissCount() {
1028    return this.cacheStats != null ? this.cacheStats.getMetaMissCount() : 0L;
1029  }
1030
1031  @Override
1032  public long getRootIndexMissCount() {
1033    return this.cacheStats != null ? this.cacheStats.getRootIndexMissCount() : 0L;
1034  }
1035
1036  @Override
1037  public long getIntermediateIndexMissCount() {
1038    return this.cacheStats != null ? this.cacheStats.getIntermediateIndexMissCount() : 0L;
1039  }
1040
1041  @Override
1042  public long getFileInfoMissCount() {
1043    return this.cacheStats != null ? this.cacheStats.getFileInfoMissCount() : 0L;
1044  }
1045
1046  @Override
1047  public long getGeneralBloomMetaMissCount() {
1048    return this.cacheStats != null ? this.cacheStats.getGeneralBloomMetaMissCount() : 0L;
1049  }
1050
1051  @Override
1052  public long getDeleteFamilyBloomMissCount() {
1053    return this.cacheStats != null ? this.cacheStats.getDeleteFamilyBloomMissCount() : 0L;
1054  }
1055
1056  @Override
1057  public long getTrailerMissCount() {
1058    return this.cacheStats != null ? this.cacheStats.getTrailerMissCount() : 0L;
1059  }
1060
1061  @Override
1062  public long getDataHitCount() {
1063    return this.cacheStats != null ? this.cacheStats.getDataHitCount() : 0L;
1064  }
1065
1066  @Override
1067  public long getLeafIndexHitCount() {
1068    return this.cacheStats != null ? this.cacheStats.getLeafIndexHitCount() : 0L;
1069  }
1070
1071  @Override
1072  public long getBloomChunkHitCount() {
1073    return this.cacheStats != null ? this.cacheStats.getBloomChunkHitCount() : 0L;
1074  }
1075
1076  @Override
1077  public long getMetaHitCount() {
1078    return this.cacheStats != null ? this.cacheStats.getMetaHitCount() : 0L;
1079  }
1080
1081  @Override
1082  public long getRootIndexHitCount() {
1083    return this.cacheStats != null ? this.cacheStats.getRootIndexHitCount() : 0L;
1084  }
1085
1086  @Override
1087  public long getIntermediateIndexHitCount() {
1088    return this.cacheStats != null ? this.cacheStats.getIntermediateIndexHitCount() : 0L;
1089  }
1090
1091  @Override
1092  public long getFileInfoHitCount() {
1093    return this.cacheStats != null ? this.cacheStats.getFileInfoHitCount() : 0L;
1094  }
1095
1096  @Override
1097  public long getGeneralBloomMetaHitCount() {
1098    return this.cacheStats != null ? this.cacheStats.getGeneralBloomMetaHitCount() : 0L;
1099  }
1100
1101  @Override
1102  public long getDeleteFamilyBloomHitCount() {
1103    return this.cacheStats != null ? this.cacheStats.getDeleteFamilyBloomHitCount() : 0L;
1104  }
1105
1106  @Override
1107  public long getTrailerHitCount() {
1108    return this.cacheStats != null ? this.cacheStats.getTrailerHitCount() : 0L;
1109  }
1110
1111  @Override
1112  public long getByteBuffAllocatorHeapAllocationBytes() {
1113    return ByteBuffAllocator.getHeapAllocationBytes(allocator, ByteBuffAllocator.HEAP);
1114  }
1115
1116  @Override
1117  public long getByteBuffAllocatorPoolAllocationBytes() {
1118    return this.allocator.getPoolAllocationBytes();
1119  }
1120
1121  @Override
1122  public double getByteBuffAllocatorHeapAllocRatio() {
1123    return ByteBuffAllocator.getHeapAllocationRatio(allocator, ByteBuffAllocator.HEAP);
1124  }
1125
1126  @Override
1127  public long getByteBuffAllocatorTotalBufferCount() {
1128    return this.allocator.getTotalBufferCount();
1129  }
1130
1131  @Override
1132  public long getByteBuffAllocatorUsedBufferCount() {
1133    return this.allocator.getUsedBufferCount();
1134  }
1135}