View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.regionserver;
19  
20  import java.io.IOException;
21  import java.util.Collection;
22  import java.util.concurrent.ScheduledExecutorService;
23  import java.util.concurrent.TimeUnit;
24  
25  import org.apache.commons.lang.StringUtils;
26  import org.apache.commons.logging.Log;
27  import org.apache.commons.logging.LogFactory;
28  import org.apache.hadoop.hbase.classification.InterfaceAudience;
29  import org.apache.hadoop.hbase.CompatibilitySingletonFactory;
30  import org.apache.hadoop.hbase.HConstants;
31  import org.apache.hadoop.hbase.HDFSBlocksDistribution;
32  import org.apache.hadoop.hbase.ServerName;
33  import org.apache.hadoop.hbase.io.hfile.BlockCache;
34  import org.apache.hadoop.hbase.io.hfile.CacheConfig;
35  import org.apache.hadoop.hbase.io.hfile.CacheStats;
36  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
37  import org.apache.hadoop.hbase.util.FSUtils;
38  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
39  import org.apache.hadoop.hdfs.DFSHedgedReadMetrics;
40  import org.apache.hadoop.metrics2.MetricsExecutor;
41  
42  /**
43   * Impl for exposing HRegionServer Information through Hadoop's metrics 2 system.
44   */
45  @InterfaceAudience.Private
46  class MetricsRegionServerWrapperImpl
47      implements MetricsRegionServerWrapper {
48  
49    public static final Log LOG = LogFactory.getLog(MetricsRegionServerWrapperImpl.class);
50  
51    private final HRegionServer regionServer;
52  
53    private BlockCache blockCache;
54  
55    private volatile long numStores = 0;
56    private volatile long numHLogFiles = 0;
57    private volatile long hlogFileSize = 0;
58    private volatile long numStoreFiles = 0;
59    private volatile long memstoreSize = 0;
60    private volatile long storeFileSize = 0;
61    private volatile double requestsPerSecond = 0.0;
62    private volatile long readRequestsCount = 0;
63    private volatile long writeRequestsCount = 0;
64    private volatile long checkAndMutateChecksFailed = 0;
65    private volatile long checkAndMutateChecksPassed = 0;
66    private volatile long storefileIndexSize = 0;
67    private volatile long totalStaticIndexSize = 0;
68    private volatile long totalStaticBloomSize = 0;
69    private volatile long numMutationsWithoutWAL = 0;
70    private volatile long dataInMemoryWithoutWAL = 0;
71    private volatile int percentFileLocal = 0;
72    private volatile long flushedCellsCount = 0;
73    private volatile long compactedCellsCount = 0;
74    private volatile long majorCompactedCellsCount = 0;
75    private volatile long flushedCellsSize = 0;
76    private volatile long compactedCellsSize = 0;
77    private volatile long majorCompactedCellsSize = 0;
78  
79    private CacheStats cacheStats;
80    private ScheduledExecutorService executor;
81    private Runnable runnable;
82    private long period;
83  
84    /**
85     * Can be null if not on hdfs.
86     */
87    private DFSHedgedReadMetrics dfsHedgedReadMetrics;
88  
89    public MetricsRegionServerWrapperImpl(final HRegionServer regionServer) {
90      this.regionServer = regionServer;
91      initBlockCache();
92  
93      this.period =
94          regionServer.conf.getLong(HConstants.REGIONSERVER_METRICS_PERIOD,
95            HConstants.DEFAULT_REGIONSERVER_METRICS_PERIOD);
96  
97      this.executor = CompatibilitySingletonFactory.getInstance(MetricsExecutor.class).getExecutor();
98      this.runnable = new RegionServerMetricsWrapperRunnable();
99      this.executor.scheduleWithFixedDelay(this.runnable, this.period, this.period,
100       TimeUnit.MILLISECONDS);
101 
102     try {
103       this.dfsHedgedReadMetrics = FSUtils.getDFSHedgedReadMetrics(regionServer.getConfiguration());
104     } catch (IOException e) {
105       LOG.warn("Failed to get hedged metrics", e);
106     }
107     if (LOG.isInfoEnabled()) {
108       LOG.info("Computing regionserver metrics every " + this.period + " milliseconds");
109     }
110   }
111 
112   /**
113    * It's possible that due to threading the block cache could not be initialized
114    * yet (testing multiple region servers in one jvm).  So we need to try and initialize
115    * the blockCache and cacheStats reference multiple times until we succeed.
116    */
117   private synchronized  void initBlockCache() {
118     CacheConfig cacheConfig = this.regionServer.cacheConfig;
119     if (cacheConfig != null && this.blockCache == null) {
120       this.blockCache = cacheConfig.getBlockCache();
121     }
122 
123     if (this.blockCache != null && this.cacheStats == null) {
124       this.cacheStats = blockCache.getStats();
125     }
126   }
127 
128   @Override
129   public String getClusterId() {
130     return regionServer.getClusterId();
131   }
132 
133   @Override
134   public long getStartCode() {
135     return regionServer.getStartcode();
136   }
137 
138   @Override
139   public String getZookeeperQuorum() {
140     ZooKeeperWatcher zk = regionServer.getZooKeeper();
141     if (zk == null) {
142       return "";
143     }
144     return zk.getQuorum();
145   }
146 
147   @Override
148   public String getCoprocessors() {
149     String[] coprocessors = regionServer.getRegionServerCoprocessors();
150     if (coprocessors == null || coprocessors.length == 0) {
151       return "";
152     }
153     return StringUtils.join(coprocessors, ", ");
154   }
155 
156   @Override
157   public String getServerName() {
158     ServerName serverName = regionServer.getServerName();
159     if (serverName == null) {
160       return "";
161     }
162     return serverName.getServerName();
163   }
164 
165   @Override
166   public long getNumOnlineRegions() {
167     Collection<HRegion> onlineRegionsLocalContext = regionServer.getOnlineRegionsLocalContext();
168     if (onlineRegionsLocalContext == null) {
169       return 0;
170     }
171     return onlineRegionsLocalContext.size();
172   }
173 
174   @Override
175   public long getTotalRequestCount() {
176     return regionServer.rpcServices.requestCount.get();
177   }
178 
179   @Override
180   public int getCompactionQueueSize() {
181     //The thread could be zero.  if so assume there is no queue.
182     if (this.regionServer.compactSplitThread == null) {
183       return 0;
184     }
185     return this.regionServer.compactSplitThread.getCompactionQueueSize();
186   }
187 
188   @Override
189   public int getSmallCompactionQueueSize() {
190     //The thread could be zero.  if so assume there is no queue.
191     if (this.regionServer.compactSplitThread == null) {
192       return 0;
193     }
194     return this.regionServer.compactSplitThread.getSmallCompactionQueueSize();
195   }
196 
197   @Override
198   public int getLargeCompactionQueueSize() {
199     //The thread could be zero.  if so assume there is no queue.
200     if (this.regionServer.compactSplitThread == null) {
201       return 0;
202     }
203     return this.regionServer.compactSplitThread.getLargeCompactionQueueSize();
204   }
205 
206   @Override
207   public int getFlushQueueSize() {
208     //If there is no flusher there should be no queue.
209     if (this.regionServer.cacheFlusher == null) {
210       return 0;
211     }
212     return this.regionServer.cacheFlusher.getFlushQueueSize();
213   }
214 
215   @Override
216   public long getBlockCacheCount() {
217     if (this.blockCache == null) {
218       return 0;
219     }
220     return this.blockCache.getBlockCount();
221   }
222 
223   @Override
224   public long getBlockCacheSize() {
225     if (this.blockCache == null) {
226       return 0;
227     }
228     return this.blockCache.getCurrentSize();
229   }
230 
231   @Override
232   public long getBlockCacheFreeSize() {
233     if (this.blockCache == null) {
234       return 0;
235     }
236     return this.blockCache.getFreeSize();
237   }
238 
239   @Override
240   public long getBlockCacheHitCount() {
241     if (this.cacheStats == null) {
242       return 0;
243     }
244     return this.cacheStats.getHitCount();
245   }
246 
247   @Override
248   public long getBlockCacheMissCount() {
249     if (this.cacheStats == null) {
250       return 0;
251     }
252     return this.cacheStats.getMissCount();
253   }
254 
255   @Override
256   public long getBlockCacheEvictedCount() {
257     if (this.cacheStats == null) {
258       return 0;
259     }
260     return this.cacheStats.getEvictedCount();
261   }
262 
263   @Override
264   public double getBlockCacheHitPercent() {
265     if (this.cacheStats == null) {
266       return 0;
267     }
268     return (int) (this.cacheStats.getHitRatio() * 100);
269   }
270 
271   @Override
272   public int getBlockCacheHitCachingPercent() {
273     if (this.cacheStats == null) {
274       return 0;
275     }
276     return (int) (this.cacheStats.getHitCachingRatio() * 100);
277   }
278 
279   @Override public void forceRecompute() {
280     this.runnable.run();
281   }
282 
283   @Override
284   public long getNumStores() {
285     return numStores;
286   }
287   
288   @Override
289   public long getNumHLogFiles() {
290     return numHLogFiles;
291   }
292 
293   @Override
294   public long getHLogFileSize() {
295     return hlogFileSize;
296   }
297   
298   @Override
299   public long getNumStoreFiles() {
300     return numStoreFiles;
301   }
302 
303   @Override
304   public long getMemstoreSize() {
305     return memstoreSize;
306   }
307 
308   @Override
309   public long getStoreFileSize() {
310     return storeFileSize;
311   }
312 
313   @Override public double getRequestsPerSecond() {
314     return requestsPerSecond;
315   }
316 
317   @Override
318   public long getReadRequestsCount() {
319     return readRequestsCount;
320   }
321 
322   @Override
323   public long getWriteRequestsCount() {
324     return writeRequestsCount;
325   }
326 
327   @Override
328   public long getCheckAndMutateChecksFailed() {
329     return checkAndMutateChecksFailed;
330   }
331 
332   @Override
333   public long getCheckAndMutateChecksPassed() {
334     return checkAndMutateChecksPassed;
335   }
336 
337   @Override
338   public long getStoreFileIndexSize() {
339     return storefileIndexSize;
340   }
341 
342   @Override
343   public long getTotalStaticIndexSize() {
344     return totalStaticIndexSize;
345   }
346 
347   @Override
348   public long getTotalStaticBloomSize() {
349     return totalStaticBloomSize;
350   }
351 
352   @Override
353   public long getNumMutationsWithoutWAL() {
354     return numMutationsWithoutWAL;
355   }
356 
357   @Override
358   public long getDataInMemoryWithoutWAL() {
359     return dataInMemoryWithoutWAL;
360   }
361 
362   @Override
363   public int getPercentFileLocal() {
364     return percentFileLocal;
365   }
366 
367   @Override
368   public long getUpdatesBlockedTime() {
369     if (this.regionServer.cacheFlusher == null) {
370       return 0;
371     }
372     return this.regionServer.cacheFlusher.getUpdatesBlockedMsHighWater().get();
373   }
374 
375   @Override
376   public long getFlushedCellsCount() {
377     return flushedCellsCount;
378   }
379 
380   @Override
381   public long getCompactedCellsCount() {
382     return compactedCellsCount;
383   }
384 
385   @Override
386   public long getMajorCompactedCellsCount() {
387     return majorCompactedCellsCount;
388   }
389 
390   @Override
391   public long getFlushedCellsSize() {
392     return flushedCellsSize;
393   }
394 
395   @Override
396   public long getCompactedCellsSize() {
397     return compactedCellsSize;
398   }
399 
400   @Override
401   public long getMajorCompactedCellsSize() {
402     return majorCompactedCellsSize;
403   }
404 
405   /**
406    * This is the runnable that will be executed on the executor every PERIOD number of seconds
407    * It will take metrics/numbers from all of the regions and use them to compute point in
408    * time metrics.
409    */
410   public class RegionServerMetricsWrapperRunnable implements Runnable {
411 
412     private long lastRan = 0;
413     private long lastRequestCount = 0;
414 
415     @Override
416     synchronized public void run() {
417       initBlockCache();
418       cacheStats = blockCache.getStats();
419 
420       HDFSBlocksDistribution hdfsBlocksDistribution =
421           new HDFSBlocksDistribution();
422 
423       long tempNumStores = 0;
424       long tempNumStoreFiles = 0;
425       long tempMemstoreSize = 0;
426       long tempStoreFileSize = 0;
427       long tempReadRequestsCount = 0;
428       long tempWriteRequestsCount = 0;
429       long tempCheckAndMutateChecksFailed = 0;
430       long tempCheckAndMutateChecksPassed = 0;
431       long tempStorefileIndexSize = 0;
432       long tempTotalStaticIndexSize = 0;
433       long tempTotalStaticBloomSize = 0;
434       long tempNumMutationsWithoutWAL = 0;
435       long tempDataInMemoryWithoutWAL = 0;
436       int tempPercentFileLocal = 0;
437       long tempFlushedCellsCount = 0;
438       long tempCompactedCellsCount = 0;
439       long tempMajorCompactedCellsCount = 0;
440       long tempFlushedCellsSize = 0;
441       long tempCompactedCellsSize = 0;
442       long tempMajorCompactedCellsSize = 0;
443 
444       for (HRegion r : regionServer.getOnlineRegionsLocalContext()) {
445         tempNumMutationsWithoutWAL += r.numMutationsWithoutWAL.get();
446         tempDataInMemoryWithoutWAL += r.dataInMemoryWithoutWAL.get();
447         tempReadRequestsCount += r.readRequestsCount.get();
448         tempWriteRequestsCount += r.writeRequestsCount.get();
449         tempCheckAndMutateChecksFailed += r.checkAndMutateChecksFailed.get();
450         tempCheckAndMutateChecksPassed += r.checkAndMutateChecksPassed.get();
451         tempNumStores += r.stores.size();
452         for (Store store : r.stores.values()) {
453           tempNumStoreFiles += store.getStorefilesCount();
454           tempMemstoreSize += store.getMemStoreSize();
455           tempStoreFileSize += store.getStorefilesSize();
456           tempStorefileIndexSize += store.getStorefilesIndexSize();
457           tempTotalStaticBloomSize += store.getTotalStaticBloomSize();
458           tempTotalStaticIndexSize += store.getTotalStaticIndexSize();
459           tempFlushedCellsCount += store.getFlushedCellsCount();
460           tempCompactedCellsCount += store.getCompactedCellsCount();
461           tempMajorCompactedCellsCount += store.getMajorCompactedCellsCount();
462           tempFlushedCellsSize += store.getFlushedCellsSize();
463           tempCompactedCellsSize += store.getCompactedCellsSize();
464           tempMajorCompactedCellsSize += store.getMajorCompactedCellsSize();
465         }
466 
467         hdfsBlocksDistribution.add(r.getHDFSBlocksDistribution());
468       }
469 
470       float localityIndex = hdfsBlocksDistribution.getBlockLocalityIndex(
471           regionServer.getServerName().getHostname());
472       tempPercentFileLocal = (int) (localityIndex * 100);
473 
474 
475       //Compute the number of requests per second
476       long currentTime = EnvironmentEdgeManager.currentTime();
477 
478       // assume that it took PERIOD seconds to start the executor.
479       // this is a guess but it's a pretty good one.
480       if (lastRan == 0) {
481         lastRan = currentTime - period;
482       }
483 
484 
485       //If we've time traveled keep the last requests per second.
486       if ((currentTime - lastRan) > 0) {
487         long currentRequestCount = getTotalRequestCount();
488         requestsPerSecond = (currentRequestCount - lastRequestCount) / ((currentTime - lastRan) / 1000.0);
489         lastRequestCount = currentRequestCount;
490       }
491       lastRan = currentTime;
492 
493       //Copy over computed values so that no thread sees half computed values.
494       numStores = tempNumStores;
495       long tempNumHLogFiles = regionServer.hlog.getNumLogFiles();
496       // meta logs
497       if (regionServer.hlogForMeta != null) {
498         tempNumHLogFiles += regionServer.hlogForMeta.getNumLogFiles();
499       }
500       numHLogFiles = tempNumHLogFiles;
501       
502       long tempHlogFileSize = regionServer.hlog.getLogFileSize();
503       if (regionServer.hlogForMeta != null) {
504         tempHlogFileSize += regionServer.hlogForMeta.getLogFileSize();
505       }
506       hlogFileSize = tempHlogFileSize;
507       
508       numStoreFiles = tempNumStoreFiles;
509       memstoreSize = tempMemstoreSize;
510       storeFileSize = tempStoreFileSize;
511       readRequestsCount = tempReadRequestsCount;
512       writeRequestsCount = tempWriteRequestsCount;
513       checkAndMutateChecksFailed = tempCheckAndMutateChecksFailed;
514       checkAndMutateChecksPassed = tempCheckAndMutateChecksPassed;
515       storefileIndexSize = tempStorefileIndexSize;
516       totalStaticIndexSize = tempTotalStaticIndexSize;
517       totalStaticBloomSize = tempTotalStaticBloomSize;
518       numMutationsWithoutWAL = tempNumMutationsWithoutWAL;
519       dataInMemoryWithoutWAL = tempDataInMemoryWithoutWAL;
520       percentFileLocal = tempPercentFileLocal;
521       flushedCellsCount = tempFlushedCellsCount;
522       compactedCellsCount = tempCompactedCellsCount;
523       majorCompactedCellsCount = tempMajorCompactedCellsCount;
524       flushedCellsSize = tempFlushedCellsSize;
525       compactedCellsSize = tempCompactedCellsSize;
526       majorCompactedCellsSize = tempMajorCompactedCellsSize;
527     }
528   }
529 
530   @Override
531   public long getHedgedReadOps() {
532     return this.dfsHedgedReadMetrics == null? 0: this.dfsHedgedReadMetrics.getHedgedReadOps();
533   }
534 
535   @Override
536   public long getHedgedReadWins() {
537     return this.dfsHedgedReadMetrics == null? 0: this.dfsHedgedReadMetrics.getHedgedReadWins();
538   }
539 }