001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import java.io.Closeable;
021import java.io.IOException;
022import java.util.HashMap;
023import java.util.Map;
024import java.util.Set;
025import java.util.concurrent.ConcurrentHashMap;
026import java.util.concurrent.ScheduledExecutorService;
027import java.util.concurrent.ScheduledFuture;
028import java.util.concurrent.TimeUnit;
029import org.apache.hadoop.hbase.CompatibilitySingletonFactory;
030import org.apache.hadoop.hbase.HConstants;
031import org.apache.hadoop.hbase.TableName;
032import org.apache.hadoop.metrics2.MetricsExecutor;
033import org.apache.yetus.audience.InterfaceAudience;
034
035import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
036
037@InterfaceAudience.Private
038public class MetricsTableWrapperAggregateImpl implements MetricsTableWrapperAggregate, Closeable {
039  private final HRegionServer regionServer;
040  private ScheduledExecutorService executor;
041  private Runnable runnable;
042  private long period;
043  private ScheduledFuture<?> tableMetricsUpdateTask;
044  private ConcurrentHashMap<TableName, MetricsTableValues> metricsTableMap =
045    new ConcurrentHashMap<>();
046
047  public MetricsTableWrapperAggregateImpl(final HRegionServer regionServer) {
048    this.regionServer = regionServer;
049    this.period = regionServer.getConfiguration().getLong(HConstants.REGIONSERVER_METRICS_PERIOD,
050      HConstants.DEFAULT_REGIONSERVER_METRICS_PERIOD) + 1000;
051    this.executor = CompatibilitySingletonFactory.getInstance(MetricsExecutor.class).getExecutor();
052    this.runnable = new TableMetricsWrapperRunnable();
053    this.tableMetricsUpdateTask =
054      this.executor.scheduleWithFixedDelay(this.runnable, period, period, TimeUnit.MILLISECONDS);
055  }
056
057  public class TableMetricsWrapperRunnable implements Runnable {
058
059    @Override
060    public void run() {
061      Map<TableName, MetricsTableValues> localMetricsTableMap = new HashMap<>();
062      for (Region r : regionServer.getOnlineRegionsLocalContext()) {
063        TableName tbl = r.getTableDescriptor().getTableName();
064        MetricsTableValues mt = localMetricsTableMap.get(tbl);
065        if (mt == null) {
066          mt = new MetricsTableValues();
067          localMetricsTableMap.put(tbl, mt);
068        }
069        long memstoreReadCount = 0L;
070        long mixedReadCount = 0L;
071        String tempKey = null;
072        if (r.getStores() != null) {
073          String familyName = null;
074          for (Store store : r.getStores()) {
075            familyName = store.getColumnFamilyName();
076
077            mt.storeFileCount += store.getStorefilesCount();
078            mt.memstoreSize += (store.getMemStoreSize().getDataSize()
079              + store.getMemStoreSize().getHeapSize() + store.getMemStoreSize().getOffHeapSize());
080            mt.storeFileSize += store.getStorefilesSize();
081            mt.referenceFileCount += store.getNumReferenceFiles();
082            if (store.getMaxStoreFileAge().isPresent()) {
083              mt.maxStoreFileAge =
084                Math.max(mt.maxStoreFileAge, store.getMaxStoreFileAge().getAsLong());
085            }
086            if (store.getMinStoreFileAge().isPresent()) {
087              mt.minStoreFileAge =
088                Math.min(mt.minStoreFileAge, store.getMinStoreFileAge().getAsLong());
089            }
090            if (store.getAvgStoreFileAge().isPresent()) {
091              mt.totalStoreFileAge =
092                (long) store.getAvgStoreFileAge().getAsDouble() * store.getStorefilesCount();
093            }
094            mt.storeCount += 1;
095            tempKey = tbl.getNameAsString() + HASH + familyName;
096            Long tempVal = mt.perStoreMemstoreOnlyReadCount.get(tempKey);
097            if (tempVal == null) {
098              tempVal = 0L;
099            }
100            memstoreReadCount = store.getMemstoreOnlyRowReadsCount() + tempVal;
101            tempVal = mt.perStoreMixedReadCount.get(tempKey);
102            if (tempVal == null) {
103              tempVal = 0L;
104            }
105            mixedReadCount = store.getMixedRowReadsCount() + tempVal;
106            // accumulate the count
107            mt.perStoreMemstoreOnlyReadCount.put(tempKey, memstoreReadCount);
108            mt.perStoreMixedReadCount.put(tempKey, mixedReadCount);
109          }
110
111          mt.regionCount += 1;
112
113          mt.readRequestCount += r.getReadRequestsCount();
114          mt.filteredReadRequestCount += r.getFilteredReadRequestsCount();
115          mt.writeRequestCount += r.getWriteRequestsCount();
116        }
117      }
118
119      for (Map.Entry<TableName, MetricsTableValues> entry : localMetricsTableMap.entrySet()) {
120        TableName tbl = entry.getKey();
121        if (metricsTableMap.get(tbl) == null) {
122          // this will add the Wrapper to the list of TableMetrics
123          CompatibilitySingletonFactory.getInstance(MetricsRegionServerSourceFactory.class)
124            .getTableAggregate()
125            .getOrCreateTableSource(tbl.getNameAsString(), MetricsTableWrapperAggregateImpl.this);
126        }
127        metricsTableMap.put(entry.getKey(), entry.getValue());
128      }
129      Set<TableName> existingTableNames = Sets.newHashSet(metricsTableMap.keySet());
130      existingTableNames.removeAll(localMetricsTableMap.keySet());
131      MetricsTableAggregateSource agg = CompatibilitySingletonFactory
132        .getInstance(MetricsRegionServerSourceFactory.class).getTableAggregate();
133      for (TableName table : existingTableNames) {
134        agg.deleteTableSource(table.getNameAsString());
135        if (metricsTableMap.get(table) != null) {
136          metricsTableMap.remove(table);
137        }
138      }
139    }
140  }
141
142  @Override
143  public long getReadRequestCount(String table) {
144    MetricsTableValues metricsTable = metricsTableMap.get(TableName.valueOf(table));
145    if (metricsTable == null) {
146      return 0;
147    } else {
148      return metricsTable.readRequestCount;
149    }
150  }
151
152  @Override
153  public Map<String, Long> getMemstoreOnlyRowReadsCount(String table) {
154    MetricsTableValues metricsTable = metricsTableMap.get(TableName.valueOf(table));
155    if (metricsTable == null) {
156      return null;
157    } else {
158      return metricsTable.perStoreMemstoreOnlyReadCount;
159    }
160  }
161
162  @Override
163  public Map<String, Long> getMixedRowReadsCount(String table) {
164    MetricsTableValues metricsTable = metricsTableMap.get(TableName.valueOf(table));
165    if (metricsTable == null) {
166      return null;
167    } else {
168      return metricsTable.perStoreMixedReadCount;
169    }
170  }
171
172  @Override
173  public long getCpRequestsCount(String table) {
174    MetricsTableValues metricsTable = metricsTableMap.get(TableName.valueOf(table));
175    if (metricsTable == null) {
176      return 0;
177    } else {
178      return metricsTable.cpRequestCount;
179    }
180  }
181
182  public long getFilteredReadRequestCount(String table) {
183    MetricsTableValues metricsTable = metricsTableMap.get(TableName.valueOf(table));
184    if (metricsTable == null) {
185      return 0;
186    }
187    return metricsTable.filteredReadRequestCount;
188  }
189
190  @Override
191  public long getWriteRequestCount(String table) {
192    MetricsTableValues metricsTable = metricsTableMap.get(TableName.valueOf(table));
193    if (metricsTable == null) {
194      return 0;
195    } else {
196      return metricsTable.writeRequestCount;
197    }
198  }
199
200  @Override
201  public long getTotalRequestsCount(String table) {
202    MetricsTableValues metricsTable = metricsTableMap.get(TableName.valueOf(table));
203    if (metricsTable == null) {
204      return 0;
205    } else {
206      return metricsTable.readRequestCount + metricsTable.writeRequestCount;
207    }
208  }
209
210  @Override
211  public long getMemStoreSize(String table) {
212    MetricsTableValues metricsTable = metricsTableMap.get(TableName.valueOf(table));
213    if (metricsTable == null) {
214      return 0;
215    } else {
216      return metricsTable.memstoreSize;
217    }
218  }
219
220  @Override
221  public long getStoreFileSize(String table) {
222    MetricsTableValues metricsTable = metricsTableMap.get(TableName.valueOf(table));
223    if (metricsTable == null) {
224      return 0;
225    } else {
226      return metricsTable.storeFileSize;
227    }
228  }
229
230  @Override
231  public long getTableSize(String table) {
232    MetricsTableValues metricsTable = metricsTableMap.get(TableName.valueOf(table));
233    if (metricsTable == null) {
234      return 0;
235    } else {
236      return metricsTable.memstoreSize + metricsTable.storeFileSize;
237    }
238  }
239
240  public long getNumRegions(String table) {
241    MetricsTableValues metricsTable = metricsTableMap.get(TableName.valueOf(table));
242    if (metricsTable == null) {
243      return 0;
244    }
245    return metricsTable.regionCount;
246  }
247
248  @Override
249  public long getNumStores(String table) {
250    MetricsTableValues metricsTable = metricsTableMap.get(TableName.valueOf(table));
251    if (metricsTable == null) {
252      return 0;
253    }
254    return metricsTable.storeCount;
255  }
256
257  @Override
258  public long getNumStoreFiles(String table) {
259    MetricsTableValues metricsTable = metricsTableMap.get(TableName.valueOf(table));
260    if (metricsTable == null) {
261      return 0;
262    }
263    return metricsTable.storeFileCount;
264  }
265
266  @Override
267  public long getMaxStoreFileAge(String table) {
268    MetricsTableValues metricsTable = metricsTableMap.get(TableName.valueOf(table));
269    if (metricsTable == null) {
270      return 0;
271    }
272    return metricsTable.maxStoreFileAge;
273  }
274
275  @Override
276  public long getMinStoreFileAge(String table) {
277    MetricsTableValues metricsTable = metricsTableMap.get(TableName.valueOf(table));
278    if (metricsTable == null) {
279      return 0;
280    }
281    return metricsTable.minStoreFileAge == Long.MAX_VALUE ? 0 : metricsTable.minStoreFileAge;
282  }
283
284  @Override
285  public long getAvgStoreFileAge(String table) {
286    MetricsTableValues metricsTable = metricsTableMap.get(TableName.valueOf(table));
287    if (metricsTable == null) {
288      return 0;
289    }
290
291    return metricsTable.storeFileCount == 0
292      ? 0
293      : (metricsTable.totalStoreFileAge / metricsTable.storeFileCount);
294  }
295
296  @Override
297  public long getNumReferenceFiles(String table) {
298    MetricsTableValues metricsTable = metricsTableMap.get(TableName.valueOf(table));
299    if (metricsTable == null) {
300      return 0;
301    }
302    return metricsTable.referenceFileCount;
303  }
304
305  @Override
306  public long getAvgRegionSize(String table) {
307    MetricsTableValues metricsTable = metricsTableMap.get(TableName.valueOf(table));
308    if (metricsTable == null) {
309      return 0;
310    }
311    return metricsTable.regionCount == 0
312      ? 0
313      : (metricsTable.memstoreSize + metricsTable.storeFileSize) / metricsTable.regionCount;
314  }
315
316  public long getCpRequestCount(String table) {
317    MetricsTableValues metricsTable = metricsTableMap.get(TableName.valueOf(table));
318    if (metricsTable == null) {
319      return 0;
320    }
321    return metricsTable.cpRequestCount;
322  }
323
324  @Override
325  public void close() throws IOException {
326    tableMetricsUpdateTask.cancel(true);
327  }
328
329  private static class MetricsTableValues {
330    long readRequestCount;
331    long filteredReadRequestCount;
332    long writeRequestCount;
333    long memstoreSize;
334    long regionCount;
335    long storeCount;
336    long storeFileCount;
337    long storeFileSize;
338    long maxStoreFileAge;
339    long minStoreFileAge = Long.MAX_VALUE;
340    long totalStoreFileAge;
341    long referenceFileCount;
342    long cpRequestCount;
343    Map<String, Long> perStoreMemstoreOnlyReadCount = new HashMap<>();
344    Map<String, Long> perStoreMixedReadCount = new HashMap<>();
345  }
346
347}