001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase;
019
020import edu.umd.cs.findbugs.annotations.Nullable;
021import java.util.ArrayList;
022import java.util.Collection;
023import java.util.Collections;
024import java.util.HashMap;
025import java.util.List;
026import java.util.Map;
027import java.util.Set;
028import java.util.TreeMap;
029import java.util.TreeSet;
030import java.util.stream.Collectors;
031import org.apache.hadoop.hbase.replication.ReplicationLoadSink;
032import org.apache.hadoop.hbase.replication.ReplicationLoadSource;
033import org.apache.hadoop.hbase.util.Bytes;
034import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
035import org.apache.hadoop.hbase.util.Strings;
036import org.apache.yetus.audience.InterfaceAudience;
037
038import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
039
040import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
041import org.apache.hadoop.hbase.shaded.protobuf.generated.ClusterStatusProtos;
042import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos;
043
044@InterfaceAudience.Private
045public final class ServerMetricsBuilder {
046
047  public static ServerMetrics of(ServerName sn) {
048    return newBuilder(sn).build();
049  }
050
051  public static ServerMetrics of(ServerName sn, int versionNumber, String version) {
052    return newBuilder(sn).setVersionNumber(versionNumber).setVersion(version).build();
053  }
054
055  public static ServerMetrics toServerMetrics(ClusterStatusProtos.LiveServerInfo serverInfo) {
056    return toServerMetrics(ProtobufUtil.toServerName(serverInfo.getServer()), 0, "0.0.0",
057      serverInfo.getServerLoad());
058  }
059
060  public static ServerMetrics toServerMetrics(ServerName serverName,
061    ClusterStatusProtos.ServerLoad serverLoadPB) {
062    return toServerMetrics(serverName, 0, "0.0.0", serverLoadPB);
063  }
064
065  public static ServerMetrics toServerMetrics(ServerName serverName, int versionNumber,
066    String version, ClusterStatusProtos.ServerLoad serverLoadPB) {
067    return ServerMetricsBuilder.newBuilder(serverName)
068      .setRequestCountPerSecond(serverLoadPB.getNumberOfRequests())
069      .setRequestCount(serverLoadPB.getTotalNumberOfRequests())
070      .setInfoServerPort(serverLoadPB.getInfoServerPort())
071      .setReadRequestCount(serverLoadPB.getReadRequestsCount())
072      .setWriteRequestCount(serverLoadPB.getWriteRequestsCount())
073      .setMaxHeapSize(new Size(serverLoadPB.getMaxHeapMB(), Size.Unit.MEGABYTE))
074      .setUsedHeapSize(new Size(serverLoadPB.getUsedHeapMB(), Size.Unit.MEGABYTE))
075      .setCoprocessorNames(serverLoadPB.getCoprocessorsList().stream()
076        .map(HBaseProtos.Coprocessor::getName).collect(Collectors.toList()))
077      .setRegionMetrics(serverLoadPB.getRegionLoadsList().stream()
078        .map(RegionMetricsBuilder::toRegionMetrics).collect(Collectors.toList()))
079      .setUserMetrics(serverLoadPB.getUserLoadsList().stream()
080        .map(UserMetricsBuilder::toUserMetrics).collect(Collectors.toList()))
081      .setReplicationLoadSources(serverLoadPB.getReplLoadSourceList().stream()
082        .map(ProtobufUtil::toReplicationLoadSource).collect(Collectors.toList()))
083      .setReplicationLoadSink(serverLoadPB.hasReplLoadSink()
084        ? ProtobufUtil.toReplicationLoadSink(serverLoadPB.getReplLoadSink())
085        : null)
086      .setTasks(serverLoadPB.getTasksList().stream().map(ProtobufUtil::getServerTask)
087        .collect(Collectors.toList()))
088      .setRegionCachedInfo(serverLoadPB.getRegionCachedInfoMap())
089      .setReportTimestamp(serverLoadPB.getReportEndTime())
090      .setLastReportTimestamp(serverLoadPB.getReportStartTime()).setVersionNumber(versionNumber)
091      .setVersion(version).build();
092  }
093
094  public static List<HBaseProtos.Coprocessor> toCoprocessor(Collection<String> names) {
095    return names.stream().map(n -> HBaseProtos.Coprocessor.newBuilder().setName(n).build())
096      .collect(Collectors.toList());
097  }
098
099  public static ClusterStatusProtos.ServerLoad toServerLoad(ServerMetrics metrics) {
100    ClusterStatusProtos.ServerLoad.Builder builder = ClusterStatusProtos.ServerLoad.newBuilder()
101      .setNumberOfRequests(metrics.getRequestCountPerSecond())
102      .setTotalNumberOfRequests(metrics.getRequestCount())
103      .setInfoServerPort(metrics.getInfoServerPort())
104      .setMaxHeapMB((int) metrics.getMaxHeapSize().get(Size.Unit.MEGABYTE))
105      .setUsedHeapMB((int) metrics.getUsedHeapSize().get(Size.Unit.MEGABYTE))
106      .addAllCoprocessors(toCoprocessor(metrics.getCoprocessorNames()))
107      .addAllRegionLoads(metrics.getRegionMetrics().values().stream()
108        .map(RegionMetricsBuilder::toRegionLoad).collect(Collectors.toList()))
109      .addAllUserLoads(metrics.getUserMetrics().values().stream()
110        .map(UserMetricsBuilder::toUserMetrics).collect(Collectors.toList()))
111      .addAllReplLoadSource(metrics.getReplicationLoadSourceList().stream()
112        .map(ProtobufUtil::toReplicationLoadSource).collect(Collectors.toList()))
113      .addAllTasks(
114        metrics.getTasks().stream().map(ProtobufUtil::toServerTask).collect(Collectors.toList()))
115      .putAllRegionCachedInfo(metrics.getRegionCachedInfo())
116      .setReportStartTime(metrics.getLastReportTimestamp())
117      .setReportEndTime(metrics.getReportTimestamp());
118    if (metrics.getReplicationLoadSink() != null) {
119      builder.setReplLoadSink(ProtobufUtil.toReplicationLoadSink(metrics.getReplicationLoadSink()));
120    }
121    return builder.build();
122  }
123
124  public static ServerMetricsBuilder newBuilder(ServerName sn) {
125    return new ServerMetricsBuilder(sn);
126  }
127
128  private final ServerName serverName;
129  private int versionNumber;
130  private String version = "0.0.0";
131  private long requestCountPerSecond;
132  private long requestCount;
133  private long readRequestCount;
134  private long writeRequestCount;
135  private Size usedHeapSize = Size.ZERO;
136  private Size maxHeapSize = Size.ZERO;
137  private int infoServerPort;
138  private List<ReplicationLoadSource> sources = Collections.emptyList();
139  @Nullable
140  private ReplicationLoadSink sink = null;
141  private final Map<byte[], RegionMetrics> regionStatus = new TreeMap<>(Bytes.BYTES_COMPARATOR);
142  private final Map<byte[], UserMetrics> userMetrics = new TreeMap<>(Bytes.BYTES_COMPARATOR);
143  private final Set<String> coprocessorNames = new TreeSet<>();
144  private long reportTimestamp = EnvironmentEdgeManager.currentTime();
145  private long lastReportTimestamp = 0;
146  private final List<ServerTask> tasks = new ArrayList<>();
147  private Map<String, Integer> regionCachedInfo = new HashMap<>();
148
149  private ServerMetricsBuilder(ServerName serverName) {
150    this.serverName = serverName;
151  }
152
153  public ServerMetricsBuilder setVersionNumber(int versionNumber) {
154    this.versionNumber = versionNumber;
155    return this;
156  }
157
158  public ServerMetricsBuilder setVersion(String version) {
159    this.version = version;
160    return this;
161  }
162
163  public ServerMetricsBuilder setRequestCountPerSecond(long value) {
164    this.requestCountPerSecond = value;
165    return this;
166  }
167
168  public ServerMetricsBuilder setRequestCount(long value) {
169    this.requestCount = value;
170    return this;
171  }
172
173  public ServerMetricsBuilder setReadRequestCount(long value) {
174    this.readRequestCount = value;
175    return this;
176  }
177
178  public ServerMetricsBuilder setWriteRequestCount(long value) {
179    this.writeRequestCount = value;
180    return this;
181  }
182
183  public ServerMetricsBuilder setUsedHeapSize(Size value) {
184    this.usedHeapSize = value;
185    return this;
186  }
187
188  public ServerMetricsBuilder setMaxHeapSize(Size value) {
189    this.maxHeapSize = value;
190    return this;
191  }
192
193  public ServerMetricsBuilder setInfoServerPort(int value) {
194    this.infoServerPort = value;
195    return this;
196  }
197
198  public ServerMetricsBuilder setReplicationLoadSources(List<ReplicationLoadSource> value) {
199    this.sources = value;
200    return this;
201  }
202
203  public ServerMetricsBuilder setReplicationLoadSink(ReplicationLoadSink value) {
204    this.sink = value;
205    return this;
206  }
207
208  public ServerMetricsBuilder setRegionMetrics(List<RegionMetrics> value) {
209    value.forEach(v -> this.regionStatus.put(v.getRegionName(), v));
210    return this;
211  }
212
213  public ServerMetricsBuilder setUserMetrics(List<UserMetrics> value) {
214    value.forEach(v -> this.userMetrics.put(v.getUserName(), v));
215    return this;
216  }
217
218  public ServerMetricsBuilder setCoprocessorNames(List<String> value) {
219    coprocessorNames.addAll(value);
220    return this;
221  }
222
223  public ServerMetricsBuilder setReportTimestamp(long value) {
224    this.reportTimestamp = value;
225    return this;
226  }
227
228  public ServerMetricsBuilder setLastReportTimestamp(long value) {
229    this.lastReportTimestamp = value;
230    return this;
231  }
232
233  public ServerMetricsBuilder setTasks(List<ServerTask> tasks) {
234    this.tasks.addAll(tasks);
235    return this;
236  }
237
238  public ServerMetricsBuilder setRegionCachedInfo(Map<String, Integer> value) {
239    this.regionCachedInfo = value;
240    return this;
241  }
242
243  public ServerMetrics build() {
244    return new ServerMetricsImpl(serverName, versionNumber, version, requestCountPerSecond,
245      requestCount, readRequestCount, writeRequestCount, usedHeapSize, maxHeapSize, infoServerPort,
246      sources, sink, regionStatus, coprocessorNames, reportTimestamp, lastReportTimestamp,
247      userMetrics, tasks, regionCachedInfo);
248  }
249
250  private static class ServerMetricsImpl implements ServerMetrics {
251    private final ServerName serverName;
252    private final int versionNumber;
253    private final String version;
254    private final long requestCountPerSecond;
255    private final long requestCount;
256    private final long readRequestsCount;
257    private final long writeRequestsCount;
258    private final Size usedHeapSize;
259    private final Size maxHeapSize;
260    private final int infoServerPort;
261    private final List<ReplicationLoadSource> sources;
262    @Nullable
263    private final ReplicationLoadSink sink;
264    private final Map<byte[], RegionMetrics> regionStatus;
265    private final Set<String> coprocessorNames;
266    private final long reportTimestamp;
267    private final long lastReportTimestamp;
268    private final Map<byte[], UserMetrics> userMetrics;
269    private final List<ServerTask> tasks;
270    private final Map<String, Integer> regionCachedInfo;
271
272    ServerMetricsImpl(ServerName serverName, int versionNumber, String version,
273      long requestCountPerSecond, long requestCount, long readRequestsCount,
274      long writeRequestsCount, Size usedHeapSize, Size maxHeapSize, int infoServerPort,
275      List<ReplicationLoadSource> sources, ReplicationLoadSink sink,
276      Map<byte[], RegionMetrics> regionStatus, Set<String> coprocessorNames, long reportTimestamp,
277      long lastReportTimestamp, Map<byte[], UserMetrics> userMetrics, List<ServerTask> tasks,
278      Map<String, Integer> regionCachedInfo) {
279      this.serverName = Preconditions.checkNotNull(serverName);
280      this.versionNumber = versionNumber;
281      this.version = version;
282      this.requestCountPerSecond = requestCountPerSecond;
283      this.requestCount = requestCount;
284      this.readRequestsCount = readRequestsCount;
285      this.writeRequestsCount = writeRequestsCount;
286      this.usedHeapSize = Preconditions.checkNotNull(usedHeapSize);
287      this.maxHeapSize = Preconditions.checkNotNull(maxHeapSize);
288      this.infoServerPort = infoServerPort;
289      this.sources = Preconditions.checkNotNull(sources);
290      this.sink = sink;
291      this.regionStatus = Preconditions.checkNotNull(regionStatus);
292      this.userMetrics = Preconditions.checkNotNull(userMetrics);
293      this.coprocessorNames = Preconditions.checkNotNull(coprocessorNames);
294      this.reportTimestamp = reportTimestamp;
295      this.lastReportTimestamp = lastReportTimestamp;
296      this.tasks = tasks;
297      this.regionCachedInfo = regionCachedInfo;
298    }
299
300    @Override
301    public ServerName getServerName() {
302      return serverName;
303    }
304
305    @Override
306    public int getVersionNumber() {
307      return versionNumber;
308    }
309
310    @Override
311    public String getVersion() {
312      return version;
313    }
314
315    @Override
316    public long getRequestCountPerSecond() {
317      return requestCountPerSecond;
318    }
319
320    @Override
321    public long getRequestCount() {
322      return requestCount;
323    }
324
325    @Override
326    public long getReadRequestsCount() {
327      return readRequestsCount;
328    }
329
330    @Override
331    public long getWriteRequestsCount() {
332      return writeRequestsCount;
333    }
334
335    @Override
336    public Size getUsedHeapSize() {
337      return usedHeapSize;
338    }
339
340    @Override
341    public Size getMaxHeapSize() {
342      return maxHeapSize;
343    }
344
345    @Override
346    public int getInfoServerPort() {
347      return infoServerPort;
348    }
349
350    @Override
351    public List<ReplicationLoadSource> getReplicationLoadSourceList() {
352      return Collections.unmodifiableList(sources);
353    }
354
355    @Override
356    public Map<String, List<ReplicationLoadSource>> getReplicationLoadSourceMap() {
357      Map<String, List<ReplicationLoadSource>> sourcesMap = new HashMap<>();
358      for (ReplicationLoadSource loadSource : sources) {
359        sourcesMap.computeIfAbsent(loadSource.getPeerID(), peerId -> new ArrayList<>())
360          .add(loadSource);
361      }
362      return sourcesMap;
363    }
364
365    @Override
366    public ReplicationLoadSink getReplicationLoadSink() {
367      return sink;
368    }
369
370    @Override
371    public Map<byte[], RegionMetrics> getRegionMetrics() {
372      return Collections.unmodifiableMap(regionStatus);
373    }
374
375    @Override
376    public Map<byte[], UserMetrics> getUserMetrics() {
377      return Collections.unmodifiableMap(userMetrics);
378    }
379
380    @Override
381    public Set<String> getCoprocessorNames() {
382      return Collections.unmodifiableSet(coprocessorNames);
383    }
384
385    @Override
386    public long getReportTimestamp() {
387      return reportTimestamp;
388    }
389
390    @Override
391    public long getLastReportTimestamp() {
392      return lastReportTimestamp;
393    }
394
395    @Override
396    public List<ServerTask> getTasks() {
397      return tasks;
398    }
399
400    @Override
401    public Map<String, Integer> getRegionCachedInfo() {
402      return Collections.unmodifiableMap(regionCachedInfo);
403    }
404
405    @Override
406    public String toString() {
407      int storeCount = 0;
408      int storeFileCount = 0;
409      int storeRefCount = 0;
410      int maxCompactedStoreFileRefCount = 0;
411      long uncompressedStoreFileSizeMB = 0;
412      long storeFileSizeMB = 0;
413      long memStoreSizeMB = 0;
414      long storefileIndexSizeKB = 0;
415      long rootLevelIndexSizeKB = 0;
416      long readRequestsCount = 0;
417      long cpRequestsCount = 0;
418      long writeRequestsCount = 0;
419      long filteredReadRequestsCount = 0;
420      long bloomFilterSizeMB = 0;
421      long compactingCellCount = 0;
422      long compactedCellCount = 0;
423      for (RegionMetrics r : getRegionMetrics().values()) {
424        storeCount += r.getStoreCount();
425        storeFileCount += r.getStoreFileCount();
426        storeRefCount += r.getStoreRefCount();
427        int currentMaxCompactedStoreFileRefCount = r.getMaxCompactedStoreFileRefCount();
428        maxCompactedStoreFileRefCount =
429          Math.max(maxCompactedStoreFileRefCount, currentMaxCompactedStoreFileRefCount);
430        uncompressedStoreFileSizeMB +=
431          (long) r.getUncompressedStoreFileSize().get(Size.Unit.MEGABYTE);
432        storeFileSizeMB += (long) r.getStoreFileSize().get(Size.Unit.MEGABYTE);
433        memStoreSizeMB += (long) r.getMemStoreSize().get(Size.Unit.MEGABYTE);
434        storefileIndexSizeKB +=
435          (long) r.getStoreFileUncompressedDataIndexSize().get(Size.Unit.KILOBYTE);
436        readRequestsCount += r.getReadRequestCount();
437        cpRequestsCount += r.getCpRequestCount();
438        writeRequestsCount += r.getWriteRequestCount();
439        filteredReadRequestsCount += r.getFilteredReadRequestCount();
440        rootLevelIndexSizeKB += (long) r.getStoreFileRootLevelIndexSize().get(Size.Unit.KILOBYTE);
441        bloomFilterSizeMB += (long) r.getBloomFilterSize().get(Size.Unit.MEGABYTE);
442        compactedCellCount += r.getCompactedCellCount();
443        compactingCellCount += r.getCompactingCellCount();
444      }
445      StringBuilder sb = Strings.appendKeyValue(new StringBuilder(), "requestsPerSecond",
446        Double.valueOf(getRequestCountPerSecond()));
447      Strings.appendKeyValue(sb, "numberOfOnlineRegions",
448        Integer.valueOf(getRegionMetrics().size()));
449      Strings.appendKeyValue(sb, "usedHeapMB", getUsedHeapSize());
450      Strings.appendKeyValue(sb, "maxHeapMB", getMaxHeapSize());
451      Strings.appendKeyValue(sb, "numberOfStores", storeCount);
452      Strings.appendKeyValue(sb, "numberOfStorefiles", storeFileCount);
453      Strings.appendKeyValue(sb, "storeRefCount", storeRefCount);
454      Strings.appendKeyValue(sb, "maxCompactedStoreFileRefCount", maxCompactedStoreFileRefCount);
455      Strings.appendKeyValue(sb, "storefileUncompressedSizeMB", uncompressedStoreFileSizeMB);
456      Strings.appendKeyValue(sb, "storefileSizeMB", storeFileSizeMB);
457      if (uncompressedStoreFileSizeMB != 0) {
458        Strings.appendKeyValue(sb, "compressionRatio",
459          String.format("%.4f", (float) storeFileSizeMB / (float) uncompressedStoreFileSizeMB));
460      }
461      Strings.appendKeyValue(sb, "memstoreSizeMB", memStoreSizeMB);
462      Strings.appendKeyValue(sb, "readRequestsCount", readRequestsCount);
463      Strings.appendKeyValue(sb, "cpRequestsCount", cpRequestsCount);
464      Strings.appendKeyValue(sb, "filteredReadRequestsCount", filteredReadRequestsCount);
465      Strings.appendKeyValue(sb, "writeRequestsCount", writeRequestsCount);
466      Strings.appendKeyValue(sb, "rootIndexSizeKB", rootLevelIndexSizeKB);
467      Strings.appendKeyValue(sb, "totalStaticIndexSizeKB", storefileIndexSizeKB);
468      Strings.appendKeyValue(sb, "totalStaticBloomSizeKB", bloomFilterSizeMB);
469      Strings.appendKeyValue(sb, "totalCompactingKVs", compactingCellCount);
470      Strings.appendKeyValue(sb, "currentCompactedKVs", compactedCellCount);
471      float compactionProgressPct = Float.NaN;
472      if (compactingCellCount > 0) {
473        compactionProgressPct = Float.valueOf((float) compactedCellCount / compactingCellCount);
474      }
475      Strings.appendKeyValue(sb, "compactionProgressPct", compactionProgressPct);
476      Strings.appendKeyValue(sb, "coprocessors", getCoprocessorNames());
477      return sb.toString();
478    }
479  }
480}