001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase;
019
020import edu.umd.cs.findbugs.annotations.Nullable;
021import java.util.ArrayList;
022import java.util.Collection;
023import java.util.Collections;
024import java.util.HashMap;
025import java.util.List;
026import java.util.Map;
027import java.util.Set;
028import java.util.TreeMap;
029import java.util.TreeSet;
030import java.util.stream.Collectors;
031import org.apache.hadoop.hbase.replication.ReplicationLoadSink;
032import org.apache.hadoop.hbase.replication.ReplicationLoadSource;
033import org.apache.hadoop.hbase.util.Bytes;
034import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
035import org.apache.hadoop.hbase.util.Strings;
036import org.apache.yetus.audience.InterfaceAudience;
037
038import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
039
040import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
041import org.apache.hadoop.hbase.shaded.protobuf.generated.ClusterStatusProtos;
042import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos;
043
044@InterfaceAudience.Private
045public final class ServerMetricsBuilder {
046
047  public static ServerMetrics of(ServerName sn) {
048    return newBuilder(sn).build();
049  }
050
051  public static ServerMetrics of(ServerName sn, int versionNumber, String version) {
052    return newBuilder(sn).setVersionNumber(versionNumber).setVersion(version).build();
053  }
054
055  public static ServerMetrics toServerMetrics(ClusterStatusProtos.LiveServerInfo serverInfo) {
056    return toServerMetrics(ProtobufUtil.toServerName(serverInfo.getServer()), 0, "0.0.0",
057      serverInfo.getServerLoad());
058  }
059
060  public static ServerMetrics toServerMetrics(ServerName serverName,
061    ClusterStatusProtos.ServerLoad serverLoadPB) {
062    return toServerMetrics(serverName, 0, "0.0.0", serverLoadPB);
063  }
064
065  public static ServerMetrics toServerMetrics(ServerName serverName, int versionNumber,
066    String version, ClusterStatusProtos.ServerLoad serverLoadPB) {
067    return ServerMetricsBuilder.newBuilder(serverName)
068      .setRequestCountPerSecond(serverLoadPB.getNumberOfRequests())
069      .setRequestCount(serverLoadPB.getTotalNumberOfRequests())
070      .setInfoServerPort(serverLoadPB.getInfoServerPort())
071      .setMaxHeapSize(new Size(serverLoadPB.getMaxHeapMB(), Size.Unit.MEGABYTE))
072      .setUsedHeapSize(new Size(serverLoadPB.getUsedHeapMB(), Size.Unit.MEGABYTE))
073      .setCoprocessorNames(serverLoadPB.getCoprocessorsList().stream()
074        .map(HBaseProtos.Coprocessor::getName).collect(Collectors.toList()))
075      .setRegionMetrics(serverLoadPB.getRegionLoadsList().stream()
076        .map(RegionMetricsBuilder::toRegionMetrics).collect(Collectors.toList()))
077      .setUserMetrics(serverLoadPB.getUserLoadsList().stream()
078        .map(UserMetricsBuilder::toUserMetrics).collect(Collectors.toList()))
079      .setReplicationLoadSources(serverLoadPB.getReplLoadSourceList().stream()
080        .map(ProtobufUtil::toReplicationLoadSource).collect(Collectors.toList()))
081      .setReplicationLoadSink(serverLoadPB.hasReplLoadSink()
082        ? ProtobufUtil.toReplicationLoadSink(serverLoadPB.getReplLoadSink())
083        : null)
084      .setTasks(serverLoadPB.getTasksList().stream().map(ProtobufUtil::getServerTask)
085        .collect(Collectors.toList()))
086      .setReportTimestamp(serverLoadPB.getReportEndTime())
087      .setLastReportTimestamp(serverLoadPB.getReportStartTime()).setVersionNumber(versionNumber)
088      .setVersion(version).build();
089  }
090
091  public static List<HBaseProtos.Coprocessor> toCoprocessor(Collection<String> names) {
092    return names.stream().map(n -> HBaseProtos.Coprocessor.newBuilder().setName(n).build())
093      .collect(Collectors.toList());
094  }
095
096  public static ClusterStatusProtos.ServerLoad toServerLoad(ServerMetrics metrics) {
097    ClusterStatusProtos.ServerLoad.Builder builder = ClusterStatusProtos.ServerLoad.newBuilder()
098      .setNumberOfRequests(metrics.getRequestCountPerSecond())
099      .setTotalNumberOfRequests(metrics.getRequestCount())
100      .setInfoServerPort(metrics.getInfoServerPort())
101      .setMaxHeapMB((int) metrics.getMaxHeapSize().get(Size.Unit.MEGABYTE))
102      .setUsedHeapMB((int) metrics.getUsedHeapSize().get(Size.Unit.MEGABYTE))
103      .addAllCoprocessors(toCoprocessor(metrics.getCoprocessorNames()))
104      .addAllRegionLoads(metrics.getRegionMetrics().values().stream()
105        .map(RegionMetricsBuilder::toRegionLoad).collect(Collectors.toList()))
106      .addAllUserLoads(metrics.getUserMetrics().values().stream()
107        .map(UserMetricsBuilder::toUserMetrics).collect(Collectors.toList()))
108      .addAllReplLoadSource(metrics.getReplicationLoadSourceList().stream()
109        .map(ProtobufUtil::toReplicationLoadSource).collect(Collectors.toList()))
110      .addAllTasks(
111        metrics.getTasks().stream().map(ProtobufUtil::toServerTask).collect(Collectors.toList()))
112      .setReportStartTime(metrics.getLastReportTimestamp())
113      .setReportEndTime(metrics.getReportTimestamp());
114    if (metrics.getReplicationLoadSink() != null) {
115      builder.setReplLoadSink(ProtobufUtil.toReplicationLoadSink(metrics.getReplicationLoadSink()));
116    }
117    return builder.build();
118  }
119
120  public static ServerMetricsBuilder newBuilder(ServerName sn) {
121    return new ServerMetricsBuilder(sn);
122  }
123
124  private final ServerName serverName;
125  private int versionNumber;
126  private String version = "0.0.0";
127  private long requestCountPerSecond;
128  private long requestCount;
129  private Size usedHeapSize = Size.ZERO;
130  private Size maxHeapSize = Size.ZERO;
131  private int infoServerPort;
132  private List<ReplicationLoadSource> sources = Collections.emptyList();
133  @Nullable
134  private ReplicationLoadSink sink = null;
135  private final Map<byte[], RegionMetrics> regionStatus = new TreeMap<>(Bytes.BYTES_COMPARATOR);
136  private final Map<byte[], UserMetrics> userMetrics = new TreeMap<>(Bytes.BYTES_COMPARATOR);
137  private final Set<String> coprocessorNames = new TreeSet<>();
138  private long reportTimestamp = EnvironmentEdgeManager.currentTime();
139  private long lastReportTimestamp = 0;
140  private final List<ServerTask> tasks = new ArrayList<>();
141
142  private ServerMetricsBuilder(ServerName serverName) {
143    this.serverName = serverName;
144  }
145
146  public ServerMetricsBuilder setVersionNumber(int versionNumber) {
147    this.versionNumber = versionNumber;
148    return this;
149  }
150
151  public ServerMetricsBuilder setVersion(String version) {
152    this.version = version;
153    return this;
154  }
155
156  public ServerMetricsBuilder setRequestCountPerSecond(long value) {
157    this.requestCountPerSecond = value;
158    return this;
159  }
160
161  public ServerMetricsBuilder setRequestCount(long value) {
162    this.requestCount = value;
163    return this;
164  }
165
166  public ServerMetricsBuilder setUsedHeapSize(Size value) {
167    this.usedHeapSize = value;
168    return this;
169  }
170
171  public ServerMetricsBuilder setMaxHeapSize(Size value) {
172    this.maxHeapSize = value;
173    return this;
174  }
175
176  public ServerMetricsBuilder setInfoServerPort(int value) {
177    this.infoServerPort = value;
178    return this;
179  }
180
181  public ServerMetricsBuilder setReplicationLoadSources(List<ReplicationLoadSource> value) {
182    this.sources = value;
183    return this;
184  }
185
186  public ServerMetricsBuilder setReplicationLoadSink(ReplicationLoadSink value) {
187    this.sink = value;
188    return this;
189  }
190
191  public ServerMetricsBuilder setRegionMetrics(List<RegionMetrics> value) {
192    value.forEach(v -> this.regionStatus.put(v.getRegionName(), v));
193    return this;
194  }
195
196  public ServerMetricsBuilder setUserMetrics(List<UserMetrics> value) {
197    value.forEach(v -> this.userMetrics.put(v.getUserName(), v));
198    return this;
199  }
200
201  public ServerMetricsBuilder setCoprocessorNames(List<String> value) {
202    coprocessorNames.addAll(value);
203    return this;
204  }
205
206  public ServerMetricsBuilder setReportTimestamp(long value) {
207    this.reportTimestamp = value;
208    return this;
209  }
210
211  public ServerMetricsBuilder setLastReportTimestamp(long value) {
212    this.lastReportTimestamp = value;
213    return this;
214  }
215
216  public ServerMetricsBuilder setTasks(List<ServerTask> tasks) {
217    this.tasks.addAll(tasks);
218    return this;
219  }
220
221  public ServerMetrics build() {
222    return new ServerMetricsImpl(serverName, versionNumber, version, requestCountPerSecond,
223      requestCount, usedHeapSize, maxHeapSize, infoServerPort, sources, sink, regionStatus,
224      coprocessorNames, reportTimestamp, lastReportTimestamp, userMetrics, tasks);
225  }
226
227  private static class ServerMetricsImpl implements ServerMetrics {
228    private final ServerName serverName;
229    private final int versionNumber;
230    private final String version;
231    private final long requestCountPerSecond;
232    private final long requestCount;
233    private final Size usedHeapSize;
234    private final Size maxHeapSize;
235    private final int infoServerPort;
236    private final List<ReplicationLoadSource> sources;
237    @Nullable
238    private final ReplicationLoadSink sink;
239    private final Map<byte[], RegionMetrics> regionStatus;
240    private final Set<String> coprocessorNames;
241    private final long reportTimestamp;
242    private final long lastReportTimestamp;
243    private final Map<byte[], UserMetrics> userMetrics;
244    private final List<ServerTask> tasks;
245
246    ServerMetricsImpl(ServerName serverName, int versionNumber, String version,
247      long requestCountPerSecond, long requestCount, Size usedHeapSize, Size maxHeapSize,
248      int infoServerPort, List<ReplicationLoadSource> sources, ReplicationLoadSink sink,
249      Map<byte[], RegionMetrics> regionStatus, Set<String> coprocessorNames, long reportTimestamp,
250      long lastReportTimestamp, Map<byte[], UserMetrics> userMetrics, List<ServerTask> tasks) {
251      this.serverName = Preconditions.checkNotNull(serverName);
252      this.versionNumber = versionNumber;
253      this.version = version;
254      this.requestCountPerSecond = requestCountPerSecond;
255      this.requestCount = requestCount;
256      this.usedHeapSize = Preconditions.checkNotNull(usedHeapSize);
257      this.maxHeapSize = Preconditions.checkNotNull(maxHeapSize);
258      this.infoServerPort = infoServerPort;
259      this.sources = Preconditions.checkNotNull(sources);
260      this.sink = sink;
261      this.regionStatus = Preconditions.checkNotNull(regionStatus);
262      this.userMetrics = Preconditions.checkNotNull(userMetrics);
263      this.coprocessorNames = Preconditions.checkNotNull(coprocessorNames);
264      this.reportTimestamp = reportTimestamp;
265      this.lastReportTimestamp = lastReportTimestamp;
266      this.tasks = tasks;
267    }
268
269    @Override
270    public ServerName getServerName() {
271      return serverName;
272    }
273
274    @Override
275    public int getVersionNumber() {
276      return versionNumber;
277    }
278
279    @Override
280    public String getVersion() {
281      return version;
282    }
283
284    @Override
285    public long getRequestCountPerSecond() {
286      return requestCountPerSecond;
287    }
288
289    @Override
290    public long getRequestCount() {
291      return requestCount;
292    }
293
294    @Override
295    public Size getUsedHeapSize() {
296      return usedHeapSize;
297    }
298
299    @Override
300    public Size getMaxHeapSize() {
301      return maxHeapSize;
302    }
303
304    @Override
305    public int getInfoServerPort() {
306      return infoServerPort;
307    }
308
309    @Override
310    public List<ReplicationLoadSource> getReplicationLoadSourceList() {
311      return Collections.unmodifiableList(sources);
312    }
313
314    @Override
315    public Map<String, List<ReplicationLoadSource>> getReplicationLoadSourceMap() {
316      Map<String, List<ReplicationLoadSource>> sourcesMap = new HashMap<>();
317      for (ReplicationLoadSource loadSource : sources) {
318        sourcesMap.computeIfAbsent(loadSource.getPeerID(), peerId -> new ArrayList<>())
319          .add(loadSource);
320      }
321      return sourcesMap;
322    }
323
324    @Override
325    public ReplicationLoadSink getReplicationLoadSink() {
326      return sink;
327    }
328
329    @Override
330    public Map<byte[], RegionMetrics> getRegionMetrics() {
331      return Collections.unmodifiableMap(regionStatus);
332    }
333
334    @Override
335    public Map<byte[], UserMetrics> getUserMetrics() {
336      return Collections.unmodifiableMap(userMetrics);
337    }
338
339    @Override
340    public Set<String> getCoprocessorNames() {
341      return Collections.unmodifiableSet(coprocessorNames);
342    }
343
344    @Override
345    public long getReportTimestamp() {
346      return reportTimestamp;
347    }
348
349    @Override
350    public long getLastReportTimestamp() {
351      return lastReportTimestamp;
352    }
353
354    @Override
355    public List<ServerTask> getTasks() {
356      return tasks;
357    }
358
359    @Override
360    public String toString() {
361      int storeCount = 0;
362      int storeFileCount = 0;
363      int storeRefCount = 0;
364      int maxCompactedStoreFileRefCount = 0;
365      long uncompressedStoreFileSizeMB = 0;
366      long storeFileSizeMB = 0;
367      long memStoreSizeMB = 0;
368      long storefileIndexSizeKB = 0;
369      long rootLevelIndexSizeKB = 0;
370      long readRequestsCount = 0;
371      long writeRequestsCount = 0;
372      long filteredReadRequestsCount = 0;
373      long bloomFilterSizeMB = 0;
374      long compactingCellCount = 0;
375      long compactedCellCount = 0;
376      for (RegionMetrics r : getRegionMetrics().values()) {
377        storeCount += r.getStoreCount();
378        storeFileCount += r.getStoreFileCount();
379        storeRefCount += r.getStoreRefCount();
380        int currentMaxCompactedStoreFileRefCount = r.getMaxCompactedStoreFileRefCount();
381        maxCompactedStoreFileRefCount =
382          Math.max(maxCompactedStoreFileRefCount, currentMaxCompactedStoreFileRefCount);
383        uncompressedStoreFileSizeMB +=
384          (long) r.getUncompressedStoreFileSize().get(Size.Unit.MEGABYTE);
385        storeFileSizeMB += (long) r.getStoreFileSize().get(Size.Unit.MEGABYTE);
386        memStoreSizeMB += (long) r.getMemStoreSize().get(Size.Unit.MEGABYTE);
387        storefileIndexSizeKB +=
388          (long) r.getStoreFileUncompressedDataIndexSize().get(Size.Unit.KILOBYTE);
389        readRequestsCount += r.getReadRequestCount();
390        writeRequestsCount += r.getWriteRequestCount();
391        filteredReadRequestsCount += r.getFilteredReadRequestCount();
392        rootLevelIndexSizeKB += (long) r.getStoreFileRootLevelIndexSize().get(Size.Unit.KILOBYTE);
393        bloomFilterSizeMB += (long) r.getBloomFilterSize().get(Size.Unit.MEGABYTE);
394        compactedCellCount += r.getCompactedCellCount();
395        compactingCellCount += r.getCompactingCellCount();
396      }
397      StringBuilder sb = Strings.appendKeyValue(new StringBuilder(), "requestsPerSecond",
398        Double.valueOf(getRequestCountPerSecond()));
399      Strings.appendKeyValue(sb, "numberOfOnlineRegions",
400        Integer.valueOf(getRegionMetrics().size()));
401      Strings.appendKeyValue(sb, "usedHeapMB", getUsedHeapSize());
402      Strings.appendKeyValue(sb, "maxHeapMB", getMaxHeapSize());
403      Strings.appendKeyValue(sb, "numberOfStores", storeCount);
404      Strings.appendKeyValue(sb, "numberOfStorefiles", storeFileCount);
405      Strings.appendKeyValue(sb, "storeRefCount", storeRefCount);
406      Strings.appendKeyValue(sb, "maxCompactedStoreFileRefCount", maxCompactedStoreFileRefCount);
407      Strings.appendKeyValue(sb, "storefileUncompressedSizeMB", uncompressedStoreFileSizeMB);
408      Strings.appendKeyValue(sb, "storefileSizeMB", storeFileSizeMB);
409      if (uncompressedStoreFileSizeMB != 0) {
410        Strings.appendKeyValue(sb, "compressionRatio",
411          String.format("%.4f", (float) storeFileSizeMB / (float) uncompressedStoreFileSizeMB));
412      }
413      Strings.appendKeyValue(sb, "memstoreSizeMB", memStoreSizeMB);
414      Strings.appendKeyValue(sb, "readRequestsCount", readRequestsCount);
415      Strings.appendKeyValue(sb, "filteredReadRequestsCount", filteredReadRequestsCount);
416      Strings.appendKeyValue(sb, "writeRequestsCount", writeRequestsCount);
417      Strings.appendKeyValue(sb, "rootIndexSizeKB", rootLevelIndexSizeKB);
418      Strings.appendKeyValue(sb, "totalStaticIndexSizeKB", storefileIndexSizeKB);
419      Strings.appendKeyValue(sb, "totalStaticBloomSizeKB", bloomFilterSizeMB);
420      Strings.appendKeyValue(sb, "totalCompactingKVs", compactingCellCount);
421      Strings.appendKeyValue(sb, "currentCompactedKVs", compactedCellCount);
422      float compactionProgressPct = Float.NaN;
423      if (compactingCellCount > 0) {
424        compactionProgressPct = Float.valueOf((float) compactedCellCount / compactingCellCount);
425      }
426      Strings.appendKeyValue(sb, "compactionProgressPct", compactionProgressPct);
427      Strings.appendKeyValue(sb, "coprocessors", getCoprocessorNames());
428      return sb.toString();
429    }
430  }
431}