001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase; 019 020import edu.umd.cs.findbugs.annotations.Nullable; 021import java.util.ArrayList; 022import java.util.Collection; 023import java.util.Collections; 024import java.util.HashMap; 025import java.util.List; 026import java.util.Map; 027import java.util.Set; 028import java.util.TreeMap; 029import java.util.TreeSet; 030import java.util.stream.Collectors; 031import org.apache.hadoop.hbase.replication.ReplicationLoadSink; 032import org.apache.hadoop.hbase.replication.ReplicationLoadSource; 033import org.apache.hadoop.hbase.util.Bytes; 034import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; 035import org.apache.hadoop.hbase.util.Strings; 036import org.apache.yetus.audience.InterfaceAudience; 037 038import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; 039 040import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; 041import org.apache.hadoop.hbase.shaded.protobuf.generated.ClusterStatusProtos; 042import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; 043 044@InterfaceAudience.Private 045public final class ServerMetricsBuilder { 046 047 public static ServerMetrics of(ServerName sn) { 048 return newBuilder(sn).build(); 049 } 050 051 public static ServerMetrics of(ServerName sn, int versionNumber, String version) { 052 return newBuilder(sn).setVersionNumber(versionNumber).setVersion(version).build(); 053 } 054 055 public static ServerMetrics toServerMetrics(ClusterStatusProtos.LiveServerInfo serverInfo) { 056 return toServerMetrics(ProtobufUtil.toServerName(serverInfo.getServer()), 0, "0.0.0", 057 serverInfo.getServerLoad()); 058 } 059 060 public static ServerMetrics toServerMetrics(ServerName serverName, 061 ClusterStatusProtos.ServerLoad serverLoadPB) { 062 return toServerMetrics(serverName, 0, "0.0.0", serverLoadPB); 063 } 064 065 public static ServerMetrics toServerMetrics(ServerName serverName, int versionNumber, 066 String version, ClusterStatusProtos.ServerLoad serverLoadPB) { 067 return ServerMetricsBuilder.newBuilder(serverName) 068 .setRequestCountPerSecond(serverLoadPB.getNumberOfRequests()) 069 .setRequestCount(serverLoadPB.getTotalNumberOfRequests()) 070 .setInfoServerPort(serverLoadPB.getInfoServerPort()) 071 .setReadRequestCount(serverLoadPB.getReadRequestsCount()) 072 .setWriteRequestCount(serverLoadPB.getWriteRequestsCount()) 073 .setMaxHeapSize(new Size(serverLoadPB.getMaxHeapMB(), Size.Unit.MEGABYTE)) 074 .setUsedHeapSize(new Size(serverLoadPB.getUsedHeapMB(), Size.Unit.MEGABYTE)) 075 .setCoprocessorNames(serverLoadPB.getCoprocessorsList().stream() 076 .map(HBaseProtos.Coprocessor::getName).collect(Collectors.toList())) 077 .setRegionMetrics(serverLoadPB.getRegionLoadsList().stream() 078 .map(RegionMetricsBuilder::toRegionMetrics).collect(Collectors.toList())) 079 .setUserMetrics(serverLoadPB.getUserLoadsList().stream() 080 .map(UserMetricsBuilder::toUserMetrics).collect(Collectors.toList())) 081 .setReplicationLoadSources(serverLoadPB.getReplLoadSourceList().stream() 082 .map(ProtobufUtil::toReplicationLoadSource).collect(Collectors.toList())) 083 .setReplicationLoadSink(serverLoadPB.hasReplLoadSink() 084 ? ProtobufUtil.toReplicationLoadSink(serverLoadPB.getReplLoadSink()) 085 : null) 086 .setTasks(serverLoadPB.getTasksList().stream().map(ProtobufUtil::getServerTask) 087 .collect(Collectors.toList())) 088 .setRegionCachedInfo(serverLoadPB.getRegionCachedInfoMap()) 089 .setReportTimestamp(serverLoadPB.getReportEndTime()) 090 .setLastReportTimestamp(serverLoadPB.getReportStartTime()).setVersionNumber(versionNumber) 091 .setVersion(version).build(); 092 } 093 094 public static List<HBaseProtos.Coprocessor> toCoprocessor(Collection<String> names) { 095 return names.stream().map(n -> HBaseProtos.Coprocessor.newBuilder().setName(n).build()) 096 .collect(Collectors.toList()); 097 } 098 099 public static ClusterStatusProtos.ServerLoad toServerLoad(ServerMetrics metrics) { 100 ClusterStatusProtos.ServerLoad.Builder builder = ClusterStatusProtos.ServerLoad.newBuilder() 101 .setNumberOfRequests(metrics.getRequestCountPerSecond()) 102 .setTotalNumberOfRequests(metrics.getRequestCount()) 103 .setInfoServerPort(metrics.getInfoServerPort()) 104 .setMaxHeapMB((int) metrics.getMaxHeapSize().get(Size.Unit.MEGABYTE)) 105 .setUsedHeapMB((int) metrics.getUsedHeapSize().get(Size.Unit.MEGABYTE)) 106 .addAllCoprocessors(toCoprocessor(metrics.getCoprocessorNames())) 107 .addAllRegionLoads(metrics.getRegionMetrics().values().stream() 108 .map(RegionMetricsBuilder::toRegionLoad).collect(Collectors.toList())) 109 .addAllUserLoads(metrics.getUserMetrics().values().stream() 110 .map(UserMetricsBuilder::toUserMetrics).collect(Collectors.toList())) 111 .addAllReplLoadSource(metrics.getReplicationLoadSourceList().stream() 112 .map(ProtobufUtil::toReplicationLoadSource).collect(Collectors.toList())) 113 .addAllTasks( 114 metrics.getTasks().stream().map(ProtobufUtil::toServerTask).collect(Collectors.toList())) 115 .putAllRegionCachedInfo(metrics.getRegionCachedInfo()) 116 .setReportStartTime(metrics.getLastReportTimestamp()) 117 .setReportEndTime(metrics.getReportTimestamp()); 118 if (metrics.getReplicationLoadSink() != null) { 119 builder.setReplLoadSink(ProtobufUtil.toReplicationLoadSink(metrics.getReplicationLoadSink())); 120 } 121 return builder.build(); 122 } 123 124 public static ServerMetricsBuilder newBuilder(ServerName sn) { 125 return new ServerMetricsBuilder(sn); 126 } 127 128 private final ServerName serverName; 129 private int versionNumber; 130 private String version = "0.0.0"; 131 private long requestCountPerSecond; 132 private long requestCount; 133 private long readRequestCount; 134 private long writeRequestCount; 135 private Size usedHeapSize = Size.ZERO; 136 private Size maxHeapSize = Size.ZERO; 137 private int infoServerPort; 138 private List<ReplicationLoadSource> sources = Collections.emptyList(); 139 @Nullable 140 private ReplicationLoadSink sink = null; 141 private final Map<byte[], RegionMetrics> regionStatus = new TreeMap<>(Bytes.BYTES_COMPARATOR); 142 private final Map<byte[], UserMetrics> userMetrics = new TreeMap<>(Bytes.BYTES_COMPARATOR); 143 private final Set<String> coprocessorNames = new TreeSet<>(); 144 private long reportTimestamp = EnvironmentEdgeManager.currentTime(); 145 private long lastReportTimestamp = 0; 146 private final List<ServerTask> tasks = new ArrayList<>(); 147 private Map<String, Integer> regionCachedInfo = new HashMap<>(); 148 149 private ServerMetricsBuilder(ServerName serverName) { 150 this.serverName = serverName; 151 } 152 153 public ServerMetricsBuilder setVersionNumber(int versionNumber) { 154 this.versionNumber = versionNumber; 155 return this; 156 } 157 158 public ServerMetricsBuilder setVersion(String version) { 159 this.version = version; 160 return this; 161 } 162 163 public ServerMetricsBuilder setRequestCountPerSecond(long value) { 164 this.requestCountPerSecond = value; 165 return this; 166 } 167 168 public ServerMetricsBuilder setRequestCount(long value) { 169 this.requestCount = value; 170 return this; 171 } 172 173 public ServerMetricsBuilder setReadRequestCount(long value) { 174 this.readRequestCount = value; 175 return this; 176 } 177 178 public ServerMetricsBuilder setWriteRequestCount(long value) { 179 this.writeRequestCount = value; 180 return this; 181 } 182 183 public ServerMetricsBuilder setUsedHeapSize(Size value) { 184 this.usedHeapSize = value; 185 return this; 186 } 187 188 public ServerMetricsBuilder setMaxHeapSize(Size value) { 189 this.maxHeapSize = value; 190 return this; 191 } 192 193 public ServerMetricsBuilder setInfoServerPort(int value) { 194 this.infoServerPort = value; 195 return this; 196 } 197 198 public ServerMetricsBuilder setReplicationLoadSources(List<ReplicationLoadSource> value) { 199 this.sources = value; 200 return this; 201 } 202 203 public ServerMetricsBuilder setReplicationLoadSink(ReplicationLoadSink value) { 204 this.sink = value; 205 return this; 206 } 207 208 public ServerMetricsBuilder setRegionMetrics(List<RegionMetrics> value) { 209 value.forEach(v -> this.regionStatus.put(v.getRegionName(), v)); 210 return this; 211 } 212 213 public ServerMetricsBuilder setUserMetrics(List<UserMetrics> value) { 214 value.forEach(v -> this.userMetrics.put(v.getUserName(), v)); 215 return this; 216 } 217 218 public ServerMetricsBuilder setCoprocessorNames(List<String> value) { 219 coprocessorNames.addAll(value); 220 return this; 221 } 222 223 public ServerMetricsBuilder setReportTimestamp(long value) { 224 this.reportTimestamp = value; 225 return this; 226 } 227 228 public ServerMetricsBuilder setLastReportTimestamp(long value) { 229 this.lastReportTimestamp = value; 230 return this; 231 } 232 233 public ServerMetricsBuilder setTasks(List<ServerTask> tasks) { 234 this.tasks.addAll(tasks); 235 return this; 236 } 237 238 public ServerMetricsBuilder setRegionCachedInfo(Map<String, Integer> value) { 239 this.regionCachedInfo = value; 240 return this; 241 } 242 243 public ServerMetrics build() { 244 return new ServerMetricsImpl(serverName, versionNumber, version, requestCountPerSecond, 245 requestCount, readRequestCount, writeRequestCount, usedHeapSize, maxHeapSize, infoServerPort, 246 sources, sink, regionStatus, coprocessorNames, reportTimestamp, lastReportTimestamp, 247 userMetrics, tasks, regionCachedInfo); 248 } 249 250 private static class ServerMetricsImpl implements ServerMetrics { 251 private final ServerName serverName; 252 private final int versionNumber; 253 private final String version; 254 private final long requestCountPerSecond; 255 private final long requestCount; 256 private final long readRequestsCount; 257 private final long writeRequestsCount; 258 private final Size usedHeapSize; 259 private final Size maxHeapSize; 260 private final int infoServerPort; 261 private final List<ReplicationLoadSource> sources; 262 @Nullable 263 private final ReplicationLoadSink sink; 264 private final Map<byte[], RegionMetrics> regionStatus; 265 private final Set<String> coprocessorNames; 266 private final long reportTimestamp; 267 private final long lastReportTimestamp; 268 private final Map<byte[], UserMetrics> userMetrics; 269 private final List<ServerTask> tasks; 270 private final Map<String, Integer> regionCachedInfo; 271 272 ServerMetricsImpl(ServerName serverName, int versionNumber, String version, 273 long requestCountPerSecond, long requestCount, long readRequestsCount, 274 long writeRequestsCount, Size usedHeapSize, Size maxHeapSize, int infoServerPort, 275 List<ReplicationLoadSource> sources, ReplicationLoadSink sink, 276 Map<byte[], RegionMetrics> regionStatus, Set<String> coprocessorNames, long reportTimestamp, 277 long lastReportTimestamp, Map<byte[], UserMetrics> userMetrics, List<ServerTask> tasks, 278 Map<String, Integer> regionCachedInfo) { 279 this.serverName = Preconditions.checkNotNull(serverName); 280 this.versionNumber = versionNumber; 281 this.version = version; 282 this.requestCountPerSecond = requestCountPerSecond; 283 this.requestCount = requestCount; 284 this.readRequestsCount = readRequestsCount; 285 this.writeRequestsCount = writeRequestsCount; 286 this.usedHeapSize = Preconditions.checkNotNull(usedHeapSize); 287 this.maxHeapSize = Preconditions.checkNotNull(maxHeapSize); 288 this.infoServerPort = infoServerPort; 289 this.sources = Preconditions.checkNotNull(sources); 290 this.sink = sink; 291 this.regionStatus = Preconditions.checkNotNull(regionStatus); 292 this.userMetrics = Preconditions.checkNotNull(userMetrics); 293 this.coprocessorNames = Preconditions.checkNotNull(coprocessorNames); 294 this.reportTimestamp = reportTimestamp; 295 this.lastReportTimestamp = lastReportTimestamp; 296 this.tasks = tasks; 297 this.regionCachedInfo = regionCachedInfo; 298 } 299 300 @Override 301 public ServerName getServerName() { 302 return serverName; 303 } 304 305 @Override 306 public int getVersionNumber() { 307 return versionNumber; 308 } 309 310 @Override 311 public String getVersion() { 312 return version; 313 } 314 315 @Override 316 public long getRequestCountPerSecond() { 317 return requestCountPerSecond; 318 } 319 320 @Override 321 public long getRequestCount() { 322 return requestCount; 323 } 324 325 @Override 326 public long getReadRequestsCount() { 327 return readRequestsCount; 328 } 329 330 @Override 331 public long getWriteRequestsCount() { 332 return writeRequestsCount; 333 } 334 335 @Override 336 public Size getUsedHeapSize() { 337 return usedHeapSize; 338 } 339 340 @Override 341 public Size getMaxHeapSize() { 342 return maxHeapSize; 343 } 344 345 @Override 346 public int getInfoServerPort() { 347 return infoServerPort; 348 } 349 350 @Override 351 public List<ReplicationLoadSource> getReplicationLoadSourceList() { 352 return Collections.unmodifiableList(sources); 353 } 354 355 @Override 356 public Map<String, List<ReplicationLoadSource>> getReplicationLoadSourceMap() { 357 Map<String, List<ReplicationLoadSource>> sourcesMap = new HashMap<>(); 358 for (ReplicationLoadSource loadSource : sources) { 359 sourcesMap.computeIfAbsent(loadSource.getPeerID(), peerId -> new ArrayList<>()) 360 .add(loadSource); 361 } 362 return sourcesMap; 363 } 364 365 @Override 366 public ReplicationLoadSink getReplicationLoadSink() { 367 return sink; 368 } 369 370 @Override 371 public Map<byte[], RegionMetrics> getRegionMetrics() { 372 return Collections.unmodifiableMap(regionStatus); 373 } 374 375 @Override 376 public Map<byte[], UserMetrics> getUserMetrics() { 377 return Collections.unmodifiableMap(userMetrics); 378 } 379 380 @Override 381 public Set<String> getCoprocessorNames() { 382 return Collections.unmodifiableSet(coprocessorNames); 383 } 384 385 @Override 386 public long getReportTimestamp() { 387 return reportTimestamp; 388 } 389 390 @Override 391 public long getLastReportTimestamp() { 392 return lastReportTimestamp; 393 } 394 395 @Override 396 public List<ServerTask> getTasks() { 397 return tasks; 398 } 399 400 @Override 401 public Map<String, Integer> getRegionCachedInfo() { 402 return Collections.unmodifiableMap(regionCachedInfo); 403 } 404 405 @Override 406 public String toString() { 407 int storeCount = 0; 408 int storeFileCount = 0; 409 int storeRefCount = 0; 410 int maxCompactedStoreFileRefCount = 0; 411 long uncompressedStoreFileSizeMB = 0; 412 long storeFileSizeMB = 0; 413 long memStoreSizeMB = 0; 414 long storefileIndexSizeKB = 0; 415 long rootLevelIndexSizeKB = 0; 416 long readRequestsCount = 0; 417 long cpRequestsCount = 0; 418 long writeRequestsCount = 0; 419 long filteredReadRequestsCount = 0; 420 long bloomFilterSizeMB = 0; 421 long compactingCellCount = 0; 422 long compactedCellCount = 0; 423 for (RegionMetrics r : getRegionMetrics().values()) { 424 storeCount += r.getStoreCount(); 425 storeFileCount += r.getStoreFileCount(); 426 storeRefCount += r.getStoreRefCount(); 427 int currentMaxCompactedStoreFileRefCount = r.getMaxCompactedStoreFileRefCount(); 428 maxCompactedStoreFileRefCount = 429 Math.max(maxCompactedStoreFileRefCount, currentMaxCompactedStoreFileRefCount); 430 uncompressedStoreFileSizeMB += 431 (long) r.getUncompressedStoreFileSize().get(Size.Unit.MEGABYTE); 432 storeFileSizeMB += (long) r.getStoreFileSize().get(Size.Unit.MEGABYTE); 433 memStoreSizeMB += (long) r.getMemStoreSize().get(Size.Unit.MEGABYTE); 434 storefileIndexSizeKB += 435 (long) r.getStoreFileUncompressedDataIndexSize().get(Size.Unit.KILOBYTE); 436 readRequestsCount += r.getReadRequestCount(); 437 cpRequestsCount += r.getCpRequestCount(); 438 writeRequestsCount += r.getWriteRequestCount(); 439 filteredReadRequestsCount += r.getFilteredReadRequestCount(); 440 rootLevelIndexSizeKB += (long) r.getStoreFileRootLevelIndexSize().get(Size.Unit.KILOBYTE); 441 bloomFilterSizeMB += (long) r.getBloomFilterSize().get(Size.Unit.MEGABYTE); 442 compactedCellCount += r.getCompactedCellCount(); 443 compactingCellCount += r.getCompactingCellCount(); 444 } 445 StringBuilder sb = Strings.appendKeyValue(new StringBuilder(), "requestsPerSecond", 446 Double.valueOf(getRequestCountPerSecond())); 447 Strings.appendKeyValue(sb, "numberOfOnlineRegions", 448 Integer.valueOf(getRegionMetrics().size())); 449 Strings.appendKeyValue(sb, "usedHeapMB", getUsedHeapSize()); 450 Strings.appendKeyValue(sb, "maxHeapMB", getMaxHeapSize()); 451 Strings.appendKeyValue(sb, "numberOfStores", storeCount); 452 Strings.appendKeyValue(sb, "numberOfStorefiles", storeFileCount); 453 Strings.appendKeyValue(sb, "storeRefCount", storeRefCount); 454 Strings.appendKeyValue(sb, "maxCompactedStoreFileRefCount", maxCompactedStoreFileRefCount); 455 Strings.appendKeyValue(sb, "storefileUncompressedSizeMB", uncompressedStoreFileSizeMB); 456 Strings.appendKeyValue(sb, "storefileSizeMB", storeFileSizeMB); 457 if (uncompressedStoreFileSizeMB != 0) { 458 Strings.appendKeyValue(sb, "compressionRatio", 459 String.format("%.4f", (float) storeFileSizeMB / (float) uncompressedStoreFileSizeMB)); 460 } 461 Strings.appendKeyValue(sb, "memstoreSizeMB", memStoreSizeMB); 462 Strings.appendKeyValue(sb, "readRequestsCount", readRequestsCount); 463 Strings.appendKeyValue(sb, "cpRequestsCount", cpRequestsCount); 464 Strings.appendKeyValue(sb, "filteredReadRequestsCount", filteredReadRequestsCount); 465 Strings.appendKeyValue(sb, "writeRequestsCount", writeRequestsCount); 466 Strings.appendKeyValue(sb, "rootIndexSizeKB", rootLevelIndexSizeKB); 467 Strings.appendKeyValue(sb, "totalStaticIndexSizeKB", storefileIndexSizeKB); 468 Strings.appendKeyValue(sb, "totalStaticBloomSizeKB", bloomFilterSizeMB); 469 Strings.appendKeyValue(sb, "totalCompactingKVs", compactingCellCount); 470 Strings.appendKeyValue(sb, "currentCompactedKVs", compactedCellCount); 471 float compactionProgressPct = Float.NaN; 472 if (compactingCellCount > 0) { 473 compactionProgressPct = Float.valueOf((float) compactedCellCount / compactingCellCount); 474 } 475 Strings.appendKeyValue(sb, "compactionProgressPct", compactionProgressPct); 476 Strings.appendKeyValue(sb, "coprocessors", getCoprocessorNames()); 477 return sb.toString(); 478 } 479 } 480}