001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase; 019 020import edu.umd.cs.findbugs.annotations.Nullable; 021import java.util.ArrayList; 022import java.util.Collection; 023import java.util.Collections; 024import java.util.HashMap; 025import java.util.List; 026import java.util.Map; 027import java.util.Set; 028import java.util.TreeMap; 029import java.util.TreeSet; 030import java.util.stream.Collectors; 031import org.apache.hadoop.hbase.replication.ReplicationLoadSink; 032import org.apache.hadoop.hbase.replication.ReplicationLoadSource; 033import org.apache.hadoop.hbase.util.Bytes; 034import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; 035import org.apache.hadoop.hbase.util.Strings; 036import org.apache.yetus.audience.InterfaceAudience; 037 038import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; 039 040import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; 041import org.apache.hadoop.hbase.shaded.protobuf.generated.ClusterStatusProtos; 042import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; 043 044@InterfaceAudience.Private 045public final class ServerMetricsBuilder { 046 047 public static ServerMetrics of(ServerName sn) { 048 return newBuilder(sn).build(); 049 } 050 051 public static ServerMetrics of(ServerName sn, int versionNumber, String version) { 052 return newBuilder(sn).setVersionNumber(versionNumber).setVersion(version).build(); 053 } 054 055 public static ServerMetrics toServerMetrics(ClusterStatusProtos.LiveServerInfo serverInfo) { 056 return toServerMetrics(ProtobufUtil.toServerName(serverInfo.getServer()), 0, "0.0.0", 057 serverInfo.getServerLoad()); 058 } 059 060 public static ServerMetrics toServerMetrics(ServerName serverName, 061 ClusterStatusProtos.ServerLoad serverLoadPB) { 062 return toServerMetrics(serverName, 0, "0.0.0", serverLoadPB); 063 } 064 065 public static ServerMetrics toServerMetrics(ServerName serverName, int versionNumber, 066 String version, ClusterStatusProtos.ServerLoad serverLoadPB) { 067 return ServerMetricsBuilder.newBuilder(serverName) 068 .setRequestCountPerSecond(serverLoadPB.getNumberOfRequests()) 069 .setRequestCount(serverLoadPB.getTotalNumberOfRequests()) 070 .setInfoServerPort(serverLoadPB.getInfoServerPort()) 071 .setMaxHeapSize(new Size(serverLoadPB.getMaxHeapMB(), Size.Unit.MEGABYTE)) 072 .setUsedHeapSize(new Size(serverLoadPB.getUsedHeapMB(), Size.Unit.MEGABYTE)) 073 .setCoprocessorNames(serverLoadPB.getCoprocessorsList().stream() 074 .map(HBaseProtos.Coprocessor::getName).collect(Collectors.toList())) 075 .setRegionMetrics(serverLoadPB.getRegionLoadsList().stream() 076 .map(RegionMetricsBuilder::toRegionMetrics).collect(Collectors.toList())) 077 .setUserMetrics(serverLoadPB.getUserLoadsList().stream() 078 .map(UserMetricsBuilder::toUserMetrics).collect(Collectors.toList())) 079 .setReplicationLoadSources(serverLoadPB.getReplLoadSourceList().stream() 080 .map(ProtobufUtil::toReplicationLoadSource).collect(Collectors.toList())) 081 .setReplicationLoadSink(serverLoadPB.hasReplLoadSink() 082 ? ProtobufUtil.toReplicationLoadSink(serverLoadPB.getReplLoadSink()) 083 : null) 084 .setTasks(serverLoadPB.getTasksList().stream().map(ProtobufUtil::getServerTask) 085 .collect(Collectors.toList())) 086 .setReportTimestamp(serverLoadPB.getReportEndTime()) 087 .setLastReportTimestamp(serverLoadPB.getReportStartTime()).setVersionNumber(versionNumber) 088 .setVersion(version).build(); 089 } 090 091 public static List<HBaseProtos.Coprocessor> toCoprocessor(Collection<String> names) { 092 return names.stream().map(n -> HBaseProtos.Coprocessor.newBuilder().setName(n).build()) 093 .collect(Collectors.toList()); 094 } 095 096 public static ClusterStatusProtos.ServerLoad toServerLoad(ServerMetrics metrics) { 097 ClusterStatusProtos.ServerLoad.Builder builder = ClusterStatusProtos.ServerLoad.newBuilder() 098 .setNumberOfRequests(metrics.getRequestCountPerSecond()) 099 .setTotalNumberOfRequests(metrics.getRequestCount()) 100 .setInfoServerPort(metrics.getInfoServerPort()) 101 .setMaxHeapMB((int) metrics.getMaxHeapSize().get(Size.Unit.MEGABYTE)) 102 .setUsedHeapMB((int) metrics.getUsedHeapSize().get(Size.Unit.MEGABYTE)) 103 .addAllCoprocessors(toCoprocessor(metrics.getCoprocessorNames())) 104 .addAllRegionLoads(metrics.getRegionMetrics().values().stream() 105 .map(RegionMetricsBuilder::toRegionLoad).collect(Collectors.toList())) 106 .addAllUserLoads(metrics.getUserMetrics().values().stream() 107 .map(UserMetricsBuilder::toUserMetrics).collect(Collectors.toList())) 108 .addAllReplLoadSource(metrics.getReplicationLoadSourceList().stream() 109 .map(ProtobufUtil::toReplicationLoadSource).collect(Collectors.toList())) 110 .addAllTasks( 111 metrics.getTasks().stream().map(ProtobufUtil::toServerTask).collect(Collectors.toList())) 112 .setReportStartTime(metrics.getLastReportTimestamp()) 113 .setReportEndTime(metrics.getReportTimestamp()); 114 if (metrics.getReplicationLoadSink() != null) { 115 builder.setReplLoadSink(ProtobufUtil.toReplicationLoadSink(metrics.getReplicationLoadSink())); 116 } 117 return builder.build(); 118 } 119 120 public static ServerMetricsBuilder newBuilder(ServerName sn) { 121 return new ServerMetricsBuilder(sn); 122 } 123 124 private final ServerName serverName; 125 private int versionNumber; 126 private String version = "0.0.0"; 127 private long requestCountPerSecond; 128 private long requestCount; 129 private Size usedHeapSize = Size.ZERO; 130 private Size maxHeapSize = Size.ZERO; 131 private int infoServerPort; 132 private List<ReplicationLoadSource> sources = Collections.emptyList(); 133 @Nullable 134 private ReplicationLoadSink sink = null; 135 private final Map<byte[], RegionMetrics> regionStatus = new TreeMap<>(Bytes.BYTES_COMPARATOR); 136 private final Map<byte[], UserMetrics> userMetrics = new TreeMap<>(Bytes.BYTES_COMPARATOR); 137 private final Set<String> coprocessorNames = new TreeSet<>(); 138 private long reportTimestamp = EnvironmentEdgeManager.currentTime(); 139 private long lastReportTimestamp = 0; 140 private final List<ServerTask> tasks = new ArrayList<>(); 141 142 private ServerMetricsBuilder(ServerName serverName) { 143 this.serverName = serverName; 144 } 145 146 public ServerMetricsBuilder setVersionNumber(int versionNumber) { 147 this.versionNumber = versionNumber; 148 return this; 149 } 150 151 public ServerMetricsBuilder setVersion(String version) { 152 this.version = version; 153 return this; 154 } 155 156 public ServerMetricsBuilder setRequestCountPerSecond(long value) { 157 this.requestCountPerSecond = value; 158 return this; 159 } 160 161 public ServerMetricsBuilder setRequestCount(long value) { 162 this.requestCount = value; 163 return this; 164 } 165 166 public ServerMetricsBuilder setUsedHeapSize(Size value) { 167 this.usedHeapSize = value; 168 return this; 169 } 170 171 public ServerMetricsBuilder setMaxHeapSize(Size value) { 172 this.maxHeapSize = value; 173 return this; 174 } 175 176 public ServerMetricsBuilder setInfoServerPort(int value) { 177 this.infoServerPort = value; 178 return this; 179 } 180 181 public ServerMetricsBuilder setReplicationLoadSources(List<ReplicationLoadSource> value) { 182 this.sources = value; 183 return this; 184 } 185 186 public ServerMetricsBuilder setReplicationLoadSink(ReplicationLoadSink value) { 187 this.sink = value; 188 return this; 189 } 190 191 public ServerMetricsBuilder setRegionMetrics(List<RegionMetrics> value) { 192 value.forEach(v -> this.regionStatus.put(v.getRegionName(), v)); 193 return this; 194 } 195 196 public ServerMetricsBuilder setUserMetrics(List<UserMetrics> value) { 197 value.forEach(v -> this.userMetrics.put(v.getUserName(), v)); 198 return this; 199 } 200 201 public ServerMetricsBuilder setCoprocessorNames(List<String> value) { 202 coprocessorNames.addAll(value); 203 return this; 204 } 205 206 public ServerMetricsBuilder setReportTimestamp(long value) { 207 this.reportTimestamp = value; 208 return this; 209 } 210 211 public ServerMetricsBuilder setLastReportTimestamp(long value) { 212 this.lastReportTimestamp = value; 213 return this; 214 } 215 216 public ServerMetricsBuilder setTasks(List<ServerTask> tasks) { 217 this.tasks.addAll(tasks); 218 return this; 219 } 220 221 public ServerMetrics build() { 222 return new ServerMetricsImpl(serverName, versionNumber, version, requestCountPerSecond, 223 requestCount, usedHeapSize, maxHeapSize, infoServerPort, sources, sink, regionStatus, 224 coprocessorNames, reportTimestamp, lastReportTimestamp, userMetrics, tasks); 225 } 226 227 private static class ServerMetricsImpl implements ServerMetrics { 228 private final ServerName serverName; 229 private final int versionNumber; 230 private final String version; 231 private final long requestCountPerSecond; 232 private final long requestCount; 233 private final Size usedHeapSize; 234 private final Size maxHeapSize; 235 private final int infoServerPort; 236 private final List<ReplicationLoadSource> sources; 237 @Nullable 238 private final ReplicationLoadSink sink; 239 private final Map<byte[], RegionMetrics> regionStatus; 240 private final Set<String> coprocessorNames; 241 private final long reportTimestamp; 242 private final long lastReportTimestamp; 243 private final Map<byte[], UserMetrics> userMetrics; 244 private final List<ServerTask> tasks; 245 246 ServerMetricsImpl(ServerName serverName, int versionNumber, String version, 247 long requestCountPerSecond, long requestCount, Size usedHeapSize, Size maxHeapSize, 248 int infoServerPort, List<ReplicationLoadSource> sources, ReplicationLoadSink sink, 249 Map<byte[], RegionMetrics> regionStatus, Set<String> coprocessorNames, long reportTimestamp, 250 long lastReportTimestamp, Map<byte[], UserMetrics> userMetrics, List<ServerTask> tasks) { 251 this.serverName = Preconditions.checkNotNull(serverName); 252 this.versionNumber = versionNumber; 253 this.version = version; 254 this.requestCountPerSecond = requestCountPerSecond; 255 this.requestCount = requestCount; 256 this.usedHeapSize = Preconditions.checkNotNull(usedHeapSize); 257 this.maxHeapSize = Preconditions.checkNotNull(maxHeapSize); 258 this.infoServerPort = infoServerPort; 259 this.sources = Preconditions.checkNotNull(sources); 260 this.sink = sink; 261 this.regionStatus = Preconditions.checkNotNull(regionStatus); 262 this.userMetrics = Preconditions.checkNotNull(userMetrics); 263 this.coprocessorNames = Preconditions.checkNotNull(coprocessorNames); 264 this.reportTimestamp = reportTimestamp; 265 this.lastReportTimestamp = lastReportTimestamp; 266 this.tasks = tasks; 267 } 268 269 @Override 270 public ServerName getServerName() { 271 return serverName; 272 } 273 274 @Override 275 public int getVersionNumber() { 276 return versionNumber; 277 } 278 279 @Override 280 public String getVersion() { 281 return version; 282 } 283 284 @Override 285 public long getRequestCountPerSecond() { 286 return requestCountPerSecond; 287 } 288 289 @Override 290 public long getRequestCount() { 291 return requestCount; 292 } 293 294 @Override 295 public Size getUsedHeapSize() { 296 return usedHeapSize; 297 } 298 299 @Override 300 public Size getMaxHeapSize() { 301 return maxHeapSize; 302 } 303 304 @Override 305 public int getInfoServerPort() { 306 return infoServerPort; 307 } 308 309 @Override 310 public List<ReplicationLoadSource> getReplicationLoadSourceList() { 311 return Collections.unmodifiableList(sources); 312 } 313 314 @Override 315 public Map<String, List<ReplicationLoadSource>> getReplicationLoadSourceMap() { 316 Map<String, List<ReplicationLoadSource>> sourcesMap = new HashMap<>(); 317 for (ReplicationLoadSource loadSource : sources) { 318 sourcesMap.computeIfAbsent(loadSource.getPeerID(), peerId -> new ArrayList<>()) 319 .add(loadSource); 320 } 321 return sourcesMap; 322 } 323 324 @Override 325 public ReplicationLoadSink getReplicationLoadSink() { 326 return sink; 327 } 328 329 @Override 330 public Map<byte[], RegionMetrics> getRegionMetrics() { 331 return Collections.unmodifiableMap(regionStatus); 332 } 333 334 @Override 335 public Map<byte[], UserMetrics> getUserMetrics() { 336 return Collections.unmodifiableMap(userMetrics); 337 } 338 339 @Override 340 public Set<String> getCoprocessorNames() { 341 return Collections.unmodifiableSet(coprocessorNames); 342 } 343 344 @Override 345 public long getReportTimestamp() { 346 return reportTimestamp; 347 } 348 349 @Override 350 public long getLastReportTimestamp() { 351 return lastReportTimestamp; 352 } 353 354 @Override 355 public List<ServerTask> getTasks() { 356 return tasks; 357 } 358 359 @Override 360 public String toString() { 361 int storeCount = 0; 362 int storeFileCount = 0; 363 int storeRefCount = 0; 364 int maxCompactedStoreFileRefCount = 0; 365 long uncompressedStoreFileSizeMB = 0; 366 long storeFileSizeMB = 0; 367 long memStoreSizeMB = 0; 368 long storefileIndexSizeKB = 0; 369 long rootLevelIndexSizeKB = 0; 370 long readRequestsCount = 0; 371 long writeRequestsCount = 0; 372 long filteredReadRequestsCount = 0; 373 long bloomFilterSizeMB = 0; 374 long compactingCellCount = 0; 375 long compactedCellCount = 0; 376 for (RegionMetrics r : getRegionMetrics().values()) { 377 storeCount += r.getStoreCount(); 378 storeFileCount += r.getStoreFileCount(); 379 storeRefCount += r.getStoreRefCount(); 380 int currentMaxCompactedStoreFileRefCount = r.getMaxCompactedStoreFileRefCount(); 381 maxCompactedStoreFileRefCount = 382 Math.max(maxCompactedStoreFileRefCount, currentMaxCompactedStoreFileRefCount); 383 uncompressedStoreFileSizeMB += 384 (long) r.getUncompressedStoreFileSize().get(Size.Unit.MEGABYTE); 385 storeFileSizeMB += (long) r.getStoreFileSize().get(Size.Unit.MEGABYTE); 386 memStoreSizeMB += (long) r.getMemStoreSize().get(Size.Unit.MEGABYTE); 387 storefileIndexSizeKB += 388 (long) r.getStoreFileUncompressedDataIndexSize().get(Size.Unit.KILOBYTE); 389 readRequestsCount += r.getReadRequestCount(); 390 writeRequestsCount += r.getWriteRequestCount(); 391 filteredReadRequestsCount += r.getFilteredReadRequestCount(); 392 rootLevelIndexSizeKB += (long) r.getStoreFileRootLevelIndexSize().get(Size.Unit.KILOBYTE); 393 bloomFilterSizeMB += (long) r.getBloomFilterSize().get(Size.Unit.MEGABYTE); 394 compactedCellCount += r.getCompactedCellCount(); 395 compactingCellCount += r.getCompactingCellCount(); 396 } 397 StringBuilder sb = Strings.appendKeyValue(new StringBuilder(), "requestsPerSecond", 398 Double.valueOf(getRequestCountPerSecond())); 399 Strings.appendKeyValue(sb, "numberOfOnlineRegions", 400 Integer.valueOf(getRegionMetrics().size())); 401 Strings.appendKeyValue(sb, "usedHeapMB", getUsedHeapSize()); 402 Strings.appendKeyValue(sb, "maxHeapMB", getMaxHeapSize()); 403 Strings.appendKeyValue(sb, "numberOfStores", storeCount); 404 Strings.appendKeyValue(sb, "numberOfStorefiles", storeFileCount); 405 Strings.appendKeyValue(sb, "storeRefCount", storeRefCount); 406 Strings.appendKeyValue(sb, "maxCompactedStoreFileRefCount", maxCompactedStoreFileRefCount); 407 Strings.appendKeyValue(sb, "storefileUncompressedSizeMB", uncompressedStoreFileSizeMB); 408 Strings.appendKeyValue(sb, "storefileSizeMB", storeFileSizeMB); 409 if (uncompressedStoreFileSizeMB != 0) { 410 Strings.appendKeyValue(sb, "compressionRatio", 411 String.format("%.4f", (float) storeFileSizeMB / (float) uncompressedStoreFileSizeMB)); 412 } 413 Strings.appendKeyValue(sb, "memstoreSizeMB", memStoreSizeMB); 414 Strings.appendKeyValue(sb, "readRequestsCount", readRequestsCount); 415 Strings.appendKeyValue(sb, "filteredReadRequestsCount", filteredReadRequestsCount); 416 Strings.appendKeyValue(sb, "writeRequestsCount", writeRequestsCount); 417 Strings.appendKeyValue(sb, "rootIndexSizeKB", rootLevelIndexSizeKB); 418 Strings.appendKeyValue(sb, "totalStaticIndexSizeKB", storefileIndexSizeKB); 419 Strings.appendKeyValue(sb, "totalStaticBloomSizeKB", bloomFilterSizeMB); 420 Strings.appendKeyValue(sb, "totalCompactingKVs", compactingCellCount); 421 Strings.appendKeyValue(sb, "currentCompactedKVs", compactedCellCount); 422 float compactionProgressPct = Float.NaN; 423 if (compactingCellCount > 0) { 424 compactionProgressPct = Float.valueOf((float) compactedCellCount / compactingCellCount); 425 } 426 Strings.appendKeyValue(sb, "compactionProgressPct", compactionProgressPct); 427 Strings.appendKeyValue(sb, "coprocessors", getCoprocessorNames()); 428 return sb.toString(); 429 } 430 } 431}