001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.http; 019 020import java.io.File; 021import java.io.IOException; 022import java.util.ArrayList; 023import java.util.List; 024import java.util.concurrent.TimeUnit; 025import java.util.concurrent.atomic.AtomicInteger; 026import java.util.concurrent.locks.Lock; 027import java.util.concurrent.locks.ReentrantLock; 028import javax.servlet.http.HttpServlet; 029import javax.servlet.http.HttpServletRequest; 030import javax.servlet.http.HttpServletResponse; 031import org.apache.hadoop.hbase.util.ProcessUtils; 032import org.apache.yetus.audience.InterfaceAudience; 033import org.slf4j.Logger; 034import org.slf4j.LoggerFactory; 035 036import org.apache.hbase.thirdparty.com.google.common.base.Joiner; 037 038/** 039 * Servlet that runs async-profiler as web-endpoint. Following options from async-profiler can be 040 * specified as query paramater. // -e event profiling event: cpu|alloc|lock|cache-misses etc. // -d 041 * duration run profiling for 'duration' seconds (integer) // -i interval sampling interval in 042 * nanoseconds (long) // -j jstackdepth maximum Java stack depth (integer) // -b bufsize frame 043 * buffer size (long) // -t profile different threads separately // -s simple class names instead of 044 * FQN // -o fmt[,fmt...] output format: summary|traces|flat|collapsed|svg|tree|jfr|html // --width 045 * px SVG width pixels (integer) // --height px SVG frame height pixels (integer) // --minwidth px 046 * skip frames smaller than px (double) // --reverse generate stack-reversed FlameGraph / Call tree 047 * Example: - To collect 30 second CPU profile of current process (returns FlameGraph svg) curl 048 * "http://localhost:10002/prof" - To collect 1 minute CPU profile of current process and output in 049 * tree format (html) curl "http://localhost:10002/prof?output=tree&duration=60" - To collect 30 050 * second heap allocation profile of current process (returns FlameGraph svg) curl 051 * "http://localhost:10002/prof?event=alloc" - To collect lock contention profile of current process 052 * (returns FlameGraph svg) curl "http://localhost:10002/prof?event=lock" Following event types are 053 * supported (default is 'cpu') (NOTE: not all OS'es support all events) // Perf events: // cpu // 054 * page-faults // context-switches // cycles // instructions // cache-references // cache-misses // 055 * branches // branch-misses // bus-cycles // L1-dcache-load-misses // LLC-load-misses // 056 * dTLB-load-misses // mem:breakpoint // trace:tracepoint // Java events: // alloc // lock 057 */ 058@InterfaceAudience.Private 059public class ProfileServlet extends HttpServlet { 060 061 private static final long serialVersionUID = 1L; 062 private static final Logger LOG = LoggerFactory.getLogger(ProfileServlet.class); 063 064 private static final String ACCESS_CONTROL_ALLOW_METHODS = "Access-Control-Allow-Methods"; 065 private static final String ALLOWED_METHODS = "GET"; 066 private static final String ACCESS_CONTROL_ALLOW_ORIGIN = "Access-Control-Allow-Origin"; 067 private static final String CONTENT_TYPE_TEXT = "text/plain; charset=utf-8"; 068 private static final String ASYNC_PROFILER_HOME_ENV = "ASYNC_PROFILER_HOME"; 069 private static final String ASYNC_PROFILER_HOME_SYSTEM_PROPERTY = "async.profiler.home"; 070 private static final String PROFILER_SCRIPT = "/profiler.sh"; 071 private static final int DEFAULT_DURATION_SECONDS = 10; 072 private static final AtomicInteger ID_GEN = new AtomicInteger(0); 073 static final String OUTPUT_DIR = System.getProperty("java.io.tmpdir") + "/prof-output-hbase"; 074 075 enum Event { 076 CPU("cpu"), 077 ALLOC("alloc"), 078 LOCK("lock"), 079 PAGE_FAULTS("page-faults"), 080 CONTEXT_SWITCHES("context-switches"), 081 CYCLES("cycles"), 082 INSTRUCTIONS("instructions"), 083 CACHE_REFERENCES("cache-references"), 084 CACHE_MISSES("cache-misses"), 085 BRANCHES("branches"), 086 BRANCH_MISSES("branch-misses"), 087 BUS_CYCLES("bus-cycles"), 088 L1_DCACHE_LOAD_MISSES("L1-dcache-load-misses"), 089 LLC_LOAD_MISSES("LLC-load-misses"), 090 DTLB_LOAD_MISSES("dTLB-load-misses"), 091 MEM_BREAKPOINT("mem:breakpoint"), 092 TRACE_TRACEPOINT("trace:tracepoint"),; 093 094 private final String internalName; 095 096 Event(final String internalName) { 097 this.internalName = internalName; 098 } 099 100 public String getInternalName() { 101 return internalName; 102 } 103 104 public static Event fromInternalName(final String name) { 105 for (Event event : values()) { 106 if (event.getInternalName().equalsIgnoreCase(name)) { 107 return event; 108 } 109 } 110 111 return null; 112 } 113 } 114 115 enum Output { 116 SUMMARY, 117 TRACES, 118 FLAT, 119 COLLAPSED, 120 // No SVG in 2.x asyncprofiler. 121 SVG, 122 TREE, 123 JFR, 124 // In 2.x asyncprofiler, this is how you get flamegraphs. 125 HTML 126 } 127 128 @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "SE_TRANSIENT_FIELD_NOT_RESTORED", 129 justification = "This class is never serialized nor restored.") 130 private transient Lock profilerLock = new ReentrantLock(); 131 private transient volatile Process process; 132 private String asyncProfilerHome; 133 private Integer pid; 134 135 public ProfileServlet() { 136 this.asyncProfilerHome = getAsyncProfilerHome(); 137 this.pid = ProcessUtils.getPid(); 138 LOG.info("Servlet process PID: " + pid + " asyncProfilerHome: " + asyncProfilerHome); 139 } 140 141 @Override 142 protected void doGet(final HttpServletRequest req, final HttpServletResponse resp) 143 throws IOException { 144 if (!HttpServer.isInstrumentationAccessAllowed(getServletContext(), req, resp)) { 145 resp.setStatus(HttpServletResponse.SC_UNAUTHORIZED); 146 setResponseHeader(resp); 147 resp.getWriter().write("Unauthorized: Instrumentation access is not allowed!"); 148 return; 149 } 150 151 // make sure async profiler home is set 152 if (asyncProfilerHome == null || asyncProfilerHome.trim().isEmpty()) { 153 resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); 154 setResponseHeader(resp); 155 resp.getWriter() 156 .write("ASYNC_PROFILER_HOME env is not set.\n\n" 157 + "Please ensure the prerequsites for the Profiler Servlet have been installed and the\n" 158 + "environment is properly configured. For more information please see\n" 159 + "http://hbase.apache.org/book.html#profiler\n"); 160 return; 161 } 162 163 // if pid is explicitly specified, use it else default to current process 164 pid = getInteger(req, "pid", pid); 165 166 // if pid is not specified in query param and if current process pid cannot be determined 167 if (pid == null) { 168 resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); 169 setResponseHeader(resp); 170 resp.getWriter() 171 .write("'pid' query parameter unspecified or unable to determine PID of current process."); 172 return; 173 } 174 175 final int duration = getInteger(req, "duration", DEFAULT_DURATION_SECONDS); 176 final Output output = getOutput(req); 177 final Event event = getEvent(req); 178 final Long interval = getLong(req, "interval"); 179 final Integer jstackDepth = getInteger(req, "jstackdepth", null); 180 final Long bufsize = getLong(req, "bufsize"); 181 final boolean thread = req.getParameterMap().containsKey("thread"); 182 final boolean simple = req.getParameterMap().containsKey("simple"); 183 final Integer width = getInteger(req, "width", null); 184 final Integer height = getInteger(req, "height", null); 185 final Double minwidth = getMinWidth(req); 186 final boolean reverse = req.getParameterMap().containsKey("reverse"); 187 188 if (process == null || !process.isAlive()) { 189 try { 190 int lockTimeoutSecs = 3; 191 if (profilerLock.tryLock(lockTimeoutSecs, TimeUnit.SECONDS)) { 192 try { 193 File outputFile = 194 new File(OUTPUT_DIR, "async-prof-pid-" + pid + "-" + event.name().toLowerCase() + "-" 195 + ID_GEN.incrementAndGet() + "." + output.name().toLowerCase()); 196 List<String> cmd = new ArrayList<>(); 197 cmd.add(asyncProfilerHome + PROFILER_SCRIPT); 198 cmd.add("-e"); 199 cmd.add(event.getInternalName()); 200 cmd.add("-d"); 201 cmd.add("" + duration); 202 cmd.add("-o"); 203 cmd.add(output.name().toLowerCase()); 204 cmd.add("-f"); 205 cmd.add(outputFile.getAbsolutePath()); 206 if (interval != null) { 207 cmd.add("-i"); 208 cmd.add(interval.toString()); 209 } 210 if (jstackDepth != null) { 211 cmd.add("-j"); 212 cmd.add(jstackDepth.toString()); 213 } 214 if (bufsize != null) { 215 cmd.add("-b"); 216 cmd.add(bufsize.toString()); 217 } 218 if (thread) { 219 cmd.add("-t"); 220 } 221 if (simple) { 222 cmd.add("-s"); 223 } 224 if (width != null) { 225 cmd.add("--width"); 226 cmd.add(width.toString()); 227 } 228 if (height != null) { 229 cmd.add("--height"); 230 cmd.add(height.toString()); 231 } 232 if (minwidth != null) { 233 cmd.add("--minwidth"); 234 cmd.add(minwidth.toString()); 235 } 236 if (reverse) { 237 cmd.add("--reverse"); 238 } 239 cmd.add(pid.toString()); 240 process = ProcessUtils.runCmdAsync(cmd); 241 242 // set response and set refresh header to output location 243 setResponseHeader(resp); 244 resp.setStatus(HttpServletResponse.SC_ACCEPTED); 245 String relativeUrl = "/prof-output-hbase/" + outputFile.getName(); 246 resp.getWriter() 247 .write("Started [" + event.getInternalName() 248 + "] profiling. This page will automatically redirect to " + relativeUrl + " after " 249 + duration + " seconds. " 250 + "If empty diagram and Linux 4.6+, see 'Basic Usage' section on the Async " 251 + "Profiler Home Page, https://github.com/jvm-profiling-tools/async-profiler." 252 + "\n\nCommand:\n" + Joiner.on(" ").join(cmd)); 253 254 // to avoid auto-refresh by ProfileOutputServlet, refreshDelay can be specified 255 // via url param 256 int refreshDelay = getInteger(req, "refreshDelay", 0); 257 258 // instead of sending redirect, set auto-refresh so that browsers will refresh 259 // with redirected url 260 resp.setHeader("Refresh", (duration + refreshDelay) + ";" + relativeUrl); 261 resp.getWriter().flush(); 262 } finally { 263 profilerLock.unlock(); 264 } 265 } else { 266 setResponseHeader(resp); 267 resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); 268 resp.getWriter() 269 .write("Unable to acquire lock. Another instance of profiler might be running."); 270 LOG.warn("Unable to acquire lock in " + lockTimeoutSecs 271 + " seconds. Another instance of profiler might be running."); 272 } 273 } catch (InterruptedException e) { 274 LOG.warn("Interrupted while acquiring profile lock.", e); 275 resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); 276 } 277 } else { 278 setResponseHeader(resp); 279 resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); 280 resp.getWriter().write("Another instance of profiler is already running."); 281 } 282 } 283 284 private Integer getInteger(final HttpServletRequest req, final String param, 285 final Integer defaultValue) { 286 final String value = req.getParameter(param); 287 if (value != null) { 288 try { 289 return Integer.valueOf(value); 290 } catch (NumberFormatException e) { 291 return defaultValue; 292 } 293 } 294 return defaultValue; 295 } 296 297 private Long getLong(final HttpServletRequest req, final String param) { 298 final String value = req.getParameter(param); 299 if (value != null) { 300 try { 301 return Long.valueOf(value); 302 } catch (NumberFormatException e) { 303 return null; 304 } 305 } 306 return null; 307 } 308 309 private Double getMinWidth(final HttpServletRequest req) { 310 final String value = req.getParameter("minwidth"); 311 if (value != null) { 312 try { 313 return Double.valueOf(value); 314 } catch (NumberFormatException e) { 315 return null; 316 } 317 } 318 return null; 319 } 320 321 private Event getEvent(final HttpServletRequest req) { 322 final String eventArg = req.getParameter("event"); 323 if (eventArg != null) { 324 Event event = Event.fromInternalName(eventArg); 325 return event == null ? Event.CPU : event; 326 } 327 return Event.CPU; 328 } 329 330 private Output getOutput(final HttpServletRequest req) { 331 final String outputArg = req.getParameter("output"); 332 if (req.getParameter("output") != null) { 333 try { 334 return Output.valueOf(outputArg.trim().toUpperCase()); 335 } catch (IllegalArgumentException e) { 336 return Output.HTML; 337 } 338 } 339 return Output.HTML; 340 } 341 342 static void setResponseHeader(final HttpServletResponse response) { 343 response.setHeader(ACCESS_CONTROL_ALLOW_METHODS, ALLOWED_METHODS); 344 response.setHeader(ACCESS_CONTROL_ALLOW_ORIGIN, "*"); 345 response.setContentType(CONTENT_TYPE_TEXT); 346 } 347 348 static String getAsyncProfilerHome() { 349 String asyncProfilerHome = System.getenv(ASYNC_PROFILER_HOME_ENV); 350 // if ENV is not set, see if -Dasync.profiler.home=/path/to/async/profiler/home is set 351 if (asyncProfilerHome == null || asyncProfilerHome.trim().isEmpty()) { 352 asyncProfilerHome = System.getProperty(ASYNC_PROFILER_HOME_SYSTEM_PROPERTY); 353 } 354 355 return asyncProfilerHome; 356 } 357 358 public static class DisabledServlet extends HttpServlet { 359 360 private static final long serialVersionUID = 1L; 361 362 @Override 363 protected void doGet(final HttpServletRequest req, final HttpServletResponse resp) 364 throws IOException { 365 resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); 366 setResponseHeader(resp); 367 resp.getWriter() 368 .write("The profiler servlet was disabled at startup.\n\n" 369 + "Please ensure the prerequisites for the Profiler Servlet have been installed and the\n" 370 + "environment is properly configured. For more information please see\n" 371 + "http://hbase.apache.org/book.html#profiler\n"); 372 return; 373 } 374 375 } 376 377}