1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.http;
19
20 import com.google.common.base.Joiner;
21
22 import java.io.File;
23 import java.io.IOException;
24 import java.lang.reflect.Method;
25 import java.util.ArrayList;
26 import java.util.List;
27 import java.util.concurrent.TimeUnit;
28 import java.util.concurrent.atomic.AtomicInteger;
29 import java.util.concurrent.locks.Lock;
30 import java.util.concurrent.locks.ReentrantLock;
31
32 import javax.servlet.http.HttpServlet;
33 import javax.servlet.http.HttpServletRequest;
34 import javax.servlet.http.HttpServletResponse;
35
36 import org.apache.commons.logging.Log;
37 import org.apache.commons.logging.LogFactory;
38 import org.apache.hadoop.hbase.util.ProcessUtils;
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85 public class ProfileServlet extends HttpServlet {
86 private static final long serialVersionUID = 1L;
87 private static final Log LOG = LogFactory.getLog(ProfileServlet.class);
88
89 private static final String ACCESS_CONTROL_ALLOW_METHODS = "Access-Control-Allow-Methods";
90 private static final String ALLOWED_METHODS = "GET";
91 private static final String ACCESS_CONTROL_ALLOW_ORIGIN = "Access-Control-Allow-Origin";
92 private static final String CONTENT_TYPE_TEXT = "text/plain; charset=utf-8";
93 private static final String ASYNC_PROFILER_HOME_ENV = "ASYNC_PROFILER_HOME";
94 private static final String ASYNC_PROFILER_HOME_SYSTEM_PROPERTY = "async.profiler.home";
95 private static final String PROFILER_SCRIPT = "/profiler.sh";
96 private static final int DEFAULT_DURATION_SECONDS = 10;
97 private static final AtomicInteger ID_GEN = new AtomicInteger(0);
98 static final String OUTPUT_DIR = System.getProperty("java.io.tmpdir") + "/prof-output";
99
100 enum Event {
101 CPU("cpu"),
102 ALLOC("alloc"),
103 LOCK("lock"),
104 PAGE_FAULTS("page-faults"),
105 CONTEXT_SWITCHES("context-switches"),
106 CYCLES("cycles"),
107 INSTRUCTIONS("instructions"),
108 CACHE_REFERENCES("cache-references"),
109 CACHE_MISSES("cache-misses"),
110 BRANCHES("branches"),
111 BRANCH_MISSES("branch-misses"),
112 BUS_CYCLES("bus-cycles"),
113 L1_DCACHE_LOAD_MISSES("L1-dcache-load-misses"),
114 LLC_LOAD_MISSES("LLC-load-misses"),
115 DTLB_LOAD_MISSES("dTLB-load-misses"),
116 MEM_BREAKPOINT("mem:breakpoint"),
117 TRACE_TRACEPOINT("trace:tracepoint"),;
118
119 private final String internalName;
120
121 Event(final String internalName) {
122 this.internalName = internalName;
123 }
124
125 public String getInternalName() {
126 return internalName;
127 }
128
129 public static Event fromInternalName(final String name) {
130 for (Event event : values()) {
131 if (event.getInternalName().equalsIgnoreCase(name)) {
132 return event;
133 }
134 }
135
136 return null;
137 }
138 }
139
140 enum Output {
141 SUMMARY,
142 TRACES,
143 FLAT,
144 COLLAPSED,
145 SVG,
146 TREE,
147 JFR
148 }
149
150 @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "SE_TRANSIENT_FIELD_NOT_RESTORED",
151 justification = "This class is never serialized nor restored.")
152 private transient Lock profilerLock = new ReentrantLock();
153 private transient volatile Process process;
154 private String asyncProfilerHome;
155 private Integer pid;
156
157 public ProfileServlet() {
158 this.asyncProfilerHome = getAsyncProfilerHome();
159 this.pid = ProcessUtils.getPid();
160 LOG.info("Servlet process PID: " + pid + " asyncProfilerHome: " + asyncProfilerHome);
161 }
162
163 @Override
164 protected void doGet(final HttpServletRequest req, final HttpServletResponse resp)
165 throws IOException {
166 if (!HttpServer.isInstrumentationAccessAllowed(getServletContext(), req, resp)) {
167 resp.setStatus(HttpServletResponse.SC_UNAUTHORIZED);
168 setResponseHeader(resp);
169 resp.getWriter().write("Unauthorized: Instrumentation access is not allowed!");
170 return;
171 }
172
173
174 if (asyncProfilerHome == null || asyncProfilerHome.trim().isEmpty()) {
175 resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
176 setResponseHeader(resp);
177 resp.getWriter().write("ASYNC_PROFILER_HOME env is not set.");
178 return;
179 }
180
181
182 pid = getInteger(req, "pid", pid);
183
184
185 if (pid == null) {
186 resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
187 setResponseHeader(resp);
188 resp.getWriter().write(
189 "'pid' query parameter unspecified or unable to determine PID of current process.");
190 return;
191 }
192
193 final int duration = getInteger(req, "duration", DEFAULT_DURATION_SECONDS);
194 final Output output = getOutput(req);
195 final Event event = getEvent(req);
196 final Long interval = getLong(req, "interval");
197 final Integer jstackDepth = getInteger(req, "jstackdepth", null);
198 final Long bufsize = getLong(req, "bufsize");
199 final boolean thread = req.getParameterMap().containsKey("thread");
200 final boolean simple = req.getParameterMap().containsKey("simple");
201 final Integer width = getInteger(req, "width", null);
202 final Integer height = getInteger(req, "height", null);
203 final Double minwidth = getMinWidth(req);
204 final boolean reverse = req.getParameterMap().containsKey("reverse");
205
206 if (process == null || !isAlive(process)) {
207 try {
208 int lockTimeoutSecs = 3;
209 if (profilerLock.tryLock(lockTimeoutSecs, TimeUnit.SECONDS)) {
210 try {
211 File outputFile = new File(OUTPUT_DIR, "async-prof-pid-" + pid + "-" +
212 event.name().toLowerCase() + "-" + ID_GEN.incrementAndGet() + "." +
213 output.name().toLowerCase());
214 List<String> cmd = new ArrayList<>();
215 cmd.add(asyncProfilerHome + PROFILER_SCRIPT);
216 cmd.add("-e");
217 cmd.add(event.getInternalName());
218 cmd.add("-d");
219 cmd.add("" + duration);
220 cmd.add("-o");
221 cmd.add(output.name().toLowerCase());
222 cmd.add("-f");
223 cmd.add(outputFile.getAbsolutePath());
224 if (interval != null) {
225 cmd.add("-i");
226 cmd.add(interval.toString());
227 }
228 if (jstackDepth != null) {
229 cmd.add("-j");
230 cmd.add(jstackDepth.toString());
231 }
232 if (bufsize != null) {
233 cmd.add("-b");
234 cmd.add(bufsize.toString());
235 }
236 if (thread) {
237 cmd.add("-t");
238 }
239 if (simple) {
240 cmd.add("-s");
241 }
242 if (width != null) {
243 cmd.add("--width");
244 cmd.add(width.toString());
245 }
246 if (height != null) {
247 cmd.add("--height");
248 cmd.add(height.toString());
249 }
250 if (minwidth != null) {
251 cmd.add("--minwidth");
252 cmd.add(minwidth.toString());
253 }
254 if (reverse) {
255 cmd.add("--reverse");
256 }
257 cmd.add(pid.toString());
258 process = ProcessUtils.runCmdAsync(cmd);
259
260
261 setResponseHeader(resp);
262 resp.setStatus(HttpServletResponse.SC_ACCEPTED);
263 String relativeUrl = "/prof-output/" + outputFile.getName();
264 resp.getWriter().write(
265 "Started [" + event.getInternalName() +
266 "] profiling. This page will automatically redirect to " +
267 relativeUrl + " after " + duration + " seconds.\n\ncommand:\n" +
268 Joiner.on(" ").join(cmd));
269
270
271
272 int refreshDelay = getInteger(req, "refreshDelay", 0);
273
274
275
276 resp.setHeader("Refresh", (duration + refreshDelay) + ";" + relativeUrl);
277 resp.getWriter().flush();
278 } finally {
279 profilerLock.unlock();
280 }
281 } else {
282 setResponseHeader(resp);
283 resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
284 resp.getWriter().write(
285 "Unable to acquire lock. Another instance of profiler might be running.");
286 LOG.warn("Unable to acquire lock in " + lockTimeoutSecs +
287 " seconds. Another instance of profiler might be running.");
288 }
289 } catch (InterruptedException e) {
290 LOG.warn("Interrupted while acquiring profile lock.", e);
291 resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
292 }
293 } else {
294 setResponseHeader(resp);
295 resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
296 resp.getWriter().write("Another instance of profiler is already running.");
297 }
298 }
299
300
301 private static Method isAliveMethod;
302 static {
303 try {
304 isAliveMethod = Process.class.getDeclaredMethod("isAlive");
305 } catch (Exception e) {
306 isAliveMethod = null;
307 }
308 }
309
310 private static boolean isAlive(Process process) {
311
312 if (isAliveMethod != null) {
313 try {
314 return (boolean) isAliveMethod.invoke(process);
315 } catch (Exception e) {
316 if (LOG.isTraceEnabled()) {
317 LOG.trace("Failed to invoke Process#isAlive on " + process, e);
318 }
319
320 }
321 }
322
323
324 try {
325 int exitValue = process.exitValue();
326 if (LOG.isTraceEnabled()) {
327 LOG.trace("Process " + process + " is dead with exitValue " + exitValue);
328 }
329 } catch (IllegalThreadStateException e) {
330
331 return true;
332 }
333
334 return false;
335 }
336
337 private Integer getInteger(final HttpServletRequest req, final String param,
338 final Integer defaultValue) {
339 final String value = req.getParameter(param);
340 if (value != null) {
341 try {
342 return Integer.valueOf(value);
343 } catch (NumberFormatException e) {
344 return defaultValue;
345 }
346 }
347 return defaultValue;
348 }
349
350 private Long getLong(final HttpServletRequest req, final String param) {
351 final String value = req.getParameter(param);
352 if (value != null) {
353 try {
354 return Long.valueOf(value);
355 } catch (NumberFormatException e) {
356 return null;
357 }
358 }
359 return null;
360 }
361
362 private Double getMinWidth(final HttpServletRequest req) {
363 final String value = req.getParameter("minwidth");
364 if (value != null) {
365 try {
366 return Double.valueOf(value);
367 } catch (NumberFormatException e) {
368 return null;
369 }
370 }
371 return null;
372 }
373
374 private Event getEvent(final HttpServletRequest req) {
375 final String eventArg = req.getParameter("event");
376 if (eventArg != null) {
377 Event event = Event.fromInternalName(eventArg);
378 return event == null ? Event.CPU : event;
379 }
380 return Event.CPU;
381 }
382
383 private Output getOutput(final HttpServletRequest req) {
384 final String outputArg = req.getParameter("output");
385 if (req.getParameter("output") != null) {
386 try {
387 return Output.valueOf(outputArg.trim().toUpperCase());
388 } catch (IllegalArgumentException e) {
389 return Output.SVG;
390 }
391 }
392 return Output.SVG;
393 }
394
395 private void setResponseHeader(final HttpServletResponse response) {
396 response.setHeader(ACCESS_CONTROL_ALLOW_METHODS, ALLOWED_METHODS);
397 response.setHeader(ACCESS_CONTROL_ALLOW_ORIGIN, "*");
398 response.setContentType(CONTENT_TYPE_TEXT);
399 }
400
401 static String getAsyncProfilerHome() {
402 String asyncProfilerHome = System.getenv(ASYNC_PROFILER_HOME_ENV);
403
404 if (asyncProfilerHome == null || asyncProfilerHome.trim().isEmpty()) {
405 asyncProfilerHome = System.getProperty(ASYNC_PROFILER_HOME_SYSTEM_PROPERTY);
406 }
407
408 return asyncProfilerHome;
409 }
410 }