001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.http;
019
020import java.io.File;
021import java.io.IOException;
022import java.util.ArrayList;
023import java.util.List;
024import java.util.concurrent.TimeUnit;
025import java.util.concurrent.atomic.AtomicInteger;
026import java.util.concurrent.locks.Lock;
027import java.util.concurrent.locks.ReentrantLock;
028
029import javax.servlet.http.HttpServlet;
030import javax.servlet.http.HttpServletRequest;
031import javax.servlet.http.HttpServletResponse;
032
033import org.apache.hadoop.hbase.util.ProcessUtils;
034import org.apache.yetus.audience.InterfaceAudience;
035
036import org.slf4j.Logger;
037import org.slf4j.LoggerFactory;
038
039import org.apache.hbase.thirdparty.com.google.common.base.Joiner;
040
041/**
042 * Servlet that runs async-profiler as web-endpoint.
043 * Following options from async-profiler can be specified as query paramater.
044 * //  -e event          profiling event: cpu|alloc|lock|cache-misses etc.
045 * //  -d duration       run profiling for 'duration' seconds (integer)
046 * //  -i interval       sampling interval in nanoseconds (long)
047 * //  -j jstackdepth    maximum Java stack depth (integer)
048 * //  -b bufsize        frame buffer size (long)
049 * //  -t                profile different threads separately
050 * //  -s                simple class names instead of FQN
051 * //  -o fmt[,fmt...]   output format: summary|traces|flat|collapsed|svg|tree|jfr
052 * //  --width px        SVG width pixels (integer)
053 * //  --height px       SVG frame height pixels (integer)
054 * //  --minwidth px     skip frames smaller than px (double)
055 * //  --reverse         generate stack-reversed FlameGraph / Call tree
056 * Example:
057 * - To collect 30 second CPU profile of current process (returns FlameGraph svg)
058 * curl "http://localhost:10002/prof"
059 * - To collect 1 minute CPU profile of current process and output in tree format (html)
060 * curl "http://localhost:10002/prof?output=tree&duration=60"
061 * - To collect 30 second heap allocation profile of current process (returns FlameGraph svg)
062 * curl "http://localhost:10002/prof?event=alloc"
063 * - To collect lock contention profile of current process (returns FlameGraph svg)
064 * curl "http://localhost:10002/prof?event=lock"
065 * Following event types are supported (default is 'cpu') (NOTE: not all OS'es support all events)
066 * // Perf events:
067 * //    cpu
068 * //    page-faults
069 * //    context-switches
070 * //    cycles
071 * //    instructions
072 * //    cache-references
073 * //    cache-misses
074 * //    branches
075 * //    branch-misses
076 * //    bus-cycles
077 * //    L1-dcache-load-misses
078 * //    LLC-load-misses
079 * //    dTLB-load-misses
080 * //    mem:breakpoint
081 * //    trace:tracepoint
082 * // Java events:
083 * //    alloc
084 * //    lock
085 */
086@InterfaceAudience.Private
087public class ProfileServlet extends HttpServlet {
088
089  private static final long serialVersionUID = 1L;
090  private static final Logger LOG = LoggerFactory.getLogger(ProfileServlet.class);
091
092  private static final String ACCESS_CONTROL_ALLOW_METHODS = "Access-Control-Allow-Methods";
093  private static final String ALLOWED_METHODS = "GET";
094  private static final String ACCESS_CONTROL_ALLOW_ORIGIN = "Access-Control-Allow-Origin";
095  private static final String CONTENT_TYPE_TEXT = "text/plain; charset=utf-8";
096  private static final String ASYNC_PROFILER_HOME_ENV = "ASYNC_PROFILER_HOME";
097  private static final String ASYNC_PROFILER_HOME_SYSTEM_PROPERTY = "async.profiler.home";
098  private static final String PROFILER_SCRIPT = "/profiler.sh";
099  private static final int DEFAULT_DURATION_SECONDS = 10;
100  private static final AtomicInteger ID_GEN = new AtomicInteger(0);
101  static final String OUTPUT_DIR = System.getProperty("java.io.tmpdir") + "/prof-output";
102
103  enum Event {
104    CPU("cpu"),
105    ALLOC("alloc"),
106    LOCK("lock"),
107    PAGE_FAULTS("page-faults"),
108    CONTEXT_SWITCHES("context-switches"),
109    CYCLES("cycles"),
110    INSTRUCTIONS("instructions"),
111    CACHE_REFERENCES("cache-references"),
112    CACHE_MISSES("cache-misses"),
113    BRANCHES("branches"),
114    BRANCH_MISSES("branch-misses"),
115    BUS_CYCLES("bus-cycles"),
116    L1_DCACHE_LOAD_MISSES("L1-dcache-load-misses"),
117    LLC_LOAD_MISSES("LLC-load-misses"),
118    DTLB_LOAD_MISSES("dTLB-load-misses"),
119    MEM_BREAKPOINT("mem:breakpoint"),
120    TRACE_TRACEPOINT("trace:tracepoint"),;
121
122    private final String internalName;
123
124    Event(final String internalName) {
125      this.internalName = internalName;
126    }
127
128    public String getInternalName() {
129      return internalName;
130    }
131
132    public static Event fromInternalName(final String name) {
133      for (Event event : values()) {
134        if (event.getInternalName().equalsIgnoreCase(name)) {
135          return event;
136        }
137      }
138
139      return null;
140    }
141  }
142
143  enum Output {
144    SUMMARY,
145    TRACES,
146    FLAT,
147    COLLAPSED,
148    SVG,
149    TREE,
150    JFR
151  }
152
153  @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "SE_TRANSIENT_FIELD_NOT_RESTORED",
154    justification = "This class is never serialized nor restored.")
155  private transient Lock profilerLock = new ReentrantLock();
156  private transient volatile Process process;
157  private String asyncProfilerHome;
158  private Integer pid;
159
160  public ProfileServlet() {
161    this.asyncProfilerHome = getAsyncProfilerHome();
162    this.pid = ProcessUtils.getPid();
163    LOG.info("Servlet process PID: " + pid + " asyncProfilerHome: " + asyncProfilerHome);
164  }
165
166  @Override
167  protected void doGet(final HttpServletRequest req, final HttpServletResponse resp)
168      throws IOException {
169    if (!HttpServer.isInstrumentationAccessAllowed(getServletContext(), req, resp)) {
170      resp.setStatus(HttpServletResponse.SC_UNAUTHORIZED);
171      setResponseHeader(resp);
172      resp.getWriter().write("Unauthorized: Instrumentation access is not allowed!");
173      return;
174    }
175
176    // make sure async profiler home is set
177    if (asyncProfilerHome == null || asyncProfilerHome.trim().isEmpty()) {
178      resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
179      setResponseHeader(resp);
180      resp.getWriter().write("ASYNC_PROFILER_HOME env is not set.\n\n" +
181        "Please ensure the prerequsites for the Profiler Servlet have been installed and the\n" +
182        "environment is properly configured. For more information please see\n" +
183        "http://hbase.apache.org/book.html#profiler\n");
184      return;
185    }
186
187    // if pid is explicitly specified, use it else default to current process
188    pid = getInteger(req, "pid", pid);
189
190    // if pid is not specified in query param and if current process pid cannot be determined
191    if (pid == null) {
192      resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
193      setResponseHeader(resp);
194      resp.getWriter().write(
195        "'pid' query parameter unspecified or unable to determine PID of current process.");
196      return;
197    }
198
199    final int duration = getInteger(req, "duration", DEFAULT_DURATION_SECONDS);
200    final Output output = getOutput(req);
201    final Event event = getEvent(req);
202    final Long interval = getLong(req, "interval");
203    final Integer jstackDepth = getInteger(req, "jstackdepth", null);
204    final Long bufsize = getLong(req, "bufsize");
205    final boolean thread = req.getParameterMap().containsKey("thread");
206    final boolean simple = req.getParameterMap().containsKey("simple");
207    final Integer width = getInteger(req, "width", null);
208    final Integer height = getInteger(req, "height", null);
209    final Double minwidth = getMinWidth(req);
210    final boolean reverse = req.getParameterMap().containsKey("reverse");
211
212    if (process == null || !process.isAlive()) {
213      try {
214        int lockTimeoutSecs = 3;
215        if (profilerLock.tryLock(lockTimeoutSecs, TimeUnit.SECONDS)) {
216          try {
217            File outputFile = new File(OUTPUT_DIR, "async-prof-pid-" + pid + "-" +
218              event.name().toLowerCase() + "-" + ID_GEN.incrementAndGet() + "." +
219              output.name().toLowerCase());
220            List<String> cmd = new ArrayList<>();
221            cmd.add(asyncProfilerHome + PROFILER_SCRIPT);
222            cmd.add("-e");
223            cmd.add(event.getInternalName());
224            cmd.add("-d");
225            cmd.add("" + duration);
226            cmd.add("-o");
227            cmd.add(output.name().toLowerCase());
228            cmd.add("-f");
229            cmd.add(outputFile.getAbsolutePath());
230            if (interval != null) {
231              cmd.add("-i");
232              cmd.add(interval.toString());
233            }
234            if (jstackDepth != null) {
235              cmd.add("-j");
236              cmd.add(jstackDepth.toString());
237            }
238            if (bufsize != null) {
239              cmd.add("-b");
240              cmd.add(bufsize.toString());
241            }
242            if (thread) {
243              cmd.add("-t");
244            }
245            if (simple) {
246              cmd.add("-s");
247            }
248            if (width != null) {
249              cmd.add("--width");
250              cmd.add(width.toString());
251            }
252            if (height != null) {
253              cmd.add("--height");
254              cmd.add(height.toString());
255            }
256            if (minwidth != null) {
257              cmd.add("--minwidth");
258              cmd.add(minwidth.toString());
259            }
260            if (reverse) {
261              cmd.add("--reverse");
262            }
263            cmd.add(pid.toString());
264            process = ProcessUtils.runCmdAsync(cmd);
265
266            // set response and set refresh header to output location
267            setResponseHeader(resp);
268            resp.setStatus(HttpServletResponse.SC_ACCEPTED);
269            String relativeUrl = "/prof-output/" + outputFile.getName();
270            resp.getWriter().write(
271              "Started [" + event.getInternalName() +
272              "] profiling. This page will automatically redirect to " +
273              relativeUrl + " after " + duration + " seconds. " +
274              "If empty diagram and Linux 4.6+, see 'Basic Usage' section on the Async " +
275              "Profiler Home Page, https://github.com/jvm-profiling-tools/async-profiler." +
276              "\n\nCommand:\n" +
277              Joiner.on(" ").join(cmd));
278
279            // to avoid auto-refresh by ProfileOutputServlet, refreshDelay can be specified
280            // via url param
281            int refreshDelay = getInteger(req, "refreshDelay", 0);
282
283            // instead of sending redirect, set auto-refresh so that browsers will refresh
284            // with redirected url
285            resp.setHeader("Refresh", (duration + refreshDelay) + ";" + relativeUrl);
286            resp.getWriter().flush();
287          } finally {
288            profilerLock.unlock();
289          }
290        } else {
291          setResponseHeader(resp);
292          resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
293          resp.getWriter().write(
294            "Unable to acquire lock. Another instance of profiler might be running.");
295          LOG.warn("Unable to acquire lock in " + lockTimeoutSecs +
296            " seconds. Another instance of profiler might be running.");
297        }
298      } catch (InterruptedException e) {
299        LOG.warn("Interrupted while acquiring profile lock.", e);
300        resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
301      }
302    } else {
303      setResponseHeader(resp);
304      resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
305      resp.getWriter().write("Another instance of profiler is already running.");
306    }
307  }
308
309  private Integer getInteger(final HttpServletRequest req, final String param,
310      final Integer defaultValue) {
311    final String value = req.getParameter(param);
312    if (value != null) {
313      try {
314        return Integer.valueOf(value);
315      } catch (NumberFormatException e) {
316        return defaultValue;
317      }
318    }
319    return defaultValue;
320  }
321
322  private Long getLong(final HttpServletRequest req, final String param) {
323    final String value = req.getParameter(param);
324    if (value != null) {
325      try {
326        return Long.valueOf(value);
327      } catch (NumberFormatException e) {
328        return null;
329      }
330    }
331    return null;
332  }
333
334  private Double getMinWidth(final HttpServletRequest req) {
335    final String value = req.getParameter("minwidth");
336    if (value != null) {
337      try {
338        return Double.valueOf(value);
339      } catch (NumberFormatException e) {
340        return null;
341      }
342    }
343    return null;
344  }
345
346  private Event getEvent(final HttpServletRequest req) {
347    final String eventArg = req.getParameter("event");
348    if (eventArg != null) {
349      Event event = Event.fromInternalName(eventArg);
350      return event == null ? Event.CPU : event;
351    }
352    return Event.CPU;
353  }
354
355  private Output getOutput(final HttpServletRequest req) {
356    final String outputArg = req.getParameter("output");
357    if (req.getParameter("output") != null) {
358      try {
359        return Output.valueOf(outputArg.trim().toUpperCase());
360      } catch (IllegalArgumentException e) {
361        return Output.SVG;
362      }
363    }
364    return Output.SVG;
365  }
366
367  static void setResponseHeader(final HttpServletResponse response) {
368    response.setHeader(ACCESS_CONTROL_ALLOW_METHODS, ALLOWED_METHODS);
369    response.setHeader(ACCESS_CONTROL_ALLOW_ORIGIN, "*");
370    response.setContentType(CONTENT_TYPE_TEXT);
371  }
372
373  static String getAsyncProfilerHome() {
374    String asyncProfilerHome = System.getenv(ASYNC_PROFILER_HOME_ENV);
375    // if ENV is not set, see if -Dasync.profiler.home=/path/to/async/profiler/home is set
376    if (asyncProfilerHome == null || asyncProfilerHome.trim().isEmpty()) {
377      asyncProfilerHome = System.getProperty(ASYNC_PROFILER_HOME_SYSTEM_PROPERTY);
378    }
379
380    return asyncProfilerHome;
381  }
382
383  public static class DisabledServlet extends HttpServlet {
384
385    private static final long serialVersionUID = 1L;
386
387    @Override
388    protected void doGet(final HttpServletRequest req, final HttpServletResponse resp)
389        throws IOException {
390      resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
391      setResponseHeader(resp);
392      resp.getWriter().write("The profiler servlet was disabled at startup.\n\n" +
393        "Please ensure the prerequsites for the Profiler Servlet have been installed and the\n" +
394        "environment is properly configured. For more information please see\n" +
395        "http://hbase.apache.org/book.html#profiler\n");
396      return;
397    }
398
399  }
400
401}