001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.http;
019
020import java.io.File;
021import java.io.IOException;
022import java.nio.file.Files;
023import java.nio.file.Path;
024import java.nio.file.Paths;
025import java.util.ArrayList;
026import java.util.List;
027import java.util.concurrent.TimeUnit;
028import java.util.concurrent.atomic.AtomicInteger;
029import java.util.concurrent.locks.Lock;
030import java.util.concurrent.locks.ReentrantLock;
031import javax.servlet.http.HttpServlet;
032import javax.servlet.http.HttpServletRequest;
033import javax.servlet.http.HttpServletResponse;
034import org.apache.hadoop.hbase.util.ProcessUtils;
035import org.apache.yetus.audience.InterfaceAudience;
036import org.slf4j.Logger;
037import org.slf4j.LoggerFactory;
038
039import org.apache.hbase.thirdparty.com.google.common.base.Joiner;
040
041/**
042 * Servlet that runs async-profiler as web-endpoint. Following options from async-profiler can be
043 * specified as query paramater. // -e event profiling event: cpu|alloc|lock|cache-misses etc. // -d
044 * duration run profiling for 'duration' seconds (integer) // -i interval sampling interval in
045 * nanoseconds (long) // -j jstackdepth maximum Java stack depth (integer) // -b bufsize frame
046 * buffer size (long) // -t profile different threads separately // -s simple class names instead of
047 * FQN // -o fmt[,fmt...] output format: summary|traces|flat|collapsed|svg|tree|jfr|html // --width
048 * px SVG width pixels (integer) // --height px SVG frame height pixels (integer) // --minwidth px
049 * skip frames smaller than px (double) // --reverse generate stack-reversed FlameGraph / Call tree
050 * Example: - To collect 30 second CPU profile of current process (returns FlameGraph svg) curl
051 * "http://localhost:10002/prof" - To collect 1 minute CPU profile of current process and output in
052 * tree format (html) curl "http://localhost:10002/prof?output=tree&duration=60" - To collect 30
053 * second heap allocation profile of current process (returns FlameGraph svg) curl
054 * "http://localhost:10002/prof?event=alloc" - To collect lock contention profile of current process
055 * (returns FlameGraph svg) curl "http://localhost:10002/prof?event=lock" Following event types are
056 * supported (default is 'cpu') (NOTE: not all OS'es support all events) // Perf events: // cpu //
057 * page-faults // context-switches // cycles // instructions // cache-references // cache-misses //
058 * branches // branch-misses // bus-cycles // L1-dcache-load-misses // LLC-load-misses //
059 * dTLB-load-misses // mem:breakpoint // trace:tracepoint // Java events: // alloc // lock
060 */
061@InterfaceAudience.Private
062public class ProfileServlet extends HttpServlet {
063
064  private static final long serialVersionUID = 1L;
065  private static final Logger LOG = LoggerFactory.getLogger(ProfileServlet.class);
066
067  private static final String ACCESS_CONTROL_ALLOW_METHODS = "Access-Control-Allow-Methods";
068  private static final String ALLOWED_METHODS = "GET";
069  private static final String ACCESS_CONTROL_ALLOW_ORIGIN = "Access-Control-Allow-Origin";
070  private static final String CONTENT_TYPE_TEXT = "text/plain; charset=utf-8";
071  private static final String ASYNC_PROFILER_HOME_ENV = "ASYNC_PROFILER_HOME";
072  private static final String ASYNC_PROFILER_HOME_SYSTEM_PROPERTY = "async.profiler.home";
073  private static final String OLD_PROFILER_SCRIPT = "profiler.sh";
074  private static final String PROFILER_SCRIPT = "asprof";
075  private static final int DEFAULT_DURATION_SECONDS = 10;
076  private static final AtomicInteger ID_GEN = new AtomicInteger(0);
077  static final String OUTPUT_DIR = System.getProperty("java.io.tmpdir") + "/prof-output-hbase";
078
079  enum Event {
080    CPU("cpu"),
081    WALL("wall"),
082    ALLOC("alloc"),
083    LOCK("lock"),
084    PAGE_FAULTS("page-faults"),
085    CONTEXT_SWITCHES("context-switches"),
086    CYCLES("cycles"),
087    INSTRUCTIONS("instructions"),
088    CACHE_REFERENCES("cache-references"),
089    CACHE_MISSES("cache-misses"),
090    BRANCHES("branches"),
091    BRANCH_MISSES("branch-misses"),
092    BUS_CYCLES("bus-cycles"),
093    L1_DCACHE_LOAD_MISSES("L1-dcache-load-misses"),
094    LLC_LOAD_MISSES("LLC-load-misses"),
095    DTLB_LOAD_MISSES("dTLB-load-misses"),
096    MEM_BREAKPOINT("mem:breakpoint"),
097    TRACE_TRACEPOINT("trace:tracepoint"),;
098
099    private final String internalName;
100
101    Event(final String internalName) {
102      this.internalName = internalName;
103    }
104
105    public String getInternalName() {
106      return internalName;
107    }
108
109    public static Event fromInternalName(final String name) {
110      for (Event event : values()) {
111        if (event.getInternalName().equalsIgnoreCase(name)) {
112          return event;
113        }
114      }
115
116      return null;
117    }
118  }
119
120  enum Output {
121    SUMMARY,
122    TRACES,
123    FLAT,
124    COLLAPSED,
125    // No SVG in 2.x asyncprofiler.
126    SVG,
127    TREE,
128    JFR,
129    // In 2.x asyncprofiler, this is how you get flamegraphs.
130    HTML
131  }
132
133  @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "SE_TRANSIENT_FIELD_NOT_RESTORED",
134      justification = "This class is never serialized nor restored.")
135  private transient Lock profilerLock = new ReentrantLock();
136  private transient volatile Process process;
137  private String asyncProfilerHome;
138  private Integer pid;
139
140  public ProfileServlet() {
141    this.asyncProfilerHome = getAsyncProfilerHome();
142    this.pid = ProcessUtils.getPid();
143    LOG.info("Servlet process PID: " + pid + " asyncProfilerHome: " + asyncProfilerHome);
144  }
145
146  @Override
147  protected void doGet(final HttpServletRequest req, final HttpServletResponse resp)
148    throws IOException {
149    if (!HttpServer.isInstrumentationAccessAllowed(getServletContext(), req, resp)) {
150      resp.setStatus(HttpServletResponse.SC_UNAUTHORIZED);
151      setResponseHeader(resp);
152      resp.getWriter().write("Unauthorized: Instrumentation access is not allowed!");
153      return;
154    }
155
156    // make sure async profiler home is set
157    if (asyncProfilerHome == null || asyncProfilerHome.trim().isEmpty()) {
158      resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
159      setResponseHeader(resp);
160      resp.getWriter()
161        .write("ASYNC_PROFILER_HOME env is not set.\n\n"
162          + "Please ensure the prerequsites for the Profiler Servlet have been installed and the\n"
163          + "environment is properly configured. For more information please see\n"
164          + "http://hbase.apache.org/book.html#profiler\n");
165      return;
166    }
167
168    // if pid is explicitly specified, use it else default to current process
169    pid = getInteger(req, "pid", pid);
170
171    // if pid is not specified in query param and if current process pid cannot be determined
172    if (pid == null) {
173      resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
174      setResponseHeader(resp);
175      resp.getWriter()
176        .write("'pid' query parameter unspecified or unable to determine PID of current process.");
177      return;
178    }
179
180    final int duration = getInteger(req, "duration", DEFAULT_DURATION_SECONDS);
181    final Output output = getOutput(req);
182    final Event event = getEvent(req);
183    final Long interval = getLong(req, "interval");
184    final Integer jstackDepth = getInteger(req, "jstackdepth", null);
185    final Long bufsize = getLong(req, "bufsize");
186    final boolean thread = req.getParameterMap().containsKey("thread");
187    final boolean simple = req.getParameterMap().containsKey("simple");
188    final Integer width = getInteger(req, "width", null);
189    final Integer height = getInteger(req, "height", null);
190    final Double minwidth = getMinWidth(req);
191    final boolean reverse = req.getParameterMap().containsKey("reverse");
192
193    if (process == null || !process.isAlive()) {
194      try {
195        int lockTimeoutSecs = 3;
196        if (profilerLock.tryLock(lockTimeoutSecs, TimeUnit.SECONDS)) {
197          try {
198            File outputFile =
199              new File(OUTPUT_DIR, "async-prof-pid-" + pid + "-" + event.name().toLowerCase() + "-"
200                + ID_GEN.incrementAndGet() + "." + output.name().toLowerCase());
201            Files.createDirectories(Paths.get(OUTPUT_DIR));
202            List<String> cmd = new ArrayList<>();
203            Path profilerScriptPath = Paths.get(asyncProfilerHome, "bin", PROFILER_SCRIPT);
204            if (!Files.exists(profilerScriptPath)) {
205              LOG.info(
206                "async-profiler script {} does not exist, fallback to use old script {}(version <= 2.9).",
207                PROFILER_SCRIPT, OLD_PROFILER_SCRIPT);
208              profilerScriptPath = Paths.get(asyncProfilerHome, OLD_PROFILER_SCRIPT);
209            }
210            cmd.add(profilerScriptPath.toString());
211            cmd.add("-e");
212            cmd.add(event.getInternalName());
213            cmd.add("-d");
214            cmd.add("" + duration);
215            cmd.add("-o");
216            cmd.add(output.name().toLowerCase());
217            cmd.add("-f");
218            cmd.add(outputFile.getAbsolutePath());
219            if (interval != null) {
220              cmd.add("-i");
221              cmd.add(interval.toString());
222            }
223            if (jstackDepth != null) {
224              cmd.add("-j");
225              cmd.add(jstackDepth.toString());
226            }
227            if (bufsize != null) {
228              cmd.add("-b");
229              cmd.add(bufsize.toString());
230            }
231            if (thread) {
232              cmd.add("-t");
233            }
234            if (simple) {
235              cmd.add("-s");
236            }
237            if (width != null) {
238              cmd.add("--width");
239              cmd.add(width.toString());
240            }
241            if (height != null) {
242              cmd.add("--height");
243              cmd.add(height.toString());
244            }
245            if (minwidth != null) {
246              cmd.add("--minwidth");
247              cmd.add(minwidth.toString());
248            }
249            if (reverse) {
250              cmd.add("--reverse");
251            }
252            cmd.add(pid.toString());
253            process = ProcessUtils.runCmdAsync(cmd);
254
255            // set response and set refresh header to output location
256            setResponseHeader(resp);
257            resp.setStatus(HttpServletResponse.SC_ACCEPTED);
258            String relativeUrl = "/prof-output-hbase/" + outputFile.getName();
259            resp.getWriter()
260              .write("Started [" + event.getInternalName()
261                + "] profiling. This page will automatically redirect to " + relativeUrl + " after "
262                + duration + " seconds. "
263                + "If empty diagram and Linux 4.6+, see 'Basic Usage' section on the Async "
264                + "Profiler Home Page, https://github.com/jvm-profiling-tools/async-profiler."
265                + "\n\nCommand:\n" + Joiner.on(" ").join(cmd));
266
267            // to avoid auto-refresh by ProfileOutputServlet, refreshDelay can be specified
268            // via url param
269            int refreshDelay = getInteger(req, "refreshDelay", 0);
270
271            // instead of sending redirect, set auto-refresh so that browsers will refresh
272            // with redirected url
273            resp.setHeader("Refresh", (duration + refreshDelay) + ";" + relativeUrl);
274            resp.getWriter().flush();
275          } finally {
276            profilerLock.unlock();
277          }
278        } else {
279          setResponseHeader(resp);
280          resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
281          resp.getWriter()
282            .write("Unable to acquire lock. Another instance of profiler might be running.");
283          LOG.warn("Unable to acquire lock in " + lockTimeoutSecs
284            + " seconds. Another instance of profiler might be running.");
285        }
286      } catch (InterruptedException e) {
287        LOG.warn("Interrupted while acquiring profile lock.", e);
288        resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
289      }
290    } else {
291      setResponseHeader(resp);
292      resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
293      resp.getWriter().write("Another instance of profiler is already running.");
294    }
295  }
296
297  private Integer getInteger(final HttpServletRequest req, final String param,
298    final Integer defaultValue) {
299    final String value = req.getParameter(param);
300    if (value != null) {
301      try {
302        return Integer.valueOf(value);
303      } catch (NumberFormatException e) {
304        return defaultValue;
305      }
306    }
307    return defaultValue;
308  }
309
310  private Long getLong(final HttpServletRequest req, final String param) {
311    final String value = req.getParameter(param);
312    if (value != null) {
313      try {
314        return Long.valueOf(value);
315      } catch (NumberFormatException e) {
316        return null;
317      }
318    }
319    return null;
320  }
321
322  private Double getMinWidth(final HttpServletRequest req) {
323    final String value = req.getParameter("minwidth");
324    if (value != null) {
325      try {
326        return Double.valueOf(value);
327      } catch (NumberFormatException e) {
328        return null;
329      }
330    }
331    return null;
332  }
333
334  private Event getEvent(final HttpServletRequest req) {
335    final String eventArg = req.getParameter("event");
336    if (eventArg != null) {
337      Event event = Event.fromInternalName(eventArg);
338      return event == null ? Event.CPU : event;
339    }
340    return Event.CPU;
341  }
342
343  private Output getOutput(final HttpServletRequest req) {
344    final String outputArg = req.getParameter("output");
345    if (req.getParameter("output") != null) {
346      try {
347        return Output.valueOf(outputArg.trim().toUpperCase());
348      } catch (IllegalArgumentException e) {
349        return Output.HTML;
350      }
351    }
352    return Output.HTML;
353  }
354
355  static void setResponseHeader(final HttpServletResponse response) {
356    response.setHeader(ACCESS_CONTROL_ALLOW_METHODS, ALLOWED_METHODS);
357    response.setHeader(ACCESS_CONTROL_ALLOW_ORIGIN, "*");
358    response.setContentType(CONTENT_TYPE_TEXT);
359  }
360
361  static String getAsyncProfilerHome() {
362    String asyncProfilerHome = System.getenv(ASYNC_PROFILER_HOME_ENV);
363    // if ENV is not set, see if -Dasync.profiler.home=/path/to/async/profiler/home is set
364    if (asyncProfilerHome == null || asyncProfilerHome.trim().isEmpty()) {
365      asyncProfilerHome = System.getProperty(ASYNC_PROFILER_HOME_SYSTEM_PROPERTY);
366    }
367
368    return asyncProfilerHome;
369  }
370
371  public static class DisabledServlet extends HttpServlet {
372
373    private static final long serialVersionUID = 1L;
374
375    @Override
376    protected void doGet(final HttpServletRequest req, final HttpServletResponse resp)
377      throws IOException {
378      resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
379      setResponseHeader(resp);
380      resp.getWriter()
381        .write("The profiler servlet was disabled at startup.\n\n"
382          + "Please ensure the prerequisites for the Profiler Servlet have been installed and the\n"
383          + "environment is properly configured. For more information please see\n"
384          + "http://hbase.apache.org/book.html#profiler\n");
385      return;
386    }
387
388  }
389
390}