Repository: oozie Updated Branches: refs/heads/master b84a3e45c -> 1add349e0
OOZIE-1388 Add a admin servlet to show thread stack trace and CPU usage per thread (rohini) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/1add349e Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/1add349e Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/1add349e Branch: refs/heads/master Commit: 1add349e0fd74653bf0b8821a88c7171fd656c16 Parents: b84a3e4 Author: Rohini Palaniswamy <[email protected]> Authored: Mon Jun 23 14:54:51 2014 -0700 Committer: Rohini Palaniswamy <[email protected]> Committed: Mon Jun 23 14:54:51 2014 -0700 ---------------------------------------------------------------------- .../java/org/apache/oozie/servlet/JVMInfo.java | 275 +++++++++++++++++++ docs/src/site/twiki/AG_Monitoring.twiki | 7 + release-log.txt | 1 + webapp/src/main/webapp/WEB-INF/web-common.xml | 5 + webapp/src/main/webapp/admin/jvminfo.jsp | 91 ++++++ 5 files changed, 379 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/1add349e/core/src/main/java/org/apache/oozie/servlet/JVMInfo.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/servlet/JVMInfo.java b/core/src/main/java/org/apache/oozie/servlet/JVMInfo.java new file mode 100644 index 0000000..9a8248e --- /dev/null +++ b/core/src/main/java/org/apache/oozie/servlet/JVMInfo.java @@ -0,0 +1,275 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.servlet; + +import java.io.Serializable; +import java.lang.Thread.State; +import java.lang.management.ClassLoadingMXBean; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; +import java.lang.management.MemoryUsage; +import java.lang.management.ThreadInfo; +import java.lang.management.ThreadMXBean; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@SuppressWarnings("serial") +public class JVMInfo implements Serializable { + + private static Map<State, Integer> threadStateOrdering; + private final MemoryMXBean memoryMXBean; + private final ClassLoadingMXBean classLoadingMXBean; + private final ThreadMXBean threadMXBean; + private List<JThreadInfo> jThreadInfos; + private String threadSortOrder; + private Integer cpuMonitorTime = 0; + private int blockedThreads = 0; + private int runnableThreads = 0; + private int waitingThreads = 0; + private int timedWaitingThreads = 0; + private int newThreads = 0; + private int terminatedThreads = 0; + + static { + threadStateOrdering = new HashMap<State, Integer>(); + threadStateOrdering.put(State.RUNNABLE, 1); + threadStateOrdering.put(State.BLOCKED, 2); + threadStateOrdering.put(State.WAITING, 3); + threadStateOrdering.put(State.TIMED_WAITING, 4); + threadStateOrdering.put(State.NEW, 5); + threadStateOrdering.put(State.TERMINATED, 6); + } + + public JVMInfo() { + memoryMXBean = ManagementFactory.getMemoryMXBean(); + classLoadingMXBean = ManagementFactory.getClassLoadingMXBean(); + threadMXBean = ManagementFactory.getThreadMXBean(); + } + + public String getThreadSortOrder() { + return threadSortOrder; + } + + public void setThreadSortOrder(String threadSortOrder) { + this.threadSortOrder = threadSortOrder; + } + + public String getCpuMonitorTime() { + return cpuMonitorTime.toString(); + } + + public void setCpuMonitorTime(String sleepTime) { + if (sleepTime != null) { + this.cpuMonitorTime = Integer.parseInt(sleepTime); + } + } + + public String getHeapMemoryUsage() { + MemoryUsage hmu = memoryMXBean.getHeapMemoryUsage(); + StringBuffer sb = new StringBuffer(60); + sb.append("INIT=").append(hmu.getInit()); + sb.append(" USED=").append(hmu.getUsed()); + sb.append(" COMMITTED=").append(hmu.getCommitted()); + sb.append(" MAX=").append(hmu.getMax()); + return sb.toString(); + } + + public String getNonHeapMemoryUsage() { + MemoryUsage nhmu = memoryMXBean.getNonHeapMemoryUsage(); + StringBuffer sb = new StringBuffer(60); + sb.append("INIT=").append(nhmu.getInit()); + sb.append(" USED=").append(nhmu.getUsed()); + sb.append(" COMMITTED=").append(nhmu.getCommitted()); + sb.append(" MAX=").append(nhmu.getMax()); + return sb.toString(); + } + + public String getClassLoadingInfo() { + StringBuffer sb = new StringBuffer(150); + sb.append("Total Loaded Classes=").append(classLoadingMXBean.getTotalLoadedClassCount()); + sb.append(" Loaded Classes=").append(classLoadingMXBean.getLoadedClassCount()); + sb.append(" Unloaded Classes=").append(classLoadingMXBean.getUnloadedClassCount()); + return sb.toString(); + } + + public String getThreadInfo() throws InterruptedException { + getThreads(); + StringBuffer sb = new StringBuffer(150); + sb.append("Thread Count=").append(threadMXBean.getThreadCount()); + sb.append(" Peak Thread Count=").append(threadMXBean.getPeakThreadCount()); + sb.append(" Total Started Threads=").append(threadMXBean.getTotalStartedThreadCount()); + sb.append(" Deamon Threads=").append(threadMXBean.getDaemonThreadCount()); + sb.append("<br /> RUNNABLE=").append(runnableThreads); + sb.append(" BLOCKED=").append(blockedThreads); + sb.append(" WAITING=").append(waitingThreads); + sb.append(" TIMED_WAITING=").append(timedWaitingThreads); + sb.append(" NEW=").append(newThreads); + sb.append(" TERMINATED=").append(terminatedThreads); + sb.append("<br /> Time of Thread Dump=").append(new Date().toString()); + return sb.toString(); + } + + public Collection<JThreadInfo> getThreads() throws InterruptedException { + if (jThreadInfos == null) { + jThreadInfos = getThreadInfos(); + Collections.sort(jThreadInfos, new ThreadComparator(threadSortOrder)); + } + return jThreadInfos; + } + + private List<JThreadInfo> getThreadInfos() throws InterruptedException { + ThreadInfo[] threads = threadMXBean.dumpAllThreads(false, false); + Map<Long, JThreadInfo> oldThreadInfoMap = new HashMap<Long, JThreadInfo>(); + for (ThreadInfo thread : threads) { + long cpuTime = -1L; + long userTime = -1L; + long threadId = thread.getThreadId(); + if (threadMXBean.isThreadCpuTimeSupported() && threadMXBean.isThreadCpuTimeEnabled()) { + cpuTime = threadMXBean.getThreadCpuTime(threadId); + userTime = threadMXBean.getThreadUserTime(threadId); + } + oldThreadInfoMap.put(thread.getThreadId(), new JThreadInfo(cpuTime, userTime, thread)); + } + Thread.sleep(cpuMonitorTime); + ThreadInfo[] threadInfos = threadMXBean.dumpAllThreads(false, false); + List<JThreadInfo> jThreadInfos = new ArrayList<JThreadInfo>(threadInfos.length); + for (int i = 0; i < threadInfos.length; i++) { + ThreadInfo threadInfo = threadInfos[i]; + long threadId = threadInfo.getThreadId(); + long cpuTime = -1L; + long userTime = -1L; + if (threadMXBean.isThreadCpuTimeSupported() && threadMXBean.isThreadCpuTimeEnabled()) { + JThreadInfo oldThread = oldThreadInfoMap.get(threadInfo.getThreadId()); + if (oldThread == null) { + cpuTime = threadMXBean.getThreadCpuTime(threadId); + userTime = threadMXBean.getThreadUserTime(threadId); + } + else { + cpuTime = threadMXBean.getThreadCpuTime(threadId) - oldThread.getCpuTime(); + userTime = threadMXBean.getThreadUserTime(threadId) - oldThread.getUserTime(); + } + } + jThreadInfos.add(new JThreadInfo(cpuTime, userTime, threadInfo)); + switch (threadInfo.getThreadState()) { + case RUNNABLE: + runnableThreads++; + break; + case BLOCKED: + blockedThreads++; + break; + case WAITING: + waitingThreads++; + break; + case TIMED_WAITING: + timedWaitingThreads++; + break; + case NEW: + newThreads++; + break; + case TERMINATED: + terminatedThreads++; + break; + } + } + return jThreadInfos; + } + + public static final class JThreadInfo { + private long cpuTime; + private long userTime; + private ThreadInfo threadInfo; + + public JThreadInfo(long cpuTime, long userTime, ThreadInfo threadInfo) { + this.cpuTime = cpuTime; + this.userTime = userTime; + this.threadInfo = threadInfo; + } + + public long getCpuTime() { + return cpuTime; + } + + public long getUserTime() { + return userTime; + } + + public ThreadInfo getThreadInfo() { + return threadInfo; + } + + } + + private static final class ThreadComparator implements Comparator<JThreadInfo> { + + private String threadSortOrder; + + public ThreadComparator(String threadSortOrder) { + this.threadSortOrder = threadSortOrder; + } + + @Override + public int compare(JThreadInfo jt1, JThreadInfo jt2) { + ThreadInfo o1 = jt1.getThreadInfo(); + ThreadInfo o2 = jt2.getThreadInfo(); + if ("cpu".equals(threadSortOrder)) { + int result = (int) (jt2.getCpuTime() - jt1.getCpuTime()); + if (result == 0) { + return compareId(o1, o2); + } + return result; + } + else if ("name".equals(threadSortOrder)) { + return compareName(o1, o2); + } + else { + // state + if (o1.getThreadState().equals(o2.getThreadState())) { + return compareName(o1, o2); + } + else { + return (int) threadStateOrdering.get(o1.getThreadState()) + - threadStateOrdering.get(o2.getThreadState()); + } + } + } + + private int compareId(ThreadInfo o1, ThreadInfo o2) { + Long id1 = o1.getThreadId(); + Long id2 = o2.getThreadId(); + return id1.compareTo(id2); + } + + private int compareName(ThreadInfo o1, ThreadInfo o2) { + int result = o1.getThreadName().compareTo(o2.getThreadName()); + if (result == 0) { + Long id1 = o1.getThreadId(); + Long id2 = o2.getThreadId(); + return id1.compareTo(id2); + } + return result; + } + + } + +} http://git-wip-us.apache.org/repos/asf/oozie/blob/1add349e/docs/src/site/twiki/AG_Monitoring.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/AG_Monitoring.twiki b/docs/src/site/twiki/AG_Monitoring.twiki index c6ff6fe..a37fd20 100644 --- a/docs/src/site/twiki/AG_Monitoring.twiki +++ b/docs/src/site/twiki/AG_Monitoring.twiki @@ -151,6 +151,13 @@ Oozie work on a 1 minute interval. * version * version-GET +---++ Oozie JVM Thread Dump + The =admin/jvminfo.jsp= servlet can be used to get some basic jvm stats and thread dump. +For eg: http://localhost:11000/oozie/admin/jvminfo.jsp?cpuwatch=1000&threadsort=cpu. It takes the following optional +query parameters: + * threadsort - The order in which the threads are sorted for display. Valid values are name, cpu, state. Default is state. + * cpuwatch - Time interval in milliseconds to monitor cpu usage of threads. Default value is 0. + [[index][::Go back to Oozie Documentation Index::]] </noautolink> http://git-wip-us.apache.org/repos/asf/oozie/blob/1add349e/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 5f4602b..a879090 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.1.0 release (trunk - unreleased) +OOZIE-1388 Add a admin servlet to show thread stack trace and CPU usage per thread (rohini) OOZIE-1893 Recovery service will never recover bundle action if CoordSubmitXCommand command is lost (puru via rohini) OOZIE-1878 Can't execute dryrun on the CLI (puru via rohini) OOZIE-1741 Add new coord EL function to get input partitions value string (satish.mittal via rohini) http://git-wip-us.apache.org/repos/asf/oozie/blob/1add349e/webapp/src/main/webapp/WEB-INF/web-common.xml ---------------------------------------------------------------------- diff --git a/webapp/src/main/webapp/WEB-INF/web-common.xml b/webapp/src/main/webapp/WEB-INF/web-common.xml index 3d7a0a9..4353cb7 100644 --- a/webapp/src/main/webapp/WEB-INF/web-common.xml +++ b/webapp/src/main/webapp/WEB-INF/web-common.xml @@ -222,6 +222,11 @@ <filter-mapping> <filter-name>authenticationfilter</filter-name> + <url-pattern>/admin/*</url-pattern> + </filter-mapping> + + <filter-mapping> + <filter-name>authenticationfilter</filter-name> <url-pattern>*.js</url-pattern> </filter-mapping> http://git-wip-us.apache.org/repos/asf/oozie/blob/1add349e/webapp/src/main/webapp/admin/jvminfo.jsp ---------------------------------------------------------------------- diff --git a/webapp/src/main/webapp/admin/jvminfo.jsp b/webapp/src/main/webapp/admin/jvminfo.jsp new file mode 100644 index 0000000..d169b32 --- /dev/null +++ b/webapp/src/main/webapp/admin/jvminfo.jsp @@ -0,0 +1,91 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<%@ page import="java.lang.management.ThreadInfo"%> +<%@ page import="org.apache.oozie.servlet.JVMInfo.JThreadInfo"%> +<%@page contentType="text/html; charset=UTF-8"%> + +<jsp:useBean id="jvmInfo" class="org.apache.oozie.servlet.JVMInfo" scope="request"> + <jsp:setProperty name="jvmInfo" property="threadSortOrder" value='<%=request.getParameter("threadsort")%>' /> + <jsp:setProperty name="jvmInfo" property="cpuMonitorTime" value='<%=request.getParameter("cpuwatch")%>' /> +</jsp:useBean> + +<HTML> +<HEAD> +<TITLE>Oozie JVM Info</TITLE> +</HEAD> +<BODY> + <h2>Memory Information:</h2> + <ul> + <li>Heap Memory: <br /> <%=jvmInfo.getHeapMemoryUsage()%></li> + <li>NonHeap Memory: <br /> <%=jvmInfo.getNonHeapMemoryUsage()%></li> + <li>ClassLoading: <br /> <%=jvmInfo.getClassLoadingInfo()%></li> + <li>Threads: <br /> <%=jvmInfo.getThreadInfo()%></li> + </ul> + + <h2>Thread Summary:</h2> + <table cellpadding="5" cellspacing="5"> + <tr> + <th>Thread</th> + <th>State</th> + <th>CPU Time (ns)</th> + <th>User Time (ns)</th> + </tr> + <% + for (JThreadInfo jthread : jvmInfo.getThreads()) { + ThreadInfo thread = jthread.getThreadInfo(); + %> + <tr> + <td><a href="#<%=thread.getThreadId()%>"><%=thread.getThreadName()%></a></td> + <td><%=thread.getThreadState()%></td> + <td><%=jthread.getCpuTime()%></td> + <td><%=jthread.getUserTime()%></td> + </tr> + <% + } + %> + </table> + + <h2>Stack Trace of JVM:</h2> + <% + for (JThreadInfo jthread : jvmInfo.getThreads()) { + ThreadInfo thread = jthread.getThreadInfo(); + %> + <h4> + <a name="<%=thread.getThreadId()%>"><%=thread.getThreadName()%> tid=<%=thread.getThreadId()%> + <%=thread.getThreadState()%></a> + </h4> + <pre> + <table> + <% + StackTraceElement[] traceElements = thread.getStackTrace(); + for (StackTraceElement traceElement : traceElements) { + %> + <tr> + <td> </td> + <td> at <%=traceElement%> </td> + </tr> + <% + } + %> + </table> + </pre> + <% + } + %> +</BODY> +</HTML> \ No newline at end of file
