Repository: giraph Updated Branches: refs/heads/trunk dd47ce6ab -> 2ce3efbb7
GIRAPH-889: Update Yourkit Profiler (yhdong via pavanka) Project: http://git-wip-us.apache.org/repos/asf/giraph/repo Commit: http://git-wip-us.apache.org/repos/asf/giraph/commit/2ce3efbb Tree: http://git-wip-us.apache.org/repos/asf/giraph/tree/2ce3efbb Diff: http://git-wip-us.apache.org/repos/asf/giraph/diff/2ce3efbb Branch: refs/heads/trunk Commit: 2ce3efbb7e872f11c49f92a8830ff2ad2df5a22b Parents: dd47ce6 Author: Pavan Kumar <[email protected]> Authored: Fri May 2 15:40:40 2014 -0700 Committer: Pavan Kumar <[email protected]> Committed: Fri May 2 15:40:40 2014 -0700 ---------------------------------------------------------------------- CHANGELOG | 2 + .../org/apache/giraph/utils/YourKitContext.java | 80 +++++++++++++++----- .../apache/giraph/utils/YourKitProfiler.java | 71 +++++++++++++---- pom.xml | 2 +- 4 files changed, 118 insertions(+), 37 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/giraph/blob/2ce3efbb/CHANGELOG ---------------------------------------------------------------------- diff --git a/CHANGELOG b/CHANGELOG index 2b4db33..c38a65f 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,8 @@ Giraph Change Log Release 1.1.0 - unreleased + GIRAPH-889: Update Yourkit Profiler (yhdong via pavanka) + GIRAPH-891: Make MessageStoreFactory configurable (rohankarwa via majakabiljo) GIRAPH-825: Fix DiskBackedPartitionStore to work with current trunk (armax00 via claudio) http://git-wip-us.apache.org/repos/asf/giraph/blob/2ce3efbb/giraph-core/src/main/java/org/apache/giraph/utils/YourKitContext.java ---------------------------------------------------------------------- diff --git a/giraph-core/src/main/java/org/apache/giraph/utils/YourKitContext.java b/giraph-core/src/main/java/org/apache/giraph/utils/YourKitContext.java index 8de4537..39220a3 100644 --- a/giraph-core/src/main/java/org/apache/giraph/utils/YourKitContext.java +++ b/giraph-core/src/main/java/org/apache/giraph/utils/YourKitContext.java @@ -54,10 +54,9 @@ public class YourKitContext { /** * Capture a snapshot * @param flags See {@link com.yourkit.api.ProfilingModes} - * @param context map context - * @param name unique name for this snapshot + * @param destPath where to store snapshot */ - private void snapshot(long flags, Mapper.Context context, String name) { + private void snapshot(long flags, String destPath) { if (yourKitController != null) { String path; try { @@ -67,56 +66,95 @@ public class YourKitContext { // CHECKSTYLE: resume IllegalCatch return; } - File destFile = new File(SLASH_JOINER.join( - "/tmp", context.getJobID(), context.getTaskAttemptID(), - name + ".snapshot")); try { + File destFile = new File(destPath); Files.createParentDirs(destFile); Files.move(new File(path), destFile); } catch (IOException e) { LOG.error("Failed to move YourKit snapshot file from " + path + - " to " + destFile.getPath(), e); + " to " + destPath, e); } } } /** - * This method is just a convenient replacement of - * {@link #captureSnapshot(long, java.io.File)} with - * {@link com.yourkit.api.ProfilingModes#SNAPSHOT_WITH_HEAP} for the flags. + * Capture snapshot with all recorded data including heap dump. * * WARNING: This is likely to be VERY slow for large jobs. * - * @param context map context - * @param name unique name for this snapshot + * @param destPath path to store snapshot file + */ + public void snapshotWithMemory(String destPath) { + snapshot(ProfilingModes.SNAPSHOT_WITH_HEAP, destPath); + } + + /** + * Capture snapshot with all recorded data including heap dump. + * The snapshot file is saved in log directory, or /tmp as default. + * + * WARNING: This is likely to be VERY slow for large jobs. + * + * @param context context + * @param name snapshot file name */ public void snapshotWithMemory(Mapper.Context context, String name) { - snapshot(ProfilingModes.SNAPSHOT_WITH_HEAP, context, name); + snapshot(ProfilingModes.SNAPSHOT_WITH_HEAP, + SLASH_JOINER.join(System.getProperty("hadoop.log.dir", "/tmp"), + "userlogs", context.getTaskAttemptID(), name + ".snapshot")); + } + + /** + * Capture snapshot with all recorded data except for heap dump. + * + * @param destPath path to store snapshot file + */ + public void snapshotCPUOnly(String destPath) { + snapshot(ProfilingModes.SNAPSHOT_WITHOUT_HEAP, destPath); } /** - * This method is just a convenient replacement of - * {@link #captureSnapshot(long, java.io.File)} with - * {@link com.yourkit.api.ProfilingModes#SNAPSHOT_WITHOUT_HEAP} for the flags. + * Capture snapshot with all recorded data except for heap dump. + * The snapshot file is saved in log directory, or /tmp as default. * - * @param context map context - * @param name unique name for this snapshot + * @param context context + * @param name snapshot file name */ public void snapshotCPUOnly(Mapper.Context context, String name) { - snapshot(ProfilingModes.SNAPSHOT_WITHOUT_HEAP, context, name); + snapshot(ProfilingModes.SNAPSHOT_WITHOUT_HEAP, + SLASH_JOINER.join(System.getProperty("hadoop.log.dir", "/tmp"), + "userlogs", context.getTaskAttemptID(), name + ".snapshot")); } /** - * Stop profiling CPU + * Stop recording */ public void stop() { if (yourKitController != null) { try { + yourKitController.disableStackTelemetry(); + LOG.info("Disabled YourKit stack telemetry"); + // CHECKSTYLE: stop IllegalCatch + } catch (Exception e) { + // CHECKSTYLE: resume IllegalCatch + LOG.error("Failed to stop stack telemetry", e); + } + + try { yourKitController.stopCPUProfiling(); + LOG.info("Stopped Yourkit CPU profiling"); + // CHECKSTYLE: stop IllegalCatch + } catch (Exception e) { + // CHECKSTYLE: resume IllegalCatch + LOG.error("Failed to stop YourKit profiling", e); + } + + try { + yourKitController.stopAllocationRecording(); + LOG.info("Stopped Yourkit allocation recording"); // CHECKSTYLE: stop IllegalCatch } catch (Exception e) { // CHECKSTYLE: resume IllegalCatch - LOG.error("Failed to stop YourKit CPU profiling", e); + LOG.error("Failed to stop YourKit allocation recording", e); } } } http://git-wip-us.apache.org/repos/asf/giraph/blob/2ce3efbb/giraph-core/src/main/java/org/apache/giraph/utils/YourKitProfiler.java ---------------------------------------------------------------------- diff --git a/giraph-core/src/main/java/org/apache/giraph/utils/YourKitProfiler.java b/giraph-core/src/main/java/org/apache/giraph/utils/YourKitProfiler.java index 88832b0..4b36d96 100644 --- a/giraph-core/src/main/java/org/apache/giraph/utils/YourKitProfiler.java +++ b/giraph-core/src/main/java/org/apache/giraph/utils/YourKitProfiler.java @@ -18,7 +18,6 @@ package org.apache.giraph.utils; -import org.apache.giraph.conf.GiraphConfiguration; import org.apache.log4j.Logger; import com.yourkit.api.Controller; @@ -31,9 +30,7 @@ import com.yourkit.api.ProfilingModes; * - http://www.yourkit.com/docs/95/help/api.jsp * - http://www.yourkit.com/docs/95/api/index.html * - * This class is a simple helper around the API mentioned above that allows you - * to easily wrap code with - * {@link YourKitProfiler#startProfile(GiraphConfiguration)} + * This class is a simple helper around the API mentioned above * followed by any amount of snapshotX calls and finally * {@link YourKitContext#stop()}. * See also {@link YourKitContext}. @@ -45,30 +42,74 @@ import com.yourkit.api.ProfilingModes; public class YourKitProfiler { /** Logger */ private static final Logger LOG = Logger.getLogger(YourKitProfiler.class); + /** Record every ALLOCATION_RECORDING_INTERVAL'th allocation */ + private static final int ALLOCATION_RECORDING_INTERVAL = 1000; /** Don't construct, allow inheritance */ protected YourKitProfiler() { } /** - * Convenient replacement of {@link #startProfilingCPU(long)} with - * {@link ProfilingModes#CPU_TRACING} for the mode. + * Create a YourKit controller and do some or all of + * {@link Controller#enableExceptionTelemetry()} + * {@link Controller#startCPUProfiling(long, String, String)} + * {@link Controller#startAllocationRecording(boolean, int, boolean, + * int, boolean, boolean)} + * based on boolean config options passed as method parameters * - * @param conf GiraphConfiguration - * @return profiler context + * @param enableStackTelemetry enable stack telementry + * @param enableCPUProfilling enable CPU profilling + * @param enableAllocationRecording enable allocation recording + * + * @return profiler context, or null if controller cannot be created */ - public static YourKitContext startProfile(GiraphConfiguration conf) { - Controller controller = null; + public static YourKitContext startProfile(boolean enableStackTelemetry, + boolean enableCPUProfilling, + boolean enableAllocationRecording) { + Controller controller; try { controller = new Controller(); - controller.enableStackTelemetry(); - controller.startCPUProfiling(ProfilingModes.CPU_SAMPLING, - Controller.DEFAULT_FILTERS); - LOG.debug("Started YourKit profiling CPU"); // CHECKSTYLE: stop IllegalCatch } catch (Exception e) { // CHECKSTYLE: resume IllegalCatch - LOG.debug("Failed to start YourKit CPU profiling", e); + LOG.info("Failed to set up YourKit controller", e); + return null; + } + + try { + if (enableStackTelemetry) { + controller.enableStackTelemetry(); + LOG.info("Enabled Yourkit stack telemetry"); + } + // CHECKSTYLE: stop IllegalCatch + } catch (Exception e) { + // CHECKSTYLE: resume IllegalCatch + LOG.info("Failed to enable YourKit stack telemetry", e); + } + + try { + if (enableCPUProfilling) { + controller.startCPUProfiling(ProfilingModes.CPU_SAMPLING, + Controller.DEFAULT_FILTERS, Controller.DEFAULT_WALLTIME_SPEC); + LOG.info("Started YourKit CPU profiling"); + } + // CHECKSTYLE: stop IllegalCatch + } catch (Exception e) { + // CHECKSTYLE: resume IllegalCatch + LOG.info("Failed to start YourKit CPU profiling", e); + } + + try { + if (enableAllocationRecording) { + controller.startAllocationRecording(true, ALLOCATION_RECORDING_INTERVAL, + false, -1, true, false); + LOG.info("Started YourKit allocation recording"); + } + // CHECKSTYLE: stop IllegalCatch + } catch (Exception e) { + // CHECKSTYLE: resume IllegalCatch + LOG.info("Failed to start YourKit allocation recording", e); } + return new YourKitContext(controller); } } http://git-wip-us.apache.org/repos/asf/giraph/blob/2ce3efbb/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index f963585..2a321d3 100644 --- a/pom.xml +++ b/pom.xml @@ -307,7 +307,7 @@ under the License. <dep.tinkerpop.rexter.version>2.4.0</dep.tinkerpop.rexter.version> <dep.typetools.version>0.2.1</dep.typetools.version> <dep.yammer-metrics.version>2.2.0</dep.yammer-metrics.version> - <dep.yourkit-api.version>9.5.6</dep.yourkit-api.version> + <dep.yourkit-api.version>11.0.10</dep.yourkit-api.version> <dep.zookeeper.version>3.4.5</dep.zookeeper.version> <forHadoop>for-hadoop-${hadoop.version}</forHadoop>
