[beam] branch master updated: Add explicit PortablePipelineOptions.enableHeapDumps to control heap dumping. (#26982)

[yhu]

This is an automated email from the ASF dual-hosted git repository.

yhu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
     new a8cf6964e07 Add explicit PortablePipelineOptions.enableHeapDumps to 
control heap dumping. (#26982)
a8cf6964e07 is described below

commit a8cf6964e07939293d78ac197198c2bb944ca31e
Author: Sam Whittle <scwhit...@users.noreply.github.com>
AuthorDate: Wed Jun 28 17:34:59 2023 +0200

    Add explicit PortablePipelineOptions.enableHeapDumps to control heap 
dumping. (#26982)
---
 .../beam/sdk/options/PortablePipelineOptions.java       | 17 +++++++++++++++++
 .../apache/beam/fn/harness/status/MemoryMonitor.java    |  4 +++-
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PortablePipelineOptions.java
 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PortablePipelineOptions.java
index 6bfab14ed0e..ec82aebbef6 100644
--- 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PortablePipelineOptions.java
+++ 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PortablePipelineOptions.java
@@ -123,4 +123,21 @@ public interface PortablePipelineOptions extends 
PipelineOptions, FileStagingOpt
 
     return "";
   }
+
+  /**
+   * If {@literal true} and PipelineOption tempLocation is set, save a heap 
dump before shutting
+   * down the JVM due to GC thrashing or out of memory. The heap will be 
dumped to local disk and
+   * then uploaded to the tempLocation.
+   *
+   * <p>CAUTION: Heap dumps can take up more disk than the JVM memory. Ensure 
the local disk is
+   * configured to have sufficient free space before enabling this option.
+   */
+  @Description(
+      "If {@literal true} and PipelineOption tempLocation is set, save a heap 
dump before shutting"
+          + " down the JVM due to GC thrashing or out of memory. The heap will 
be dumped to local"
+          + " disk and then uploaded to the tempLocation.")
+  @Default.Boolean(false)
+  boolean getEnableHeapDumps();
+
+  void setEnableHeapDumps(boolean enableHeapDumps);
 }
diff --git 
a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/status/MemoryMonitor.java
 
b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/status/MemoryMonitor.java
index f9e91eda7f6..13ac433c355 100644
--- 
a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/status/MemoryMonitor.java
+++ 
b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/status/MemoryMonitor.java
@@ -43,6 +43,7 @@ import org.apache.beam.sdk.io.FileSystems;
 import org.apache.beam.sdk.io.fs.CreateOptions.StandardCreateOptions;
 import org.apache.beam.sdk.io.fs.ResourceId;
 import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.PortablePipelineOptions;
 import 
org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting;
 import 
org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions;
 import 
org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet;
@@ -202,7 +203,8 @@ public class MemoryMonitor implements Runnable {
 
   public static MemoryMonitor fromOptions(PipelineOptions options) {
     String uploadFilePath = options.getTempLocation();
-    boolean canDumpHeap = uploadFilePath != null;
+    PortablePipelineOptions portableOptions = 
options.as(PortablePipelineOptions.class);
+    boolean canDumpHeap = uploadFilePath != null && 
portableOptions.getEnableHeapDumps();
 
     return new MemoryMonitor(
         new SystemGCStatsProvider(),