This is an automated email from the ASF dual-hosted git repository. heneveld pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/brooklyn-server.git
commit 6d6e90af2ec759144853a5f5dff01a0f0a72d050 Author: Alex Heneveld <[email protected]> AuthorDate: Fri Jun 9 14:19:17 2023 +0100 some of workflow scratch vars stored --- .../core/workflow/WorkflowExecutionContext.java | 52 ++++++++++++++++++++-- .../brooklyn/core/workflow/WorkflowSizeTest.java | 4 +- 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/apache/brooklyn/core/workflow/WorkflowExecutionContext.java b/core/src/main/java/org/apache/brooklyn/core/workflow/WorkflowExecutionContext.java index 74be491f3e..61d6e59f6f 100644 --- a/core/src/main/java/org/apache/brooklyn/core/workflow/WorkflowExecutionContext.java +++ b/core/src/main/java/org/apache/brooklyn/core/workflow/WorkflowExecutionContext.java @@ -18,10 +18,11 @@ */ package org.apache.brooklyn.core.workflow; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.*; +import com.fasterxml.jackson.databind.JavaType; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.type.TypeFactory; +import com.fasterxml.jackson.databind.util.Converter; import com.google.common.collect.Iterables; import com.google.common.reflect.TypeToken; import org.apache.brooklyn.api.entity.Entity; @@ -81,6 +82,7 @@ import java.util.function.Supplier; import static org.apache.brooklyn.core.workflow.WorkflowReplayUtils.ReplayResumeDepthCheck.RESUMABLE_WHENEVER_NESTED_WORKFLOWS_PRESENT; @JsonInclude(JsonInclude.Include.NON_NULL) +@JsonDeserialize(converter = WorkflowExecutionContext.Converter.class) public class WorkflowExecutionContext { private static final Logger log = LoggerFactory.getLogger(WorkflowExecutionContext.class); @@ -205,6 +207,7 @@ public class WorkflowExecutionContext { String nextTaskId; } + // note: when persisted, this may be omitted and restored from the oldStepInfo map in the Converter Map<String,Object> workflowScratchVariables = MutableMap.of(); @JsonInclude(JsonInclude.Include.NON_EMPTY) @@ -366,9 +369,16 @@ public class WorkflowExecutionContext { return parentTag; } + @JsonIgnore public Map<String, Object> getWorkflowScratchVariables() { return workflowScratchVariables; } + @JsonGetter("workflowScratchVariables") + public Map<String, Object> getWorkflowScratchVariablesForPersistence() { + Pair<Map<String, Object>, Set<Integer>> prev = getPreviousStepWorkflowScratchAndBacktrackedSteps(); + if (prev!=null && Objects.equals(prev.getLeft(), workflowScratchVariables)) return null; + return workflowScratchVariables; + } public Map<String, List<Instant>> getRetryRecords() { return retryRecords; @@ -680,6 +690,20 @@ public class WorkflowExecutionContext { return null; } + @JsonIgnore + public Pair<Map<String,Object>,Set<Integer>> getPreviousStepWorkflowScratchAndBacktrackedSteps() { + Integer prevSI = previousStepIndex; + Set<Integer> previousSteps = MutableSet.of(); + while (prevSI!=null && previousSteps.add(prevSI)) { + OldStepRecord last = oldStepInfo.get(prevSI); + if (last==null) break; + if (last.workflowScratch!=null) return Pair.of(last.workflowScratch, previousSteps); + if (last.previous.isEmpty()) break; + prevSI = last.previous.iterator().next(); + } + return null; + } + public Object getOutput() { return output; } @@ -1513,4 +1537,26 @@ public class WorkflowExecutionContext { return "neg-"+(index); // unknown } + public static class Converter implements com.fasterxml.jackson.databind.util.Converter<WorkflowExecutionContext,WorkflowExecutionContext> { + @Override + public WorkflowExecutionContext convert(WorkflowExecutionContext value) { + if (value.workflowScratchVariables==null || value.workflowScratchVariables.isEmpty()) { + Pair<Map<String, Object>, Set<Integer>> prev = value.getPreviousStepWorkflowScratchAndBacktrackedSteps(); + if (prev!=null && prev.getLeft()!=null) value.workflowScratchVariables = prev.getLeft(); + } + // note: no special handling for output; currently we set that as null as we go along + + return value; + } + + @Override + public JavaType getInputType(TypeFactory typeFactory) { + return typeFactory.constructType(WorkflowExecutionContext.class); + } + + @Override + public JavaType getOutputType(TypeFactory typeFactory) { + return typeFactory.constructType(WorkflowExecutionContext.class); + } + } } diff --git a/core/src/test/java/org/apache/brooklyn/core/workflow/WorkflowSizeTest.java b/core/src/test/java/org/apache/brooklyn/core/workflow/WorkflowSizeTest.java index 7c16b1d02e..4edcb936cc 100644 --- a/core/src/test/java/org/apache/brooklyn/core/workflow/WorkflowSizeTest.java +++ b/core/src/test/java/org/apache/brooklyn/core/workflow/WorkflowSizeTest.java @@ -95,8 +95,10 @@ public class WorkflowSizeTest extends BrooklynMgmtUnitTestSupport { sizes.forEach((k,v) -> { log.info("Sensor "+k+": "+v); }); Asserts.assertThat(sizes.values().stream().reduce(0, (v0,v1)->v0+v1), result -> result < 20*1000); - // 100k payload now -> bumps sensor size from 5k to 3MB (before any optimization) + // 100k payload now -> bumps sensor (json) size from 5k to 3MB (before any optimization) // removing output which is identical to the previous gives minor savings (in this test): 3380416 -> 3176074 + // removing scratch at workflow which matches a step reduces further: -> 2869522 + // [xml persistence is less of an issue because it will use a shared reference] for (int i=0; i<1000; i++) { for (int j=0; j<10; j++) sampleData += "0123456789"; sampleData += "\n";
