This is an automated email from the ASF dual-hosted git repository. mhubail pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push: new 48a3a9d4f8 [ASTERIXDB-3386][CONF] Add runtime memory overhead config 48a3a9d4f8 is described below commit 48a3a9d4f8d223c163e00b6d04fea9186626713c Author: Murtadha Hubail <mhub...@apache.org> AuthorDate: Mon Apr 22 22:41:01 2024 +0300 [ASTERIXDB-3386][CONF] Add runtime memory overhead config - user model changes: yes - storage format changes: no - interface changes: no Details: - Add a new compiler configuration that allows specifying a percentage of memory to be added to the job's required memory to account for runtime memory overhead and default it to 5%. Change-Id: I33a1911ff6b0df64b273ba10a7626dd10cc4cca2 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18246 Integration-Tests: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Reviewed-by: Murtadha Hubail <mhub...@apache.org> Reviewed-by: Ali Alsuliman <ali.al.solai...@gmail.com> Tested-by: Jenkins <jenk...@fulliautomatix.ics.uci.edu> --- .../java/org/apache/asterix/api/common/APIFramework.java | 11 +++++++++++ .../results/api/cluster_state_1/cluster_state_1.1.regexadm | 1 + .../cluster_state_1_full/cluster_state_1_full.1.regexadm | 1 + .../cluster_state_1_less/cluster_state_1_less.1.regexadm | 1 + .../misc/dataset-resources/dataset-resources.6.regex | 2 +- .../misc/dataset-resources/dataset-resources.7.regex | 2 +- .../test/resources/runtimets/results/misc/jobs/jobs.2.regex | 2 +- .../test/resources/runtimets/results/misc/ping/ping.2.regex | 2 +- .../apache/asterix/common/config/CompilerProperties.java | 13 ++++++++----- 9 files changed, 26 insertions(+), 9 deletions(-) diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java index 3da25fe311..16f922f6d7 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java @@ -342,6 +342,7 @@ public class APIFramework { getJobLocations(spec, nodeJobTracker, computationLocations); final IClusterCapacity jobRequiredCapacity = ResourceUtils.getRequiredCapacity(plan, jobLocations, physOptConf); + addRuntimeMemoryOverhead(jobRequiredCapacity, compilerProperties); spec.setRequiredClusterCapacity(jobRequiredCapacity); } } @@ -600,4 +601,14 @@ public class APIFramework { return new AlgebricksAbsolutePartitionConstraint(Arrays.stream(clusterLocations.getLocations()) .filter(jobParticipatingNodes::contains).toArray(String[]::new)); } + + private static void addRuntimeMemoryOverhead(IClusterCapacity jobRequiredCapacity, + CompilerProperties compilerProperties) { + int runtimeMemoryOverheadPercentage = compilerProperties.getRuntimeMemoryOverheadPercentage(); + if (runtimeMemoryOverheadPercentage > 0) { + double multiplier = 1 + runtimeMemoryOverheadPercentage / 100.0; + long aggregatedMemoryByteSize = jobRequiredCapacity.getAggregatedMemoryByteSize(); + jobRequiredCapacity.setAggregatedMemoryByteSize((long) (aggregatedMemoryByteSize * multiplier)); + } + } } diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm index 1bb6ef9c97..6888accc37 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm @@ -32,6 +32,7 @@ "compiler\.min\.memory\.allocation" : true, "compiler\.parallelism" : 0, "compiler.queryplanshape" : "zigzag", + "compiler.runtime.memory.overhead" : 5, "compiler\.sort\.parallel" : false, "compiler\.sort\.samples" : 100, "compiler\.sortmemory" : 327680, diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm index c2883ae935..31daf991a3 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm @@ -32,6 +32,7 @@ "compiler\.min\.memory\.allocation" : true, "compiler\.parallelism" : -1, "compiler.queryplanshape" : "zigzag", + "compiler.runtime.memory.overhead" : 5, "compiler\.sort\.parallel" : true, "compiler\.sort\.samples" : 100, "compiler\.sortmemory" : 327680, diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm index 8c54bb88ad..a47b3be23b 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm @@ -32,6 +32,7 @@ "compiler\.min\.memory\.allocation" : true, "compiler\.parallelism" : 3, "compiler.queryplanshape" : "zigzag", + "compiler.runtime.memory.overhead" : 5, "compiler\.sort\.parallel" : true, "compiler\.sort\.samples" : 100, "compiler\.sortmemory" : 327680, diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.6.regex b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.6.regex index e2d166b9ad..13bbeceba7 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.6.regex +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.6.regex @@ -1 +1 @@ -/memory\D+229376/ \ No newline at end of file +/memory\D+240844/ \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.7.regex b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.7.regex index e2d166b9ad..13bbeceba7 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.7.regex +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.7.regex @@ -1 +1 @@ -/memory\D+229376/ \ No newline at end of file +/memory\D+240844/ \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/jobs/jobs.2.regex b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/jobs/jobs.2.regex index e2d166b9ad..13bbeceba7 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/jobs/jobs.2.regex +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/jobs/jobs.2.regex @@ -1 +1 @@ -/memory\D+229376/ \ No newline at end of file +/memory\D+240844/ \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/ping/ping.2.regex b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/ping/ping.2.regex index e2d166b9ad..13bbeceba7 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/ping/ping.2.regex +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/ping/ping.2.regex @@ -1 +1 @@ -/memory\D+229376/ \ No newline at end of file +/memory\D+240844/ \ No newline at end of file diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java index 85b80a6136..edd25bf0f3 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java @@ -22,6 +22,7 @@ import static org.apache.hyracks.control.common.config.OptionTypes.BOOLEAN; import static org.apache.hyracks.control.common.config.OptionTypes.INTEGER; import static org.apache.hyracks.control.common.config.OptionTypes.INTEGER_BYTE_UNIT; import static org.apache.hyracks.control.common.config.OptionTypes.LONG_BYTE_UNIT; +import static org.apache.hyracks.control.common.config.OptionTypes.NONNEGATIVE_INTEGER; import static org.apache.hyracks.control.common.config.OptionTypes.POSITIVE_INTEGER; import static org.apache.hyracks.control.common.config.OptionTypes.STRING; import static org.apache.hyracks.util.StorageUtil.StorageUnit.KILOBYTE; @@ -124,7 +125,11 @@ public class CompilerProperties extends AbstractProperties { AlgebricksConfig.COLUMN_FILTER_DEFAULT, "Enable/disable the use of column min/max filters"), //TODO(DB): remove after - COMPILER_ENABLE_DB_RESOLUTION(BOOLEAN, true, "Enable/disable the resolution of namespaces to database"); + COMPILER_ENABLE_DB_RESOLUTION(BOOLEAN, true, "Enable/disable the resolution of namespaces to database"), + COMPILER_RUNTIME_MEMORY_OVERHEAD( + NONNEGATIVE_INTEGER, + 5, + "A percentage of the job's required memory to be added to account for runtime memory overhead"); private final IOptionType type; private final Object defaultValue; @@ -207,8 +212,6 @@ public class CompilerProperties extends AbstractProperties { public static final String COMPILER_COLUMN_FILTER_KEY = Option.COMPILER_COLUMN_FILTER.ini(); - public static final String COMPILER_ENABLE_DB_RESOLUTION_KEY = Option.COMPILER_ENABLE_DB_RESOLUTION.ini(); - public static final int COMPILER_PARALLELISM_AS_STORAGE = 0; public CompilerProperties(PropertiesAccessor accessor) { @@ -312,7 +315,7 @@ public class CompilerProperties extends AbstractProperties { return accessor.getBoolean(Option.COMPILER_COLUMN_FILTER); } - public boolean isDbResolutionEnabled() { - return accessor.getBoolean(Option.COMPILER_ENABLE_DB_RESOLUTION); + public int getRuntimeMemoryOverheadPercentage() { + return accessor.getInt(Option.COMPILER_RUNTIME_MEMORY_OVERHEAD); } }