>From Murtadha Hubail <[email protected]>: Murtadha Hubail has submitted this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18246 )
Change subject: [ASTERIXDB-3386][CONF] Add runtime memory overhead config ...................................................................... [ASTERIXDB-3386][CONF] Add runtime memory overhead config - user model changes: yes - storage format changes: no - interface changes: no Details: - Add a new compiler configuration that allows specifying a percentage of memory to be added to the job's required memory to account for runtime memory overhead and default it to 5%. Change-Id: I33a1911ff6b0df64b273ba10a7626dd10cc4cca2 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18246 Integration-Tests: Jenkins <[email protected]> Reviewed-by: Murtadha Hubail <[email protected]> Reviewed-by: Ali Alsuliman <[email protected]> Tested-by: Jenkins <[email protected]> --- M asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm M asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java M asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm M asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.7.regex M asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.6.regex M asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java M asterixdb/asterix-app/src/test/resources/runtimets/results/misc/jobs/jobs.2.regex M asterixdb/asterix-app/src/test/resources/runtimets/results/misc/ping/ping.2.regex 9 files changed, 51 insertions(+), 9 deletions(-) Approvals: Murtadha Hubail: Looks good to me, but someone else must approve Ali Alsuliman: Looks good to me, approved Jenkins: Verified; Verified diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java index 3da25fe..16f922f 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java @@ -342,6 +342,7 @@ getJobLocations(spec, nodeJobTracker, computationLocations); final IClusterCapacity jobRequiredCapacity = ResourceUtils.getRequiredCapacity(plan, jobLocations, physOptConf); + addRuntimeMemoryOverhead(jobRequiredCapacity, compilerProperties); spec.setRequiredClusterCapacity(jobRequiredCapacity); } } @@ -600,4 +601,14 @@ return new AlgebricksAbsolutePartitionConstraint(Arrays.stream(clusterLocations.getLocations()) .filter(jobParticipatingNodes::contains).toArray(String[]::new)); } + + private static void addRuntimeMemoryOverhead(IClusterCapacity jobRequiredCapacity, + CompilerProperties compilerProperties) { + int runtimeMemoryOverheadPercentage = compilerProperties.getRuntimeMemoryOverheadPercentage(); + if (runtimeMemoryOverheadPercentage > 0) { + double multiplier = 1 + runtimeMemoryOverheadPercentage / 100.0; + long aggregatedMemoryByteSize = jobRequiredCapacity.getAggregatedMemoryByteSize(); + jobRequiredCapacity.setAggregatedMemoryByteSize((long) (aggregatedMemoryByteSize * multiplier)); + } + } } diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm index 1bb6ef9..6888acc 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm @@ -32,6 +32,7 @@ "compiler\.min\.memory\.allocation" : true, "compiler\.parallelism" : 0, "compiler.queryplanshape" : "zigzag", + "compiler.runtime.memory.overhead" : 5, "compiler\.sort\.parallel" : false, "compiler\.sort\.samples" : 100, "compiler\.sortmemory" : 327680, diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm index c2883ae..31daf99 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm @@ -32,6 +32,7 @@ "compiler\.min\.memory\.allocation" : true, "compiler\.parallelism" : -1, "compiler.queryplanshape" : "zigzag", + "compiler.runtime.memory.overhead" : 5, "compiler\.sort\.parallel" : true, "compiler\.sort\.samples" : 100, "compiler\.sortmemory" : 327680, diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm index 8c54bb8..a47b3be 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm @@ -32,6 +32,7 @@ "compiler\.min\.memory\.allocation" : true, "compiler\.parallelism" : 3, "compiler.queryplanshape" : "zigzag", + "compiler.runtime.memory.overhead" : 5, "compiler\.sort\.parallel" : true, "compiler\.sort\.samples" : 100, "compiler\.sortmemory" : 327680, diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.6.regex b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.6.regex index e2d166b..13bbece 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.6.regex +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.6.regex @@ -1 +1 @@ -/memory\D+229376/ \ No newline at end of file +/memory\D+240844/ \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.7.regex b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.7.regex index e2d166b..13bbece 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.7.regex +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.7.regex @@ -1 +1 @@ -/memory\D+229376/ \ No newline at end of file +/memory\D+240844/ \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/jobs/jobs.2.regex b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/jobs/jobs.2.regex index e2d166b..13bbece 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/jobs/jobs.2.regex +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/jobs/jobs.2.regex @@ -1 +1 @@ -/memory\D+229376/ \ No newline at end of file +/memory\D+240844/ \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/ping/ping.2.regex b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/ping/ping.2.regex index e2d166b..13bbece 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/ping/ping.2.regex +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/ping/ping.2.regex @@ -1 +1 @@ -/memory\D+229376/ \ No newline at end of file +/memory\D+240844/ \ No newline at end of file diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java index 85b80a6..edd25bf 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java @@ -22,6 +22,7 @@ import static org.apache.hyracks.control.common.config.OptionTypes.INTEGER; import static org.apache.hyracks.control.common.config.OptionTypes.INTEGER_BYTE_UNIT; import static org.apache.hyracks.control.common.config.OptionTypes.LONG_BYTE_UNIT; +import static org.apache.hyracks.control.common.config.OptionTypes.NONNEGATIVE_INTEGER; import static org.apache.hyracks.control.common.config.OptionTypes.POSITIVE_INTEGER; import static org.apache.hyracks.control.common.config.OptionTypes.STRING; import static org.apache.hyracks.util.StorageUtil.StorageUnit.KILOBYTE; @@ -124,7 +125,11 @@ AlgebricksConfig.COLUMN_FILTER_DEFAULT, "Enable/disable the use of column min/max filters"), //TODO(DB): remove after - COMPILER_ENABLE_DB_RESOLUTION(BOOLEAN, true, "Enable/disable the resolution of namespaces to database"); + COMPILER_ENABLE_DB_RESOLUTION(BOOLEAN, true, "Enable/disable the resolution of namespaces to database"), + COMPILER_RUNTIME_MEMORY_OVERHEAD( + NONNEGATIVE_INTEGER, + 5, + "A percentage of the job's required memory to be added to account for runtime memory overhead"); private final IOptionType type; private final Object defaultValue; @@ -207,8 +212,6 @@ public static final String COMPILER_COLUMN_FILTER_KEY = Option.COMPILER_COLUMN_FILTER.ini(); - public static final String COMPILER_ENABLE_DB_RESOLUTION_KEY = Option.COMPILER_ENABLE_DB_RESOLUTION.ini(); - public static final int COMPILER_PARALLELISM_AS_STORAGE = 0; public CompilerProperties(PropertiesAccessor accessor) { @@ -312,7 +315,7 @@ return accessor.getBoolean(Option.COMPILER_COLUMN_FILTER); } - public boolean isDbResolutionEnabled() { - return accessor.getBoolean(Option.COMPILER_ENABLE_DB_RESOLUTION); + public int getRuntimeMemoryOverheadPercentage() { + return accessor.getInt(Option.COMPILER_RUNTIME_MEMORY_OVERHEAD); } } -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18246 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Change-Id: I33a1911ff6b0df64b273ba10a7626dd10cc4cca2 Gerrit-Change-Number: 18246 Gerrit-PatchSet: 3 Gerrit-Owner: Murtadha Hubail <[email protected]> Gerrit-Reviewer: Ali Alsuliman <[email protected]> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Murtadha Hubail <[email protected]> Gerrit-Reviewer: Wail Alkowaileet <[email protected]> Gerrit-MessageType: merged
