This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 48a3a9d4f8 [ASTERIXDB-3386][CONF] Add runtime memory overhead config
48a3a9d4f8 is described below
commit 48a3a9d4f8d223c163e00b6d04fea9186626713c
Author: Murtadha Hubail <mhub...@apache.org>
AuthorDate: Mon Apr 22 22:41:01 2024 +0300
[ASTERIXDB-3386][CONF] Add runtime memory overhead config
- user model changes: yes
- storage format changes: no
- interface changes: no
Details:
- Add a new compiler configuration that allows specifying
a percentage of memory to be added to the job's required
memory to account for runtime memory overhead and default
it to 5%.
Change-Id: I33a1911ff6b0df64b273ba10a7626dd10cc4cca2
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18246
Integration-Tests: Jenkins <jenk...@fulliautomatix.ics.uci.edu>
Reviewed-by: Murtadha Hubail <mhub...@apache.org>
Reviewed-by: Ali Alsuliman <ali.al.solai...@gmail.com>
Tested-by: Jenkins <jenk...@fulliautomatix.ics.uci.edu>
---
.../java/org/apache/asterix/api/common/APIFramework.java | 11 +++++++++++
.../results/api/cluster_state_1/cluster_state_1.1.regexadm | 1 +
.../cluster_state_1_full/cluster_state_1_full.1.regexadm | 1 +
.../cluster_state_1_less/cluster_state_1_less.1.regexadm | 1 +
.../misc/dataset-resources/dataset-resources.6.regex | 2 +-
.../misc/dataset-resources/dataset-resources.7.regex | 2 +-
.../test/resources/runtimets/results/misc/jobs/jobs.2.regex | 2 +-
.../test/resources/runtimets/results/misc/ping/ping.2.regex | 2 +-
.../apache/asterix/common/config/CompilerProperties.java | 13 ++++++++-----
9 files changed, 26 insertions(+), 9 deletions(-)
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
index 3da25fe311..16f922f6d7 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
@@ -342,6 +342,7 @@ public class APIFramework {
getJobLocations(spec, nodeJobTracker,
computationLocations);
final IClusterCapacity jobRequiredCapacity =
ResourceUtils.getRequiredCapacity(plan,
jobLocations, physOptConf);
+ addRuntimeMemoryOverhead(jobRequiredCapacity,
compilerProperties);
spec.setRequiredClusterCapacity(jobRequiredCapacity);
}
}
@@ -600,4 +601,14 @@ public class APIFramework {
return new
AlgebricksAbsolutePartitionConstraint(Arrays.stream(clusterLocations.getLocations())
.filter(jobParticipatingNodes::contains).toArray(String[]::new));
}
+
+ private static void addRuntimeMemoryOverhead(IClusterCapacity
jobRequiredCapacity,
+ CompilerProperties compilerProperties) {
+ int runtimeMemoryOverheadPercentage =
compilerProperties.getRuntimeMemoryOverheadPercentage();
+ if (runtimeMemoryOverheadPercentage > 0) {
+ double multiplier = 1 + runtimeMemoryOverheadPercentage / 100.0;
+ long aggregatedMemoryByteSize =
jobRequiredCapacity.getAggregatedMemoryByteSize();
+ jobRequiredCapacity.setAggregatedMemoryByteSize((long)
(aggregatedMemoryByteSize * multiplier));
+ }
+ }
}
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
index 1bb6ef9c97..6888accc37 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
@@ -32,6 +32,7 @@
"compiler\.min\.memory\.allocation" : true,
"compiler\.parallelism" : 0,
"compiler.queryplanshape" : "zigzag",
+ "compiler.runtime.memory.overhead" : 5,
"compiler\.sort\.parallel" : false,
"compiler\.sort\.samples" : 100,
"compiler\.sortmemory" : 327680,
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
index c2883ae935..31daf991a3 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
@@ -32,6 +32,7 @@
"compiler\.min\.memory\.allocation" : true,
"compiler\.parallelism" : -1,
"compiler.queryplanshape" : "zigzag",
+ "compiler.runtime.memory.overhead" : 5,
"compiler\.sort\.parallel" : true,
"compiler\.sort\.samples" : 100,
"compiler\.sortmemory" : 327680,
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
index 8c54bb88ad..a47b3be23b 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
@@ -32,6 +32,7 @@
"compiler\.min\.memory\.allocation" : true,
"compiler\.parallelism" : 3,
"compiler.queryplanshape" : "zigzag",
+ "compiler.runtime.memory.overhead" : 5,
"compiler\.sort\.parallel" : true,
"compiler\.sort\.samples" : 100,
"compiler\.sortmemory" : 327680,
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.6.regex
b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.6.regex
index e2d166b9ad..13bbeceba7 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.6.regex
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.6.regex
@@ -1 +1 @@
-/memory\D+229376/
\ No newline at end of file
+/memory\D+240844/
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.7.regex
b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.7.regex
index e2d166b9ad..13bbeceba7 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.7.regex
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.7.regex
@@ -1 +1 @@
-/memory\D+229376/
\ No newline at end of file
+/memory\D+240844/
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/jobs/jobs.2.regex
b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/jobs/jobs.2.regex
index e2d166b9ad..13bbeceba7 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/jobs/jobs.2.regex
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/jobs/jobs.2.regex
@@ -1 +1 @@
-/memory\D+229376/
\ No newline at end of file
+/memory\D+240844/
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/ping/ping.2.regex
b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/ping/ping.2.regex
index e2d166b9ad..13bbeceba7 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/ping/ping.2.regex
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/ping/ping.2.regex
@@ -1 +1 @@
-/memory\D+229376/
\ No newline at end of file
+/memory\D+240844/
\ No newline at end of file
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
index 85b80a6136..edd25bf0f3 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
@@ -22,6 +22,7 @@ import static
org.apache.hyracks.control.common.config.OptionTypes.BOOLEAN;
import static org.apache.hyracks.control.common.config.OptionTypes.INTEGER;
import static
org.apache.hyracks.control.common.config.OptionTypes.INTEGER_BYTE_UNIT;
import static
org.apache.hyracks.control.common.config.OptionTypes.LONG_BYTE_UNIT;
+import static
org.apache.hyracks.control.common.config.OptionTypes.NONNEGATIVE_INTEGER;
import static
org.apache.hyracks.control.common.config.OptionTypes.POSITIVE_INTEGER;
import static org.apache.hyracks.control.common.config.OptionTypes.STRING;
import static org.apache.hyracks.util.StorageUtil.StorageUnit.KILOBYTE;
@@ -124,7 +125,11 @@ public class CompilerProperties extends AbstractProperties
{
AlgebricksConfig.COLUMN_FILTER_DEFAULT,
"Enable/disable the use of column min/max filters"),
//TODO(DB): remove after
- COMPILER_ENABLE_DB_RESOLUTION(BOOLEAN, true, "Enable/disable the
resolution of namespaces to database");
+ COMPILER_ENABLE_DB_RESOLUTION(BOOLEAN, true, "Enable/disable the
resolution of namespaces to database"),
+ COMPILER_RUNTIME_MEMORY_OVERHEAD(
+ NONNEGATIVE_INTEGER,
+ 5,
+ "A percentage of the job's required memory to be added to
account for runtime memory overhead");
private final IOptionType type;
private final Object defaultValue;
@@ -207,8 +212,6 @@ public class CompilerProperties extends AbstractProperties {
public static final String COMPILER_COLUMN_FILTER_KEY =
Option.COMPILER_COLUMN_FILTER.ini();
- public static final String COMPILER_ENABLE_DB_RESOLUTION_KEY =
Option.COMPILER_ENABLE_DB_RESOLUTION.ini();
-
public static final int COMPILER_PARALLELISM_AS_STORAGE = 0;
public CompilerProperties(PropertiesAccessor accessor) {
@@ -312,7 +315,7 @@ public class CompilerProperties extends AbstractProperties {
return accessor.getBoolean(Option.COMPILER_COLUMN_FILTER);
}
- public boolean isDbResolutionEnabled() {
- return accessor.getBoolean(Option.COMPILER_ENABLE_DB_RESOLUTION);
+ public int getRuntimeMemoryOverheadPercentage() {
+ return accessor.getInt(Option.COMPILER_RUNTIME_MEMORY_OVERHEAD);
}
}
