This is an automated email from the ASF dual-hosted git repository.
tcodehuber028 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/amoro.git
The following commit(s) were added to refs/heads/master by this push:
new 345e28bfe [Hotfix] Unlock some configs of the spark optimizer (#2950)
345e28bfe is described below
commit 345e28bfe12c5e25ba90be1f241a90b6cb72a3b1
Author: tcodehuber <[email protected]>
AuthorDate: Thu Jun 20 14:34:21 2024 +0800
[Hotfix] Unlock some configs of the spark optimizer (#2950)
---
.github/ISSUE_TEMPLATE/bug_report.yml | 2 +-
.../amoro/server/manager/SparkOptimizerContainer.java | 17 -----------------
charts/amoro/values.yaml | 2 +-
3 files changed, 2 insertions(+), 19 deletions(-)
diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml
b/.github/ISSUE_TEMPLATE/bug_report.yml
index caf35ff23..e73e7f73b 100644
--- a/.github/ISSUE_TEMPLATE/bug_report.yml
+++ b/.github/ISSUE_TEMPLATE/bug_report.yml
@@ -50,7 +50,7 @@ body:
- type: dropdown
id: table-format
attributes:
- label: What table format are you seeing the problem on?
+ label: What table formats are you seeing the problem on?
multiple: true
options:
- Iceberg
diff --git
a/amoro-ams/amoro-ams-server/src/main/java/org/apache/amoro/server/manager/SparkOptimizerContainer.java
b/amoro-ams/amoro-ams-server/src/main/java/org/apache/amoro/server/manager/SparkOptimizerContainer.java
index 1c708f6d5..fe6b6cad8 100644
---
a/amoro-ams/amoro-ams-server/src/main/java/org/apache/amoro/server/manager/SparkOptimizerContainer.java
+++
b/amoro-ams/amoro-ams-server/src/main/java/org/apache/amoro/server/manager/SparkOptimizerContainer.java
@@ -157,8 +157,6 @@ public class SparkOptimizerContainer extends
AbstractResourceContainer {
if (deployedOnKubernetes()) {
addKubernetesProperties(resource, resourceSparkConf);
- } else {
- addYarnProperties(resourceSparkConf);
}
String sparkOptions = resourceSparkConf.toConfOptions();
String proxyUser =
@@ -224,16 +222,6 @@ public class SparkOptimizerContainer extends
AbstractResourceContainer {
SparkConfKeys.KUBERNETES_EXECUTOR_LABEL_PREFIX + "optimizer-id",
resource.getResourceId());
}
- private void addYarnProperties(SparkOptimizerContainer.SparkConf sparkConf) {
- // Load optimizer jar first
- sparkConf.putToOptions(
- SparkConfKeys.DRIVER_CLASSPATH_INCLUDE_USER_JAR,
-
SparkOptimizerContainer.SparkConfKeys.CLASSPATH_INCLUDE_USER_JAR_DEFAULT);
- sparkConf.putToOptions(
- SparkConfKeys.EXECUTOR_CLASSPATH_INCLUDE_USER_JAR,
-
SparkOptimizerContainer.SparkConfKeys.CLASSPATH_INCLUDE_USER_JAR_DEFAULT);
- }
-
private <T> T fetchCommandOutput(Process exec, Function<String, T>
commandReader) {
T value = null;
try (InputStreamReader inputStreamReader = new
InputStreamReader(exec.getInputStream())) {
@@ -337,11 +325,6 @@ public class SparkOptimizerContainer extends
AbstractResourceContainer {
}
public static class SparkConfKeys {
- public static final String DRIVER_CLASSPATH_INCLUDE_USER_JAR =
- "spark.driver.userClassPathFirst";
- public static final String EXECUTOR_CLASSPATH_INCLUDE_USER_JAR =
- "spark.executor.userClassPathFirst";
- public static final String CLASSPATH_INCLUDE_USER_JAR_DEFAULT = "true";
public static final String KUBERNETES_IMAGE_REF =
"spark.kubernetes.container.image";
public static final String KUBERNETES_DRIVER_NAME =
"spark.kubernetes.driver.pod.name";
public static final String KUBERNETES_NAMESPACE =
"spark.kubernetes.namespace";
diff --git a/charts/amoro/values.yaml b/charts/amoro/values.yaml
index e32f406af..ae085dbee 100644
--- a/charts/amoro/values.yaml
+++ b/charts/amoro/values.yaml
@@ -266,7 +266,7 @@ optimizer:
pullPolicy: "IfNotPresent"
extra: []
-## Configure the plugin that allows you to access the metric reporters and
events listeners.
+## Configure the plugin that allows you to access the metric reporters and
event listeners.
##
plugin:
##
