This is an automated email from the ASF dual-hosted git repository.
kirs pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-dolphinscheduler.git
The following commit(s) were added to refs/heads/dev by this push:
new 2d8c4ec [Improvement][Worker-Task] Python Command (#5036)
2d8c4ec is described below
commit 2d8c4ec86bdc8d00baee4066f40699078735c5dc
Author: 游侠 <[email protected]>
AuthorDate: Mon Mar 15 10:22:13 2021 +0800
[Improvement][Worker-Task] Python Command (#5036)
* create the interpreter for python command
* Update README.md
Co-authored-by: zhouhong <[email protected]>
---
docker/build/README.md | 2 +-
docker/build/README_zh_CN.md | 2 +-
.../conf/dolphinscheduler/env/dolphinscheduler_env.sh.tpl | 2 +-
docker/build/startup-init-conf.sh | 2 +-
docker/docker-swarm/docker-compose.yml | 2 +-
docker/docker-swarm/docker-stack.yml | 2 +-
docker/kubernetes/dolphinscheduler/values.yaml | 4 ++--
.../server/worker/task/PythonCommandExecutor.java | 14 +++++---------
.../apache/dolphinscheduler/server/worker/EnvFileTest.java | 3 ++-
script/env/dolphinscheduler_env.sh | 2 +-
10 files changed, 16 insertions(+), 19 deletions(-)
diff --git a/docker/build/README.md b/docker/build/README.md
index 69273f9..4d3ac87 100644
--- a/docker/build/README.md
+++ b/docker/build/README.md
@@ -193,7 +193,7 @@ This environment variable sets `SPARK_HOME2`. The default
value is `/opt/soft/sp
**`PYTHON_HOME`**
-This environment variable sets `PYTHON_HOME`. The default value is
`/usr/bin/python`.
+This environment variable sets `PYTHON_HOME`. The default value is `/usr`.
**`JAVA_HOME`**
diff --git a/docker/build/README_zh_CN.md b/docker/build/README_zh_CN.md
index 96bfb80..1dfc69c 100644
--- a/docker/build/README_zh_CN.md
+++ b/docker/build/README_zh_CN.md
@@ -193,7 +193,7 @@ DolphinScheduler Docker 容器通过环境变量进行配置，缺省时将会
**`PYTHON_HOME`**
-配置`dolphinscheduler`的`PYTHON_HOME`，默认值 `/usr/bin/python`。
+配置`dolphinscheduler`的`PYTHON_HOME`，默认值 `/usr`。
**`JAVA_HOME`**
diff --git a/docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh.tpl
b/docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh.tpl
index b09e497..4463a3f 100755
--- a/docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh.tpl
+++ b/docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh.tpl
@@ -25,4 +25,4 @@ export HIVE_HOME=$HIVE_HOME
export FLINK_HOME=$FLINK_HOME
export DATAX_HOME=$DATAX_HOME
-export
PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH:$FLINK_HOME/bin:$DATAX_HOME:$PATH
+export
PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH:$FLINK_HOME/bin:$DATAX_HOME:$PATH
diff --git a/docker/build/startup-init-conf.sh
b/docker/build/startup-init-conf.sh
index 9549126..b16100c 100755
--- a/docker/build/startup-init-conf.sh
+++ b/docker/build/startup-init-conf.sh
@@ -42,7 +42,7 @@ export HADOOP_HOME=${HADOOP_HOME:-"/opt/soft/hadoop"}
export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-"/opt/soft/hadoop/etc/hadoop"}
export SPARK_HOME1=${SPARK_HOME1:-"/opt/soft/spark1"}
export SPARK_HOME2=${SPARK_HOME2:-"/opt/soft/spark2"}
-export PYTHON_HOME=${PYTHON_HOME:-"/usr/bin/python"}
+export PYTHON_HOME=${PYTHON_HOME:-"/usr"}
export JAVA_HOME=${JAVA_HOME:-"/usr/lib/jvm/java-1.8-openjdk"}
export HIVE_HOME=${HIVE_HOME:-"/opt/soft/hive"}
export FLINK_HOME=${FLINK_HOME:-"/opt/soft/flink"}
diff --git a/docker/docker-swarm/docker-compose.yml
b/docker/docker-swarm/docker-compose.yml
index e860170..5b2cf46 100644
--- a/docker/docker-swarm/docker-compose.yml
+++ b/docker/docker-swarm/docker-compose.yml
@@ -177,7 +177,7 @@ services:
HADOOP_CONF_DIR: "/opt/soft/hadoop/etc/hadoop"
SPARK_HOME1: "/opt/soft/spark1"
SPARK_HOME2: "/opt/soft/spark2"
- PYTHON_HOME: "/usr/bin/python"
+ #PYTHON_HOME: "/opt/soft/python"
JAVA_HOME: "/usr/lib/jvm/java-1.8-openjdk"
HIVE_HOME: "/opt/soft/hive"
FLINK_HOME: "/opt/soft/flink"
diff --git a/docker/docker-swarm/docker-stack.yml
b/docker/docker-swarm/docker-stack.yml
index 094890d..aab44a3 100644
--- a/docker/docker-swarm/docker-stack.yml
+++ b/docker/docker-swarm/docker-stack.yml
@@ -171,7 +171,7 @@ services:
HADOOP_CONF_DIR: "/opt/soft/hadoop/etc/hadoop"
SPARK_HOME1: "/opt/soft/spark1"
SPARK_HOME2: "/opt/soft/spark2"
- PYTHON_HOME: "/usr/bin/python"
+ #PYTHON_HOME: "/opt/soft/python"
JAVA_HOME: "/usr/lib/jvm/java-1.8-openjdk"
HIVE_HOME: "/opt/soft/hive"
FLINK_HOME: "/opt/soft/flink"
diff --git a/docker/kubernetes/dolphinscheduler/values.yaml
b/docker/kubernetes/dolphinscheduler/values.yaml
index 896ae77..ba6b8eb 100644
--- a/docker/kubernetes/dolphinscheduler/values.yaml
+++ b/docker/kubernetes/dolphinscheduler/values.yaml
@@ -76,12 +76,12 @@ common:
- "export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop"
- "export SPARK_HOME1=/opt/soft/spark1"
- "export SPARK_HOME2=/opt/soft/spark2"
- - "export PYTHON_HOME=/usr/bin/python"
+ #- "export PYTHON_HOME=/opt/soft/python"
- "export JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk"
- "export HIVE_HOME=/opt/soft/hive"
- "export FLINK_HOME=/opt/soft/flink"
- "export DATAX_HOME=/opt/soft/datax/bin/datax.py"
- - "export
PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$PATH"
+ - "export
PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$PATH"
DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler"
RESOURCE_STORAGE_TYPE: "HDFS"
RESOURCE_UPLOAD_PATH: "/dolphinscheduler"
diff --git
a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/PythonCommandExecutor.java
b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/PythonCommandExecutor.java
index 344d00f..8340b78 100644
---
a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/PythonCommandExecutor.java
+++
b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/PythonCommandExecutor.java
@@ -109,8 +109,8 @@ public class PythonCommandExecutor extends
AbstractCommandExecutor {
}
/**
- * get python home
- * @return python home
+ * Gets the command path to which Python can execute
+ * @return python command path
*/
@Override
protected String commandInterpreter() {
@@ -118,21 +118,17 @@ public class PythonCommandExecutor extends
AbstractCommandExecutor {
if (StringUtils.isEmpty(pythonHome)){
return PYTHON;
}
- return pythonHome;
+ return pythonHome + "/bin/python";
}
/**
- * get the absolute path of the Python command
+ * get the absolute path of the Python are installed
* note :
- * common.properties
- * PYTHON_HOME configured under common.properties is Python absolute
path, not PYTHON_HOME itself
*
* for example :
- * your PYTHON_HOM is /opt/python3.7/
- * you must set PYTHON_HOME is /opt/python3.7/python under nder
common.properties
- * dolphinscheduler.env.path file.
+ * your PYTHON_HOM is /opt/python3.7
*
* @param envPath env path
* @return python home
diff --git
a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/EnvFileTest.java
b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/EnvFileTest.java
index dc2a600..34f5d2f 100644
---
a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/EnvFileTest.java
+++
b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/EnvFileTest.java
@@ -17,6 +17,7 @@
package org.apache.dolphinscheduler.server.worker;
+import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import java.io.BufferedReader;
@@ -51,7 +52,7 @@ public class EnvFileTest {
try {
br = new BufferedReader(new InputStreamReader(new
FileInputStream(path)));
while ((line = br.readLine()) != null){
- if (line.contains("PYTHON_HOME")){
+ if (line.contains(Constants.PYTHON_HOME)) {
sb.append(line);
break;
}
diff --git a/script/env/dolphinscheduler_env.sh
b/script/env/dolphinscheduler_env.sh
index 026b620..9fb9f55 100755
--- a/script/env/dolphinscheduler_env.sh
+++ b/script/env/dolphinscheduler_env.sh
@@ -25,4 +25,4 @@ export HIVE_HOME=/opt/soft/hive
export FLINK_HOME=/opt/soft/flink
export DATAX_HOME=/opt/soft/datax/bin/datax.py
-export
PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH:$FLINK_HOME/bin:$DATAX_HOME:$PATH
+export
PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH:$FLINK_HOME/bin:$DATAX_HOME:$PATH
