This is an automated email from the ASF dual-hosted git repository.
wenhemin pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-dolphinscheduler.git
The following commit(s) were added to refs/heads/dev by this push:
new 1cd62b4 [Improvement][Datax] Re-assign DATAX_HOME to /opt/soft/datax
instead of /opt/soft/datax/bin/datax.py (#5019)
1cd62b4 is described below
commit 1cd62b4a5e6d5e8ac3adf8b7e2ffd61a8e87f71b
Author: Shiwen Cheng <[email protected]>
AuthorDate: Wed Mar 17 14:38:04 2021 +0800
[Improvement][Datax] Re-assign DATAX_HOME to /opt/soft/datax instead of
/opt/soft/datax/bin/datax.py (#5019)
---
.../DOLPHIN/1.3.0/configuration/dolphin-env.xml | 3 ++-
.../DOLPHIN/1.3.3/configuration/dolphin-env.xml | 3 ++-
docker/build/README.md | 2 +-
docker/build/README_zh_CN.md | 2 +-
.../conf/dolphinscheduler/env/dolphinscheduler_env.sh.tpl | 2 +-
docker/build/startup-init-conf.sh | 2 +-
docker/docker-swarm/docker-compose.yml | 2 +-
docker/docker-swarm/docker-stack.yml | 2 +-
docker/kubernetes/dolphinscheduler/values.yaml | 4 ++--
.../dolphinscheduler/common/task/DataxParametersTest.java | 4 ++--
.../dolphinscheduler/server/worker/task/datax/DataxTask.java | 10 +++++-----
script/env/dolphinscheduler_env.sh | 4 ++--
12 files changed, 21 insertions(+), 19 deletions(-)
diff --git
a/ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-env.xml
b/ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-env.xml
index 8e14716..18501c6 100644
--- a/ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-env.xml
+++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-env.xml
@@ -112,7 +112,8 @@ export SPARK_HOME2=/opt/soft/spark2
export PYTHON_HOME=/opt/soft/python
export JAVA_HOME=/opt/soft/java
export HIVE_HOME=/opt/soft/hive
-export FLINK_HOME=/opt/soft/flink</value>
+export FLINK_HOME=/opt/soft/flink
+export DATAX_HOME=/opt/soft/datax</value>
<value-attributes>
<type>content</type>
<empty-value-valid>false</empty-value-valid>
diff --git
a/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-env.xml
b/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-env.xml
index 8e14716..18501c6 100644
--- a/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-env.xml
+++ b/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-env.xml
@@ -112,7 +112,8 @@ export SPARK_HOME2=/opt/soft/spark2
export PYTHON_HOME=/opt/soft/python
export JAVA_HOME=/opt/soft/java
export HIVE_HOME=/opt/soft/hive
-export FLINK_HOME=/opt/soft/flink</value>
+export FLINK_HOME=/opt/soft/flink
+export DATAX_HOME=/opt/soft/datax</value>
<value-attributes>
<type>content</type>
<empty-value-valid>false</empty-value-valid>
diff --git a/docker/build/README.md b/docker/build/README.md
index 4d3ac87..5a1b28a 100644
--- a/docker/build/README.md
+++ b/docker/build/README.md
@@ -209,7 +209,7 @@ This environment variable sets `FLINK_HOME`. The default
value is `/opt/soft/fli
**`DATAX_HOME`**
-This environment variable sets `DATAX_HOME`. The default value is
`/opt/soft/datax/bin/datax.py`.
+This environment variable sets `DATAX_HOME`. The default value is
`/opt/soft/datax`.
**`DOLPHINSCHEDULER_DATA_BASEDIR_PATH`**
diff --git a/docker/build/README_zh_CN.md b/docker/build/README_zh_CN.md
index 1dfc69c..f3b8ea7 100644
--- a/docker/build/README_zh_CN.md
+++ b/docker/build/README_zh_CN.md
@@ -209,7 +209,7 @@ DolphinScheduler Docker 容器通过环境变量进行配置，缺省时将会
**`DATAX_HOME`**
-配置`dolphinscheduler`的`DATAX_HOME`，默认值 `/opt/soft/datax/bin/datax。py`。
+配置`dolphinscheduler`的`DATAX_HOME`，默认值 `/opt/soft/datax`。
**`DOLPHINSCHEDULER_DATA_BASEDIR_PATH`**
diff --git a/docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh.tpl
b/docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh.tpl
index 4463a3f..b9b04eb 100755
--- a/docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh.tpl
+++ b/docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh.tpl
@@ -25,4 +25,4 @@ export HIVE_HOME=$HIVE_HOME
export FLINK_HOME=$FLINK_HOME
export DATAX_HOME=$DATAX_HOME
-export
PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH:$FLINK_HOME/bin:$DATAX_HOME:$PATH
+export
PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$PATH
diff --git a/docker/build/startup-init-conf.sh
b/docker/build/startup-init-conf.sh
index b16100c..80baa92 100755
--- a/docker/build/startup-init-conf.sh
+++ b/docker/build/startup-init-conf.sh
@@ -46,7 +46,7 @@ export PYTHON_HOME=${PYTHON_HOME:-"/usr"}
export JAVA_HOME=${JAVA_HOME:-"/usr/lib/jvm/java-1.8-openjdk"}
export HIVE_HOME=${HIVE_HOME:-"/opt/soft/hive"}
export FLINK_HOME=${FLINK_HOME:-"/opt/soft/flink"}
-export DATAX_HOME=${DATAX_HOME:-"/opt/soft/datax/bin/datax.py"}
+export DATAX_HOME=${DATAX_HOME:-"/opt/soft/datax"}
# common env
export
DOLPHINSCHEDULER_DATA_BASEDIR_PATH=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH:-"/tmp/dolphinscheduler"}
export DOLPHINSCHEDULER_OPTS=${DOLPHINSCHEDULER_OPTS:-""}
diff --git a/docker/docker-swarm/docker-compose.yml
b/docker/docker-swarm/docker-compose.yml
index 5b2cf46..8f6a0b8 100644
--- a/docker/docker-swarm/docker-compose.yml
+++ b/docker/docker-swarm/docker-compose.yml
@@ -181,7 +181,7 @@ services:
JAVA_HOME: "/usr/lib/jvm/java-1.8-openjdk"
HIVE_HOME: "/opt/soft/hive"
FLINK_HOME: "/opt/soft/flink"
- DATAX_HOME: "/opt/soft/datax/bin/datax.py"
+ DATAX_HOME: "/opt/soft/datax"
DOLPHINSCHEDULER_DATA_BASEDIR_PATH: /tmp/dolphinscheduler
DOLPHINSCHEDULER_OPTS: "-Xms1g -Xmx1g -Xmn512m"
DATABASE_TYPE: postgresql
diff --git a/docker/docker-swarm/docker-stack.yml
b/docker/docker-swarm/docker-stack.yml
index aab44a3..d78e726 100644
--- a/docker/docker-swarm/docker-stack.yml
+++ b/docker/docker-swarm/docker-stack.yml
@@ -175,7 +175,7 @@ services:
JAVA_HOME: "/usr/lib/jvm/java-1.8-openjdk"
HIVE_HOME: "/opt/soft/hive"
FLINK_HOME: "/opt/soft/flink"
- DATAX_HOME: "/opt/soft/datax/bin/datax.py"
+ DATAX_HOME: "/opt/soft/datax"
DOLPHINSCHEDULER_DATA_BASEDIR_PATH: /tmp/dolphinscheduler
DOLPHINSCHEDULER_OPTS: "-Xms1g -Xmx1g -Xmn512m"
DATABASE_TYPE: postgresql
diff --git a/docker/kubernetes/dolphinscheduler/values.yaml
b/docker/kubernetes/dolphinscheduler/values.yaml
index ba6b8eb..af7da46 100644
--- a/docker/kubernetes/dolphinscheduler/values.yaml
+++ b/docker/kubernetes/dolphinscheduler/values.yaml
@@ -80,8 +80,8 @@ common:
- "export JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk"
- "export HIVE_HOME=/opt/soft/hive"
- "export FLINK_HOME=/opt/soft/flink"
- - "export DATAX_HOME=/opt/soft/datax/bin/datax.py"
- - "export
PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$PATH"
+ - "export DATAX_HOME=/opt/soft/datax"
+ - "export
PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$PATH"
DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler"
RESOURCE_STORAGE_TYPE: "HDFS"
RESOURCE_UPLOAD_PATH: "/dolphinscheduler"
diff --git
a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/DataxParametersTest.java
b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/DataxParametersTest.java
index d6e2f69..dd11aab 100644
---
a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/DataxParametersTest.java
+++
b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/DataxParametersTest.java
@@ -27,7 +27,7 @@ public class DataxParametersTest {
/**
* jvm parameters
*/
- public static final String JVM_EVN = " --jvm=\"-Xms%sG -Xmx%sG\" ";
+ public static final String JVM_PARAM = " --jvm=\"-Xms%sG -Xmx%sG\" ";
@Test
public void testLoadJvmEnv() {
@@ -88,6 +88,6 @@ public class DataxParametersTest {
public String loadJvmEnvTest(DataxParameters dataXParameters) {
int xms = dataXParameters.getXms() < 1 ? 1 : dataXParameters.getXms();
int xmx = dataXParameters.getXmx() < 1 ? 1 : dataXParameters.getXmx();
- return String.format(JVM_EVN, xms, xmx);
+ return String.format(JVM_PARAM, xms, xmx);
}
}
diff --git
a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java
b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java
index caf4879..8f0b20d 100755
---
a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java
+++
b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java
@@ -81,15 +81,15 @@ public class DataxTask extends AbstractTask {
/**
* jvm parameters
*/
- public static final String JVM_EVN = " --jvm=\"-Xms%sG -Xmx%sG\" ";
+ public static final String JVM_PARAM = " --jvm=\"-Xms%sG -Xmx%sG\" ";
/**
* python process(datax only supports version 2.7 by default)
*/
private static final String DATAX_PYTHON = "python2.7";
/**
- * datax home path
+ * datax path
*/
- private static final String DATAX_HOME_EVN = "${DATAX_HOME}";
+ private static final String DATAX_PATH = "${DATAX_HOME}/bin/datax.py";
/**
* datax channel count
*/
@@ -396,7 +396,7 @@ public class DataxTask extends AbstractTask {
StringBuilder sbr = new StringBuilder();
sbr.append(DATAX_PYTHON);
sbr.append(" ");
- sbr.append(DATAX_HOME_EVN);
+ sbr.append(DATAX_PATH);
sbr.append(" ");
sbr.append(loadJvmEnv(dataXParameters));
sbr.append(jobConfigFilePath);
@@ -424,7 +424,7 @@ public class DataxTask extends AbstractTask {
public String loadJvmEnv(DataxParameters dataXParameters) {
int xms = dataXParameters.getXms() < 1 ? 1 : dataXParameters.getXms();
int xmx = dataXParameters.getXmx() < 1 ? 1 : dataXParameters.getXmx();
- return String.format(JVM_EVN, xms, xmx);
+ return String.format(JVM_PARAM, xms, xmx);
}
/**
diff --git a/script/env/dolphinscheduler_env.sh
b/script/env/dolphinscheduler_env.sh
index 9fb9f55..e62f54e 100755
--- a/script/env/dolphinscheduler_env.sh
+++ b/script/env/dolphinscheduler_env.sh
@@ -23,6 +23,6 @@ export PYTHON_HOME=/opt/soft/python
export JAVA_HOME=/opt/soft/java
export HIVE_HOME=/opt/soft/hive
export FLINK_HOME=/opt/soft/flink
-export DATAX_HOME=/opt/soft/datax/bin/datax.py
+export DATAX_HOME=/opt/soft/datax
-export
PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH:$FLINK_HOME/bin:$DATAX_HOME:$PATH
+export
PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$PATH
