This is an automated email from the ASF dual-hosted git repository.
kirs pushed a commit to branch 1.3.7-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/1.3.7-prepare by this push:
new 6679492 [1.3.7-prepare][Fix-5570][Worker] Fix worker.groups in
worker.properties is still commented and improve install scripts and
configurations (#5899)
6679492 is described below
commit 6679492a436cb92c205601dce52605a2c8f0dce7
Author: Shiwen Cheng <[email protected]>
AuthorDate: Fri Jul 30 15:23:47 2021 +0800
[1.3.7-prepare][Fix-5570][Worker] Fix worker.groups in worker.properties is
still commented and improve install scripts and configurations (#5899)
* [Improvement][Install] Add resourceManagerHttpAddressPort in install.sh
and install_config.conf
* [Improvement][Install] Add dataBasedirPath and kerberosExpireTime in
install.sh and install_config.conf
* [Fix-5570][Worker] Fix worker.groups in worker.properties is still
commented
---
.../apache/dolphinscheduler/common/Constants.java | 27 ++++++++--------------
.../src/main/resources/config/install_config.conf | 14 +++++++----
.../src/main/resources/zookeeper.properties | 2 +-
install.sh | 13 +++++++----
script/scp-hosts.sh | 2 +-
5 files changed, 29 insertions(+), 29 deletions(-)
diff --git
a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
index 864af98..f56b409 100644
---
a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
+++
b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
@@ -93,6 +93,15 @@ public final class Constants {
/**
+ * hadoop configuration
+ */
+ public static final String HADOOP_RM_STATE_ACTIVE = "ACTIVE";
+
+ public static final String HADOOP_RM_STATE_STANDBY = "STANDBY";
+
+ public static final String HADOOP_RESOURCE_MANAGER_HTTPADDRESS_PORT =
"resource.manager.httpaddress.port";
+
+ /**
* yarn.resourcemanager.ha.rm.ids
*/
public static final String YARN_RESOURCEMANAGER_HA_RM_IDS =
"yarn.resourcemanager.ha.rm.ids";
@@ -258,15 +267,6 @@ public final class Constants {
public static final String AT_SIGN = "@";
- public static final String WORKER_MAX_CPULOAD_AVG =
"worker.max.cpuload.avg";
-
- public static final String WORKER_RESERVED_MEMORY =
"worker.reserved.memory";
-
- public static final String MASTER_MAX_CPULOAD_AVG =
"master.max.cpuload.avg";
-
- public static final String MASTER_RESERVED_MEMORY =
"master.reserved.memory";
-
-
/**
* date format of yyyy-MM-dd HH:mm:ss
*/
@@ -466,15 +466,6 @@ public final class Constants {
*/
public static final String CMDPARAM_COMPLEMENT_DATA_END_DATE =
"complementEndDate";
- /**
- * hadoop configuration
- */
- public static final String HADOOP_RM_STATE_ACTIVE = "ACTIVE";
-
- public static final String HADOOP_RM_STATE_STANDBY = "STANDBY";
-
- public static final String HADOOP_RESOURCE_MANAGER_HTTPADDRESS_PORT =
"resource.manager.httpaddress.port";
-
/**
* data source config
diff --git
a/dolphinscheduler-server/src/main/resources/config/install_config.conf
b/dolphinscheduler-server/src/main/resources/config/install_config.conf
index 236617f..f722a8c 100644
--- a/dolphinscheduler-server/src/main/resources/config/install_config.conf
+++ b/dolphinscheduler-server/src/main/resources/config/install_config.conf
@@ -73,10 +73,15 @@ sslEnable="false"
#note: sslTrust is the same as mailServerHost
sslTrust="smtp.exmail.qq.com"
+# user data local directory path, please make sure the directory exists and
have read write permissions
+dataBasedirPath="/tmp/dolphinscheduler"
# resource storage type: HDFS, S3, NONE
resourceStorageType="NONE"
+# resource store on HDFS/S3 path, resource file will store to this hadoop hdfs
path, self configuration, please make sure the directory exists on hdfs and
have read write permissions. "/dolphinscheduler" is recommended
+resourceUploadPath="/dolphinscheduler"
+
# if resourceStorageType is HDFS，defaultFS write namenode address，HA you need
to put core-site.xml and hdfs-site.xml in the conf directory.
# if S3，write S3 address，HA，for example ：s3a://dolphinscheduler，
# Note，s3 be sure to create the root directory /dolphinscheduler
@@ -87,15 +92,15 @@ s3Endpoint="http://192.168.xx.xx:9010";
s3AccessKey="xxxxxxxxxx"
s3SecretKey="xxxxxxxxxx"
+# resourcemanager port, the default value is 8088 if not specified
+resourceManagerHttpAddressPort="8088"
+
# if resourcemanager HA is enabled, please set the HA IPs; if resourcemanager
is single, keep this value empty
yarnHaIps="192.168.xx.xx,192.168.xx.xx"
# if resourcemanager HA is enabled or not use resourcemanager, please keep the
default value; If resourcemanager is single, you only need to replace ds1 to
actual resourcemanager hostname
singleYarnIp="yarnIp1"
-# resource store on HDFS/S3 path, resource file will store to this hadoop hdfs
path, self configuration, please make sure the directory exists on hdfs and
have read write permissions. "/dolphinscheduler" is recommended
-resourceUploadPath="/dolphinscheduler"
-
# who have permissions to create directory under HDFS/S3 root path
# Note: if kerberos is enabled, please config hdfsRootUser=
hdfsRootUser="hdfs"
@@ -109,7 +114,8 @@ krb5ConfPath="$installPath/conf/krb5.conf"
keytabUserName="[email protected]"
# username keytab path
keytabPath="$installPath/conf/hdfs.headless.keytab"
-
+# kerberos expire time, the unit is hour
+kerberosExpireTime="2"
# api server port
apiServerPort="12345"
diff --git a/dolphinscheduler-service/src/main/resources/zookeeper.properties
b/dolphinscheduler-service/src/main/resources/zookeeper.properties
index 006b120..ad1fb8e 100644
--- a/dolphinscheduler-service/src/main/resources/zookeeper.properties
+++ b/dolphinscheduler-service/src/main/resources/zookeeper.properties
@@ -19,7 +19,7 @@
zookeeper.quorum=localhost:2181
# dolphinscheduler root directory
-#zookeeper.dolphinscheduler.root=/dolphinscheduler
+zookeeper.dolphinscheduler.root=/dolphinscheduler
# dolphinscheduler failover directory
#zookeeper.session.timeout=60000
diff --git a/install.sh b/install.sh
index a74fe67..df0dd9e 100755
--- a/install.sh
+++ b/install.sh
@@ -34,24 +34,27 @@ datasourceDriverClassname="com.mysql.jdbc.Driver"
if [ $dbtype == "postgresql" ];then
datasourceDriverClassname="org.postgresql.Driver"
fi
-sed -i ${txt}
"s#spring.datasource.driver-class-name.*#spring.datasource.driver-class-name=${datasourceDriverClassname}#g"
conf/datasource.properties
-sed -i ${txt}
"s#spring.datasource.url.*#spring.datasource.url=jdbc:${dbtype}://${dbhost}/${dbname}?characterEncoding=UTF-8\&allowMultiQueries=true#g"
conf/datasource.properties
-sed -i ${txt}
"s#spring.datasource.username.*#spring.datasource.username=${username}#g"
conf/datasource.properties
-sed -i ${txt}
"s#spring.datasource.password.*#spring.datasource.password=${password}#g"
conf/datasource.properties
+sed -i ${txt}
"s#^spring.datasource.driver-class-name.*#spring.datasource.driver-class-name=${datasourceDriverClassname}#g"
conf/datasource.properties
+sed -i ${txt}
"s#^spring.datasource.url.*#spring.datasource.url=jdbc:${dbtype}://${dbhost}/${dbname}?characterEncoding=UTF-8\&allowMultiQueries=true#g"
conf/datasource.properties
+sed -i ${txt}
"s#^spring.datasource.username.*#spring.datasource.username=${username}#g"
conf/datasource.properties
+sed -i ${txt}
"s#^spring.datasource.password.*#spring.datasource.password=${password}#g"
conf/datasource.properties
sed -i ${txt} "s#fs.defaultFS.*#fs.defaultFS=${defaultFS}#g"
conf/common.properties
sed -i ${txt} "s#fs.s3a.endpoint.*#fs.s3a.endpoint=${s3Endpoint}#g"
conf/common.properties
sed -i ${txt} "s#fs.s3a.access.key.*#fs.s3a.access.key=${s3AccessKey}#g"
conf/common.properties
sed -i ${txt} "s#fs.s3a.secret.key.*#fs.s3a.secret.key=${s3SecretKey}#g"
conf/common.properties
+sed -i ${txt}
"s#resource.manager.httpaddress.port.*#resource.manager.httpaddress.port=${resourceManagerHttpAddressPort}#g"
conf/common.properties
sed -i ${txt}
"s#yarn.resourcemanager.ha.rm.ids.*#yarn.resourcemanager.ha.rm.ids=${yarnHaIps}#g"
conf/common.properties
-sed -i ${txt}
"s#yarn.application.status.address.*#yarn.application.status.address=http://${singleYarnIp}:8088/ws/v1/cluster/apps/%s#g";
conf/common.properties
+sed -i ${txt}
"s#yarn.application.status.address.*#yarn.application.status.address=http://${singleYarnIp}:%s/ws/v1/cluster/apps/%s#g";
conf/common.properties
sed -i ${txt} "s#hdfs.root.user.*#hdfs.root.user=${hdfsRootUser}#g"
conf/common.properties
sed -i ${txt}
"s#resource.upload.path.*#resource.upload.path=${resourceUploadPath}#g"
conf/common.properties
+sed -i ${txt} "s#data.basedir.path=.*#data.basedir.path=${dataBasedirPath}#g"
conf/common.properties
sed -i ${txt}
"s#resource.storage.type.*#resource.storage.type=${resourceStorageType}#g"
conf/common.properties
sed -i ${txt}
"s#hadoop.security.authentication.startup.state.*#hadoop.security.authentication.startup.state=${kerberosStartUp}#g"
conf/common.properties
sed -i ${txt}
"s#java.security.krb5.conf.path.*#java.security.krb5.conf.path=${krb5ConfPath}#g"
conf/common.properties
sed -i ${txt}
"s#login.user.keytab.username.*#login.user.keytab.username=${keytabUserName}#g"
conf/common.properties
sed -i ${txt}
"s#login.user.keytab.path.*#login.user.keytab.path=${keytabPath}#g"
conf/common.properties
+sed -i ${txt}
"s#kerberos.expire.time=.*#kerberos.expire.time=${kerberosExpireTime}#g"
conf/common.properties
sed -i ${txt} "s#zookeeper.quorum.*#zookeeper.quorum=${zkQuorum}#g"
conf/zookeeper.properties
sed -i ${txt} "s#server.port.*#server.port=${apiServerPort}#g"
conf/application-api.properties
sed -i ${txt} "s#mail.server.host.*#mail.server.host=${mailServerHost}#g"
conf/alert.properties
diff --git a/script/scp-hosts.sh b/script/scp-hosts.sh
index 258ebe5..21d5701 100755
--- a/script/scp-hosts.sh
+++ b/script/scp-hosts.sh
@@ -58,7 +58,7 @@ do
do
# if worker in workersGroupMap
if [[ "${workersGroupMap[${host}]}" ]] && [[ "${dsDir}" == "conf" ]]; then
- sed -i ${txt}
"s#worker.groups.*#worker.groups=${workersGroupMap[${host}]}#g"
${dsDir}/worker.properties
+ sed -i ${txt}
"s@^#\?worker.groups=.*@worker.groups=${workersGroupMap[${host}]}@g"
${dsDir}/worker.properties
fi
echo "start to scp $dsDir to $host/$installPath"
