This is an automated email from the ASF dual-hosted git repository.
journey pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-dolphinscheduler.git
The following commit(s) were added to refs/heads/dev by this push:
new 4f9d8ca Use docker-compose for deployment . Failed to execute python
script (#2539)
4f9d8ca is described below
commit 4f9d8caeed66b978cbc03007e652a5b4e8d213c0
Author: liwenhe1993 <[email protected]>
AuthorDate: Mon Apr 27 16:18:21 2020 +0800
Use docker-compose for deployment . Failed to execute python script (#2539)
* Modify docker-compose and dockerfile
1. Add python3 in Dockerfile
2. Modify startup.sh file
3. Modify docker-compose.yml and add docker-stack.yml
4. Modify dolphinscheduler_env.sh
* Add logback-xx.xml and Modify Dockerfile
---
docker/docker-swarm/docker-compose.yml | 122 ++++++---------
.../{docker-compose.yml => docker-stack.yml} | 165 +++++++++------------
docker/docker-swarm/dolphinscheduler_env.sh | 12 +-
dockerfile/Dockerfile | 3 +-
.../dolphinscheduler/env/dolphinscheduler_env.sh | 12 +-
.../dolphinscheduler/logback/logback-alert.xml | 52 +++++++
.../worker_logback.xml => logback/logback-api.xml} | 51 +++----
.../logback-master.xml} | 33 +++--
.../logback-worker.xml} | 26 ++--
dockerfile/startup.sh | 19 ++-
10 files changed, 240 insertions(+), 255 deletions(-)
diff --git a/docker/docker-swarm/docker-compose.yml
b/docker/docker-swarm/docker-compose.yml
index 20fb0cc..ffa91a0 100644
--- a/docker/docker-swarm/docker-compose.yml
+++ b/docker/docker-swarm/docker-compose.yml
@@ -16,36 +16,6 @@
version: "3.4"
-networks:
- dolphinscheduler-postgresql:
- driver: bridge
- dolphinscheduler-zookeeper:
- driver: bridge
- dolphinscheduler-api:
- driver: bridge
- dolphinscheduler-frontend:
- driver: bridge
- dolphinscheduler-alert:
- driver: bridge
- dolphinscheduler-master:
- driver: bridge
- dolphinscheduler-worker:
- driver: bridge
-
-volumes:
- dolphinscheduler-postgresql:
- dolphinscheduler-zookeeper:
- dolphinscheduler-api:
- dolphinscheduler-frontend:
- dolphinscheduler-alert:
- dolphinscheduler-master:
- dolphinscheduler-worker-data:
- dolphinscheduler-worker-logs:
-
-configs:
- dolphinscheduler-worker-task-env:
- file: ./dolphinscheduler_env.sh
-
services:
dolphinscheduler-postgresql:
@@ -58,16 +28,10 @@ services:
POSTGRESQL_USERNAME: root
POSTGRESQL_PASSWORD: root
POSTGRESQL_DATABASE: dolphinscheduler
- healthcheck:
- test: ["CMD", "pg_isready", "-U", "${POSTGRESQL_USERNAME}", "-d",
"{POSTGRESQL_PASSWORD}", "-h", "localhost", "5432"]
- interval: 30s
- timeout: 5s
- retries: 3
- # start_period: 30s
- volumes:
+ volumes:
- dolphinscheduler-postgresql:/bitnami/postgresql
networks:
- - dolphinscheduler-postgresql
+ - dolphinscheduler
dolphinscheduler-zookeeper:
image: bitnami/zookeeper:latest
@@ -77,19 +41,13 @@ services:
environment:
TZ: Asia/Shanghai
ALLOW_ANONYMOUS_LOGIN: "yes"
- healthcheck:
- test: ["CMD-SHELL", "nc -z localhost 2181"]
- interval: 30s
- timeout: 5s
- retries: 3
- # start_period: 30s
volumes:
- dolphinscheduler-zookeeper:/bitnami/zookeeper
networks:
- - dolphinscheduler-zookeeper
+ - dolphinscheduler
dolphinscheduler-api:
- image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:1.2.1
+ image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
container_name: dolphinscheduler-api
command: ["api-server"]
ports:
@@ -103,23 +61,21 @@ services:
POSTGRESQL_DATABASE: dolphinscheduler
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
healthcheck:
- test: ["CMD-SHELL", "curl -f http://localhost:12345";]
+ test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"]
interval: 30s
timeout: 5s
retries: 3
- # start_period: 30s
+ start_period: 30s
depends_on:
- dolphinscheduler-postgresql
- dolphinscheduler-zookeeper
volumes:
- dolphinscheduler-api:/opt/dolphinscheduler/logs
networks:
- - dolphinscheduler-api
- - dolphinscheduler-postgresql
- - dolphinscheduler-zookeeper
+ - dolphinscheduler
dolphinscheduler-frontend:
- image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:1.2.1
+ image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
container_name: dolphinscheduler-frontend
command: ["frontend"]
ports:
@@ -129,21 +85,20 @@ services:
FRONTEND_API_SERVER_HOST: dolphinscheduler-api
FRONTEND_API_SERVER_PORT: 12345
healthcheck:
- test: ["CMD-SHELL", "curl -f http://localhost:8888";]
+ test: ["CMD", "nc", "-z", "localhost", "8888"]
interval: 30s
timeout: 5s
retries: 3
- # start_period: 30s
+ start_period: 30s
depends_on:
- dolphinscheduler-api
volumes:
- dolphinscheduler-frontend:/var/log/nginx
networks:
- - dolphinscheduler-frontend
- - dolphinscheduler-api
+ - dolphinscheduler
dolphinscheduler-alert:
- image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:1.2.1
+ image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
container_name: dolphinscheduler-alert
command: ["alert-server"]
environment:
@@ -172,17 +127,16 @@ services:
interval: 30s
timeout: 5s
retries: 3
- # start_period: 30s
+ start_period: 30s
depends_on:
- dolphinscheduler-postgresql
volumes:
- dolphinscheduler-alert:/opt/dolphinscheduler/logs
networks:
- - dolphinscheduler-alert
- - dolphinscheduler-postgresql
+ - dolphinscheduler
dolphinscheduler-master:
- image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:1.2.1
+ image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
container_name: dolphinscheduler-master
command: ["master-server"]
ports:
@@ -207,19 +161,17 @@ services:
interval: 30s
timeout: 5s
retries: 3
- # start_period: 30s
+ start_period: 30s
depends_on:
- dolphinscheduler-postgresql
- dolphinscheduler-zookeeper
volumes:
- dolphinscheduler-master:/opt/dolphinscheduler/logs
networks:
- - dolphinscheduler-master
- - dolphinscheduler-postgresql
- - dolphinscheduler-zookeeper
+ - dolphinscheduler
dolphinscheduler-worker:
- image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:1.2.1
+ image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
container_name: dolphinscheduler-worker
command: ["worker-server"]
ports:
@@ -245,17 +197,37 @@ services:
interval: 30s
timeout: 5s
retries: 3
- # start_period: 30s
+ start_period: 30s
depends_on:
- dolphinscheduler-postgresql
- dolphinscheduler-zookeeper
- volumes:
- - dolphinscheduler-worker-data:/tmp/dolphinscheduler
- - dolphinscheduler-worker-logs:/opt/dolphinscheduler/logs
- configs:
- - source: dolphinscheduler-worker-task-env
+ volumes:
+ - type: bind
+ source: ./dolphinscheduler_env.sh
target: /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh
+ - type: volume
+ source: dolphinscheduler-worker-data
+ target: /tmp/dolphinscheduler
+ - type: volume
+ source: dolphinscheduler-worker-logs
+ target: /opt/dolphinscheduler/logs
networks:
- - dolphinscheduler-worker
- - dolphinscheduler-postgresql
- - dolphinscheduler-zookeeper
\ No newline at end of file
+ - dolphinscheduler
+
+networks:
+ dolphinscheduler:
+ driver: bridge
+
+volumes:
+ dolphinscheduler-postgresql:
+ dolphinscheduler-zookeeper:
+ dolphinscheduler-api:
+ dolphinscheduler-frontend:
+ dolphinscheduler-alert:
+ dolphinscheduler-master:
+ dolphinscheduler-worker-data:
+ dolphinscheduler-worker-logs:
+
+configs:
+ dolphinscheduler-worker-task-env:
+ file: ./dolphinscheduler_env.sh
\ No newline at end of file
diff --git a/docker/docker-swarm/docker-compose.yml
b/docker/docker-swarm/docker-stack.yml
similarity index 70%
copy from docker/docker-swarm/docker-compose.yml
copy to docker/docker-swarm/docker-stack.yml
index 20fb0cc..e628205 100644
--- a/docker/docker-swarm/docker-compose.yml
+++ b/docker/docker-swarm/docker-stack.yml
@@ -16,41 +16,10 @@
version: "3.4"
-networks:
- dolphinscheduler-postgresql:
- driver: bridge
- dolphinscheduler-zookeeper:
- driver: bridge
- dolphinscheduler-api:
- driver: bridge
- dolphinscheduler-frontend:
- driver: bridge
- dolphinscheduler-alert:
- driver: bridge
- dolphinscheduler-master:
- driver: bridge
- dolphinscheduler-worker:
- driver: bridge
-
-volumes:
- dolphinscheduler-postgresql:
- dolphinscheduler-zookeeper:
- dolphinscheduler-api:
- dolphinscheduler-frontend:
- dolphinscheduler-alert:
- dolphinscheduler-master:
- dolphinscheduler-worker-data:
- dolphinscheduler-worker-logs:
-
-configs:
- dolphinscheduler-worker-task-env:
- file: ./dolphinscheduler_env.sh
-
services:
dolphinscheduler-postgresql:
image: bitnami/postgresql:latest
- container_name: dolphinscheduler-postgresql
ports:
- 5432:5432
environment:
@@ -58,39 +27,31 @@ services:
POSTGRESQL_USERNAME: root
POSTGRESQL_PASSWORD: root
POSTGRESQL_DATABASE: dolphinscheduler
- healthcheck:
- test: ["CMD", "pg_isready", "-U", "${POSTGRESQL_USERNAME}", "-d",
"{POSTGRESQL_PASSWORD}", "-h", "localhost", "5432"]
- interval: 30s
- timeout: 5s
- retries: 3
- # start_period: 30s
- volumes:
+ volumes:
- dolphinscheduler-postgresql:/bitnami/postgresql
networks:
- - dolphinscheduler-postgresql
+ - dolphinscheduler
+ deploy:
+ mode: replicated
+ replicas: 1
dolphinscheduler-zookeeper:
image: bitnami/zookeeper:latest
- container_name: dolphinscheduler-zookeeper
ports:
- 2181:2181
environment:
TZ: Asia/Shanghai
ALLOW_ANONYMOUS_LOGIN: "yes"
- healthcheck:
- test: ["CMD-SHELL", "nc -z localhost 2181"]
- interval: 30s
- timeout: 5s
- retries: 3
- # start_period: 30s
volumes:
- dolphinscheduler-zookeeper:/bitnami/zookeeper
networks:
- - dolphinscheduler-zookeeper
+ - dolphinscheduler
+ deploy:
+ mode: replicated
+ replicas: 1
dolphinscheduler-api:
- image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:1.2.1
- container_name: dolphinscheduler-api
+ image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
command: ["api-server"]
ports:
- 12345:12345
@@ -103,24 +64,21 @@ services:
POSTGRESQL_DATABASE: dolphinscheduler
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
healthcheck:
- test: ["CMD-SHELL", "curl -f http://localhost:12345";]
- interval: 30s
+ test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"]
+ interval: 30
timeout: 5s
retries: 3
- # start_period: 30s
- depends_on:
- - dolphinscheduler-postgresql
- - dolphinscheduler-zookeeper
+ start_period: 30s
volumes:
- dolphinscheduler-api:/opt/dolphinscheduler/logs
networks:
- - dolphinscheduler-api
- - dolphinscheduler-postgresql
- - dolphinscheduler-zookeeper
+ - dolphinscheduler
+ deploy:
+ mode: replicated
+ replicas: 1
dolphinscheduler-frontend:
- image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:1.2.1
- container_name: dolphinscheduler-frontend
+ image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
command: ["frontend"]
ports:
- 8888:8888
@@ -129,22 +87,21 @@ services:
FRONTEND_API_SERVER_HOST: dolphinscheduler-api
FRONTEND_API_SERVER_PORT: 12345
healthcheck:
- test: ["CMD-SHELL", "curl -f http://localhost:8888";]
- interval: 30s
+ test: ["CMD", "nc", "-z", "localhost", "8888"]
+ interval: 30
timeout: 5s
retries: 3
- # start_period: 30s
- depends_on:
- - dolphinscheduler-api
+ start_period: 30s
volumes:
- dolphinscheduler-frontend:/var/log/nginx
networks:
- - dolphinscheduler-frontend
- - dolphinscheduler-api
+ - dolphinscheduler
+ deploy:
+ mode: replicated
+ replicas: 1
dolphinscheduler-alert:
- image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:1.2.1
- container_name: dolphinscheduler-alert
+ image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
command: ["alert-server"]
environment:
TZ: Asia/Shanghai
@@ -169,21 +126,20 @@ services:
POSTGRESQL_DATABASE: dolphinscheduler
healthcheck:
test: ["CMD", "/root/checkpoint.sh", "AlertServer"]
- interval: 30s
+ interval: 30
timeout: 5s
retries: 3
- # start_period: 30s
- depends_on:
- - dolphinscheduler-postgresql
- volumes:
+ start_period: 30s
+ volumes:
- dolphinscheduler-alert:/opt/dolphinscheduler/logs
networks:
- - dolphinscheduler-alert
- - dolphinscheduler-postgresql
+ - dolphinscheduler
+ deploy:
+ mode: replicated
+ replicas: 1
dolphinscheduler-master:
- image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:1.2.1
- container_name: dolphinscheduler-master
+ image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
command: ["master-server"]
ports:
- 5678:5678
@@ -204,23 +160,20 @@ services:
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
healthcheck:
test: ["CMD", "/root/checkpoint.sh", "MasterServer"]
- interval: 30s
+ interval: 30
timeout: 5s
retries: 3
- # start_period: 30s
- depends_on:
- - dolphinscheduler-postgresql
- - dolphinscheduler-zookeeper
- volumes:
+ start_period: 30s
+ volumes:
- dolphinscheduler-master:/opt/dolphinscheduler/logs
networks:
- - dolphinscheduler-master
- - dolphinscheduler-postgresql
- - dolphinscheduler-zookeeper
+ - dolphinscheduler
+ deploy:
+ mode: replicated
+ replicas: 1
dolphinscheduler-worker:
- image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:1.2.1
- container_name: dolphinscheduler-worker
+ image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
command: ["worker-server"]
ports:
- 1234:1234
@@ -242,20 +195,36 @@ services:
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
healthcheck:
test: ["CMD", "/root/checkpoint.sh", "WorkerServer"]
- interval: 30s
+ interval: 30
timeout: 5s
retries: 3
- # start_period: 30s
- depends_on:
- - dolphinscheduler-postgresql
- - dolphinscheduler-zookeeper
- volumes:
+ start_period: 30s
+ volumes:
- dolphinscheduler-worker-data:/tmp/dolphinscheduler
- dolphinscheduler-worker-logs:/opt/dolphinscheduler/logs
configs:
- source: dolphinscheduler-worker-task-env
target: /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh
networks:
- - dolphinscheduler-worker
- - dolphinscheduler-postgresql
- - dolphinscheduler-zookeeper
\ No newline at end of file
+ - dolphinscheduler
+ deploy:
+ mode: replicated
+ replicas: 1
+
+networks:
+ dolphinscheduler:
+ driver: overlay
+
+volumes:
+ dolphinscheduler-postgresql:
+ dolphinscheduler-zookeeper:
+ dolphinscheduler-api:
+ dolphinscheduler-frontend:
+ dolphinscheduler-alert:
+ dolphinscheduler-master:
+ dolphinscheduler-worker-data:
+ dolphinscheduler-worker-logs:
+
+configs:
+ dolphinscheduler-worker-task-env:
+ file: ./dolphinscheduler_env.sh
\ No newline at end of file
diff --git a/docker/docker-swarm/dolphinscheduler_env.sh
b/docker/docker-swarm/dolphinscheduler_env.sh
index 790e306..654318c 100644
--- a/docker/docker-swarm/dolphinscheduler_env.sh
+++ b/docker/docker-swarm/dolphinscheduler_env.sh
@@ -15,12 +15,6 @@
# limitations under the License.
#
-export HADOOP_HOME=/opt/soft/hadoop
-export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop
-export SPARK_HOME1=/opt/soft/spark1
-export SPARK_HOME2=/opt/soft/spark2
-export PYTHON_HOME=/opt/soft/python
-export JAVA_HOME=/opt/soft/java
-export HIVE_HOME=/opt/soft/hive
-export FLINK_HOME=/opt/soft/flink
-export
PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$PATH
\ No newline at end of file
+export PYTHON_HOME=/usr/bin/python2
+export JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk
+export PATH=$PYTHON_HOME:$JAVA_HOME/bin:$PATH
\ No newline at end of file
diff --git a/dockerfile/Dockerfile b/dockerfile/Dockerfile
index c48b51e..bed8f65 100644
--- a/dockerfile/Dockerfile
+++ b/dockerfile/Dockerfile
@@ -27,7 +27,7 @@ ENV DEBIAN_FRONTEND noninteractive
#If install slowly, you can replcae alpine's mirror with aliyun's mirror,
Example:
#RUN sed -i "s/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g"
/etc/apk/repositories
RUN apk update && \
- apk add dos2unix shadow bash openrc python sudo vim wget iputils net-tools
openssh-server py2-pip tini && \
+ apk add dos2unix shadow bash openrc python python3 sudo vim wget iputils
net-tools openssh-server py2-pip tini && \
apk add --update procps && \
openrc boot && \
pip install kazoo
@@ -67,6 +67,7 @@ ADD ./checkpoint.sh /root/checkpoint.sh
ADD ./startup-init-conf.sh /root/startup-init-conf.sh
ADD ./startup.sh /root/startup.sh
ADD ./conf/dolphinscheduler/*.tpl /opt/dolphinscheduler/conf/
+ADD ./conf/dolphinscheduler/logback/* /opt/dolphinscheduler/conf/
ADD conf/dolphinscheduler/env/dolphinscheduler_env.sh
/opt/dolphinscheduler/conf/env/
RUN chmod +x /root/checkpoint.sh && \
chmod +x /root/startup-init-conf.sh && \
diff --git a/dockerfile/conf/dolphinscheduler/env/dolphinscheduler_env.sh
b/dockerfile/conf/dolphinscheduler/env/dolphinscheduler_env.sh
index 070c438..78c8f98 100644
--- a/dockerfile/conf/dolphinscheduler/env/dolphinscheduler_env.sh
+++ b/dockerfile/conf/dolphinscheduler/env/dolphinscheduler_env.sh
@@ -15,12 +15,6 @@
# limitations under the License.
#
-export HADOOP_HOME=/opt/soft/hadoop
-export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop
-export SPARK_HOME1=/opt/soft/spark1
-export SPARK_HOME2=/opt/soft/spark2
-export PYTHON_HOME=/opt/soft/python
-export JAVA_HOME=/opt/soft/java
-export HIVE_HOME=/opt/soft/hive
-export FLINK_HOME=/opt/soft/flink
-export
PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$PATH
+export PYTHON_HOME=/usr/bin/python2
+export JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk
+export PATH=$PYTHON_HOME/bin:$JAVA_HOME/bin:$PATH
diff --git a/dockerfile/conf/dolphinscheduler/logback/logback-alert.xml
b/dockerfile/conf/dolphinscheduler/logback/logback-alert.xml
new file mode 100644
index 0000000..5d1c078
--- /dev/null
+++ b/dockerfile/conf/dolphinscheduler/logback/logback-alert.xml
@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<!-- Logback configuration. See http://logback.qos.ch/manual/index.html -->
+<configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" -->
+
+ <property name="log.base" value="logs"/>
+ <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>
+ [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] -
%msg%n
+ </pattern>
+ <charset>UTF-8</charset>
+ </encoder>
+ </appender>
+
+ <appender name="ALERTLOGFILE"
class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <file>${log.base}/dolphinscheduler-alert.log</file>
+ <rollingPolicy
class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+
<fileNamePattern>${log.base}/dolphinscheduler-alert.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>
+ <maxHistory>20</maxHistory>
+ <maxFileSize>64MB</maxFileSize>
+ </rollingPolicy>
+ <encoder>
+ <pattern>
+ [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] -
%msg%n
+ </pattern>
+ <charset>UTF-8</charset>
+ </encoder>
+ </appender>
+
+ <root level="INFO">
+ <appender-ref ref="STDOUT"/>
+ <appender-ref ref="APILOGFILE"/>
+ </root>
+
+</configuration>
\ No newline at end of file
diff --git a/dockerfile/conf/dolphinscheduler/conf/worker_logback.xml
b/dockerfile/conf/dolphinscheduler/logback/logback-api.xml
similarity index 53%
copy from dockerfile/conf/dolphinscheduler/conf/worker_logback.xml
copy to dockerfile/conf/dolphinscheduler/logback/logback-api.xml
index 1b09260..2df90d8 100644
--- a/dockerfile/conf/dolphinscheduler/conf/worker_logback.xml
+++ b/dockerfile/conf/dolphinscheduler/logback/logback-api.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="UTF-8" ?>
+<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
@@ -17,7 +17,8 @@
-->
<!-- Logback configuration. See http://logback.qos.ch/manual/index.html -->
-<configuration scan="true" scanPeriod="120 seconds">
+<configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" -->
+
<property name="log.base" value="logs"/>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
@@ -27,53 +28,35 @@
<charset>UTF-8</charset>
</encoder>
</appender>
- <appender name="TASKLOGFILE"
class="ch.qos.logback.classic.sift.SiftingAppender">
- <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
- <level>INFO</level>
- </filter>
- <filter
class="org.apache.dolphinscheduler.server.log.TaskLogFilter"></filter>
- <Discriminator
class="org.apache.dolphinscheduler.server.log.TaskLogDiscriminator">
- <key>taskAppId</key>
- <logBase>${log.base}</logBase>
- </Discriminator>
- <sift>
- <appender name="FILE-${taskAppId}"
class="ch.qos.logback.core.FileAppender">
- <file>${log.base}/${taskAppId}.log</file>
- <encoder>
- <pattern>
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS}
%logger{96}:[%line] - %msg%n
- </pattern>
- <charset>UTF-8</charset>
- </encoder>
- <append>true</append>
- </appender>
- </sift>
- </appender>
- <appender name="WORKERLOGFILE"
class="ch.qos.logback.core.rolling.RollingFileAppender">
- <file>${log.base}/dolphinscheduler-worker.log</file>
- <filter class="org.apache.dolphinscheduler.server.log.WorkerLogFilter">
+ <!-- api server logback config start -->
+ <appender name="APILOGFILE"
class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <file>${log.base}/dolphinscheduler-api-server.log</file>
+ <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>INFO</level>
</filter>
-
<rollingPolicy
class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
-
<fileNamePattern>${log.base}/dolphinscheduler-worker.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>
+
<fileNamePattern>${log.base}/dolphinscheduler-api-server.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>
<maxHistory>168</maxHistory>
- <maxFileSize>200MB</maxFileSize>
+ <maxFileSize>64MB</maxFileSize>
</rollingPolicy>
-
<encoder>
<pattern>
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] -
%msg%n
</pattern>
<charset>UTF-8</charset>
</encoder>
-
</appender>
+ <!-- api server logback config end -->
+
+ <logger name="org.apache.zookeeper" level="WARN"/>
+ <logger name="org.apache.hbase" level="WARN"/>
+ <logger name="org.apache.hadoop" level="WARN"/>
<root level="INFO">
- <appender-ref ref="TASKLOGFILE"/>
- <appender-ref ref="WORKERLOGFILE"/>
+ <appender-ref ref="STDOUT"/>
+ <appender-ref ref="APILOGFILE"/>
</root>
+
</configuration>
\ No newline at end of file
diff --git a/dockerfile/conf/dolphinscheduler/conf/worker_logback.xml
b/dockerfile/conf/dolphinscheduler/logback/logback-master.xml
similarity index 77%
copy from dockerfile/conf/dolphinscheduler/conf/worker_logback.xml
copy to dockerfile/conf/dolphinscheduler/logback/logback-master.xml
index 1b09260..7410c01 100644
--- a/dockerfile/conf/dolphinscheduler/conf/worker_logback.xml
+++ b/dockerfile/conf/dolphinscheduler/logback/logback-master.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="UTF-8" ?>
+<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
@@ -17,7 +17,8 @@
-->
<!-- Logback configuration. See http://logback.qos.ch/manual/index.html -->
-<configuration scan="true" scanPeriod="120 seconds">
+<configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" -->
+
<property name="log.base" value="logs"/>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
@@ -27,11 +28,14 @@
<charset>UTF-8</charset>
</encoder>
</appender>
+
+ <conversionRule conversionWord="messsage"
+
converterClass="org.apache.dolphinscheduler.server.log.SensitiveDataConverter"/>
<appender name="TASKLOGFILE"
class="ch.qos.logback.classic.sift.SiftingAppender">
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>INFO</level>
</filter>
- <filter
class="org.apache.dolphinscheduler.server.log.TaskLogFilter"></filter>
+ <filter class="org.apache.dolphinscheduler.server.log.TaskLogFilter"/>
<Discriminator
class="org.apache.dolphinscheduler.server.log.TaskLogDiscriminator">
<key>taskAppId</key>
<logBase>${log.base}</logBase>
@@ -41,7 +45,7 @@
<file>${log.base}/${taskAppId}.log</file>
<encoder>
<pattern>
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS}
%logger{96}:[%line] - %msg%n
+ [%level] %date{yyyy-MM-dd HH:mm:ss.SSS}
%logger{96}:[%line] - %messsage%n
</pattern>
<charset>UTF-8</charset>
</encoder>
@@ -49,31 +53,30 @@
</appender>
</sift>
</appender>
-
- <appender name="WORKERLOGFILE"
class="ch.qos.logback.core.rolling.RollingFileAppender">
- <file>${log.base}/dolphinscheduler-worker.log</file>
- <filter class="org.apache.dolphinscheduler.server.log.WorkerLogFilter">
+ <!-- master server logback config start -->
+ <appender name="MASTERLOGFILE"
class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <file>${log.base}/dolphinscheduler-master.log</file>
+ <!--<filter
class="org.apache.dolphinscheduler.server.log.MasterLogFilter">
<level>INFO</level>
- </filter>
-
+ </filter>-->
<rollingPolicy
class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
-
<fileNamePattern>${log.base}/dolphinscheduler-worker.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>
+
<fileNamePattern>${log.base}/dolphinscheduler-master.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>
<maxHistory>168</maxHistory>
<maxFileSize>200MB</maxFileSize>
</rollingPolicy>
-
<encoder>
<pattern>
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] -
%msg%n
</pattern>
<charset>UTF-8</charset>
</encoder>
-
</appender>
-
+ <!-- master server logback config end -->
<root level="INFO">
+ <appender-ref ref="STDOUT"/>
<appender-ref ref="TASKLOGFILE"/>
- <appender-ref ref="WORKERLOGFILE"/>
+ <appender-ref ref="MASTERLOGFILE"/>
</root>
+
</configuration>
\ No newline at end of file
diff --git a/dockerfile/conf/dolphinscheduler/conf/worker_logback.xml
b/dockerfile/conf/dolphinscheduler/logback/logback-worker.xml
similarity index 84%
rename from dockerfile/conf/dolphinscheduler/conf/worker_logback.xml
rename to dockerfile/conf/dolphinscheduler/logback/logback-worker.xml
index 1b09260..be1d0ac 100644
--- a/dockerfile/conf/dolphinscheduler/conf/worker_logback.xml
+++ b/dockerfile/conf/dolphinscheduler/logback/logback-worker.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="UTF-8" ?>
+<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
@@ -17,7 +17,8 @@
-->
<!-- Logback configuration. See http://logback.qos.ch/manual/index.html -->
-<configuration scan="true" scanPeriod="120 seconds">
+<configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" -->
+
<property name="log.base" value="logs"/>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
@@ -27,11 +28,15 @@
<charset>UTF-8</charset>
</encoder>
</appender>
+
+ <!-- worker server logback config start -->
+ <conversionRule conversionWord="messsage"
+
converterClass="org.apache.dolphinscheduler.server.log.SensitiveDataConverter"/>
<appender name="TASKLOGFILE"
class="ch.qos.logback.classic.sift.SiftingAppender">
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>INFO</level>
</filter>
- <filter
class="org.apache.dolphinscheduler.server.log.TaskLogFilter"></filter>
+ <filter class="org.apache.dolphinscheduler.server.log.TaskLogFilter"/>
<Discriminator
class="org.apache.dolphinscheduler.server.log.TaskLogDiscriminator">
<key>taskAppId</key>
<logBase>${log.base}</logBase>
@@ -41,7 +46,7 @@
<file>${log.base}/${taskAppId}.log</file>
<encoder>
<pattern>
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS}
%logger{96}:[%line] - %msg%n
+ [%level] %date{yyyy-MM-dd HH:mm:ss.SSS}
%logger{96}:[%line] - %messsage%n
</pattern>
<charset>UTF-8</charset>
</encoder>
@@ -49,31 +54,30 @@
</appender>
</sift>
</appender>
-
<appender name="WORKERLOGFILE"
class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${log.base}/dolphinscheduler-worker.log</file>
- <filter class="org.apache.dolphinscheduler.server.log.WorkerLogFilter">
+ <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>INFO</level>
</filter>
-
+ <filter
class="org.apache.dolphinscheduler.server.log.WorkerLogFilter"/>
<rollingPolicy
class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
<fileNamePattern>${log.base}/dolphinscheduler-worker.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>
<maxHistory>168</maxHistory>
<maxFileSize>200MB</maxFileSize>
</rollingPolicy>
-
<encoder>
<pattern>
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] -
%msg%n
+ [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] -
%messsage%n
</pattern>
<charset>UTF-8</charset>
</encoder>
-
</appender>
-
+ <!-- worker server logback config end -->
<root level="INFO">
+ <appender-ref ref="STDOUT"/>
<appender-ref ref="TASKLOGFILE"/>
<appender-ref ref="WORKERLOGFILE"/>
</root>
+
</configuration>
\ No newline at end of file
diff --git a/dockerfile/startup.sh b/dockerfile/startup.sh
index af3c456..71ab506 100644
--- a/dockerfile/startup.sh
+++ b/dockerfile/startup.sh
@@ -25,7 +25,9 @@ DOLPHINSCHEDULER_LOGS=${DOLPHINSCHEDULER_HOME}/logs
# start postgresql
initPostgreSQL() {
echo "checking postgresql"
- if [ -n "$(ifconfig | grep ${POSTGRESQL_HOST})" ]; then
+ if [[ "${POSTGRESQL_HOST}" = "127.0.0.1" || "${POSTGRESQL_HOST}" =
"localhost" ]]; then
+ export PGPORT=${POSTGRESQL_PORT}
+
echo "start postgresql service"
rc-service postgresql restart
@@ -47,10 +49,21 @@ initPostgreSQL() {
sudo -u postgres psql -tAc "grant all privileges on database
dolphinscheduler to ${POSTGRESQL_USERNAME}"
fi
+ echo "test postgresql service"
+ while ! nc -z ${POSTGRESQL_HOST} ${POSTGRESQL_PORT}; do
+ counter=$((counter+1))
+ if [ $counter == 30 ]; then
+ echo "Error: Couldn't connect to postgresql."
+ exit 1
+ fi
+ echo "Trying to connect to postgresql at
${POSTGRESQL_HOST}:${POSTGRESQL_PORT}. Attempt $counter."
+ sleep 5
+ done
+
echo "connect postgresql service"
- v=$(sudo -u postgres PGPASSWORD=${POSTGRESQL_PASSWORD} psql -h
${POSTGRESQL_HOST} -U ${POSTGRESQL_USERNAME} -d dolphinscheduler -tAc "select
1")
+ v=$(sudo -u postgres PGPASSWORD=${POSTGRESQL_PASSWORD} psql -h
${POSTGRESQL_HOST} -p ${POSTGRESQL_PORT} -U ${POSTGRESQL_USERNAME} -d
dolphinscheduler -tAc "select 1")
if [ "$(echo '${v}' | grep 'FATAL' | wc -l)" -eq 1 ]; then
- echo "Can't connect to database...${v}"
+ echo "Error: Can't connect to database...${v}"
exit 1
fi
