This is an automated email from the ASF dual-hosted git repository.
liuxiaochun pushed a commit to branch dev-1.3.0
in repository https://gitbox.apache.org/repos/asf/incubator-dolphinscheduler.git
The following commit(s) were added to refs/heads/dev-1.3.0 by this push:
new 29a353e Using docker/docker-swarm/docker-compose.yml to run pg and zk
in ci_ut.yml (#2692)
29a353e is described below
commit 29a353e689730cac822165979ce17c5e3ac29835
Author: liwenhe1993 <[email protected]>
AuthorDate: Tue May 12 20:24:17 2020 +0800
Using docker/docker-swarm/docker-compose.yml to run pg and zk in ci_ut.yml
(#2692)
* Using docker/docker-swarm/docker-compose.yml to run pg and zk in ci_ut.yml
1. add initdb volume for pg service in docker-compose.yml
2. modify ci_ut.yml
* add sudo in ci_ut.yml
* Query the metadata of indexes is not correct in
postgresql/dolphinscheduler_ddl.sql
1. replace information_schema.STATISTICS to pg_stat_all_indexes
2. modify ProcessDefinitionMapperTest.java
* Remove inner zk and pg
* add inner pg
---
.github/workflows/ci_ut.yml | 10 ++--
docker/docker-swarm/docker-compose.yml | 6 ++-
dockerfile/Dockerfile | 33 ++++----------
dockerfile/startup.sh | 53 +++++-----------------
.../dao/mapper/ProcessDefinitionMapperTest.java | 18 +++++++-
.../postgresql/dolphinscheduler_ddl.sql | 6 +--
6 files changed, 51 insertions(+), 75 deletions(-)
diff --git a/.github/workflows/ci_ut.yml b/.github/workflows/ci_ut.yml
index 6d0b2a2..55f1259 100644
--- a/.github/workflows/ci_ut.yml
+++ b/.github/workflows/ci_ut.yml
@@ -21,7 +21,6 @@ on:
branches:
- dev
env:
- DOCKER_DIR: ./docker
LOG_DIR: /tmp/dolphinscheduler
name: Unit Test
@@ -47,7 +46,11 @@ jobs:
restore-keys: |
${{ runner.os }}-maven-
- name: Bootstrap database
- run: cd ${DOCKER_DIR} && docker-compose up -d
+ run: |
+ sed -i "s/: root/: test/g"
$(pwd)/docker/docker-swarm/docker-compose.yml
+ docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml
create --force-recreate dolphinscheduler-zookeeper dolphinscheduler-postgresql
+ sudo cp $(pwd)/sql/dolphinscheduler-postgre.sql $(docker volume
inspect docker-swarm_dolphinscheduler-postgresql-initdb | grep "Mountpoint" |
awk -F "\"" '{print $4}')
+ docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml up
-d dolphinscheduler-zookeeper dolphinscheduler-postgresql
- name: Set up JDK 1.8
uses: actions/setup-java@v1
with:
@@ -82,6 +85,5 @@ jobs:
- name: Collect logs
run: |
mkdir -p ${LOG_DIR}
- cd ${DOCKER_DIR}
- docker-compose logs db > ${LOG_DIR}/db.txt
+ docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml logs
dolphinscheduler-postgresql > ${LOG_DIR}/db.txt
continue-on-error: true
diff --git a/docker/docker-swarm/docker-compose.yml
b/docker/docker-swarm/docker-compose.yml
index b564eda..ee8be25 100644
--- a/docker/docker-swarm/docker-compose.yml
+++ b/docker/docker-swarm/docker-compose.yml
@@ -30,6 +30,7 @@ services:
POSTGRESQL_DATABASE: dolphinscheduler
volumes:
- dolphinscheduler-postgresql:/bitnami/postgresql
+ - dolphinscheduler-postgresql-initdb:/docker-entrypoint-initdb.d
networks:
- dolphinscheduler
@@ -175,10 +176,10 @@ services:
image: apache/dolphinscheduler:latest
container_name: dolphinscheduler-worker
command: ["worker-server"]
- ports:
+ ports:
- 1234:1234
- 50051:50051
- environment:
+ environment:
TZ: Asia/Shanghai
WORKER_EXEC_THREADS: "100"
WORKER_HEARTBEAT_INTERVAL: "10"
@@ -221,6 +222,7 @@ networks:
volumes:
dolphinscheduler-postgresql:
+ dolphinscheduler-postgresql-initdb:
dolphinscheduler-zookeeper:
dolphinscheduler-worker-data:
dolphinscheduler-logs:
diff --git a/dockerfile/Dockerfile b/dockerfile/Dockerfile
index 53e362b..101e0b2 100644
--- a/dockerfile/Dockerfile
+++ b/dockerfile/Dockerfile
@@ -37,32 +37,20 @@ RUN apk add openjdk8
ENV JAVA_HOME /usr/lib/jvm/java-1.8-openjdk
ENV PATH $JAVA_HOME/bin:$PATH
-#3. install zk
-RUN cd /opt && \
- wget
https://downloads.apache.org/zookeeper/zookeeper-3.5.7/apache-zookeeper-3.5.7-bin.tar.gz
&& \
- tar -zxvf apache-zookeeper-3.5.7-bin.tar.gz && \
- mv apache-zookeeper-3.5.7-bin zookeeper && \
- mkdir -p /tmp/zookeeper && \
- rm -rf ./zookeeper-*tar.gz && \
- rm -rf /opt/zookeeper/conf/zoo_sample.cfg
-ADD ./conf/zookeeper/zoo.cfg /opt/zookeeper/conf
-ENV ZK_HOME /opt/zookeeper
-ENV PATH $ZK_HOME/bin:$PATH
-
-#4. install pg
-RUN apk add postgresql postgresql-contrib
-
-#5. add dolphinscheduler
+#3. add dolphinscheduler
ADD
./apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz
/opt/
RUN mv
/opt/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin/
/opt/dolphinscheduler/
ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler
-#6. modify nginx
+#4. install pg
+RUN apk add postgresql postgresql-contrib
+
+#5. modify nginx
RUN echo "daemon off;" >> /etc/nginx/nginx.conf && \
rm -rf /etc/nginx/conf.d/*
ADD ./conf/nginx/dolphinscheduler.conf /etc/nginx/conf.d
-#7. add configuration and modify permissions and set soft links
+#6. add configuration and modify permissions and set soft links
ADD ./checkpoint.sh /root/checkpoint.sh
ADD ./startup-init-conf.sh /root/startup-init-conf.sh
ADD ./startup.sh /root/startup.sh
@@ -75,22 +63,21 @@ RUN chmod +x /root/checkpoint.sh && \
chmod +x /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh && \
chmod +x /opt/dolphinscheduler/script/*.sh && \
chmod +x /opt/dolphinscheduler/bin/*.sh && \
- chmod +x /opt/zookeeper/bin/*.sh && \
dos2unix /root/checkpoint.sh && \
dos2unix /root/startup-init-conf.sh && \
dos2unix /root/startup.sh && \
dos2unix /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh && \
dos2unix /opt/dolphinscheduler/script/*.sh && \
dos2unix /opt/dolphinscheduler/bin/*.sh && \
- dos2unix /opt/zookeeper/bin/*.sh && \
rm -rf /bin/sh && \
ln -s /bin/bash /bin/sh && \
mkdir -p /tmp/xls
-#8. remove apk index cache
-RUN rm -rf /var/cache/apk/*
+#7. remove apk index cache and disable coredup for sudo
+RUN rm -rf /var/cache/apk/* && \
+ echo "Set disable_coredump false" >> /etc/sudo.conf
-#9. expose port
+#8. expose port
EXPOSE 2181 2888 3888 5432 5678 1234 12345 50051 8888
ENTRYPOINT ["/sbin/tini", "--", "/root/startup.sh"]
\ No newline at end of file
diff --git a/dockerfile/startup.sh b/dockerfile/startup.sh
index 71ab506..00bb9cd 100644
--- a/dockerfile/startup.sh
+++ b/dockerfile/startup.sh
@@ -24,31 +24,6 @@ DOLPHINSCHEDULER_LOGS=${DOLPHINSCHEDULER_HOME}/logs
# start postgresql
initPostgreSQL() {
- echo "checking postgresql"
- if [[ "${POSTGRESQL_HOST}" = "127.0.0.1" || "${POSTGRESQL_HOST}" =
"localhost" ]]; then
- export PGPORT=${POSTGRESQL_PORT}
-
- echo "start postgresql service"
- rc-service postgresql restart
-
- # role if not exists, create
- flag=$(sudo -u postgres psql -tAc "SELECT 1 FROM pg_roles WHERE
rolname='${POSTGRESQL_USERNAME}'")
- if [ -z "${flag}" ]; then
- echo "create user"
- sudo -u postgres psql -tAc "create user ${POSTGRESQL_USERNAME}
with password '${POSTGRESQL_PASSWORD}'"
- fi
-
- # database if not exists, create
- flag=$(sudo -u postgres psql -tAc "select 1 from pg_database where
datname='dolphinscheduler'")
- if [ -z "${flag}" ]; then
- echo "init db"
- sudo -u postgres psql -tAc "create database dolphinscheduler owner
${POSTGRESQL_USERNAME}"
- fi
-
- # grant
- sudo -u postgres psql -tAc "grant all privileges on database
dolphinscheduler to ${POSTGRESQL_USERNAME}"
- fi
-
echo "test postgresql service"
while ! nc -z ${POSTGRESQL_HOST} ${POSTGRESQL_PORT}; do
counter=$((counter+1))
@@ -73,24 +48,18 @@ initPostgreSQL() {
# start zk
initZK() {
- echo -e "checking zookeeper"
- if [[ "${ZOOKEEPER_QUORUM}" = "127.0.0.1:2181" || "${ZOOKEEPER_QUORUM}" =
"localhost:2181" ]]; then
- echo "start local zookeeper"
- /opt/zookeeper/bin/zkServer.sh restart
- else
- echo "connect remote zookeeper"
- echo "${ZOOKEEPER_QUORUM}" | awk -F ',' 'BEGIN{ i=1 }{ while( i <= NF
){ print $i; i++ } }' | while read line; do
- while ! nc -z ${line%:*} ${line#*:}; do
- counter=$((counter+1))
- if [ $counter == 30 ]; then
- echo "Error: Couldn't connect to zookeeper."
- exit 1
- fi
- echo "Trying to connect to zookeeper at ${line}. Attempt
$counter."
- sleep 5
- done
+ echo "connect remote zookeeper"
+ echo "${ZOOKEEPER_QUORUM}" | awk -F ',' 'BEGIN{ i=1 }{ while( i <= NF ){
print $i; i++ } }' | while read line; do
+ while ! nc -z ${line%:*} ${line#*:}; do
+ counter=$((counter+1))
+ if [ $counter == 30 ]; then
+ echo "Error: Couldn't connect to zookeeper."
+ exit 1
+ fi
+ echo "Trying to connect to zookeeper at ${line}. Attempt $counter."
+ sleep 5
done
- fi
+ done
}
# start nginx
diff --git
a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapperTest.java
b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapperTest.java
index 9dafbe1..e92ec54 100644
---
a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapperTest.java
+++
b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapperTest.java
@@ -74,6 +74,22 @@ public class ProcessDefinitionMapperTest {
}
/**
+ * insert
+ * @return ProcessDefinition
+ */
+ private ProcessDefinition insertTwo(){
+ //insertOne
+ ProcessDefinition processDefinition = new ProcessDefinition();
+ processDefinition.setName("def 2");
+ processDefinition.setProjectId(1010);
+ processDefinition.setUserId(101);
+ processDefinition.setUpdateTime(new Date());
+ processDefinition.setCreateTime(new Date());
+ processDefinitionMapper.insert(processDefinition);
+ return processDefinition;
+ }
+
+ /**
* test update
*/
@Test
@@ -177,7 +193,7 @@ public class ProcessDefinitionMapperTest {
public void testQueryDefinitionListByIdList() {
ProcessDefinition processDefinition = insertOne();
- ProcessDefinition processDefinition1 = insertOne();
+ ProcessDefinition processDefinition1 = insertTwo();
Integer[] array = new Integer[2];
array[0] = processDefinition.getId();
diff --git a/sql/upgrade/1.3.0_schema/postgresql/dolphinscheduler_ddl.sql
b/sql/upgrade/1.3.0_schema/postgresql/dolphinscheduler_ddl.sql
index 61cbfed..16a49a8 100644
--- a/sql/upgrade/1.3.0_schema/postgresql/dolphinscheduler_ddl.sql
+++ b/sql/upgrade/1.3.0_schema/postgresql/dolphinscheduler_ddl.sql
@@ -368,9 +368,9 @@ DROP FUNCTION
dc_dolphin_T_t_ds_error_command_D_worker_group_id();
delimiter d//
CREATE OR REPLACE FUNCTION
uc_dolphin_T_t_ds_process_definition_A_process_definition_unique() RETURNS void
AS $$
BEGIN
- IF NOT EXISTS (SELECT 1 FROM information_schema.STATISTICS
- WHERE TABLE_NAME='t_ds_process_definition'
- AND INDEX_NAME ='process_definition_unique')
+ IF NOT EXISTS (SELECT 1 FROM pg_stat_all_indexes
+ WHERE relname='t_ds_process_definition'
+ AND indexrelname ='process_definition_unique')
THEN
ALTER TABLE t_ds_process_definition ADD CONSTRAINT
process_definition_unique UNIQUE (name,project_id);
END IF;
