This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch integration-tests-v2
in repository https://gitbox.apache.org/repos/asf/incubator-hop.git
The following commit(s) were added to refs/heads/integration-tests-v2 by this
push:
new 3469dbc HOP-2914: split compose files and make tests more modular
new d5e34c3 Merge pull request #844 from hansva/integration-tests-v2
3469dbc is described below
commit 3469dbc67cb2232e8d735ab81b05cd4154b0a291
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Wed Jun 2 10:43:46 2021 +0200
HOP-2914: split compose files and make tests more modular
---
Jenkinsfile.daily | 29 +++---
docker/integration-tests/docker-compose-ci.yaml | 89 -----------------
.../integration-tests/integration-tests-base.yaml | 2 +-
...-base.yaml => integration-tests-cassandra.yaml} | 34 ++++---
...ests-base.yaml => integration-tests-mongo.yaml} | 33 +++---
...ests-base.yaml => integration-tests-neo4j.yaml} | 34 ++++---
.../scripts/main-0001-xml-join-test.hwf | 111 ---------------------
integration-tests/scripts/run-tests-docker.sh | 59 +++++++++++
integration-tests/scripts/run-tests.sh | 11 +-
.../spreadsheet/files/exelwriter-testfile.xlsx | Bin 3624 -> 3598 bytes
10 files changed, 141 insertions(+), 261 deletions(-)
diff --git a/Jenkinsfile.daily b/Jenkinsfile.daily
index f6d9292..738f61d 100644
--- a/Jenkinsfile.daily
+++ b/Jenkinsfile.daily
@@ -84,28 +84,25 @@ pipeline {
sh "mvn $MAVEN_PARAMS clean install"
}
}
- stage('Code Quality') {
- tools{
- jdk 'jdk_11_latest'
- }
- steps {
- echo 'Checking Code Quality on SonarCloud'
- withCredentials([string(credentialsId:
'sonarcloud-key-apache-hop', variable: 'SONAR_TOKEN')]) {
- sh 'mvn sonar:sonar -Dsonar.host.url=https://sonarcloud.io
-Dsonar.organization=apache -Dsonar.projectKey=apache_incubator-hop
-Dsonar.branch.name=${BRANCH_NAME} -Dsonar.login=${SONAR_TOKEN}'
- }
- }
- }
+ // stage('Code Quality') {
+ // tools{
+ // jdk 'jdk_11_latest'
+ // }
+ // steps {
+ // echo 'Checking Code Quality on SonarCloud'
+ // withCredentials([string(credentialsId:
'sonarcloud-key-apache-hop', variable: 'SONAR_TOKEN')]) {
+ // sh 'mvn sonar:sonar
-Dsonar.host.url=https://sonarcloud.io -Dsonar.organization=apache
-Dsonar.projectKey=apache_incubator-hop -Dsonar.branch.name=${BRANCH_NAME}
-Dsonar.login=${SONAR_TOKEN}'
+ // }
+ // }
+ // }
stage('Create Docker image & Run Tests'){
steps {
- sh "docker-compose -f
'docker/integration-tests/docker-compose-ci.yaml' build --build-arg
JENKINS_USER=${jenkins_user} --build-arg JENKINS_UID=${jenkins_uid} --build-arg
JENKINS_GROUP=${jenkins_group} --build-arg JENKINS_GID=${jenkins_gid}"
- sh "docker-compose -f
'docker/integration-tests/docker-compose-ci.yaml' up -d"
- sh "docker wait integration-tests_integration_test_1"
- sh "docker logs integration-tests_integration_test_1"
+ sh "docker-compose -f
'docker/integration-tests/integration-tests-base.yaml' build --build-arg
JENKINS_USER=${jenkins_user} --build-arg JENKINS_UID=${jenkins_uid} --build-arg
JENKINS_GROUP=${jenkins_group} --build-arg JENKINS_GID=${jenkins_gid}"
+ sh "integration-tests/scripts/run-tests-docker.sh"
}
post {
always {
junit(testResults:
'integration-tests/**/surefire-reports/*.xml', allowEmptyResults: true)
- sh "docker-compose -f
'docker/integration-tests/docker-compose-ci.yaml' down --rmi all"
}
}
}
diff --git a/docker/integration-tests/docker-compose-ci.yaml
b/docker/integration-tests/docker-compose-ci.yaml
deleted file mode 100644
index 89807fe..0000000
--- a/docker/integration-tests/docker-compose-ci.yaml
+++ /dev/null
@@ -1,89 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-version: '3'
-services:
- integration_test:
- build:
- context: ../../.
- dockerfile: docker/integration-tests/Dockerfile.unit-tests
- args:
- - JENKINS_USER=jenkins
- - JENKINS_UID=1000
- - JENKINS_GROUP=jenkins
- - JENKINS_GID=1000
- volumes:
- - ../../integration-tests/:/files
- depends_on:
- postgres:
- condition: service_healthy
- neo4j:
- condition: service_healthy
- mongo:
- condition: service_healthy
- cassandra:
- condition: service_healthy
- links:
- - postgres
- - neo4j
- - mongo
- - cassandra
- environment:
- - FLASK_ENV=docker
- command: [ "bash", "-c", "/files/scripts/run-tests.sh" ]
- postgres:
- image: postgres:latest
- env_file:
- - database.env
- ports:
- - "5432"
- healthcheck:
- test: [ "CMD-SHELL", "pg_isready -U postgres" ]
- interval: 10s
- timeout: 5s
- retries: 5
- neo4j:
- image: neo4j:latest
- ports:
- - "7474"
- - "7687"
- environment:
- - NEO4J_AUTH=neo4j/abcd
- healthcheck:
- test: echo 'ok'
- interval: 10s
- timeout: 10s
- retries: 5
- mongo:
- image: mongo:latest
- ports:
- - "27017"
- healthcheck:
- test: echo 'db.runCommand("ping").ok' | mongo 127.0.0.1:27017/test
--quiet
- interval: 10s
- timeout: 10s
- retries: 5
- cassandra:
- image: cassandra:latest
- ports:
- - "9042"
- - "9160"
- healthcheck:
- test: [ "CMD-SHELL", "[ $$(nodetool statusgossip) = running ]" ]
- interval: 10s
- timeout: 5s
- retries: 50
\ No newline at end of file
diff --git a/docker/integration-tests/integration-tests-base.yaml
b/docker/integration-tests/integration-tests-base.yaml
index ed9f0cd..31360bf 100644
--- a/docker/integration-tests/integration-tests-base.yaml
+++ b/docker/integration-tests/integration-tests-base.yaml
@@ -30,4 +30,4 @@ services:
- ../../integration-tests/:/files
environment:
- FLASK_ENV=docker
- command: [ "bash", "-c", "/files/scripts/run-tests.sh" ]
\ No newline at end of file
+ command: [ "bash", "-c", "/files/scripts/run-tests.sh ${PROJECT_NAME}" ]
\ No newline at end of file
diff --git a/docker/integration-tests/integration-tests-base.yaml
b/docker/integration-tests/integration-tests-cassandra.yaml
similarity index 64%
copy from docker/integration-tests/integration-tests-base.yaml
copy to docker/integration-tests/integration-tests-cassandra.yaml
index ed9f0cd..6f2b5c1 100644
--- a/docker/integration-tests/integration-tests-base.yaml
+++ b/docker/integration-tests/integration-tests-cassandra.yaml
@@ -17,17 +17,23 @@
version: '3'
services:
- integration_test:
- build:
- context: ../../.
- dockerfile: docker/integration-tests/Dockerfile.unit-tests
- args:
- - JENKINS_USER=jenkins
- - JENKINS_UID=1000
- - JENKINS_GROUP=jenkins
- - JENKINS_GID=1000
- volumes:
- - ../../integration-tests/:/files
- environment:
- - FLASK_ENV=docker
- command: [ "bash", "-c", "/files/scripts/run-tests.sh" ]
\ No newline at end of file
+ integration_test_cassandra:
+ extends:
+ file: integration-tests-base.yaml
+ service: integration_test
+ depends_on:
+ cassandra:
+ condition: service_healthy
+ links:
+ - cassandra
+
+ cassandra:
+ image: cassandra:latest
+ ports:
+ - "9042"
+ - "9160"
+ healthcheck:
+ test: [ "CMD-SHELL", "[ $$(nodetool statusgossip) = running ]" ]
+ interval: 10s
+ timeout: 5s
+ retries: 50
\ No newline at end of file
diff --git a/docker/integration-tests/integration-tests-base.yaml
b/docker/integration-tests/integration-tests-mongo.yaml
similarity index 66%
copy from docker/integration-tests/integration-tests-base.yaml
copy to docker/integration-tests/integration-tests-mongo.yaml
index ed9f0cd..16ee0da 100644
--- a/docker/integration-tests/integration-tests-base.yaml
+++ b/docker/integration-tests/integration-tests-mongo.yaml
@@ -17,17 +17,22 @@
version: '3'
services:
- integration_test:
- build:
- context: ../../.
- dockerfile: docker/integration-tests/Dockerfile.unit-tests
- args:
- - JENKINS_USER=jenkins
- - JENKINS_UID=1000
- - JENKINS_GROUP=jenkins
- - JENKINS_GID=1000
- volumes:
- - ../../integration-tests/:/files
- environment:
- - FLASK_ENV=docker
- command: [ "bash", "-c", "/files/scripts/run-tests.sh" ]
\ No newline at end of file
+ integration_test_mongo:
+ extends:
+ file: integration-tests-base.yaml
+ service: integration_test
+ depends_on:
+ mongo:
+ condition: service_healthy
+ links:
+ - mongo
+
+ mongo:
+ image: mongo:latest
+ ports:
+ - "27017"
+ healthcheck:
+ test: echo 'db.runCommand("ping").ok' | mongo 127.0.0.1:27017/test
--quiet
+ interval: 10s
+ timeout: 10s
+ retries: 5
\ No newline at end of file
diff --git a/docker/integration-tests/integration-tests-base.yaml
b/docker/integration-tests/integration-tests-neo4j.yaml
similarity index 67%
copy from docker/integration-tests/integration-tests-base.yaml
copy to docker/integration-tests/integration-tests-neo4j.yaml
index ed9f0cd..6926281 100644
--- a/docker/integration-tests/integration-tests-base.yaml
+++ b/docker/integration-tests/integration-tests-neo4j.yaml
@@ -17,17 +17,25 @@
version: '3'
services:
- integration_test:
- build:
- context: ../../.
- dockerfile: docker/integration-tests/Dockerfile.unit-tests
- args:
- - JENKINS_USER=jenkins
- - JENKINS_UID=1000
- - JENKINS_GROUP=jenkins
- - JENKINS_GID=1000
- volumes:
- - ../../integration-tests/:/files
+ integration_test_neo4j:
+ extends:
+ file: integration-tests-base.yaml
+ service: integration_test
+ depends_on:
+ neo4j:
+ condition: service_healthy
+ links:
+ - neo4j
+
+ neo4j:
+ image: neo4j:latest
+ ports:
+ - "7474"
+ - "7687"
environment:
- - FLASK_ENV=docker
- command: [ "bash", "-c", "/files/scripts/run-tests.sh" ]
\ No newline at end of file
+ - NEO4J_AUTH=neo4j/abcd
+ healthcheck:
+ test: echo 'ok'
+ interval: 10s
+ timeout: 10s
+ retries: 5
\ No newline at end of file
diff --git a/integration-tests/scripts/main-0001-xml-join-test.hwf
b/integration-tests/scripts/main-0001-xml-join-test.hwf
deleted file mode 100644
index 94139ef..0000000
--- a/integration-tests/scripts/main-0001-xml-join-test.hwf
+++ /dev/null
@@ -1,111 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<workflow>
- <name>main-0001-xml-join-test</name>
- <name_sync_with_filename>Y</name_sync_with_filename>
- <description/>
- <extended_description/>
- <workflow_version/>
- <created_user>-</created_user>
- <created_date>2020/12/09 13:24:02.438</created_date>
- <modified_user>-</modified_user>
- <modified_date>2020/12/09 13:24:02.438</modified_date>
- <parameters>
- </parameters>
- <actions>
- <action>
- <name>START</name>
- <description/>
- <type>SPECIAL</type>
- <attributes/>
- <repeat>N</repeat>
- <schedulerType>0</schedulerType>
- <intervalSeconds>0</intervalSeconds>
- <intervalMinutes>60</intervalMinutes>
- <hour>12</hour>
- <minutes>0</minutes>
- <weekDay>1</weekDay>
- <DayOfMonth>1</DayOfMonth>
- <parallel>N</parallel>
- <nr>0</nr>
- <xloc>64</xloc>
- <yloc>64</yloc>
- <attributes_hac/>
- </action>
- <action>
- <name>0001-xml-join-test.hpl</name>
- <description/>
- <type>PIPELINE</type>
- <attributes/>
- <filename>${PROJECT_HOME}/0001-xml-join-test.hpl</filename>
- <params_from_previous>N</params_from_previous>
- <exec_per_row>N</exec_per_row>
- <clear_rows>N</clear_rows>
- <clear_files>N</clear_files>
- <set_logfile>N</set_logfile>
- <logfile/>
- <logext/>
- <add_date>N</add_date>
- <add_time>N</add_time>
- <loglevel>Basic</loglevel>
- <set_append_logfile>N</set_append_logfile>
- <wait_until_finished>Y</wait_until_finished>
- <follow_abort_remote>N</follow_abort_remote>
- <create_parent_folder>N</create_parent_folder>
- <run_configuration>local</run_configuration>
- <parameters>
- <pass_all_parameters>Y</pass_all_parameters>
- </parameters>
- <parallel>N</parallel>
- <nr>0</nr>
- <xloc>224</xloc>
- <yloc>64</yloc>
- <attributes_hac/>
- </action>
- <action>
- <name>verify XML</name>
- <description/>
- <type>SIMPLE_EVAL</type>
- <attributes/>
- <valuetype>field</valuetype>
- <fieldname>xmlOutput</fieldname>
- <variablename/>
- <fieldtype>string</fieldtype>
- <mask/>
- <comparevalue><?xml version="1.0" encoding="ISO-8859-1"
standalone="no"?><OrderList businessUnitId="EU10" plantId="EU11" source="XY"
timestamp="19700101"><OrderHeaders><OrderHeader customerNumber="1000"
orderDate="19700101" orderNumber="4711" orderType="AN"
originator="G"><OrderLines><OrderLine orderLineNumber="1" principal="XY"
productNumber="500000" productText="red" quantity="10"
uom="KG"><OrderSubLines><OrderSubLine batch="B61" expiryDate="19700101"
orde [...]
- <minvalue/>
- <maxvalue/>
- <successcondition>equal</successcondition>
- <successnumbercondition>equal</successnumbercondition>
- <successbooleancondition>false</successbooleancondition>
- <successwhenvarset>N</successwhenvarset>
- <parallel>N</parallel>
- <nr>0</nr>
- <xloc>400</xloc>
- <yloc>64</yloc>
- <attributes_hac/>
- </action>
- </actions>
- <hops>
- <hop>
- <from>START</from>
- <to>0001-xml-join-test.hpl</to>
- <from_nr>0</from_nr>
- <to_nr>0</to_nr>
- <enabled>Y</enabled>
- <evaluation>Y</evaluation>
- <unconditional>Y</unconditional>
- </hop>
- <hop>
- <from>0001-xml-join-test.hpl</from>
- <to>verify XML</to>
- <from_nr>0</from_nr>
- <to_nr>0</to_nr>
- <enabled>Y</enabled>
- <evaluation>Y</evaluation>
- <unconditional>N</unconditional>
- </hop>
- </hops>
- <notepads>
- </notepads>
- <attributes/>
-</workflow>
diff --git a/integration-tests/scripts/run-tests-docker.sh
b/integration-tests/scripts/run-tests-docker.sh
new file mode 100755
index 0000000..6a69395
--- /dev/null
+++ b/integration-tests/scripts/run-tests-docker.sh
@@ -0,0 +1,59 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+
+CURRENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+DOCKER_FILES_DIR="$(cd ${CURRENT_DIR}/../../docker/integration-tests/ && pwd)"
+PROJECT_NAME="$1"
+
+if [ -z "${PROJECT_NAME}" ]; then
+ PROJECT_NAME="*"
+fi
+
+#Cleanup surefire reports
+rm -rf "${CURRENT_DIR}"/../surefire-reports
+mkdir -p "${CURRENT_DIR}"/../surefire-reports/
+
+#Loop over project folders
+for d in "${CURRENT_DIR}"/../${PROJECT_NAME}/ ; do
+
+ if [[ "$d" != *"scripts/" ]] && [[ "$d" != *"surefire-reports/" ]] && [[
"$d" != *"hopweb/" ]]; then
+
+ PROJECT_NAME=$(basename $d)
+
+ echo "Project name: ${PROJECT_NAME}"
+ echo "project path: $d"
+ echo "docker compose path: ${DOCKER_FILES_DIR}"
+
+ #Check if specific compose exists
+
+ if [ -f "${DOCKER_FILES_DIR}/integration-tests-${PROJECT_NAME}.yaml"
]; then
+ echo "Project compose exists."
+ PROJECT_NAME=${PROJECT_NAME} docker-compose -f
${DOCKER_FILES_DIR}/integration-tests-${PROJECT_NAME}.yaml up
--abort-on-container-exit
+ docker-compose -f
${DOCKER_FILES_DIR}/integration-tests-${PROJECT_NAME}.yaml down --rmi all
+ else
+ echo "Project compose does not exists."
+ PROJECT_NAME=${PROJECT_NAME} docker-compose -f
${DOCKER_FILES_DIR}/integration-tests-base.yaml up --abort-on-container-exit
+ docker-compose -f ${DOCKER_FILES_DIR}/integration-tests-base.yaml
down
+ fi
+ fi
+done
+
+#Cleanup all images
+docker-compose -f ${DOCKER_FILES_DIR}/integration-tests-base.yaml down --rmi
all --remove-orphans
\ No newline at end of file
diff --git a/integration-tests/scripts/run-tests.sh
b/integration-tests/scripts/run-tests.sh
index 4a9b60c..f0b0ef2 100755
--- a/integration-tests/scripts/run-tests.sh
+++ b/integration-tests/scripts/run-tests.sh
@@ -19,6 +19,9 @@
#
CURRENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+PROJECT_NAME="$1"
+
+echo "The Project Name = ${PROJECT_NAME}"
if [ -z "${HOP_LOCATION}" ]; then
HOP_LOCATION=/opt/hop
@@ -49,6 +52,10 @@ if [ -z "${POSTGRES_PASSWORD}" ]; then
POSTGRES_PASSWORD=hop_password
fi
+if [ -z "${PROJECT_NAME}" ]; then
+ PROJECT_NAME="*"
+fi
+
#set global variables
SPACER="==========================================="
@@ -60,8 +67,6 @@ mkdir -p "${TMP_FOLDER}"
#cleanup Temp
export TMP_TESTCASES="${TMP_FOLDER}"/testcases.xml
rm -f "${TMP_TESTCASES}"
-rm -rf "${CURRENT_DIR}"/../surefire-reports
-mkdir -p "${CURRENT_DIR}"/../surefire-reports/
# Set up auditing
# Start with a new blank slate every time
@@ -75,7 +80,7 @@ mkdir -p "${HOP_AUDIT_FOLDER}"
TMP_CONFIG_FOLDER="${HOP_CONFIG_FOLDER}"
#Loop over project folders
-for d in "${CURRENT_DIR}"/../*/ ; do
+for d in "${CURRENT_DIR}"/../${PROJECT_NAME}/ ; do
#cleanup project testcases
rm -f "${TMP_TESTCASES}"
diff --git a/integration-tests/spreadsheet/files/exelwriter-testfile.xlsx
b/integration-tests/spreadsheet/files/exelwriter-testfile.xlsx
index eeaee0c..f50090a 100644
Binary files a/integration-tests/spreadsheet/files/exelwriter-testfile.xlsx and
b/integration-tests/spreadsheet/files/exelwriter-testfile.xlsx differ
