This is an automated email from the ASF dual-hosted git repository.
jbarrett pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/geode.git
The following commit(s) were added to refs/heads/develop by this push:
new 4e0bab8 GEODE-5476: Migrate pipeline changes to pull request pipeline.
4e0bab8 is described below
commit 4e0bab82e4b1c457755f876f490e2f9bcab5cb50
Author: FSOUTHERLAND <[email protected]>
AuthorDate: Thu Jul 26 15:26:21 2018 -0700
GEODE-5476: Migrate pipeline changes to pull request pipeline.
Co-authored-by: Finn Southerland <[email protected]>
---
ci/pipelines/geode-build/test-template.yml | 1 +
ci/pipelines/images/deploy_images_pipeline.sh | 1 -
ci/pipelines/pull-request/base.yml | 151 +++-------------------
ci/pipelines/pull-request/deploy_pr_pipeline.sh | 15 ++-
ci/pipelines/pull-request/pr-template.yml | 163 ++++++++++++++++--------
ci/scripts/start_instance.sh | 2 +-
6 files changed, 140 insertions(+), 193 deletions(-)
diff --git a/ci/pipelines/geode-build/test-template.yml
b/ci/pipelines/geode-build/test-template.yml
index ecbb47f..a06121e 100644
--- a/ci/pipelines/geode-build/test-template.yml
+++ b/ci/pipelines/geode-build/test-template.yml
@@ -17,6 +17,7 @@
---
jobs:
- name: (( grab metadata.job.name ))
+ serial: false
public: true
plan:
- do:
diff --git a/ci/pipelines/images/deploy_images_pipeline.sh
b/ci/pipelines/images/deploy_images_pipeline.sh
index 7e9844c..bcbeb50 100755
--- a/ci/pipelines/images/deploy_images_pipeline.sh
+++ b/ci/pipelines/images/deploy_images_pipeline.sh
@@ -47,7 +47,6 @@ if [ "${GEODE_BRANCH}" = "HEAD" ]; then
fi
SANITIZED_GEODE_BRANCH=$(echo ${GEODE_BRANCH} | tr "/" "-" | tr '[:upper:]'
'[:lower:]')
-MAX_IN_FLIGHT=5
BIN_DIR=${OUTPUT_DIRECTORY}/bin
TMP_DIR=${OUTPUT_DIRECTORY}/tmp
diff --git a/ci/pipelines/pull-request/base.yml
b/ci/pipelines/pull-request/base.yml
index e4812b1..92f0fbe 100644
--- a/ci/pipelines/pull-request/base.yml
+++ b/ci/pipelines/pull-request/base.yml
@@ -26,6 +26,13 @@ resource_types:
type: docker-image
source:
repository: jtarchie/pr
+ - name: concourse-metadata-resource
+ type: docker-image
+ source:
+ username: ((!docker-username))
+ password: ((!docker-password))
+ repository:
gcr.io/apachegeode-ci/((!docker-image-prefix))concourse-metadata-resource
+ tag: latest
resources:
- name: docker-geode-build-image
@@ -33,9 +40,15 @@ resources:
source:
username: ((!docker-username))
password: ((!docker-password))
- repository: gcr.io/apachegeode-ci/((!docker-image-name))
+ repository:
gcr.io/apachegeode-ci/((!docker-image-prefix))((!docker-image-name))
+ tag: latest
+- name: alpine-tools-image
+ type: docker-image
+ source:
+ username: ((!docker-username))
+ password: ((!docker-password))
+ repository: gcr.io/apachegeode-ci/((!docker-image-prefix))alpine-tools
tag: latest
-
- name: geode
type: pull-request
source:
@@ -45,6 +58,8 @@ resources:
ignore_paths:
- geode-docs/*
- geode-book/*
+- name: concourse-metadata-resource
+ type: concourse-metadata-resource
groups:
- name: main
@@ -59,7 +74,6 @@ groups:
jobs:
- name: Build
serial: false
- max_in_flight: (( grab metadata.max_in_flight ))
public: true
plan:
- aggregate:
@@ -116,134 +130,3 @@ jobs:
path: geode-unmerged-request
status: success
context: $BUILD_JOB_NAME
-
-- name: DistributedTest
- serial: false
- max_in_flight: (( grab metadata.max_in_flight ))
- public: true
- plan:
- - aggregate:
- - get: geode
- trigger: true
- version: every
- params:
- fetch_merge: true
- git:
- depth: 100
- - get: docker-geode-build-image
- params:
- rootfs: true
- # Unmerged pull request SHA, for writing status into GitHub
- - get: geode-unmerged-request
- resource: geode
- version: every
- params:
- fetch_merge: false
-
- - aggregate:
- - put: geode
- params:
- path: geode-unmerged-request
- context: $BUILD_JOB_NAME
- status: pending
-
- - task: run-distributed-core
- image: docker-geode-build-image
- tags: [pr_large]
- privileged: true
- timeout: 8h
- config:
- inputs:
- - name: geode
- - name: docker-geode-build-image
- platform: linux
- outputs:
- - name: built-geode
- params:
- MAINTENANCE_VERSION: (( grab metadata.geode-build-branch ))
- SERVICE_ACCOUNT: ((!concourse-gcp-account))
- PUBLIC_BUCKET: ((!public-bucket))
- PARALLEL_DUNIT: true
- DUNIT_PARALLEL_FORKS: 7
- CALL_STACK_TIMEOUT: 25200
- run:
- args:
- - geode-core:distributedTest
- - distributedtestcore
- path: geode/ci/scripts/test-run.sh
-
- ensure:
- aggregate:
- - task: archive-results-core
- image: docker-geode-build-image
- config:
- inputs:
- - name: geode
- - name: built-geode
- platform: linux
- params:
- MAINTENANCE_VERSION: (( grab metadata.geode-build-branch ))
- SERVICE_ACCOUNT: ((!concourse-gcp-account))
- PUBLIC_BUCKET: ((!public-bucket))
- run:
- args:
- - geode-core:distributedTest
- - distributedtestcore
- path: geode/ci/scripts/test-archive.sh
- - task: run-distributed-everything-else
- image: docker-geode-build-image
- tags: [pr_large]
- privileged: true
- timeout: 8h
- config:
- inputs:
- - name: geode
- - name: docker-geode-build-image
- platform: linux
- outputs:
- - name: built-geode
- params:
- MAINTENANCE_VERSION: (( grab metadata.geode-build-branch ))
- SERVICE_ACCOUNT: ((!concourse-gcp-account))
- PUBLIC_BUCKET: ((!public-bucket))
- PARALLEL_DUNIT: true
- DUNIT_PARALLEL_FORKS: 7
- CALL_STACK_TIMEOUT: 25200
- GRADLE_TASK_OPTIONS: -x geode-core:distributedTest
- run:
- args:
- - distributedTest
- - distributedtesteverythingelse
- path: geode/ci/scripts/test-run.sh
- ensure:
- aggregate:
- - task: archive-results-everythingelse
- image: docker-geode-build-image
- config:
- inputs:
- - name: geode
- - name: built-geode
- platform: linux
- params:
- MAINTENANCE_VERSION: (( grab metadata.geode-build-branch ))
- SERVICE_ACCOUNT: ((!concourse-gcp-account))
- PUBLIC_BUCKET: ((!public-bucket))
- run:
- args:
- - disitributedTest
- - distributedtesteverythingelse
- path: geode/ci/scripts/test-archive.sh
- on_failure:
- aggregate:
- - put: geode
- params:
- path: geode-unmerged-request
- status: failure
- context: $BUILD_JOB_NAME
- on_success:
- aggregate:
- - put: geode
- params:
- path: geode-unmerged-request
- context: $BUILD_JOB_NAME
- status: success
diff --git a/ci/pipelines/pull-request/deploy_pr_pipeline.sh
b/ci/pipelines/pull-request/deploy_pr_pipeline.sh
index da2a2e1..440828e 100755
--- a/ci/pipelines/pull-request/deploy_pr_pipeline.sh
+++ b/ci/pipelines/pull-request/deploy_pr_pipeline.sh
@@ -47,7 +47,6 @@ if [ "${GEODE_BRANCH}" = "HEAD" ]; then
fi
SANITIZED_GEODE_BRANCH=$(echo ${GEODE_BRANCH} | tr "/" "-" | tr '[:upper:]'
'[:lower:]')
-MAX_IN_FLIGHT=5
BIN_DIR=${OUTPUT_DIRECTORY}/bin
TMP_DIR=${OUTPUT_DIRECTORY}/tmp
@@ -64,7 +63,7 @@ for i in ${GEODEBUILDDIR}/test-stubs/*.yml; do
<(echo "metadata:"; \
echo " geode-build-branch: ${GEODE_BRANCH}"; \
echo " geode-fork: ${GEODE_FORK}"; \
- echo " max_in_flight: ${MAX_IN_FLIGHT}") \
+ echo " ") \
${SCRIPTDIR}/pr-template.yml \
${i} > ${TMP_DIR}/${X}
done
@@ -75,7 +74,6 @@ ${SPRUCE} merge --prune metadata \
<(echo "metadata:"; \
echo " geode-build-branch: ${GEODE_BRANCH}"; \
echo " geode-fork: ${GEODE_FORK}"; \
- echo " max_in_flight: ${MAX_IN_FLIGHT}"; \
echo " ") \
${TMP_DIR}/*.yml > ${TMP_DIR}/final.yml
@@ -88,10 +86,19 @@ TEAM=${CONCOURSE_TEAM}
# TEAM="main"
#fi
+if [[ "${GEODE_FORK}" == "apache" ]]; then
+ PIPELINE_PREFIX=""
+ DOCKER_IMAGE_PREFIX=""
+else
+ PIPELINE_PREFIX="${GEODE_FORK}-${SANITIZED_GEODE_BRANCH}-"
+ DOCKER_IMAGE_PREFIX=${PIPELINE_PREFIX}
+fi
+
fly login -t ${TARGET} -n ${TEAM} -c https://concourse.apachegeode-ci.info -u
${CONCOURSE_USERNAME} -p ${CONCOURSE_PASSWORD}
fly -t ${TARGET} set-pipeline \
--non-interactive \
- --pipeline pr-${SANITIZED_GEODE_BRANCH} \
+ --pipeline ${PIPELINE_PREFIX}pr \
--config ${TMP_DIR}/final.yml \
+ --var docker-image-prefix=${DOCKER_IMAGE_PREFIX} \
--var concourse-team=${TEAM}
diff --git a/ci/pipelines/pull-request/pr-template.yml
b/ci/pipelines/pull-request/pr-template.yml
index f91f216..919a2f7 100644
--- a/ci/pipelines/pull-request/pr-template.yml
+++ b/ci/pipelines/pull-request/pr-template.yml
@@ -15,14 +15,14 @@
# limitations under the License.
---
-
jobs:
- name: (( grab metadata.job.name ))
serial: false
- max_in_flight: (( grab metadata.max_in_flight ))
public: true
plan:
+ - do:
- aggregate:
+ - put: concourse-metadata-resource
- get: geode
trigger: true
version: every
@@ -30,7 +30,7 @@ jobs:
fetch_merge: true
git:
depth: 100
- - get: docker-geode-build-image
+ - get: alpine-tools-image
params:
rootfs: true
# Unmerged pull request SHA, for writing status into GitHub
@@ -46,60 +46,117 @@ jobs:
context: $BUILD_JOB_NAME
status: pending
- - task: runtests
- image: docker-geode-build-image
- tags: [pr]
- privileged: true
- timeout: (( grab metadata.job.timeout ))
+ - task: start_instance
+ image: alpine-tools-image
+ timeout: 5m
+ config:
+ inputs:
+ - name: concourse-metadata-resource
+ - name: geode
+ outputs:
+ - name: instance-data
+ platform: linux
+ params:
+ CPUS: (( grab metadata.job.cpus ))
+ RAM: (( grab metadata.job.ram ))
+ GEODE_BRANCH: (( grab metadata.geode-build-branch ))
+ GEODE_FORK: (( grab metadata.geode-fork ))
+ run:
+ path: geode/ci/scripts/start_instance.sh
+ - task: rsync_code_up
+ image: alpine-tools-image
+ timeout: 5m
+ config:
+ inputs:
+ - name: geode
+ - name: instance-data
+ platform: linux
+ run:
+ path: geode/ci/scripts/rsync_code_up.sh
+ - task: prepopulate_build
+ image: alpine-tools-image
+ config:
+ inputs:
+ - name: geode
+ - name: instance-data
+ platform: linux
+ run:
+ path: geode/ci/scripts/prepopulate_build.sh
+ - task: execute_tests
+ image: alpine-tools-image
+ timeout: (( grab metadata.job.timeout ))
+ tags: [pr]
+ config:
+ inputs:
+ - name: geode
+ - name: instance-data
+ platform: linux
+ params:
+ MAINTENANCE_VERSION: (( grab metadata.geode-build-branch ))
+ SERVICE_ACCOUNT: ((!concourse-gcp-account))
+ PUBLIC_BUCKET: ((!public-bucket))
+ PARALLEL_DUNIT: (( grab metadata.job.dunit.parallel ))
+ DUNIT_PARALLEL_FORKS: (( grab metadata.job.dunit.forks ))
+ CALL_STACK_TIMEOUT: (( grab metadata.job.call_stack_timeout ))
+ GRADLE_TASK: (( grab metadata.job.gradle_task ))
+ GRADLE_TASK_OPTIONS: (( grab metadata.job.gradle_task_options || ""
))
+ ARTIFACT_SLUG: (( grab metadata.job.artifact_slug ))
+ run:
+ path: geode/ci/scripts/execute_tests.sh
+ ensure:
+ do:
+ - task: rsync_code_down
+ image: alpine-tools-image
+ timeout: 5m
config:
inputs:
- name: geode
- - name: docker-geode-build-image
- platform: linux
+ - name: instance-data
outputs:
- - name: built-geode
- params:
- MAINTENANCE_VERSION: (( grab metadata.geode-build-branch ))
- SERVICE_ACCOUNT: ((!concourse-gcp-account))
- PUBLIC_BUCKET: ((!public-bucket))
- PARALLEL_DUNIT: (( grab metadata.job.dunit.parallel ))
- DUNIT_PARALLEL_FORKS: (( grab metadata.job.dunit.forks ))
- CALL_STACK_TIMEOUT: (( grab metadata.job.call_stack_timeout ))
+ - name: geode-results
+ platform: linux
run:
- args:
- - (( grab metadata.job.gradle_task ))
- - (( grab metadata.job.artifact_slug ))
- path: geode/ci/scripts/test-run.sh
- on_failure:
- aggregate:
- - put: geode
- params:
- path: geode-unmerged-request
- status: failure
- context: $BUILD_JOB_NAME
- on_success:
- aggregate:
- - put: geode
+ path: geode/ci/scripts/rsync_code_down.sh
+ ensure:
+ aggregate:
+ - task: archive-results
+ image: alpine-tools-image
+ config:
+ inputs:
+ - name: concourse-metadata-resource
+ - name: geode
+ - name: geode-results
+ platform: linux
params:
- path: geode-unmerged-request
- context: $BUILD_JOB_NAME
- status: success
+ MAINTENANCE_VERSION: (( grab metadata.geode-build-branch ))
+ SERVICE_ACCOUNT: ((!concourse-gcp-account))
+ PUBLIC_BUCKET: ((!public-bucket))
+ GRADLE_TASK: (( grab metadata.job.gradle_task ))
+ ARTIFACT_SLUG: (( grab metadata.job.artifact_slug ))
+ run:
+ path: geode/ci/scripts/archive_results.sh
+ - task: stop_instance
+ image: alpine-tools-image
+ timeout: 1h
+ config:
+ inputs:
+ - name: geode
+ - name: instance-data
+ platform: linux
+ run:
+ path: geode/ci/scripts/stop_instance.sh
+ on_failure:
+ aggregate:
+ - put: geode
+ params:
+ path: geode-unmerged-request
+ status: failure
+ context: $BUILD_JOB_NAME
+ on_success:
+ aggregate:
+ - put: geode
+ params:
+ path: geode-unmerged-request
+ context: $BUILD_JOB_NAME
+ status: success
- ensure:
- aggregate:
- - task: archive-results
- image: docker-geode-build-image
- config:
- inputs:
- - name: geode
- - name: built-geode
- platform: linux
- params:
- MAINTENANCE_VERSION: (( grab metadata.geode-build-branch ))
- SERVICE_ACCOUNT: ((!concourse-gcp-account))
- PUBLIC_BUCKET: ((!public-bucket))
- run:
- args:
- - (( grab metadata.job.gradle_task ))
- - (( grab metadata.job.artifact_slug ))
- path: geode/ci/scripts/test-archive.sh
diff --git a/ci/scripts/start_instance.sh b/ci/scripts/start_instance.sh
index 261ab86..9e72c8b 100755
--- a/ci/scripts/start_instance.sh
+++ b/ci/scripts/start_instance.sh
@@ -52,7 +52,7 @@ if [[ "${GEODE_FORK}" != "apache" ]]; then
IMAGE_FAMILY_PREFIX="${GEODE_FORK}-${SANITIZED_GEODE_BRANCH}-"
fi
-INSTANCE_NAME="$(echo
"geode-builder-${BUILD_PIPELINE_NAME}-${BUILD_JOB_NAME}-${BUILD_NAME}" | tr
'[:upper:]' '[:lower:]')"
+INSTANCE_NAME="$(echo
"build-${BUILD_PIPELINE_NAME}-${BUILD_JOB_NAME}-${BUILD_NAME}" | tr '[:upper:]'
'[:lower:]')"
PROJECT=apachegeode-ci
ZONE=us-central1-f
echo "${INSTANCE_NAME}" > "instance-data/instance-name"
