This is an automated email from the ASF dual-hosted git repository. abaker pushed a commit to branch develop in repository https://gitbox.apache.org/repos/asf/geode.git
The following commit(s) were added to refs/heads/develop by this push: new 0fa83b4 [GEODE-5449] Refactor almost the entire pipeline to be fork-friendly. Add new image. 0fa83b4 is described below commit 0fa83b4faad1f94a18e42ee48ece3174a0aa6b19 Author: Sean Goller <sgol...@pivotal.io> AuthorDate: Wed Jul 18 12:36:03 2018 -0700 [GEODE-5449] Refactor almost the entire pipeline to be fork-friendly. Add new image. * Add new job to the image pipeline to create a google cloud instance image for executing workloads outside of concourse. * Refactor as much of the pipeline as possible to be fork-friendly. Forked pipelines are prefixed with "<github user>-<branch>-" * Metrics and pr pipeline jobs for a fork's meta pipeline are paused by default. * PR pipeline job will be removed from fork metas in the future. Signed-off-by: Jacob Barrett <jbarr...@pivotal.io> --- ci/docker/Dockerfile | 1 + .../google-geode-builder/build_image.sh} | 35 +++++------ ci/images/google-geode-builder/packer.json | 29 +++++++++ .../scripts/cache_dependencies.sh} | 41 ++++++------ ci/images/google-geode-builder/scripts/setup.sh | 73 ++++++++++++++++++++++ ci/pipelines/deploy_meta.sh | 22 ++++++- ci/pipelines/geode-build/base.yml | 5 +- ci/pipelines/geode-build/deploy_pipeline.sh | 13 +++- .../deploy_images_pipeline.sh} | 49 ++++++++------- .../{docker-images.yml => images/images.yml} | 38 +++++++++-- ci/pipelines/meta.yml | 38 +++++++---- 11 files changed, 261 insertions(+), 83 deletions(-) diff --git a/ci/docker/Dockerfile b/ci/docker/Dockerfile index a6276e6..7e9067b 100644 --- a/ci/docker/Dockerfile +++ b/ci/docker/Dockerfile @@ -21,6 +21,7 @@ ENV SPRUCE_VERSION 1.17.0 ENV GRADLE_USER_HOME /usr/local/maven_files WORKDIR /tmp/work +COPY --from=hashicorp/packer:latest /bin/packer /usr/local/bin/packer ADD https://github.com/geofffranks/spruce/releases/download/v${SPRUCE_VERSION}/spruce-linux-amd64 /usr/local/bin/spruce ADD https://github.com/krallin/tini/releases/download/v0.14.0/tini-static-amd64 /usr/local/bin/tini diff --git a/ci/pipelines/deploy_meta.sh b/ci/images/google-geode-builder/build_image.sh similarity index 53% copy from ci/pipelines/deploy_meta.sh copy to ci/images/google-geode-builder/build_image.sh index 93ff869..e033de6 100755 --- a/ci/pipelines/deploy_meta.sh +++ b/ci/images/google-geode-builder/build_image.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,25 +16,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -GEODE_BRANCH=$(git rev-parse --abbrev-ref HEAD) -SANITIZED_GEODE_BRANCH=$(echo ${GEODE_BRANCH} | tr "/" "-") -TARGET=geode -GEODE_FORK=${1:-apache} -TEAM=$(fly targets | grep ^${TARGET} | awk '{print $3}') - -PUBLIC=true - -echo "Deploying pipline for ${GEODE_FORK}/${GEODE_BRANCH}" +SOURCE="${BASH_SOURCE[0]}" +while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink + SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located +done +SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" -if [ "${TEAM}" = "staging" ]; then - PUBLIC=false -fi +pushd ${SCRIPTDIR} +echo "Setting up credentials for packer" -set -x -fly -t ${TARGET} set-pipeline \ - -p meta-${SANITIZED_GEODE_BRANCH} \ - -c meta.yml \ - --var geode-build-branch=${GEODE_BRANCH} \ - --var geode-fork=${GEODE_FORK} \ - --var concourse-team=${TEAM} \ - --yaml-var public-pipelines=${PUBLIC} +#CREDENTIALS_FILE=concourse-key.json +#echo "${GCP_CONCOURSE_KEY}" | base64 -d > ${CREDENTIALS_FILE} +#export GOOGLE_APPLICATION_CREDENTIALS=${CREDENTIALS_FILE} +echo "Running packer" +packer build packer.json diff --git a/ci/images/google-geode-builder/packer.json b/ci/images/google-geode-builder/packer.json new file mode 100644 index 0000000..59cc529 --- /dev/null +++ b/ci/images/google-geode-builder/packer.json @@ -0,0 +1,29 @@ +{ + "provisioners": [ + { + "type": "shell", + "inline": ["mkdir -p /tmp/work"] + }, + { + "type": "shell", + "execute_command": "sudo -S sh -c '{{ .Vars }} {{ .Path }}'", + "script": "scripts/setup.sh" + }, + { + "type": "shell", + "execute_command": "sudo -S sh -c '{{ .Vars }} {{ .Path }}'", + "script": "scripts/cache_dependencies.sh" + } + ], + "builders": [ + { + "type": "googlecompute", + "project_id": "apachegeode-ci", + "source_image_family": "debian-9", + "ssh_username": "packer", + "zone": "us-central1-f", + "image_family": "geode-builder", + "image_name": "geode-builder-{{timestamp}}" + } + ] +} diff --git a/ci/pipelines/deploy_meta.sh b/ci/images/google-geode-builder/scripts/cache_dependencies.sh old mode 100755 new mode 100644 similarity index 56% copy from ci/pipelines/deploy_meta.sh copy to ci/images/google-geode-builder/scripts/cache_dependencies.sh index 93ff869..c75cbdb --- a/ci/pipelines/deploy_meta.sh +++ b/ci/images/google-geode-builder/scripts/cache_dependencies.sh @@ -15,25 +15,30 @@ # See the License for the specific language governing permissions and # limitations under the License. -GEODE_BRANCH=$(git rev-parse --abbrev-ref HEAD) -SANITIZED_GEODE_BRANCH=$(echo ${GEODE_BRANCH} | tr "/" "-") -TARGET=geode -GEODE_FORK=${1:-apache} -TEAM=$(fly targets | grep ^${TARGET} | awk '{print $3}') +set -e -PUBLIC=true +WORK_DIR=$(mktemp -d) +export GRADLE_USER_HOME=/usr/local/maven_files -echo "Deploying pipline for ${GEODE_FORK}/${GEODE_BRANCH}" +pushd ${WORK_DIR} + git clone -b develop --depth 1 https://github.com/apache/geode.git geode -if [ "${TEAM}" = "staging" ]; then - PUBLIC=false -fi + pushd geode + cat << EOF >> build.gradle + subprojects { + task getDeps(type: Copy) { + from project.sourceSets.main.runtimeClasspath + from project.sourceSets.test.runtimeClasspath + from configurations.testRuntime + into 'runtime/' + } + } +EOF + + # Include rat to get its runtime dependencies, which are apparently not captured by the 'getDeps' task above + ./gradlew --no-daemon :rat getDeps + + popd +popd +rm -rf ${WORK_DIR} -set -x -fly -t ${TARGET} set-pipeline \ - -p meta-${SANITIZED_GEODE_BRANCH} \ - -c meta.yml \ - --var geode-build-branch=${GEODE_BRANCH} \ - --var geode-fork=${GEODE_FORK} \ - --var concourse-team=${TEAM} \ - --yaml-var public-pipelines=${PUBLIC} diff --git a/ci/images/google-geode-builder/scripts/setup.sh b/ci/images/google-geode-builder/scripts/setup.sh new file mode 100755 index 0000000..3699a55 --- /dev/null +++ b/ci/images/google-geode-builder/scripts/setup.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x + +export CHROME_DRIVER_VERSION=2.35 +export GRADLE_USER_HOME=/usr/local/maven_files +export LOCAL_USER=geode + +chmod +x /usr/local/bin/initdocker +mkdir -p ${GRADLE_USER_HOME} +apt-get update +apt-get install -y --no-install-recommends \ + apt-transport-https \ + lsb-release + +echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" > /etc/apt/sources.list.d/google-chrome.list +echo "deb http://packages.cloud.google.com/apt cloud-sdk-$(lsb_release -c -s) main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list +echo "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list +curl -sSL https://dl.google.com/linux/linux_signing_key.pub | apt-key add - +curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - +curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - +apt-get update +apt-get purge lxc-docker +apt-get install -y --no-install-recommends \ + aptitude \ + ca-certificates \ + cgroupfs-mount \ + docker-compose \ + docker-ce \ + git \ + golang \ + google-chrome-stable \ + google-cloud-sdk \ + htop \ + jq \ + openjdk-8-jdk-headless \ + python3 \ + python3-pip \ + unzip \ + vim +gcloud config set core/disable_usage_reporting true +gcloud config set component_manager/disable_update_check true +gcloud config set metrics/environment github_docker_image +curl -Lo /usr/local/bin/dunit-progress https://github.com/jdeppe-pivotal/progress-util/releases/download/0.2/progress.linux +chmod +x /usr/local/bin/dunit-progress +wget --no-verbose -O /tmp/chromedriver_linux64.zip https://chromedriver.storage.googleapis.com/${CHROME_DRIVER_VERSION}/chromedriver_linux64.zip +rm -rf /opt/selenium/chromedriver +unzip /tmp/chromedriver_linux64.zip -d /opt/selenium +rm /tmp/chromedriver_linux64.zip +mv /opt/selenium/chromedriver /opt/selenium/chromedriver-${CHROME_DRIVER_VERSION} +chmod 755 /opt/selenium/chromedriver-${CHROME_DRIVER_VERSION} +ln -fs /opt/selenium/chromedriver-${CHROME_DRIVER_VERSION} /usr/bin/chromedriver +adduser --disabled-password --gecos "" ${LOCAL_USER} +usermod -G docker,google-sudoers -a ${LOCAL_USER} +chown -R ${LOCAL_USER} ${GRADLE_USER_HOME} +echo "export GRADLE_USER_HOME=${GRADLE_USER_HOME}" > /etc/profile.d/gradle_home.sh +apt-get clean +rm -rf /var/lib/apt/lists/* diff --git a/ci/pipelines/deploy_meta.sh b/ci/pipelines/deploy_meta.sh index 93ff869..ceaa11a 100755 --- a/ci/pipelines/deploy_meta.sh +++ b/ci/pipelines/deploy_meta.sh @@ -23,17 +23,35 @@ TEAM=$(fly targets | grep ^${TARGET} | awk '{print $3}') PUBLIC=true -echo "Deploying pipline for ${GEODE_FORK}/${GEODE_BRANCH}" +echo "Deploying pipline for ${GEODE_FORK}/${GEODE_BRANCH} on team ${TEAM}" if [ "${TEAM}" = "staging" ]; then PUBLIC=false fi +if [[ "${GEODE_FORK}" == "apache" ]]; then + META_PIPELINE="meta-${SANITIZED_GEODE_BRANCH}" + PIPELINE_PREFIX="" +else + META_PIPELINE="meta-${GEODE_FORK}-${SANITIZED_GEODE_BRANCH}" + PIPELINE_PREFIX="${GEODE_FORK}-${SANITIZED_GEODE_BRANCH}-" +fi set -x fly -t ${TARGET} set-pipeline \ - -p meta-${SANITIZED_GEODE_BRANCH} \ + -p ${META_PIPELINE} \ -c meta.yml \ --var geode-build-branch=${GEODE_BRANCH} \ --var geode-fork=${GEODE_FORK} \ + --var pipeline-prefix=${PIPELINE_PREFIX} \ --var concourse-team=${TEAM} \ --yaml-var public-pipelines=${PUBLIC} +set +x +if [[ "${GEODE_FORK}" != "apache" ]]; then + echo "Disabling unnecessary jobs for forks." + set -x + fly -t ${TARGET} pause-job \ + -j ${META_PIPELINE}/set-pr-pipeline + fly -t ${TARGET} pause-job \ + -j ${META_PIPELINE}/set-metrics-pipeline + set +x +fi \ No newline at end of file diff --git a/ci/pipelines/geode-build/base.yml b/ci/pipelines/geode-build/base.yml index d6e3d3f..073a990 100644 --- a/ci/pipelines/geode-build/base.yml +++ b/ci/pipelines/geode-build/base.yml @@ -33,7 +33,7 @@ resources: source: username: ((!docker-username)) password: ((!docker-password)) - repository: gcr.io/apachegeode-ci/((!docker-image-name)) + repository: gcr.io/apachegeode-ci/((!docker-image-prefix))((!docker-image-name)) tag: latest - name: geode type: git @@ -49,7 +49,8 @@ resources: uri: (( concat "https://github.com/" metadata.geode-fork "/geode.git" )) branch: (( grab metadata.geode-build-branch )) paths: - - ci/* + - ci/pipelines/geode-build/* + - ci/scripts/* - name: geode-examples type: git source: diff --git a/ci/pipelines/geode-build/deploy_pipeline.sh b/ci/pipelines/geode-build/deploy_pipeline.sh index 0e596d2..ebb2db0 100755 --- a/ci/pipelines/geode-build/deploy_pipeline.sh +++ b/ci/pipelines/geode-build/deploy_pipeline.sh @@ -83,6 +83,17 @@ if [[ "${GEODE_BRANCH}" == "develop" ]] || [[ ${GEODE_BRANCH} =~ ^release/* ]]; TEAM="main" fi +if [[ "${GEODE_FORK}" == "apache" ]]; then + PIPELINE_NAME=${SANITIZED_GEODE_BRANCH} + DOCKER_IMAGE_PREFIX="" +else + PIPELINE_NAME="${GEODE_FORK}-${SANITIZED_GEODE_BRANCH}" + DOCKER_IMAGE_PREFIX="${PIPELINE_NAME}-" +fi + fly login -t ${TARGET} -n ${TEAM} -c https://concourse.apachegeode-ci.info -u ${CONCOURSE_USERNAME} -p ${CONCOURSE_PASSWORD} -fly -t ${TARGET} set-pipeline --non-interactive --pipeline ${SANITIZED_GEODE_BRANCH} --config ${TMP_DIR}/final.yml +fly -t ${TARGET} set-pipeline --non-interactive \ + --pipeline ${PIPELINE_NAME} \ + --var docker-image-prefix=${DOCKER_IMAGE_PREFIX} \ + --config ${TMP_DIR}/final.yml diff --git a/ci/pipelines/geode-build/deploy_pipeline.sh b/ci/pipelines/images/deploy_images_pipeline.sh similarity index 72% copy from ci/pipelines/geode-build/deploy_pipeline.sh copy to ci/pipelines/images/deploy_images_pipeline.sh index 0e596d2..96b060c 100755 --- a/ci/pipelines/geode-build/deploy_pipeline.sh +++ b/ci/pipelines/images/deploy_images_pipeline.sh @@ -23,6 +23,7 @@ while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symli [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located done SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" +GEODEBUILDDIR="${SCRIPTDIR}/../geode-build" if ! [ -x "$(command -v spruce)" ]; then echo "Spruce must be installed for pipeline deployment to work." @@ -46,6 +47,7 @@ if [ "${GEODE_BRANCH}" = "HEAD" ]; then fi SANITIZED_GEODE_BRANCH=$(echo ${GEODE_BRANCH} | tr "/" "-") +MAX_IN_FLIGHT=5 BIN_DIR=${OUTPUT_DIRECTORY}/bin TMP_DIR=${OUTPUT_DIRECTORY}/tmp @@ -55,34 +57,33 @@ chmod +x ${BIN_DIR}/fly PATH=${PATH}:${BIN_DIR} -for i in ${SCRIPTDIR}/test-stubs/*.yml; do - X=$(basename $i) - echo "Merging ${i} into ${TMP_DIR}/${X}" - ${SPRUCE} merge --prune metadata \ - <(echo "metadata:"; \ - echo " geode-build-branch: ${GEODE_BRANCH}"; \ - echo " geode-fork: ${GEODE_FORK}") \ - ${SCRIPTDIR}/test-template.yml \ - ${i} > ${TMP_DIR}/${X} -done +TARGET="geode" -echo "Spruce branch-name into resources" -${SPRUCE} merge --prune metadata \ - ${SCRIPTDIR}/base.yml \ - <(echo "metadata:"; \ - echo " geode-build-branch: ${GEODE_BRANCH}"; \ - echo " geode-fork: ${GEODE_FORK}"; \ - echo " ") \ - ${TMP_DIR}/*.yml > ${TMP_DIR}/final.yml +TEAM=${CONCOURSE_TEAM} +if [[ "${GEODE_FORK}" == "apache" ]]; then + PIPELINE_PREFIX="" + DOCKER_IMAGE_PREFIX="" +else + PIPELINE_PREFIX="${GEODE_FORK}-${SANITIZED_GEODE_BRANCH}-" + DOCKER_IMAGE_PREFIX=${PIPELINE_PREFIX} +fi -TARGET="geode" +PIPELINE_NAME="${PIPELINE_PREFIX}images" -TEAM="staging" -if [[ "${GEODE_BRANCH}" == "develop" ]] || [[ ${GEODE_BRANCH} =~ ^release/* ]]; then - TEAM="main" -fi +#if [[ "${GEODE_BRANCH}" == "develop" ]] || [[ ${GEODE_BRANCH} =~ ^release/* ]]; then +# TEAM="main" +#fi fly login -t ${TARGET} -n ${TEAM} -c https://concourse.apachegeode-ci.info -u ${CONCOURSE_USERNAME} -p ${CONCOURSE_PASSWORD} -fly -t ${TARGET} set-pipeline --non-interactive --pipeline ${SANITIZED_GEODE_BRANCH} --config ${TMP_DIR}/final.yml +set -x +fly -t ${TARGET} set-pipeline \ + --non-interactive \ + --pipeline ${PIPELINE_NAME} \ + --config geode-images-pipeline/ci/pipelines/images/images.yml \ + --var concourse-team=${TEAM} \ + --var geode-fork=${GEODE_FORK} \ + --var geode-build-branch=${GEODE_BRANCH} \ + --var docker-image-prefix=${DOCKER_IMAGE_PREFIX} \ + --yaml-var public-pipelines=${PUBLIC_PIPELINES} diff --git a/ci/pipelines/docker-images.yml b/ci/pipelines/images/images.yml similarity index 62% rename from ci/pipelines/docker-images.yml rename to ci/pipelines/images/images.yml index 7f4e0a9..f62eb9c 100644 --- a/ci/pipelines/docker-images.yml +++ b/ci/pipelines/images/images.yml @@ -21,8 +21,8 @@ resources: - name: apachegeode-build-concourse-dockerfile type: git source: - uri: https://github.com/apache/geode.git - branch: develop + uri: https://github.com/((!geode-fork))/geode.git + branch: ((!geode-build-branch)) paths: - ci/docker/* @@ -31,7 +31,7 @@ resources: source: username: ((!docker-username)) password: ((!docker-password)) - repository: gcr.io/apachegeode-ci/((!docker-image-name)) + repository: gcr.io/apachegeode-ci/((!docker-image-prefix))((!docker-image-name)) - name: openjdk8-docker-image type: docker-image @@ -39,10 +39,19 @@ resources: repository: openjdk tag: 8 skip_download: true + +- name: google-geode-builder + type: git + source: + uri: https://github.com/((!geode-fork))/geode.git + branch: ((!geode-build-branch)) + paths: + - ci/images/google-geode-builder + jobs: # apachegeode-build-concourse - name: build-apachegeode-build-concourse-docker-image - public: true + public: ((!public-pipelines)) serial: true plan: - aggregate: @@ -54,3 +63,24 @@ jobs: params: build: apachegeode-build-concourse-dockerfile/ci/docker tag_as_latest: true + +- name: build-google-geode-builder + public: ((!public-pipelines)) + serial: true + plan: + - aggregate: + - get: google-geode-builder + trigger: true + - get: apachegeode-build-concourse-docker-image + - task: build-image + image: apachegeode-build-concourse-docker-image + config: + inputs: + - name: google-geode-builder + outputs: + - name: results + platform: linux + params: + GCP_CONCOURSE_KEY: ((!concourse-gcp-key-encoded)) + run: + path: google-geode-builder/ci/images/google-geode-builder/build_image.sh diff --git a/ci/pipelines/meta.yml b/ci/pipelines/meta.yml index e2929b1..d538d90 100644 --- a/ci/pipelines/meta.yml +++ b/ci/pipelines/meta.yml @@ -38,14 +38,14 @@ resources: uri: https://github.com/((!geode-fork))/geode.git branch: ((!geode-build-branch)) paths: - - ci/* -- name: geode-docker-images-pipeline + - ci/pipelines/geode-build/* +- name: geode-images-pipeline type: git source: uri: https://github.com/((!geode-fork))/geode.git branch: ((!geode-build-branch)) paths: - - ci/pipelines/docker-images.yml + - ci/pipelines/images/* - name: geode-metrics-pipeline type: git source: @@ -111,18 +111,32 @@ jobs: run: path: geode-pipeline/ci/pipelines/pull-request/deploy_pr_pipeline.sh -- name: set-docker-images-pipeline +- name: set-images-pipeline serial: true public: ((!public-pipelines)) plan: - - get: geode-docker-images-pipeline + - get: geode-images-pipeline trigger: true - - put: apachegeode-concourse - params: - pipelines: - - name: docker-images - team: ((!concourse-team)) - config_file: geode-docker-images-pipeline/ci/pipelines/docker-images.yml + - get: apachegeode-build-concourse-docker-image + - task: deploy-images + image: apachegeode-build-concourse-docker-image + config: + inputs: + - name: geode-images-pipeline + outputs: + - name: results + platform: linux + params: + OUTPUT_DIRECTORY: results + GEODE_BRANCH: ((!geode-build-branch)) + GEODE_FORK: ((!geode-fork)) + CONCOURSE_USERNAME: ((!concourse-username)) + CONCOURSE_PASSWORD: ((!concourse-password)) + CONCOURSE_TEAM: ((!concourse-team)) + PUBLIC_PIPELINES: ((!public-pipelines)) + run: + path: geode-images-pipeline/ci/pipelines/images/deploy_images_pipeline.sh + - name: set-metrics-pipeline serial: true public: ((!public-pipelines)) @@ -132,6 +146,6 @@ jobs: - put: apachegeode-concourse params: pipelines: - - name: develop-metrics + - name: ((!geode-build-branch))-metrics team: ((!concourse-team)) config_file: geode-metrics-pipeline/ci/pipelines/metrics.yml