This is an automated email from the ASF dual-hosted git repository.
jbarrett pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/geode.git
The following commit(s) were added to refs/heads/develop by this push:
new 9723637 [GEODE-5476] Refactor CI to use external instances for test
execution. (#2198)
9723637 is described below
commit 972363747bdd48b55c7ae1ea96caf7c879550833
Author: Sean Goller <[email protected]>
AuthorDate: Wed Jul 25 16:09:53 2018 -0700
[GEODE-5476] Refactor CI to use external instances for test execution.
(#2198)
* Build Google Cloud-based images for launching during test jobs
* refactor test template to use new images.
Signed-off-by: Sean Goller <[email protected]>
Signed-off-by: Dick Cavender <[email protected]>
Signed-off-by: Jacob Barrett <[email protected]>
---
.../alpine-tools/Dockerfile} | 27 ++--
.../concourse-metadata-resource/Dockerfile} | 18 +--
.../concourse-metadata-resource/files/check} | 22 +--
ci/images/concourse-metadata-resource/files/in | 48 +++++++
.../concourse-metadata-resource/files/out} | 22 +--
ci/images/google-geode-builder/build_image.sh | 23 +++-
ci/images/google-geode-builder/packer.json | 13 +-
.../scripts/cache_dependencies.sh | 1 -
ci/images/google-geode-builder/scripts/setup.sh | 25 ++--
ci/pipelines/clean_fork_pipelines.sh | 58 ++++++++
ci/pipelines/deploy_meta.sh | 2 +-
ci/pipelines/geode-build/base.yml | 149 ++++-----------------
ci/pipelines/geode-build/deploy_pipeline.sh | 2 +-
ci/pipelines/geode-build/test-stubs/acceptance.yml | 8 +-
.../{integration.yml => distributed.yml} | 17 ++-
.../geode-build/test-stubs/integration.yml | 12 +-
ci/pipelines/geode-build/test-stubs/ui.yml | 12 +-
.../geode-build/test-stubs/{ui.yml => upgrade.yml} | 15 ++-
ci/pipelines/geode-build/test-template.yml | 135 +++++++++++++------
ci/pipelines/images/deploy_images_pipeline.sh | 2 +-
ci/pipelines/images/images.yml | 84 +++++++++++-
ci/pipelines/pull-request/deploy_pr_pipeline.sh | 2 +-
ci/scripts/archive_results.sh | 136 +++++++++++++++++++
ci/scripts/capture-call-stacks.sh | 27 +++-
ci/scripts/execute_tests.sh | 74 ++++++++++
.../prepopulate_build.sh} | 28 +++-
.../build_image.sh => scripts/rsync_code_down.sh} | 23 +++-
.../build_image.sh => scripts/rsync_code_up.sh} | 23 +++-
ci/scripts/start_instance.sh | 84 ++++++++++++
.../build_image.sh => scripts/stop_instance.sh} | 21 ++-
gradle/test.gradle | 1 +
31 files changed, 825 insertions(+), 289 deletions(-)
diff --git a/ci/pipelines/geode-build/test-stubs/integration.yml
b/ci/images/alpine-tools/Dockerfile
similarity index 59%
copy from ci/pipelines/geode-build/test-stubs/integration.yml
copy to ci/images/alpine-tools/Dockerfile
index e672f64..ab49980 100644
--- a/ci/pipelines/geode-build/test-stubs/integration.yml
+++ b/ci/images/alpine-tools/Dockerfile
@@ -13,15 +13,20 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+FROM alpine:3.7
-metadata:
- job:
- name: IntegrationTest
- gradle_task: integrationTest
- artifact_slug: integrationtestfiles
- dunit:
- parallel: true
- forks: 1
- call_stack_timeout: 25200
- timeout: 8h
- size: []
+COPY --from=google/cloud-sdk:alpine /google-cloud-sdk /google-cloud-sdk
+COPY --from=hashicorp/packer:latest /bin/packer /usr/local/bin/packer
+ENV PATH /google-cloud-sdk/bin:$PATH
+RUN apk --no-cache add \
+ bash \
+ curl \
+ git \
+ jq \
+ openssh-client \
+ python \
+ py2-pip \
+ rsync \
+ && gcloud config set core/disable_usage_reporting true \
+ && gcloud config set component_manager/disable_update_check true \
+ && gcloud config set metrics/environment github_docker_image
\ No newline at end of file
diff --git a/ci/pipelines/geode-build/test-stubs/flaky.yml
b/ci/images/concourse-metadata-resource/Dockerfile
similarity index 79%
rename from ci/pipelines/geode-build/test-stubs/flaky.yml
rename to ci/images/concourse-metadata-resource/Dockerfile
index b5d7afb..cdcfe9e 100644
--- a/ci/pipelines/geode-build/test-stubs/flaky.yml
+++ b/ci/images/concourse-metadata-resource/Dockerfile
@@ -13,15 +13,11 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+FROM alpine
-metadata:
- job:
- name: FlakyTest
- gradle_task: flakyTest
- artifact_slug: flakytestfiles
- dunit:
- parallel: false
- forks: 0
- call_stack_timeout: 25200
- timeout: 8h
- size: []
+RUN apk --no-cache add \
+ bash \
+ jq \
+ util-linux
+RUN mkdir -p /opt/resource
+ADD files/check files/in files/out /opt/resource/
diff --git a/ci/pipelines/geode-build/test-stubs/ui.yml
b/ci/images/concourse-metadata-resource/files/check
old mode 100644
new mode 100755
similarity index 80%
copy from ci/pipelines/geode-build/test-stubs/ui.yml
copy to ci/images/concourse-metadata-resource/files/check
index 60404c6..21302c0
--- a/ci/pipelines/geode-build/test-stubs/ui.yml
+++ b/ci/images/concourse-metadata-resource/files/check
@@ -1,3 +1,4 @@
+#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -14,14 +15,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-metadata:
- job:
- name: UITests
- gradle_task: uiTest
- artifact_slug: uitestfiles
- dunit:
- parallel: true
- forks: 1
- call_stack_timeout: 25200
- timeout: 2h
- size: []
+echo "Running concourse-metadata script" 1>&2
+
+SCRIPT_INPUT='/tmp/input'
+cat > $SCRIPT_INPUT <&0 # STDIN params
+
+OUTPUT_STRING="[ ]"
+
+echo $OUTPUT_STRING
+
+exit 0
\ No newline at end of file
diff --git a/ci/images/concourse-metadata-resource/files/in
b/ci/images/concourse-metadata-resource/files/in
new file mode 100755
index 0000000..de756f7
--- /dev/null
+++ b/ci/images/concourse-metadata-resource/files/in
@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+DEST_DIR=$1
+
+SCRIPT_INPUT='/tmp/input'
+cat > $SCRIPT_INPUT <&0 # STDIN params
+
+VERSION=$(jq -r '.version.ref // ""' < $SCRIPT_INPUT)
+
+OUTPUT_STRING=$(cat <<EOF
+{ "version": { "ref": "$VERSION" },
+ "metadata": [
+ { "name": "BUILD_ID", "value": "$BUILD_ID" },
+ { "name": "BUILD_NAME", "value": "$BUILD_NAME" },
+ { "name": "BUILD_JOB_NAME", "value": "$BUILD_JOB_NAME" },
+ { "name": "BUILD_PIPELINE_NAME", "value": "$BUILD_PIPELINE_NAME" },
+ { "name": "BUILD_TEAM_NAME", "value": "$BUILD_TEAM_NAME" },
+ { "name": "ATC_EXTERNAL_URL", "value": "$ATC_EXTERNAL_URL" }
+ ]
+}
+EOF
+)
+
+echo "export BUILD_ID=$BUILD_ID" > $DEST_DIR/concourse_metadata
+echo "export BUILD_NAME=$BUILD_NAME" >> $DEST_DIR/concourse_metadata
+echo "export BUILD_JOB_NAME=$BUILD_JOB_NAME" >> $DEST_DIR/concourse_metadata
+echo "export BUILD_PIPELINE_NAME=$BUILD_PIPELINE_NAME" >>
$DEST_DIR/concourse_metadata
+echo "export BUILD_TEAM_NAME=$BUILD_TEAM_NAME" >> $DEST_DIR/concourse_metadata
+echo "export ATC_EXTERNAL_URL=$ATC_EXTERNAL_URL" >>
$DEST_DIR/concourse_metadata
+
+echo $OUTPUT_STRING
+
+exit 0
\ No newline at end of file
diff --git a/ci/pipelines/geode-build/test-stubs/ui.yml
b/ci/images/concourse-metadata-resource/files/out
old mode 100644
new mode 100755
similarity index 80%
copy from ci/pipelines/geode-build/test-stubs/ui.yml
copy to ci/images/concourse-metadata-resource/files/out
index 60404c6..7044e8a
--- a/ci/pipelines/geode-build/test-stubs/ui.yml
+++ b/ci/images/concourse-metadata-resource/files/out
@@ -1,3 +1,4 @@
+#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -14,14 +15,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-metadata:
- job:
- name: UITests
- gradle_task: uiTest
- artifact_slug: uitestfiles
- dunit:
- parallel: true
- forks: 1
- call_stack_timeout: 25200
- timeout: 2h
- size: []
+SCRIPT_INPUT='/tmp/input'
+cat > $SCRIPT_INPUT <&0 # STDIN params
+
+VERSION=$(uuidgen)
+
+OUTPUT_STRING="{ \"version\": { \"ref\": \"$VERSION\" } }"
+
+echo $OUTPUT_STRING
+
+exit 0
diff --git a/ci/images/google-geode-builder/build_image.sh
b/ci/images/google-geode-builder/build_image.sh
index e033de6..4a369a8 100755
--- a/ci/images/google-geode-builder/build_image.sh
+++ b/ci/images/google-geode-builder/build_image.sh
@@ -25,10 +25,23 @@ done
SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
pushd ${SCRIPTDIR}
-echo "Setting up credentials for packer"
-#CREDENTIALS_FILE=concourse-key.json
-#echo "${GCP_CONCOURSE_KEY}" | base64 -d > ${CREDENTIALS_FILE}
-#export GOOGLE_APPLICATION_CREDENTIALS=${CREDENTIALS_FILE}
+GEODE_BRANCH=${GEODE_BRANCH:-$(git rev-parse --abbrev-ref HEAD)}
+SANITIZED_GEODE_BRANCH=$(echo ${GEODE_BRANCH} | tr "/" "-" | tr '[:upper:]'
'[:lower:]')
+IMAGE_FAMILY_PREFIX=""
+GEODE_DOCKER_IMAGE=${GEODE_DOCKER_IMAGE:-"gcr.io/apachegeode-ci/apachegeode-build-concourse"}
+if [[ -z "${GEODE_FORK}" ]]; then
+ echo "GEODE_FORK environment variable must be set for this script to work."
+ exit 1
+fi
+
+
+if [[ "${GEODE_FORK}" != "apache" ]]; then
+ IMAGE_FAMILY_PREFIX="${GEODE_FORK}-${SANITIZED_GEODE_BRANCH}-"
+fi
+
echo "Running packer"
-packer build packer.json
+packer build \
+ --var "geode_docker_image=${GEODE_DOCKER_IMAGE}" \
+ --var "image_family_prefix=${IMAGE_FAMILY_PREFIX}" \
+ packer.json
diff --git a/ci/images/google-geode-builder/packer.json
b/ci/images/google-geode-builder/packer.json
index 59cc529..b930fd1 100644
--- a/ci/images/google-geode-builder/packer.json
+++ b/ci/images/google-geode-builder/packer.json
@@ -1,4 +1,8 @@
{
+ "variables": {
+ "image_family_prefix": "",
+ "geode_docker_image": ""
+ },
"provisioners": [
{
"type": "shell",
@@ -6,12 +10,15 @@
},
{
"type": "shell",
+ "environment_vars": [
+ "GEODE_DOCKER_IMAGE={{user `geode_docker_image`}}"
+ ],
"execute_command": "sudo -S sh -c '{{ .Vars }} {{ .Path }}'",
"script": "scripts/setup.sh"
},
{
"type": "shell",
- "execute_command": "sudo -S sh -c '{{ .Vars }} {{ .Path }}'",
+ "execute_command": "sudo -u geode -S sh -c '{{ .Vars }} {{ .Path }}'",
"script": "scripts/cache_dependencies.sh"
}
],
@@ -22,8 +29,8 @@
"source_image_family": "debian-9",
"ssh_username": "packer",
"zone": "us-central1-f",
- "image_family": "geode-builder",
- "image_name": "geode-builder-{{timestamp}}"
+ "image_family": "{{user `image_family_prefix`}}geode-builder",
+ "image_name": "{{user `image_family_prefix`}}geode-builder-{{timestamp}}"
}
]
}
diff --git a/ci/images/google-geode-builder/scripts/cache_dependencies.sh
b/ci/images/google-geode-builder/scripts/cache_dependencies.sh
index c75cbdb..954bcf4 100644
--- a/ci/images/google-geode-builder/scripts/cache_dependencies.sh
+++ b/ci/images/google-geode-builder/scripts/cache_dependencies.sh
@@ -18,7 +18,6 @@
set -e
WORK_DIR=$(mktemp -d)
-export GRADLE_USER_HOME=/usr/local/maven_files
pushd ${WORK_DIR}
git clone -b develop --depth 1 https://github.com/apache/geode.git geode
diff --git a/ci/images/google-geode-builder/scripts/setup.sh
b/ci/images/google-geode-builder/scripts/setup.sh
index 50c4d61..19f66e4 100755
--- a/ci/images/google-geode-builder/scripts/setup.sh
+++ b/ci/images/google-geode-builder/scripts/setup.sh
@@ -17,25 +17,21 @@
set -x
set -e
-
+export CLOUD_SDK_VERSION=209.0.0
export CHROME_DRIVER_VERSION=2.35
-export GRADLE_USER_HOME=/usr/local/maven_files
export LOCAL_USER=geode
-mkdir -p ${GRADLE_USER_HOME}
apt-get update
apt-get install -y --no-install-recommends \
apt-transport-https \
lsb-release
echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >
/etc/apt/sources.list.d/google-chrome.list
-echo "deb http://packages.cloud.google.com/apt cloud-sdk-$(lsb_release -c -s)
main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
echo "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release
-cs) stable" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
curl -sSL https://dl.google.com/linux/linux_signing_key.pub | apt-key add -
-curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add -
apt-get update
-apt-get purge lxc-docker
+apt-get purge -y google-cloud-sdk lxc-docker
apt-get install -y --no-install-recommends \
aptitude \
ca-certificates \
@@ -43,20 +39,29 @@ apt-get install -y --no-install-recommends \
docker-compose \
docker-ce \
git \
- golang \
google-chrome-stable \
- google-cloud-sdk \
htop \
jq \
+ less \
openjdk-8-jdk-headless \
python3 \
python3-pip \
rsync \
+ tmux \
unzip \
vim
+pushd /tmp
+ curl -O
https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-x86_64.tar.gz
+ tar xzf google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-x86_64.tar.gz -C /
+ rm google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-x86_64.tar.gz
+popd
+export PATH=/google-cloud-sdk/bin:${PATH}
gcloud config set core/disable_usage_reporting true
gcloud config set component_manager/disable_update_check true
gcloud config set metrics/environment github_docker_image
+gcloud components install docker-credential-gcr --quiet
+gcloud auth configure-docker --quiet
+docker pull ${GEODE_DOCKER_IMAGE}
curl -Lo /usr/local/bin/dunit-progress
https://github.com/jdeppe-pivotal/progress-util/releases/download/0.2/progress.linux
chmod +x /usr/local/bin/dunit-progress
wget --no-verbose -O /tmp/chromedriver_linux64.zip
https://chromedriver.storage.googleapis.com/${CHROME_DRIVER_VERSION}/chromedriver_linux64.zip
@@ -68,7 +73,7 @@ chmod 755 /opt/selenium/chromedriver-${CHROME_DRIVER_VERSION}
ln -fs /opt/selenium/chromedriver-${CHROME_DRIVER_VERSION}
/usr/bin/chromedriver
adduser --disabled-password --gecos "" ${LOCAL_USER}
usermod -G docker,google-sudoers -a ${LOCAL_USER}
-chown -R ${LOCAL_USER} ${GRADLE_USER_HOME}
-echo "export GRADLE_USER_HOME=${GRADLE_USER_HOME}" >
/etc/profile.d/gradle_home.sh
+echo "export PATH=/google-cloud-sdk/bin:${PATH}" >
/etc/profile.d/google_sdk_path.sh
+
apt-get clean
rm -rf /var/lib/apt/lists/*
diff --git a/ci/pipelines/clean_fork_pipelines.sh
b/ci/pipelines/clean_fork_pipelines.sh
new file mode 100755
index 0000000..4d6c03b
--- /dev/null
+++ b/ci/pipelines/clean_fork_pipelines.sh
@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+TARGET=geode
+GEODE_FORK=${1}
+CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
+GEODE_BRANCH=${2:-${CURRENT_BRANCH}}
+SANITIZED_GEODE_BRANCH=$(echo ${GEODE_BRANCH} | tr "/" "-" | tr '[:upper:]'
'[:lower:]')
+TEAM=$(fly targets | grep ^${TARGET} | awk '{print $3}')
+
+if [[ -z "${GEODE_FORK}" ]]; then
+ echo "No fork provided!"
+ exit 1
+fi
+
+echo "Fork is ${GEODE_FORK}"
+echo "Branch is ${GEODE_BRANCH}"
+
+echo "Deleting meta pipeline if it exists..."
+META_PIPELINE="meta-${GEODE_FORK}-${SANITIZED_GEODE_BRANCH}"
+fly -t ${TARGET} destroy-pipeline --non-interactive -p ${META_PIPELINE}
+
+echo "Deleting images pipeline if it exists..."
+IMAGES_PIPELINE="${GEODE_FORK}-${SANITIZED_GEODE_BRANCH}-images"
+fly -t ${TARGET} destroy-pipeline --non-interactive -p ${IMAGES_PIPELINE}
+
+echo "Deleting build pipeline if it exists..."
+BUILD_PIPELINE="${GEODE_FORK}-${SANITIZED_GEODE_BRANCH}"
+fly -t ${TARGET} destroy-pipeline --non-interactive -p ${BUILD_PIPELINE}
+
+gcloud container images list | grep "${GEODE_FORK}-${SANITIZED_GEODE_BRANCH}"
| while IFS= read -r line; do
+ echo "Deleting image: ${line}"
+ gcloud container images delete ${line}:latest --quiet
+ gcloud container images list-tags ${line} --filter='-tags:*'
--format='get(digest)' | while IFS= read -r line2; do
+ echo "Deleting image: ${line2}"
+ gcloud container images delete ${line}@${line2} --quiet
+ done
+done
+
+gcloud compute images list | awk "/^${GEODE_FORK}-${SANITIZED_GEODE_BRANCH}/
{print \$1}" | while IFS= read -r line; do
+ echo "Deleting image: ${line}"
+ gcloud compute images delete ${line} --quiet
+done
+
diff --git a/ci/pipelines/deploy_meta.sh b/ci/pipelines/deploy_meta.sh
index ceaa11a..611164f 100755
--- a/ci/pipelines/deploy_meta.sh
+++ b/ci/pipelines/deploy_meta.sh
@@ -16,7 +16,7 @@
# limitations under the License.
GEODE_BRANCH=$(git rev-parse --abbrev-ref HEAD)
-SANITIZED_GEODE_BRANCH=$(echo ${GEODE_BRANCH} | tr "/" "-")
+SANITIZED_GEODE_BRANCH=$(echo ${GEODE_BRANCH} | tr "/" "-" | tr '[:upper:]'
'[:lower:]')
TARGET=geode
GEODE_FORK=${1:-apache}
TEAM=$(fly targets | grep ^${TARGET} | awk '{print $3}')
diff --git a/ci/pipelines/geode-build/base.yml
b/ci/pipelines/geode-build/base.yml
index 073a990..3733b9d 100644
--- a/ci/pipelines/geode-build/base.yml
+++ b/ci/pipelines/geode-build/base.yml
@@ -26,6 +26,13 @@ resource_types:
type: docker-image
source:
repository: pcfseceng/email-resource
+ - name: concourse-metadata-resource
+ type: docker-image
+ source:
+ username: ((!docker-username))
+ password: ((!docker-password))
+ repository:
gcr.io/apachegeode-ci/((!docker-image-prefix))concourse-metadata-resource
+ tag: latest
resources:
- name: docker-geode-build-image
@@ -35,9 +42,17 @@ resources:
password: ((!docker-password))
repository:
gcr.io/apachegeode-ci/((!docker-image-prefix))((!docker-image-name))
tag: latest
+- name: alpine-tools-image
+ type: docker-image
+ source:
+ username: ((!docker-username))
+ password: ((!docker-password))
+ repository: gcr.io/apachegeode-ci/((!docker-image-prefix))alpine-tools
+ tag: latest
- name: geode
type: git
source:
+ depth: 1
uri: (( concat "https://github.com/"; metadata.geode-fork "/geode.git" ))
branch: (( grab metadata.geode-build-branch ))
ignore_paths:
@@ -81,6 +96,8 @@ resources:
password: ((!source-email-password))
from: ((!source-email-address))
to: [ ((!notification-email-address)) ]
+- name: concourse-metadata-resource
+ type: concourse-metadata-resource
groups:
- name: main
@@ -89,7 +106,7 @@ groups:
- DistributedTest
- AcceptanceTest
- IntegrationTest
- - FlakyTest
+ - UpgradeTest
- TestExamples
- UpdatePassingRef
- name: UITests
@@ -107,11 +124,16 @@ jobs:
- get: geode-ci
- get: geode-build-version
params: {pre: build}
- - get: docker-geode-build-image
- task: build
- image: docker-geode-build-image
config:
platform: linux
+ image_resource:
+ type: docker-image
+ source:
+ username: ((!docker-username))
+ password: ((!docker-password))
+ repository:
gcr.io/apachegeode-ci/((!docker-image-prefix))((!docker-image-name))
+ tag: latest
inputs:
- name: geode
- name: geode-ci
@@ -195,124 +217,3 @@ jobs:
PUBLIC_BUCKET: ((!public-bucket))
run:
path: geode-ci/ci/scripts/update-passing-ref.sh
-
-- name: DistributedTest
- serial: true
- public: true
- plan:
- - aggregate:
- - get: geode
- passed: [Build]
- trigger: true
- - get: geode-ci
- - get: geode-build-version
- passed: [Build]
- - get: docker-geode-build-image
- params:
- rootfs: true
- - aggregate:
- - task: run-distributed-core
- image: docker-geode-build-image
- tags: [large]
- privileged: true
- timeout: 8h
- config:
- inputs:
- - name: geode
- - name: geode-ci
- - name: docker-geode-build-image
- - name: geode-build-version
- platform: linux
- outputs:
- - name: built-geode
- params:
- MAINTENANCE_VERSION: (( grab metadata.geode-build-branch ))
- SERVICE_ACCOUNT: ((!concourse-gcp-account))
- PUBLIC_BUCKET: ((!public-bucket))
- PARALLEL_DUNIT: true
- DUNIT_PARALLEL_FORKS: 7
- CALL_STACK_TIMEOUT: 25200
- run:
- args:
- - geode-core:distributedTest
- - distributedtestcore
- path: geode-ci/ci/scripts/test-run.sh
- on_failure:
- aggregate:
- - put: send-notification-email
- params:
- subject: built-geode/subject
- body: built-geode/body
- ensure:
- aggregate:
- - task: archive-results-core
- image: docker-geode-build-image
- config:
- inputs:
- - name: geode
- - name: geode-ci
- - name: geode-build-version
- - name: built-geode
- platform: linux
- params:
- MAINTENANCE_VERSION: (( grab metadata.geode-build-branch ))
- SERVICE_ACCOUNT: ((!concourse-gcp-account))
- PUBLIC_BUCKET: ((!public-bucket))
- run:
- args:
- - geode-core:distributedTest
- - distributedtestcore
- path: geode-ci/ci/scripts/test-archive.sh
- - task: run-distributed-everything-else
- image: docker-geode-build-image
- tags: [large]
- privileged: true
- timeout: 8h
- config:
- inputs:
- - name: geode
- - name: geode-ci
- - name: docker-geode-build-image
- - name: geode-build-version
- platform: linux
- outputs:
- - name: built-geode
- params:
- MAINTENANCE_VERSION: (( grab metadata.geode-build-branch ))
- SERVICE_ACCOUNT: ((!concourse-gcp-account))
- PUBLIC_BUCKET: ((!public-bucket))
- PARALLEL_DUNIT: true
- DUNIT_PARALLEL_FORKS: 7
- CALL_STACK_TIMEOUT: 25200
- GRADLE_TASK_OPTIONS: -x geode-core:distributedTest
- run:
- args:
- - distributedTest
- - distributedtesteverythingelse
- path: geode-ci/ci/scripts/test-run.sh
- on_failure:
- aggregate:
- - put: send-notification-email
- params:
- subject: built-geode/subject
- body: built-geode/body
- ensure:
- aggregate:
- - task: archive-results-everythingelse
- image: docker-geode-build-image
- config:
- inputs:
- - name: geode
- - name: geode-ci
- - name: geode-build-version
- - name: built-geode
- platform: linux
- params:
- MAINTENANCE_VERSION: (( grab metadata.geode-build-branch ))
- SERVICE_ACCOUNT: ((!concourse-gcp-account))
- PUBLIC_BUCKET: ((!public-bucket))
- run:
- args:
- - disitributedTest
- - distributedtesteverythingelse
- path: geode-ci/ci/scripts/test-archive.sh
diff --git a/ci/pipelines/geode-build/deploy_pipeline.sh
b/ci/pipelines/geode-build/deploy_pipeline.sh
index ebb2db0..59c22ab 100755
--- a/ci/pipelines/geode-build/deploy_pipeline.sh
+++ b/ci/pipelines/geode-build/deploy_pipeline.sh
@@ -45,7 +45,7 @@ if [ "${GEODE_BRANCH}" = "HEAD" ]; then
exit 1
fi
-SANITIZED_GEODE_BRANCH=$(echo ${GEODE_BRANCH} | tr "/" "-")
+SANITIZED_GEODE_BRANCH=$(echo ${GEODE_BRANCH} | tr "/" "-" | tr '[:upper:]'
'[:lower:]')
BIN_DIR=${OUTPUT_DIRECTORY}/bin
TMP_DIR=${OUTPUT_DIRECTORY}/tmp
diff --git a/ci/pipelines/geode-build/test-stubs/acceptance.yml
b/ci/pipelines/geode-build/test-stubs/acceptance.yml
index 45da578..0cd23bc 100644
--- a/ci/pipelines/geode-build/test-stubs/acceptance.yml
+++ b/ci/pipelines/geode-build/test-stubs/acceptance.yml
@@ -22,6 +22,10 @@ metadata:
dunit:
parallel: false
forks: 0
- call_stack_timeout: 25200
- timeout: 8h
+ cpus: 8
+# specified in Gigabytes.
+ ram: 30
+# specified in seconds
+ call_stack_timeout: 1800
+ timeout: 1h
size: []
diff --git a/ci/pipelines/geode-build/test-stubs/integration.yml
b/ci/pipelines/geode-build/test-stubs/distributed.yml
similarity index 74%
copy from ci/pipelines/geode-build/test-stubs/integration.yml
copy to ci/pipelines/geode-build/test-stubs/distributed.yml
index e672f64..6a4c42d 100644
--- a/ci/pipelines/geode-build/test-stubs/integration.yml
+++ b/ci/pipelines/geode-build/test-stubs/distributed.yml
@@ -16,12 +16,17 @@
metadata:
job:
- name: IntegrationTest
- gradle_task: integrationTest
- artifact_slug: integrationtestfiles
+ name: DistributedTest
+ gradle_task: distributedTest
+ artifact_slug: distributedtestfiles
dunit:
parallel: true
- forks: 1
- call_stack_timeout: 25200
- timeout: 8h
+# max number of docker containers to run, generally cpus/2
+ forks: 48
+ cpus: 96
+# specified in Gigabytes.
+ ram: 210
+# specified in seconds
+ call_stack_timeout: 7200
+ timeout: 3h
size: []
diff --git a/ci/pipelines/geode-build/test-stubs/integration.yml
b/ci/pipelines/geode-build/test-stubs/integration.yml
index e672f64..df921a4 100644
--- a/ci/pipelines/geode-build/test-stubs/integration.yml
+++ b/ci/pipelines/geode-build/test-stubs/integration.yml
@@ -20,8 +20,12 @@ metadata:
gradle_task: integrationTest
artifact_slug: integrationtestfiles
dunit:
- parallel: true
- forks: 1
- call_stack_timeout: 25200
- timeout: 8h
+ parallel: false
+ forks: 0
+ cpus: 8
+# specified in Gigabytes.
+ ram: 30
+# specified in seconds
+ call_stack_timeout: 7200
+ timeout: 3h
size: []
diff --git a/ci/pipelines/geode-build/test-stubs/ui.yml
b/ci/pipelines/geode-build/test-stubs/ui.yml
index 60404c6..ff6ee8d 100644
--- a/ci/pipelines/geode-build/test-stubs/ui.yml
+++ b/ci/pipelines/geode-build/test-stubs/ui.yml
@@ -20,8 +20,12 @@ metadata:
gradle_task: uiTest
artifact_slug: uitestfiles
dunit:
- parallel: true
- forks: 1
- call_stack_timeout: 25200
- timeout: 2h
+ parallel: false
+ forks: 0
+ cpus: 8
+# specified in Gigabytes.
+ ram: 30
+# specified in seconds
+ call_stack_timeout: 1800
+ timeout: 1h
size: []
diff --git a/ci/pipelines/geode-build/test-stubs/ui.yml
b/ci/pipelines/geode-build/test-stubs/upgrade.yml
similarity index 76%
copy from ci/pipelines/geode-build/test-stubs/ui.yml
copy to ci/pipelines/geode-build/test-stubs/upgrade.yml
index 60404c6..4e94b4e 100644
--- a/ci/pipelines/geode-build/test-stubs/ui.yml
+++ b/ci/pipelines/geode-build/test-stubs/upgrade.yml
@@ -16,12 +16,17 @@
metadata:
job:
- name: UITests
- gradle_task: uiTest
- artifact_slug: uitestfiles
+ name: UpgradeTest
+ gradle_task: upgradeTest
+ artifact_slug: upgradetestfiles
dunit:
parallel: true
- forks: 1
- call_stack_timeout: 25200
+# max number of docker containers to run, generally cpus/2
+ forks: 48
+ cpus: 96
+# specified in Gigabytes.
+ ram: 210
+# specified in seconds
+ call_stack_timeout: 3600
timeout: 2h
size: []
diff --git a/ci/pipelines/geode-build/test-template.yml
b/ci/pipelines/geode-build/test-template.yml
index a29e762..ecbb47f 100644
--- a/ci/pipelines/geode-build/test-template.yml
+++ b/ci/pipelines/geode-build/test-template.yml
@@ -17,33 +17,67 @@
---
jobs:
- name: (( grab metadata.job.name ))
- serial: true
public: true
plan:
+ - do:
- aggregate:
+ - put: concourse-metadata-resource
- get: geode
passed: [Build]
- trigger: true
- get: geode-ci
- get: geode-build-version
passed: [Build]
- - get: docker-geode-build-image
+ trigger: true
+ - get: alpine-tools-image
params:
rootfs: true
- - task: runtests
- image: docker-geode-build-image
- tags: (( grab metadata.job.size ))
- privileged: true
- timeout: (( grab metadata.job.timeout ))
+ - task: start_instance
+ image: alpine-tools-image
+ timeout: 5m
+ config:
+ inputs:
+ - name: concourse-metadata-resource
+ - name: geode-ci
+ outputs:
+ - name: instance-data
+ platform: linux
+ params:
+ CPUS: (( grab metadata.job.cpus ))
+ RAM: (( grab metadata.job.ram ))
+ GEODE_BRANCH: (( grab metadata.geode-build-branch ))
+ GEODE_FORK: (( grab metadata.geode-fork ))
+ run:
+ path: geode-ci/ci/scripts/start_instance.sh
+ - task: rsync_code_up
+ image: alpine-tools-image
+ timeout: 5m
+ config:
+ inputs:
+ - name: geode-ci
+ - name: geode
+ - name: instance-data
+ platform: linux
+ run:
+ path: geode-ci/ci/scripts/rsync_code_up.sh
+ - task: prepopulate_build
+ image: alpine-tools-image
config:
inputs:
+ - name: geode-ci
- name: geode
+ - name: instance-data
+ platform: linux
+ run:
+ path: geode-ci/ci/scripts/prepopulate_build.sh
+ - task: execute_tests
+ image: alpine-tools-image
+ timeout: (( grab metadata.job.timeout ))
+ config:
+ inputs:
- name: geode-ci
- - name: docker-geode-build-image
- - name: geode-build-version
+ - name: geode
+ - name: instance-data
platform: linux
- outputs:
- - name: built-geode
params:
MAINTENANCE_VERSION: (( grab metadata.geode-build-branch ))
SERVICE_ACCOUNT: ((!concourse-gcp-account))
@@ -51,34 +85,53 @@ jobs:
PARALLEL_DUNIT: (( grab metadata.job.dunit.parallel ))
DUNIT_PARALLEL_FORKS: (( grab metadata.job.dunit.forks ))
CALL_STACK_TIMEOUT: (( grab metadata.job.call_stack_timeout ))
+ GRADLE_TASK: (( grab metadata.job.gradle_task ))
+ GRADLE_TASK_OPTIONS: (( grab metadata.job.gradle_task_options || ""
))
+ ARTIFACT_SLUG: (( grab metadata.job.artifact_slug ))
run:
- args:
- - (( grab metadata.job.gradle_task ))
- - (( grab metadata.job.artifact_slug ))
- path: geode-ci/ci/scripts/test-run.sh
- on_failure:
- aggregate:
- - put: send-notification-email
- params:
- subject: built-geode/subject
- body: built-geode/body
+ path: geode-ci/ci/scripts/execute_tests.sh
+ ensure:
+ do:
+ - task: rsync_code_down
+ image: alpine-tools-image
+ timeout: 5m
+ config:
+ inputs:
+ - name: geode-ci
+ - name: instance-data
+ outputs:
+ - name: geode-results
+ platform: linux
+ run:
+ path: geode-ci/ci/scripts/rsync_code_down.sh
ensure:
- aggregate:
- - task: archive-results
- image: docker-geode-build-image
- config:
- inputs:
- - name: geode
- - name: geode-ci
- - name: geode-build-version
- - name: built-geode
- platform: linux
- params:
- MAINTENANCE_VERSION: (( grab metadata.geode-build-branch ))
- SERVICE_ACCOUNT: ((!concourse-gcp-account))
- PUBLIC_BUCKET: ((!public-bucket))
- run:
- args:
- - (( grab metadata.job.gradle_task ))
- - (( grab metadata.job.artifact_slug ))
- path: geode-ci/ci/scripts/test-archive.sh
+ do:
+ - aggregate:
+ - task: archive_results
+ image: alpine-tools-image
+ timeout: 1h
+ config:
+ inputs:
+ - name: concourse-metadata-resource
+ - name: geode-ci
+ - name: geode-build-version
+ - name: geode-results
+ platform: linux
+ params:
+ MAINTENANCE_VERSION: (( grab metadata.geode-build-branch ))
+ SERVICE_ACCOUNT: ((!concourse-gcp-account))
+ PUBLIC_BUCKET: ((!public-bucket))
+ GRADLE_TASK: (( grab metadata.job.gradle_task ))
+ ARTIFACT_SLUG: (( grab metadata.job.artifact_slug ))
+ run:
+ path: geode-ci/ci/scripts/archive_results.sh
+ - task: stop_instance
+ image: alpine-tools-image
+ timeout: 1h
+ config:
+ inputs:
+ - name: geode-ci
+ - name: instance-data
+ platform: linux
+ run:
+ path: geode-ci/ci/scripts/stop_instance.sh
diff --git a/ci/pipelines/images/deploy_images_pipeline.sh
b/ci/pipelines/images/deploy_images_pipeline.sh
index 96b060c..7e9844c 100755
--- a/ci/pipelines/images/deploy_images_pipeline.sh
+++ b/ci/pipelines/images/deploy_images_pipeline.sh
@@ -46,7 +46,7 @@ if [ "${GEODE_BRANCH}" = "HEAD" ]; then
exit 1
fi
-SANITIZED_GEODE_BRANCH=$(echo ${GEODE_BRANCH} | tr "/" "-")
+SANITIZED_GEODE_BRANCH=$(echo ${GEODE_BRANCH} | tr "/" "-" | tr '[:upper:]'
'[:lower:]')
MAX_IN_FLIGHT=5
BIN_DIR=${OUTPUT_DIRECTORY}/bin
diff --git a/ci/pipelines/images/images.yml b/ci/pipelines/images/images.yml
index f62eb9c..373eee2 100644
--- a/ci/pipelines/images/images.yml
+++ b/ci/pipelines/images/images.yml
@@ -38,7 +38,6 @@ resources:
source:
repository: openjdk
tag: 8
- skip_download: true
- name: google-geode-builder
type: git
@@ -48,6 +47,50 @@ resources:
paths:
- ci/images/google-geode-builder
+- name: alpine37-docker-image
+ type: docker-image
+ source:
+ repository: alpine
+ tag: 3.7
+ skip_download: true
+
+- name: alpine-docker-image
+ type: docker-image
+ source:
+ repository: alpine
+ tag: latest
+ skip_download: true
+
+- name: alpine-tools-dockerfile
+ type: git
+ source:
+ uri: https://github.com/((!geode-fork))/geode.git
+ branch: ((!geode-build-branch))
+ paths:
+ - ci/images/alpine-tools/*
+
+- name: alpine-tools-docker-image
+ type: docker-image
+ source:
+ username: ((!docker-username))
+ password: ((!docker-password))
+ repository: gcr.io/apachegeode-ci/((!docker-image-prefix))alpine-tools
+
+- name: concourse-metadata-resource-dockerfile
+ type: git
+ source:
+ uri: https://github.com/((!geode-fork))/geode.git
+ branch: ((!geode-build-branch))
+ paths:
+ - ci/images/concourse-metadata-resource/*
+
+- name: concourse-metadata-resource-docker-image
+ type: docker-image
+ source:
+ username: ((!docker-username))
+ password: ((!docker-password))
+ repository:
gcr.io/apachegeode-ci/((!docker-image-prefix))concourse-metadata-resource
+
jobs:
# apachegeode-build-concourse
- name: build-apachegeode-build-concourse-docker-image
@@ -69,11 +112,14 @@ jobs:
serial: true
plan:
- aggregate:
+ - get: apachegeode-build-concourse-docker-image
+ passed: [build-apachegeode-build-concourse-docker-image]
+ trigger: true
- get: google-geode-builder
trigger: true
- - get: apachegeode-build-concourse-docker-image
+ - get: alpine-tools-docker-image
- task: build-image
- image: apachegeode-build-concourse-docker-image
+ image: alpine-tools-docker-image
config:
inputs:
- name: google-geode-builder
@@ -81,6 +127,36 @@ jobs:
- name: results
platform: linux
params:
- GCP_CONCOURSE_KEY: ((!concourse-gcp-key-encoded))
+ GEODE_FORK: ((!geode-fork))
+ GEODE_BRANCH: ((!geode-build-branch))
+ GEODE_DOCKER_IMAGE:
gcr.io/apachegeode-ci/((!docker-image-prefix))((!docker-image-name))
run:
path:
google-geode-builder/ci/images/google-geode-builder/build_image.sh
+
+- name: build-alpine-tools-docker-image
+ public: ((!public-pipelines))
+ serial: true
+ plan:
+ - aggregate:
+ - get: alpine37-docker-image
+ trigger: true
+ - get: alpine-tools-dockerfile
+ trigger: true
+ - put: alpine-tools-docker-image
+ params:
+ build: alpine-tools-dockerfile/ci/images/alpine-tools
+ tag_as_latest: true
+
+- name: build-concourse-metadata-resource-docker-image
+ public: ((!public-pipelines))
+ serial: true
+ plan:
+ - aggregate:
+ - get: alpine-docker-image
+ trigger: true
+ - get: concourse-metadata-resource-dockerfile
+ trigger: true
+ - put: concourse-metadata-resource-docker-image
+ params:
+ build:
concourse-metadata-resource-dockerfile/ci/images/concourse-metadata-resource
+ tag_as_latest: true
diff --git a/ci/pipelines/pull-request/deploy_pr_pipeline.sh
b/ci/pipelines/pull-request/deploy_pr_pipeline.sh
index ca102db..da2a2e1 100755
--- a/ci/pipelines/pull-request/deploy_pr_pipeline.sh
+++ b/ci/pipelines/pull-request/deploy_pr_pipeline.sh
@@ -46,7 +46,7 @@ if [ "${GEODE_BRANCH}" = "HEAD" ]; then
exit 1
fi
-SANITIZED_GEODE_BRANCH=$(echo ${GEODE_BRANCH} | tr "/" "-")
+SANITIZED_GEODE_BRANCH=$(echo ${GEODE_BRANCH} | tr "/" "-" | tr '[:upper:]'
'[:lower:]')
MAX_IN_FLIGHT=5
BIN_DIR=${OUTPUT_DIRECTORY}/bin
diff --git a/ci/scripts/archive_results.sh b/ci/scripts/archive_results.sh
new file mode 100755
index 0000000..bce4fcd
--- /dev/null
+++ b/ci/scripts/archive_results.sh
@@ -0,0 +1,136 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+BASE_DIR=$(pwd)
+
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a
symlink
+ SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+ SOURCE="$(readlink "$SOURCE")"
+ [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative
symlink, we need to resolve it relative to the path where the symlink file was
located
+done
+SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+
+source ${BASE_DIR}/concourse-metadata-resource/concourse_metadata
+
+BUILDROOT=$(pwd)
+DEST_DIR=${BUILDROOT}/geode-results
+
+if [[ -z "${GRADLE_TASK}" ]]; then
+ echo "GRADLE_TASK must be set. exiting..."
+ exit 1
+fi
+
+if [[ -z "${ARTIFACT_SLUG}" ]]; then
+ echo "ARTIFACT_SLUG must be set. exiting..."
+ exit 1
+fi
+
+SANITIZED_GRADLE_TASK=${GRADLE_TASK##*:}
+TMPDIR=${DEST_DIR}/tmp
+GEODE_BUILD=${DEST_DIR}/geode
+GEODE_BUILD_VERSION_NUMBER=$(grep "versionNumber *="
${GEODE_BUILD}/gradle.properties | awk -F "=" '{print $2}' | tr -d ' ')
+BUILD_TIMESTAMP=$(date +%s)
+
+GEODE_PULL_REQUEST_ID_FILE=${GEODE_BUILD}/.git/id
+if [ -e "${GEODE_PULL_REQUEST_ID_FILE}" ]; then
+ GEODE_PULL_REQUEST_ID=$(cat ${GEODE_PULL_REQUEST_ID_FILE})
+fi
+
+
+GEODE_BUILD_VERSION_FILE=${BUILDROOT}/geode-build-version/number
+
+if [ ! -e "${GEODE_BUILD_VERSION_FILE}" ] && [ -z "${GEODE_PULL_REQUEST_ID}"
]; then
+ echo "${GEODE_BUILD_VERSION_FILE} file does not exist. Concourse is probably
not configured correctly."
+ exit 1
+fi
+if [ -z ${MAINTENANCE_VERSION+x} ]; then
+ echo "MAINTENANCE_VERSION is unset. Check your pipeline configuration and
make sure this script is called properly."
+ exit 1
+fi
+
+if [ -z "${GEODE_PULL_REQUEST_ID}" ]; then
+CONCOURSE_VERSION=$(cat ${GEODE_BUILD_VERSION_FILE})
+ CONCOURSE_PRODUCT_VERSION=${CONCOURSE_VERSION%%-*}
+ GEODE_PRODUCT_VERSION=${GEODE_BUILD_VERSION_NUMBER}
+ CONCOURSE_BUILD_SLUG=${CONCOURSE_VERSION##*-}
+ BUILD_ID=${CONCOURSE_VERSION##*.}
+ FULL_PRODUCT_VERSION=${GEODE_PRODUCT_VERSION}-${CONCOURSE_BUILD_SLUG}
+
+ echo "Concourse VERSION is ${CONCOURSE_VERSION}"
+ echo "Geode product VERSION is ${GEODE_PRODUCT_VERSION}"
+ echo "Build ID is ${BUILD_ID}"
+else
+ FULL_PRODUCT_VERSION="geode-pr-${GEODE_PULL_REQUEST_ID}"
+fi
+
+directories_file=${DEST_DIR}/artifact_directories
+mkdir -p ${TMPDIR}
+
+echo "TMPDIR = ${TMPDIR}"
+echo "GRADLE_TASK = ${GRADLE_TASK}"
+echo "ARTIFACT_SLUG = ${ARTIFACT_SLUG}"
+
+gcloud config set account ${SERVICE_ACCOUNT}
+
+
+FILENAME=${ARTIFACT_SLUG}-${FULL_PRODUCT_VERSION}.tgz
+
+pushd ${GEODE_BUILD}
+
+ set +e
+ find . -type d -name "reports" > ${directories_file}
+ find . -type d -name "test-results" >> ${directories_file}
+ (find . -type d -name "*Test" | grep "build/[^/]*Test$") >>
${directories_file}
+ find . -name "*-progress*txt" >> ${directories_file}
+ find . -type d -name "callstacks" >> ${directories_file}
+ echo "Collecting the following artifacts..."
+ cat ${directories_file}
+ echo ""
+ tar zcf ${DEST_DIR}/${FILENAME} -T ${directories_file}
+popd
+
+ARTIFACTS_DESTINATION="${PUBLIC_BUCKET}/builds/${FULL_PRODUCT_VERSION}"
+TEST_RESULTS_DESTINATION="${ARTIFACTS_DESTINATION}/test-results/${SANITIZED_GRADLE_TASK}/${BUILD_TIMESTAMP}/"
+TEST_ARTIFACTS_DESTINATION="${ARTIFACTS_DESTINATION}/test-artifacts/${BUILD_TIMESTAMP}/"
+
+
+if [ ! -d "${GEODE_BUILD}/build/reports/combined" ]; then
+ echo "No tests exist, compile failed."
+ mkdir -p ${GEODE_BUILD}/build/reports/combined
+ echo "<html><head><title>No Test Results Were
Captured</title></head><body><h1>No Test Results Were
Captured</h1></body></html>" > ${GEODE_BUILD}/build/reports/combined/index.html
+fi
+
+pushd ${GEODE_BUILD}/build/reports/combined
+gsutil -q -m cp -r * gs://${TEST_RESULTS_DESTINATION}
+popd
+
+echo ""
+printf "\033[92m=-=-=-=-=-=-=-=-=-=-=-=-=-= Test Results Website
=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=\033[0m\n"
+printf "\033[92mhttp://${TEST_RESULTS_DESTINATION}\033[0m\n";
+printf
"\033[92m=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=\033[0m\n"
+printf "\n"
+
+gsutil cp ${DEST_DIR}/${FILENAME} gs://${TEST_ARTIFACTS_DESTINATION}
+
+printf "\033[92mTest artifacts from this job are available at:\033[0m\n"
+printf "\n"
+printf "\033[92mhttp://${TEST_ARTIFACTS_DESTINATION}${FILENAME}\033[0m\n";
diff --git a/ci/scripts/capture-call-stacks.sh
b/ci/scripts/capture-call-stacks.sh
index 1db552c..d162540 100755
--- a/ci/scripts/capture-call-stacks.sh
+++ b/ci/scripts/capture-call-stacks.sh
@@ -20,16 +20,35 @@
export TERM=${TERM:-dumb}
export PAGER=cat
-export BUILDROOT=$(pwd)
-export DEST_DIR=${BUILDROOT}/built-geode
-export GEODE_BUILD=${DEST_DIR}/test
+
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a
symlink
+ SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+ SOURCE="$(readlink "$SOURCE")"
+ [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative
symlink, we need to resolve it relative to the path where the symlink file was
located
+done
+SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+
+
+export GEODE_BUILD=~/geode
export CALLSTACKS_DIR=${GEODE_BUILD}/callstacks
#SLEEP_TIME is in seconds
-SLEEP_TIME=${1}
+PARALLEL_DUNIT=${1}
+SLEEP_TIME=${2}
COUNT=3
STACK_INTERVAL=5
+if [[ -z "${PARALLEL_DUNIT}" ]]; then
+ echo "PARALLEL_DUNIT must be set. exiting..."
+ exit 1
+fi
+
+if [[ -z "${SLEEP_TIME}" ]]; then
+ echo "SLEEP_TIME must be set. exiting..."
+ exit 1
+fi
+
mkdir -p ${CALLSTACKS_DIR}
diff --git a/ci/scripts/execute_tests.sh b/ci/scripts/execute_tests.sh
new file mode 100755
index 0000000..bf2dbb2
--- /dev/null
+++ b/ci/scripts/execute_tests.sh
@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+BASE_DIR=$(pwd)
+
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a
symlink
+ SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+ SOURCE="$(readlink "$SOURCE")"
+ [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative
symlink, we need to resolve it relative to the path where the symlink file was
located
+done
+SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+
+if [[ -z "${GRADLE_TASK}" ]]; then
+ echo "GRADLE_TASK must be set. exiting..."
+ exit 1
+fi
+
+REPODIR=$(cd geode; git rev-parse --show-toplevel)
+
+DEFAULT_GRADLE_TASK_OPTIONS="--parallel --console=plain --no-daemon -x javadoc
-x spotlessCheck -x rat"
+
+
+SSHKEY_FILE="instance-data/sshkey"
+
+INSTANCE_NAME="$(cat instance-data/instance-name)"
+INSTANCE_IP_ADDRESS="$(cat instance-data/instance-ip-address)"
+PROJECT="$(cat instance-data/project)"
+ZONE="$(cat instance-data/zone)"
+
+
+echo 'StrictHostKeyChecking no' >> /etc/ssh/ssh_config
+
+scp -i ${SSHKEY_FILE} ${SCRIPTDIR}/capture-call-stacks.sh
geode@${INSTANCE_IP_ADDRESS}:.
+
+
+
+if [[ -n "${PARALLEL_DUNIT}" && "${PARALLEL_DUNIT}" == "true" ]]; then
+ PARALLEL_DUNIT="-PparallelDunit"
+ if [ -n "${DUNIT_PARALLEL_FORKS}" ]; then
+ DUNIT_PARALLEL_FORKS="-PdunitParallelForks=${DUNIT_PARALLEL_FORKS}"
+ fi
+else
+ PARALLEL_DUNIT=""
+ DUNIT_PARALLEL_FORKS=""
+fi
+
+
+if [ -v CALL_STACK_TIMEOUT ]; then
+ ssh -i ${SSHKEY_FILE} geode@${INSTANCE_IP_ADDRESS} "tmux new-session -d -s
callstacks; tmux send-keys ~/capture-call-stacks.sh\ ${PARALLEL_DUNIT}\
${CALL_STACK_TIMEOUT} C-m"
+fi
+
+ssh -i ${SSHKEY_FILE} geode@${INSTANCE_IP_ADDRESS} "mkdir -p tmp && cd geode
&& ./gradlew ${DEFAULT_GRADLE_TASK_OPTIONS} -Dskip.tests=true build"
+
+ssh -i ${SSHKEY_FILE} geode@${INSTANCE_IP_ADDRESS} "mkdir -p tmp && cd geode
&& ./gradlew ${PARALLEL_DUNIT} ${DUNIT_PARALLEL_FORKS}
-PdunitDockerImage=\$(docker images --format '{{.Repository}}:{{.Tag}}') \
+ --system-prop java.io.tmpdir=/home/geode/tmp
${DEFAULT_GRADLE_TASK_OPTIONS} ${GRADLE_TASK} ${GRADLE_TASK_OPTIONS}"
diff --git a/ci/images/google-geode-builder/build_image.sh
b/ci/scripts/prepopulate_build.sh
similarity index 66%
copy from ci/images/google-geode-builder/build_image.sh
copy to ci/scripts/prepopulate_build.sh
index e033de6..ed1f432 100755
--- a/ci/images/google-geode-builder/build_image.sh
+++ b/ci/scripts/prepopulate_build.sh
@@ -1,4 +1,5 @@
#!/usr/bin/env bash
+
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -16,6 +17,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+set -e
+
+BASE_DIR=$(pwd)
+
SOURCE="${BASH_SOURCE[0]}"
while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a
symlink
SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
@@ -24,11 +29,20 @@ while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file
is no longer a symli
done
SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
-pushd ${SCRIPTDIR}
-echo "Setting up credentials for packer"
+REPODIR=$(cd geode; git rev-parse --show-toplevel)
+
+DEFAULT_GRADLE_TASK_OPTIONS="--parallel --console=plain --no-daemon -x javadoc
-x spotlessCheck -x rat"
+
+
+SSHKEY_FILE="instance-data/sshkey"
+
+INSTANCE_NAME="$(cat instance-data/instance-name)"
+INSTANCE_IP_ADDRESS="$(cat instance-data/instance-ip-address)"
+PROJECT="$(cat instance-data/project)"
+ZONE="$(cat instance-data/zone)"
+
+
+echo 'StrictHostKeyChecking no' >> /etc/ssh/ssh_config
+
+ssh -i ${SSHKEY_FILE} geode@${INSTANCE_IP_ADDRESS} "mkdir -p tmp && cd geode
&& ./gradlew ${DEFAULT_GRADLE_TASK_OPTIONS} -Dskip.tests=true build"
-#CREDENTIALS_FILE=concourse-key.json
-#echo "${GCP_CONCOURSE_KEY}" | base64 -d > ${CREDENTIALS_FILE}
-#export GOOGLE_APPLICATION_CREDENTIALS=${CREDENTIALS_FILE}
-echo "Running packer"
-packer build packer.json
diff --git a/ci/images/google-geode-builder/build_image.sh
b/ci/scripts/rsync_code_down.sh
similarity index 74%
copy from ci/images/google-geode-builder/build_image.sh
copy to ci/scripts/rsync_code_down.sh
index e033de6..242326f 100755
--- a/ci/images/google-geode-builder/build_image.sh
+++ b/ci/scripts/rsync_code_down.sh
@@ -1,4 +1,5 @@
#!/usr/bin/env bash
+
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -16,6 +17,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+set -e
+
+BASE_DIR=$(pwd)
+
SOURCE="${BASH_SOURCE[0]}"
while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a
symlink
SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
@@ -24,11 +29,15 @@ while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file
is no longer a symli
done
SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
-pushd ${SCRIPTDIR}
-echo "Setting up credentials for packer"
+SSHKEY_FILE="instance-data/sshkey"
+
+INSTANCE_NAME="$(cat instance-data/instance-name)"
+INSTANCE_IP_ADDRESS="$(cat instance-data/instance-ip-address)"
+PROJECT="$(cat instance-data/project)"
+ZONE="$(cat instance-data/zone)"
+
+echo 'StrictHostKeyChecking no' >> /etc/ssh/ssh_config
+
+OUTPUT_DIR=${BASE_DIR}/geode-results
-#CREDENTIALS_FILE=concourse-key.json
-#echo "${GCP_CONCOURSE_KEY}" | base64 -d > ${CREDENTIALS_FILE}
-#export GOOGLE_APPLICATION_CREDENTIALS=${CREDENTIALS_FILE}
-echo "Running packer"
-packer build packer.json
+time rsync -e "ssh -i ${SSHKEY_FILE}" -ah geode@${INSTANCE_IP_ADDRESS}:.
${OUTPUT_DIR}/.
diff --git a/ci/images/google-geode-builder/build_image.sh
b/ci/scripts/rsync_code_up.sh
similarity index 73%
copy from ci/images/google-geode-builder/build_image.sh
copy to ci/scripts/rsync_code_up.sh
index e033de6..bd5cd56 100755
--- a/ci/images/google-geode-builder/build_image.sh
+++ b/ci/scripts/rsync_code_up.sh
@@ -1,4 +1,5 @@
#!/usr/bin/env bash
+
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -16,6 +17,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+set -e
+
+BASE_DIR=$(pwd)
+
SOURCE="${BASH_SOURCE[0]}"
while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a
symlink
SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
@@ -24,11 +29,15 @@ while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file
is no longer a symli
done
SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
-pushd ${SCRIPTDIR}
-echo "Setting up credentials for packer"
+REPODIR=$(cd geode; git rev-parse --show-toplevel)
+
+SSHKEY_FILE="instance-data/sshkey"
+
+INSTANCE_NAME="$(cat instance-data/instance-name)"
+INSTANCE_IP_ADDRESS="$(cat instance-data/instance-ip-address)"
+PROJECT="$(cat instance-data/project)"
+ZONE="$(cat instance-data/zone)"
+
+echo 'StrictHostKeyChecking no' >> /etc/ssh/ssh_config
-#CREDENTIALS_FILE=concourse-key.json
-#echo "${GCP_CONCOURSE_KEY}" | base64 -d > ${CREDENTIALS_FILE}
-#export GOOGLE_APPLICATION_CREDENTIALS=${CREDENTIALS_FILE}
-echo "Running packer"
-packer build packer.json
+time rsync -e "ssh -i ${SSHKEY_FILE}" -ah ${REPODIR}
geode@${INSTANCE_IP_ADDRESS}:.
diff --git a/ci/scripts/start_instance.sh b/ci/scripts/start_instance.sh
new file mode 100755
index 0000000..261ab86
--- /dev/null
+++ b/ci/scripts/start_instance.sh
@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+BASE_DIR=$(pwd)
+
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a
symlink
+ SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+ SOURCE="$(readlink "$SOURCE")"
+ [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative
symlink, we need to resolve it relative to the path where the symlink file was
located
+done
+SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+
+source ${BASE_DIR}/concourse-metadata-resource/concourse_metadata
+
+SSHKEY_FILE="instance-data/sshkey"
+
+if [[ -z "${GEODE_FORK}" ]]; then
+ echo "GEODE_FORK environment variable must be set for this script to work."
+ exit 1
+fi
+
+if [[ -z "${GEODE_BRANCH}" ]]; then
+ echo "GEODE_BRANCH environment variable must be set for this script to work."
+ exit 1
+fi
+
+
+
+SANITIZED_GEODE_BRANCH=$(echo ${GEODE_BRANCH} | tr "/" "-" | tr '[:upper:]'
'[:lower:]')
+IMAGE_FAMILY_PREFIX=""
+
+if [[ "${GEODE_FORK}" != "apache" ]]; then
+ IMAGE_FAMILY_PREFIX="${GEODE_FORK}-${SANITIZED_GEODE_BRANCH}-"
+fi
+
+INSTANCE_NAME="$(echo
"geode-builder-${BUILD_PIPELINE_NAME}-${BUILD_JOB_NAME}-${BUILD_NAME}" | tr
'[:upper:]' '[:lower:]')"
+PROJECT=apachegeode-ci
+ZONE=us-central1-f
+echo "${INSTANCE_NAME}" > "instance-data/instance-name"
+echo "${PROJECT}" > "instance-data/project"
+echo "${ZONE}" > "instance-data/zone"
+
+echo 'StrictHostKeyChecking no' >> /etc/ssh/ssh_config
+echo "RAM is ${RAM}"
+RAM_MEGABYTES=$( expr ${RAM} \* 1024 )
+echo "RAM_MEGABYTES is ${RAM_MEGABYTES}"
+INSTANCE_INFORMATION=$(gcloud compute --project=${PROJECT} instances create
${INSTANCE_NAME} \
+ --zone=${ZONE} \
+ --machine-type=custom-${CPUS}-${RAM_MEGABYTES} \
+ --min-cpu-platform=Intel\ Skylake \
+ --image-family="${IMAGE_FAMILY_PREFIX}geode-builder" \
+ --image-project=${PROJECT} \
+ --boot-disk-size=100GB \
+ --boot-disk-type=pd-ssd \
+ --format=json)
+CREATE_EXIT_STATUS=$?
+
+
+while ! gcloud compute --project=${PROJECT} ssh geode@${INSTANCE_NAME}
--zone=${ZONE} --ssh-key-file=${SSHKEY_FILE} --quiet -- true; do
+ echo -n .
+done
+echo "${INSTANCE_INFORMATION}" > instance-data/instance-information
+
+INSTANCE_IP_ADDRESS=$(echo ${INSTANCE_INFORMATION} | jq -r
'.[].networkInterfaces[0].accessConfigs[0].natIP')
+echo "${INSTANCE_IP_ADDRESS}" > "instance-data/instance-ip-address"
diff --git a/ci/images/google-geode-builder/build_image.sh
b/ci/scripts/stop_instance.sh
similarity index 80%
copy from ci/images/google-geode-builder/build_image.sh
copy to ci/scripts/stop_instance.sh
index e033de6..e7f39f7 100755
--- a/ci/images/google-geode-builder/build_image.sh
+++ b/ci/scripts/stop_instance.sh
@@ -1,4 +1,5 @@
#!/usr/bin/env bash
+
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -16,6 +17,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+set -e
+
+BASE_DIR=$(pwd)
+
SOURCE="${BASH_SOURCE[0]}"
while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a
symlink
SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
@@ -24,11 +29,13 @@ while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file
is no longer a symli
done
SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
-pushd ${SCRIPTDIR}
-echo "Setting up credentials for packer"
+INSTANCE_NAME="$(cat instance-data/instance-name)"
+PROJECT="$(cat instance-data/project)"
+ZONE="$(cat instance-data/zone)"
+
+
+echo 'StrictHostKeyChecking no' >> /etc/ssh/ssh_config
-#CREDENTIALS_FILE=concourse-key.json
-#echo "${GCP_CONCOURSE_KEY}" | base64 -d > ${CREDENTIALS_FILE}
-#export GOOGLE_APPLICATION_CREDENTIALS=${CREDENTIALS_FILE}
-echo "Running packer"
-packer build packer.json
+gcloud compute --project=${PROJECT} instances delete ${INSTANCE_NAME} \
+ --zone=${ZONE} \
+ --quiet
diff --git a/gradle/test.gradle b/gradle/test.gradle
index 898cf99..610d6c0 100644
--- a/gradle/test.gradle
+++ b/gradle/test.gradle
@@ -282,6 +282,7 @@ subprojects {
afterTest { desc, result ->
def now = new Date().format('yyyy-MM-dd HH:mm:ss.SSS Z')
progress << "$now Completed test $desc.className $desc.name with
result: ${result.resultType}$eol"
+ logger.quiet("Test: ${desc.className}.${desc.name}:
${result.resultType}")
}
doFirst {
