This is an automated email from the ASF dual-hosted git repository.
aw pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/yetus.git
The following commit(s) were added to refs/heads/main by this push:
new 81f34a9 YETUS-1085. Add support for Buildkite (#206)
81f34a9 is described below
commit 81f34a9a9e8c698c1deb8443c2d1ce3ba7f249f5
Author: Allen Wittenauer <[email protected]>
AuthorDate: Sun Nov 22 10:13:20 2020 -0800
YETUS-1085. Add support for Buildkite (#206)
---
.buildkite/pipeline.yml | 35 +++
.../in-progress/precommit/index.html.md | 1 +
.../in-progress/precommit/robots/buildkite.html.md | 44 ++++
.../in-progress/precommit/robots/index.html.md | 1 +
precommit/src/main/shell/buildkite-recovery.sh | 173 ++++++++++++++
precommit/src/main/shell/robots.d/buildkite.sh | 256 +++++++++++++++++++++
precommit/src/main/shell/test-patch.sh | 4 +
7 files changed, 514 insertions(+)
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
new file mode 100644
index 0000000..8b9df1d
--- /dev/null
+++ b/.buildkite/pipeline.yml
@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+steps:
+ - label: "Clean patch-dir"
+ command: \[ -d /tmp/yetus-out ] && rm -rf /tmp/yetus-out/*
+ - label: "Apache Yetus"
+ command: >
+ precommit/src/main/shell/test-patch.sh
+ --brief-report-file=/tmp/yetus-out/brief.txt
+ --bugcomments=briefreport,htmlout,buildkiteannotate
+ --console-report-file=/tmp/yetus-out/console.txt
+ --docker
+ --docker-cache-from=apache/yetus:main
+ --html-report-file=/tmp/yetus-out/report.html
+ --java-home=/usr/lib/jvm/java-8-openjdk-amd64
+ --patch-dir=/tmp/yetus-out
+ --plugins=all
+ --tests-filter=checkstyle,test4tests
+ - label: 'Buildkite Recovery'
+ command: >
+ precommit/src/main/shell/buildkite-recovery.sh
+ --patch-dir=/tmp/yetus-out
diff --git
a/asf-site-src/source/documentation/in-progress/precommit/index.html.md
b/asf-site-src/source/documentation/in-progress/precommit/index.html.md
index b9d3a9f..2cfeb49 100644
--- a/asf-site-src/source/documentation/in-progress/precommit/index.html.md
+++ b/asf-site-src/source/documentation/in-progress/precommit/index.html.md
@@ -109,6 +109,7 @@ These plug-ins are native to Apace Yetus and are (usually!)
always available:
Automation and Isolation:
* [Azure Pipelines](robots/azurepipelines)
+* [Buildkite](robots/buildkite)
* [Circle CI](robots/circleci)
* [Cirrus CI](robots/cirrusci)
* [Docker](docker) version 1.7.0+
diff --git
a/asf-site-src/source/documentation/in-progress/precommit/robots/buildkite.html.md
b/asf-site-src/source/documentation/in-progress/precommit/robots/buildkite.html.md
new file mode 100644
index 0000000..f5501ef
--- /dev/null
+++
b/asf-site-src/source/documentation/in-progress/precommit/robots/buildkite.html.md
@@ -0,0 +1,44 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+# Robots: Buildkite
+
+TRIGGER: ${BUILDKITE}=true
+
+The recommended configuration is to have Apache Yetus installed on the host
running the build agent.
+
+If `buildkite-agent` is available and working, then artifacts will be
automatically uploaded and an
+annotation added to the build console if `buildkiteannotate` bugsystem is
enabled.
+If `${ARTIFACTS_PATH}` is configured, then `--patch-dir` is set to the first
listed directory path
+with a `/yetus` suffix. However, links to the location logs must still be
configured manually.
+
+# Buildkite Annotations
+
+If precommit is running in a mode such that `buildkite-agent` isn't available
(e.g., using the Apache Yetus container image)
+the `buildkite-annotation-recovery` tool may be used to send the annotations
and artifacts to Buildkite:
+
+The usage is simple:
+
+```bash
+$ buildkite-recovery --patch-dir=<pre-existing patch directory>
+```
+
+See also:
+
+* Apache Yetus' source tree
[.buildkite](https://github.com/apache/yetus/blob/main/.buildkite) for some
tips and tricks.
diff --git
a/asf-site-src/source/documentation/in-progress/precommit/robots/index.html.md
b/asf-site-src/source/documentation/in-progress/precommit/robots/index.html.md
index 41431ae..16d79b9 100644
---
a/asf-site-src/source/documentation/in-progress/precommit/robots/index.html.md
+++
b/asf-site-src/source/documentation/in-progress/precommit/robots/index.html.md
@@ -47,6 +47,7 @@
The following systems are detected automatically when run in one of these
environments:
* [Azure Pipelines](azurepipelines)
+* [Buildkite](buildkite)
* [Circle CI](circleci)
* [Cirrus CI](cirrusci)
* [GitHub Actions](githubactions)
diff --git a/precommit/src/main/shell/buildkite-recovery.sh
b/precommit/src/main/shell/buildkite-recovery.sh
new file mode 100755
index 0000000..74463f8
--- /dev/null
+++ b/precommit/src/main/shell/buildkite-recovery.sh
@@ -0,0 +1,173 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# no shelldocs required from this file
+# SHELLDOC-IGNORE
+
+# Make sure that bash version meets the pre-requisite
+
+if [[ -z "${BASH_VERSINFO[0]}" ]] \
+ || [[ "${BASH_VERSINFO[0]}" -lt 3 ]] \
+ || [[ "${BASH_VERSINFO[0]}" -eq 3 && "${BASH_VERSINFO[1]}" -lt 2 ]]; then
+ echo "bash v3.2+ is required. Sorry."
+ exit 1
+fi
+
+this="${BASH_SOURCE-$0}"
+BINDIR=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
+BINNAME=${this##*/}
+BINNAME=${BINNAME%.sh}
+#shellcheck disable=SC2034
+STARTINGDIR=$(pwd)
+#shellcheck disable=SC2034
+USER_PARAMS=("$@")
+#shellcheck disable=SC2034
+QATESTMODE=false
+
+
+## @description import core library routines
+## @audience private
+## @stability evolving
+function import_core
+{
+ declare filename
+
+ for filename in "${BINDIR}/core.d"/*; do
+ # shellcheck disable=SC1091
+ # shellcheck source=core.d/01-common.sh
+ . "${filename}"
+ done
+}
+
+## @description import plugins then remove the stuff we don't need
+## @audience public
+## @stability stable
+## @replaceable no
+function import_including_buildkite
+{
+ #shellcheck disable=SC2034
+ ENABLED_PLUGINS='buildkiteannotate'
+ importplugins
+ yetus_debug "Removing BUILDTOOLS, TESTTYPES, and TESTFORMATS from installed
plug-in list"
+ #shellcheck disable=SC2034
+ BUILDTOOLS=()
+ #shellcheck disable=SC2034
+ TESTTYPES=()
+ #shellcheck disable=SC2034
+ TESTFORMATS=()
+ #shellcheck disable=SC2034
+ BUGSYSTEMS=('buildkiteannotate')
+ #shellcheck disable=SC2034
+ BUGLINECOMMENTS='buildkiteannotate'
+}
+
+## @description Setup the default global variables
+## @audience public
+## @stability stable
+## @replaceable no
+function setup_defaults
+{
+ common_defaults
+}
+
+## @description Interpret the command line parameters
+## @audience private
+## @stability stable
+## @replaceable no
+## @param $@
+## @return May exit on failure
+function parse_args
+{
+ declare i
+ common_args "$@"
+}
+
+## @description Print the usage information
+## @audience public
+## @stability stable
+## @replaceable no
+function yetus_usage
+{
+ import_including_github
+ github_usage
+
+ echo "${BINNAME} [OPTIONS]"
+
+ yetus_add_option "--debug" "If set, then output some extra stuff to stderr"
+ yetus_add_option "--ignore-unknown-options=<bool>" "Continue despite unknown
options (default: ${IGNORE_UNKNOWN_OPTIONS})"
+ yetus_add_option "--patch-dir=<dir>" "The directory for working and output
files (default '/tmp/test-patch-${PROJECT_NAME}/pid')"
+ yetus_add_option "--sed-cmd=<cmd>" "The 'sed' command to use (default 'sed')"
+
+ yetus_generic_columnprinter "${YETUS_OPTION_USAGE[@]}"
+ yetus_reset_usage
+}
+
+## @description Large display for the user console
+## @audience public
+## @stability stable
+## @replaceable no
+## @param string
+## @return large chunk of text
+function big_console_header
+{
+ local text="$*"
+ local spacing=$(( (75+${#text}) /2 ))
+ printf '\n\n'
+ echo
"============================================================================"
+ echo
"============================================================================"
+ printf '%*s\n' ${spacing} "${text}"
+ echo
"============================================================================"
+ echo
"============================================================================"
+ printf '\n\n'
+}
+
+## @description setup the parameter tracker for param errors
+## @audience private
+## @stability evolving
+function setup_parameter_tracker
+{
+ declare i
+
+ for i in "${USER_PARAMS[@]}"; do
+ if [[ "${i}" =~ ^-- ]]; then
+ i=${i%=*}
+ PARAMETER_TRACKER+=("${i}")
+ fi
+ done
+}
+
+trap "cleanup_and_exit 1" HUP INT QUIT TERM
+
+setup_parameter_tracker
+
+import_core
+
+setup_defaults
+
+parse_args "$@"
+
+import_including_buildkite
+
+parse_args_plugins "$@"
+
+if [[ "${#PARAMETER_TRACKER}" -gt 0 ]]; then
+ yetus_error "ERROR: Unprocessed flag(s): ${PARAMETER_TRACKER[*]}"
+ if [[ "${IGNORE_UNKNOWN_OPTIONS}" == false ]]; then
+ cleanup_and_exit 1
+ fi
+fi
+
+buildkite_recovery
diff --git a/precommit/src/main/shell/robots.d/buildkite.sh
b/precommit/src/main/shell/robots.d/buildkite.sh
new file mode 100755
index 0000000..f52c515
--- /dev/null
+++ b/precommit/src/main/shell/robots.d/buildkite.sh
@@ -0,0 +1,256 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# SHELLDOC-IGNORE
+
+if [[ "${BUILDKITE}" == true ]] &&
+ declare -f compile_cycle >/dev/null; then
+ # shellcheck disable=SC2034
+ ROBOT=true
+
+ add_bugsystem buildkiteannotate
+
+ # shellcheck disable=SC2034
+ if [[ -n "${BUILDKITE_ARTIFACT_PATHS}" ]]; then
+ PATCH_DIR=${BUILDKITE_ARTIFACT_PATHS%%;*}/yetus
+ fi
+
+ # shellcheck disable=SC2034
+ INSTANCE=${BUILDKITE_BUILD_ID}
+ # shellcheck disable=SC2034
+ ROBOTTYPE=buildkite
+ # shellcheck disable=SC2034
+ BUILD_URL="${BUILDKITE_BUILD_URL}"
+
+ # shellcheck disable=SC2034
+ if [[ "${BUILDKITE_PULL_REQUEST}" == false ]]; then
+ BUILDMODE=full
+ PATCH_BRANCH=${BUILDKITE_BRANCH}
+ else
+ # shellcheck disable=SC2034
+ BUILDMODE='patch'
+
+ case ${BUILDKITE_PIPELINE_PROVIDER} in
+ github)
+ PATCH_OR_ISSUE=GH:${BUILDKITE_PULL_REQUEST}
+ USER_PARAMS+=("GH:${BUILDKITE_PULL_REQUEST}")
+ ;;
+ gitlab)
+ # shellcheck disable=SC2034
+ PATCH_OR_ISSUE=GL:${BUILDKITE_PULL_REQUEST}
+ USER_PARAMS+=("GL:${BUILDKITE_PULL_REQUEST}")
+ ;;
+ esac
+ fi
+
+ add_docker_env \
+ BUILDKITE \
+ BUILDKITE_BUILD_ID \
+ BUILDKITE_BUILD_URL \
+ BUILDKITE_BRANCH \
+ BUILDKITE_PIPELINE_PROVIDER \
+ BUILDKITE_PULL_REQUEST \
+ BUILDKITE_REPO
+
+ # shellcheck disable=SC2034
+ BUILD_URL_CONSOLE=""
+ # shellcheck disable=SC2034
+ CONSOLE_USE_BUILD_URL=false
+
+ if [[ -d ${BASEDIR}/.git ]]; then
+ echo "Updating the local git repo to include all branches/tags:"
+ pushd "${BASEDIR}" >/dev/null || exit 1
+ "${GIT}" config --replace-all remote.origin.fetch
+refs/heads/*:refs/remotes/origin/*
+ "${GIT}" fetch --tags
+ popd >/dev/null || exit 1
+ fi
+
+ yetus_add_array_element EXEC_MODES Buildkite
+fi
+
+function buildkite_finalreport
+{
+ add_footer_table "Console output" "${BUILDKITE_BUILD_URL}"
+}
+
+function buildkite_artifact_url
+{
+ echo "artifact:/${PATCH_DIR}"
+}
+
+function buildkiteannotate_finalreport
+{
+ declare result=$1
+ declare i
+ declare recoverydir="${PATCH_DIR}/buildkite-recovery"
+ declare comment
+ declare url
+ declare ela
+ declare subs
+ declare logfile
+ declare calctime
+ declare vote
+ declare emoji
+ declare buildkite_style
+
+ mkdir -p "${recoverydir}"
+ commentfile="${recoverydir}/buildkite-recovery.md"
+
+ big_console_header "Creating Buildkite annotation"
+
+ rm "${commentfile}" 2>/dev/null
+
+ url=$(get_artifact_url)
+
+ if [[ ${result} == 0 ]]; then
+ echo ":confetti_ball: **+1 overall**" >> "${commentfile}"
+ echo "success" > "${recoverydir}/style.txt"
+ else
+ echo ":broken_heart: **-1 overall**" >> "${commentfile}"
+ echo "error" > "${recoverydir}/style.txt"
+ fi
+ printf '\n\n\n\n' >> "${commentfile}"
+
+ i=0
+ until [[ ${i} -ge ${#TP_HEADER[@]} ]]; do
+ printf '%s\n\n' "${TP_HEADER[${i}]}" >> "${commentfile}"
+ ((i=i+1))
+ done
+
+ {
+ printf '\n\n'
+ echo "| Vote | Subsystem | Runtime | Logfile | Comment |"
+ echo "|:----:|----------:|--------:|:--------:|:-------:|"
+ } >> "${commentfile}"
+
+ i=0
+ until [[ ${i} -ge ${#TP_VOTE_TABLE[@]} ]]; do
+ ourstring=$(echo "${TP_VOTE_TABLE[${i}]}" | tr -s ' ')
+ vote=$(echo "${ourstring}" | cut -f2 -d\| | tr -d ' ')
+ subs=$(echo "${ourstring}" | cut -f3 -d\|)
+ ela=$(echo "${ourstring}" | cut -f4 -d\|)
+ calctime=$(clock_display "${ela}")
+ logfile=$(echo "${ourstring}" | cut -f5 -d\| | tr -d ' ')
+ comment=$(echo "${ourstring}" | cut -f6 -d\|)
+
+ if [[ "${vote}" = "H" ]]; then
+ echo "|||| _${comment}_ |" >> "${commentfile}"
+ ((i=i+1))
+ continue
+ fi
+
+ emoji=""
+ case ${vote} in
+ 1|"+1")
+ emoji="+1 :green_heart:"
+ ;;
+ -1)
+ emoji="-1 :x:"
+ ;;
+ 0)
+ emoji="+0 :ok:"
+ ;;
+ -0)
+ emoji="-0 :warning:"
+ ;;
+ H)
+ # this never gets called (see above) but this is here so others know
the color is taken
+ emoji=""
+ ;;
+ *)
+ # usually this should not happen but let's keep the old vote result if
it happens
+ emoji=${vote}
+ ;;
+ esac
+
+ if [[ -n "${logfile}" ]]; then
+ t1=${logfile/@@BASE@@/}
+ t2=$(echo "${logfile}" | "${SED}" -e "s,@@BASE@@,${url},g")
+ t2="[${t1}](${t2})"
+ else
+ t2=""
+ fi
+
+ printf '| %s | %s | %s | %s | %s |\n' \
+ "${emoji}" \
+ "${subs}" \
+ "${calctime}" \
+ "${t2}" \
+ "${comment}" \
+ >> "${commentfile}"
+
+ ((i=i+1))
+ done
+
+ if [[ ${#TP_TEST_TABLE[@]} -gt 0 ]]; then
+ {
+ printf '\n\n'
+ echo "| Reason | Tests |"
+ echo "|-------:|:------|"
+ } >> "${commentfile}"
+ i=0
+ until [[ ${i} -ge ${#TP_TEST_TABLE[@]} ]]; do
+ echo "${TP_TEST_TABLE[${i}]}" >> "${commentfile}"
+ ((i=i+1))
+ done
+ fi
+
+ {
+ printf '\n\n'
+ echo "| Subsystem | Report/Notes |"
+ echo "|----------:|:-------------|"
+ } >> "${commentfile}"
+
+ i=0
+ until [[ $i -ge ${#TP_FOOTER_TABLE[@]} ]]; do
+ comment=$(echo "${TP_FOOTER_TABLE[${i}]}" | "${SED}" -e
"s,@@BASE@@,${url},g")
+ printf '%s\n' "${comment}" >> "${commentfile}"
+ ((i=i+1))
+ done
+
+ buildkite_recovery
+}
+
+function buildkite_recovery
+{
+ declare buildkite_style
+ declare recoverydir="${PATCH_DIR}/buildkite-recovery"
+
+ buildkite_agent=$(command -v buildkite-agent 2>/dev/null)
+
+ commentfile="${recoverydir}/buildkite-recovery.md"
+
+ if [[ -z "${buildkite_agent}" ]]; then
+ yetus_error "ERROR: buildkite-agent is not available. Skipping Buildkite
annotation."
+ return 0
+ fi
+
+ if [[ ! -f "${recoverydir}/style.txt" ]]; then
+ yetus_error "WARNING: No buildkite status to recovery. Maybe it was
successful?"
+ return 0
+ fi
+
+ big_console_header "Uploading Buildkite artifacts"
+
+ buildkite-agent artifact upload "${PATCH_DIR}/*"
+
+ big_console_header "Adding Buildkite annotation"
+
+ buildkite_style=$(cat "${recoverydir}/style.txt")
+
+ "${buildkite_agent}" annotate --style "${buildkite_style}" < "${commentfile}"
+ rm -rf "${recoverydir}"
+}
\ No newline at end of file
diff --git a/precommit/src/main/shell/test-patch.sh
b/precommit/src/main/shell/test-patch.sh
index 66c6542..7dfeaf8 100755
--- a/precommit/src/main/shell/test-patch.sh
+++ b/precommit/src/main/shell/test-patch.sh
@@ -408,6 +408,10 @@ function big_console_header
echo "::group::${text}"
fi
+ if [[ "${ROBOTTYPE}" == 'buildkite' ]]; then
+ echo "--- ${text}"
+ fi
+
printf '\n\n'
echo
"============================================================================"
echo
"============================================================================"
