This is an automated email from the ASF dual-hosted git repository. aw pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/yetus.git
The following commit(s) were added to refs/heads/master by this push: new 0dc9e47 YETUS-947. Support github actions as a robot (#95) 0dc9e47 is described below commit 0dc9e47fa4554ecbe6f8b5e040b64c724529b583 Author: Allen Wittenauer <a...@apache.org> AuthorDate: Fri Feb 28 18:14:50 2020 -0800 YETUS-947. Support github actions as a robot (#95) --- .github/workflows/yetus.yml | 51 +++++++++++++ .../in-progress/precommit-basic.html.md | 2 +- .../in-progress/precommit-robots.html.md | 14 +++- precommit/src/main/shell/core.d/00-yetuslib.sh | 2 +- precommit/src/main/shell/core.d/docker.sh | 55 ++++++++------ precommit/src/main/shell/robots.d/githubactions.sh | 85 ++++++++++++++++++++++ precommit/src/main/shell/test-patch.d/github.sh | 31 +++++--- precommit/src/main/shell/test-patch.sh | 1 + 8 files changed, 203 insertions(+), 38 deletions(-) diff --git a/.github/workflows/yetus.yml b/.github/workflows/yetus.yml new file mode 100644 index 0000000..f6e76d2 --- /dev/null +++ b/.github/workflows/yetus.yml @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +name: Apache Yetus + +on: [push, pull_request] # yamllint disable-line rule:truthy + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + with: + path: src + - name: test-patch + run: >- + "${GITHUB_WORKSPACE}/src/precommit/src/main/shell/test-patch.sh" + --basedir="${GITHUB_WORKSPACE}/src" + --patch-dir="${GITHUB_WORKSPACE}/out" + --docker + --project=yetus + --brief-report-file="${GITHUB_WORKSPACE}/out/brief.txt" + --console-report-file="${GITHUB_WORKSPACE}/out/console.txt" + --html-report-file="${GITHUB_WORKSPACE}/out/report.html" + --junit-report-xml="${GITHUB_WORKSPACE}/out/junit-report.xml" + --linecomments='' + --proclimit=2000 + --archive-list="checkstyle-errors.xml,spotbugsXml.xml" + --plugins=all + --tests-filter=checkstyle,javadoc,rubocop,test4tests + --docker-cache-from=apache/yetus-base:master + - name: Artifact output + uses: actions/upload-artifact@v1 + with: + name: apacheyetuspatchdir + path: out diff --git a/asf-site-src/source/documentation/in-progress/precommit-basic.html.md b/asf-site-src/source/documentation/in-progress/precommit-basic.html.md index 004782a..e38bc6c 100644 --- a/asf-site-src/source/documentation/in-progress/precommit-basic.html.md +++ b/asf-site-src/source/documentation/in-progress/precommit-basic.html.md @@ -97,7 +97,7 @@ Build Tools: Automation and Isolation: * [Circle CI](https://www.circleci.com) -* [Docker](https://www.docker.com) version 1.7.0+ +* [Docker](https://www.docker.com) Minimum: 17.0+ (moby Client API 1.27.0+) - Recommended: 18.0+ (moby Client API 1.38.0+) * [Gitlab CI](https://www.gitlab.com) * [Jenkins](https://www.jenkins-ci.org) * [Travis CI](https://www.travis-ci.com) diff --git a/asf-site-src/source/documentation/in-progress/precommit-robots.html.md b/asf-site-src/source/documentation/in-progress/precommit-robots.html.md index b0fa588..dd50b75 100644 --- a/asf-site-src/source/documentation/in-progress/precommit-robots.html.md +++ b/asf-site-src/source/documentation/in-progress/precommit-robots.html.md @@ -55,8 +55,6 @@ TRIGGER: ${TF_BUILD}=True Azure Pipelines support has only been tested on the Ubuntu VM with GitHub as the source repository. It automatically configures `--patch-dir` to be `${BUILD_ARTIFACTSTAGINGDIRECTORY}/yetus`. While the URL to the console is provided in the report, links are not provided due to the URLs to artifacts not being available at runtime. -As of this writing, Azure Pipelines has moved to a custom moby build for the 'docker' executable. As a result, `--docker` is not supported. - # Circle CI TRIGGER: ${CIRCLECI}=true @@ -135,6 +133,18 @@ See also: * Apache Yetus' source tree [.cirrus.yml](https://github.com/apache/yetus/blob/master/.cirrus.yml) for some tips and tricks. +# GitHub Actions + + NOTE: GitHub Actions support is not stable and should be viewed as experimental, at best. + +TRIGGER: ${GITHUB_ACTIONS}=True + +GitHub Actions support has only been tested on the ubuntu-latest image. It automatically configures `--patch-dir` to be `${GITHUB_WORKSAPCE}/yetus` if not previously set. + +See also: + +* Apache Yetus' source tree [yetus.yaml](https://github.com/apache/yetus/blob/master/.github/workflows/yetus.yml) for some tips and tricks. + # Gitlab CI TRIGGER: ${GITLAB_CI}=true diff --git a/precommit/src/main/shell/core.d/00-yetuslib.sh b/precommit/src/main/shell/core.d/00-yetuslib.sh index 780dc2a..0cb2e15 100755 --- a/precommit/src/main/shell/core.d/00-yetuslib.sh +++ b/precommit/src/main/shell/core.d/00-yetuslib.sh @@ -180,7 +180,7 @@ function yetus_generic_columnprinter if [[ -n "${COLUMNS}" ]]; then numcols=${COLUMNS} else - numcols=$(tput cols) 2>/dev/null + numcols=$(tput cols 2>/dev/null) fi if [[ -z "${numcols}" diff --git a/precommit/src/main/shell/core.d/docker.sh b/precommit/src/main/shell/core.d/docker.sh index e47cea4..4d210e9 100755 --- a/precommit/src/main/shell/core.d/docker.sh +++ b/precommit/src/main/shell/core.d/docker.sh @@ -146,7 +146,7 @@ function docker_parse_args ;; --docker-work-dir=*) delete_parameter "${i}" - DOCKER_TAG=${i#*=} + DOCKER_WORK_DIR=${i#*=} ;; esac done @@ -196,8 +196,9 @@ function docker_initialize dockvers=$(docker_version Client) IFS='.' read -r -a DOCKER_VERSION <<< "${dockvers}" - if [[ "${DOCKER_VERSION[0]}" -lt 17 ]]; then - add_vote_table -1 docker "Docker command '${DOCKERCMD}' is too old (${dockvers} < 17.0)." + if [[ "${DOCKER_VERSION[0]}" -lt 1 ]] || \ + [[ "${DOCKER_VERSION[0]}" -lt 2 && "${DOCKER_VERSION[1]}" -lt 27 ]]; then + add_vote_table -1 docker "Docker command '${DOCKERCMD}' is too old (${dockvers} < API v 1.27.0)." bugsystem_finalreport 1 cleanup_and_exit 1 fi @@ -247,7 +248,7 @@ function docker_fileverify dockplat=('--platform' "${DOCKER_PLATFORM}") fi - echo "No --dockerfile or --dockertag provided. Attempting to pull apache/yetus:${VERSION}." + echo "No --dockerfile or --docker-tag provided. Attempting to pull apache/yetus:${VERSION}." if dockercmd pull "${dockplat[@]}" "apache/yetus:${VERSION}"; then echo "Pull succeeded; will build with pulled image." @@ -559,7 +560,7 @@ function docker_version declare ret # new version command - val=$(dockercmd version --format "{{.${vertype}.Version}}" 2>/dev/null) + val=$(dockercmd version --format "{{.${vertype}.APIVersion}}" 2>/dev/null) ret=$? if [[ ${ret} != 0 ]];then @@ -676,20 +677,23 @@ function docker_run_image lines=$("${AWK}" '/YETUS CUT HERE/ {print FNR; exit}' "${DOCKERFILE}") buildfile="${PATCH_DIR}/Dockerfile" - if [[ "${DOCKER_VERSION[0]}" -lt 18 ]] && [[ ${lines} -gt 0 ]]; then - - # versions less than 18 don't support having the - # Dockerfile outside of the build context. Let's fall back to - # pre-YETUS-723 behavior and put the re-written Dockerfile - # outside of the source tree rather than go through a lot of - # machinations. This means COPY, ADD, etc do not work, but - # whatever - - popd >/dev/null || return 1 - buildfile="${PATCH_DIR}/test-patch-docker/Dockerfile" - dockerdir="${PATCH_DIR}/test-patch-docker" - mkdir -p "${dockerdir}" - pushd "${PATCH_DIR}/test-patch-docker" >/dev/null || return 1 + if [[ ${lines} -gt 0 ]]; then + if [[ "${DOCKER_VERSION[0]}" -lt 1 ]] || \ + [[ "${DOCKER_VERSION[0]}" -lt 2 && "${DOCKER_VERSION[1]}" -lt 38 ]]; then + + # versions less than 18 don't support having the + # Dockerfile outside of the build context. Let's fall back to + # pre-YETUS-723 behavior and put the re-written Dockerfile + # outside of the source tree rather than go through a lot of + # machinations. This means COPY, ADD, etc do not work, but + # whatever + + popd >/dev/null || return 1 + buildfile="${PATCH_DIR}/test-patch-docker/Dockerfile" + dockerdir="${PATCH_DIR}/test-patch-docker" + mkdir -p "${dockerdir}" + pushd "${PATCH_DIR}/test-patch-docker" >/dev/null || return 1 + fi fi ( @@ -700,10 +704,13 @@ function docker_run_image fi ) > "${buildfile}" - if [[ "${DOCKER_VERSION[0]}" -lt 18 ]] && [[ ${lines} -gt 0 ]]; then - # Need to put our re-constructed Dockerfile in a place - # where it can be referenced in the output post-run - cp -p "${buildfile}" "${PATCH_DIR}/Dockerfile" + if [[ ${lines} -gt 0 ]]; then + if [[ "${DOCKER_VERSION[0]}" -lt 1 ]] || \ + [[ "${DOCKER_VERSION[0]}" -lt 2 && "${DOCKER_VERSION[1]}" -lt 38 ]]; then + # Need to put our re-constructed Dockerfile in a place + # where it can be referenced in the output post-run + cp -p "${buildfile}" "${PATCH_DIR}/Dockerfile" + fi fi if [[ -n "${DOCKER_CACHE_FROM}" ]]; then @@ -799,7 +806,7 @@ function docker_run_image client=$(docker_version Client) server=$(docker_version Server) - dockerversion="Client=${client} Server=${server}" + dockerversion="ClientAPI=${client} ServerAPI=${server}" # make the kernel prefer to kill us if we run out of RAM DOCKER_EXTRAARGS+=("--oom-score-adj" "500") diff --git a/precommit/src/main/shell/robots.d/githubactions.sh b/precommit/src/main/shell/robots.d/githubactions.sh new file mode 100755 index 0000000..f59eb85 --- /dev/null +++ b/precommit/src/main/shell/robots.d/githubactions.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# SHELLDOC-IGNORE + +if [[ "${GITHUB_ACTIONS}" == true ]] && + declare -f compile_cycle >/dev/null; then + # shellcheck disable=SC2034 + ROBOT=true + # shellcheck disable=SC2034 + PATCH_DIR=${PATCH_DIR:-${GITHUB_WORKSPACE}/yetus} + # shellcheck disable=SC2034 + PATCH_OR_ISSUE="GHSHA:${GITHUB_SHA}" + + # shellcheck disable=SC2034 + INSTANCE=${GITHUB_RUN_NUMBER} + + # shellcheck disable=SC2034 + ROBOTTYPE=githubactions + + # shellcheck disable=SC2034 + GITHUB_REPO="${GITHUB_REPOSITORY}" + + if [[ "${GITHUB_EVENT_NAME}" == push ]]; then + # shellcheck disable=SC2034 + PATCH_OR_ISSUE="" + #shellcheck disable=SC2034,SC2153 + PATCH_BRANCH=$(echo "${GITHUB_REF}" | cut -f3- -d/) + # shellcheck disable=SC2034 + BUILDMODE=full + add_docker_env BUILDMODE + elif [[ "${GITHUB_EVENT_NAME}" == pull_request ]]; then + # shellcheck disable=SC2034 + PATCH_OR_ISSUE=$(echo "${GITHUB_REF}" | cut -f3 -d/) + PATCH_OR_ISSUE="GH:${PATCH_OR_ISSUE}" + # shellcheck disable=SC2034 + PATCH_BRANCH=${GITHUB_BASE_REF} + fi + + add_docker_env \ + GITHUB_WORKSPACE \ + GITHUB_SHA \ + GITHUB_REPOSITORY \ + GITHUB_RUN_NUMBER \ + GITHUB_REF \ + GITHUB_ACTIONS \ + GITHUB_EVENT_NAME \ + GITHUB_BASE_REF + + # shellcheck disable=SC2034 + BUILD_URL_CONSOLE=console + + # shellcheck disable=SC2034 + CONSOLE_USE_BUILD_URL=true + + if [[ -d ${BASEDIR}/.git ]]; then + echo "Updating the local git repo to include all branches/tags:" + pushd "${BASEDIR}" >/dev/null || exit 1 + "${GIT}" config --replace-all remote.origin.fetch +refs/heads/*:refs/remotes/origin/* + "${GIT}" fetch --tags + popd >/dev/null || exit 1 + fi + + yetus_add_array_element EXEC_MODES GitHubActions +fi + +function githubactions_set_plugin_defaults +{ + # shellcheck disable=SC2034 + GITHUB_REPO="${GITHUB_REPOSITORY}" +} + diff --git a/precommit/src/main/shell/test-patch.d/github.sh b/precommit/src/main/shell/test-patch.d/github.sh index 32b5456..921368d 100755 --- a/precommit/src/main/shell/test-patch.d/github.sh +++ b/precommit/src/main/shell/test-patch.d/github.sh @@ -414,22 +414,33 @@ function github_locate_sha_patch number=$("${GREP}" number "${PATCH_DIR}/github-search.json" | \ head -1 | \ "${AWK}" '{print $NF}') - number=${number//\s/} + number=${number//\s} number=${number%,} # Semaphore CI doesn't tell us if the sha is a PR or not, so... - if [[ -z "${number}" ]] && [[ "${ROBOTTYPE}" = semaphoreci ]]; then + if [[ -z "${number}" ]]; then + if [[ "${ROBOTTYPE}" = semaphoreci ]]; then - echo "This appears to be a full build on Semaphore CI. Switching modes." + echo "This appears to be a full build on Semaphore CI. Switching modes." - PATCH_BRANCH=${SEMAPHORE_GIT_BRANCH} + PATCH_BRANCH=${SEMAPHORE_GIT_BRANCH} - # shellcheck disable=SC2034 - PATCH_OR_ISSUE="" - # shellcheck disable=SC2034 - BUILDMODE=full - set_buildmode - return 0 + # shellcheck disable=SC2034 + PATCH_OR_ISSUE="" + # shellcheck disable=SC2034 + BUILDMODE=full + add_docker_env BUILDMODE + set_buildmode + return 0 + else + # shellcheck disable=SC2034 + PATCH_OR_ISSUE="" + # shellcheck disable=SC2034 + BUILDMODE=full + add_docker_env BUILDMODE + set_buildmode + return 0 + fi fi github_locate_pr_patch "GH:${number}" "${patchout}" "${diffout}" diff --git a/precommit/src/main/shell/test-patch.sh b/precommit/src/main/shell/test-patch.sh index 58d011d..46cef73 100755 --- a/precommit/src/main/shell/test-patch.sh +++ b/precommit/src/main/shell/test-patch.sh @@ -972,6 +972,7 @@ function parse_args if [[ -z "${PATCH_OR_ISSUE}" && "${BUILDMODE}" = patch ]]; then + yetus_error "ERROR: No patch given." yetus_usage exit 1 fi