This is an automated email from the ASF dual-hosted git repository.
aw pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/yetus.git
The following commit(s) were added to refs/heads/main by this push:
new b77d7259 YETUS-277. full report in csv format (#319)
b77d7259 is described below
commit b77d7259141654082483a257abf8ae74f2e9fb82
Author: Allen Wittenauer <a...@apache.org>
AuthorDate: Tue Sep 12 07:40:46 2023 -0700
YETUS-277. full report in csv format (#319)
Signed-off-by: Nick Dimiduk <ndimi...@apache.org>
---
.buildkite/pipeline.yml | 3 +-
.circleci/config.yml | 3 +-
.cirrus.yml | 3 +-
.github/workflows/yetus.yml | 3 +-
.gitlab-ci.yml | 3 +-
.semaphore/semaphore-build.sh | 2 +
Jenkinsfile | 3 +-
precommit/src/main/shell/core.d/00-yetuslib.sh | 11 ++
precommit/src/main/shell/plugins.d/csvout.sh | 126 +++++++++++++++++++++
precommit/src/main/shell/plugins.d/htmlout.sh | 2 +-
.../src/test/shell/yetus_trim.bats | 33 +++---
11 files changed, 172 insertions(+), 20 deletions(-)
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 2b8eb5d8..d8ccfc3e 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -20,10 +20,11 @@ steps:
command: >
precommit/src/main/shell/test-patch.sh
--brief-report-file=/tmp/yetus-out/brief.txt
- --bugcomments=briefreport,htmlout,buildkiteannotate
+ --bugcomments=briefreport,csvout,htmlout,buildkiteannotate
--console-report-file=/tmp/yetus-out/console.txt
--docker
--docker-cache-from=ghcr.io/apache/yetus-base:main
+ --csv-report-file=/tmp/yetus-out/report.csv
--html-report-file=/tmp/yetus-out/report.html
--java-home=/usr/lib/jvm/java-17-openjdk-amd64
--patch-dir=/tmp/yetus-out
diff --git a/.circleci/config.yml b/.circleci/config.yml
index 7b68ed76..08798682 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -42,8 +42,9 @@ jobs:
--patch-dir=/tmp/yetus-out
--html-report-file=/tmp/yetus-out/report.html
--console-report-file=/tmp/yetus-out/console.txt
+ --csv-report-file=/tmp/yetus-out/report.csv
--brief-report-file=/tmp/yetus-out/brief.txt
- --bugcomments=briefreport,htmlout,junit
+ --bugcomments=briefreport,csvout,htmlout,junit
--tests-filter=checkstyle,test4tests
--junit-report-xml=/tmp/yetus-out/junit-report.xml
diff --git a/.cirrus.yml b/.cirrus.yml
index 93695f5e..086c2744 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -22,8 +22,9 @@ yetus_task:
${CIRRUS_WORKING_DIR}/precommit/src/main/shell/test-patch.sh
--basedir="${CIRRUS_WORKING_DIR}"
--brief-report-file=/tmp/yetus-out/brief.txt
- --bugcomments=briefreport,htmlout,junit
+ --bugcomments=briefreport,console,csvout,htmlout,junit
--console-report-file=/tmp/yetus-out/console.txt
+ --csv-report-file=/tmp/yetus-out/report.csv
--html-report-file=/tmp/yetus-out/report.html
--java-home=/usr/lib/jvm/java-11-openjdk-amd64
--junit-report-xml=/tmp/yetus-out/junit.xml
diff --git a/.github/workflows/yetus.yml b/.github/workflows/yetus.yml
index b7d42800..5e4eb07e 100644
--- a/.github/workflows/yetus.yml
+++ b/.github/workflows/yetus.yml
@@ -44,8 +44,9 @@ jobs:
--archive-list="checkstyle-errors.xml,spotbugsXml.xml"
--basedir="${GITHUB_WORKSPACE}/src"
--brief-report-file="${GITHUB_WORKSPACE}/out/brief.txt"
- --bugcomments=console,briefreport,github,htmlout,junit
+ --bugcomments=console,briefreport,csvout,github,htmlout,junit
--console-report-file="${GITHUB_WORKSPACE}/out/console.txt"
+ --csv-report-file="${GITHUB_WORKSPACE}/out/report.csv"
--docker
--docker-cache-from=ghcr.io/apache/yetus-base:main
--github-token="${{ secrets.GITHUB_TOKEN }}"
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 1190222d..eae1a60c 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -25,9 +25,10 @@ buretoolbox-job:
--plugins=all
--java-home=/usr/lib/jvm/java-11-openjdk-amd64
--html-report-file=/tmp/yetus-out/report.html
+ --csv-report-file=/tmp/yetus-out/report.csv
--console-report-file=/tmp/yetus-out/console.txt
--brief-report-file=/tmp/yetus-out/brief.txt
- --bugcomments=briefreport,htmlout,gitlab,junit
+ --bugcomments=briefreport,csvout,htmlout,gitlab,junit
--tests-filter=checkstyle,test4tests
--junit-report-xml=/tmp/yetus-out/junit-report.xml
diff --git a/.semaphore/semaphore-build.sh b/.semaphore/semaphore-build.sh
index 4561894b..ac7d7677 100755
--- a/.semaphore/semaphore-build.sh
+++ b/.semaphore/semaphore-build.sh
@@ -19,10 +19,12 @@ PRECOMMITDIR=precommit/src/main/shell
"${PRECOMMITDIR}/test-patch.sh" \
--plugins=all \
+ --bugcomments=briefreport,csvout,htmlout,junit \
--mvn-custom-repos \
--mvn-custom-repos-dir=/tmp/yetus-m2 \
--patch-dir=/tmp/yetus-out \
--tests-filter=checkstyle,test4tests \
+ --csv-report-file=/tmp/yetus-out/report.csv \
--junit-report-xml=/tmp/yetus-out/junit-results.xml \
--junit-report-style=full \
--docker \
diff --git a/Jenkinsfile b/Jenkinsfile
index 5d545f14..2bc9da6c 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -86,11 +86,12 @@ pipeline {
YETUS_ARGS+=(--sentinel)
# lots of different output formats
+
YETUS_ARGS+=("--bugcomments=briefreport,console,csvout,htmlout,junit")
YETUS_ARGS+=("--brief-report-file=${WORKSPACE}/${YETUS_RELATIVE_PATCHDIR}/brief.txt")
YETUS_ARGS+=("--console-report-file=${WORKSPACE}/${YETUS_RELATIVE_PATCHDIR}/console.txt")
YETUS_ARGS+=("--html-report-file=${WORKSPACE}/${YETUS_RELATIVE_PATCHDIR}/report.html")
YETUS_ARGS+=("--junit-report-xml=${WORKSPACE}/${YETUS_RELATIVE_PATCHDIR}/junit-report.xml")
-
+
YETUS_ARGS+=("--csv-report-file=${WORKSPACE}/${YETUS_RELATIVE_PATCHDIR}/yetus-out/report.csv")
# enable writing back to Github
YETUS_ARGS+=(--github-token="${GITHUB_TOKEN}")
diff --git a/precommit/src/main/shell/core.d/00-yetuslib.sh
b/precommit/src/main/shell/core.d/00-yetuslib.sh
index 9b2177a5..0c9bbbbf 100755
--- a/precommit/src/main/shell/core.d/00-yetuslib.sh
+++ b/precommit/src/main/shell/core.d/00-yetuslib.sh
@@ -609,3 +609,14 @@ function yetus_is_container
YETUS_CONTAINER_STATE=false
return 1
}
+
+## @description Given a string, trim extra surrounding whitespace
+## @audience public
+## @stability stable
+## @replaceable no
+function yetus_trim {
+ declare var="$*"
+ var="${var#"${var%%[![:space:]]*}"}"
+ var="${var%"${var##*[![:space:]]}"}"
+ printf '%s' "$var"
+}
diff --git a/precommit/src/main/shell/plugins.d/csvout.sh
b/precommit/src/main/shell/plugins.d/csvout.sh
new file mode 100755
index 00000000..17d848df
--- /dev/null
+++ b/precommit/src/main/shell/plugins.d/csvout.sh
@@ -0,0 +1,126 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+add_bugsystem csvout
+
+## @description Usage info for csvout plugin
+## @audience private
+## @stability evolving
+## @replaceable no
+function csvout_usage
+{
+ yetus_add_option "--csv-report-file=<file>" "Save the final report to an
CSV-formated file"
+}
+
+## @description Option parsing for csvout plugin
+## @audience private
+## @stability evolving
+## @replaceable no
+function csvout_parse_args
+{
+ declare i
+ declare fn
+
+ for i in "$@"; do
+ case ${i} in
+ --csv-report-file=*)
+ delete_parameter "${i}"
+ fn=${i#*=}
+ ;;
+ esac
+ done
+
+ if [[ -n "${fn}" ]]; then
+ if : > "${fn}"; then
+ CSVOUT_REPORTFILE_ORIG="${fn}"
+ CSVOUT_REPORTFILE=$(yetus_abs "${CSVOUT_REPORTFILE_ORIG}")
+ else
+ yetus_error "WARNING: cannot create CSV report file ${fn}. Ignoring."
+ fi
+ fi
+}
+
+## @description Give access to the CSV report file in docker mode
+## @audience private
+## @stability evolving
+## @replaceable no
+function csvout_docker_support
+{
+ # if for some reason the report file is in PATCH_DIR, then if
+ # PATCH_DIR gets cleaned out we lose access to the file on the 'outside'
+ # so we put it into the workdir which never gets cleaned.
+
+ if [[ -n ${CSVOUT_REPORTFILE} ]]; then
+ DOCKER_EXTRAARGS+=("-v"
"${CSVOUT_REPORTFILE}:${DOCKER_WORK_DIR}/report.csv")
+ USER_PARAMS+=("--csv-report-file=${DOCKER_WORK_DIR}/report.csv")
+ fi
+}
+
+## @description Write out an CSV version of the final report to a file
+## @audience private
+## @stability evolving
+## @replaceable no
+## @param runresult
+function csvout_report_writer
+{
+ declare result=$1
+ shift
+ declare i=0
+ declare ourstring
+ declare vote
+ declare subs
+ declare ela
+ declare comment
+
+ if [[ -z "${CSVOUT_REPORTFILE}" ]]; then
+ return
+ fi
+
+ : > "${CSVOUT_REPORTFILE}"
+ i=0
+ until [[ $i -ge ${#TP_VOTE_TABLE[@]} ]]; do
+ ourstring=$(echo "${TP_VOTE_TABLE[i]}" | tr -s ' ')
+ vote=$(echo "${ourstring}" | cut -f2 -d\|)
+ vote=$(yetus_trim "${vote}")
+ subs=$(echo "${ourstring}" | cut -f3 -d\|)
+ subs=$(yetus_trim "${subs}")
+ ela=$(echo "${ourstring}" | cut -f4 -d\|)
+ ela=$(yetus_trim "${ela}")
+ comment=$(echo "${ourstring}" | cut -f6 -d\|)
+ comment=$(yetus_trim "${comment}")
+
+ echo "${vote},${subs},${ela},${comment}" >> "${CSVOUT_REPORTFILE}"
+ ((i=i+1))
+ done
+}
+
+## @description Write out the CSV version of the final report to a file
+## @audience private
+## @stability evolving
+## @replaceable no
+## @param runresult
+function csvout_finalreport
+{
+ declare result=$1
+
+ if [[ -z "${CSVOUT_REPORTFILE}" ]]; then
+ return
+ fi
+
+ big_console_header "Writing CSV to ${CSVOUT_REPORTFILE}"
+
+ csvout_report_writer "${result}" "${CSVOUT_REPORTFILE}"
+}
diff --git a/precommit/src/main/shell/plugins.d/htmlout.sh
b/precommit/src/main/shell/plugins.d/htmlout.sh
index b14056ea..c92854b8 100755
--- a/precommit/src/main/shell/plugins.d/htmlout.sh
+++ b/precommit/src/main/shell/plugins.d/htmlout.sh
@@ -314,7 +314,7 @@ function htmlout_finalreport
return
fi
- big_console_header "Writing HTML to ${commentfile}"
+ big_console_header "Writing HTML to ${HTMLOUT_REPORTFILE}"
htmlout_report_writer "${result}" "${HTMLOUT_REPORTFILE}"
}
diff --git a/.semaphore/semaphore-build.sh
b/precommit/src/test/shell/yetus_trim.bats
similarity index 63%
copy from .semaphore/semaphore-build.sh
copy to precommit/src/test/shell/yetus_trim.bats
index 4561894b..32c3cdff 100755
--- a/.semaphore/semaphore-build.sh
+++ b/precommit/src/test/shell/yetus_trim.bats
@@ -1,5 +1,4 @@
#!/usr/bin/env bash
-#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -15,16 +14,24 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-PRECOMMITDIR=precommit/src/main/shell
+load functions_test_helper
+
+@test "yetus_trim (no trim)" {
+ run yetus_trim fake
+ [ "${output}" = "fake" ]
+}
+
+@test "yetus_trim (left trim)" {
+ run yetus_trim " fake"
+ [ "${output}" = "fake" ]
+}
+
+@test "yetus_trim (right trim)" {
+ run yetus_trim "fake "
+ [ "${output}" = "fake" ]
+}
-"${PRECOMMITDIR}/test-patch.sh" \
- --plugins=all \
- --mvn-custom-repos \
- --mvn-custom-repos-dir=/tmp/yetus-m2 \
- --patch-dir=/tmp/yetus-out \
- --tests-filter=checkstyle,test4tests \
- --junit-report-xml=/tmp/yetus-out/junit-results.xml \
- --junit-report-style=full \
- --docker \
- --dockerfile="${PRECOMMITDIR}/test-patch-docker/Dockerfile" \
- --docker-cache-from=ghcr.io/apache/yetus-base:main,ubuntu:jammy
+@test "yetus_trim (dual trim)" {
+ run yetus_trim " fake "
+ [ "${output}" = "fake" ]
+}
