This is an automated email from the ASF dual-hosted git repository.
abhishek pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new d242a9314b Adds license and security vulnerabilities checks for
Hadoop3 build (#13270)
d242a9314b is described below
commit d242a9314b8a4979e5233f1b3d4ac76267bdc4e1
Author: Tejaswini Bandlamudi <[email protected]>
AuthorDate: Wed Nov 9 14:50:31 2022 +0530
Adds license and security vulnerabilities checks for Hadoop3 build (#13270)
* adds license and security vulnerabilities check for Hadoop3 builds
* spacing
* fixes bugs
* updates check_test_suite.py to always run license checks with Hadoop3
* nit
* run analyze dependencies, analyze hadoop 3 dependencies
* run tests
* revert analyze dependencies, analyze hadoop 3 dependencies addition in
check_test_suite.py
* fixes bug
* revert code change
---
.travis.yml | 51 ++++++++++++++++++++++++---------------------------
check_test_suite.py | 2 +-
2 files changed, 25 insertions(+), 28 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index efee82a862..81f906f72c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -89,7 +89,8 @@ jobs:
- name: "spotbugs checks"
script: ${MVN} spotbugs:check --fail-at-end -pl '!benchmarks'
- - name: "license checks"
+ - &license_checks
+ name: "license checks"
before_script: &setup_generate_license
- sudo apt-get update && sudo apt-get install python3 -y
- curl https://bootstrap.pypa.io/pip/3.5/get-pip.py | sudo -H python3
@@ -100,12 +101,18 @@ jobs:
- >
${MVN} apache-rat:check -Prat --fail-at-end
-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn
- -Drat.consoleOutput=true
+ -Drat.consoleOutput=true ${HADOOP_PROFILE}
# Generate dependency reports and checks they are valid. When running
on Travis CI, 2 cores are available
#
(https://docs.travis-ci.com/user/reference/overview/#virtualisation-environment-vs-operating-system).
- mkdir -p target
- distribution/bin/generate-license-dependency-reports.py . target
--clean-maven-artifact-transfer --parallel 2
- distribution/bin/check-licenses.py licenses.yaml
target/license-reports
+
+ - <<: *license_checks
+ name: "license checks with Hadoop3"
+ env:
+ - HADOOP_PROFILE='-Phadoop3'
+
- name: "script checks"
install: skip
# who watches the watchers?
@@ -120,9 +127,10 @@ jobs:
./check_test_suite.py && travis_terminate 0 || MAVEN_OPTS='-Xmx3000m'
${MVN} clean -DstrictCompile compile test-compile --fail-at-end
${MAVEN_SKIP} ${MAVEN_SKIP_TESTS}
- - name: "analyze dependencies"
+ - &analyze_dependencies
+ name: "analyze dependencies"
script: |-
- MAVEN_OPTS='-Xmx3000m' ${MVN} ${MAVEN_SKIP} dependency:analyze
-DoutputXML=true -DignoreNonCompile=true -DfailOnWarning=true || { echo "
+ MAVEN_OPTS='-Xmx3000m' ${MVN} ${MAVEN_SKIP} dependency:analyze
-DoutputXML=true -DignoreNonCompile=true -DfailOnWarning=true ${HADOOP_PROFILE}
|| { echo "
The dependency analysis has found a dependency that is either:
@@ -142,27 +150,10 @@ jobs:
" && false; }
- - name: "analyze hadoop 3 dependencies"
- script: |-
- MAVEN_OPTS='-Xmx3000m' ${MVN} ${MAVEN_SKIP} dependency:analyze
-DoutputXML=true -DignoreNonCompile=true -DfailOnWarning=true -Phadoop3 || {
echo "
-
- The dependency analysis has found a dependency that is either:
-
- 1) Used and undeclared: These are available as a transitive dependency
but should be explicitly
- added to the POM to ensure the dependency version. The XML to add the
dependencies to the POM is
- shown above.
-
- 2) Unused and declared: These are not needed and removing them from
the POM will speed up the build
- and reduce the artifact size. The dependencies to remove are shown
above.
-
- If there are false positive dependency analysis warnings, they can be
suppressed:
-
https://maven.apache.org/plugins/maven-dependency-plugin/analyze-mojo.html#usedDependencies
-
https://maven.apache.org/plugins/maven-dependency-plugin/examples/exclude-dependencies-from-dependency-analysis.html
-
- For more information, refer to:
-
https://maven.apache.org/plugins/maven-dependency-plugin/analyze-mojo.html
-
- " && false; }
+ - <<: *analyze_dependencies
+ name: "analyze hadoop 3 dependencies"
+ env:
+ - HADOOP_PROFILE='-Phadoop3'
- name: "intellij inspections"
script: >
@@ -803,11 +794,12 @@ jobs:
echo
"------------------------druid-tiny-cluster-"$v"s-0-------------------------";
sudo /usr/local/bin/kubectl logs --tail 1000
druid-tiny-cluster-"$v"s-0;
done
- - name: "security vulnerabilities"
+ - &security_vulnerabilities
+ name: "security vulnerabilities"
stage: cron
install: skip
script: |-
- ${MVN} dependency-check:purge dependency-check:check || { echo "
+ ${MVN} dependency-check:purge dependency-check:check ${HADOOP_PROFILE}
|| { echo "
The OWASP dependency check has found security vulnerabilities. Please
use a newer version
of the dependency that does not have vulnerabilities. To see a report
run
@@ -818,6 +810,11 @@ jobs:
" && false; }
+ - <<: *security_vulnerabilities
+ name: "security vulnerabilities with Hadoop3"
+ env:
+ - HADOOP_PROFILE='-Phadoop3'
+
# Travis CI only supports per build (and not per-job notifications):
https://github.com/travis-ci/travis-ci/issues/9888
notifications:
email:
diff --git a/check_test_suite.py b/check_test_suite.py
index 8fd7391114..741b14180a 100755
--- a/check_test_suite.py
+++ b/check_test_suite.py
@@ -22,7 +22,7 @@ import sys
# this script does some primitive examination of git diff to determine if a
test suite needs to be run or not
# these jobs should always be run, no matter what
-always_run_jobs = ['license checks', '(openjdk8) packaging check',
'(openjdk11) packaging check']
+always_run_jobs = ['license checks', 'license checks with Hadoop3',
'(openjdk8) packaging check', '(openjdk11) packaging check']
# ignore changes to these files completely since they don't impact CI, if the
changes are only to these files then all
# of CI can be skipped. however, jobs which are always run will still be run
even if only these files are changed
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
