This is an automated email from the ASF dual-hosted git repository.
marcoabreu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push:
new fdf2283 [MXNET-612] Optimize RAT setup and move it into PR stage
(#11492)
fdf2283 is described below
commit fdf22837c063ee58e3c7013fe22f47a1e876591c
Author: Marco de Abreu <[email protected]>
AuthorDate: Fri Jun 29 19:07:07 2018 +0200
[MXNET-612] Optimize RAT setup and move it into PR stage (#11492)
* Move Apache RAT installation into Docker layers instead of doing it
during runtime
* Address comments
* Move RAT from nightly to PR
---
Jenkinsfile | 20 ++++++---
ci/docker/Dockerfile.build.ubuntu_rat | 35 +++++++++++++++
ci/docker/install/ubuntu_rat.sh | 33 ++++++++++++++
ci/docker/runtime_functions.sh | 22 +++++++++-
tests/nightly/Jenkinsfile | 10 +----
tests/nightly/apache_rat_license_check/README.md | 6 +++
.../apache_rat_license_check/license_check.sh | 51 ----------------------
7 files changed, 110 insertions(+), 67 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index 7156dc1..9b652e8 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -163,11 +163,21 @@ def python3_gpu_ut(docker_container_name) {
}
try {
- stage("Sanity Check") {
- node('mxnetlinux-cpu') {
- ws('workspace/sanity') {
- init_git()
- docker_run('ubuntu_cpu', 'sanity_check', false)
+ stage('Sanity Check') {
+ parallel 'Lint': {
+ node('mxnetlinux-cpu') {
+ ws('workspace/sanity-lint') {
+ init_git()
+ docker_run('ubuntu_cpu', 'sanity_check', false)
+ }
+ }
+ },
+ 'RAT License': {
+ node('mxnetlinux-cpu') {
+ ws('workspace/sanity-rat') {
+ init_git()
+ docker_run('ubuntu_rat', 'nightly_test_rat_check', false)
+ }
}
}
}
diff --git a/ci/docker/Dockerfile.build.ubuntu_rat
b/ci/docker/Dockerfile.build.ubuntu_rat
new file mode 100755
index 0000000..c77261c
--- /dev/null
+++ b/ci/docker/Dockerfile.build.ubuntu_rat
@@ -0,0 +1,35 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to run the Apache RAT license check
+
+FROM ubuntu:16.04
+
+WORKDIR /work/deps
+
+COPY install/ubuntu_rat.sh /work/
+RUN /work/ubuntu_rat.sh
+
+ARG USER_ID=0
+COPY install/ubuntu_adduser.sh /work/
+RUN /work/ubuntu_adduser.sh
+
+COPY runtime_functions.sh /work/
+
+WORKDIR /work/mxnet
+ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib
diff --git a/ci/docker/install/ubuntu_rat.sh b/ci/docker/install/ubuntu_rat.sh
new file mode 100755
index 0000000..afaea52
--- /dev/null
+++ b/ci/docker/install/ubuntu_rat.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -ex
+
+echo "Install dependencies"
+apt-get update
+apt-get install -y subversion maven openjdk-8-jdk openjdk-8-jre
+
+echo "download RAT"
+svn co http://svn.apache.org/repos/asf/creadur/rat/trunk/
+
+echo "cd into directory"
+cd trunk
+
+echo "mvn install"
+mvn -Dmaven.test.skip=true install
diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index e496399..c82f993 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -783,8 +783,26 @@ build_docs() {
#Runs Apache RAT Check on MXNet Source for License Headers
nightly_test_rat_check() {
- set -ex
- ./tests/nightly/apache_rat_license_check/license_check.sh
+ set -e
+ pushd .
+
+ cd /work/deps/trunk/apache-rat/target
+
+ # Use shell number 5 to duplicate the log output. It get sprinted and
stored in $OUTPUT at the same time https://stackoverflow.com/a/12451419
+ exec 5>&1
+ OUTPUT=$(java -jar apache-rat-0.13-SNAPSHOT.jar -E
/work/mxnet/tests/nightly/apache_rat_license_check/rat-excludes -d
/work/mxnet|tee >(cat - >&5))
+ ERROR_MESSAGE="Printing headers for text files without a valid license
header"
+
+
+ echo "-------Process The Output-------"
+
+ if [[ $OUTPUT =~ $ERROR_MESSAGE ]]; then
+ echo "ERROR: RAT Check detected files with unknown licenses. Please
fix and run test again!";
+ exit 1
+ else
+ echo "SUCCESS: There are no files with an Unknown License.";
+ fi
+ popd
}
#Checks MXNet for Compilation Warnings
diff --git a/tests/nightly/Jenkinsfile b/tests/nightly/Jenkinsfile
index fb9fbfd..d869b4f 100755
--- a/tests/nightly/Jenkinsfile
+++ b/tests/nightly/Jenkinsfile
@@ -69,15 +69,7 @@ def docker_run(platform, function_name, use_nvidia,
shared_mem = '500m') {
try {
stage('NightlyTests'){
- parallel 'RATCheck: CPU': {
- node('mxnetlinux-cpu') {
- ws('workspace/nt-RATTest') {
- init_git()
- docker_run('ubuntu_nightly_cpu', 'nightly_test_rat_check', false)
- }
- }
- },
- 'CompilationWarnings: CPU': {
+ parallel 'CompilationWarnings: CPU': {
node('mxnetlinux-cpu') {
ws('workspace/nt-compilationTest') {
init_git()
diff --git a/tests/nightly/apache_rat_license_check/README.md
b/tests/nightly/apache_rat_license_check/README.md
index 477db37..04def91 100755
--- a/tests/nightly/apache_rat_license_check/README.md
+++ b/tests/nightly/apache_rat_license_check/README.md
@@ -12,6 +12,12 @@ The license check script called by the Jenkinsfile
### How to run the RAT check locally
The following commands can be used to run a Apache RAT check locally -
+Docker based 1-click-method:
+```
+ci/build.py --platform ubuntu_rat /work/runtime_functions.sh
nightly_test_rat_check
+```
+
+Manual method:
```
#install maven
sudo apt-get install maven -y #>/dev/null
diff --git a/tests/nightly/apache_rat_license_check/license_check.sh
b/tests/nightly/apache_rat_license_check/license_check.sh
deleted file mode 100755
index 79e86c0..0000000
--- a/tests/nightly/apache_rat_license_check/license_check.sh
+++ /dev/null
@@ -1,51 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-
-#mvn and svn are installed in the docker container via the nghtly test script
-
-echo "download RAT"
-svn co http://svn.apache.org/repos/asf/creadur/rat/trunk/ #>/dev/null
-
-echo "cd into directory"
-cd trunk
-
-echo "mvn install"
-mvn -Dmaven.test.skip=true install #>/dev/null
-
-echo "build success, cd into target"
-cd apache-rat/target
-
-
-echo "-------Run Apache RAT check on MXNet-------"
-
-#Command has been run twice, once for the logs and once to store in the
variable to parse.
-java -jar apache-rat-0.13-SNAPSHOT.jar -E
/work/mxnet/tests/nightly/apache_rat_license_check/rat-excludes -d /work/mxnet
-OUTPUT="$(java -jar apache-rat-0.13-SNAPSHOT.jar -E
/work/mxnet/tests/nightly/apache_rat_license_check/rat-excludes -d /work/mxnet)"
-SOURCE="^0 Unknown Licenses"
-
-
-echo "-------Process The Output-------"
-
-if [[ "$OUTPUT" =~ $SOURCE ]]; then
- echo "SUCCESS: There are no files with an Unknown License.";
-else
- echo "ERROR: RAT Check detected files with unknown licenses. Please fix
and run test again!";
- exit 1
-fi
\ No newline at end of file
