This is an automated email from the ASF dual-hosted git repository. szetszwo pushed a commit to branch RATIS-4 in repository https://gitbox.apache.org/repos/asf/ratis-hadoop-projects.git
commit a4f368271739e8197678b85fc7d6caa4371398b6 Author: Jitendra Pandey <[email protected]> AuthorDate: Wed Nov 1 10:01:10 2017 -0700 RATIS-4. Addendum patch. Add ci scripts. Contributed by Marton Elek. --- Jenkinsfile | 22 ------------------ dev-support/ci/README.md | 40 ++++++++++++++++++++++++++++++++ dev-support/ci/common.sh | 37 +++++++++++++++++++++++++++++ dev-support/ci/nightly-build.sh | 47 +++++++++++++++++++++++++++++++++++++ dev-support/ci/precommit-build.sh | 49 +++++++++++++++++++++++++++++++++++++++ dev-support/docker/Dockerfile | 15 ++++++++---- 6 files changed, 183 insertions(+), 27 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile deleted file mode 100644 index 6093303..0000000 --- a/Jenkinsfile +++ /dev/null @@ -1,22 +0,0 @@ -pipeline { - agent any - stages { - stage('Checkout') { - steps { - checkout scm - } - } - stage('Build') { - steps { - sh "./start-build-env.sh mvn clean -Pclean-shade" - sh "./start-build-env.sh mvn install -DskipTests" - } - } - stage('Test') { - steps { - sh "./start-build-env.sh mvn test" - } - } - } -} - diff --git a/dev-support/ci/README.md b/dev-support/ci/README.md new file mode 100644 index 0000000..7d6bbef --- /dev/null +++ b/dev-support/ci/README.md @@ -0,0 +1,40 @@ +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> + +RATIS CI scripts +---------------- + +These scripts are called from jenkins for ratis pre-commit and nightly build jobs. + +On jenkins, the repository is checked out to a subdirectory (sourcedir) and the script is called from the parent directory which is also used to store temporary files (yetus/out/...). + +IT'S NOT RECOMMENDED to run it locally unless you know what could be expected. The script runs in sentinel/robot mode, so: + + * Old docker images are removed by the script + * Local commits are removed by the script + + +## Running locally + +To test the jenkins build locally: + + 1. create a new directory + 2. Clone the ratis repository to a subdirectory: `git clone git://github.com/apache/incubator-ratis.git sourcedir` + 3. Run the script fro the parent directory ./sourcedir/dev-support/ci/nightly-build.sh + +For nightly-build.sh you can set the BRANCH environment variable to define which branch should be tested. + +For precommit-build.sh you should set ISSUE_NUM (eg. 122, RATIS prefix should not be added). + +The variables are set by the jenkins. diff --git a/dev-support/ci/common.sh b/dev-support/ci/common.sh new file mode 100644 index 0000000..a3129e6 --- /dev/null +++ b/dev-support/ci/common.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +#Workspace is set by the jenkins, by default (local run) is the parent directory of the checkout. +WORKSPACE=${WORKSPACE:-$DIR/../../..} +cd $WORKSPACE + +YETUSDIR=${WORKSPACE}/yetus +TESTPATCHBIN=${YETUSDIR}/bin/test-patch +ARTIFACTS=${WORKSPACE}/out +BASEDIR=${WORKSPACE}/sourcedir +TOOLS=${WORKSPACE}/tools +rm -rf "${ARTIFACTS}" "${YETUSDIR}" +mkdir -p "${ARTIFACTS}" "${YETUSDIR}" "${TOOLS}" + +#It's not on all the branches, so we need to copy it from the checkout out source +cp $BASEDIR/dev-support/yetus-personality.sh $WORKSPACE/ +cp $BASEDIR/dev-support/docker/Dockerfile $WORKSPACE/ + +echo "Downloading Yetus" +curl -L https://archive.apache.org/dist/yetus/0.5.0/yetus-0.5.0-bin.tar.gz -o yetus.tar.gz +gunzip -c yetus.tar.gz | tar xpf - -C "${YETUSDIR}" --strip-components 1 diff --git a/dev-support/ci/nightly-build.sh b/dev-support/ci/nightly-build.sh new file mode 100644 index 0000000..ff11058 --- /dev/null +++ b/dev-support/ci/nightly-build.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source $DIR/common.sh + +YETUS_ARGS+=("--archive-list=checkstyle-errors.xml,findbugsXml.xml") +YETUS_ARGS+=("--basedir=${BASEDIR}") +YETUS_ARGS+=("--branch=${BRANCH:-master}") +YETUS_ARGS+=("--brief-report-file=${ARTIFACTS}/email-report.txt") +YETUS_ARGS+=("--build-url-artifacts=artifact/out") +YETUS_ARGS+=("--console-report-file=${ARTIFACTS}/console-report.txt") +YETUS_ARGS+=("--console-urls") +YETUS_ARGS+=("--docker") +YETUS_ARGS+=("--dockerfile=${WORKSPACE}/Dockerfile") +YETUS_ARGS+=("--dockermemlimit=20g") +YETUS_ARGS+=("--empty-patch") +YETUS_ARGS+=("--html-report-file=${ARTIFACTS}/console-report.html") +YETUS_ARGS+=("--java-home=/usr/lib/jvm/java-8-openjdk-amd64") +YETUS_ARGS+=("--jenkins") +YETUS_ARGS+=("--mvn-custom-repos") +YETUS_ARGS+=("--patch-dir=${ARTIFACTS}") +YETUS_ARGS+=("--plugins=all,-author") +YETUS_ARGS+=("--proclimit=5000") +YETUS_ARGS+=("--project=ratis") +YETUS_ARGS+=("--personality=${WORKSPACE}/yetus-personality.sh") +YETUS_ARGS+=("--resetrepo") +YETUS_ARGS+=("--sentinel") +YETUS_ARGS+=("--shelldocs=/testptch/hadoop/dev-support/bin/shelldocs") +YETUS_ARGS+=("--tests-filter=cc,checkstyle,javac,javadoc,pylint,shellcheck,shelldocs,whitespace") + +TESTPATCHBIN=${YETUSDIR}/bin/test-patch + +/bin/bash ${TESTPATCHBIN} "${YETUS_ARGS[@]}" diff --git a/dev-support/ci/precommit-build.sh b/dev-support/ci/precommit-build.sh new file mode 100644 index 0000000..7cae44f --- /dev/null +++ b/dev-support/ci/precommit-build.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source $DIR/common.sh + +YETUS_ARGS+=("--archive-list=checkstyle-errors.xml,findbugsXml.xml") +YETUS_ARGS+=("--basedir=${BASEDIR}") +YETUS_ARGS+=("--brief-report-file=${ARTIFACTS}/email-report.txt") +YETUS_ARGS+=("--build-url-artifacts=artifact/out") +YETUS_ARGS+=("--console-report-file=${ARTIFACTS}/console-report.txt") +YETUS_ARGS+=("--console-urls") +YETUS_ARGS+=("--docker") +YETUS_ARGS+=("--dockerfile=${WORKSPACE}/Dockerfile") +YETUS_ARGS+=("--dockermemlimit=20g") +YETUS_ARGS+=("--findbugs-strict-precheck") +YETUS_ARGS+=("--html-report-file=${ARTIFACTS}/console-report.html") +YETUS_ARGS+=("--jenkins") +YETUS_ARGS+=("--jira-password=${JIRA_PASSWORD}") +YETUS_ARGS+=("--jira-user=hadoopqa") +YETUS_ARGS+=("--multijdkdirs=/usr/lib/jvm/java-8-oracle") +YETUS_ARGS+=("--mvn-custom-repos") +YETUS_ARGS+=("--patch-dir=${ARTIFACTS}") +YETUS_ARGS+=("--project=ratis") +YETUS_ARGS+=("--personality=${WORKSPACE}/yetus-personality.sh") +YETUS_ARGS+=("--proclimit=5000") +YETUS_ARGS+=("--resetrepo") +YETUS_ARGS+=("--sentinel") +YETUS_ARGS+=("--shelldocs=/testptch/hadoop/dev-support/bin/shelldocs") +YETUS_ARGS+=("--skip-dir=dev-support") +YETUS_ARGS+=("--tests-filter=checkstyle,pylint,shelldocs") + +YETUS_ARGS+=("RATIS-${ISSUE_NUM}") + + +/bin/bash ${TESTPATCHBIN} "${YETUS_ARGS[@]}" diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index a5d29cb..8381a21 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -19,7 +19,7 @@ # See BUILDING.txt. -FROM ubuntu:trusty +FROM ubuntu:xenial WORKDIR /root @@ -35,15 +35,20 @@ ENV DEBCONF_TERSE true RUN apt-get -q update && apt-get -q install --no-install-recommends -y \ curl \ unzip \ - git + git \ + wget \ + rsync -####### -# OpenJDK Java -####### +# wget configuration RUN echo "dot_style = mega" > "/root/.wgetrc" RUN echo "quiet = on" >> "/root/.wgetrc" + +####### +# OpenJDK Java +####### + RUN apt-get -q install -y openjdk-8-jdk ######
