This is an automated email from the ASF dual-hosted git repository. szetszwo pushed a commit to branch RATIS-4 in repository https://gitbox.apache.org/repos/asf/ratis-hadoop-projects.git
commit 105138da02755134f55804c6fe3cb5331117715e Author: Jitendra Pandey <[email protected]> AuthorDate: Mon Oct 30 23:13:39 2017 -0700 RATIS-4. Setup Jenkins. Contributed by Marton Elek. --- .gitignore | 1 + .travis.yml | 12 +++ Jenkinsfile | 22 +++++ dev-support/docker/Dockerfile | 91 ++++++++++++++++++++ dev-support/test-patch | 18 ++++ dev-support/yetus-personality.sh | 32 +++++++ dev-support/yetus-wrapper | 177 +++++++++++++++++++++++++++++++++++++++ ratis-hadoop-shaded/pom.xml | 55 ++++++------ ratis-proto-shaded/pom.xml | 55 ++++++------ start-build-env.sh | 55 ++++++++++++ 10 files changed, 460 insertions(+), 58 deletions(-) diff --git a/.gitignore b/.gitignore index d208ca9..688c2ea 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ .settings target build +patchprocess diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..333873f --- /dev/null +++ b/.travis.yml @@ -0,0 +1,12 @@ +language: java + +sudo: required + +service: docker + +script: + - ./start-build-env.sh mvn clean -Pclean-shade + - ./start-build-env.sh mvn install -DskipTests + - ./start-build-env.sh mvn test + + diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000..6093303 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,22 @@ +pipeline { + agent any + stages { + stage('Checkout') { + steps { + checkout scm + } + } + stage('Build') { + steps { + sh "./start-build-env.sh mvn clean -Pclean-shade" + sh "./start-build-env.sh mvn install -DskipTests" + } + } + stage('Test') { + steps { + sh "./start-build-env.sh mvn test" + } + } + } +} + diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile new file mode 100644 index 0000000..606d314 --- /dev/null +++ b/dev-support/docker/Dockerfile @@ -0,0 +1,91 @@ + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Dockerfile for installing the necessary dependencies for building Hadoop. +# See BUILDING.txt. + + +FROM ubuntu:trusty + +WORKDIR /root + +ENV DEBIAN_FRONTEND noninteractive +ENV DEBCONF_TERSE true + +###### +# Install common dependencies from packages +# +# WARNING: DO NOT PUT JAVA APPS HERE! Otherwise they will install default +# Ubuntu Java. See Java section below! +###### +RUN apt-get -q update && apt-get -q install --no-install-recommends -y \ + curl \ + unzip \ + git + +####### +# Oracle Java +####### + +RUN echo "dot_style = mega" > "/root/.wgetrc" +RUN echo "quiet = on" >> "/root/.wgetrc" + +RUN apt-get -q install --no-install-recommends -y software-properties-common +RUN add-apt-repository -y ppa:webupd8team/java +RUN apt-get -q update + +# Auto-accept the Oracle JDK license +RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | sudo /usr/bin/debconf-set-selections +RUN apt-get -q install --no-install-recommends -y oracle-java8-installer + +###### +# Install protobuf compiler 3.1 +###### + +RUN mkdir -p $HOME/protobuf && \ + cd $HOME/protobuf && wget https://github.com/google/protobuf/releases/download/v3.1.0/protoc-3.1.0-linux-x86_64.zip && \ + unzip protoc-3.1.0-linux-x86_64.zip && \ + mv $HOME/protobuf/bin/protoc /usr/local/bin && \ + chmod 755 /usr/local/bin/protoc + + +###### +# Install Apache Maven +###### +RUN mkdir -p /opt/maven && \ + curl -L -s -S \ + http://www-us.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz \ + -o /opt/maven.tar.gz && \ + tar xzf /opt/maven.tar.gz --strip-components 1 -C /opt/maven +ENV MAVEN_HOME /opt/maven +ENV PATH "$PATH:/opt/maven/bin" + +###### +# Install findbugs +###### +#RUN mkdir -p /opt/findbugs && \ +# curl -L -s -S \ +# https://sourceforge.net/projects/findbugs/files/findbugs/3.0.1/findbugs-noUpdateChecks-3.0.1.tar.gz/download \ +# -o /opt/findbugs.tar.gz && \ +# tar xzf /opt/findbugs.tar.gz --strip-components 1 -C /opt/findbugs +#ENV FINDBUGS_HOME /opt/findbugs + +### +# Avoid out of memory errors in builds +### +ENV MAVEN_OPTS -Xms256m -Xmx512m + diff --git a/dev-support/test-patch b/dev-support/test-patch new file mode 100644 index 0000000..25d9d81 --- /dev/null +++ b/dev-support/test-patch @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +BINDIR=$(cd -P -- "$(dirname -- "${BASH_SOURCE-0}")" >/dev/null && pwd -P) +exec "${BINDIR}/yetus-wrapper" test-patch --project=ratis --dockerfile="$BINDIR/docker/Dockerfile" --personality=$BINDIR/yetus-personality.sh --skip-dir=dev-support "$@" diff --git a/dev-support/yetus-personality.sh b/dev-support/yetus-personality.sh new file mode 100644 index 0000000..d700512 --- /dev/null +++ b/dev-support/yetus-personality.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +personality_plugins "all" + +## @description Globals specific to this personality +## @audience private +## @stability evolving +function personality_globals +{ + # shellcheck disable=SC2034 + BUILDTOOL=maven + #shellcheck disable=SC2034 + PATCH_BRANCH_DEFAULT=master + #shellcheck disable=SC2034 + JIRA_ISSUE_RE='^RATIS-[0-9]+$' + #shellcheck disable=SC2034 + GITHUB_REPO="apache/incubator-ratis" +} diff --git a/dev-support/yetus-wrapper b/dev-support/yetus-wrapper new file mode 100644 index 0000000..dc40a7e --- /dev/null +++ b/dev-support/yetus-wrapper @@ -0,0 +1,177 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# you must be this high to ride the ride +if [[ -z "${BASH_VERSINFO[0]}" ]] \ + || [[ "${BASH_VERSINFO[0]}" -lt 3 ]] \ + || [[ "${BASH_VERSINFO[0]}" -eq 3 && "${BASH_VERSINFO[1]}" -lt 2 ]]; then + echo "bash v3.2+ is required. Sorry." + exit 1 +fi + +set -o pipefail + +## @description Print a message to stderr +## @audience public +## @stability stable +## @replaceable no +## @param string +function yetus_error +{ + echo "$*" 1>&2 +} + +## @description Given a filename or dir, return the absolute version of it +## @audience public +## @stability stable +## @param directory +## @replaceable no +## @return 0 success +## @return 1 failure +## @return stdout abspath +function yetus_abs +{ + declare obj=$1 + declare dir + declare fn + declare dirret + + if [[ ! -e ${obj} ]]; then + return 1 + elif [[ -d ${obj} ]]; then + dir=${obj} + else + dir=$(dirname -- "${obj}") + fn=$(basename -- "${obj}") + fn="/${fn}" + fi + + dir=$(cd -P -- "${dir}" >/dev/null 2>/dev/null && pwd -P) + dirret=$? + if [[ ${dirret} = 0 ]]; then + echo "${dir}${fn}" + return 0 + fi + return 1 +} + + +WANTED="$1" +shift +ARGV=("$@") + +HADOOP_YETUS_VERSION=${HADOOP_YETUS_VERSION:-0.4.0} +BIN=$(yetus_abs "${BASH_SOURCE-$0}") +BINDIR=$(dirname "${BIN}") +echo "${WANTED}" +### +### if YETUS_HOME is set, then try to use it +### +if [[ -n "${YETUS_HOME}" + && -x "${YETUS_HOME}/bin/${WANTED}" ]]; then + exec "${YETUS_HOME}/bin/${WANTED}" "${ARGV[@]}" +fi + +# +# this directory is ignored by git and maven +# +HADOOP_PATCHPROCESS=${HADOOP_PATCHPROCESS:-"${BINDIR}/../../patchprocess"} + +if [[ ! -d "${HADOOP_PATCHPROCESS}" ]]; then + mkdir -p "${HADOOP_PATCHPROCESS}" +fi + +mytmpdir=$(yetus_abs "${HADOOP_PATCHPROCESS}") +ret=$? +if [[ ${ret} != 0 ]]; then + yetus_error "yetus-dl: Unable to cwd to ${HADOOP_PATCHPROCESS}" + exit 1 +fi +HADOOP_PATCHPROCESS=${mytmpdir} + +## +## if we've already DL'd it, then short cut +## +if [[ -x "${HADOOP_PATCHPROCESS}/yetus-${HADOOP_YETUS_VERSION}/bin/${WANTED}" ]]; then + exec "${HADOOP_PATCHPROCESS}/yetus-${HADOOP_YETUS_VERSION}/bin/${WANTED}" "${ARGV[@]}" +fi + +## +## need to DL, etc +## + +BASEURL="https://archive.apache.org/dist/yetus/${HADOOP_YETUS_VERSION}/"; +TARBALL="yetus-${HADOOP_YETUS_VERSION}-bin.tar" + +GPGBIN=$(command -v gpg) +CURLBIN=$(command -v curl) + +if ! pushd "${HADOOP_PATCHPROCESS}" >/dev/null; then + yetus_error "ERROR: yetus-dl: Cannot pushd to ${HADOOP_PATCHPROCESS}" + exit 1 +fi + +if [[ -n "${CURLBIN}" ]]; then + if ! "${CURLBIN}" -f -s -L -O "${BASEURL}/${TARBALL}.gz"; then + yetus_error "ERROR: yetus-dl: unable to download ${BASEURL}/${TARBALL}.gz" + exit 1 + fi +else + yetus_error "ERROR: yetus-dl requires curl." + exit 1 +fi + +if [[ -n "${GPGBIN}" ]]; then + if ! mkdir -p .gpg; then + yetus_error "ERROR: yetus-dl: Unable to create ${HADOOP_PATCHPROCESS}/.gpg" + exit 1 + fi + if ! chmod -R 700 .gpg; then + yetus_error "ERROR: yetus-dl: Unable to chmod ${HADOOP_PATCHPROCESS}/.gpg" + exit 1 + fi + if ! "${CURLBIN}" -s -L -o KEYS_YETUS https://dist.apache.org/repos/dist/release/yetus/KEYS; then + yetus_error "ERROR: yetus-dl: unable to fetch https://dist.apache.org/repos/dist/release/yetus/KEYS"; + exit 1 + fi + if ! "${CURLBIN}" -s -L -O "${BASEURL}/${TARBALL}.gz.asc"; then + yetus_error "ERROR: yetus-dl: unable to fetch ${BASEURL}/${TARBALL}.gz.asc" + exit 1 + fi + if ! "${GPGBIN}" --homedir "${HADOOP_PATCHPROCESS}/.gpg" --import "${HADOOP_PATCHPROCESS}/KEYS_YETUS" >/dev/null 2>&1; then + yetus_error "ERROR: yetus-dl: gpg unable to import ${HADOOP_PATCHPROCESS}/KEYS_YETUS" + exit 1 + fi + if ! "${GPGBIN}" --homedir "${HADOOP_PATCHPROCESS}/.gpg" --verify "${TARBALL}.gz.asc" >/dev/null 2>&1; then + yetus_error "ERROR: yetus-dl: gpg verify of tarball in ${HADOOP_PATCHPROCESS} failed" + exit 1 + fi +fi + +if ! (gunzip -c "${TARBALL}.gz" | tar xpf -); then + yetus_error "ERROR: ${TARBALL}.gz is corrupt. Investigate and then remove ${HADOOP_PATCHPROCESS} to try again." + exit 1 +fi + +if [[ -x "${HADOOP_PATCHPROCESS}/yetus-${HADOOP_YETUS_VERSION}/bin/${WANTED}" ]]; then + popd >/dev/null + exec "${HADOOP_PATCHPROCESS}/yetus-${HADOOP_YETUS_VERSION}/bin/${WANTED}" "${ARGV[@]}" +fi + +## +## give up +## +yetus_error "ERROR: ${WANTED} is not part of Apache Yetus ${HADOOP_YETUS_VERSION}" +exit 1 diff --git a/ratis-hadoop-shaded/pom.xml b/ratis-hadoop-shaded/pom.xml index cc47466..e42bd4e 100644 --- a/ratis-hadoop-shaded/pom.xml +++ b/ratis-hadoop-shaded/pom.xml @@ -76,6 +76,32 @@ </execution> </executions> </plugin> + <plugin> + <artifactId>maven-clean-plugin</artifactId> + <executions> + <execution> + <id>clean-shade</id> + <goals> + <goal>clean</goal> + </goals> + <configuration> + <filesets> + <fileset> + <directory>${shaded.sources.dir}</directory> + <followSymlinks>false</followSymlinks> + </fileset> + <fileset> + <directory>.</directory> + <includes> + <include>dependency-reduced-pom.xml</include> + </includes> + <followSymlinks>false</followSymlinks> + </fileset> + </filesets> + </configuration> + </execution> + </executions> + </plugin> </plugins> </build> @@ -134,35 +160,6 @@ </profile> <profile> - <id>clean-shade</id> - <activation> - <activeByDefault>false</activeByDefault> - </activation> - <build> - <plugins> - <plugin> - <artifactId>maven-clean-plugin</artifactId> - <configuration> - <filesets> - <fileset> - <directory>${shaded.sources.dir}</directory> - <followSymlinks>false</followSymlinks> - </fileset> - <fileset> - <directory>.</directory> - <includes> - <include>dependency-reduced-pom.xml</include> - </includes> - <followSymlinks>false</followSymlinks> - </fileset> - </filesets> - </configuration> - </plugin> - </plugins> - </build> - </profile> - - <profile> <id>compile-protobuf</id> <!-- Shade and drop the generated java files under src/main/java. diff --git a/ratis-proto-shaded/pom.xml b/ratis-proto-shaded/pom.xml index 7824500..260ffa1 100644 --- a/ratis-proto-shaded/pom.xml +++ b/ratis-proto-shaded/pom.xml @@ -87,6 +87,32 @@ </execution> </executions> </plugin> + <plugin> + <artifactId>maven-clean-plugin</artifactId> + <executions> + <execution> + <id>clean-shade</id> + <goals> + <goal>clean</goal> + </goals> + <configuration> + <filesets> + <fileset> + <directory>${shaded.sources.dir}</directory> + <followSymlinks>false</followSymlinks> + </fileset> + <fileset> + <directory>.</directory> + <includes> + <include>dependency-reduced-pom.xml</include> + </includes> + <followSymlinks>false</followSymlinks> + </fileset> + </filesets> + </configuration> + </execution> + </executions> + </plugin> </plugins> </build> @@ -431,35 +457,6 @@ </profile> <profile> - <id>clean-shade</id> - <activation> - <activeByDefault>false</activeByDefault> - </activation> - <build> - <plugins> - <plugin> - <artifactId>maven-clean-plugin</artifactId> - <configuration> - <filesets> - <fileset> - <directory>${shaded.sources.dir}</directory> - <followSymlinks>false</followSymlinks> - </fileset> - <fileset> - <directory>.</directory> - <includes> - <include>dependency-reduced-pom.xml</include> - </includes> - <followSymlinks>false</followSymlinks> - </fileset> - </filesets> - </configuration> - </plugin> - </plugins> - </build> - </profile> - - <profile> <id>compile-protobuf</id> <!-- Shade and drop the generated java files under src/main/java. diff --git a/start-build-env.sh b/start-build-env.sh new file mode 100644 index 0000000..db182b3 --- /dev/null +++ b/start-build-env.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e # exit on error + +cd "$(dirname "$0")" # connect to root + +docker build -t ratis-build dev-support/docker + +if [ "$(uname -s)" == "Linux" ]; then + USER_NAME=${SUDO_USER:=$USER} + USER_ID=$(id -u "${USER_NAME}") + GROUP_ID=$(id -g "${USER_NAME}") +else # boot2docker uid and gid + USER_NAME=$USER + USER_ID=1000 + GROUP_ID=50 +fi + +docker build -t "ratis-build-${USER_ID}" - <<UserSpecificDocker +FROM ratis-build +RUN groupadd --non-unique -g ${GROUP_ID} ${USER_NAME} +RUN useradd -g ${GROUP_ID} -u ${USER_ID} -k /root -m ${USER_NAME} +ENV HOME /home/${USER_NAME} +UserSpecificDocker + +TTY_MODE="-t -i" +if [ "$#" -gt 0 ]; then + TTY_MODE="" +fi +# By mapping the .m2 directory you can do an mvn install from +# within the container and use the result on your normal +# system. And this also is a significant speedup in subsequent +# builds because the dependencies are downloaded only once. +docker run --rm=true $TTY_MODE \ + -v "${PWD}:/home/${USER_NAME}/ratis" \ + -w "/home/${USER_NAME}/ratis" \ + -v "${HOME}/.m2:/home/${USER_NAME}/.m2" \ + -u "${USER_NAME}" \ + "ratis-build-${USER_ID}" \ + "$@"
