Build: Update docker files to enable Jenkins This commit updates the Jenkins script to enable automatic creation of builds.
Add JUnit export format for Jenkins Add sys import Duration in seconds not millisec Add cast to float Update jenkins build script Comments in build script Project: http://git-wip-us.apache.org/repos/asf/incubator-madlib/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-madlib/commit/5984d827 Tree: http://git-wip-us.apache.org/repos/asf/incubator-madlib/tree/5984d827 Diff: http://git-wip-us.apache.org/repos/asf/incubator-madlib/diff/5984d827 Branch: refs/heads/latest_release Commit: 5984d8277556c6271b516f7726640e923152c90f Parents: 7be6893 Author: Rahul Iyer <ri...@apache.org> Authored: Fri Mar 3 17:31:00 2017 -0800 Committer: Rahul Iyer <ri...@apache.org> Committed: Wed Mar 15 14:27:01 2017 -0700 ---------------------------------------------------------------------- .../docker/base/Dockerfile_postgres_9_6_Jenkins | 2 +- tool/jenkins/jenkins_build.sh | 81 +++++++++++++---- tool/jenkins/junit_export.py | 96 ++++++++++++++++++++ 3 files changed, 160 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/5984d827/tool/docker/base/Dockerfile_postgres_9_6_Jenkins ---------------------------------------------------------------------- diff --git a/tool/docker/base/Dockerfile_postgres_9_6_Jenkins b/tool/docker/base/Dockerfile_postgres_9_6_Jenkins index 137842e..fe6a95a 100644 --- a/tool/docker/base/Dockerfile_postgres_9_6_Jenkins +++ b/tool/docker/base/Dockerfile_postgres_9_6_Jenkins @@ -27,7 +27,7 @@ RUN apt-get update && apt-get install -y wget \ libssl-dev \ libboost-all-dev \ m4 \ - wget + rpm ### Build custom CMake with SSQL support RUN wget https://cmake.org/files/v3.6/cmake-3.6.1.tar.gz && \ http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/5984d827/tool/jenkins/jenkins_build.sh ---------------------------------------------------------------------- diff --git a/tool/jenkins/jenkins_build.sh b/tool/jenkins/jenkins_build.sh index 72ada55..f03bc78 100644 --- a/tool/jenkins/jenkins_build.sh +++ b/tool/jenkins/jenkins_build.sh @@ -1,4 +1,4 @@ -# +#!/bin/bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,28 +16,73 @@ # specific language governing permissions and limitations # under the License. -#!/bin/sh - -##################################################################################### -### If this bash script is executed as a stand-alone file, assuming this -### is not part of the MADlib source code, then the following two commands -### may have to be used: -# git clone https://github.com/apache/incubator-madlib.git -# pushd incubator-madlib ##################################################################################### +workdir=`pwd` +user_name=`whoami` +echo "Build by user $user_name in directory $workdir" +echo "-------------------------------" +echo "ls -la" +ls -la +echo "-------------------------------" +echo "rm -rf build" +rm -rf build +echo "-------------------------------" +echo "rm -rf logs" +rm -rf logs +echo "mkdir logs" +mkdir logs +echo "-------------------------------" +echo "docker kill madlib" +docker kill madlib +echo "docker rm madlib" +docker rm madlib + +echo "Creating docker container" # Pull down the base docker images -docker pull madlib/postgres_9_6:jenkins -# Assuming git clone of incubator-madlib has been done, launch a container with the volume mounted -docker run -d --name madlib -v incubator-madlib:/incubator-madlib madlib/postgres_9.6:jenkins +docker pull madlib/postgres_9.6:jenkins +# Launch docker container with volume mounted from workdir +echo "-------------------------------" +cat <<EOF +docker run -d --name madlib -v "${workdir}/incubator-madlib":/incubator-madlib madlib/postgres_9.6:jenkins | tee logs/docker_setup.log +EOF +docker run -d --name madlib -v "${workdir}/incubator-madlib":/incubator-madlib madlib/postgres_9.6:jenkins | tee logs/docker_setup.log +echo "-------------------------------" + ## This sleep is required since it takes a couple of seconds for the docker ## container to come up, which is required by the docker exec command that follows. sleep 5 -# cmake, make and make install MADlib -docker exec madlib bash -c 'mkdir /incubator-madlib/build ; cd /incubator-madlib/build ; cmake .. ; make ; make install' + +echo "---------- Building package -----------" +# cmake, make, make install, and make package +cat <<EOF +docker exec madlib bash -c 'rm -rf /build; mkdir /build; cd /build; cmake ../incubator-madlib; make clean; make; make install; make package' | tee $workdir/logs/madlib_compile.log +EOF +docker exec madlib bash -c 'rm -rf /build; mkdir /build; cd /build; cmake ../incubator-madlib; make clean; make; make install; make package' | tee $workdir/logs/madlib_compile.log + +echo "---------- Installing and running install-check --------------------" # Install MADlib and run install check -docker exec -it madlib /incubator-madlib/build/src/bin/madpack -p postgres -c postgres/postgres@localhost:5432/postgres install -docker exec -it madlib /incubator-madlib/build/src/bin/madpack -p postgres -c postgres/postgres@localhost:5432/postgres install-check +cat <<EOF +docker exec madlib /build/src/bin/madpack -p postgres -c postgres/postgres@localhost:5432/postgres install | tee $workdir/logs/madlib_install.log +docker exec madlib /build/src/bin/madpack -p postgres -c postgres/postgres@localhost:5432/postgres install-check | tee $workdir/logs/madlib_install_check.log +EOF +docker exec madlib /build/src/bin/madpack -p postgres -c postgres/postgres@localhost:5432/postgres install | tee $workdir/logs/madlib_install.log +docker exec madlib /build/src/bin/madpack -p postgres -c postgres/postgres@localhost:5432/postgres install-check | tee $workdir/logs/madlib_install_check.log -docker kill madlib -docker rm madlib +echo "--------- Copying packages -----------------" +echo "docker cp madlib:build $workdir" +docker cp madlib:build $workdir + +echo "-------------------------------" +echo "ls -la" +ls -la +echo "-------------------------------" +echo "ls -la build" +ls -la build/ +echo "-------------------------------" + +# convert install-check test results to junit format for reporting +cat <<EOF +python incubator-madlib/tool/jenkins/junit_export.py $workdir/logs/madlib_install_check.log $workdir/logs/madlib_install_check.xml +EOF +python incubator-madlib/tool/jenkins/junit_export.py $workdir/logs/madlib_install_check.log $workdir/logs/madlib_install_check.xml http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/5984d827/tool/jenkins/junit_export.py ---------------------------------------------------------------------- diff --git a/tool/jenkins/junit_export.py b/tool/jenkins/junit_export.py new file mode 100644 index 0000000..ce30320 --- /dev/null +++ b/tool/jenkins/junit_export.py @@ -0,0 +1,96 @@ +#!/usr/env python + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import re +import sys +from collections import namedtuple + +""" Convert install-check results into a standardized JUnit XML format + +Example of JUnit output: + +<?xml version="1.0" encoding="UTF-8"?> +<testsuite tests="3"> + <testcase classname="foo1" name="ASuccessfulTest"/> + <testcase classname="foo2" name="AnotherSuccessfulTest"/> + <testcase classname="foo3" name="AFailingTest"> + <failure type="NotEnoughFoo"> details about failure </failure> + </testcase> +</testsuite> +""" + + +TestResult = namedtuple("TestResult", 'name suite status duration') + + +def _test_result_factory(install_check_log): + """ + Args: + @param install_check_log: File name containing results from install-check + + Returns: + Next result of type test_result + """ + with open(install_check_log, 'r') as ic_log: + for line in ic_log: + m = re.match(r"^TEST CASE RESULT\|Module: (.*)\|(.*)\|(.*)\|Time: ([0-9]+)(.*)", line) + if m: + yield TestResult(name=m.group(2), suite=m.group(1), + status=m.group(3), duration=m.group(4)) +# ---------------------------------------------------------------------- + + +def _add_header(out_log, n_tests): + header = ['<?xml version="1.0" encoding="UTF-8"?>', + '<testsuite tests="{0}">'.format(n_tests), ''] + out_log.write('\n'.join(header)) + + +def _add_footer(out_log): + header = ['', '</testsuite>'] + out_log.write('\n'.join(header)) + + +def _add_test_case(out_log, test_results): + for res in test_results: + try: + # convert duration from milliseconds to seconds + duration = float(res.duration)/1000 + except TypeError: + duration = 0.0 + output = ['<testcase classname="{t.suite}" name="{t.name}" ' + 'status="{t.status}" time="{d}">'. + format(t=res, d=duration)] + output.append('</testcase>') + out_log.write('\n'.join(output)) + + +def main(install_check_log, test_output_log): + + # need number of test results - so have to create the iterable + all_test_results = [i for i in _test_result_factory(install_check_log)] + + with open(test_output_log, 'w') as out_log: + _add_header(out_log, len(all_test_results)) + _add_test_case(out_log, all_test_results) + _add_footer(out_log) + + +if __name__ == "__main__": + main(sys.argv[1], sys.argv[2])
