Repository: beam Updated Branches: refs/heads/master 7478da997 -> 29b828235
Retire Travis-CI coverage in Apache Beam Project: http://git-wip-us.apache.org/repos/asf/beam/repo Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/15d8eacc Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/15d8eacc Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/15d8eacc Branch: refs/heads/master Commit: 15d8eacc8514197ca18289fe4b88ef245dd3379e Parents: 7478da9 Author: Davor Bonaci <[email protected]> Authored: Fri Apr 28 14:19:09 2017 -0700 Committer: Davor Bonaci <[email protected]> Committed: Fri Apr 28 16:23:30 2017 -0700 ---------------------------------------------------------------------- .test-infra/jenkins/test_wordcount.sh | 125 +++++++++++++++++++++++++++++ .test-infra/travis/README.md | 23 ------ .test-infra/travis/settings.xml | 33 -------- .test-infra/travis/test_wordcount.sh | 125 ----------------------------- .travis.yml | 93 --------------------- README.md | 3 +- 6 files changed, 126 insertions(+), 276 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/beam/blob/15d8eacc/.test-infra/jenkins/test_wordcount.sh ---------------------------------------------------------------------- diff --git a/.test-infra/jenkins/test_wordcount.sh b/.test-infra/jenkins/test_wordcount.sh new file mode 100755 index 0000000..e059a35 --- /dev/null +++ b/.test-infra/jenkins/test_wordcount.sh @@ -0,0 +1,125 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This script runs WordCount example locally in a few different ways. +# Specifically, all combinations of: +# a) using mvn exec, or java -cp with a bundled jar file; +# b) input filename with no directory component, with a relative directory, or +# with an absolute directory; AND +# c) input filename containing wildcards or not. +# +# The one optional parameter is a path from the directory containing the script +# to the directory containing the top-level (parent) pom.xml. If no parameter +# is provided, the script assumes that directory is equal to the directory +# containing the script itself. +# +# The exit-code of the script indicates success or a failure. + +set -e +set -o pipefail + +PASS=1 +VERSION=$(mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | grep -v '\[') +JAR_FILE=examples/java/target/beam-examples-java-bundled-${VERSION}.jar + +function check_result_hash { + local name=$1 + local outfile_prefix=$2 + local expected=$3 + + local actual=$(LC_ALL=C sort $outfile_prefix-* | md5sum | awk '{print $1}' \ + || LC_ALL=C sort $outfile_prefix-* | md5 -q) || exit 2 # OSX + if [[ "$actual" != "$expected" ]] + then + echo "FAIL $name: Output hash mismatch. Got $actual, expected $expected." + PASS="" + echo "head hexdump of actual:" + head $outfile_prefix-* | hexdump -c + else + echo "pass $name" + # Output files are left behind in /tmp + fi +} + +function get_outfile_prefix { + local name=$1 + # NOTE: mktemp on OSX doesn't support --tmpdir + mktemp -u "/tmp/$name.out.XXXXXXXXXX" +} + +function run_via_mvn { + local name=$1 + local input=$2 + local expected_hash=$3 + + local outfile_prefix="$(get_outfile_prefix "$name")" || exit 2 + local cmd='mvn exec:java -f pom.xml -pl examples/java \ + -Dexec.mainClass=org.apache.beam.examples.WordCount \ + -Dexec.args="--runner=DirectRunner --inputFile='"$input"' --output='"$outfile_prefix"'"' + echo "$name: Running $cmd" >&2 + sh -c "$cmd" + check_result_hash "$name" "$outfile_prefix" "$expected_hash" +} + +function run_bundled { + local name=$1 + local input=$2 + local expected_hash=$3 + + local outfile_prefix="$(get_outfile_prefix "$name")" || exit 2 + local cmd='java -cp '"$JAR_FILE"' \ + org.apache.beam.examples.WordCount \ + --runner=DirectRunner \ + --inputFile='"'$input'"' \ + --output='"$outfile_prefix" + echo "$name: Running $cmd" >&2 + sh -c "$cmd" + check_result_hash "$name" "$outfile_prefix" "$expected_hash" +} + +function run_all_ways { + local name=$1 + local input=$2 + local expected_hash=$3 + + run_via_mvn ${name}a "$input" $expected_hash + check_for_jar_file + run_bundled ${name}b "$input" $expected_hash +} + +function check_for_jar_file { + if [[ ! -f $JAR_FILE ]] + then + echo "Jar file $JAR_FILE not created" >&2 + exit 2 + fi +} + +run_all_ways wordcount1 "LICENSE" c5350a5ad4bb51e3e018612b4b044097 +run_all_ways wordcount2 "./LICENSE" c5350a5ad4bb51e3e018612b4b044097 +run_all_ways wordcount3 "$PWD/LICENSE" c5350a5ad4bb51e3e018612b4b044097 +run_all_ways wordcount4 "L*N?E*" c5350a5ad4bb51e3e018612b4b044097 +run_all_ways wordcount5 "./LICE*N?E" c5350a5ad4bb51e3e018612b4b044097 +run_all_ways wordcount6 "$PWD/*LIC?NSE" c5350a5ad4bb51e3e018612b4b044097 + +if [[ ! "$PASS" ]] +then + echo "One or more tests FAILED." + exit 1 +fi +echo "All tests PASS" http://git-wip-us.apache.org/repos/asf/beam/blob/15d8eacc/.test-infra/travis/README.md ---------------------------------------------------------------------- diff --git a/.test-infra/travis/README.md b/.test-infra/travis/README.md deleted file mode 100644 index 526995a..0000000 --- a/.test-infra/travis/README.md +++ /dev/null @@ -1,23 +0,0 @@ -<!-- - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> - -# Travis Scripts - -This directory contains scripts used for [Travis CI](https://travis-ci.org/apache/beam/) -testing. http://git-wip-us.apache.org/repos/asf/beam/blob/15d8eacc/.test-infra/travis/settings.xml ---------------------------------------------------------------------- diff --git a/.test-infra/travis/settings.xml b/.test-infra/travis/settings.xml deleted file mode 100644 index e086aec..0000000 --- a/.test-infra/travis/settings.xml +++ /dev/null @@ -1,33 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<settings> - <servers> - <server> - <id>central</id> - <configuration> - <httpConfiguration> - <all> - <connectionTimeout>1000</connectionTimeout> - </all> - </httpConfiguration> - <timeout>5000</timeout> - </configuration> - </server> - </servers> -</settings> http://git-wip-us.apache.org/repos/asf/beam/blob/15d8eacc/.test-infra/travis/test_wordcount.sh ---------------------------------------------------------------------- diff --git a/.test-infra/travis/test_wordcount.sh b/.test-infra/travis/test_wordcount.sh deleted file mode 100755 index e059a35..0000000 --- a/.test-infra/travis/test_wordcount.sh +++ /dev/null @@ -1,125 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# This script runs WordCount example locally in a few different ways. -# Specifically, all combinations of: -# a) using mvn exec, or java -cp with a bundled jar file; -# b) input filename with no directory component, with a relative directory, or -# with an absolute directory; AND -# c) input filename containing wildcards or not. -# -# The one optional parameter is a path from the directory containing the script -# to the directory containing the top-level (parent) pom.xml. If no parameter -# is provided, the script assumes that directory is equal to the directory -# containing the script itself. -# -# The exit-code of the script indicates success or a failure. - -set -e -set -o pipefail - -PASS=1 -VERSION=$(mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | grep -v '\[') -JAR_FILE=examples/java/target/beam-examples-java-bundled-${VERSION}.jar - -function check_result_hash { - local name=$1 - local outfile_prefix=$2 - local expected=$3 - - local actual=$(LC_ALL=C sort $outfile_prefix-* | md5sum | awk '{print $1}' \ - || LC_ALL=C sort $outfile_prefix-* | md5 -q) || exit 2 # OSX - if [[ "$actual" != "$expected" ]] - then - echo "FAIL $name: Output hash mismatch. Got $actual, expected $expected." - PASS="" - echo "head hexdump of actual:" - head $outfile_prefix-* | hexdump -c - else - echo "pass $name" - # Output files are left behind in /tmp - fi -} - -function get_outfile_prefix { - local name=$1 - # NOTE: mktemp on OSX doesn't support --tmpdir - mktemp -u "/tmp/$name.out.XXXXXXXXXX" -} - -function run_via_mvn { - local name=$1 - local input=$2 - local expected_hash=$3 - - local outfile_prefix="$(get_outfile_prefix "$name")" || exit 2 - local cmd='mvn exec:java -f pom.xml -pl examples/java \ - -Dexec.mainClass=org.apache.beam.examples.WordCount \ - -Dexec.args="--runner=DirectRunner --inputFile='"$input"' --output='"$outfile_prefix"'"' - echo "$name: Running $cmd" >&2 - sh -c "$cmd" - check_result_hash "$name" "$outfile_prefix" "$expected_hash" -} - -function run_bundled { - local name=$1 - local input=$2 - local expected_hash=$3 - - local outfile_prefix="$(get_outfile_prefix "$name")" || exit 2 - local cmd='java -cp '"$JAR_FILE"' \ - org.apache.beam.examples.WordCount \ - --runner=DirectRunner \ - --inputFile='"'$input'"' \ - --output='"$outfile_prefix" - echo "$name: Running $cmd" >&2 - sh -c "$cmd" - check_result_hash "$name" "$outfile_prefix" "$expected_hash" -} - -function run_all_ways { - local name=$1 - local input=$2 - local expected_hash=$3 - - run_via_mvn ${name}a "$input" $expected_hash - check_for_jar_file - run_bundled ${name}b "$input" $expected_hash -} - -function check_for_jar_file { - if [[ ! -f $JAR_FILE ]] - then - echo "Jar file $JAR_FILE not created" >&2 - exit 2 - fi -} - -run_all_ways wordcount1 "LICENSE" c5350a5ad4bb51e3e018612b4b044097 -run_all_ways wordcount2 "./LICENSE" c5350a5ad4bb51e3e018612b4b044097 -run_all_ways wordcount3 "$PWD/LICENSE" c5350a5ad4bb51e3e018612b4b044097 -run_all_ways wordcount4 "L*N?E*" c5350a5ad4bb51e3e018612b4b044097 -run_all_ways wordcount5 "./LICE*N?E" c5350a5ad4bb51e3e018612b4b044097 -run_all_ways wordcount6 "$PWD/*LIC?NSE" c5350a5ad4bb51e3e018612b4b044097 - -if [[ ! "$PASS" ]] -then - echo "One or more tests FAILED." - exit 1 -fi -echo "All tests PASS" http://git-wip-us.apache.org/repos/asf/beam/blob/15d8eacc/.travis.yml ---------------------------------------------------------------------- diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 0b5d700..0000000 --- a/.travis.yml +++ /dev/null @@ -1,93 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -language: java - -sudo: false - -notifications: - email: - # Group email notifications are disabled for now, since we cannot do it on a per-branch basis. - # Right now, it would trigger a notification for each fork, which generates a lot of spam. - # recipients: - # - [email protected] - on_success: change - on_failure: always - -addons: - apt: - packages: - - python2.7 -env: - global: - - MAVEN_OVERRIDE="--settings=.test-infra/travis/settings.xml" - - MAVEN_CONTAINER_OVERRIDE="-DbeamSurefireArgline='-Xmx512m'" - -matrix: - include: - # On OSX, run with default JDK only. - - os: osx - - # On Linux, run with specific JDKs only. - - os: linux - env: CUSTOM_JDK="oraclejdk8" MAVEN_OVERRIDE="$MAVEN_OVERRIDE $MAVEN_CONTAINER_OVERRIDE" - - os: linux - env: CUSTOM_JDK="oraclejdk7" MAVEN_OVERRIDE="$MAVEN_OVERRIDE $MAVEN_CONTAINER_OVERRIDE" - - os: linux - env: CUSTOM_JDK="openjdk7" MAVEN_OVERRIDE="$MAVEN_OVERRIDE $MAVEN_CONTAINER_OVERRIDE" - - os: linux - env: MAVEN_OVERRIDE="-Peclipse-jdt -DskipTests $MAVEN_OVERRIDE $MAVEN_CONTAINER_OVERRIDE" CUSTOM_JDK="oraclejdk8" - - # Python SDK tests. - - os: osx - env: TEST_PYTHON="1" - - os: linux - env: TEST_PYTHON="1" - -before_install: - # The -XX:+TieredCompilation -XX:TieredStopAtLevel=1 JVM options enable - # tiered compilation to make the JVM startup times faster during the tests. - - echo 'MAVEN_OPTS="$MAVEN_OPTS -Xmx1024m -XX:MaxPermSize=512m -XX:+TieredCompilation -XX:TieredStopAtLevel=1 -XX:+BytecodeVerificationLocal"' >> ~/.mavenrc - - echo $'MAVEN_OPTS="$MAVEN_OPTS -Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=yyyy-MM-dd\'T\'HH:mm:ss.SSS"' >> ~/.mavenrc - - cat ~/.mavenrc - - if [ "$TRAVIS_OS_NAME" == "osx" ]; then export JAVA_HOME=$(/usr/libexec/java_home); fi - - if [ "$TRAVIS_OS_NAME" == "linux" ]; then jdk_switcher use "$CUSTOM_JDK"; fi - - export BEAM_SUREFIRE_ARGLINE="-Xmx512m" - # Python SDK environment settings. - - if [ "$TRAVIS_OS_NAME" == "osx" ]; then export TOX_HOME=$HOME/Library/Python/2.7/bin; fi - - if [ "$TRAVIS_OS_NAME" == "linux" ]; then export TOX_HOME=$HOME/.local/bin; fi - -install: - - if [ ! "$TEST_PYTHON" ]; then travis_retry mvn -B install clean -U -DskipTests=true; fi - - if [ "$TEST_PYTHON" ] && pip list | grep tox; then TOX_FILE=`which tox` ; export TOX_HOME=`dirname $TOX_FILE`; fi - - if [ "$TEST_PYTHON" ] && ! pip list | grep tox; then travis_retry pip install tox --user; fi - # Removing this here protects from inadvertent caching - - rm -rf "$HOME/.m2/repository/org/apache/beam" - -script: - - if [ "$TEST_PYTHON" ]; then travis_retry $TOX_HOME/tox -c sdks/python/tox.ini; fi - - if [ ! "$TEST_PYTHON" ]; then travis_retry mvn --batch-mode --update-snapshots --no-snapshot-updates --threads 1C $MAVEN_OVERRIDE install && travis_retry bash -ex .test-infra/travis/test_wordcount.sh; fi - -cache: - directories: - - $HOME/.m2/repository - -before_cache: - # Removing here increases cache hits (makes the above - # rm in `install` redundant unless our config has a bug, - # but it will be idempotent) - - rm -rf "$HOME/.m2/repository/org/apache/beam" http://git-wip-us.apache.org/repos/asf/beam/blob/15d8eacc/README.md ---------------------------------------------------------------------- diff --git a/README.md b/README.md index 23768ce..1202c54 100644 --- a/README.md +++ b/README.md @@ -23,8 +23,7 @@ ## Status -[](https://travis-ci.org/apache/beam?branch=master) -[](https://builds.apache.org/job/beam_PostCommit_MavenVerify/) +[](https://builds.apache.org/job/beam_PostCommit_Java_MavenInstall/) [](https://coveralls.io/github/apache/beam?branch=master) ## Overview
