This is an automated email from the ASF dual-hosted git repository.
cutting pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/avro.git
The following commit(s) were added to refs/heads/master by this push:
new a11bd49 [AVRO-2229] Test using a Docker image (#336)
a11bd49 is described below
commit a11bd49e86d50f5a840ab2cdfe4f6927bedb4f93
Author: Fokko Driesprong <[email protected]>
AuthorDate: Fri Oct 5 08:49:56 2018 -0700
[AVRO-2229] Test using a Docker image (#336)
* [AVRO-2229] Fix the docker-based build/test.
---
BUILD.txt => BUILD.md | 34 +++++++---
build.sh | 13 ++--
lang/csharp/src/apache/codegen/app.config | 7 +-
lang/csharp/src/apache/perf/app.config | 7 +-
lang/java/build.sh | 2 +-
lang/py3/setup.py | 0
share/docker/Dockerfile | 90 +++++++++++++++++--------
lang/java/build.sh => share/docker/run-tests.sh | 57 +++++-----------
8 files changed, 124 insertions(+), 86 deletions(-)
diff --git a/BUILD.txt b/BUILD.md
similarity index 67%
rename from BUILD.txt
rename to BUILD.md
index 223bc4d..882338a 100644
--- a/BUILD.txt
+++ b/BUILD.md
@@ -1,6 +1,6 @@
-Apache Avro Build Instructions
+# Apache Avro Build Instructions
-REQUIREMENTS
+## Requirements
The following packages must be installed before Avro can be built:
@@ -18,13 +18,18 @@ The following packages must be installed before Avro can be
built:
IO::String, Object::Tiny, Compress::ZLib, Test::More,
Test::Exception, Test::Pod
- Apache Ant 1.7
- - Apache Forrest 0.8 (for documentation)
+ - Apache Forrest 0.9 (for documentation)
- md5sum, sha1sum, used by top-level dist target
To simplify this, you can run a Docker container with all the above
-dependencies installed by installing docker.io and typing:
+dependencies installed by installing Docker and run:
- ./build.sh docker
+```bash
+./build.sh docker
+docker@539f6535c9db:~/avro$ cd lang/java/
+docker@539f6535c9db:~/avro/lang/java$ ./build.sh test
+[INFO] Scanning for projects...
+```
When this completes you will be in a shell running in the
container. Building the image the first time may take a while (20
@@ -36,11 +41,22 @@ The working directory in the container is mounted from your
host. This
allows you to access the files in your Avro development tree from the
Docker container.
-BUILDING
+## Building
Once the requirements are installed (or from the Docker container),
build.sh can be used as follows:
- './build.sh test' runs tests for all languages
- './build.sh dist' creates all release distribution files in dist/
- './build.sh clean' removes all generated artifacts
+```
+./build.sh test # runs tests for all languages
+./build.sh dist # creates all release distribution files in dist/
+./build.sh clean # removes all generated artifacts
+```
+
+## Testing
+
+Testing is done with the same Docker container as mentioned in the building
+step. The difference is that it will do clean run of the full test suite:
+
+```bash
+./build.sh docker-test
+```
diff --git a/build.sh b/build.sh
index 255e46a..8810d6b 100755
--- a/build.sh
+++ b/build.sh
@@ -22,7 +22,7 @@ cd `dirname "$0"` # connect to root
VERSION=`cat share/VERSION.txt`
function usage {
- echo "Usage: $0 {test|dist|sign|clean|docker|rat|githooks}"
+ echo "Usage: $0 {test|dist|sign|clean|docker|rat|githooks|docker-test}"
exit 1
}
@@ -197,7 +197,7 @@ do
;;
docker)
- docker build -t avro-build share/docker
+ docker build -t avro-build -f share/docker/Dockerfile .
if [ "$(uname -s)" == "Linux" ]; then
USER_NAME=${SUDO_USER:=$USER}
USER_ID=$(id -u $USER_NAME)
@@ -217,13 +217,13 @@ UserSpecificDocker
# within the container and use the result on your normal
# system. And this also is a significant speedup in subsequent
# builds because the dependencies are downloaded only once.
- docker run --rm=true -t -i \
+ docker run --rm -t -i \
-v ${PWD}:/home/${USER_NAME}/avro \
-w /home/${USER_NAME}/avro \
-v ${HOME}/.m2:/home/${USER_NAME}/.m2 \
-v ${HOME}/.gnupg:/home/${USER_NAME}/.gnupg \
-u ${USER_NAME} \
- avro-build-${USER_NAME}
+ avro-build-${USER_NAME} bash
;;
rat)
@@ -236,6 +236,11 @@ UserSpecificDocker
find .git/hooks/ -type f | fgrep -v sample | xargs chmod 755
;;
+ docker-test)
+ docker build -t avro-test -f share/docker/Dockerfile .
+ docker run --rm -v ${PWD}:/avro/ avro-test
+ ;;
+
*)
usage
;;
diff --git a/lang/csharp/src/apache/codegen/app.config
b/lang/csharp/src/apache/codegen/app.config
index ae24790..e8a4254 100755
--- a/lang/csharp/src/apache/codegen/app.config
+++ b/lang/csharp/src/apache/codegen/app.config
@@ -15,5 +15,8 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<configuration>
-<startup><supportedRuntime version="v2.0.50727"/></startup></configuration>
+<configuration useLegacyV2RuntimeActivationPolicy="true">
+ <startup>
+ <supportedRuntime version="v4.0.30319" />
+ </startup>
+</configuration>
diff --git a/lang/csharp/src/apache/perf/app.config
b/lang/csharp/src/apache/perf/app.config
index ae24790..e8a4254 100755
--- a/lang/csharp/src/apache/perf/app.config
+++ b/lang/csharp/src/apache/perf/app.config
@@ -15,5 +15,8 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<configuration>
-<startup><supportedRuntime version="v2.0.50727"/></startup></configuration>
+<configuration useLegacyV2RuntimeActivationPolicy="true">
+ <startup>
+ <supportedRuntime version="v4.0.30319" />
+ </startup>
+</configuration>
diff --git a/lang/java/build.sh b/lang/java/build.sh
index bf1a4fb..76f7f57 100755
--- a/lang/java/build.sh
+++ b/lang/java/build.sh
@@ -43,7 +43,7 @@ function do_dist() {
case "$target" in
test)
- mvn test
+ mvn -B test
;;
dist)
diff --git a/lang/py3/setup.py b/lang/py3/setup.py
old mode 100644
new mode 100755
diff --git a/share/docker/Dockerfile b/share/docker/Dockerfile
index 535ae31..de09220 100644
--- a/share/docker/Dockerfile
+++ b/share/docker/Dockerfile
@@ -21,52 +21,88 @@ FROM java:8-jdk
WORKDIR /root
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
# Add the repository for node.js 4.x
RUN curl -sL https://deb.nodesource.com/setup_4.x | bash -
# Install dependencies from packages
-RUN apt-get update && apt-get install --no-install-recommends -y \
- git subversion curl ant make maven \
- gcc cmake libjansson-dev asciidoc source-highlight \
- g++ flex bison libboost-all-dev doxygen \
- mono-devel mono-gmcs nunit \
- nodejs \
- perl \
- php5 phpunit php5-gmp bzip2 \
- python python-setuptools python3-setuptools \
- ruby ruby-dev rake \
- libsnappy1 libsnappy-dev
+RUN apt-get update && \
+ apt-get install --no-install-recommends -y \
+ ant \
+ asciidoc \
+ bison \
+ bzip2 \
+ cmake \
+ curl \
+ doxygen \
+ flex \
+ g++ \
+ gcc \
+ git \
+ libboost-all-dev \
+ libfontconfig1-dev \
+ libfreetype6-dev \
+ libglib2.0-dev \
+ libjansson-dev \
+ libsnappy-dev \
+ libsnappy1 \
+ make \
+ maven \
+ mono-devel \
+ nodejs \
+ nunit \
+ perl \
+ php5 \
+ php5-gmp \
+ phpunit \
+ python \
+ python-setuptools \
+ python3-setuptools \
+ rake \
+ ruby \
+ ruby-dev \
+ source-highlight \
+ subversion && \
+ apt-get clean && \
+ rm -rf /var/lib/apt/lists/*
# Install Forrest in /usr/local/apache-forrest
-# Download
-RUN cd /usr/local/ && wget
"http://www.apache.org/dyn/closer.lua?action=download&filename=/forrest/apache-forrest-0.9-sources.tar.gz";
-O "apache-forrest-0.9-sources.tar.gz"
-RUN cd /usr/local/ && wget
"http://www.apache.org/dyn/closer.lua?action=download&filename=/forrest/apache-forrest-0.9-dependencies.tar.gz";
-O "apache-forrest-0.9-dependencies.tar.gz"
-
-# Unpack Apache Forrest
-RUN cd /usr/local/ && \
- tar xzf apache-forrest-0.9-sources.tar.gz && \
- tar xzf apache-forrest-0.9-dependencies.tar.gz && \
- mv apache-forrest-0.9 apache-forrest
-RUN cd /usr/local/apache-forrest/main && ./build.sh
+RUN curl -L -s
"http://www.apache.org/dyn/closer.lua?action=download&filename=/forrest/apache-forrest-0.9-sources.tar.gz";
| tar -xzf - -C /usr/local/ && \
+ curl -L -s
"http://www.apache.org/dyn/closer.lua?action=download&filename=/forrest/apache-forrest-0.9-dependencies.tar.gz";
| tar -xzf - -C /usr/local/ && \
+ mv /usr/local/apache-forrest-0.9 /usr/local/apache-forrest && \
+ cd /usr/local/apache-forrest/main && \
+ ./build.sh
# The solution for https://issues.apache.org/jira/browse/PIG-3906
-RUN mkdir -p /usr/local/apache-forrest/plugins && chmod a+rwX -R
/usr/local/apache-forrest/plugins
-RUN mkdir -p /usr/local/apache-forrest/build/plugins && chmod a+rwX -R
/usr/local/apache-forrest/build/plugins
-
# Configure where forrest can be found
-RUN echo 'forrest.home=/usr/local/apache-forrest' > build.properties
+RUN mkdir -p /usr/local/apache-forrest/plugins && chmod a+rwX -R
/usr/local/apache-forrest/plugins && \
+ mkdir -p /usr/local/apache-forrest/build/plugins && chmod a+rwX -R
/usr/local/apache-forrest/build/plugins && \
+ echo 'forrest.home=/usr/local/apache-forrest' > build.properties
+
ENV FORREST_HOME /usr/local/apache-forrest
# Install Perl modules
-RUN curl -L http://cpanmin.us | perl - --self-upgrade # non-interactive cpan
-RUN cpanm install Module::Install Module::Install::ReadmeFromPod \
+RUN curl -L http://cpanmin.us | perl - --self-upgrade && \
+ cpanm install Module::Install Module::Install::ReadmeFromPod \
Module::Install::Repository \
Math::BigInt JSON::XS Try::Tiny Regexp::Common Encode \
IO::String Object::Tiny Compress::Zlib Test::More \
Test::Exception Test::Pod
+# Install mono modules
+RUN mkdir -p /tmp/nunit/ && \
+ cd /tmp/nunit/ && \
+ curl -L -s -o nunit.zip
https://github.com/nunit-legacy/nunitv2/releases/download/2.7.0/NUnit-2.7.0.zip
&& \
+ unzip nunit.zip && \
+ rm nunit.zip
+
+ENV MONO_PATH /tmp/nunit/bin/lib
+
# Install Ruby modules
RUN gem install echoe yajl-ruby multi_json snappy
# Install global Node modules
RUN npm install -g grunt-cli
+
+CMD ["/avro/share/docker/run-tests.sh"]
diff --git a/lang/java/build.sh b/share/docker/run-tests.sh
similarity index 61%
copy from lang/java/build.sh
copy to share/docker/run-tests.sh
index bf1a4fb..e9919a7 100755
--- a/lang/java/build.sh
+++ b/share/docker/run-tests.sh
@@ -15,49 +15,24 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-set -e # exit on error
-
-function usage {
- echo "Usage: $0 {test|dist|clean}"
- exit 1
+headline(){
+ echo -e
"\e[1;34m#################################################################"
+ echo -e "##### $1 \e[1;37m"
+ echo -e
"\e[1;34m#################################################################\e[0m"
}
-if [ $# -eq 0 ]
-then
- usage
-fi
-
-if [ -f VERSION.txt ]
-then
- VERSION=`cat VERSION.txt`
-else
- VERSION=`cat ../../share/VERSION.txt`
-fi
+set -e
-for target in "$@"
+for lang in /avro/lang/*/
do
-
-function do_dist() {
- mvn -P dist package -DskipTests -Davro.version=$VERSION javadoc:aggregate
-}
-
-case "$target" in
- test)
- mvn test
- ;;
-
- dist)
- do_dist
- ;;
-
- clean)
- mvn clean
- ;;
-
- *)
- usage
-esac
-
+ headline "Run tests: $lang"
+ cd "$lang"
+
+ if [[ "$lang" = *"c++"* ]]; then
+ # The current cpp tests are failing:
+ # https://issues.apache.org/jira/projects/AVRO/issues/AVRO-2230
+ ./build.sh test || true
+ else
+ ./build.sh test
+ fi
done
-
-exit 0
