This is an automated email from the ASF dual-hosted git repository.
wesm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 54634dd ARROW-3456: [CI] Reuse docker images and optimize
docker-compose containers
54634dd is described below
commit 54634dd6745dccad393c1461f11a787047dbc41a
Author: Krisztián Szűcs <[email protected]>
AuthorDate: Sat Oct 13 17:41:45 2018 -0400
ARROW-3456: [CI] Reuse docker images and optimize docker-compose containers
Author: Krisztián Szűcs <[email protected]>
Closes #2751 from kszucs/ARROW-3456 and squashes the following commits:
649c49f0a <Krisztián Szűcs> uncomment hdfs dockerfile
0e8e8fc25 <Krisztián Szűcs> optimize docker images
---
.dockerignore | 7 ++
.gitignore | 2 +-
c_glib/Dockerfile | 44 +++--------
ci/conda_env_python.yml | 1 -
ci/docker_build_c_glib.sh | 17 +++--
ci/docker_build_cpp.sh | 7 +-
ci/docker_build_python.sh | 7 +-
ci/docker_install_conda.sh | 6 +-
cpp/Dockerfile | 44 ++++++-----
cpp/build-support/iwyu/iwyu.sh | 6 +-
dev/iwyu/run_iwyu.sh | 72 ------------------
dev/lint/Dockerfile | 37 +++-------
dev/lint/run_clang_format.sh | 8 +-
dev/{iwyu/Dockerfile => lint/run_iwyu.sh} | 21 +++++-
dev/lint/{run_checks.sh => run_linters.sh} | 14 ++--
docker-compose.yml | 115 +++++++++++++++++++----------
go/Dockerfile | 11 ++-
integration/hdfs/Dockerfile | 3 +
python/Dockerfile | 51 +++----------
rust/Dockerfile | 6 +-
20 files changed, 201 insertions(+), 278 deletions(-)
diff --git a/.dockerignore b/.dockerignore
index 61f42f0..1c6bc1e 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -16,9 +16,11 @@
# under the License.
.git
+docker_cache
# IDE
.vscode
+*/.vscode
# c_glib
c_glib/build
@@ -34,7 +36,12 @@ c_glib/*/.libs
cpp/.idea
cpp/build
cpp/*-build
+cpp/*_build
cpp/Testing
+cpp/thirdparty
+!cpp/thirdparty/jemalloc
+!cpp/thirdparty/versions.txt
+!cpp/thirdparty/hadoop/include
# python
python/build
diff --git a/.gitignore b/.gitignore
index e54be5f..79a2a8e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -41,4 +41,4 @@ python/.eggs/
pkgs
.Rproj.user
arrow.Rcheck/
-
+docker_cache
diff --git a/c_glib/Dockerfile b/c_glib/Dockerfile
index 76fc8cf..5d64a5f 100644
--- a/c_glib/Dockerfile
+++ b/c_glib/Dockerfile
@@ -15,37 +15,19 @@
# specific language governing permissions and limitations
# under the License.
-FROM ubuntu:18.04
+FROM arrow:cpp
ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get -q update && \
- apt-get -q install --no-install-recommends -y \
- gcc \
- g++ \
- git \
- wget \
+RUN apt-get -q install --no-install-recommends -y \
tzdata \
ruby-dev \
pkg-config \
- ninja-build \
autoconf-archive \
gtk-doc-tools \
libgirepository1.0-dev
-ENV CC=gcc \
- CXX=g++ \
- PATH=/opt/conda/bin:$PATH \
- CONDA_PREFIX=/opt/conda
-
-# install dependencies
-ADD ci/docker_install_conda.sh \
- ci/conda_env_cpp.yml \
- /arrow/ci/
ADD c_glib/Gemfile /arrow/c_glib/
-RUN arrow/ci/docker_install_conda.sh && \
- conda install -c conda-forge \
- --file arrow/ci/conda_env_cpp.yml \
- meson=0.47.1 && \
+RUN conda install -c conda-forge meson=0.47.1 && \
conda clean --all && \
gem install bundler && \
bundle install --gemfile arrow/c_glib/Gemfile
@@ -53,20 +35,12 @@ RUN arrow/ci/docker_install_conda.sh && \
# build cpp
ENV ARROW_BUILD_TESTS=OFF \
ARROW_BUILD_UTILITIES=OFF \
- ARROW_INSTALL_NAME_RPATH=OFF
-ADD ci/docker_build_cpp.sh /arrow/ci/
-ADD cpp /arrow/cpp
-ADD format /arrow/format
-ADD java/pom.xml /arrow/java/pom.xml
-RUN arrow/ci/docker_build_cpp.sh
-
-# build c_glib
-ENV LD_LIBRARY_PATH="${CONDA_PREFIX}/lib" \
+ ARROW_INSTALL_NAME_RPATH=OFF \
+ LD_LIBRARY_PATH="${CONDA_PREFIX}/lib" \
PKG_CONFIG_PATH="${CONDA_PREFIX}/lib/pkgconfig" \
GI_TYPELIB_PATH="${CONDA_PREFIX}/lib/girepository-1.0"
-ADD ci/docker_build_c_glib.sh /arrow/ci/
-ADD c_glib /arrow/c_glib
-RUN arrow/ci/docker_build_c_glib.sh
-WORKDIR arrow/c_glib
-CMD test/run-test.rb
+# build, install and test
+CMD arrow/ci/docker_build_cpp.sh && \
+ arrow/ci/docker_build_c_glib.sh && \
+ arrow/c_glib/test/run-test.rb
diff --git a/ci/conda_env_python.yml b/ci/conda_env_python.yml
index 43022ae..37ec654 100644
--- a/ci/conda_env_python.yml
+++ b/ci/conda_env_python.yml
@@ -16,7 +16,6 @@
# under the License.
cython
-ipython
nomkl
numpy
pandas
diff --git a/ci/docker_build_c_glib.sh b/ci/docker_build_c_glib.sh
index 7390e79..28ef901 100755
--- a/ci/docker_build_c_glib.sh
+++ b/ci/docker_build_c_glib.sh
@@ -24,14 +24,15 @@ export ARROW_C_GLIB_HOME=$CONDA_PREFIX
export CFLAGS="-DARROW_NO_DEPRECATED_API"
export CXXFLAGS="-DARROW_NO_DEPRECATED_API -D_GLIBCXX_USE_CXX11_ABI=0"
-pushd arrow/c_glib
- mkdir build
+mkdir -p /build/c_glib
- # Build with Meson
- meson build --prefix=$ARROW_C_GLIB_HOME --libdir=lib
+# Build with Meson
+meson --prefix=$ARROW_C_GLIB_HOME \
+ --libdir=lib \
+ /build/c_glib \
+ /arrow/c_glib
- pushd build
- ninja
- ninja install
- popd
+pushd /build/c_glib
+ ninja
+ ninja install
popd
diff --git a/ci/docker_build_cpp.sh b/ci/docker_build_cpp.sh
index 16a09f3..f1cf43f 100755
--- a/ci/docker_build_cpp.sh
+++ b/ci/docker_build_cpp.sh
@@ -17,6 +17,7 @@
# under the License.
set -e
+set -o xtrace
# Arrow specific environment variables
export ARROW_BUILD_TOOLCHAIN=$CONDA_PREFIX
@@ -26,8 +27,8 @@ export PARQUET_HOME=$CONDA_PREFIX
# https://arrow.apache.org/docs/python/development.html#known-issues
export CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0"
-mkdir -p arrow/cpp/build
-pushd arrow/cpp/build
+mkdir -p /build/cpp
+pushd /build/cpp
cmake -GNinja \
-DCMAKE_BUILD_TYPE=${ARROW_BUILD_TYPE:-debug} \
@@ -42,7 +43,7 @@ cmake -GNinja \
-DARROW_INSTALL_NAME_RPATH=${ARROW_INSTALL_NAME_RPATH:-ON} \
-DARROW_EXTRA_ERROR_CONTEXT=ON \
-DCMAKE_CXX_FLAGS=$CXXFLAGS \
- ..
+ /arrow/cpp
ninja
ninja install
diff --git a/ci/docker_build_python.sh b/ci/docker_build_python.sh
index f145694..e89a0b4 100755
--- a/ci/docker_build_python.sh
+++ b/ci/docker_build_python.sh
@@ -27,12 +27,13 @@ export PYARROW_CXXFLAGS=$CXXFLAGS
export PYARROW_CMAKE_GENERATOR=Ninja
# Build pyarrow
-pushd arrow/python
+pushd /arrow/python
python setup.py build_ext \
- --build-type=${ARROW_BUILD_TYPE:-debug} \
+ --build-temp=/build/python \
+ --build-type=${PYARROW_BUILD_TYPE:-debug} \
--with-parquet \
--with-plasma \
- --inplace
+ install
popd
diff --git a/ci/docker_install_conda.sh b/ci/docker_install_conda.sh
index 81d5256..427ee76 100755
--- a/ci/docker_install_conda.sh
+++ b/ci/docker_install_conda.sh
@@ -20,9 +20,9 @@
# Exit on any error
set -e
-wget --quiet
https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O
miniconda.sh
-bash miniconda.sh -b -q -p ${CONDA_PREFIX:=/opt/conda}
-rm miniconda.sh
+wget --quiet
https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O
/tmp/miniconda.sh
+bash /tmp/miniconda.sh -b -q -p ${CONDA_PREFIX:=/opt/conda}
+rm /tmp/miniconda.sh
ln -s ${CONDA_PREFIX}/etc/profile.d/conda.sh /etc/profile.d/conda.sh
echo ". ${CONDA_PREFIX}/etc/profile.d/conda.sh" >> ~/.bashrc
diff --git a/cpp/Dockerfile b/cpp/Dockerfile
index de5b40e..2cd0e01 100644
--- a/cpp/Dockerfile
+++ b/cpp/Dockerfile
@@ -17,20 +17,23 @@
FROM ubuntu:18.04
-RUN apt-get update && \
- apt-get install -y \
- gcc \
- g++ \
- git \
- wget \
- pkg-config \
- ninja-build
+# install build essentials
+RUN apt-get update -y -q && \
+ apt-get install -y -q --no-install-recommends \
+ autoconf \
+ automake \
+ ca-certificates \
+ ccache \
+ g++ \
+ gcc \
+ git \
+ ninja-build \
+ pkg-config \
+ wget
-ENV CC=gcc \
- CXX=g++ \
- PATH=/opt/conda/bin:$PATH \
+# install conda and required packages
+ENV PATH=/opt/conda/bin:$PATH \
CONDA_PREFIX=/opt/conda
-
ADD ci/docker_install_conda.sh \
ci/conda_env_cpp.yml \
/arrow/ci/
@@ -39,15 +42,10 @@ RUN arrow/ci/docker_install_conda.sh && \
--file arrow/ci/conda_env_cpp.yml && \
conda clean --all
-# build cpp with tests
-ENV ARROW_BUILD_TESTS=ON
-ADD ci/docker_build_cpp.sh /arrow/ci/
-ADD cpp /arrow/cpp
-ADD format /arrow/format
-ADD java/pom.xml /arrow/java/pom.xml
-RUN arrow/ci/docker_build_cpp.sh
+ENV CC=gcc \
+ CXX=g++ \
+ ARROW_BUILD_TESTS=ON
-# execute the tests
-WORKDIR arrow/cpp/build
-ENV PARQUET_TEST_DATA=/arrow/cpp/submodules/parquet-testing/data
-CMD ninja test
+# build, install and test
+CMD arrow/ci/docker_build_cpp.sh && \
+ ninja -C /build/cpp test
diff --git a/cpp/build-support/iwyu/iwyu.sh b/cpp/build-support/iwyu/iwyu.sh
index 865ac8e..b970e13 100755
--- a/cpp/build-support/iwyu/iwyu.sh
+++ b/cpp/build-support/iwyu/iwyu.sh
@@ -33,12 +33,12 @@ IWYU_ARGS="--mapping_file=$IWYU_MAPPINGS_PATH/boost-all.imp
\
--mapping_file=$IWYU_MAPPINGS_PATH/gflags.imp \
--mapping_file=$IWYU_MAPPINGS_PATH/glog.imp \
--mapping_file=$IWYU_MAPPINGS_PATH/gtest.imp \
- --mapping_file=$IWYU_MAPPINGS_PATH/arrow-misc.imp"
+ --mapping_file=$IWYU_MAPPINGS_PATH/arrow-misc.imp"
set -e
if [ "$1" == "all" ]; then
- python $ROOT/cpp/build-support/iwyu/iwyu_tool.py -p . \
+ python $ROOT/cpp/build-support/iwyu/iwyu_tool.py -p
${IWYU_COMPILATION_DATABASE_PATH:-.} \
-- $IWYU_ARGS | awk -f $ROOT/cpp/build-support/iwyu/iwyu-filter.awk
else
# Build the list of updated files which are of IWYU interest.
@@ -56,7 +56,7 @@ else
IWYU_FILE_LIST="$IWYU_FILE_LIST $ROOT/$p"
done
- python $ROOT/cpp/build-support/iwyu/iwyu_tool.py -p . $IWYU_FILE_LIST -- \
+ python $ROOT/cpp/build-support/iwyu/iwyu_tool.py -p
${IWYU_COMPILATION_DATABASE_PATH:-.} $IWYU_FILE_LIST -- \
$IWYU_ARGS | awk -f $ROOT/cpp/build-support/iwyu/iwyu-filter.awk | \
tee $IWYU_LOG
fi
diff --git a/dev/iwyu/run_iwyu.sh b/dev/iwyu/run_iwyu.sh
deleted file mode 100755
index 9e2bd7e..0000000
--- a/dev/iwyu/run_iwyu.sh
+++ /dev/null
@@ -1,72 +0,0 @@
-#!/usr/bin/env bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Set up environment and working directory
-CLANG_VERSION=6.0
-IWYU_BUILD_DIR=/arrow/cpp/docker-iwyu
-IWYU_SH=/arrow/cpp/build-support/iwyu/iwyu.sh
-IWYU_URL=https://github.com/include-what-you-use/include-what-you-use/archive/clang_$CLANG_VERSION.tar.gz
-
-rm -rf $IWYU_BUILD_DIR
-mkdir -p $IWYU_BUILD_DIR
-pushd $IWYU_BUILD_DIR
-
-function cleanup {
- popd
- rm -rf $IWYU_BUILD_DIR
-}
-
-trap cleanup EXIT
-
-# Build IWYU
-wget -O iwyu.tar.gz $IWYU_URL
-tar xzf iwyu.tar.gz
-rm -f iwyu.tar.gz
-
-IWYU_SRC=`pwd`/include-what-you-use-clang_$CLANG_VERSION
-
-export CC=clang-$CLANG_VERSION
-export CXX=clang++-$CLANG_VERSION
-
-mkdir -p iwyu-build
-pushd iwyu-build
-
-# iwyu needs this
-apt-get install -y zlib1g-dev
-
-source activate pyarrow-dev
-
-cmake -G "Unix Makefiles" -DIWYU_LLVM_ROOT_PATH=/usr/lib/llvm-$CLANG_VERSION
$IWYU_SRC
-make -j4
-popd
-
-# Add iwyu and iwyu_tool.py to path
-export PATH=$IWYU_BUILD_DIR/iwyu-build:$PATH
-
-export ARROW_BUILD_TOOLCHAIN=$CONDA_PREFIX
-
-cmake -GNinja \
- -DARROW_PARQUET=ON \
- -DARROW_PYTHON=ON \
- -DCMAKE_CXX_FLAGS='-D_GLIBCXX_USE_CXX11_ABI=0' \
- -DCMAKE_EXPORT_COMPILE_COMMANDS=ON ..
-
-# Make so that vendored bits are built
-ninja arrow_shared
-
-$IWYU_SH all
diff --git a/dev/lint/Dockerfile b/dev/lint/Dockerfile
index 9b9d2c5..71d7ec8 100644
--- a/dev/lint/Dockerfile
+++ b/dev/lint/Dockerfile
@@ -15,34 +15,17 @@
# specific language governing permissions and limitations
# under the License.
-FROM ubuntu:18.04
+FROM arrow:python-3.6
-RUN apt-get update && \
- apt-get install -y \
- gcc \
- g++ \
- git \
- wget \
- pkg-config \
- ninja-build \
- clang-format \
- clang-tidy
+RUN apt-get install -y -q \
+ libclang-dev \
+ clang \
+ clang-format \
+ clang-tidy \
+ iwyu
-ENV CC=gcc \
- CXX=g++ \
- PATH=/opt/conda/bin:$PATH \
- CONDA_PREFIX=/opt/conda
-
-ADD ci/docker_install_conda.sh \
- ci/conda_env_cpp.yml \
- /arrow/ci/
-RUN arrow/ci/docker_install_conda.sh && \
- conda install -c conda-forge \
- --file arrow/ci/conda_env_cpp.yml flake8 && \
+RUN conda install -c conda-forge flake8 && \
conda clean --all -y
-ADD dev/lint/run_checks.sh /run_checks.sh
-ADD dev/lint/run_clang_format.sh /run_clang_format.sh
-
-WORKDIR /tmp
-CMD /run_checks.sh
+# https://bugs.launchpad.net/ubuntu/+source/iwyu/+bug/1769334
+RUN ln -sv /usr/lib/clang/6.0 /usr/lib/clang/5.0.1
diff --git a/dev/lint/run_clang_format.sh b/dev/lint/run_clang_format.sh
index 2f70836..1f0f28b 100755
--- a/dev/lint/run_clang_format.sh
+++ b/dev/lint/run_clang_format.sh
@@ -18,8 +18,8 @@
set -ex
-mkdir build_cpp
-pushd build_cpp
-cmake -GNinja /arrow/cpp
-ninja format
+mkdir -p /build/lint
+pushd /build/lint
+ cmake -GNinja /arrow/cpp
+ ninja format
popd
diff --git a/dev/iwyu/Dockerfile b/dev/lint/run_iwyu.sh
old mode 100644
new mode 100755
similarity index 63%
rename from dev/iwyu/Dockerfile
rename to dev/lint/run_iwyu.sh
index 8174720..3f39357
--- a/dev/iwyu/Dockerfile
+++ b/dev/lint/run_iwyu.sh
@@ -1,3 +1,4 @@
+#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
@@ -15,9 +16,21 @@
# limitations under the License.
#
-FROM arrow_integration_xenial_base
+export ARROW_BUILD_TOOLCHAIN=$CONDA_PREFIX
-ADD dev/iwyu/run_iwyu.sh /run_iwyu.sh
+mkdir -p /build/lint
+pushd /build/lint
-WORKDIR /tmp
-CMD /run_iwyu.sh
\ No newline at end of file
+cmake -GNinja \
+ -DARROW_PARQUET=ON \
+ -DARROW_PYTHON=ON \
+ -DCMAKE_CXX_FLAGS='-D_GLIBCXX_USE_CXX11_ABI=0' \
+ -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
+ /arrow/cpp
+# Make so that vendored bits are built
+ninja arrow_shared
+
+popd
+
+export IWYU_COMPILATION_DATABASE_PATH=/build/lint
+/arrow/cpp/build-support/iwyu/iwyu.sh all
diff --git a/dev/lint/run_checks.sh b/dev/lint/run_linters.sh
similarity index 82%
rename from dev/lint/run_checks.sh
rename to dev/lint/run_linters.sh
index 6e7a63d..1b549aa 100755
--- a/dev/lint/run_checks.sh
+++ b/dev/lint/run_linters.sh
@@ -18,14 +18,14 @@
set -ex
-mkdir build_cpp
-pushd build_cpp
-cmake -GNinja /arrow/cpp
-ninja check-format
-ninja lint
+mkdir -p /build/lint
+pushd /build/lint
+ cmake -GNinja /arrow/cpp
+ ninja check-format
+ ninja lint
popd
pushd /arrow/python
-flake8 --count pyarrow
-flake8 --count --config=.flake8.cython pyarrow
+ flake8 --count pyarrow
+ flake8 --count --config=.flake8.cython pyarrow
popd
diff --git a/docker-compose.yml b/docker-compose.yml
index 62c3003..9ee2b44 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -15,33 +15,54 @@
# specific language governing permissions and limitations
# under the License.
+# Development containers for Arrow
+
version: '3.5'
-services:
- # we can further improve the caching mechanism for go rust and js via
- # early adding the dependencies explicitly (cargo.toml etc) to prevent
- # reinstalling the dependencies on each modification
+x-volumes:
+ &volumes
+ - .:/arrow:delegated
+ - ${ARROW_DOCKER_CACHE_DIR:-./docker_cache}:/build:delegated
+
+services:
######################### Language Containers ###############################
+ #TODO(kszucs): R
+
c_glib:
+ # Usage:
+ # docker-compose build cpp
+ # docker-compose build c_glib
+ # docker-compose run c_glib
image: arrow:c_glib
build:
context: .
dockerfile: c_glib/Dockerfile
+ volumes: *volumes
cpp:
+ # Usage:
+ # docker-compose build cpp
+ # docker-compose run cpp
image: arrow:cpp
shm_size: 2G
build:
context: .
dockerfile: cpp/Dockerfile
+ environment:
+ PARQUET_TEST_DATA: /arrow/cpp/submodules/parquet-testing/data
+ volumes: *volumes
go:
+ # Usage:
+ # docker-compose build go
+ # docker-compose run go
image: arrow:go
build:
context: .
dockerfile: go/Dockerfile
+ volumes: *volumes
java:
image: arrow:java
@@ -56,6 +77,11 @@ services:
dockerfile: js/Dockerfile
python:
+ # Usage:
+ # export PYTHON_VERSION=3.6
+ # docker-compose build cpp
+ # docker-compose build python
+ # docker-compose run python
image: arrow:python-${PYTHON_VERSION:-3.6}
shm_size: 2G
build:
@@ -63,14 +89,52 @@ services:
dockerfile: python/Dockerfile
args:
PYTHON_VERSION: ${PYTHON_VERSION:-3.6}
-
- #TODO(kszucs): R
+ volumes: *volumes
rust:
+ # Usage:
+ # docker-compose build rust
+ # docker-compose run rust
image: arrow:rust
build:
context: .
dockerfile: rust/Dockerfile
+ volumes: *volumes
+
+ ######################### Tools and Linters #################################
+
+ # TODO(kszucs): site
+ # TODO(kszucs): apidoc
+
+ lint:
+ # Usage:
+ # docker-compose build lint
+ # docker-compose run lint
+ image: arrow:lint
+ build:
+ context: .
+ dockerfile: dev/lint/Dockerfile
+ command: arrow/dev/lint/run_linters.sh
+ volumes: *volumes
+
+ iwyu:
+ # Usage:
+ # docker-compose build lint
+ # docker-compose run iwyu
+ image: arrow:lint
+ environment:
+ CC: clang
+ CXX: clang++
+ command: arrow/dev/lint/run_iwyu.sh
+ volumes: *volumes
+
+ clang-format:
+ # Usage:
+ # docker-compose build lint
+ # docker-compose run clang-format
+ image: arrow:lint
+ command: arrow/dev/lint/run_clang_format.sh
+ volumes: *volumes
######################### Integration Tests #################################
@@ -100,6 +164,12 @@ services:
- hdfs-namenode:hdfs-namenode
hdfs-integration:
+ # Usage:
+ # export PYTHON_VERSION=3.6
+ # docker-compose build cpp
+ # docker-compose build python
+ # docker-compose build hdfs-integration
+ # docker-compose run hdfs-integration
links:
- hdfs-namenode:hdfs-namenode
- hdfs-datanode:hdfs-datanode
@@ -111,39 +181,6 @@ services:
context: .
dockerfile: integration/hdfs/Dockerfile
- lint:
- image: arrow:lint
- shm_size: 2G
- build:
- context: .
- dockerfile: dev/lint/Dockerfile
- volumes:
- - .:/arrow:delegated
-
- # Depends on xenial base image, see dev/README.md
- iwyu:
- image: arrow:iwyu
- build:
- context: .
- dockerfile: dev/iwyu/Dockerfile
- volumes:
- - .:/arrow:delegated
-
- clang-format:
- image: arrow:lint
- shm_size: 2G
- command: '/run_clang_format.sh'
- build:
- context: .
- dockerfile: dev/lint/Dockerfile
- volumes:
- - .:/arrow:delegated
-
# TODO(kszucs): dask-integration
# TODO(kszucs): hive-integration
# TODO(kszucs): spark-integration
-
- ######################### Documentation #####################################
-
- # TODO(kszucs): site
- # TODO(kszucs): apidoc
diff --git a/go/Dockerfile b/go/Dockerfile
index 860f7d6..8261e1f 100644
--- a/go/Dockerfile
+++ b/go/Dockerfile
@@ -17,9 +17,12 @@
FROM golang
-ADD go /arrow/go
+ADD go/arrow/Gopkg.lock \
+ go/arrow/Gopkg.toml \
+ /arrow/go/arrow/
WORKDIR /arrow/go/arrow
-RUN go get -d -t -v ./... && \
- go install -v ./...
-CMD go test
+RUN go get -d -t -v ./...
+
+CMD go install -v ./... && \
+ for d in $(go list ./... | grep -v vendor); do go test $d; done
diff --git a/integration/hdfs/Dockerfile b/integration/hdfs/Dockerfile
index 87d4e31..a1d3e4e 100644
--- a/integration/hdfs/Dockerfile
+++ b/integration/hdfs/Dockerfile
@@ -19,6 +19,9 @@ FROM gelog/hadoop
RUN apt-get update && \
apt-get install -y \
+ autoconf \
+ automake \
+ make \
gcc \
g++ \
git \
diff --git a/python/Dockerfile b/python/Dockerfile
index f143cca..41d0441 100644
--- a/python/Dockerfile
+++ b/python/Dockerfile
@@ -15,48 +15,21 @@
# specific language governing permissions and limitations
# under the License.
-FROM ubuntu:18.04
+FROM arrow:cpp
-RUN apt-get update && \
- apt-get install -y \
- gcc \
- g++ \
- git \
- wget \
- pkg-config \
- ninja-build
-
-ENV CC=gcc \
- CXX=g++ \
- PATH=/opt/conda/bin:$PATH \
- CONDA_PREFIX=/opt/conda
-
-# install dependencies
+# install python specific packages
ARG PYTHON_VERSION=3.6
-ADD ci/docker_install_conda.sh \
- ci/conda_env_cpp.yml \
- ci/conda_env_python.yml \
- /arrow/ci/
-RUN arrow/ci/docker_install_conda.sh && \
- conda install -c conda-forge \
- --file arrow/ci/conda_env_cpp.yml \
- --file arrow/ci/conda_env_python.yml \
+ADD ci/conda_env_python.yml /arrow/ci/
+RUN conda install -c conda-forge \
+ --file arrow/ci/conda_env_python.yml && \
python=$PYTHON_VERSION && \
conda clean --all
-# build cpp without tests
-ENV ARROW_PYTHON=ON \
- ARROW_BUILD_TESTS=OFF
-ADD ci/docker_build_cpp.sh /arrow/ci/
-ADD cpp /arrow/cpp
-ADD format /arrow/format
-ADD java/pom.xml /arrow/java/pom.xml
-RUN arrow/ci/docker_build_cpp.sh
-
-# build python
-ADD ci/docker_build_python.sh /arrow/ci/
-ADD python /arrow/python
-RUN arrow/ci/docker_build_python.sh
+ENV CC=gcc \
+ CXX=g++ \
+ ARROW_PYTHON=ON
-WORKDIR arrow/python
-CMD pytest -v pyarrow
+# build and test
+CMD arrow/ci/docker_build_cpp.sh && \
+ arrow/ci/docker_build_python.sh && \
+ pytest -v --pyargs pyarrow
diff --git a/rust/Dockerfile b/rust/Dockerfile
index 6ad57c3..c63dcda 100644
--- a/rust/Dockerfile
+++ b/rust/Dockerfile
@@ -17,8 +17,10 @@
FROM rust
-ADD rust /arrow/rust
+# sadly cargo doesn't have a command to fetch and build the
+# dependencies without building the library itself
+ENV CARGO_TARGET_DIR=/build/rust
+ADD rust/Cargo.toml /arrow/rust/Cargo.toml
WORKDIR /arrow/rust
-RUN cargo build
CMD cargo test
