This is an automated email from the ASF dual-hosted git repository.
jorgecarleitao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
The following commit(s) were added to refs/heads/master by this push:
new 8345b5e [Ballista] Fix Ballista IT (#5)
8345b5e is described below
commit 8345b5e33ae3bae05cf0a998c183a31fd6c2fa71
Author: Andy Grove <[email protected]>
AuthorDate: Mon Apr 19 22:44:10 2021 -0600
[Ballista] Fix Ballista IT (#5)
---
.dockerignore | 42 ++-------------------------
ballista/.dockerignore | 18 ------------
ballista/docker/README.md | 29 ------------------
{ballista/dev => dev}/build-rust-base.sh | 2 +-
{ballista/dev => dev}/build-rust.sh | 2 +-
{ballista => dev}/docker/rust-base.dockerfile | 2 +-
{ballista => dev}/docker/rust.dockerfile | 30 +++++++++++++++----
{ballista/dev => dev}/integration-tests.sh | 3 +-
8 files changed, 32 insertions(+), 96 deletions(-)
diff --git a/.dockerignore b/.dockerignore
index eb71138..9a64a12 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -21,42 +21,6 @@
# This setup requires to all of our docker containers have arrow's source
# as a mounted directory.
-# exclude everything
-**
-
-# include explicitly
-!ci/**
-!c_glib/Gemfile
-!dev/archery/requirements*.txt
-!python/requirements*.txt
-!python/manylinux1/**
-!python/manylinux2010/**
-!r/DESCRIPTION
-!ruby/Gemfile
-!ruby/red-arrow/Gemfile
-!ruby/red-arrow/lib/arrow/version.rb
-!ruby/red-arrow/red-arrow.gemspec
-!ruby/red-arrow-cuda/Gemfile
-!ruby/red-arrow-cuda/lib/arrow-cuda/version.rb
-!ruby/red-arrow-cuda/red-arrow-cuda.gemspec
-!ruby/red-gandiva/Gemfile
-!ruby/red-gandiva/lib/gandiva/version.rb
-!ruby/red-gandiva/red-gandiva.gemspec
-!ruby/red-parquet/Gemfile
-!ruby/red-parquet/lib/parquet/version.rb
-!ruby/red-parquet/red-parquet.gemspec
-!ruby/red-plasma/Gemfile
-!ruby/red-plasma/lib/plasma/version.rb
-!ruby/red-plasma/red-plasma.gemspec
-!rust/Cargo.toml
-!rust/benchmarks/Cargo.toml
-!rust/arrow/Cargo.toml
-!rust/arrow/benches
-!rust/arrow-flight/Cargo.toml
-!rust/parquet/Cargo.toml
-!rust/parquet/build.rs
-!rust/parquet_derive/Cargo.toml
-!rust/parquet_derive_test/Cargo.toml
-!rust/datafusion/Cargo.toml
-!rust/datafusion/benches
-!rust/integration-testing/Cargo.toml
+ci
+dev
+**/target/*
diff --git a/ballista/.dockerignore b/ballista/.dockerignore
deleted file mode 100644
index 3cde49e..0000000
--- a/ballista/.dockerignore
+++ /dev/null
@@ -1,18 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-rust/**/target
diff --git a/ballista/docker/README.md b/ballista/docker/README.md
deleted file mode 100644
index 8417d04..0000000
--- a/ballista/docker/README.md
+++ /dev/null
@@ -1,29 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-
-# Ballista Docker Images
-
-Pre-built docker images are available from [Docker
Hub](https://hub.docker.com/orgs/ballistacompute/repositories) but here are the
commands to build the images from source.
-
-Run these commands from the root directory of the project.
-
-```bash
-./dev/build-all.sh
-```
-
diff --git a/ballista/dev/build-rust-base.sh b/dev/build-rust-base.sh
similarity index 96%
rename from ballista/dev/build-rust-base.sh
rename to dev/build-rust-base.sh
index ee4b32c..e424909 100755
--- a/ballista/dev/build-rust-base.sh
+++ b/dev/build-rust-base.sh
@@ -18,4 +18,4 @@
# under the License.
BALLISTA_VERSION=0.4.2-SNAPSHOT
set -e
-docker build -t ballistacompute/rust-base:$BALLISTA_VERSION -f
docker/rust-base.dockerfile .
+docker build -t ballistacompute/rust-base:$BALLISTA_VERSION -f
dev/docker/rust-base.dockerfile .
diff --git a/ballista/dev/build-rust.sh b/dev/build-rust.sh
similarity index 96%
rename from ballista/dev/build-rust.sh
rename to dev/build-rust.sh
index 1916f8e..d31c524 100755
--- a/ballista/dev/build-rust.sh
+++ b/dev/build-rust.sh
@@ -21,4 +21,4 @@ BALLISTA_VERSION=0.4.2-SNAPSHOT
set -e
-docker build -t ballistacompute/ballista-rust:$BALLISTA_VERSION -f
docker/rust.dockerfile .
+docker build -t ballistacompute/ballista-rust:$BALLISTA_VERSION -f
dev/docker/rust.dockerfile .
diff --git a/ballista/docker/rust-base.dockerfile
b/dev/docker/rust-base.dockerfile
similarity index 98%
rename from ballista/docker/rust-base.dockerfile
rename to dev/docker/rust-base.dockerfile
index 4519225..31620b3 100644
--- a/ballista/docker/rust-base.dockerfile
+++ b/dev/docker/rust-base.dockerfile
@@ -23,7 +23,7 @@
# Base image extends debian:buster-slim
-FROM rust:1.49.0-buster AS builder
+FROM rust:1.51.0-buster AS builder
RUN apt update && apt -y install musl musl-dev musl-tools libssl-dev openssl
diff --git a/ballista/docker/rust.dockerfile b/dev/docker/rust.dockerfile
similarity index 64%
rename from ballista/docker/rust.dockerfile
rename to dev/docker/rust.dockerfile
index 8b06af3..19dd487 100644
--- a/ballista/docker/rust.dockerfile
+++ b/dev/docker/rust.dockerfile
@@ -22,13 +22,21 @@
# as a mounted directory.
ARG RELEASE_FLAG=--release
-FROM ballistacompute/rust-base:0.4.0-20210213 AS base
+FROM ballistacompute/rust-base:0.4.2-SNAPSHOT AS base
WORKDIR /tmp/ballista
RUN apt-get -y install cmake
RUN cargo install cargo-chef
FROM base as planner
-COPY rust .
+RUN mkdir /tmp/ballista/ballista
+RUN mkdir /tmp/ballista/benchmarks
+RUN mkdir /tmp/ballista/datafusion
+RUN mkdir /tmp/ballista/datafusion-examples
+ADD Cargo.toml .
+COPY benchmarks ./benchmarks/
+COPY datafusion ./datafusion/
+COPY datafusion-examples ./datafusion-examples/
+COPY ballista ./ballista/
RUN cargo chef prepare --recipe-path recipe.json
FROM base as cacher
@@ -36,25 +44,35 @@ COPY --from=planner /tmp/ballista/recipe.json recipe.json
RUN cargo chef cook $RELEASE_FLAG --recipe-path recipe.json
FROM base as builder
-COPY rust .
+RUN mkdir /tmp/ballista/ballista
+RUN mkdir /tmp/ballista/benchmarks
+RUN mkdir /tmp/ballista/datafusion
+RUN mkdir /tmp/ballista/datafusion-examples
+ADD Cargo.toml .
+COPY benchmarks ./benchmarks/
+COPY datafusion ./datafusion/
+COPY ballista ./ballista/
+COPY datafusion-examples ./datafusion-examples/
COPY --from=cacher /tmp/ballista/target target
ARG RELEASE_FLAG=--release
# force build.rs to run to generate configure_me code.
ENV FORCE_REBUILD='true'
RUN cargo build $RELEASE_FLAG
+RUN cd ballista/rust && \
+ cargo build $RELEASE_FLAG
# put the executor on /executor (need to be copied from different places
depending on FLAG)
ENV RELEASE_FLAG=${RELEASE_FLAG}
-RUN if [ -z "$RELEASE_FLAG" ]; then mv
/tmp/ballista/target/debug/ballista-executor /executor; else mv
/tmp/ballista/target/release/ballista-executor /executor; fi
+RUN if [ -z "$RELEASE_FLAG" ]; then mv
/tmp/ballista/ballista/rust/target/debug/ballista-executor /executor; else mv
/tmp/ballista/ballista/rust/target/release/ballista-executor /executor; fi
# put the scheduler on /scheduler (need to be copied from different places
depending on FLAG)
ENV RELEASE_FLAG=${RELEASE_FLAG}
-RUN if [ -z "$RELEASE_FLAG" ]; then mv
/tmp/ballista/target/debug/ballista-scheduler /scheduler; else mv
/tmp/ballista/target/release/ballista-scheduler /scheduler; fi
+RUN if [ -z "$RELEASE_FLAG" ]; then mv
/tmp/ballista/ballista/rust/target/debug/ballista-scheduler /scheduler; else mv
/tmp/ballista/ballista/rust/target/release/ballista-scheduler /scheduler; fi
# put the tpch on /tpch (need to be copied from different places depending on
FLAG)
ENV RELEASE_FLAG=${RELEASE_FLAG}
-RUN if [ -z "$RELEASE_FLAG" ]; then mv /tmp/ballista/target/debug/tpch /tpch;
else mv /tmp/ballista/target/release/tpch /tpch; fi
+RUN if [ -z "$RELEASE_FLAG" ]; then mv
/tmp/ballista/ballista/rust/target/debug/tpch /tpch; else mv
/tmp/ballista/ballista/rust/target/release/tpch /tpch; fi
# Copy the binary into a new container for a smaller docker image
FROM ballistacompute/rust-base:0.4.0-20210213
diff --git a/ballista/dev/integration-tests.sh b/dev/integration-tests.sh
similarity index 93%
rename from ballista/dev/integration-tests.sh
rename to dev/integration-tests.sh
index cc34a5c..6ed764e 100755
--- a/ballista/dev/integration-tests.sh
+++ b/dev/integration-tests.sh
@@ -17,8 +17,9 @@
# specific language governing permissions and limitations
# under the License.
set -e
+./dev/build-rust-base.sh
./dev/build-rust.sh
-pushd rust/benchmarks/tpch
+pushd ballista/rust/benchmarks/tpch
./tpch-gen.sh
docker-compose up -d
