This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-ballista.git
The following commit(s) were added to refs/heads/main by this push:
new fdd3e94a feat: publish docker containers for executor and scheduler
(#1200)
fdd3e94a is described below
commit fdd3e94ac523c079988c22779b94e6e0874391ee
Author: Marko Milenković <[email protected]>
AuthorDate: Mon Mar 17 21:02:11 2025 +0000
feat: publish docker containers for executor and scheduler (#1200)
* feat: push more docker images
* update container versions
* docker tag uses version rather than git tag
* tag docker as latest version
* remove netcat package installation ...
as it cant be find in ubuntu:24:04
* tag images based on git tag
* minor script cleanup
---
.github/workflows/docker.yml | 40 ++++++++++++++++++++++++++++---
.github/workflows/rust.yml | 30 -----------------------
dev/build-ballista-docker.sh | 13 ++++------
dev/docker/ballista-benchmarks.Dockerfile | 2 +-
dev/docker/ballista-builder.Dockerfile | 4 ++--
dev/docker/ballista-cli.Dockerfile | 2 +-
dev/docker/ballista-executor.Dockerfile | 4 +---
dev/docker/ballista-scheduler.Dockerfile | 4 +---
dev/docker/ballista-standalone.Dockerfile | 4 ++--
dev/docker/builder-entrypoint.sh | 2 +-
10 files changed, 51 insertions(+), 54 deletions(-)
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index ffeb9fa5..d3947baa 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -16,11 +16,11 @@
# under the License.
name: Docker
-on: [pull_request]
+on: [pull_request, push]
jobs:
build_docker:
- name: Run build Docker script
+ name: Run Build Docker Script
runs-on: ubuntu-latest
steps:
- name: Checkout
@@ -28,4 +28,38 @@ jobs:
- name: Installs Rust and Cargo
run: curl -y --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
- name: Run script
- run: ./dev/build-ballista-docker.sh
+ run: |
+
+ ./dev/build-ballista-docker.sh
+
+ docker tag apache/arrow-ballista-standalone:latest
ghcr.io/apache/arrow-ballista-standalone:latest
+ docker tag apache/arrow-ballista-executor:latest
ghcr.io/apache/arrow-ballista-executor:latest
+ docker tag apache/arrow-ballista-scheduler:latest
ghcr.io/apache/arrow-ballista-scheduler:latest
+
+ # release dockers only when there is a tag
+ export DOCKER_TAG="$(git describe --exact-match --tags $(git log -n1
--pretty='%h') || echo '')"
+ if [[ $DOCKER_TAG =~ ^[0-9\.]+-rc[0-9]+$ ]]
+ then
+
+ docker login ghcr.io -u $DOCKER_USER -p "$DOCKER_PASS"
+
+ echo "publishing docker tag $DOCKER_TAG"
+
+ docker tag apache/arrow-ballista-standalone:latest
ghcr.io/apache/arrow-ballista-standalone:$DOCKER_TAG
+ docker tag apache/arrow-ballista-executor:latest
ghcr.io/apache/arrow-ballista-executor:$DOCKER_TAG
+ docker tag apache/arrow-ballista-scheduler:latest
ghcr.io/apache/arrow-ballista-scheduler:$DOCKER_TAG
+
+ docker push ghcr.io/apache/arrow-ballista-standalone:$DOCKER_TAG
+ docker push ghcr.io/apache/arrow-ballista-executor:$DOCKER_TAG
+ docker push ghcr.io/apache/arrow-ballista-scheduler:$DOCKER_TAG
+
+ echo "publishing docker tag latest"
+
+ docker push ghcr.io/apache/arrow-ballista-standalone:latest
+ docker push ghcr.io/apache/arrow-ballista-executor:latest
+ docker push ghcr.io/apache/arrow-ballista-scheduler:latest
+
+ fi
+ env:
+ DOCKER_USER: ${{ github.actor }}
+ DOCKER_PASS: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 8a20601a..68ad936a 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -294,36 +294,6 @@ jobs:
CARGO_HOME: "/github/home/.cargo"
CARGO_TARGET_DIR: "/github/home/target"
- docker:
- name: Docker
- needs: [linux-build-lib]
- runs-on: ubuntu-latest
- permissions:
- contents: read
- packages: write
- steps:
- - uses: actions/checkout@v4
- - name: Restore rust artifacts
- uses: actions/download-artifact@v4
- with:
- name: rust-artifacts
- path: target/release
- - name: Build and push Docker image
- run: |
- echo "github user is $DOCKER_USER"
- docker build -t arrow-ballista-standalone:latest -f
dev/docker/ballista-standalone.Dockerfile .
- export DOCKER_TAG="$(git describe --exact-match --tags $(git log -n1
--pretty='%h') || echo '')"
- if [[ $DOCKER_TAG =~ ^[0-9\.]+-rc[0-9]+$ ]]
- then
- echo "publishing docker tag $DOCKER_TAG"
- docker tag arrow-ballista-standalone:latest
ghcr.io/apache/arrow-ballista-standalone:$DOCKER_TAG
- docker login ghcr.io -u $DOCKER_USER -p "$DOCKER_PASS"
- docker push ghcr.io/apache/arrow-ballista-standalone:$DOCKER_TAG
- fi
- env:
- DOCKER_USER: ${{ github.actor }}
- DOCKER_PASS: ${{ secrets.GITHUB_TOKEN }}
-
clippy:
name: Clippy
needs: [linux-build-lib]
diff --git a/dev/build-ballista-docker.sh b/dev/build-ballista-docker.sh
index d7f6e69b..e21838d5 100755
--- a/dev/build-ballista-docker.sh
+++ b/dev/build-ballista-docker.sh
@@ -23,13 +23,10 @@ RELEASE_FLAG=${RELEASE_FLAG:=release}
./dev/build-ballista-executables.sh
-docker compose build
-
. ./dev/build-set-env.sh
-docker build -t "apache/arrow-ballista-standalone:$BALLISTA_VERSION" -f
dev/docker/ballista-standalone.Dockerfile .
-
-docker tag ballista-executor "apache/arrow-ballista-executor:$BALLISTA_VERSION"
-docker tag ballista-scheduler
"apache/arrow-ballista-scheduler:$BALLISTA_VERSION"
-docker tag ballista-benchmarks
"apache/arrow-ballista-benchmarks:$BALLISTA_VERSION"
-docker build -t "apache/arrow-ballista-cli:$BALLISTA_VERSION" -f
dev/docker/ballista-cli.Dockerfile .
+docker build -t "apache/arrow-ballista-standalone:latest" -f
dev/docker/ballista-standalone.Dockerfile .
+docker build -t "apache/arrow-ballista-scheduler:latest" -f
dev/docker/ballista-scheduler.Dockerfile .
+docker build -t "apache/arrow-ballista-executor:latest" -f
dev/docker/ballista-executor.Dockerfile .
+docker build -t "apache/arrow-ballista-cli:latest" -f
dev/docker/ballista-cli.Dockerfile .
+docker build -t "apache/arrow-ballista-benchmarks:latest" -f
dev/docker/ballista-benchmarks.Dockerfile .
diff --git a/dev/docker/ballista-benchmarks.Dockerfile
b/dev/docker/ballista-benchmarks.Dockerfile
index f9ab3042..cf764a87 100644
--- a/dev/docker/ballista-benchmarks.Dockerfile
+++ b/dev/docker/ballista-benchmarks.Dockerfile
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-FROM ubuntu:22.04
+FROM ubuntu:24.04
ARG RELEASE_FLAG=release
diff --git a/dev/docker/ballista-builder.Dockerfile
b/dev/docker/ballista-builder.Dockerfile
index aa9a6fa5..2347f1cd 100644
--- a/dev/docker/ballista-builder.Dockerfile
+++ b/dev/docker/ballista-builder.Dockerfile
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-FROM rust:1.81-bullseye
+FROM rust:1.85-bullseye
ARG EXT_UID
@@ -24,7 +24,7 @@ ENV RUST_BACKTRACE=full
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && \
- apt-get -y install libssl-dev openssl zlib1g zlib1g-dev libpq-dev cmake
protobuf-compiler netcat curl unzip
+ apt-get -y install libssl-dev openssl zlib1g zlib1g-dev libpq-dev cmake
protobuf-compiler curl unzip
RUN curl -fsSL https://deb.nodesource.com/setup_18.x | bash - && \
apt-get update && \
diff --git a/dev/docker/ballista-cli.Dockerfile
b/dev/docker/ballista-cli.Dockerfile
index 796a91f4..a9de7279 100644
--- a/dev/docker/ballista-cli.Dockerfile
+++ b/dev/docker/ballista-cli.Dockerfile
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-FROM ubuntu:22.04
+FROM ubuntu:24.04
ARG RELEASE_FLAG=release
diff --git a/dev/docker/ballista-executor.Dockerfile
b/dev/docker/ballista-executor.Dockerfile
index 33931df6..4408b3d5 100644
--- a/dev/docker/ballista-executor.Dockerfile
+++ b/dev/docker/ballista-executor.Dockerfile
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-FROM ubuntu:22.04
+FROM ubuntu:24.04
ARG RELEASE_FLAG=release
@@ -23,8 +23,6 @@ ENV RELEASE_FLAG=${RELEASE_FLAG}
ENV RUST_LOG=info
ENV RUST_BACKTRACE=full
-RUN apt-get update && apt-get install -y netcat
-
COPY target/$RELEASE_FLAG/ballista-executor /root/ballista-executor
# Expose Ballista Executor gRPC port
diff --git a/dev/docker/ballista-scheduler.Dockerfile
b/dev/docker/ballista-scheduler.Dockerfile
index 121eb202..f96da9e5 100644
--- a/dev/docker/ballista-scheduler.Dockerfile
+++ b/dev/docker/ballista-scheduler.Dockerfile
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-FROM ubuntu:22.04
+FROM ubuntu:24.04
ARG RELEASE_FLAG=release
@@ -24,8 +24,6 @@ ENV RUST_LOG=info
ENV RUST_BACKTRACE=full
ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && apt-get install -y netcat
-
COPY target/$RELEASE_FLAG/ballista-scheduler /root/ballista-scheduler
# Expose Ballista Scheduler gRPC port
diff --git a/dev/docker/ballista-standalone.Dockerfile
b/dev/docker/ballista-standalone.Dockerfile
index 69ff7af1..ba0c0113 100644
--- a/dev/docker/ballista-standalone.Dockerfile
+++ b/dev/docker/ballista-standalone.Dockerfile
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-FROM ubuntu:22.04
+FROM ubuntu:24.04
LABEL
org.opencontainers.image.source="https://github.com/apache/datafusion-ballista";
LABEL org.opencontainers.image.description="Apache Arrow Ballista Distributed
SQL Query Engine"
@@ -28,7 +28,7 @@ ENV RUST_LOG=info
ENV RUST_BACKTRACE=full
ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get -qq update && apt-get install -qq -y netcat wget
+RUN apt-get -qq update && apt-get install -qq -y wget
COPY target/$RELEASE_FLAG/ballista-scheduler /root/ballista-scheduler
COPY target/$RELEASE_FLAG/ballista-executor /root/ballista-executor
diff --git a/dev/docker/builder-entrypoint.sh b/dev/docker/builder-entrypoint.sh
index 2578574e..fc462704 100755
--- a/dev/docker/builder-entrypoint.sh
+++ b/dev/docker/builder-entrypoint.sh
@@ -22,4 +22,4 @@ set -x
printenv
RELEASE_FLAG=${RELEASE_FLAG:=release}
-cargo build --features flight-sql --profile $RELEASE_FLAG "$@"
+cargo build --features rest-api --profile $RELEASE_FLAG "$@"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
