This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-ballista.git
The following commit(s) were added to refs/heads/master by this push:
new cc1b2fbc Add scheduler UI to ballista-scheduler Docker image (#238)
cc1b2fbc is described below
commit cc1b2fbcd52dcdfeb4c950fd70e22f38dfb4f9d5
Author: Andy Grove <[email protected]>
AuthorDate: Mon Sep 19 08:52:49 2022 -0600
Add scheduler UI to ballista-scheduler Docker image (#238)
---
.dockerignore | 5 ++-
ballista/ui/scheduler/src/components/Header.tsx | 4 +-
benchmarks/docker-compose.yaml | 10 ++---
dev/build-ballista-docker.sh | 7 ++-
.../ballista-benchmarks.dockerfile} | 16 ++++---
.../ballista-executor.dockerfile} | 16 ++++---
.../ballista-scheduler.dockerfile} | 31 +++++++++++---
dev/docker/ballista.dockerfile | 10 ++---
.../executor-entrypoint.sh} | 10 ++---
.../scheduler-entrypoint.sh} | 12 +++---
.../source/user-guide/deployment/docker-compose.md | 16 ++++---
docs/source/user-guide/deployment/docker.md | 50 +++++++++++++---------
docs/source/user-guide/deployment/kubernetes.md | 21 ++++++---
13 files changed, 125 insertions(+), 83 deletions(-)
diff --git a/.dockerignore b/.dockerignore
index 75fbb692..b8571a34 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -1,8 +1,9 @@
.git
-ballista/ui/*
ci
conbench
-dev
+dev/dist
+dev/release
+python
**/docs
**/target
**/CHANGELOG.md
diff --git a/ballista/ui/scheduler/src/components/Header.tsx
b/ballista/ui/scheduler/src/components/Header.tsx
index 5f802128..aae1830e 100644
--- a/ballista/ui/scheduler/src/components/Header.tsx
+++ b/ballista/ui/scheduler/src/components/Header.tsx
@@ -67,7 +67,7 @@ export const Header: React.FunctionComponent<HeaderProps> = ({
<a
rel={"noreferrer"}
target={"_blank"}
- href={"https://ballistacompute.org/docs/"}
+ href={"https://arrow.apache.org/ballista/"}
>
<Button
mr={4}
@@ -80,7 +80,7 @@ export const Header: React.FunctionComponent<HeaderProps> = ({
</a>
<a
rel="noreferrer"
- href={"https://github.com/apache/arrow-datafusion"}
+ href={"https://github.com/apache/arrow-ballista"}
target={"_blank"}
>
<Button colorScheme="blue" size="sm" rightIcon={<AiFillGithub
/>}>
diff --git a/benchmarks/docker-compose.yaml b/benchmarks/docker-compose.yaml
index 224a1e39..4e9f607b 100644
--- a/benchmarks/docker-compose.yaml
+++ b/benchmarks/docker-compose.yaml
@@ -22,8 +22,8 @@ services:
ports:
- 2379:2379
ballista-scheduler:
- image: ballista:0.8.0
- command: "/scheduler --config-backend etcd --etcd-urls etcd:2379
--bind-host 0.0.0.0 --bind-port 50050"
+ image: apache/arrow-ballista-scheduler:0.8.0
+ command: "--config-backend etcd --etcd-urls etcd:2379 --bind-host 0.0.0.0
--bind-port 50050"
environment:
- RUST_LOG=info
volumes:
@@ -31,8 +31,8 @@ services:
depends_on:
- etcd
ballista-executor:
- image: ballista:0.8.0
- command: "/executor --bind-host 0.0.0.0 --bind-port 50051 --scheduler-host
ballista-scheduler"
+ image: apache/arrow-ballista-executor:0.8.0
+ command: "--bind-host 0.0.0.0 --bind-port 50051 --scheduler-host
ballista-scheduler"
scale: 2
environment:
- RUST_LOG=ballista=debug,info
@@ -41,7 +41,7 @@ services:
depends_on:
- ballista-scheduler
ballista-client:
- image: ballista:0.8.0
+ image: apache/arrow-ballista-benchmarks:0.8.0
command: "/bin/sh" # do nothing
environment:
- RUST_LOG=info
diff --git a/dev/build-ballista-docker.sh b/dev/build-ballista-docker.sh
index 7add135d..371b147e 100755
--- a/dev/build-ballista-docker.sh
+++ b/dev/build-ballista-docker.sh
@@ -20,5 +20,8 @@
set -e
. ./dev/build-set-env.sh
-docker build -t ballista-base:$BALLISTA_VERSION -f
dev/docker/ballista-base.dockerfile .
-docker build -t ballista:$BALLISTA_VERSION -f dev/docker/ballista.dockerfile .
\ No newline at end of file
+docker build -t apache/arrow-ballista-base:$BALLISTA_VERSION -f
dev/docker/ballista-base.dockerfile .
+docker build --build-arg VERSION=$BALLISTA_VERSION -t
apache/arrow-ballista:$BALLISTA_VERSION -f dev/docker/ballista.dockerfile .
+docker build --build-arg VERSION=$BALLISTA_VERSION -t
apache/arrow-ballista-executor:$BALLISTA_VERSION -f
dev/docker/ballista-executor.dockerfile .
+docker build --build-arg VERSION=$BALLISTA_VERSION -t
apache/arrow-ballista-scheduler:$BALLISTA_VERSION -f
dev/docker/ballista-scheduler.dockerfile .
+docker build --build-arg VERSION=$BALLISTA_VERSION -t
apache/arrow-ballista-benchmarks:$BALLISTA_VERSION -f
dev/docker/ballista-benchmarks.dockerfile .
\ No newline at end of file
diff --git a/dev/build-ballista-docker.sh
b/dev/docker/ballista-benchmarks.dockerfile
old mode 100755
new mode 100644
similarity index 78%
copy from dev/build-ballista-docker.sh
copy to dev/docker/ballista-benchmarks.dockerfile
index 7add135d..10e05613
--- a/dev/build-ballista-docker.sh
+++ b/dev/docker/ballista-benchmarks.dockerfile
@@ -1,5 +1,3 @@
-#!/bin/bash
-
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -17,8 +15,14 @@
# specific language governing permissions and limitations
# under the License.
-set -e
+ARG VERSION
+FROM apache/arrow-ballista:$VERSION
+
+ADD benchmarks/run.sh /
+RUN mkdir /queries
+COPY benchmarks/queries/ /queries/
+
+ENV RUST_LOG=info
+ENV RUST_BACKTRACE=full
-. ./dev/build-set-env.sh
-docker build -t ballista-base:$BALLISTA_VERSION -f
dev/docker/ballista-base.dockerfile .
-docker build -t ballista:$BALLISTA_VERSION -f dev/docker/ballista.dockerfile .
\ No newline at end of file
+CMD ["/run.sh"]
\ No newline at end of file
diff --git a/dev/build-ballista-docker.sh
b/dev/docker/ballista-executor.dockerfile
old mode 100755
new mode 100644
similarity index 78%
copy from dev/build-ballista-docker.sh
copy to dev/docker/ballista-executor.dockerfile
index 7add135d..b4bb47bf
--- a/dev/build-ballista-docker.sh
+++ b/dev/docker/ballista-executor.dockerfile
@@ -1,5 +1,3 @@
-#!/bin/bash
-
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -17,8 +15,14 @@
# specific language governing permissions and limitations
# under the License.
-set -e
+ARG VERSION
+FROM apache/arrow-ballista:$VERSION
+
+ENV RUST_LOG=info
+ENV RUST_BACKTRACE=full
+
+# Expose Ballista Executor gRPC port
+EXPOSE 50051
-. ./dev/build-set-env.sh
-docker build -t ballista-base:$BALLISTA_VERSION -f
dev/docker/ballista-base.dockerfile .
-docker build -t ballista:$BALLISTA_VERSION -f dev/docker/ballista.dockerfile .
\ No newline at end of file
+ADD dev/docker/executor-entrypoint.sh /
+ENTRYPOINT ["/executor-entrypoint.sh"]
\ No newline at end of file
diff --git a/dev/build-ballista-docker.sh
b/dev/docker/ballista-scheduler.dockerfile
old mode 100755
new mode 100644
similarity index 58%
copy from dev/build-ballista-docker.sh
copy to dev/docker/ballista-scheduler.dockerfile
index 7add135d..ab8b45ff
--- a/dev/build-ballista-docker.sh
+++ b/dev/docker/ballista-scheduler.dockerfile
@@ -1,5 +1,3 @@
-#!/bin/bash
-
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -17,8 +15,29 @@
# specific language governing permissions and limitations
# under the License.
-set -e
+ARG VERSION
+
+# Use node image to build the scheduler UI
+FROM node:14.16.0-alpine as ui-build
+WORKDIR /app
+ENV PATH /app/node_modules/.bin:$PATH
+COPY ballista/ui/scheduler ./
+RUN yarn
+RUN yarn build
+
+FROM apache/arrow-ballista:$VERSION
+RUN apt -y install nginx
+RUN rm -rf /var/www/html/*
+COPY --from=ui-build /app/build /var/www/html
+
+ENV RUST_LOG=info
+ENV RUST_BACKTRACE=full
+
+# Expose Ballista Scheduler web UI port
+EXPOSE 80
+
+# Expose Ballista Scheduler gRPC port
+EXPOSE 50050
-. ./dev/build-set-env.sh
-docker build -t ballista-base:$BALLISTA_VERSION -f
dev/docker/ballista-base.dockerfile .
-docker build -t ballista:$BALLISTA_VERSION -f dev/docker/ballista.dockerfile .
\ No newline at end of file
+ADD dev/docker/scheduler-entrypoint.sh /
+ENTRYPOINT ["/scheduler-entrypoint.sh"]
diff --git a/dev/docker/ballista.dockerfile b/dev/docker/ballista.dockerfile
index f45e6c8e..e2c93d4a 100644
--- a/dev/docker/ballista.dockerfile
+++ b/dev/docker/ballista.dockerfile
@@ -22,9 +22,9 @@
# as a mounted directory.
ARG RELEASE_FLAG=--release
-ARG VERSION=0.7.0
+ARG VERSION
-FROM ballista-base:$VERSION AS base
+FROM apache/arrow-ballista-base:$VERSION AS base
WORKDIR /tmp/ballista
RUN apt-get -y install cmake
RUN cargo install cargo-chef --version 0.1.34
@@ -86,7 +86,7 @@ ENV RELEASE_FLAG=${RELEASE_FLAG}
RUN if [ -z "$RELEASE_FLAG" ]; then mv /tmp/ballista/target/debug/tpch /tpch;
else mv /tmp/ballista/target/release/tpch /tpch; fi
# Copy the binary into a new container for a smaller docker image
-FROM ballista-base:$VERSION
+FROM apache/arrow-ballista-base:$VERSION
COPY --from=builder /executor /
@@ -94,10 +94,6 @@ COPY --from=builder /scheduler /
COPY --from=builder /tpch /
-ADD benchmarks/run.sh /
-RUN mkdir /queries
-COPY benchmarks/queries/ /queries/
-
ENV RUST_LOG=info
ENV RUST_BACKTRACE=full
diff --git a/dev/build-ballista-docker.sh b/dev/docker/executor-entrypoint.sh
similarity index 79%
copy from dev/build-ballista-docker.sh
copy to dev/docker/executor-entrypoint.sh
index 7add135d..9a4492e7 100755
--- a/dev/build-ballista-docker.sh
+++ b/dev/docker/executor-entrypoint.sh
@@ -1,5 +1,5 @@
#!/bin/bash
-
+#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -16,9 +16,5 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
-set -e
-
-. ./dev/build-set-env.sh
-docker build -t ballista-base:$BALLISTA_VERSION -f
dev/docker/ballista-base.dockerfile .
-docker build -t ballista:$BALLISTA_VERSION -f dev/docker/ballista.dockerfile .
\ No newline at end of file
+#
+/executor "$@"
\ No newline at end of file
diff --git a/dev/build-ballista-docker.sh b/dev/docker/scheduler-entrypoint.sh
similarity index 79%
copy from dev/build-ballista-docker.sh
copy to dev/docker/scheduler-entrypoint.sh
index 7add135d..2cdc267f 100755
--- a/dev/build-ballista-docker.sh
+++ b/dev/docker/scheduler-entrypoint.sh
@@ -1,5 +1,5 @@
#!/bin/bash
-
+#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -16,9 +16,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
-set -e
-
-. ./dev/build-set-env.sh
-docker build -t ballista-base:$BALLISTA_VERSION -f
dev/docker/ballista-base.dockerfile .
-docker build -t ballista:$BALLISTA_VERSION -f dev/docker/ballista.dockerfile .
\ No newline at end of file
+#
+echo "Starting nginx to serve Ballista Scheduler web UI on port 80"
+nohup nginx -g "daemon off;" &
+/scheduler "$@"
\ No newline at end of file
diff --git a/docs/source/user-guide/deployment/docker-compose.md
b/docs/source/user-guide/deployment/docker-compose.md
index 1cd84176..1a2f374a 100644
--- a/docs/source/user-guide/deployment/docker-compose.md
+++ b/docs/source/user-guide/deployment/docker-compose.md
@@ -33,7 +33,10 @@ cd arrow-datafusion
./dev/build-ballista-docker.sh
```
-This will create an image with the tag `ballista:0.7.0`.
+This will create the following images:
+
+- `apache/arrow-ballista-scheduler:0.8.0`
+- `apache/arrow-ballista-executor:0.8.0`
## Start a cluster
@@ -48,10 +51,11 @@ services:
image: quay.io/coreos/etcd:v3.4.9
command: "etcd -advertise-client-urls http://etcd:2379 -listen-client-urls
http://0.0.0.0:2379";
ballista-scheduler:
- image: ballista:0.8.0
- command: "/scheduler --config-backend etcd --etcd-urls etcd:2379
--bind-host 0.0.0.0 --bind-port 50050"
+ image: apache/arrow-ballista-scheduler:0.8.0
+ command: "--config-backend etcd --etcd-urls etcd:2379 --bind-host 0.0.0.0
--bind-port 50050"
ports:
- "50050:50050"
+ - "80:8080"
environment:
- RUST_LOG=info
volumes:
@@ -59,8 +63,8 @@ services:
depends_on:
- etcd
ballista-executor:
- image: ballista:0.8.0
- command: "/executor --bind-host 0.0.0.0 --bind-port 50051 --scheduler-host
ballista-scheduler"
+ image: apache/arrow-ballista-executor:0.8.0
+ command: "--bind-host 0.0.0.0 --bind-port 50051 --scheduler-host
ballista-scheduler"
ports:
- "50051:50051"
environment:
@@ -95,3 +99,5 @@ ballista-executor_1 | [2021-08-28T15:55:22Z INFO
ballista_executor] Ballista
```
The scheduler listens on port 50050 and this is the port that clients will
need to connect to.
+
+The scheduler web UI is available on port 80 in the scheduler.
diff --git a/docs/source/user-guide/deployment/docker.md
b/docs/source/user-guide/deployment/docker.md
index e00acac9..c6188aae 100644
--- a/docs/source/user-guide/deployment/docker.md
+++ b/docs/source/user-guide/deployment/docker.md
@@ -31,7 +31,10 @@ cd arrow-datafusion
./dev/build-ballista-docker.sh
```
-This will create an image with the tag `ballista:0.7.0`.
+This will create the following images:
+
+- `apache/arrow-ballista-scheduler:0.8.0`
+- `apache/arrow-ballista-executor:0.8.0`
### Start a Scheduler
@@ -39,23 +42,27 @@ Start a scheduler using the following syntax:
```bash
docker run --network=host \
- -d ballista:0.7.0 \
- /scheduler --bind-port 50050
+ -d apache/arrow-ballista-scheduler:0.8.0 \
+ --bind-port 50050
```
Run `docker ps` to check that the process is running:
```
$ docker ps
-CONTAINER ID IMAGE COMMAND CREATED
STATUS PORTS NAMES
-1f3f8b5ed93a ballista:0.7.0 "/scheduler --bind-p…" 2 seconds ago Up 1
second tender_archimedes
+CONTAINER ID IMAGE COMMAND
CREATED STATUS PORTS NAMES
+8cdea4956c97 apache/arrow-ballista-scheduler:0.8.0 "/scheduler-entrypoi…"
2 seconds ago Up 1 second nervous_swirles
```
Run `docker logs CONTAINER_ID` to check the output from the process:
```
-$ docker logs 1f3f8b5ed93a
-[2021-08-28T15:45:11Z INFO ballista_scheduler] Ballista v0.7.0 Scheduler
listening on 0.0.0.0:50050
+$ docker logs 8cdea4956c97
+Starting nginx to serve Ballista Scheduler web UI on port 80
+2022-09-19T13:51:34.792363Z INFO main ThreadId(01) ballista_scheduler:
Ballista v0.8.0 Scheduler listening on 0.0.0.0:50050
+2022-09-19T13:51:34.792395Z INFO main ThreadId(01) ballista_scheduler:
Starting Scheduler grpc server with task scheduling policy of PullStaged
+2022-09-19T13:51:34.792494Z INFO main ThreadId(01)
ballista_scheduler::scheduler_server::query_stage_scheduler: Starting
QueryStageScheduler
+2022-09-19T13:51:34.792581Z INFO tokio-runtime-worker ThreadId(45)
ballista_core::event_loop: Starting the event loop query_stage
```
### Start executors
@@ -64,27 +71,28 @@ Start one or more executor processes. Each executor process
will need to listen
```bash
docker run --network=host \
- -d ballista:0.7.0 \
- /executor --external-host localhost --bind-port 50051
+ -d apache/arrow-ballista-executor:0.8.0 \
+ --external-host localhost --bind-port 50051
```
-Use `docker ps` to check that both the scheduer and executor(s) are now
running:
+Use `docker ps` to check that both the scheduler and executor(s) are now
running:
```
$ docker ps
-CONTAINER ID IMAGE COMMAND CREATED
STATUS PORTS NAMES
-7c6941bb8dc0 ballista:0.7.0 "/executor --externa…" 3 seconds ago Up 2
seconds tender_goldberg
-1f3f8b5ed93a ballista:0.7.0 "/scheduler --bind-p…" 50 seconds ago Up
49 seconds tender_archimedes
+CONTAINER ID IMAGE COMMAND
CREATED STATUS PORTS NAMES
+f0b21f6b5050 apache/arrow-ballista-executor:0.8.0 "/executor-entrypoin…"
2 seconds ago Up 1 second relaxed_goldberg
+8cdea4956c97 apache/arrow-ballista-scheduler:0.8.0 "/scheduler-entrypoi…"
2 minutes ago Up 2 minutes nervous_swirles
```
Use `docker logs CONTAINER_ID` to check the output from the executor(s):
```
-$ docker logs 7c6941bb8dc0
-[2021-08-28T15:45:58Z INFO ballista_executor] Running with config:
-[2021-08-28T15:45:58Z INFO ballista_executor] work_dir: /tmp/.tmpeyEM76
-[2021-08-28T15:45:58Z INFO ballista_executor] concurrent_tasks: 4
-[2021-08-28T15:45:58Z INFO ballista_executor] Ballista v0.7.0 Rust Executor
listening on 0.0.0.0:50051
+$ docker logs f0b21f6b5050
+2022-09-19T13:54:10.806231Z INFO main ThreadId(01) ballista_executor: Running
with config:
+2022-09-19T13:54:10.806261Z INFO main ThreadId(01) ballista_executor:
work_dir: /tmp/.tmp5BdxT2
+2022-09-19T13:54:10.806265Z INFO main ThreadId(01) ballista_executor:
concurrent_tasks: 48
+2022-09-19T13:54:10.807454Z INFO tokio-runtime-worker ThreadId(49)
ballista_executor: Ballista v0.8.0 Rust Executor Flight Server listening on
0.0.0.0:50051
+2022-09-19T13:54:10.807467Z INFO tokio-runtime-worker ThreadId(46)
ballista_executor::execution_loop: Starting poll work loop with scheduler
```
### Using etcd as backing store
@@ -96,11 +104,11 @@ to launch the scheduler with this option enabled.
```bash
docker run --network=host \
- -d ballista:0.7.0 \
- /scheduler --bind-port 50050 \
+ -d apache/arrow-ballista-scheduler:0.8.0 \
+ --bind-port 50050 \
--config-backend etcd \
--etcd-urls etcd:2379
```
-Please refer to the [etcd](https://etcd.io/) web site for installation
instructions. Etcd version 3.4.9 or later is
+Please refer to the [etcd](https://etcd.io/) website for installation
instructions. Etcd version 3.4.9 or later is
recommended.
diff --git a/docs/source/user-guide/deployment/kubernetes.md
b/docs/source/user-guide/deployment/kubernetes.md
index 6886b879..f77336fc 100644
--- a/docs/source/user-guide/deployment/kubernetes.md
+++ b/docs/source/user-guide/deployment/kubernetes.md
@@ -60,15 +60,20 @@ cd arrow-ballista
./dev/build-ballista-docker.sh
```
-This will create an image with the tag `ballista:0.7.0`.
+This will create the following images:
+
+- `apache/arrow-ballista-scheduler:0.8.0`
+- `apache/arrow-ballista-executor:0.8.0`
## Publishing your images
Once the images have been built, you can retag them and can push them to your
favourite docker registry.
```bash
-docker tag ballista:0.7.0 <your-repo>/ballista:0.7.0
-docker push <your-repo>/ballista:0.7.0
+docker tag apache/arrow-ballista-scheduler:0.8.0
<your-repo>/arrow-ballista-scheduler:0.8.0
+docker tag apache/arrow-ballista-executor:0.8.0
<your-repo>/arrow-ballista-executor:0.8.0
+docker push <your-repo>/arrow-ballista-scheduler:0.8.0
+docker push <your-repo>/arrow-ballista-executor:0.8.0
```
## Create Persistent Volume and Persistent Volume Claim
@@ -134,6 +139,8 @@ spec:
ports:
- port: 50050
name: scheduler
+ - port: 80
+ name: scheduler-ui
selector:
app: ballista-scheduler
---
@@ -154,12 +161,13 @@ spec:
spec:
containers:
- name: ballista-scheduler
- image: <your-repo>/ballista:0.7.0
- command: ["/scheduler"]
+ image: <your-repo>/arrow-ballista-scheduler:0.8.0
args: ["--bind-port=50050"]
ports:
- containerPort: 50050
name: flight
+ - containerPort: 80
+ name: webui
volumeMounts:
- mountPath: /mnt
name: data
@@ -185,8 +193,7 @@ spec:
spec:
containers:
- name: ballista-executor
- image: <your-repo>/ballista:0.7.0
- command: ["/executor"]
+ image: <your-repo>/arrow-ballista-executor:0.8.0
args:
- "--bind-port=50051"
- "--scheduler-host=ballista-scheduler"
