This is an automated email from the ASF dual-hosted git repository.
craigrueda pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/superset.git
The following commit(s) were added to refs/heads/master by this push:
new e1a91e2228 feat(docker): Use docker buildx and Add ARM builds for
dockerize and websocket (#25377)
e1a91e2228 is described below
commit e1a91e22285df9ed6077edf76f7c2e253c3fd997
Author: Aleksey Karpov <[email protected]>
AuthorDate: Mon Sep 25 23:08:21 2023 +0300
feat(docker): Use docker buildx and Add ARM builds for dockerize and
websocket (#25377)
---
.github/workflows/docker-release.yml | 4 ++
.github/workflows/docker.yml | 14 +++++-
.github/workflows/docker_build_push.sh | 81 ++++++++++++++++++++++------------
Dockerfile | 54 +++++++++++++----------
dockerize.Dockerfile | 9 ++--
5 files changed, 104 insertions(+), 58 deletions(-)
diff --git a/.github/workflows/docker-release.yml
b/.github/workflows/docker-release.yml
index f0253dbaec..b079206f51 100644
--- a/.github/workflows/docker-release.yml
+++ b/.github/workflows/docker-release.yml
@@ -29,6 +29,10 @@ jobs:
persist-credentials: false
submodules: recursive
ref: ${{ github.ref }}
+ - name: Set up QEMU
+ uses: docker/setup-qemu-action@v1
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v1
- shell: bash
env:
DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index 43483cf5f9..983267ff60 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -35,7 +35,10 @@ jobs:
uses: actions/checkout@v3
with:
persist-credentials: false
-
+ - name: Set up QEMU
+ uses: docker/setup-qemu-action@v1
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v1
- shell: bash
env:
DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
@@ -49,7 +52,14 @@ jobs:
mkdir -p ./build
echo ${{ github.sha }} > ./build/SHA
echo ${{ github.event.pull_request.number }} > ./build/PR-NUM
- DOCKER_BUILDKIT=1 docker build --target ci -t ${{ github.sha }} -t
"pr-${{ github.event.pull_request.number }}" .
+ docker buildx build --target ci \
+ --load \
+ --cache-from=type=local,src=/tmp/superset \
+ -t ${{ github.sha }} \
+ -t "pr-${{ github.event.pull_request.number }}" \
+ --platform linux/amd64 \
+ --label "build_actor=${GITHUB_ACTOR}" \
+ .
docker save ${{ github.sha }} | gzip > ./build/${{ github.sha
}}.tar.gz
- name: Upload build artifacts
diff --git a/.github/workflows/docker_build_push.sh
b/.github/workflows/docker_build_push.sh
index 1a7575dc2c..3e90af45f1 100755
--- a/.github/workflows/docker_build_push.sh
+++ b/.github/workflows/docker_build_push.sh
@@ -43,13 +43,50 @@ cat<<EOF
- ${REPO_NAME}:${LATEST_TAG}
EOF
+if [ -z "${DOCKERHUB_TOKEN}" ]; then
+ # Skip if secrets aren't populated -- they're only visible for actions
running in the repo (not on forks)
+ echo "Skipping Docker push"
+ # By default load it back
+ DOCKER_ARGS="--load"
+ ARCHITECTURE_FOR_BUILD="linux/amd64 linux/arm64"
+else
+ # Login and push
+ docker logout
+ docker login --username "${DOCKERHUB_USER}" --password "${DOCKERHUB_TOKEN}"
+ DOCKER_ARGS="--push"
+ ARCHITECTURE_FOR_BUILD="linux/amd64,linux/arm64"
+fi
+set -x
+
+#
+# Build the dev image
+#
+docker buildx build --target dev \
+ $DOCKER_ARGS \
+ --cache-from=type=registry,ref=apache/superset:master-dev \
+ --cache-from=type=local,src=/tmp/superset \
+ --cache-to=type=local,ignore-error=true,dest=/tmp/superset \
+ -t "${REPO_NAME}:${SHA}-dev" \
+ -t "${REPO_NAME}:${REFSPEC}-dev" \
+ -t "${REPO_NAME}:${LATEST_TAG}-dev" \
+ --platform linux/amd64 \
+ --label "sha=${SHA}" \
+ --label "built_at=$(date)" \
+ --label "target=dev" \
+ --label "build_actor=${GITHUB_ACTOR}" \
+ .
+
#
# Build the "lean" image
#
-DOCKER_BUILDKIT=1 docker build --target lean \
+docker buildx build --target lean \
+ $DOCKER_ARGS \
+ --cache-from=type=local,src=/tmp/superset \
+ --cache-to=type=local,ignore-error=true,dest=/tmp/superset \
-t "${REPO_NAME}:${SHA}" \
-t "${REPO_NAME}:${REFSPEC}" \
-t "${REPO_NAME}:${LATEST_TAG}" \
+ --platform linux/amd64 \
--label "sha=${SHA}" \
--label "built_at=$(date)" \
--label "target=lean" \
@@ -59,60 +96,48 @@ DOCKER_BUILDKIT=1 docker build --target lean \
#
# Build the "lean310" image
#
-DOCKER_BUILDKIT=1 docker build --target lean \
+docker buildx build --target lean \
+ $DOCKER_ARGS \
+ --cache-from=type=local,src=/tmp/superset \
+ --cache-to=type=local,ignore-error=true,dest=/tmp/superset \
-t "${REPO_NAME}:${SHA}-py310" \
-t "${REPO_NAME}:${REFSPEC}-py310" \
-t "${REPO_NAME}:${LATEST_TAG}-py310" \
+ --platform linux/amd64 \
--build-arg PY_VER="3.10-slim-bookworm"\
--label "sha=${SHA}" \
--label "built_at=$(date)" \
--label "target=lean310" \
--label "build_actor=${GITHUB_ACTOR}" \
.
-
+for BUILD_PLATFORM in $ARCHITECTURE_FOR_BUILD; do
#
# Build the "websocket" image
#
-DOCKER_BUILDKIT=1 docker build \
+docker buildx build \
+ $DOCKER_ARGS \
+ --cache-from=type=registry,ref=apache/superset:master-websocket \
-t "${REPO_NAME}:${SHA}-websocket" \
-t "${REPO_NAME}:${REFSPEC}-websocket" \
-t "${REPO_NAME}:${LATEST_TAG}-websocket" \
+ --platform ${BUILD_PLATFORM} \
--label "sha=${SHA}" \
--label "built_at=$(date)" \
--label "target=websocket" \
--label "build_actor=${GITHUB_ACTOR}" \
superset-websocket
-#
-# Build the dev image
-#
-DOCKER_BUILDKIT=1 docker build --target dev \
- -t "${REPO_NAME}:${SHA}-dev" \
- -t "${REPO_NAME}:${REFSPEC}-dev" \
- -t "${REPO_NAME}:${LATEST_TAG}-dev" \
- --label "sha=${SHA}" \
- --label "built_at=$(date)" \
- --label "target=dev" \
- --label "build_actor=${GITHUB_ACTOR}" \
- .
-
#
# Build the dockerize image
#
-DOCKER_BUILDKIT=1 docker build \
+docker buildx build \
+ $DOCKER_ARGS \
+ --cache-from=type=registry,ref=apache/superset:dockerize \
-t "${REPO_NAME}:dockerize" \
+ --platform ${BUILD_PLATFORM} \
--label "sha=${SHA}" \
--label "built_at=$(date)" \
--label "build_actor=${GITHUB_ACTOR}" \
-f dockerize.Dockerfile \
.
-
-if [ -z "${DOCKERHUB_TOKEN}" ]; then
- # Skip if secrets aren't populated -- they're only visible for actions
running in the repo (not on forks)
- echo "Skipping Docker push"
-else
- # Login and push
- docker logout
- docker login --username "${DOCKERHUB_USER}" --password "${DOCKERHUB_TOKEN}"
- docker push --all-tags "${REPO_NAME}"
-fi
+done
diff --git a/Dockerfile b/Dockerfile
index 5d6a64dbf7..b9714d6c69 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -22,16 +22,14 @@ ARG PY_VER=3.9-slim-bookworm
# if BUILDPLATFORM is null, set it to 'amd64' (or leave as is otherwise).
ARG BUILDPLATFORM=${BUILDPLATFORM:-amd64}
-FROM --platform=${BUILDPLATFORM} node:16-slim AS superset-node
+FROM --platform=${BUILDPLATFORM} node:16-bookworm-slim AS superset-node
ARG NPM_BUILD_CMD="build"
-RUN apt-get update -q \
- && apt-get install -yq --no-install-recommends \
- python3 \
- make \
- gcc \
- g++
+RUN apt-get update -qq \
+ && apt-get install -yqq --no-install-recommends \
+ build-essential \
+ python3
ENV BUILD_CMD=${NPM_BUILD_CMD} \
PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
@@ -41,8 +39,9 @@ WORKDIR /app/superset-frontend
RUN
--mount=type=bind,target=/frontend-mem-nag.sh,src=./docker/frontend-mem-nag.sh \
/frontend-mem-nag.sh
-COPY superset-frontend/package*.json ./
-RUN npm ci
+RUN
--mount=type=bind,target=./package.json,src=./superset-frontend/package.json \
+
--mount=type=bind,target=./package-lock.json,src=./superset-frontend/package-lock.json
\
+ npm ci
COPY ./superset-frontend ./
# This seems to be the most expensive step
@@ -62,10 +61,11 @@ ENV LANG=C.UTF-8 \
SUPERSET_HOME="/app/superset_home" \
SUPERSET_PORT=8088
-RUN mkdir -p ${PYTHONPATH} superset/static superset-frontend
apache_superset.egg-info requirements \
+RUN --mount=target=/var/lib/apt/lists,type=cache \
+ --mount=target=/var/cache/apt,type=cache \
+ mkdir -p ${PYTHONPATH} superset/static superset-frontend
apache_superset.egg-info requirements \
&& useradd --user-group -d ${SUPERSET_HOME} -m --no-log-init --shell
/bin/bash superset \
- && apt-get update -q \
- && apt-get install -yq --no-install-recommends \
+ && apt-get update -qq && apt-get install -yqq --no-install-recommends \
build-essential \
curl \
default-libmysqlclient-dev \
@@ -74,27 +74,30 @@ RUN mkdir -p ${PYTHONPATH} superset/static
superset-frontend apache_superset.egg
libpq-dev \
libecpg-dev \
libldap2-dev \
- && apt-get autoremove -yqq --purge && rm -rf /var/lib/apt/lists/*
/var/[log,tmp]/* /tmp/* && apt-get clean \
&& touch superset/static/version_info.json \
&& chown -R superset:superset ./*
-COPY --chown=superset:superset ./requirements/*.txt requirements/
COPY --chown=superset:superset setup.py MANIFEST.in README.md ./
# setup.py uses the version information in package.json
COPY --chown=superset:superset superset-frontend/package.json
superset-frontend/
-RUN pip install --no-cache-dir -r requirements/local.txt
+RUN
--mount=type=bind,target=./requirements/local.txt,src=./requirements/local.txt \
+
--mount=type=bind,target=./requirements/development.txt,src=./requirements/development.txt
\
+
--mount=type=bind,target=./requirements/base.txt,src=./requirements/base.txt \
+ --mount=type=cache,target=/root/.cache/pip \
+ pip install -r requirements/local.txt
COPY --chown=superset:superset --from=superset-node
/app/superset/static/assets superset/static/assets
## Lastly, let's install superset itself
COPY --chown=superset:superset superset superset
-RUN pip install --no-cache-dir -e . \
+RUN --mount=type=cache,target=/root/.cache/pip \
+ pip install -e . \
&& flask fab babel-compile --target superset/translations \
&& chown -R superset:superset superset/translations
COPY --chmod=755 ./docker/run-server.sh /usr/bin/
USER superset
-HEALTHCHECK CMD curl -f "http://localhost:$SUPERSET_PORT/health";
+HEALTHCHECK CMD curl -f "http://localhost:${SUPERSET_PORT}/health";
EXPOSE ${SUPERSET_PORT}
@@ -109,8 +112,9 @@ ARG GECKODRIVER_VERSION=v0.33.0 \
USER root
-RUN apt-get update -q \
- && apt-get install -yq --no-install-recommends \
+RUN --mount=target=/var/lib/apt/lists,type=cache \
+ --mount=target=/var/cache/apt,type=cache \
+ apt-get install -yqq --no-install-recommends \
libnss3 \
libdbus-glib-1-2 \
libgtk-3-0 \
@@ -119,14 +123,16 @@ RUN apt-get update -q \
libxtst6 \
wget \
# Install GeckoDriver WebDriver
- && wget
https://github.com/mozilla/geckodriver/releases/download/${GECKODRIVER_VERSION}/geckodriver-${GECKODRIVER_VERSION}-linux64.tar.gz
-O - | tar xfz - -C /usr/local/bin \
+ && wget -q
https://github.com/mozilla/geckodriver/releases/download/${GECKODRIVER_VERSION}/geckodriver-${GECKODRIVER_VERSION}-linux64.tar.gz
-O - | tar xfz - -C /usr/local/bin \
# Install Firefox
- && wget
https://download-installer.cdn.mozilla.net/pub/firefox/releases/${FIREFOX_VERSION}/linux-x86_64/en-US/firefox-${FIREFOX_VERSION}.tar.bz2
-O - | tar xfj - -C /opt \
+ && wget -q
https://download-installer.cdn.mozilla.net/pub/firefox/releases/${FIREFOX_VERSION}/linux-x86_64/en-US/firefox-${FIREFOX_VERSION}.tar.bz2
-O - | tar xfj - -C /opt \
&& ln -s /opt/firefox/firefox /usr/local/bin/firefox \
- && apt-get autoremove -yqq --purge wget && rm -rf /var/lib/apt/lists/*
/var/[log,tmp]/* /tmp/* && apt-get clean
-
+ && apt-get autoremove -yqq --purge wget && rm -rf /var/[log,tmp]/* /tmp/*
# Cache everything for dev purposes...
-RUN pip install --no-cache-dir -r requirements/docker.txt
+RUN
--mount=type=bind,target=./requirements/base.txt,src=./requirements/base.txt \
+
--mount=type=bind,target=./requirements/docker.txt,src=./requirements/docker.txt
\
+ --mount=type=cache,target=/root/.cache/pip \
+ pip install -r requirements/docker.txt
USER superset
######################################################################
diff --git a/dockerize.Dockerfile b/dockerize.Dockerfile
index 4a3fa1b6f2..923295c457 100644
--- a/dockerize.Dockerfile
+++ b/dockerize.Dockerfile
@@ -4,10 +4,11 @@ ARG DOCKERIZE_VERSION=v0.7.0
RUN apk update --no-cache \
&& apk add --no-cache wget openssl \
- && wget -O -
https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz
| tar xzf - -C /usr/local/bin \
+ && case "$(apk --print-arch)" in \
+ x86_64) ARCH=amd64 ;; \
+ aarch64) ARCH=arm64 ;; \
+ esac \
+ && wget -O -
https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-${ARCH}-${DOCKERIZE_VERSION}.tar.gz
| tar xzf - -C /usr/local/bin \
&& apk del wget
USER 10001
-
-ENTRYPOINT ["dockerize"]
-CMD ["--help"]
